@awak-app/simy-cli 0.5.34 → 0.5.35

package/src/workflow-deliverable-upload.js

@@ -0,0 +1,454 @@
+import { createHash } from "node:crypto";
+
+import { buildLocalTaskArtifactRefs } from "./local-task-artifact-contract.js";
+import { webApiHeaders, webApiUrl } from "./web-api.js";
+
+/**
+ * WorkFlow deliverable device upload (issue #2550 G3, backend
+ * contracts/workflow-device-upload.v1.json).
+ *
+ * What counts as a deliverable: only the verified files a Local Task Provider
+ * produced beneath the task's managed output directory (`outputs_root`), as
+ * collected by collectLocalTaskArtifacts() after the Provider finished. Files
+ * elsewhere in the workspace are never uploaded, and every byte read goes
+ * through the containment-checked localTaskArtifact() reader.
+ *
+ * Order (backend authorization requires a consumed, succeeded receipt):
+ *   1. the typed terminal was already accepted with outcome "succeeded";
+ *   2. POST the verified manifest to the device gateway
+ *      `workflow-dev-work-events/deliverables` (Crew submits it to Backend);
+ *   3. per file: claim at functions/v1/workflow-deliverable-upload with the
+ *      device bearer, then PUT the exact bytes to the signed upload_url.
+ *
+ * Never throws and never logs; the returned summary contains no token, signed
+ * URL or absolute path, so it is safe to persist in the Local Task snapshot.
+ */
+
+export const WORKFLOW_DELIVERABLES_PATH = "workflow-dev-work-events/deliverables";
+export const DELIVERABLE_MANIFEST_CONTRACT = "independent_workflow_deliverable_manifest.v1";
+export const DEVICE_UPLOAD_CONTRACT = "workflow_device_upload.v1";
+export const MAX_DELIVERABLE_BYTES = 52_428_800;
+export const UPLOAD_FUNCTION_PATH = "/functions/v1/workflow-deliverable-upload";
+export const UPLOAD_ENDPOINT_ENV = "SIMY_WORKFLOW_DELIVERABLE_UPLOAD_URL";
+
+const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+const SHA256 = /^[0-9a-f]{64}$/;
+const SUPABASE_HOST = /^[a-z0-9]{20}\.supabase\.co$/;
+const LOOPBACK_HOSTS = new Set(["127.0.0.1", "localhost", "[::1]"]);
+const REQUEST_TIMEOUT_MS = 10_000;
+const PUT_TIMEOUT_MS = 120_000;
+// Crew may not have consumed the terminal receipt yet when the manifest
+// arrives (409), and a claim returns nothing until the job exists.
+const MANIFEST_RETRY_DELAYS_MS = [2_000, 5_000, 10_000, 20_000];
+const CLAIM_IDLE_DELAYS_MS = [2_000, 5_000, 10_000];
+const PUT_ATTEMPTS = 3;
+const PUT_RETRY_DELAY_MS = 2_000;
+
+export async function uploadWorkflowDeliverables({
+  binding,
+  observationId,
+  localTaskId,
+  artifacts,
+  readArtifactBytes,
+  session,
+  apiOrigin,
+  fetchImpl = globalThis.fetch,
+  environment = process.env,
+  sleep = defaultSleep,
+  manifestRetryDelaysMs = MANIFEST_RETRY_DELAYS_MS,
+  claimIdleDelaysMs = CLAIM_IDLE_DELAYS_MS,
+}) {
+  let manifest;
+  try {
+    manifest = buildLocalTaskArtifactRefs(artifacts, localTaskId);
+  } catch {
+    return summary("failed", { reason_code: "deliverable_manifest_invalid" });
+  }
+  if (manifest.length === 0) return summary("skipped", { reason_code: "no_deliverables" });
+  const base = { manifest_files: manifest.length };
+  if (!session?.token || !session?.api_base_url) {
+    return summary("failed", { ...base, reason_code: "device_session_unavailable" });
+  }
+
+  try {
+    const submitted = await submitManifest({
+      binding, observationId, manifest, session, apiOrigin, fetchImpl, sleep,
+      delays: manifestRetryDelaysMs,
+    });
+    if (!submitted.ok) return summary("failed", { ...base, ...submitted.reason });
+
+    const endpoint = resolveUploadEndpoint(environment?.[UPLOAD_ENDPOINT_ENV], submitted.payload);
+    if (!endpoint) return summary("failed", { ...base, reason_code: "upload_endpoint_unavailable" });
+    const orgId = firstUuid(session.organization_id, submitted.payload.org_id);
+    const authUserId = firstUuid(session.account_id, submitted.payload.auth_user_id);
+    if (!orgId || !authUserId) {
+      return summary("failed", { ...base, reason_code: "device_identity_unavailable" });
+    }
+
+    const result = await claimAndUpload({
+      endpoint,
+      claimBody: {
+        action: "claim",
+        device_token: session.token,
+        org_id: orgId,
+        auth_user_id: authUserId,
+        workflow_run_id: binding.workflow_run_id,
+        observation_id: observationId,
+        source_task_id: localTaskId,
+      },
+      manifest,
+      localTaskId,
+      readArtifactBytes,
+      fetchImpl,
+      sleep,
+      idleDelays: claimIdleDelaysMs,
+    });
+    const status =
+      result.failures.length === 0 && result.uploaded.length === manifest.length
+        ? "completed"
+        : result.uploaded.length > 0
+          ? "partial"
+          : "failed";
+    return summary(status, {
+      ...base,
+      ...(status === "completed" ? {} : { reason_code: result.reasonCode }),
+      uploaded: result.uploaded,
+      failures: result.failures,
+    });
+  } catch {
+    return summary("failed", { ...base, reason_code: "deliverable_upload_error" });
+  }
+}
+
+async function submitManifest({
+  binding, observationId, manifest, session, apiOrigin, fetchImpl, sleep, delays,
+}) {
+  const body = JSON.stringify({
+    workflow_id: binding.workflow_id,
+    workflow_run_id: binding.workflow_run_id,
+    action_digest: binding.action_digest,
+    observation_id: observationId,
+    manifest,
+  });
+  let lastStatus = null;
+  for (let attempt = 0; attempt <= delays.length; attempt += 1) {
+    if (attempt > 0) await sleep(delays[attempt - 1]);
+    let response;
+    try {
+      response = await fetchImpl(
+        webApiUrl(WORKFLOW_DELIVERABLES_PATH, {
+          webOrigin: apiOrigin,
+          apiBaseUrl: session.api_base_url,
+        }),
+        {
+          method: "POST",
+          headers: webApiHeaders({ token: session.token }, { "Content-Type": "application/json" }),
+          body,
+          signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
+        },
+      );
+    } catch {
+      lastStatus = null;
+      continue;
+    }
+    lastStatus = response.status;
+    // 409: receipt not consumed yet; 5xx: transient. Both are safe to resend
+    // because Backend treats an identical manifest as the same submission.
+    if (response.status === 409 || response.status >= 500) continue;
+    if (!response.ok) {
+      return { ok: false, reason: httpReason(response.status, "deliverable_manifest_rejected") };
+    }
+    const payload = await response.json().catch(() => null);
+    if (
+      !payload ||
+      typeof payload !== "object" ||
+      payload.contract_version !== DELIVERABLE_MANIFEST_CONTRACT ||
+      payload.accepted !== true
+    ) {
+      return { ok: false, reason: { reason_code: "invalid_deliverable_manifest_receipt" } };
+    }
+    return { ok: true, payload };
+  }
+  return {
+    ok: false,
+    reason: lastStatus === 409
+      ? { reason_code: "deliverable_manifest_not_admitted", http_status: 409 }
+      : { reason_code: "deliverable_service_unavailable" },
+  };
+}
+
+async function claimAndUpload({
+  endpoint, claimBody, manifest, localTaskId, readArtifactBytes, fetchImpl, sleep, idleDelays,
+}) {
+  const pending = new Map(manifest.map((entry) => [entry.relative_path, entry]));
+  const putAttempts = new Map();
+  const uploaded = [];
+  const failures = [];
+  let reasonCode = null;
+  let idle = 0;
+  // Hard bound: every file may need a few claims, plus idle waits.
+  let claimBudget = manifest.length * (PUT_ATTEMPTS + 1) + idleDelays.length + 1;
+
+  const fail = (relativePath, code) => {
+    reasonCode ||= code;
+    failures.push(relativePath === null ? { reason_code: code } : { relative_path: relativePath, reason_code: code });
+    if (relativePath !== null) pending.delete(relativePath);
+  };
+
+  while (pending.size > 0 && claimBudget > 0) {
+    claimBudget -= 1;
+    const claim = await postClaim(endpoint, claimBody, fetchImpl);
+    if (claim.kind === "fatal") {
+      reasonCode ||= claim.reasonCode;
+      break;
+    }
+    if (claim.kind === "idle") {
+      // No pending file for this device/run/receipt/task yet (or transient).
+      if (idle >= idleDelays.length) {
+        reasonCode ||= claim.reasonCode;
+        break;
+      }
+      await sleep(idleDelays[idle]);
+      idle += 1;
+      continue;
+    }
+
+    const job = claim.payload;
+    if (!isSafeRelativePath(job.source_relative_path)) {
+      // Path traversal or an absolute path is never read, even if listed.
+      fail(null, "claim_path_rejected");
+      continue;
+    }
+    const entry = pending.get(job.source_relative_path);
+    if (!entry) {
+      fail(null, "claim_unknown_file");
+      continue;
+    }
+    if (job.size_bytes > MAX_DELIVERABLE_BYTES || entry.size_bytes > MAX_DELIVERABLE_BYTES) {
+      fail(entry.relative_path, "file_too_large");
+      continue;
+    }
+    if (
+      job.source_task_id !== localTaskId ||
+      (job.source_artifact_id !== undefined && job.source_artifact_id !== entry.id) ||
+      job.size_bytes !== entry.size_bytes ||
+      job.sha256 !== String(entry.sha256).toLowerCase() ||
+      job.mime_type !== entry.mime_type
+    ) {
+      fail(entry.relative_path, "claim_manifest_mismatch");
+      continue;
+    }
+    if (!trustedUploadUrl(job.upload_url, endpoint)) {
+      fail(entry.relative_path, "untrusted_upload_url");
+      continue;
+    }
+
+    let bytes;
+    try {
+      bytes = await readArtifactBytes(entry.id);
+    } catch {
+      bytes = null;
+    }
+    if (
+      !bytes ||
+      bytes.length !== job.size_bytes ||
+      createHash("sha256").update(bytes).digest("hex") !== job.sha256
+    ) {
+      fail(entry.relative_path, "local_file_changed");
+      continue;
+    }
+
+    const put = await putWithRetry(job, bytes, fetchImpl, sleep, () => {
+      const used = (putAttempts.get(entry.relative_path) || 0) + 1;
+      putAttempts.set(entry.relative_path, used);
+      return used;
+    });
+    if (put.kind === "uploaded") {
+      uploaded.push({ artifact_id: put.artifactId, relative_path: entry.relative_path });
+      pending.delete(entry.relative_path);
+    } else if (put.kind === "reclaim") {
+      // The signed URL or lease expired; the next claim reissues the same
+      // stable artifact_id unless attempts are exhausted.
+      if ((putAttempts.get(entry.relative_path) || 0) >= PUT_ATTEMPTS) {
+        fail(entry.relative_path, put.reasonCode);
+      }
+    } else {
+      fail(entry.relative_path, put.reasonCode);
+    }
+  }
+  for (const relativePath of pending.keys()) {
+    failures.push({ relative_path: relativePath, reason_code: reasonCode || "claim_unavailable" });
+  }
+  if (failures.length && !reasonCode) reasonCode = failures[0].reason_code;
+  return { uploaded, failures, reasonCode };
+}
+
+async function postClaim(endpoint, claimBody, fetchImpl) {
+  let response;
+  try {
+    response = await fetchImpl(endpoint.href, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify(claimBody),
+      redirect: "error",
+      signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
+    });
+  } catch {
+    return { kind: "idle", reasonCode: "upload_service_unavailable" };
+  }
+  if (response.status === 403) return { kind: "idle", reasonCode: "claim_unavailable" };
+  if (response.status >= 500) return { kind: "idle", reasonCode: "upload_service_unavailable" };
+  if (!response.ok) return { kind: "fatal", reasonCode: "claim_rejected" };
+  const payload = await response.json().catch(() => null);
+  if (!validClaim(payload)) return { kind: "fatal", reasonCode: "invalid_claim_response" };
+  return { kind: "claimed", payload };
+}
+
+async function putWithRetry(job, bytes, fetchImpl, sleep, countAttempt) {
+  while (true) {
+    const attempt = countAttempt();
+    let response = null;
+    try {
+      response = await fetchImpl(job.upload_url, {
+        method: "PUT",
+        headers: {
+          "Content-Type": job.mime_type,
+          "Content-Length": String(bytes.length),
+        },
+        body: bytes,
+        redirect: "error",
+        signal: AbortSignal.timeout(PUT_TIMEOUT_MS),
+      });
+    } catch {
+      response = null;
+    }
+    if (response?.ok) {
+      const payload = await response.json().catch(() => null);
+      if (
+        payload?.contract_version === DEVICE_UPLOAD_CONTRACT &&
+        typeof payload.artifact_id === "string" &&
+        payload.artifact_id.toLowerCase() === job.artifact_id.toLowerCase()
+      ) {
+        return { kind: "uploaded", artifactId: job.artifact_id.toLowerCase() };
+      }
+      return { kind: "failed", reasonCode: "invalid_upload_response" };
+    }
+    if (response && response.status === 403) {
+      return { kind: "reclaim", reasonCode: "upload_authorization_expired" };
+    }
+    if (response && response.status === 413) return { kind: "failed", reasonCode: "file_too_large" };
+    if (response && response.status === 422) {
+      return { kind: "failed", reasonCode: "file_does_not_match_manifest" };
+    }
+    if (response && response.status < 500) {
+      return { kind: "failed", reasonCode: `upload_rejected_${response.status}` };
+    }
+    // Network failure or 5xx: the signed URL is still valid for the lease, and
+    // the gateway accepts identical immutable bytes on a repeat.
+    if (attempt >= PUT_ATTEMPTS) return { kind: "failed", reasonCode: "upload_failed_after_retries" };
+    await sleep(PUT_RETRY_DELAY_MS);
+  }
+}
+
+function validClaim(value) {
+  return Boolean(
+    value &&
+      typeof value === "object" &&
+      value.contract_version === DEVICE_UPLOAD_CONTRACT &&
+      typeof value.artifact_id === "string" &&
+      UUID.test(value.artifact_id) &&
+      typeof value.source_relative_path === "string" &&
+      typeof value.source_task_id === "string" &&
+      Number.isSafeInteger(value.size_bytes) &&
+      value.size_bytes > 0 &&
+      typeof value.sha256 === "string" &&
+      SHA256.test(value.sha256) &&
+      typeof value.mime_type === "string" &&
+      value.mime_type.length > 0 &&
+      typeof value.upload_url === "string",
+  );
+}
+
+/** Mirrors Backend's source_relative_path CHECK constraint. */
+export function isSafeRelativePath(value) {
+  return (
+    typeof value === "string" &&
+    value.length >= 1 &&
+    value.length <= 2048 &&
+    !/[\u0000-\u001f\u007f\\]/.test(value) &&
+    !/(^|\/)\.\.?(\/|$)/.test(value) &&
+    !/^(\/|~\/|[A-Za-z]:)/.test(value) &&
+    !value.includes("//") &&
+    !value.endsWith("/")
+  );
+}
+
+/**
+ * The device bearer is sent only to the Backend upload function. An operator
+ * override (env) may be any https URL, or loopback http for local Supabase; an
+ * endpoint suggested by the Web/Crew response must be a hosted Supabase
+ * project URL. Both must be the exact function path with no credentials,
+ * query or fragment.
+ */
+export function resolveUploadEndpoint(override, payload) {
+  if (typeof override === "string" && override.trim()) {
+    const url = parseEndpoint(override.trim());
+    if (!url) return null;
+    if (url.protocol === "https:") return url;
+    return url.protocol === "http:" && LOOPBACK_HOSTS.has(url.hostname) ? url : null;
+  }
+  const suggested = typeof payload?.upload_endpoint === "string"
+    ? parseEndpoint(payload.upload_endpoint)
+    : null;
+  return suggested && suggested.protocol === "https:" && SUPABASE_HOST.test(suggested.hostname) &&
+    suggested.port === ""
+    ? suggested
+    : null;
+}
+
+function parseEndpoint(value) {
+  let url;
+  try {
+    url = new URL(value);
+  } catch {
+    return null;
+  }
+  if (url.username || url.password || url.search || url.hash) return null;
+  return url.pathname === UPLOAD_FUNCTION_PATH ? url : null;
+}
+
+function trustedUploadUrl(value, endpoint) {
+  let url;
+  try {
+    url = new URL(value);
+  } catch {
+    return false;
+  }
+  return (
+    url.origin === endpoint.origin &&
+    url.pathname === endpoint.pathname &&
+    !url.username &&
+    !url.password
+  );
+}
+
+function firstUuid(...values) {
+  for (const value of values) {
+    if (typeof value === "string" && UUID.test(value)) return value.toLowerCase();
+  }
+  return null;
+}
+
+function httpReason(status, fallback) {
+  if (status === 401 || status === 403) return { reason_code: "device_or_source_not_authorized", http_status: status };
+  if (status === 404 || status === 410) return { reason_code: "workflow_binding_unavailable", http_status: status };
+  return { reason_code: fallback, http_status: status };
+}
+
+function summary(status, extra = {}) {
+  return { status, contract_version: DEVICE_UPLOAD_CONTRACT, ...extra };
+}
+
+function defaultSleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}