@bli-cockpit/cli 0.1.3 → 0.1.5

package/dist/upload.js

@@ -1,8 +1,9 @@
 import { TelemetryIngestEnvelopeSchema, TelemetryIngestEventDtoSchema, } from "@bli-cockpit/telemetry-core";
-import fs from "node:fs/promises";
 import path from "node:path";
-import { getCollectorRuntimePaths, LOCAL_COLLECTOR_VERSION, readLocalCollectorConfig, readLocalCollectorSessionFile, readLocalSessionReference, readLocalWorkContext, } from "./local-state.js";
+import { getCollectorRuntimePaths, LOCAL_COLLECTOR_VERSION, readLocalCollectorConfig, readLocalCollectorSessionFile, readLocalSessionReference, readLocalWorkContextForRepo, } from "./local-state.js";
 import { runLocalSourceCollectors } from "./adapters/local-sources.js";
+import { uploadRawEvidenceFilesChunked, } from "./evidence-upload-client.js";
+import { markObjectCommitted, readRawEvidenceCursor, writeRawEvidenceCursor, } from "./cursors/raw-evidence-cursor.js";
 import { recordUploadBlocked, recordUploadFailure, recordUploadSuccess, } from "./spool/local-spool.js";
 export class LocalUploadBlockedError extends Error {
     blocker;
@@ -29,17 +30,22 @@ export async function buildLocalAmbientEnvelope(options = {}) {
             ? "Collector session expired. Run `cockpit login` or `cockpit pair` again before `cockpit sync`."
             : "Collector is not paired. Run `cockpit login` or `cockpit pair` before `cockpit sync`.", "cockpit login");
     }
-    const activeContext = await readLocalWorkContext(paths).catch(() => {
+    const repoRoot = path.resolve(options.repoRoot ?? process.cwd());
+    const activeContext = await readLocalWorkContextForRepo(paths, repoRoot).catch(() => {
         throw new LocalUploadBlockedError("missing_context", "Active work context missing. Run `cockpit start --repo \"$PWD\"` before `cockpit sync`.", "cockpit start --repo \"$PWD\"");
     });
-    const repoRoot = path.resolve(options.repoRoot ?? activeContext.repo);
-    const repoLabel = safeRepoLabel(repoRoot);
+    const repoLabel = safeRepoLabel(activeContext.repo_label ?? repoRoot);
     const uploadContext = makeUploadWorkContext({
         activeContext,
         session,
         repoLabel,
         now,
     });
+    const contextNamespacePrefix = `${session.operator_id}/${uploadContext.work_context_id}/`;
+    const skipContentHashes = options.skipContentHashes ??
+        new Set(Object.entries(options.cursorObjects ?? {})
+            .filter(([, entry]) => entry.object_key.startsWith(contextNamespacePrefix))
+            .map(([hash]) => hash));
     const sourceCollection = await runLocalSourceCollectors({
         repoRoot,
         branch: uploadContext.branch,
@@ -49,6 +55,9 @@ export async function buildLocalAmbientEnvelope(options = {}) {
         activeWorkContext: activeContext,
         rawEvidenceStateDir: paths.state_dir,
         rawEvidenceSessionsDir: path.join(paths.home_dir, ".codex", "sessions"),
+        rawEvidenceIncludeCodexJsonl: options.rawEvidenceIncludeCodexJsonl,
+        rawEvidenceCodexSessionFiles: options.codexSessionFiles,
+        rawEvidenceSkipContentHashes: skipContentHashes,
         now,
     });
     const binding = sourceCollection.binding;
@@ -93,15 +102,21 @@ export async function buildLocalAmbientEnvelope(options = {}) {
         source_scan_count: envelope.source_scan_results.length,
         risk_flag_count: safeRiskFlags.length,
         repo_label: repoLabel,
+        head_sha: uploadContext.head_sha ?? null,
         raw_evidence_upload_files: sourceCollection.facts.raw_evidence?.upload_files ?? [],
+        raw_evidence_facts: sourceCollection.facts.raw_evidence,
     };
 }
 export async function syncLocalAmbientEnvelope(options = {}) {
     const attemptedAt = (options.now ?? new Date()).toISOString();
     const paths = getCollectorRuntimePaths(options.homeDir);
+    const cursor = await readRawEvidenceCursor(paths);
     let built;
     try {
-        built = await buildLocalAmbientEnvelope(options);
+        built = await buildLocalAmbientEnvelope({
+            ...options,
+            cursorObjects: options.cursorObjects ?? cursor.objects,
+        });
     }
     catch (error) {
         if (error instanceof LocalUploadBlockedError) {
@@ -116,49 +131,84 @@ export async function syncLocalAmbientEnvelope(options = {}) {
     if (!fetchImpl) {
         throw new Error("global fetch is unavailable; use Node.js 20 or newer.");
     }
-    let uploadedRawEvidenceObjects = [];
+    const provenance = built.envelope.work_context.provenance;
+    if (built.raw_evidence_upload_files.length > 0 && !provenance) {
+        throw new Error("Raw evidence upload requires collector provenance.");
+    }
+    let uploadOutcomes = [];
+    let uploadedChunkCount = 0;
+    let ingestAccepted = false;
     try {
-        uploadedRawEvidenceObjects = await uploadRawEvidenceFiles({
-            fetchImpl,
-            dashboardUrl: built.dashboard_url,
-            deviceToken: built.device_token,
-            envelope: built.envelope,
-            files: built.raw_evidence_upload_files,
-        });
+        if (built.raw_evidence_upload_files.length > 0 && provenance) {
+            const upload = await uploadRawEvidenceFilesChunked({
+                fetchImpl,
+                dashboardUrl: built.dashboard_url,
+                deviceToken: built.device_token,
+                provenance,
+                generatedAt: built.envelope.generated_at,
+                files: built.raw_evidence_upload_files,
+            });
+            uploadOutcomes = upload.outcomes;
+            uploadedChunkCount = upload.uploaded_chunk_count;
+        }
+        const envelope = pruneUndurablePointers(built.envelope, uploadOutcomes);
         const response = await fetchImpl(`${built.dashboard_url}/api/ambient/ingest`, {
             method: "POST",
             headers: {
                 "Authorization": `Bearer ${built.device_token}`,
                 "Content-Type": "application/json",
             },
-            body: JSON.stringify(built.envelope),
+            body: JSON.stringify(envelope),
         });
         const responseBody = await readResponseJson(response);
         if (!response.ok) {
             throw new Error(responseErrorMessage(responseBody, `Ambient ingest failed with HTTP ${response.status}`));
         }
+        ingestAccepted = true;
+        for (const outcome of uploadOutcomes) {
+            if (outcome.upload_state === "upload_failed")
+                continue;
+            const contentHash = outcome.pointer.content_hash_sha256;
+            if (!contentHash)
+                continue;
+            markObjectCommitted(cursor, contentHash, {
+                object_key: outcome.object_key,
+                byte_size: outcome.pointer.byte_size ?? 0,
+                committed_at: attemptedAt,
+            });
+        }
+        cursor.updated_at = attemptedAt;
+        await writeRawEvidenceCursor(paths, cursor);
         await recordUploadSuccess(paths, { attemptedAt });
         return {
             status: "uploaded",
             dashboard_url: built.dashboard_url,
             ticket_id: built.ticket_id,
             work_context_id: built.envelope.work_context.work_context_id,
+            head_sha: built.head_sha,
             event_count: built.event_count,
             source_scan_count: built.source_scan_count,
             risk_flag_count: built.risk_flag_count,
-            raw_evidence_file_count: built.raw_evidence_upload_files.length,
             http_status: response.status,
+            ...rawEvidenceSummary(built, uploadOutcomes, uploadedChunkCount, cursor),
         };
     }
     catch (error) {
         const failureReason = error instanceof Error ? error.message : String(error);
-        const cleanupFailureReason = await cleanupUploadedRawEvidenceObjects({
-            fetchImpl,
-            dashboardUrl: built.dashboard_url,
-            deviceToken: built.device_token,
-            envelope: built.envelope,
-            uploadedObjects: uploadedRawEvidenceObjects,
-        });
+        // Once ingest accepted the envelope, the evidence refs are indexed and the
+        // objects are load-bearing — a later local write failure (cursor, spool)
+        // must not delete durable evidence.
+        const cleanupFailureReason = ingestAccepted
+            ? null
+            : await cleanupUploadedRawEvidenceObjects({
+                fetchImpl,
+                dashboardUrl: built.dashboard_url,
+                deviceToken: built.device_token,
+                envelope: built.envelope,
+                objectKeys: uploadOutcomes
+                    .filter((outcome) => outcome.upload_state === "uploaded")
+                    .map((outcome) => outcome.object_key),
+            });
         const spooledFailureReason = cleanupFailureReason
             ? `${failureReason}; raw evidence cleanup failed: ${cleanupFailureReason}`
             : failureReason;
@@ -181,16 +231,163 @@ export async function syncLocalAmbientEnvelope(options = {}) {
             dashboard_url: built.dashboard_url,
             ticket_id: built.ticket_id,
             work_context_id: built.envelope.work_context.work_context_id,
+            head_sha: built.head_sha,
             event_count: built.event_count,
             source_scan_count: built.source_scan_count,
             risk_flag_count: built.risk_flag_count,
-            raw_evidence_file_count: built.raw_evidence_upload_files.length,
             failure_reason: spooledFailureReason,
             spool_entry_id: entry.spool_id,
             retry_command: entry.retry_command,
+            ...rawEvidenceSummary(built, uploadOutcomes, uploadedChunkCount, cursor),
         };
     }
 }
+/**
+ * Posts a Codex session attribution report using the paired collector
+ * credentials and the work context of a representative repo. Failures come
+ * back as reason labels so the harvest never hard-fails on reporting.
+ */
+export async function postCodexSessionReport(options) {
+    if (options.sessions.length === 0) {
+        return { posted: false, reason: "no_sessions_observed" };
+    }
+    const fetchImpl = options.fetch ?? globalThis.fetch;
+    if (!fetchImpl) {
+        throw new Error("global fetch is unavailable; use Node.js 20 or newer.");
+    }
+    try {
+        const paths = getCollectorRuntimePaths(options.homeDir);
+        const config = await readLocalCollectorConfig(paths);
+        const sessionFile = await readLocalCollectorSessionFile(paths);
+        const session = await readLocalSessionReference(paths);
+        if (session.session_state !== "valid") {
+            return { posted: false, reason: "collector_not_paired" };
+        }
+        const repoRoot = path.resolve(options.repoRoot ?? process.cwd());
+        const activeContext = await readLocalWorkContextForRepo(paths, repoRoot);
+        const repoLabel = safeRepoLabel(activeContext.repo_label ?? repoRoot);
+        const provenance = makeCollectorProvenance({
+            context: makeUploadWorkContext({
+                activeContext,
+                session,
+                repoLabel,
+                now: options.now ?? new Date(),
+            }),
+            session,
+            repoLabel,
+        });
+        return await reportCodexSessionAttributions({
+            fetchImpl,
+            dashboardUrl: normalizeDashboardUrl(options.dashboardUrl ?? sessionFile.dashboard_url ?? config.dashboard_url),
+            deviceToken: sessionFile.device_token,
+            provenance,
+            generatedAt: (options.now ?? new Date()).toISOString(),
+            sessions: options.sessions,
+        });
+    }
+    catch {
+        return { posted: false, reason: "collector_not_ready" };
+    }
+}
+/**
+ * Reports Codex session attribution outcomes after sync. Non-fatal by design:
+ * older dashboards without the endpoint must not fail the harvest, so the
+ * caller receives a posted/skipped label instead of an exception.
+ */
+export async function reportCodexSessionAttributions(options) {
+    if (options.sessions.length === 0) {
+        return { posted: false, reason: "no_sessions_observed" };
+    }
+    try {
+        const response = await options.fetchImpl(`${options.dashboardUrl}/api/ambient/codex-sessions`, {
+            method: "POST",
+            headers: {
+                "Authorization": `Bearer ${options.deviceToken}`,
+                "Content-Type": "application/json",
+            },
+            body: JSON.stringify({
+                schema_version: "ambient-codex-session-attributions.v1",
+                generated_at: options.generatedAt,
+                provenance: options.provenance,
+                sessions: options.sessions,
+            }),
+        });
+        if (response.status === 404) {
+            return { posted: false, reason: "codex_session_api_unavailable" };
+        }
+        if (!response.ok) {
+            return { posted: false, reason: `report_failed_http_${response.status}` };
+        }
+        return { posted: true, reason: "recorded" };
+    }
+    catch {
+        return { posted: false, reason: "report_network_error" };
+    }
+}
+function rawEvidenceSummary(built, outcomes, uploadedChunkCount, cursor) {
+    const cursorReused = built.raw_evidence_facts?.reused ?? [];
+    const serverReusedCount = outcomes.filter((outcome) => outcome.upload_state === "reused_existing").length;
+    const failed = outcomes.filter((outcome) => outcome.upload_state === "upload_failed");
+    const cursorReusedOutcomes = cursorReused.map((entry) => ({
+        object_key: cursor.objects[entry.content_hash_sha256]?.object_key ?? "",
+        raw_evidence_pointer_id: cursor.objects[entry.content_hash_sha256]?.object_key ?? "",
+        kind: entry.kind,
+        codex_session_id: entry.codex_session_id,
+        upload_state: "reused_existing",
+        reason: "cursor_content_match",
+    }));
+    return {
+        raw_evidence_file_count: built.raw_evidence_upload_files.length,
+        raw_evidence_uploaded_object_count: outcomes.filter((outcome) => outcome.upload_state === "uploaded").length,
+        raw_evidence_uploaded_chunk_count: uploadedChunkCount,
+        raw_evidence_reused_count: cursorReused.length + serverReusedCount,
+        raw_evidence_failed_count: failed.length,
+        raw_evidence_failure_reasons: [
+            ...new Set(failed.map((outcome) => outcome.reason ?? "unknown")),
+        ],
+        raw_evidence_outcomes: [
+            ...outcomes.map((outcome) => ({
+                object_key: outcome.object_key,
+                raw_evidence_pointer_id: outcome.pointer.raw_evidence_pointer_id,
+                kind: outcome.kind,
+                codex_session_id: outcome.codex_session_id,
+                upload_state: outcome.upload_state,
+                reason: outcome.reason,
+            })),
+            ...cursorReusedOutcomes,
+        ],
+        cursor_tracked_object_count: Object.keys(cursor.objects).length,
+    };
+}
+/**
+ * Ingest refuses pointers whose objects never became durable, so failed
+ * uploads are pruned from the envelope instead of failing the whole sync;
+ * the failures stay visible via reason labels and the server-side ledger.
+ */
+function pruneUndurablePointers(envelope, outcomes) {
+    // Content-addressed keys mean one pointer id can carry several outcomes
+    // (byte-identical files); the pointer is durable if ANY outcome succeeded.
+    const durablePointerIds = new Set(outcomes
+        .filter((outcome) => outcome.upload_state !== "upload_failed")
+        .map((outcome) => outcome.pointer.raw_evidence_pointer_id));
+    const failedPointerIds = new Set(outcomes
+        .filter((outcome) => outcome.upload_state === "upload_failed" &&
+        !durablePointerIds.has(outcome.pointer.raw_evidence_pointer_id))
+        .map((outcome) => outcome.pointer.raw_evidence_pointer_id));
+    if (failedPointerIds.size === 0)
+        return envelope;
+    return {
+        ...envelope,
+        events: envelope.events.map((event) => ({
+            ...event,
+            raw_evidence_pointers: event.raw_evidence_pointers.filter((pointer) => !failedPointerIds.has(pointer.raw_evidence_pointer_id)),
+            redaction: {
+                ...event.redaction,
+                raw_evidence_pointer_ids: event.redaction.raw_evidence_pointer_ids.filter((pointerId) => !failedPointerIds.has(pointerId)),
+            },
+        })),
+    };
+}
 function makeUploadWorkContext(options) {
     const provenance = makeCollectorProvenance({
         context: options.activeContext,
@@ -200,6 +397,13 @@ function makeUploadWorkContext(options) {
     return {
         ...options.activeContext,
         repo: options.repoLabel,
+        repo_label: options.activeContext.repo_label ?? options.repoLabel,
+        repo_fingerprint: options.activeContext.repo_fingerprint,
+        repo_origin_url: options.activeContext.repo_origin_url,
+        head_sha: options.activeContext.head_sha,
+        worktree_label: options.activeContext.worktree_label,
+        worktree_fingerprint: options.activeContext.worktree_fingerprint,
+        worktree_is_primary: options.activeContext.worktree_is_primary,
         operator_id: options.session.operator_id,
         session_id: options.session.session_id,
         updated_at: options.now.toISOString(),
@@ -252,9 +456,14 @@ function makeSourceScanCompletedEvent(options) {
             raw_evidence_file_count: options.rawEvidenceFacts?.file_count ?? 0,
             raw_evidence_byte_size: options.rawEvidenceFacts?.byte_size ?? 0,
             raw_evidence_skipped_count: options.rawEvidenceFacts?.skipped_count ?? 0,
+            raw_evidence_reused_count: options.rawEvidenceFacts?.reused_count ?? 0,
         },
         attributes: {
             repo_label: options.context.repo,
+            repo_fingerprint: options.context.repo_fingerprint ?? "unknown",
+            worktree_label: options.context.worktree_label ?? "unknown",
+            worktree_fingerprint: options.context.worktree_fingerprint ?? "unknown",
+            worktree_is_primary: options.context.worktree_is_primary ?? false,
             branch: options.context.branch,
             ticket_binding_state: options.binding.state,
             ticket_binding_source: options.binding.selected_source ?? "none",
@@ -271,78 +480,40 @@ function makeSourceScanCompletedEvent(options) {
         raw_evidence_pointers: rawEvidencePointers,
     });
 }
-async function uploadRawEvidenceFiles(options) {
-    if (options.files.length === 0)
-        return [];
-    const provenance = options.envelope.work_context.provenance;
-    if (!provenance) {
-        throw new Error("Raw evidence upload requires collector provenance.");
-    }
-    const files = await Promise.all(options.files.map(async (file) => ({
-        pointer: file.pointer,
-        content_base64: (await fs.readFile(file.local_path)).toString("base64"),
-    })));
-    const response = await options.fetchImpl(`${options.dashboardUrl}/api/ambient/evidence/upload`, {
-        method: "POST",
-        headers: {
-            "Authorization": `Bearer ${options.deviceToken}`,
-            "Content-Type": "application/json",
-        },
-        body: JSON.stringify({
-            schema_version: "ambient-raw-evidence-upload.v1",
-            generated_at: options.envelope.generated_at,
-            provenance,
-            files,
-        }),
-    });
-    const responseBody = await readResponseJson(response);
-    if (!response.ok) {
-        throw new Error(responseErrorMessage(responseBody, `Raw evidence upload failed with HTTP ${response.status}`));
-    }
-    return readUploadedRawEvidenceObjects(responseBody);
-}
+const CLEANUP_BATCH_SIZE = 25;
 async function cleanupUploadedRawEvidenceObjects(options) {
-    if (options.uploadedObjects.length === 0)
+    if (options.objectKeys.length === 0)
         return null;
     const provenance = options.envelope.work_context.provenance;
     if (!provenance)
         return "missing collector provenance";
-    try {
-        const response = await options.fetchImpl(`${options.dashboardUrl}/api/ambient/evidence/upload`, {
-            method: "DELETE",
-            headers: {
-                "Authorization": `Bearer ${options.deviceToken}`,
-                "Content-Type": "application/json",
-            },
-            body: JSON.stringify({
-                schema_version: "ambient-raw-evidence-cleanup.v1",
-                generated_at: options.envelope.generated_at,
-                provenance,
-                object_keys: options.uploadedObjects.map((object) => object.object_key),
-            }),
-        });
-        const responseBody = await readResponseJson(response);
-        if (!response.ok) {
-            return responseErrorMessage(responseBody, `Raw evidence cleanup failed with HTTP ${response.status}`);
+    const failures = [];
+    for (let offset = 0; offset < options.objectKeys.length; offset += CLEANUP_BATCH_SIZE) {
+        const batch = options.objectKeys.slice(offset, offset + CLEANUP_BATCH_SIZE);
+        try {
+            const response = await options.fetchImpl(`${options.dashboardUrl}/api/ambient/evidence/upload`, {
+                method: "DELETE",
+                headers: {
+                    "Authorization": `Bearer ${options.deviceToken}`,
+                    "Content-Type": "application/json",
+                },
+                body: JSON.stringify({
+                    schema_version: "ambient-raw-evidence-cleanup.v1",
+                    generated_at: options.envelope.generated_at,
+                    provenance,
+                    object_keys: batch,
+                }),
+            });
+            const responseBody = await readResponseJson(response);
+            if (!response.ok) {
+                failures.push(responseErrorMessage(responseBody, `Raw evidence cleanup failed with HTTP ${response.status}`));
+            }
+        }
+        catch (error) {
+            failures.push(error instanceof Error ? error.message : String(error));
         }
-        return null;
-    }
-    catch (error) {
-        return error instanceof Error ? error.message : String(error);
     }
-}
-function readUploadedRawEvidenceObjects(value) {
-    if (!value || typeof value !== "object")
-        return [];
-    const uploaded = value.uploaded;
-    if (!Array.isArray(uploaded))
-        return [];
-    return uploaded.flatMap((entry) => {
-        if (!entry || typeof entry !== "object")
-            return [];
-        const objectKey = entry.object_key;
-        return typeof objectKey === "string" && objectKey ? [{ object_key: objectKey }] : [];
-    });
+    return failures.length > 0 ? failures.join("; ") : null;
 }
 function makeCollectorProvenance(options) {
     return {
@@ -351,6 +522,12 @@ function makeCollectorProvenance(options) {
         collector_version: LOCAL_COLLECTOR_VERSION,
         repo: options.repoLabel,
         branch: options.context.branch,
+        repo_label: options.context.repo_label ?? options.repoLabel,
+        repo_fingerprint: options.context.repo_fingerprint,
+        repo_origin_url: options.context.repo_origin_url,
+        worktree_label: options.context.worktree_label,
+        worktree_fingerprint: options.context.worktree_fingerprint,
+        worktree_is_primary: options.context.worktree_is_primary,
         operator_id: options.session.operator_id,
         session_id: options.session.session_id,
         work_context_id: options.context.work_context_id,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bli-cockpit/cli",
-  "version": "0.1.3",
+  "version": "0.1.5",
   "private": false,
   "type": "module",
   "bin": {
@@ -26,6 +26,6 @@
     "test": "node dist/cli.js --help && node ../../scripts/assert-public-package-pack.mjs --workspace=@bli-cockpit/cli"
   },
   "dependencies": {
-    "@bli-cockpit/telemetry-core": "0.1.1"
+    "@bli-cockpit/telemetry-core": "0.1.3"
   }
 }