@letta-ai/dreams 0.0.1 → 0.0.4

package/dist/cloud/upload.js

@@ -54,15 +54,19 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.runUpload = runUpload;
 const path = __importStar(require("node:path"));
 const index_1 = require("./../auth/index");
-const claude_1 = require("./../local/claude");
+const bindings_1 = require("./../local/bindings");
 const config_1 = require("./../local/config");
 const db_1 = require("./../local/db");
 const dreamignore_1 = require("./../local/dreamignore");
 const leases_1 = require("./../local/leases");
 const scan_1 = require("./../local/scan");
 const blobs_1 = require("./../local/blobs");
+const skill_1 = require("./../skill");
 const version_1 = require("./../version");
 const client_1 = require("./client");
+/** Upload schemas stay in the upload layer — not on SourceAdapter ([LET-10314]). */
+const CLAUDE_SOURCE_SCHEMA = "claude-code-jsonl-v1";
+const CODEX_SOURCE_SCHEMA = "codex-jsonl-v1";
 /**
  * Final local policy gate: ONE fresh roots/.dreamignore snapshot drives both the
  * eligibility decision and the fingerprint, so a tightening cannot be recorded
@@ -77,11 +81,28 @@ function evaluatePolicy(db, cwd) {
     const eligible = !(0, dreamignore_1.isIgnored)((0, dreamignore_1.parseDreamignore)(ignoreText), path.relative(approvedRoot, cwd), true);
     return { eligible, approvedRoot, fingerprint: (0, scan_1.policyFingerprint)(roots, approvedRoot, ignoreText) };
 }
-const SOURCE_SCHEMA = "claude-code-jsonl-v1";
-const UPLOAD_CAPABLE = new Set(["active", "backfilling", "discovered", "awaiting_approval"]);
+/** Must match Cloud upload-capable statuses (LET-10231). */
+const UPLOAD_CAPABLE = new Set(["active", "backfilling", "detected"]);
 const MAX_BACKOFF_MS = 5 * 60_000;
-function localSourceId() {
-    return `src_${claude_1.CLAUDE_SOURCE_TYPE}`;
+function sourceRegistrationInput(installationId, adapter) {
+    // Omit status so Cloud preserves lifecycle on re-register (create defaults to active).
+    return {
+        sourceType: adapter.sourceType,
+        sourceKey: adapter.sourceKey,
+        displayName: adapter.displayName,
+        installationId,
+        adapterVersion: adapter.adapterVersion,
+        cliVersion: (0, version_1.packageVersion)(),
+        skillVersion: (0, skill_1.installedSkillVersion)(),
+        platform: process.platform,
+    };
+}
+function sourceSchemaFor(adapter) {
+    if (adapter.sourceType === "claude-code")
+        return CLAUDE_SOURCE_SCHEMA;
+    if (adapter.sourceType === "codex")
+        return CODEX_SOURCE_SCHEMA;
+    throw new Error(`no upload schema registered for source type: ${adapter.sourceType}`);
 }
 function dueRows(db, localSource) {
     return db
@@ -91,7 +112,14 @@ function dueRows(db, localSource) {
        WHERE q.source_id = ? AND q.state = 'pending'
          AND (q.next_attempt_at IS NULL OR q.next_attempt_at <= ?)
        ORDER BY CASE q.priority WHEN 'live' THEN 0 WHEN 'backfill' THEN 1 ELSE 2 END,
-                q.session_id, q.generation, q.start_offset`)
+                CASE
+                  WHEN q.priority = 'backfill' AND sf.mtime_ms IS NULL THEN 1
+                  ELSE 0
+                END,
+                CASE WHEN q.priority = 'backfill' THEN sf.mtime_ms END DESC,
+                q.session_id ASC,
+                q.generation ASC,
+                q.start_offset ASC`)
         .all(localSource, (0, db_1.nowIso)());
 }
 function backoffMs(attempts, retryAfterMs) {
@@ -186,7 +214,8 @@ function refreshPolicyFingerprint(db, row, fingerprint, owner) {
 }
 async function runUpload(db, options) {
     const owner = options.leaseOwner;
-    const local = localSourceId();
+    const adapter = options.adapter;
+    const local = adapter.sourceId;
     const summary = {
         local_source_id: null,
         server_source_id: null,
@@ -199,68 +228,86 @@ async function runUpload(db, options) {
         retry_scheduled: 0,
         canceled_by_policy: 0,
         bytes_uploaded: 0,
+        rows_processed: 0,
+        budget_exhausted: false,
         stopped_reason: null,
+        stopped_code: null,
         partial: false,
     };
+    const maxRows = options.maxRows;
     const sourceExists = db.prepare("SELECT id FROM sources WHERE id = ?").get(local);
     if (!sourceExists)
         return summary; // nothing scanned yet
     summary.local_source_id = local;
-    let due = dueRows(db, local);
+    const due = dueRows(db, local);
     summary.due = due.length;
     if (due.length === 0)
         return summary; // no eligible pending rows => no auth, no network
-    // Register the source fresh (server-idempotent). Per [02] this also provisions
-    // and returns the workspace, so there is no separate workspace call. Registering
-    // per invocation, bound to the current base URL, avoids reusing a prior env's IDs.
-    let registration;
-    try {
+    // Single budgeted loop: local size/blob preflight first; lazy-register on the
+    // first sendable row so oversized-only passes make zero Cloud requests.
+    let registration = null;
+    let bindings = null;
+    let reRegistered = false;
+    let acceptedAny = false;
+    async function ensureRegistered() {
+        if (registration && bindings)
+            return true;
+        options.onProgress?.();
         if (!(0, leases_1.renewLease)(db, leases_1.SOURCE_LEASE, owner))
             throw new leases_1.LeaseLostError(leases_1.SOURCE_LEASE);
-        registration = await (0, client_1.registerSource)({
-            sourceType: claude_1.CLAUDE_SOURCE_TYPE,
-            sourceKey: claude_1.CLAUDE_SOURCE_TYPE,
-            displayName: "Claude Code",
-            installationId: options.installationId,
-            adapterVersion: claude_1.CLAUDE_ADAPTER_VERSION,
-        });
-    }
-    catch (error) {
-        if (error instanceof index_1.NotAuthenticatedError) {
-            summary.stopped_reason = "not authenticated: run `dreams auth login`";
-            summary.partial = true;
-            return summary;
+        try {
+            registration = await (0, client_1.registerSource)(sourceRegistrationInput(options.installationId, adapter));
         }
-        if (error instanceof client_1.CloudAuthError) {
-            summary.stopped_reason = `authentication failed: ${error.message}`;
-            summary.partial = true;
-            return summary;
+        catch (error) {
+            if (error instanceof index_1.NotAuthenticatedError) {
+                summary.stopped_reason = "not authenticated: run `dreams auth login`";
+                summary.stopped_code = "needs_auth";
+                summary.partial = true;
+                return false;
+            }
+            if (error instanceof client_1.CloudAuthError) {
+                summary.stopped_reason = `authentication failed: ${error.message}`;
+                summary.stopped_code = "needs_auth";
+                summary.partial = true;
+                return false;
+            }
+            if (error instanceof client_1.CloudError) {
+                summary.stopped_reason = `could not resolve source: ${error.message}`;
+                summary.stopped_code = error.retryable ? "network" : "cloud_rejected";
+                summary.partial = true;
+                return false;
+            }
+            throw error;
         }
-        if (error instanceof client_1.CloudError) {
-            summary.stopped_reason = `could not resolve source: ${error.message}`;
+        bindings = (0, bindings_1.persistSourceBindings)(db, local, registration, owner);
+        summary.server_source_id = bindings.serverSourceId;
+        summary.source_status = bindings.status;
+        summary.workspace_id = bindings.workspaceId;
+        if (!UPLOAD_CAPABLE.has(bindings.status)) {
+            summary.stopped_reason = `source is ${bindings.status} (not upload-capable)`;
+            summary.stopped_code = "cloud_rejected";
             summary.partial = true;
-            return summary;
+            return false;
         }
-        throw error;
+        return true;
     }
-    summary.server_source_id = registration.serverSourceId;
-    summary.source_status = registration.status;
-    summary.workspace_id = registration.workspaceId;
-    persistBindings(db, registration, owner);
-    if (!UPLOAD_CAPABLE.has(registration.status)) {
-        summary.stopped_reason = `source is ${registration.status} (not upload-capable)`;
-        summary.partial = true;
-        return summary;
-    }
-    let reRegistered = false;
     for (let index = 0; index < due.length; index++) {
+        if (options.shouldCancel?.()) {
+            summary.partial = true;
+            break;
+        }
+        if (maxRows !== undefined && summary.rows_processed >= maxRows) {
+            summary.budget_exhausted = true;
+            break;
+        }
         const row = due[index];
+        options.onProgress?.();
         if (!(0, leases_1.renewLease)(db, leases_1.SOURCE_LEASE, owner))
             throw new leases_1.LeaseLostError(leases_1.SOURCE_LEASE);
-        // Prepare the payload first.
+        summary.rows_processed++;
         let bytes;
         try {
-            bytes = (0, blobs_1.readBlob)(row.content_sha256); // verifies the hash
+            bytes = (0, blobs_1.readBlob)(row.content_sha256);
         }
         catch (error) {
             const code = error.code === "ENOENT" ? "blob_missing" : "blob_corrupt";
@@ -269,6 +316,13 @@ async function runUpload(db, options) {
             summary.partial = true;
             continue;
         }
+        const encoded = (0, client_1.encodeUploadContent)(bytes);
+        if (!encoded.ok) {
+            quarantineRow(db, row, encoded.code, encoded.message, owner);
+            summary.quarantined++;
+            summary.partial = true;
+            continue;
+        }
         // Final local policy gate (one snapshot for eligibility AND fingerprint),
         // immediately before the send — the policy linearization point.
         const policy = evaluatePolicy(db, row.cwd);
@@ -279,7 +333,14 @@ async function runUpload(db, options) {
         }
         if (policy.fingerprint)
             refreshPolicyFingerprint(db, row, policy.fingerprint, owner);
+        if (options.shouldCancel?.()) {
+            summary.partial = true;
+            break;
+        }
+        if (!(await ensureRegistered()))
+            return summary;
         // Renew immediately before the network attempt so the request cannot outlive the lease.
+        options.onProgress?.();
         if (!(0, leases_1.renewLease)(db, leases_1.SOURCE_LEASE, owner))
             throw new leases_1.LeaseLostError(leases_1.SOURCE_LEASE);
         let outcome;
@@ -289,21 +350,24 @@ async function runUpload(db, options) {
                 clientUploadId: row.upload_id,
                 contentSha256: row.content_sha256,
                 bytes,
-                sourceSchema: SOURCE_SCHEMA,
+                encodedContent: encoded.content,
+                sourceSchema: sourceSchemaFor(adapter),
                 sourceMetadata: { session_id: row.session_id, generation: row.generation, start_offset: row.start_offset, end_offset: row.end_offset },
                 priority: row.priority,
                 cliVersion: (0, version_1.packageVersion)(),
-                adapterVersion: claude_1.CLAUDE_ADAPTER_VERSION,
+                adapterVersion: adapter.adapterVersion,
             });
         }
         catch (error) {
             if (error instanceof index_1.NotAuthenticatedError) {
                 summary.stopped_reason = "not authenticated: run `dreams auth login`";
+                summary.stopped_code = "needs_auth";
                 summary.partial = true;
                 return summary;
             }
             if (error instanceof client_1.CloudAuthError) {
                 summary.stopped_reason = `authentication failed: ${error.message}`;
+                summary.stopped_code = "needs_auth";
                 summary.partial = true;
                 return summary;
             }
@@ -319,22 +383,27 @@ async function runUpload(db, options) {
                 markAccepted(db, row, outcome.uploadId, outcome.acceptedAt, owner);
                 summary.accepted++;
                 summary.bytes_uploaded += bytes.length;
+                acceptedAny = true;
                 break;
             case "duplicate":
                 markAccepted(db, row, outcome.uploadId, outcome.acceptedAt, owner);
                 summary.duplicate++;
+                acceptedAny = true;
                 break;
             case "unauthenticated":
                 summary.stopped_reason = `authentication failed: ${outcome.message}`;
+                summary.stopped_code = "needs_auth";
                 summary.partial = true;
                 return summary;
             case "source_unusable":
                 summary.stopped_reason = `source not upload-capable: ${outcome.message}`;
+                summary.stopped_code = "cloud_rejected";
                 summary.partial = true;
                 return summary;
             case "not_found":
                 if (reRegistered) {
                     summary.stopped_reason = `source not found after re-registration: ${outcome.message}`;
+                    summary.stopped_code = "cloud_rejected";
                     summary.partial = true;
                     return summary;
                 }
@@ -344,20 +413,15 @@ async function runUpload(db, options) {
                     // cannot exceed the lease TTL before persistBindings commits (P2).
                     if (!(0, leases_1.renewLease)(db, leases_1.SOURCE_LEASE, owner))
                         throw new leases_1.LeaseLostError(leases_1.SOURCE_LEASE);
-                    const again = await (0, client_1.registerSource)({
-                        sourceType: claude_1.CLAUDE_SOURCE_TYPE,
-                        sourceKey: claude_1.CLAUDE_SOURCE_TYPE,
-                        displayName: "Claude Code",
-                        installationId: options.installationId,
-                        adapterVersion: claude_1.CLAUDE_ADAPTER_VERSION,
-                    });
-                    summary.server_source_id = again.serverSourceId;
-                    summary.source_status = again.status;
-                    summary.workspace_id = again.workspaceId;
-                    persistBindings(db, again, owner);
-                    registration.serverSourceId = again.serverSourceId;
+                    const again = await (0, client_1.registerSource)(sourceRegistrationInput(options.installationId, adapter));
+                    bindings = (0, bindings_1.persistSourceBindings)(db, local, again, owner);
+                    summary.server_source_id = bindings.serverSourceId;
+                    summary.source_status = bindings.status;
+                    summary.workspace_id = bindings.workspaceId;
+                    registration.serverSourceId = bindings.serverSourceId;
                 }
                 index--; // retry the same row once with the refreshed source id
+                summary.rows_processed--; // retry does not consume an extra budget slot
                 break;
             case "quarantine":
                 quarantineRow(db, row, outcome.code, outcome.message, owner);
@@ -370,15 +434,31 @@ async function runUpload(db, options) {
                 break;
         }
     }
+    if (maxRows !== undefined &&
+        summary.rows_processed >= maxRows &&
+        due.length > summary.rows_processed) {
+        summary.budget_exhausted = true;
+    }
+    // Cloud promotes detected→active only after durable acceptance. Upload acks do
+    // not echo source status, so refresh via re-register when we may have flipped.
+    if (acceptedAny && summary.source_status === "detected" && registration) {
+        try {
+            // Heartbeat before the second consecutive network call so a long upload
+            // pass cannot let SYNC_WORKER_LEASE expire across register+refresh.
+            options.onProgress?.();
+            if (!(0, leases_1.renewLease)(db, leases_1.SOURCE_LEASE, owner))
+                throw new leases_1.LeaseLostError(leases_1.SOURCE_LEASE);
+            const refreshed = await (0, client_1.registerSource)(sourceRegistrationInput(options.installationId, adapter));
+            options.onProgress?.();
+            bindings = (0, bindings_1.persistSourceBindings)(db, local, refreshed, owner);
+            summary.server_source_id = bindings.serverSourceId;
+            summary.source_status = bindings.status;
+            summary.workspace_id = bindings.workspaceId;
+        }
+        catch {
+            // Best-effort refresh; accepted uploads already committed. Next detect/upload
+            // will reconcile authoritative Cloud status.
+        }
+    }
     return summary;
 }
-function persistBindings(db, registration, owner) {
-    const base = process.env.LETTA_BASE_URL || "https://api.letta.com";
-    (0, db_1.tx)(db, () => {
-        (0, leases_1.assertLeaseHeld)(db, leases_1.SOURCE_LEASE, owner);
-        db.prepare(`INSERT INTO workspace_bindings (id, api_base_url, workspace_id, organization_id, user_id, created_at, updated_at)
-       VALUES (1, ?, ?, NULL, NULL, ?, ?)
-       ON CONFLICT(id) DO UPDATE SET api_base_url = excluded.api_base_url, workspace_id = excluded.workspace_id, updated_at = excluded.updated_at`).run(base, registration.workspaceId, (0, db_1.nowIso)(), (0, db_1.nowIso)());
-        db.prepare("UPDATE sources SET server_source_id = ?, status = ?, updated_at = ? WHERE id = ?").run(registration.serverSourceId, registration.status, (0, db_1.nowIso)(), localSourceId());
-    });
-}