@letta-ai/dreams 0.0.1 → 0.0.3

package/dist/cloud/client.js

@@ -42,8 +42,11 @@ var __importStar = (this && this.__importStar) || (function () {
     };
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.CloudError = exports.CloudAuthError = void 0;
+exports.CLOUD_UPLOAD_MAX_CONTENT_CHARS = exports.CloudError = exports.CloudAuthError = void 0;
+exports.setCloudUploadMaxContentCharsForTests = setCloudUploadMaxContentCharsForTests;
+exports.encodeUploadContent = encodeUploadContent;
 exports.registerSource = registerSource;
+exports.createSnapshot = createSnapshot;
 exports.uploadObject = uploadObject;
 const zlib = __importStar(require("node:zlib"));
 const index_1 = require("../auth/index");
@@ -71,6 +74,33 @@ exports.CloudError = CloudError;
 // Comfortably below the source lease TTL (30s) so a stalled request cannot
 // outlive the lease and let another process reclaim it mid-flight.
 const REQUEST_TIMEOUT_MS = 20_000;
+/**
+ * Cloud `createUploadBodySchema.content` max: base64 of gzip'd bytes (not the
+ * uncompressed transcript). Enforced locally so oversized payloads quarantine
+ * without a wasted round trip ([Amelia] two-limit model).
+ */
+exports.CLOUD_UPLOAD_MAX_CONTENT_CHARS = 8 * 1024 * 1024;
+let cloudUploadMaxContentCharsForTests = null;
+/** Test-only override for the encoded upload size gate. Pass null to clear. */
+function setCloudUploadMaxContentCharsForTests(value) {
+    cloudUploadMaxContentCharsForTests = value;
+}
+function cloudUploadMaxContentChars() {
+    return cloudUploadMaxContentCharsForTests ?? exports.CLOUD_UPLOAD_MAX_CONTENT_CHARS;
+}
+/** Gzip + base64 encode transcript bytes and enforce the Cloud content char cap. */
+function encodeUploadContent(bytes, maxContentChars = cloudUploadMaxContentChars()) {
+    const content = zlib.gzipSync(bytes).toString("base64");
+    if (content.length > maxContentChars) {
+        return {
+            ok: false,
+            code: "payload_too_large",
+            message: `gzip+base64 upload content is ${content.length} chars; Cloud maximum is ${maxContentChars}`,
+            encodedLength: content.length,
+        };
+    }
+    return { ok: true, content, encodedLength: content.length };
+}
 function errorDetail(error) {
     const cause = error.cause;
     return cause?.code || cause?.message || (error instanceof Error ? error.message : String(error));
@@ -144,7 +174,7 @@ function classifyAuthOrRetry(status, json, path) {
  * id, so it is also how the CLI resolves the workspace.
  */
 async function registerSource(input) {
-    const r = await request("POST", "/v1/dreams/sources", {
+    const body = {
         source_type: input.sourceType,
         source_key: input.sourceKey,
         display_name: input.displayName,
@@ -152,7 +182,16 @@ async function registerSource(input) {
         adapter_version: input.adapterVersion,
         privacy_config_hash: input.privacyConfigHash ?? null,
         backfill_policy: input.backfillPolicy ?? null,
-    });
+    };
+    if (input.status != null)
+        body.status = input.status;
+    if (input.cliVersion != null)
+        body.cli_version = input.cliVersion;
+    if (input.skillVersion != null)
+        body.skill_version = input.skillVersion;
+    if (input.platform != null)
+        body.platform = input.platform;
+    const r = await request("POST", "/v1/dreams/sources", body);
     if (r.status !== 200 && r.status !== 201)
         classifyAuthOrRetry(r.status, r.json, "sources");
     const id = r.json?.id;
@@ -164,14 +203,59 @@ async function registerSource(input) {
         workspaceId: typeof r.json?.workspace_id === "string" ? r.json.workspace_id : null,
     };
 }
+/**
+ * Narrow transport for skill-authored snapshots ([11] / LET-9833).
+ * Validation beyond JSON-object + size belongs on the server.
+ *
+ * Cloud contract: POST /v1/dreams/workspaces/{workspaceId}/snapshots
+ * with body `{ type, data, client_request_id }`.
+ */
+async function createSnapshot(input) {
+    const path = `/v1/dreams/workspaces/${encodeURIComponent(input.workspaceId)}/snapshots`;
+    const r = await request("POST", path, {
+        type: input.type,
+        client_request_id: input.clientRequestId,
+        data: input.data,
+    });
+    if (r.status !== 200 && r.status !== 201)
+        classifyAuthOrRetry(r.status, r.json, "snapshots");
+    const id = r.json?.id;
+    if (typeof id !== "string")
+        throw new CloudError("snapshot response missing id", { retryable: true });
+    return {
+        id,
+        type: typeof r.json?.type === "string" ? r.json.type : input.type,
+        workspaceId: input.workspaceId,
+        clientRequestId: typeof r.json?.client_request_id === "string" ? r.json.client_request_id : input.clientRequestId,
+        status: typeof r.json?.status === "string" ? r.json.status : "stored",
+        createdAt: typeof r.json?.created_at === "string" ? r.json.created_at : null,
+    };
+}
 async function uploadObject(input) {
-    const content = zlib.gzipSync(input.bytes).toString("base64");
+    // Prefer a preflight-encoded body; otherwise encode here (defense in depth).
+    const gated = input.encodedContent !== undefined
+        ? input.encodedContent.length > cloudUploadMaxContentChars()
+            ? {
+                ok: false,
+                code: "payload_too_large",
+                message: `gzip+base64 upload content is ${input.encodedContent.length} chars; Cloud maximum is ${cloudUploadMaxContentChars()}`,
+                encodedLength: input.encodedContent.length,
+            }
+            : {
+                ok: true,
+                content: input.encodedContent,
+                encodedLength: input.encodedContent.length,
+            }
+        : encodeUploadContent(input.bytes);
+    if (!gated.ok) {
+        return { kind: "quarantine", code: gated.code, message: gated.message };
+    }
     const r = await request("POST", "/v1/dreams/uploads", {
         source_id: input.sourceId,
         client_upload_id: input.clientUploadId,
         content_sha256: input.contentSha256,
         content_encoding: "gzip+base64",
-        content,
+        content: gated.content,
         source_schema: input.sourceSchema,
         source_metadata: input.sourceMetadata,
         priority: input.priority,

package/dist/cloud/upload.js

@@ -54,15 +54,19 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.runUpload = runUpload;
 const path = __importStar(require("node:path"));
 const index_1 = require("./../auth/index");
-const claude_1 = require("./../local/claude");
+const bindings_1 = require("./../local/bindings");
 const config_1 = require("./../local/config");
 const db_1 = require("./../local/db");
 const dreamignore_1 = require("./../local/dreamignore");
 const leases_1 = require("./../local/leases");
 const scan_1 = require("./../local/scan");
 const blobs_1 = require("./../local/blobs");
+const skill_1 = require("./../skill");
 const version_1 = require("./../version");
 const client_1 = require("./client");
+/** Upload schemas stay in the upload layer — not on SourceAdapter ([LET-10314]). */
+const CLAUDE_SOURCE_SCHEMA = "claude-code-jsonl-v1";
+const CODEX_SOURCE_SCHEMA = "codex-jsonl-v1";
 /**
  * Final local policy gate: ONE fresh roots/.dreamignore snapshot drives both the
  * eligibility decision and the fingerprint, so a tightening cannot be recorded
@@ -77,11 +81,28 @@ function evaluatePolicy(db, cwd) {
     const eligible = !(0, dreamignore_1.isIgnored)((0, dreamignore_1.parseDreamignore)(ignoreText), path.relative(approvedRoot, cwd), true);
     return { eligible, approvedRoot, fingerprint: (0, scan_1.policyFingerprint)(roots, approvedRoot, ignoreText) };
 }
-const SOURCE_SCHEMA = "claude-code-jsonl-v1";
-const UPLOAD_CAPABLE = new Set(["active", "backfilling", "discovered", "awaiting_approval"]);
+/** Must match Cloud upload-capable statuses (LET-10231). */
+const UPLOAD_CAPABLE = new Set(["active", "backfilling", "detected"]);
 const MAX_BACKOFF_MS = 5 * 60_000;
-function localSourceId() {
-    return `src_${claude_1.CLAUDE_SOURCE_TYPE}`;
+function sourceRegistrationInput(installationId, adapter) {
+    // Omit status so Cloud preserves lifecycle on re-register (create defaults to active).
+    return {
+        sourceType: adapter.sourceType,
+        sourceKey: adapter.sourceKey,
+        displayName: adapter.displayName,
+        installationId,
+        adapterVersion: adapter.adapterVersion,
+        cliVersion: (0, version_1.packageVersion)(),
+        skillVersion: (0, skill_1.installedSkillVersion)(),
+        platform: process.platform,
+    };
+}
+function sourceSchemaFor(adapter) {
+    if (adapter.sourceType === "claude-code")
+        return CLAUDE_SOURCE_SCHEMA;
+    if (adapter.sourceType === "codex")
+        return CODEX_SOURCE_SCHEMA;
+    throw new Error(`no upload schema registered for source type: ${adapter.sourceType}`);
 }
 function dueRows(db, localSource) {
     return db
@@ -91,7 +112,14 @@ function dueRows(db, localSource) {
        WHERE q.source_id = ? AND q.state = 'pending'
          AND (q.next_attempt_at IS NULL OR q.next_attempt_at <= ?)
        ORDER BY CASE q.priority WHEN 'live' THEN 0 WHEN 'backfill' THEN 1 ELSE 2 END,
-                q.session_id, q.generation, q.start_offset`)
+                CASE
+                  WHEN q.priority = 'backfill' AND sf.mtime_ms IS NULL THEN 1
+                  ELSE 0
+                END,
+                CASE WHEN q.priority = 'backfill' THEN sf.mtime_ms END DESC,
+                q.session_id ASC,
+                q.generation ASC,
+                q.start_offset ASC`)
         .all(localSource, (0, db_1.nowIso)());
 }
 function backoffMs(attempts, retryAfterMs) {
@@ -186,7 +214,8 @@ function refreshPolicyFingerprint(db, row, fingerprint, owner) {
 }
 async function runUpload(db, options) {
     const owner = options.leaseOwner;
-    const local = localSourceId();
+    const adapter = options.adapter;
+    const local = adapter.sourceId;
     const summary = {
         local_source_id: null,
         server_source_id: null,
@@ -199,68 +228,86 @@ async function runUpload(db, options) {
         retry_scheduled: 0,
         canceled_by_policy: 0,
         bytes_uploaded: 0,
+        rows_processed: 0,
+        budget_exhausted: false,
         stopped_reason: null,
+        stopped_code: null,
         partial: false,
     };
+    const maxRows = options.maxRows;
     const sourceExists = db.prepare("SELECT id FROM sources WHERE id = ?").get(local);
     if (!sourceExists)
         return summary; // nothing scanned yet
     summary.local_source_id = local;
-    let due = dueRows(db, local);
+    const due = dueRows(db, local);
     summary.due = due.length;
     if (due.length === 0)
         return summary; // no eligible pending rows => no auth, no network
-    // Register the source fresh (server-idempotent). Per [02] this also provisions
-    // and returns the workspace, so there is no separate workspace call. Registering
-    // per invocation, bound to the current base URL, avoids reusing a prior env's IDs.
-    let registration;
-    try {
+    // Single budgeted loop: local size/blob preflight first; lazy-register on the
+    // first sendable row so oversized-only passes make zero Cloud requests.
+    let registration = null;
+    let bindings = null;
+    let reRegistered = false;
+    let acceptedAny = false;
+    async function ensureRegistered() {
+        if (registration && bindings)
+            return true;
+        options.onProgress?.();
         if (!(0, leases_1.renewLease)(db, leases_1.SOURCE_LEASE, owner))
             throw new leases_1.LeaseLostError(leases_1.SOURCE_LEASE);
-        registration = await (0, client_1.registerSource)({
-            sourceType: claude_1.CLAUDE_SOURCE_TYPE,
-            sourceKey: claude_1.CLAUDE_SOURCE_TYPE,
-            displayName: "Claude Code",
-            installationId: options.installationId,
-            adapterVersion: claude_1.CLAUDE_ADAPTER_VERSION,
-        });
-    }
-    catch (error) {
-        if (error instanceof index_1.NotAuthenticatedError) {
-            summary.stopped_reason = "not authenticated: run `dreams auth login`";
-            summary.partial = true;
-            return summary;
+        try {
+            registration = await (0, client_1.registerSource)(sourceRegistrationInput(options.installationId, adapter));
         }
-        if (error instanceof client_1.CloudAuthError) {
-            summary.stopped_reason = `authentication failed: ${error.message}`;
-            summary.partial = true;
-            return summary;
+        catch (error) {
+            if (error instanceof index_1.NotAuthenticatedError) {
+                summary.stopped_reason = "not authenticated: run `dreams auth login`";
+                summary.stopped_code = "needs_auth";
+                summary.partial = true;
+                return false;
+            }
+            if (error instanceof client_1.CloudAuthError) {
+                summary.stopped_reason = `authentication failed: ${error.message}`;
+                summary.stopped_code = "needs_auth";
+                summary.partial = true;
+                return false;
+            }
+            if (error instanceof client_1.CloudError) {
+                summary.stopped_reason = `could not resolve source: ${error.message}`;
+                summary.stopped_code = error.retryable ? "network" : "cloud_rejected";
+                summary.partial = true;
+                return false;
+            }
+            throw error;
         }
-        if (error instanceof client_1.CloudError) {
-            summary.stopped_reason = `could not resolve source: ${error.message}`;
+        bindings = (0, bindings_1.persistSourceBindings)(db, local, registration, owner);
+        summary.server_source_id = bindings.serverSourceId;
+        summary.source_status = bindings.status;
+        summary.workspace_id = bindings.workspaceId;
+        if (!UPLOAD_CAPABLE.has(bindings.status)) {
+            summary.stopped_reason = `source is ${bindings.status} (not upload-capable)`;
+            summary.stopped_code = "cloud_rejected";
             summary.partial = true;
-            return summary;
+            return false;
         }
-        throw error;
+        return true;
     }
-    summary.server_source_id = registration.serverSourceId;
-    summary.source_status = registration.status;
-    summary.workspace_id = registration.workspaceId;
-    persistBindings(db, registration, owner);
-    if (!UPLOAD_CAPABLE.has(registration.status)) {
-        summary.stopped_reason = `source is ${registration.status} (not upload-capable)`;
-        summary.partial = true;
-        return summary;
-    }
-    let reRegistered = false;
     for (let index = 0; index < due.length; index++) {
+        if (options.shouldCancel?.()) {
+            summary.partial = true;
+            break;
+        }
+        if (maxRows !== undefined && summary.rows_processed >= maxRows) {
+            summary.budget_exhausted = true;
+            break;
+        }
         const row = due[index];
+        options.onProgress?.();
         if (!(0, leases_1.renewLease)(db, leases_1.SOURCE_LEASE, owner))
             throw new leases_1.LeaseLostError(leases_1.SOURCE_LEASE);
-        // Prepare the payload first.
+        summary.rows_processed++;
         let bytes;
         try {
-            bytes = (0, blobs_1.readBlob)(row.content_sha256); // verifies the hash
+            bytes = (0, blobs_1.readBlob)(row.content_sha256);
         }
         catch (error) {
             const code = error.code === "ENOENT" ? "blob_missing" : "blob_corrupt";
@@ -269,6 +316,13 @@ async function runUpload(db, options) {
             summary.partial = true;
             continue;
         }
+        const encoded = (0, client_1.encodeUploadContent)(bytes);
+        if (!encoded.ok) {
+            quarantineRow(db, row, encoded.code, encoded.message, owner);
+            summary.quarantined++;
+            summary.partial = true;
+            continue;
+        }
         // Final local policy gate (one snapshot for eligibility AND fingerprint),
         // immediately before the send — the policy linearization point.
         const policy = evaluatePolicy(db, row.cwd);
@@ -279,7 +333,14 @@ async function runUpload(db, options) {
         }
         if (policy.fingerprint)
             refreshPolicyFingerprint(db, row, policy.fingerprint, owner);
+        if (options.shouldCancel?.()) {
+            summary.partial = true;
+            break;
+        }
+        if (!(await ensureRegistered()))
+            return summary;
         // Renew immediately before the network attempt so the request cannot outlive the lease.
+        options.onProgress?.();
         if (!(0, leases_1.renewLease)(db, leases_1.SOURCE_LEASE, owner))
             throw new leases_1.LeaseLostError(leases_1.SOURCE_LEASE);
         let outcome;
@@ -289,21 +350,24 @@ async function runUpload(db, options) {
                 clientUploadId: row.upload_id,
                 contentSha256: row.content_sha256,
                 bytes,
-                sourceSchema: SOURCE_SCHEMA,
+                encodedContent: encoded.content,
+                sourceSchema: sourceSchemaFor(adapter),
                 sourceMetadata: { session_id: row.session_id, generation: row.generation, start_offset: row.start_offset, end_offset: row.end_offset },
                 priority: row.priority,
                 cliVersion: (0, version_1.packageVersion)(),
-                adapterVersion: claude_1.CLAUDE_ADAPTER_VERSION,
+                adapterVersion: adapter.adapterVersion,
             });
         }
         catch (error) {
             if (error instanceof index_1.NotAuthenticatedError) {
                 summary.stopped_reason = "not authenticated: run `dreams auth login`";
+                summary.stopped_code = "needs_auth";
                 summary.partial = true;
                 return summary;
             }
             if (error instanceof client_1.CloudAuthError) {
                 summary.stopped_reason = `authentication failed: ${error.message}`;
+                summary.stopped_code = "needs_auth";
                 summary.partial = true;
                 return summary;
             }
@@ -319,22 +383,27 @@ async function runUpload(db, options) {
                 markAccepted(db, row, outcome.uploadId, outcome.acceptedAt, owner);
                 summary.accepted++;
                 summary.bytes_uploaded += bytes.length;
+                acceptedAny = true;
                 break;
             case "duplicate":
                 markAccepted(db, row, outcome.uploadId, outcome.acceptedAt, owner);
                 summary.duplicate++;
+                acceptedAny = true;
                 break;
             case "unauthenticated":
                 summary.stopped_reason = `authentication failed: ${outcome.message}`;
+                summary.stopped_code = "needs_auth";
                 summary.partial = true;
                 return summary;
             case "source_unusable":
                 summary.stopped_reason = `source not upload-capable: ${outcome.message}`;
+                summary.stopped_code = "cloud_rejected";
                 summary.partial = true;
                 return summary;
             case "not_found":
                 if (reRegistered) {
                     summary.stopped_reason = `source not found after re-registration: ${outcome.message}`;
+                    summary.stopped_code = "cloud_rejected";
                     summary.partial = true;
                     return summary;
                 }
@@ -344,20 +413,15 @@ async function runUpload(db, options) {
                     // cannot exceed the lease TTL before persistBindings commits (P2).
                     if (!(0, leases_1.renewLease)(db, leases_1.SOURCE_LEASE, owner))
                         throw new leases_1.LeaseLostError(leases_1.SOURCE_LEASE);
-                    const again = await (0, client_1.registerSource)({
-                        sourceType: claude_1.CLAUDE_SOURCE_TYPE,
-                        sourceKey: claude_1.CLAUDE_SOURCE_TYPE,
-                        displayName: "Claude Code",
-                        installationId: options.installationId,
-                        adapterVersion: claude_1.CLAUDE_ADAPTER_VERSION,
-                    });
-                    summary.server_source_id = again.serverSourceId;
-                    summary.source_status = again.status;
-                    summary.workspace_id = again.workspaceId;
-                    persistBindings(db, again, owner);
-                    registration.serverSourceId = again.serverSourceId;
+                    const again = await (0, client_1.registerSource)(sourceRegistrationInput(options.installationId, adapter));
+                    bindings = (0, bindings_1.persistSourceBindings)(db, local, again, owner);
+                    summary.server_source_id = bindings.serverSourceId;
+                    summary.source_status = bindings.status;
+                    summary.workspace_id = bindings.workspaceId;
+                    registration.serverSourceId = bindings.serverSourceId;
                 }
                 index--; // retry the same row once with the refreshed source id
+                summary.rows_processed--; // retry does not consume an extra budget slot
                 break;
             case "quarantine":
                 quarantineRow(db, row, outcome.code, outcome.message, owner);
@@ -370,15 +434,31 @@ async function runUpload(db, options) {
                 break;
         }
     }
+    if (maxRows !== undefined &&
+        summary.rows_processed >= maxRows &&
+        due.length > summary.rows_processed) {
+        summary.budget_exhausted = true;
+    }
+    // Cloud promotes detected→active only after durable acceptance. Upload acks do
+    // not echo source status, so refresh via re-register when we may have flipped.
+    if (acceptedAny && summary.source_status === "detected" && registration) {
+        try {
+            // Heartbeat before the second consecutive network call so a long upload
+            // pass cannot let SYNC_WORKER_LEASE expire across register+refresh.
+            options.onProgress?.();
+            if (!(0, leases_1.renewLease)(db, leases_1.SOURCE_LEASE, owner))
+                throw new leases_1.LeaseLostError(leases_1.SOURCE_LEASE);
+            const refreshed = await (0, client_1.registerSource)(sourceRegistrationInput(options.installationId, adapter));
+            options.onProgress?.();
+            bindings = (0, bindings_1.persistSourceBindings)(db, local, refreshed, owner);
+            summary.server_source_id = bindings.serverSourceId;
+            summary.source_status = bindings.status;
+            summary.workspace_id = bindings.workspaceId;
+        }
+        catch {
+            // Best-effort refresh; accepted uploads already committed. Next detect/upload
+            // will reconcile authoritative Cloud status.
+        }
+    }
     return summary;
 }
-function persistBindings(db, registration, owner) {
-    const base = process.env.LETTA_BASE_URL || "https://api.letta.com";
-    (0, db_1.tx)(db, () => {
-        (0, leases_1.assertLeaseHeld)(db, leases_1.SOURCE_LEASE, owner);
-        db.prepare(`INSERT INTO workspace_bindings (id, api_base_url, workspace_id, organization_id, user_id, created_at, updated_at)
-       VALUES (1, ?, ?, NULL, NULL, ?, ?)
-       ON CONFLICT(id) DO UPDATE SET api_base_url = excluded.api_base_url, workspace_id = excluded.workspace_id, updated_at = excluded.updated_at`).run(base, registration.workspaceId, (0, db_1.nowIso)(), (0, db_1.nowIso)());
-        db.prepare("UPDATE sources SET server_source_id = ?, status = ?, updated_at = ? WHERE id = ?").run(registration.serverSourceId, registration.status, (0, db_1.nowIso)(), localSourceId());
-    });
-}