@letta-ai/dreams 0.0.1 → 0.0.4

package/dist/local/scan.js

@@ -46,7 +46,7 @@ exports.reconcile = reconcile;
 exports.listPendingQueue = listPendingQueue;
 const path = __importStar(require("node:path"));
 const blobs_1 = require("./blobs");
-const claude_1 = require("./claude");
+const transcript_bytes_1 = require("./transcript-bytes");
 const config_1 = require("./config");
 const db_1 = require("./db");
 const dreamignore_1 = require("./dreamignore");
@@ -59,6 +59,13 @@ class PolicyChangedError extends Error {
         this.name = "PolicyChangedError";
     }
 }
+/** Raised when shouldCancel trips mid-session so pause can stop a large scan. */
+class ScanCancelledError extends Error {
+    constructor() {
+        super("scan cancelled");
+        this.name = "ScanCancelledError";
+    }
+}
 /**
  * Deterministic fingerprint of the policy that governs a session's root: the full
  * approved-root set plus that root's `.dreamignore`. Stored on each queued row so
@@ -67,9 +74,31 @@ class PolicyChangedError extends Error {
 function policyFingerprint(roots, approvedRoot, ignoreText) {
     return (0, ids_1.sha256Hex)(Buffer.from(JSON.stringify({ roots: [...roots].sort(), approvedRoot, ignore: ignoreText }), "utf8"));
 }
+/**
+ * Soft packing target (`targetSegmentBytes` in design notes): pack ordinary
+ * complete JSONL lines until the next line would exceed this. Shared by every
+ * SourceAdapter — not Codex-specific.
+ */
 const DEFAULT_MAX_SEGMENT_BYTES = 1024 * 1024;
-/** Pure helper retained for unit callers; oversized lines are a hard failure. */
-function chunkCompleteLines(tail, maxSegmentBytes) {
+/**
+ * Hard per-record safety limit for a single complete JSONL line. Lines above
+ * the soft target but at/under this cap flush the preceding segment and queue
+ * alone; only larger lines isolate the session. The final transport gate is
+ * Cloud's gzip+base64 content char cap (enforced at upload time).
+ */
+const DEFAULT_MAX_RECORD_BYTES = 8 * 1024 * 1024;
+/**
+ * Pure helper retained for unit callers.
+ * Lines above `maxSegmentBytes` but at/under `maxRecordBytes` become standalone
+ * segments; only lines above `maxRecordBytes` throw.
+ */
+function chunkCompleteLines(tail, maxSegmentBytes, maxRecordBytes = maxSegmentBytes) {
+    if (!Number.isSafeInteger(maxSegmentBytes) || maxSegmentBytes <= 0) {
+        throw new Error("maxSegmentBytes must be a positive integer");
+    }
+    if (!Number.isSafeInteger(maxRecordBytes) || maxRecordBytes < maxSegmentBytes) {
+        throw new Error("maxRecordBytes must be an integer >= maxSegmentBytes");
+    }
     const segments = [];
     let segmentStart = 0;
     let segmentEnd = 0;
@@ -79,8 +108,18 @@ function chunkCompleteLines(tail, maxSegmentBytes) {
             continue;
         const lineEnd = i + 1;
         const lineLength = lineEnd - lineStart;
-        if (lineLength > maxSegmentBytes)
-            throw new RangeError(`JSONL record is ${lineLength} bytes; maximum is ${maxSegmentBytes}`);
+        if (lineLength > maxRecordBytes) {
+            throw new RangeError(`JSONL record is ${lineLength} bytes; maximum is ${maxRecordBytes}`);
+        }
+        if (lineLength > maxSegmentBytes) {
+            if (segmentEnd > segmentStart)
+                segments.push({ relStart: segmentStart, relEnd: segmentEnd });
+            segments.push({ relStart: lineStart, relEnd: lineEnd });
+            segmentStart = lineEnd;
+            segmentEnd = lineEnd;
+            lineStart = lineEnd;
+            continue;
+        }
         if (segmentEnd > segmentStart && lineEnd - segmentStart > maxSegmentBytes) {
             segments.push({ relStart: segmentStart, relEnd: segmentEnd });
             segmentStart = lineStart;
@@ -92,15 +131,15 @@ function chunkCompleteLines(tail, maxSegmentBytes) {
         segments.push({ relStart: segmentStart, relEnd: segmentEnd });
     return segments;
 }
-function getOrCreateSource(db, owner) {
-    const id = `src_${claude_1.CLAUDE_SOURCE_TYPE}`;
+function getOrCreateSource(db, owner, adapter) {
+    const id = adapter.sourceId;
     return (0, db_1.tx)(db, () => {
         (0, leases_1.assertLeaseHeld)(db, leases_1.SOURCE_LEASE, owner);
         const existing = db.prepare("SELECT id FROM sources WHERE id = ?").get(id);
         if (!existing) {
             const now = (0, db_1.nowIso)();
             db.prepare(`INSERT INTO sources (id, source_type, source_key, display_name, adapter_version, status, created_at, updated_at)
-         VALUES (?, ?, ?, ?, ?, 'active', ?, ?)`).run(id, claude_1.CLAUDE_SOURCE_TYPE, claude_1.CLAUDE_SOURCE_TYPE, "Claude Code", claude_1.CLAUDE_ADAPTER_VERSION, now, now);
+         VALUES (?, ?, ?, ?, ?, 'detected', ?, ?)`).run(id, adapter.sourceType, adapter.sourceKey, adapter.displayName, adapter.adapterVersion, now, now);
         }
         return id;
     });
@@ -131,7 +170,7 @@ function isReplacement(stored, cursor, session, onProgress) {
         return true;
     const statChanged = session.size !== stored.size || session.mtime_ms !== stored.mtime_ms;
     if (statChanged && cursor.queued_through_offset > 0 && stored.prefix_fp) {
-        if ((0, claude_1.prefixFingerprint)(session.real_path, cursor.queued_through_offset, onProgress) !== stored.prefix_fp)
+        if ((0, transcript_bytes_1.prefixFingerprint)(session.real_path, cursor.queued_through_offset, onProgress) !== stored.prefix_fp)
             return true;
     }
     if (stored.first_chunk_fp && session.first_chunk_fp && stored.first_chunk_fp !== session.first_chunk_fp)
@@ -267,21 +306,43 @@ function recordSessionFailure(db, fileId, code, message, owner) {
         db.prepare("UPDATE source_files SET last_error_code = ?, last_error_message = ?, last_error_at = ?, updated_at = ? WHERE id = ?").run(code, message.slice(0, 500), (0, db_1.nowIso)(), (0, db_1.nowIso)(), fileId);
     });
 }
+function clearSessionFailure(db, fileId, owner) {
+    (0, leases_1.assertLeaseHeld)(db, leases_1.SOURCE_LEASE, owner);
+    db.prepare("UPDATE source_files SET last_error_code = NULL, last_error_message = NULL, last_error_at = NULL, updated_at = ? WHERE id = ?").run((0, db_1.nowIso)(), fileId);
+}
 function reconcile(db, options) {
     const maxSegmentBytes = options.maxSegmentBytes ?? DEFAULT_MAX_SEGMENT_BYTES;
     if (!Number.isSafeInteger(maxSegmentBytes) || maxSegmentBytes <= 0)
         throw new Error("maxSegmentBytes must be a positive integer");
+    const maxRecordBytes = options.maxRecordBytes ?? DEFAULT_MAX_RECORD_BYTES;
+    if (!Number.isSafeInteger(maxRecordBytes) || maxRecordBytes < maxSegmentBytes) {
+        throw new Error("maxRecordBytes must be an integer >= maxSegmentBytes");
+    }
     const priority = options.priority ?? "live";
     const owner = options.leaseOwner;
-    const sourceId = getOrCreateSource(db, owner);
+    const adapter = options.adapter;
+    const sourceId = getOrCreateSource(db, owner, adapter);
     const roots = (0, config_1.getApprovedRoots)(db);
     const rulesByRoot = new Map();
     reconcileOrphanBlobs(db, owner);
     const objectsCanceled = reconcilePendingPrivacy(db, sourceId, roots, rulesByRoot, owner);
-    const sessions = (0, claude_1.discoverSessions)(options.projectsDir);
+    const discovered = adapter.discoverSessions();
+    const mode = options.mode ?? "all";
+    let sessions = discovered;
+    if (mode === "live") {
+        const tracked = new Set(db
+            .prepare("SELECT session_id FROM source_files WHERE source_id = ? AND is_current = 1")
+            .all(sourceId).map((row) => row.session_id));
+        const hints = new Set(options.hintSessionIds ?? []);
+        sessions = discovered.filter((session) => tracked.has(session.session_id) || hints.has(session.session_id));
+    }
+    else if (mode === "backfill") {
+        const hints = new Set(options.hintSessionIds ?? []);
+        sessions = discovered.filter((session) => hints.has(session.session_id));
+    }
     const summary = {
         source_id: sourceId,
-        sessions_discovered: sessions.length,
+        sessions_discovered: discovered.length,
         sessions_queued: 0,
         sessions_excluded: 0,
         sessions_skipped_unapproved: 0,
@@ -291,17 +352,26 @@ function reconcile(db, options) {
         segments_enqueued: 0,
         bytes_enqueued: 0,
         queued: [],
+        processed_sessions: [],
     };
     for (const session of sessions) {
+        if (options.shouldCancel?.())
+            break;
+        options.onProgress?.();
         if (!(0, leases_1.renewLease)(db, leases_1.SOURCE_LEASE, owner))
             throw new leases_1.LeaseLostError(leases_1.SOURCE_LEASE);
         let lastRenewedAt = Date.now();
         const keepLeaseAlive = () => {
+            // Cancel checks are cheap and must run during large single-session scans.
+            if (options.shouldCancel?.())
+                throw new ScanCancelledError();
             if (Date.now() - lastRenewedAt < 5_000)
                 return;
             if (!(0, leases_1.renewLease)(db, leases_1.SOURCE_LEASE, owner))
                 throw new leases_1.LeaseLostError(leases_1.SOURCE_LEASE);
             lastRenewedAt = Date.now();
+            // Heartbeat only on the throttled renewal path (not once per chunk).
+            options.onProgress?.();
         };
         let fileId;
         let generation;
@@ -358,7 +428,7 @@ function reconcile(db, options) {
                 prepared.push({
                     upload_id: (0, ids_1.segmentId)({
                         installation_id: options.installationId,
-                        source_key: claude_1.CLAUDE_SOURCE_TYPE,
+                        source_key: adapter.sourceKey,
                         file_identity: session.session_id,
                         generation,
                         start_offset: segmentStart,
@@ -375,7 +445,7 @@ function reconcile(db, options) {
                 segmentLines = [];
                 segmentLength = 0;
             };
-            const verified = (0, claude_1.scanCompleteLinesVerified)(session.real_path, scanStart, session.inode, maxSegmentBytes, (line) => {
+            const verified = (0, transcript_bytes_1.scanCompleteLinesVerified)(session.real_path, scanStart, session.inode, maxRecordBytes, (line) => {
                 while (exclusionIndex < explicitExclusions.length && explicitExclusions[exclusionIndex].end_offset <= line.start_offset) {
                     exclusionIndex++;
                 }
@@ -387,6 +457,10 @@ function reconcile(db, options) {
                     coalesceRange(excludedLines, line.start_offset, line.end_offset);
                     return;
                 }
+                // Soft packing target: flush before adding a line that would overflow.
+                // A single complete record above the target (but under the hard cap)
+                // becomes its own segment so real Codex multimodal lines (~2.7 MiB)
+                // still upload.
                 if (segmentLength > 0 && segmentLength + line.bytes.length > maxSegmentBytes)
                     flushSegment();
                 if (segmentLength === 0)
@@ -394,6 +468,8 @@ function reconcile(db, options) {
                 segmentLines.push(line.bytes);
                 segmentLength += line.bytes.length;
                 segmentEnd = line.end_offset;
+                if (line.bytes.length > maxSegmentBytes)
+                    flushSegment();
             }, keepLeaseAlive);
             flushSegment();
             const newOffset = verified.complete_through_offset;
@@ -401,7 +477,9 @@ function reconcile(db, options) {
                 (0, db_1.tx)(db, () => {
                     (0, leases_1.assertLeaseHeld)(db, leases_1.SOURCE_LEASE, owner);
                     updateFileMeta(db, fileId, session, verified);
+                    clearSessionFailure(db, fileId, owner);
                 });
+                summary.processed_sessions.push({ session_id: session.session_id, generation });
                 continue;
             }
             const enqueued = [];
@@ -435,6 +513,7 @@ function reconcile(db, options) {
                 }
                 db.prepare("UPDATE source_cursors SET queued_through_offset = ?, updated_at = ? WHERE source_file_id = ?").run(newOffset, (0, db_1.nowIso)(), fileId);
                 updateFileMeta(db, fileId, session, verified);
+                clearSessionFailure(db, fileId, owner);
             });
             cleanupUnreferencedBlobs(db, prepared.map((entry) => entry.content_sha256), owner);
             if (excludedLines.length > 0)
@@ -447,6 +526,7 @@ function reconcile(db, options) {
                     summary.queued.push(segment);
                 }
             }
+            summary.processed_sessions.push({ session_id: session.session_id, generation });
         }
         catch (error) {
             // Blobs published before an aborted commit become orphans; remove the ones
@@ -454,10 +534,12 @@ function reconcile(db, options) {
             cleanupUnreferencedBlobs(db, prepared.map((entry) => entry.content_sha256), owner);
             if (error instanceof leases_1.LeaseLostError)
                 throw error;
-            if (error instanceof claude_1.IdentityChangedError || error instanceof PolicyChangedError)
+            if (error instanceof ScanCancelledError)
+                break;
+            if (error instanceof transcript_bytes_1.IdentityChangedError || error instanceof PolicyChangedError)
                 continue;
-            if (error instanceof claude_1.OversizedLineError || isFilesystemError(error)) {
-                const code = error instanceof claude_1.OversizedLineError ? "oversized_record" : error.code ?? "fs_error";
+            if (error instanceof transcript_bytes_1.OversizedLineError || isFilesystemError(error)) {
+                const code = error instanceof transcript_bytes_1.OversizedLineError ? "oversized_record" : error.code ?? "fs_error";
                 recordSessionFailure(db, fileId, code, error.message, owner);
                 summary.sessions_failed++;
                 continue;

package/dist/local/source-adapter.js

@@ -0,0 +1,8 @@
+"use strict";
+/**
+ * Multi-source adapter types ([LET-10313]).
+ *
+ * Types only — registration lives in `source-adapters.ts` so default selection
+ * is deterministic and not import-order dependent.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/local/source-adapters.js

@@ -0,0 +1,71 @@
+"use strict";
+/**
+ * Deterministic source-adapter registry ([LET-10313]).
+ *
+ * Imports and registers Claude as the explicit default. Callers that need the
+ * default (CLI/worker boundaries, tests) import this module — not `claude.ts`
+ * for side effects.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.registerSourceAdapter = registerSourceAdapter;
+exports.getSourceAdapterById = getSourceAdapterById;
+exports.getSourceAdapterByType = getSourceAdapterByType;
+exports.defaultSourceAdapter = defaultSourceAdapter;
+exports.resolveSourceAdapter = resolveSourceAdapter;
+exports.listSourceAdapters = listSourceAdapters;
+const claude_1 = require("./claude");
+const codex_1 = require("./codex");
+const bySourceId = new Map();
+const bySourceType = new Map();
+let defaultSourceId = null;
+function registerSourceAdapter(adapter, options = {}) {
+    // Validate every conflict before mutating registry state.
+    if (bySourceId.has(adapter.sourceId)) {
+        throw new Error(`duplicate source id registration: ${adapter.sourceId}`);
+    }
+    if (bySourceType.has(adapter.sourceType)) {
+        throw new Error(`duplicate source type registration: ${adapter.sourceType}`);
+    }
+    if (options.default && defaultSourceId !== null && defaultSourceId !== adapter.sourceId) {
+        throw new Error(`default source adapter already set: ${defaultSourceId}`);
+    }
+    bySourceId.set(adapter.sourceId, adapter);
+    bySourceType.set(adapter.sourceType, adapter);
+    if (options.default)
+        defaultSourceId = adapter.sourceId;
+}
+function getSourceAdapterById(sourceId) {
+    const adapter = bySourceId.get(sourceId);
+    if (!adapter)
+        throw new Error(`unknown source id: ${sourceId}`);
+    return adapter;
+}
+function getSourceAdapterByType(sourceType) {
+    const adapter = bySourceType.get(sourceType);
+    if (!adapter)
+        throw new Error(`unknown source type: ${sourceType}`);
+    return adapter;
+}
+/** CLI / worker default — Claude, registered explicitly below. */
+function defaultSourceAdapter() {
+    if (!defaultSourceId)
+        throw new Error("no default source adapter registered");
+    return getSourceAdapterById(defaultSourceId);
+}
+/**
+ * Resolve `--source <type>` at command boundaries (stdin never selects source).
+ * Omitted / undefined preserves the Claude default; empty/unknown fail closed.
+ */
+function resolveSourceAdapter(sourceType) {
+    if (sourceType === undefined)
+        return defaultSourceAdapter();
+    const trimmed = sourceType.trim();
+    if (!trimmed)
+        throw new Error("unknown source type: (empty)");
+    return getSourceAdapterByType(trimmed);
+}
+function listSourceAdapters() {
+    return [...bySourceId.values()];
+}
+registerSourceAdapter(claude_1.claudeSourceAdapter, { default: true });
+registerSourceAdapter(codex_1.codexSourceAdapter);

package/dist/local/state-lock.js

@@ -0,0 +1,88 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.localStateLockFilePath = localStateLockFilePath;
+exports.acquireLocalStateLock = acquireLocalStateLock;
+exports.waitForLocalStateLock = waitForLocalStateLock;
+exports.releaseLocalStateLock = releaseLocalStateLock;
+const node_sqlite_1 = require("node:sqlite");
+const fs = __importStar(require("node:fs"));
+const path = __importStar(require("node:path"));
+const store_1 = require("../store");
+function localStateLockFilePath() {
+    return path.join((0, store_1.dreamsDir)(), "state-lock.db");
+}
+function acquireLocalStateLock(options = {}) {
+    const file = localStateLockFilePath();
+    const directory = path.dirname(file);
+    fs.mkdirSync(directory, { recursive: true, mode: 0o700 });
+    const db = new node_sqlite_1.DatabaseSync(file);
+    try {
+        db.exec("PRAGMA busy_timeout = 0; BEGIN IMMEDIATE");
+        if (options.allowAbandonedReset !== true &&
+            fs.readdirSync(directory).some((name) => name.startsWith(".reset-staging-") || name.startsWith(".reset-delete-"))) {
+            db.exec("ROLLBACK");
+            db.close();
+            return null;
+        }
+        return { db };
+    }
+    catch (error) {
+        db.close();
+        const message = error instanceof Error ? error.message : String(error);
+        if (/busy|locked/i.test(message))
+            return null;
+        throw error;
+    }
+}
+async function waitForLocalStateLock(timeoutMs = 5_000) {
+    const deadline = Date.now() + timeoutMs;
+    for (;;) {
+        const lock = acquireLocalStateLock();
+        if (lock !== null)
+            return lock;
+        if (Date.now() >= deadline)
+            return null;
+        await new Promise((resolve) => setTimeout(resolve, 25));
+    }
+}
+function releaseLocalStateLock(lock) {
+    try {
+        lock.db.exec("ROLLBACK");
+    }
+    finally {
+        lock.db.close();
+    }
+}