@phigon/observer 8.1.0 → 8.3.0

package/dist/src/activity/source-reader.js

@@ -1,16 +1,93 @@
+import { makeDirectoryPrivate } from "../private-directory.js";
 import { createHash, randomUUID } from "node:crypto";
 import { chmod, mkdir, open, readFile, readdir, realpath, stat, unlink, writeFile, } from "node:fs/promises";
 import { homedir } from "node:os";
 import { basename, join, relative, resolve, sep } from "node:path";
 import { atomicReplace } from "../atomic-replace.js";
+import { DirectoryListingCache, scanJSONDocumentSource, startedAfter, } from "./json-document-source.js";
+import { SQLiteConnections } from "./sqlite-connections.js";
 const maxSourceFiles = 256;
 const historicalFilesPerScan = 16;
 const historicalRecordsPerScan = 4;
 const maxDirectoryDepth = 8;
 const maxReadBytesPerFile = 8 << 20;
 const defaultInitialReplayBytes = 256 << 10;
-const maxLineBytes = 1 << 20;
+// Normal read window. A longer complete record widens the read up to the line
+// limit; the line limit stays below the per-file read limit.
+const readWindowBytes = 1 << 20;
+const maxLineBytes = 6 << 20;
 const maxCursorEntries = 16_384;
+// A source or file that fails is skipped for a while, twice as long after
+// each further failure, so one broken agent folder or database cannot stop
+// the others or be retried on every scan.
+const failureRetryBaseMs = 5_000;
+const failureRetryMaxMs = 5 * 60_000;
+const maxTrackedFailures = 256;
+/**
+ * Source cursors in least-recently-used order, so a full store evicts the
+ * cursors of sessions nobody reads any more instead of live ones (an
+ * evicted live cursor is re-baselined after a restart, which skips what
+ * was written while the observer was stopped). Reads are only recorded and
+ * applied by `settle`, so iterating the map never observes reordering.
+ */
+class RecentCursors extends Map {
+    #used = new Set();
+    // Values from before `track`, for each cursor changed since.
+    #journal;
+    get(key) {
+        const value = super.get(key);
+        if (value !== undefined)
+            this.#used.add(key);
+        return value;
+    }
+    set(key, value) {
+        this.#remember(key);
+        super.delete(key);
+        return super.set(key, value);
+    }
+    delete(key) {
+        this.#remember(key);
+        return super.delete(key);
+    }
+    /** Start remembering the cursors a source changes. */
+    track() {
+        this.#journal = new Map();
+    }
+    /** Stop remembering; with `rollback`, undo every change since `track`. */
+    untrack(rollback) {
+        const journal = this.#journal;
+        this.#journal = undefined;
+        if (!rollback || journal === undefined)
+            return;
+        for (const [key, value] of journal) {
+            if (value === undefined)
+                super.delete(key);
+            else
+                super.set(key, value);
+        }
+    }
+    #remember(key) {
+        if (this.#journal !== undefined && !this.#journal.has(key)) {
+            this.#journal.set(key, super.get(key));
+        }
+    }
+    /** Move cursors read since the last call to the recent end, then bound. */
+    settle(limit) {
+        for (const key of this.#used) {
+            const value = super.get(key);
+            if (value === undefined)
+                continue;
+            super.delete(key);
+            super.set(key, value);
+        }
+        this.#used.clear();
+        for (const key of super.keys()) {
+            if (this.size <= limit)
+                break;
+            super.delete(key);
+        }
+    }
+}
 export class ActivityCursorStore {
     #dataDirectory;
     #path;
@@ -25,7 +102,7 @@ export class ActivityCursorStore {
         }
         catch (error) {
             if (isNodeError(error) && error.code === "ENOENT")
-                return new Map();
+                return { cursors: new Map() };
             throw error;
         }
         let value;
@@ -42,7 +119,7 @@ export class ActivityCursorStore {
             const recovered = new Map();
             await this.save(recovered);
             process.emitWarning("Saved activity checkpoints were unreadable. Available source history will be rescanned using stable record IDs.", { code: "PHIGON_CURSOR_RECOVERY" });
-            return recovered;
+            return { cursors: recovered };
         }
         if (typeof value !== "object" ||
             value === null ||
@@ -53,7 +130,8 @@ export class ActivityCursorStore {
             value.files === null) {
             throw new Error("Activity cursor state is invalid.");
         }
-        const entries = Object.entries(value.files).slice(0, maxCursorEntries);
+        // Saved oldest-first (see save); keep the most recently used cursors.
+        const entries = Object.entries(value.files).slice(-maxCursorEntries);
         const result = new Map();
         for (const [key, offset] of entries) {
             if (/^[a-f0-9]{64}$/u.test(key) &&
@@ -63,15 +141,24 @@ export class ActivityCursorStore {
                 result.set(key, offset);
             }
         }
-        return result;
+        const scannedAt = "scanned_at" in value && typeof value.scanned_at === "string"
+            ? Date.parse(value.scanned_at)
+            : Number.NaN;
+        return Number.isFinite(scannedAt)
+            ? { cursors: result, scannedAt }
+            : { cursors: result };
     }
-    async save(cursors) {
+    async save(cursors, scannedAt) {
         await mkdir(this.#dataDirectory, { recursive: true, mode: 0o700 });
-        await chmod(this.#dataDirectory, 0o700);
-        const files = Object.fromEntries([...cursors.entries()]
-            .slice(-maxCursorEntries)
-            .sort(([left], [right]) => left.localeCompare(right)));
-        const state = { version: 1, files };
+        await makeDirectoryPrivate(this.#dataDirectory);
+        // Keep the map's order (least recently used first) so a restarted
+        // reader still knows which cursors are stale.
+        const files = Object.fromEntries([...cursors.entries()].slice(-maxCursorEntries));
+        const state = {
+            version: 1,
+            files,
+            ...(scannedAt === undefined ? {} : { scanned_at: new Date(scannedAt).toISOString() }),
+        };
         const temporary = `${this.#path}.${process.pid}.${randomUUID()}.tmp`;
         try {
             await writeFile(temporary, `${JSON.stringify(state)}\n`, {
@@ -101,84 +188,120 @@ export class JSONLSourceReader {
     #initialReplaySessions;
     #mode;
     #cursors;
+    // The cursor state that is already durable. An idle scan produces an
+    // identical checkpoint every tick; rewriting and flushing it would cost a
+    // synchronous disk write per scan interval without changing anything.
+    #savedCursors;
+    #savedScannedAt;
+    // When the scan behind the saved cursors started, as loaded at startup.
+    // Undefined on the first start ever. A file without a cursor that began
+    // after it was created while the observer was stopped.
+    #lastRunAt;
+    // The cursors as the previous scan left them, which is where the next
+    // scan starts. Kept so a position costs no copy on every scan.
+    #scanStart;
     #initialScanComplete = false;
     #generation = 0;
     #committedGeneration = 0;
     #sourceCache;
+    // Canonical paths from the previous listing, keyed by resolved root and
+    // listed path. Holds only files seen in the latest listing.
+    #canonicalPaths = new Map();
     #completedHistoryFiles = new Map();
-    constructor(dataDirectory, replayOnFirstStart = false, initialReplayBytes = defaultInitialReplayBytes, initialReplaySessions = 1, mode = "live") {
+    #completedHistoryDatabases = new Map();
+    // Cursors that stopped inside a record above the line limit. The next read
+    // skips to that record's end instead of parsing its tail as a new record.
+    #oversizedContinuations = new Set();
+    // Live database connections kept between scans, so an unchanged database
+    // is recognized without reopening it and rerunning every session query.
+    #openCodeConnections = new SQLiteConnections();
+    // Folder listings for JSON-document agents, reused while folders are unchanged.
+    #documentListings = new DirectoryListingCache();
+    #gooseConnections = new SQLiteConnections();
+    #zedConnections = new SQLiteConnections();
+    #failures = new Map();
+    // Live sources that were skipped or failed while their first scan would
+    // have baselined them. Their first successful scan baselines instead, so
+    // an old archive is not read as new work once the source recovers.
+    #unbaselinedSources = new Set();
+    #clock;
+    constructor(dataDirectory, replayOnFirstStart = false, initialReplayBytes = defaultInitialReplayBytes, initialReplaySessions = 1, mode = "live", clock = Date.now) {
         this.#cursorStore = new ActivityCursorStore(dataDirectory);
         this.#replayOnFirstStart = replayOnFirstStart;
         this.#initialReplayBytes = Math.min(maxReadBytesPerFile, Math.max(1, initialReplayBytes));
         this.#initialReplaySessions = Math.min(maxSourceFiles, Math.max(1, initialReplaySessions));
         this.#mode = mode;
+        this.#clock = clock;
     }
     async scan(sources, maxRecords) {
-        this.#cursors ??= await this.#cursorStore.load();
+        if (this.#cursors === undefined) {
+            const loaded = await this.#cursorStore.load();
+            this.#cursors = new RecentCursors();
+            for (const [key, offset] of loaded.cursors)
+                this.#cursors.set(key, offset);
+            this.#savedCursors = loaded.cursors;
+            this.#savedScannedAt = loaded.scannedAt;
+            this.#lastRunAt = loaded.scannedAt;
+            this.#scanStart = loaded.cursors;
+        }
+        const scannedAt = this.#clock();
         const records = [];
         let fileCount = 0;
         let malformed = 0;
         let oversized = 0;
         let candidates = [];
+        const initialScan = !this.#initialScanComplete;
+        // Files of sources that missed their baseline, baselined on this scan.
+        const baselinePaths = new Set();
         const cacheKey = JSON.stringify(sources);
         if (this.#mode === "history" && this.#sourceCache?.key === cacheKey && Date.now() - this.#sourceCache.at < 5_000) {
             candidates = this.#sourceCache.files;
         }
         else {
+            const canonicalPaths = new Map();
             for (const source of sources) {
-                if (source.format === "opencode-sqlite")
+                if (source.format !== undefined && source.format !== "jsonl")
                     continue;
-                for (const configuredRoot of source.roots) {
-                    const root = expandedActivityRoot(configuredRoot);
-                    const rootFiles = await listJSONLFiles(root);
-                    for (const file of rootFiles) {
-                        let canonicalPath;
-                        try {
-                            canonicalPath = await realpath(file.path);
-                        }
-                        catch (error) {
-                            if (isTransientSourceError(error))
-                                continue;
-                            throw error;
-                        }
-                        candidates.push({
-                            adapterId: source.adapterId,
-                            ...(source.sourceDisplayName === undefined
-                                ? {}
-                                : { sourceDisplayName: source.sourceDisplayName }),
-                            root,
-                            path: canonicalPath,
-                            modifiedMs: file.modifiedMs,
-                        });
-                        if (candidates.length > maxCursorEntries * 2) {
-                            candidates = reconciledSourceFiles(candidates);
-                            candidates.sort(recentFileFirst);
-                            candidates.length = maxCursorEntries;
-                        }
+                const key = sourceKey(source);
+                const sourceInitial = initialScan || this.#unbaselinedSources.has(key);
+                const listed = await this.#isolated(key, source.adapterId, sourceInitial, () => this.#listJSONLSource(source, canonicalPaths));
+                if (listed === undefined)
+                    continue;
+                for (const file of listed) {
+                    candidates.push(file);
+                    if (sourceInitial && !initialScan)
+                        baselinePaths.add(file.path);
+                    if (candidates.length > maxCursorEntries * 2) {
+                        candidates = reconciledSourceFiles(candidates);
+                        candidates.sort(recentFileFirst);
+                        candidates.length = maxCursorEntries;
                     }
                 }
             }
             this.#sourceCache = { key: cacheKey, at: Date.now(), files: candidates };
+            this.#canonicalPaths = canonicalPaths;
         }
         const recentCandidates = reconciledSourceFiles(candidates)
             .sort(recentFileFirst)
             .slice(0, maxCursorEntries);
-        const initialScan = !this.#initialScanComplete;
         const selected = this.#mode === "history"
             ? await this.#pendingHistoryFiles(recentCandidates)
             : recentCandidates.slice(0, maxSourceFiles);
         const replayFiles = initialScan && this.#replayOnFirstStart
             ? newestFilesPerAdapter(selected, this.#initialReplaySessions)
             : new Set();
-        if (initialScan && this.#mode === "live") {
+        if (this.#mode === "live" && (initialScan || baselinePaths.size > 0)) {
             for (const file of [...recentCandidates].reverse()) {
+                if (!initialScan && !baselinePaths.has(file.path))
+                    continue;
                 if (!replayFiles.has(file.path)) {
                     try {
                         await this.#baselineFile(file.root, file.path);
                     }
                     catch (error) {
-                        if (!isTransientSourceError(error))
-                            throw error;
+                        if (!isTransientSourceError(error)) {
+                            this.#recordFailure(fileKey(file.path), file.adapterId, error);
+                        }
                     }
                 }
             }
@@ -190,18 +313,30 @@ export class JSONLSourceReader {
         const reservedDatabaseRecords = sources.some((source) => source.format === "opencode-sqlite")
             ? Math.floor(remainingRecords / 2)
             : 0;
+        const rootRealPaths = new Map();
         for (const file of selected) {
             if (remainingRecords <= reservedDatabaseRecords)
                 break;
+            if (this.#unchangedLiveFile(file)) {
+                fileCount++;
+                continue;
+            }
+            if (this.#skipping(fileKey(file.path)))
+                continue;
             let result;
             try {
-                result = await this.#readFile(file.adapterId, file.root, file.path, replayFiles.has(file.path), file.sourceDisplayName, remainingRecords - reservedDatabaseRecords);
+                const rootReal = rootRealPaths.get(file.root) ??
+                    await realpath(file.root);
+                rootRealPaths.set(file.root, rootReal);
+                result = await this.#readFile(file.adapterId, rootReal, file.path, replayFiles.has(file.path), file.sourceDisplayName, remainingRecords - reservedDatabaseRecords);
             }
             catch (error) {
-                if (isTransientSourceError(error))
-                    continue;
-                throw error;
+                if (!isTransientSourceError(error)) {
+                    this.#recordFailure(fileKey(file.path), file.adapterId, error);
+                }
+                continue;
             }
+            this.#failures.delete(fileKey(file.path));
             fileCount++;
             records.push(...result.records);
             malformed += result.malformed;
@@ -213,6 +348,105 @@ export class JSONLSourceReader {
                 this.#completedHistoryFiles.set(file.path, file.modifiedMs);
             }
         }
+        let documentsComplete = true;
+        let documentFiles = 0;
+        const cursors = this.#cursors;
+        for (const source of sources) {
+            if (source.format !== "json-messages")
+                continue;
+            const key = sourceKey(source);
+            const sourceInitial = initialScan || this.#unbaselinedSources.has(key);
+            const documents = await this.#isolated(key, source.adapterId, sourceInitial, () => scanJSONDocumentSource({
+                adapterId: source.adapterId,
+                roots: source.roots.map(expandedActivityRoot),
+                pathPattern: source.pathPattern ?? "\\.json$",
+                messagesField: source.messagesField ?? "messages",
+                ...source.document,
+            }, {
+                cursors,
+                listings: this.#documentListings,
+                initialScan: sourceInitial,
+                replayOnFirstStart: this.#replayOnFirstStart,
+                replaySessions: this.#initialReplaySessions,
+                maxReplayRecords: Math.max(32, Math.floor(this.#initialReplayBytes / 1_024)),
+                maxRecords: Math.max(0, remainingRecords - reservedDatabaseRecords),
+                historical: this.#mode === "history",
+                ...(this.#lastRunAt === undefined ? {} : { newSince: this.#lastRunAt }),
+            }));
+            if (documents === undefined)
+                continue;
+            records.push(...documents.records);
+            fileCount += documents.files;
+            malformed += documents.malformed;
+            oversized += documents.oversized;
+            documentFiles += documents.files;
+            remainingRecords -= documents.records.length;
+            documentsComplete = documentsComplete && documents.complete;
+        }
+        for (const source of sources) {
+            if (source.format !== "goose-sqlite")
+                continue;
+            if (remainingRecords <= reservedDatabaseRecords) {
+                documentsComplete = false;
+                break;
+            }
+            // Loaded on demand for the same reason as the OpenCode source below.
+            const { scanGooseSQLiteSources } = await import("./goose-sqlite-source.js");
+            const key = sourceKey(source);
+            const sourceInitial = initialScan || this.#unbaselinedSources.has(key);
+            const goose = await this.#isolated(key, source.adapterId, sourceInitial, () => scanGooseSQLiteSources({
+                adapterId: source.adapterId,
+                roots: source.roots.map(expandedActivityRoot),
+                cursors,
+                initialScan: sourceInitial,
+                replayOnFirstStart: this.#replayOnFirstStart,
+                maxReplayRecords: Math.max(32, Math.floor(this.#initialReplayBytes / 1_024)),
+                maxRecords: remainingRecords - reservedDatabaseRecords,
+                historical: this.#mode === "history",
+                ...(this.#mode === "live" ? { connections: this.#gooseConnections } : { completedHistory: this.#completedHistoryDatabases }),
+            }));
+            if (goose === undefined)
+                continue;
+            records.push(...goose.records);
+            fileCount += goose.files;
+            malformed += goose.malformed;
+            oversized += goose.oversized;
+            documentFiles += goose.files;
+            remainingRecords -= goose.records.length;
+            documentsComplete = documentsComplete && goose.complete;
+        }
+        for (const source of sources) {
+            if (source.format !== "zed-sqlite")
+                continue;
+            if (remainingRecords <= reservedDatabaseRecords) {
+                documentsComplete = false;
+                break;
+            }
+            const { scanZedSQLiteSources } = await import("./zed-sqlite-source.js");
+            const key = sourceKey(source);
+            const sourceInitial = initialScan || this.#unbaselinedSources.has(key);
+            const zed = await this.#isolated(key, source.adapterId, sourceInitial, () => scanZedSQLiteSources({
+                adapterId: source.adapterId,
+                roots: source.roots.map(expandedActivityRoot),
+                cursors,
+                initialScan: sourceInitial,
+                replayOnFirstStart: this.#replayOnFirstStart,
+                replaySessions: this.#initialReplaySessions,
+                maxReplayRecords: Math.max(32, Math.floor(this.#initialReplayBytes / 1_024)),
+                maxRecords: remainingRecords - reservedDatabaseRecords,
+                historical: this.#mode === "history",
+                ...(this.#mode === "live" ? { connections: this.#zedConnections } : { completedHistory: this.#completedHistoryDatabases }),
+            }));
+            if (zed === undefined)
+                continue;
+            records.push(...zed.records);
+            fileCount += zed.files;
+            malformed += zed.malformed;
+            oversized += zed.oversized;
+            documentFiles += zed.files;
+            remainingRecords -= zed.records.length;
+            documentsComplete = documentsComplete && zed.complete;
+        }
         let sqliteComplete = true;
         let sqliteFiles = 0;
         for (const source of sources) {
@@ -226,15 +460,20 @@ export class JSONLSourceReader {
             // when an OpenCode data source is actually scanned so commands such as
             // pair, doctor, and help stay warning-free.
             const { scanOpenCodeSQLiteSources } = await import("./opencode-sqlite-source.js");
-            const sqlite = await scanOpenCodeSQLiteSources({
+            const key = sourceKey(source);
+            const sourceInitial = initialScan || this.#unbaselinedSources.has(key);
+            const sqlite = await this.#isolated(key, source.adapterId, sourceInitial, () => scanOpenCodeSQLiteSources({
                 adapterId: source.adapterId,
                 roots: source.roots.map(expandedActivityRoot),
-                cursors: this.#cursors,
-                initialScan,
+                cursors,
+                initialScan: sourceInitial,
                 replayOnFirstStart: this.#replayOnFirstStart,
                 maxReplayRecords: Math.min(remainingRecords, Math.max(this.#mode === "history" ? 1 : 32, Math.floor(this.#initialReplayBytes / 1_024))),
                 historical: this.#mode === "history",
-            });
+                ...(this.#mode === "live" ? { connections: this.#openCodeConnections } : { completedHistory: this.#completedHistoryDatabases }),
+            }));
+            if (sqlite === undefined)
+                continue;
             records.push(...sqlite.records);
             fileCount += sqlite.files;
             malformed += sqlite.malformed;
@@ -243,23 +482,29 @@ export class JSONLSourceReader {
             remainingRecords -= sqlite.records.length;
             sqliteComplete = sqliteComplete && sqlite.complete;
         }
+        this.#openCodeConnections.retain(configuredRoots(sources, "opencode-sqlite"));
+        this.#gooseConnections.retain(configuredRoots(sources, "goose-sqlite"));
+        this.#zedConnections.retain(configuredRoots(sources, "zed-sqlite"));
         this.#initialScanComplete = true;
+        this.#cursors.settle(maxCursorEntries);
         const checkpoint = Object.freeze({
             generation: ++this.#generation,
             cursors: new Map(this.#cursors),
+            scannedAt,
         });
+        this.#scanStart = checkpoint.cursors;
         const jsonlComplete = this.#mode !== "history" || selected.length === 0;
         return {
             records,
             metrics: {
                 files: fileCount,
-                discoveredFiles: recentCandidates.length + sqliteFiles,
+                discoveredFiles: recentCandidates.length + sqliteFiles + documentFiles,
                 records: records.length,
                 malformed,
                 oversized,
             },
             checkpoint,
-            complete: jsonlComplete && sqliteComplete,
+            complete: jsonlComplete && sqliteComplete && documentsComplete,
         };
     }
     async commit(checkpoint) {
@@ -268,11 +513,135 @@ export class JSONLSourceReader {
         if (checkpoint.generation > this.#generation) {
             throw new Error("Activity source checkpoint is invalid.");
         }
-        await this.#cursorStore.save(checkpoint.cursors);
+        // Saved once more when the time is missing, so the next start knows
+        // when this one ran. Later idle scans keep the time of the last change:
+        // a file read since then would have changed the cursors.
+        if (this.#savedCursors === undefined ||
+            this.#savedScannedAt === undefined ||
+            !sameCursors(checkpoint.cursors, this.#savedCursors)) {
+            await this.#cursorStore.save(checkpoint.cursors, checkpoint.scannedAt);
+            this.#savedCursors = checkpoint.cursors;
+            this.#savedScannedAt = checkpoint.scannedAt;
+        }
         this.#committedGeneration = checkpoint.generation;
     }
-    async #readFile(adapterId, root, file, initialReplay, sourceDisplayName, maxRecords = Number.MAX_SAFE_INTEGER) {
-        const rootReal = await realpath(root);
+    /**
+     * Where the next scan starts. A scan moves the cursors past every record
+     * it returns, before the caller delivers them; `rewind` puts the reader
+     * back here when they could not be delivered, so the next scan returns
+     * the same records instead of skipping them.
+     */
+    position() {
+        return {
+            cursors: this.#scanStart,
+            oversizedContinuations: new Set(this.#oversizedContinuations),
+            completedHistoryFiles: new Map(this.#completedHistoryFiles),
+            completedHistoryDatabases: new Map(this.#completedHistoryDatabases),
+            initialScanComplete: this.#initialScanComplete,
+            unbaselinedSources: new Set(this.#unbaselinedSources),
+        };
+    }
+    rewind(position) {
+        if (position.cursors === undefined) {
+            // Taken before the first scan: reload what is durable.
+            this.#cursors = undefined;
+        }
+        else {
+            this.#cursors = new RecentCursors();
+            for (const [key, offset] of position.cursors) {
+                this.#cursors.set(key, offset);
+            }
+        }
+        this.#scanStart = position.cursors;
+        this.#oversizedContinuations = new Set(position.oversizedContinuations);
+        this.#completedHistoryFiles = new Map(position.completedHistoryFiles);
+        this.#completedHistoryDatabases = new Map(position.completedHistoryDatabases);
+        this.#initialScanComplete = position.initialScanComplete;
+        this.#unbaselinedSources = new Set(position.unbaselinedSources);
+    }
+    /** Sources and files that failed and are skipped until `retryAt`. */
+    sourceFailures() {
+        return [...this.#failures.values()];
+    }
+    /**
+     * Run one source's scan so that its failure cannot fail the others. A
+     * failed source is recorded, skipped until its retry time, and its cursor
+     * changes are undone so none of its records is skipped. Returns undefined
+     * when the source was skipped or failed.
+     */
+    async #isolated(key, adapterId, baselines, scan) {
+        const remember = (missed) => {
+            if (this.#mode !== "live")
+                return;
+            if (missed && baselines)
+                this.#unbaselinedSources.add(key);
+            else if (!missed)
+                this.#unbaselinedSources.delete(key);
+        };
+        if (this.#skipping(key)) {
+            remember(true);
+            return undefined;
+        }
+        this.#cursors?.track();
+        try {
+            const result = await scan();
+            this.#cursors?.untrack(false);
+            this.#failures.delete(key);
+            remember(false);
+            return result;
+        }
+        catch (error) {
+            this.#cursors?.untrack(true);
+            this.#recordFailure(key, adapterId, error);
+            remember(true);
+            return undefined;
+        }
+    }
+    #skipping(key) {
+        const failure = this.#failures.get(key);
+        return failure !== undefined && this.#clock() < failure.retryAt.getTime();
+    }
+    #recordFailure(key, adapterId, error) {
+        const failures = (this.#failures.get(key)?.failures ?? 0) + 1;
+        const now = this.#clock();
+        this.#failures.delete(key);
+        this.#failures.set(key, Object.freeze({
+            adapterId,
+            failures,
+            message: (error instanceof Error ? error.message : String(error))
+                .replace(/\s+/gu, " ")
+                .slice(0, 200),
+            failedAt: new Date(now),
+            retryAt: new Date(now + Math.min(failureRetryMaxMs, failureRetryBaseMs * 2 ** Math.min(failures - 1, 16))),
+        }));
+        for (const oldest of this.#failures.keys()) {
+            if (this.#failures.size <= maxTrackedFailures)
+                break;
+            this.#failures.delete(oldest);
+        }
+    }
+    /**
+     * Close the database connections kept between scans. A later scan opens
+     * them again, so this only releases files, for example before shutdown.
+     */
+    close() {
+        this.#openCodeConnections.closeAll();
+        this.#gooseConnections.closeAll();
+        this.#zedConnections.closeAll();
+    }
+    /**
+     * A live file whose listed size equals its cursor has nothing to read:
+     * reading it would resolve, stat and find zero bytes. Its identity comes
+     * from the listing done in the same scan, so growth is never missed.
+     */
+    #unchangedLiveFile(file) {
+        if (this.#mode !== "live" || file.identity === undefined)
+            return false;
+        const cursorId = fileCursorId(file.path, file.identity);
+        return (this.#cursors?.get(cursorId) === file.identity.size &&
+            !this.#oversizedContinuations.has(cursorId));
+    }
+    async #readFile(adapterId, rootReal, file, initialReplay, sourceDisplayName, maxRecords = Number.MAX_SAFE_INTEGER) {
         const fileReal = await realpath(file);
         if (!isWithin(rootReal, fileReal)) {
             return { records: [], malformed: 0, oversized: 0, processedRecords: 0 };
@@ -293,34 +662,50 @@ export class JSONLSourceReader {
                     (initialReplay ? this.#initialReplayBytes : maxReadBytesPerFile)));
         if (metadata.size < offset)
             offset = 0;
-        const length = Math.min(metadata.size - offset, maxReadBytesPerFile, Math.max(maxLineBytes + 1, Math.min(1_024, maxRecords) * 16_384));
+        let length = Math.min(metadata.size - offset, maxReadBytesPerFile, Math.max(readWindowBytes + 1, Math.min(1_024, maxRecords) * 16_384));
         if (length <= 0) {
             this.#setCursor(cursorId, legacyCursorId, metadata.size);
+            this.#oversizedContinuations.delete(cursorId);
             return { records: [], malformed: 0, oversized: 0, processedRecords: 0 };
         }
-        const handle = await open(fileReal, "r");
-        let content;
-        try {
-            content = Buffer.alloc(length);
-            const read = await handle.read(content, 0, length, offset);
-            content = content.subarray(0, read.bytesRead);
-        }
-        finally {
-            await handle.close();
+        // Starting mid-file, read one byte early: if it is a newline the first
+        // record starts exactly at the offset and must not be skipped as partial.
+        // A cursor left inside an oversized record also resumes after its end.
+        const continuingOversized = this.#oversizedContinuations.has(cursorId);
+        const resync = (knownOffset === undefined && offset > 0) || continuingOversized;
+        if (resync) {
+            offset -= 1;
+            length += 1;
         }
-        if (knownOffset === undefined && offset > 0) {
+        let content = await readRange(fileReal, offset, length);
+        if (resync) {
             const firstNewline = content.indexOf(0x0a);
             if (firstNewline < 0) {
                 this.#setCursor(cursorId, legacyCursorId, offset + content.length);
-                return { records: [], malformed: 0, oversized: 1, processedRecords: 1 };
+                this.#rememberOversizedContinuation(cursorId);
+                return continuingOversized
+                    ? { records: [], malformed: 0, oversized: 0, processedRecords: 1 }
+                    : { records: [], malformed: 0, oversized: 1, processedRecords: 1 };
             }
+            this.#oversizedContinuations.delete(cursorId);
             offset += firstNewline + 1;
             content = content.subarray(firstNewline + 1);
         }
-        const lastNewline = content.lastIndexOf(0x0a);
+        let lastNewline = content.lastIndexOf(0x0a);
+        if (lastNewline < 0) {
+            // One complete record can be larger than the normal read window, for
+            // example a tool result or a pasted screenshot. Widen once, bounded by
+            // the record limit, instead of discarding it as oversized.
+            const widened = Math.min(metadata.size - offset, maxLineBytes + 1);
+            if (widened > content.length) {
+                content = await readRange(fileReal, offset, widened);
+                lastNewline = content.lastIndexOf(0x0a);
+            }
+        }
         if (lastNewline < 0) {
             if (content.length > maxLineBytes) {
                 this.#setCursor(cursorId, legacyCursorId, offset + content.length);
+                this.#rememberOversizedContinuation(cursorId);
                 return { records: [], malformed: 0, oversized: 1, processedRecords: 1 };
             }
             return { records: [], malformed: 0, oversized: 0, processedRecords: 0 };
@@ -376,11 +761,71 @@ export class JSONLSourceReader {
         this.#setCursor(cursorId, legacyCursorId, offset + processedThrough);
         return { records, malformed, oversized, processedRecords };
     }
+    /** The files of one JSONL source, with their canonical paths. */
+    async #listJSONLSource(source, canonicalPaths) {
+        let files = [];
+        const pathPattern = source.pathPattern === undefined
+            ? undefined
+            : new RegExp(source.pathPattern, "u");
+        for (const configuredRoot of source.roots) {
+            const root = expandedActivityRoot(configuredRoot);
+            const rootFiles = await listJSONLFiles(root);
+            let rootReal;
+            for (const file of rootFiles) {
+                if (pathPattern !== undefined &&
+                    !pathPattern.test(relative(root, file.path).split(sep).join("/"))) {
+                    continue;
+                }
+                let canonicalPath;
+                try {
+                    // The listing never follows links, so a listed file keeps its
+                    // canonical path while its root resolves to the same place and
+                    // the file keeps its identity. Resolve each file only once.
+                    rootReal ??= await realpath(root);
+                    const key = `${rootReal}\u0000${file.path}`;
+                    const identity = fileIdentityKey(file.identity);
+                    const known = this.#canonicalPaths.get(key);
+                    canonicalPath =
+                        known !== undefined && identity !== undefined &&
+                            known.identity === identity
+                            ? known.path
+                            : await realpath(file.path);
+                    if (identity !== undefined) {
+                        canonicalPaths.set(key, { path: canonicalPath, identity });
+                    }
+                }
+                catch (error) {
+                    if (isTransientSourceError(error))
+                        continue;
+                    throw error;
+                }
+                files.push({
+                    adapterId: source.adapterId,
+                    ...(source.sourceDisplayName === undefined
+                        ? {}
+                        : { sourceDisplayName: source.sourceDisplayName }),
+                    root,
+                    path: canonicalPath,
+                    modifiedMs: file.modifiedMs,
+                    identity: file.identity,
+                });
+                if (files.length > maxCursorEntries * 2) {
+                    files = reconciledSourceFiles(files);
+                    files.sort(recentFileFirst);
+                    files.length = maxCursorEntries;
+                }
+            }
+        }
+        return files;
+    }
     async #pendingHistoryFiles(files) {
         const pending = [];
         for (const file of files) {
             if (this.#completedHistoryFiles.get(file.path) === file.modifiedMs)
                 continue;
+            // A failing file is retried later without holding the import open.
+            if (this.#skipping(fileKey(file.path)))
+                continue;
             try {
                 const metadata = await stat(file.path);
                 const offset = this.#cursors?.get(fileCursorId(file.path, metadata)) ??
@@ -395,12 +840,21 @@ export class JSONLSourceReader {
                 }
             }
             catch (error) {
-                if (!isTransientSourceError(error))
-                    throw error;
+                if (!isTransientSourceError(error)) {
+                    this.#recordFailure(fileKey(file.path), file.adapterId, error);
+                }
             }
         }
         return pending;
     }
+    #rememberOversizedContinuation(cursorId) {
+        this.#oversizedContinuations.add(cursorId);
+        if (this.#oversizedContinuations.size <= maxSourceFiles)
+            return;
+        const oldest = this.#oversizedContinuations.values().next().value;
+        if (oldest !== undefined)
+            this.#oversizedContinuations.delete(oldest);
+    }
     #setCursor(cursorId, legacyCursorId, offset) {
         this.#cursors?.set(cursorId, offset);
         if (cursorId !== legacyCursorId) {
@@ -417,7 +871,11 @@ export class JSONLSourceReader {
             return;
         if (this.#cursors?.has(fileCursorId(fileReal, metadata)) || this.#cursors?.has(digest(fileReal)))
             return;
-        this.#setCursor(fileCursorId(fileReal, metadata), digest(fileReal), metadata.size);
+        // A session that began while the observer was stopped is read from its
+        // start; any other file without a cursor starts at its end, as before.
+        this.#setCursor(fileCursorId(fileReal, metadata), digest(fileReal), this.#lastRunAt !== undefined && startedAfter(metadata, this.#lastRunAt)
+            ? 0
+            : metadata.size);
     }
 }
 export function expandedActivityRoot(root) {
@@ -470,7 +928,16 @@ async function listJSONLFiles(root) {
                 basename(entry.name).toLowerCase().endsWith(".jsonl")) {
                 try {
                     const metadata = await stat(path);
-                    result.push({ path, modifiedMs: metadata.mtimeMs });
+                    result.push({
+                        path,
+                        modifiedMs: metadata.mtimeMs,
+                        identity: {
+                            size: metadata.size,
+                            dev: metadata.dev,
+                            ino: metadata.ino,
+                            birthtimeMs: metadata.birthtimeMs,
+                        },
+                    });
                     if (result.length > maxCursorEntries * 2) {
                         result.sort(recentFileFirst);
                         result.length = maxCursorEntries;
@@ -491,6 +958,17 @@ async function listJSONLFiles(root) {
     await walk(root, 0);
     return result.sort(recentFileFirst).slice(0, maxCursorEntries);
 }
+async function readRange(path, position, length) {
+    const handle = await open(path, "r");
+    try {
+        const content = Buffer.alloc(length);
+        const read = await handle.read(content, 0, length, position);
+        return content.subarray(0, read.bytesRead);
+    }
+    finally {
+        await handle.close();
+    }
+}
 function recentFileFirst(left, right) {
     return (right.modifiedMs - left.modifiedMs || left.path.localeCompare(right.path));
 }
@@ -546,6 +1024,9 @@ function reconciledSourceFile(claims) {
         root: representative.root,
         path: representative.path,
         modifiedMs: Math.max(...ordered.map((claim) => claim.modifiedMs)),
+        ...(representative.identity === undefined
+            ? {}
+            : { identity: representative.identity }),
     };
 }
 function compareSourceClaim(left, right) {
@@ -553,6 +1034,24 @@ function compareSourceClaim(left, right) {
         (left.sourceDisplayName ?? "").localeCompare(right.sourceDisplayName ?? "") ||
         left.root.localeCompare(right.root);
 }
+function fileIdentityKey(identity) {
+    if (identity === undefined || (identity.ino <= 0 && identity.dev <= 0)) {
+        return undefined;
+    }
+    return `${identity.dev}:${identity.ino}:${identity.birthtimeMs}`;
+}
+/**
+ * Whether saved `cursors` are at the end of the file at `path`, so every
+ * record in it was read. A file that grew or was replaced since does not
+ * count.
+ */
+export async function fileReadThrough(path, cursors) {
+    const fileReal = await realpath(path);
+    const metadata = await stat(fileReal);
+    const offset = cursors.get(fileCursorId(fileReal, metadata)) ??
+        cursors.get(digest(fileReal));
+    return offset === metadata.size;
+}
 function fileCursorId(path, metadata) {
     const identity = metadata.ino > 0 || metadata.dev > 0
         ? `inode:${metadata.dev}:${metadata.ino}:${metadata.birthtimeMs}`
@@ -573,6 +1072,24 @@ function newestFilesPerAdapter(files, limit) {
     }
     return selected;
 }
+function configuredRoots(sources, format) {
+    return new Set(sources.flatMap((source) => source.format === format ? source.roots.map(expandedActivityRoot) : []));
+}
+function sourceKey(source) {
+    return JSON.stringify([source.format ?? "jsonl", source.adapterId, source.roots, source.pathPattern ?? ""]);
+}
+function fileKey(path) {
+    return `file\u0000${path}`;
+}
+function sameCursors(left, right) {
+    if (left.size !== right.size)
+        return false;
+    for (const [key, offset] of left) {
+        if (right.get(key) !== offset)
+            return false;
+    }
+    return true;
+}
 function isWithin(root, candidate) {
     const path = relative(resolve(root), resolve(candidate));
     return path === "" || (!path.startsWith(`..${sep}`) && path !== "..");
@@ -589,6 +1106,7 @@ function isTransientSourceError(error) {
             error.code === "EACCES" ||
             error.code === "EPERM" ||
             error.code === "ENOTDIR" ||
+            error.code === "EBUSY" ||
             error.code === "ESTALE"));
 }
 //# sourceMappingURL=source-reader.js.map