@phigon/observer 8.2.0 → 8.3.0

package/dist/src/activity/source-reader.js

@@ -4,7 +4,7 @@ import { chmod, mkdir, open, readFile, readdir, realpath, stat, unlink, writeFil
 import { homedir } from "node:os";
 import { basename, join, relative, resolve, sep } from "node:path";
 import { atomicReplace } from "../atomic-replace.js";
-import { DirectoryListingCache, scanJSONDocumentSource, } from "./json-document-source.js";
+import { DirectoryListingCache, scanJSONDocumentSource, startedAfter, } from "./json-document-source.js";
 import { SQLiteConnections } from "./sqlite-connections.js";
 const maxSourceFiles = 256;
 const historicalFilesPerScan = 16;
@@ -17,6 +17,12 @@ const defaultInitialReplayBytes = 256 << 10;
 const readWindowBytes = 1 << 20;
 const maxLineBytes = 6 << 20;
 const maxCursorEntries = 16_384;
+// A source or file that fails is skipped for a while, twice as long after
+// each further failure, so one broken agent folder or database cannot stop
+// the others or be retried on every scan.
+const failureRetryBaseMs = 5_000;
+const failureRetryMaxMs = 5 * 60_000;
+const maxTrackedFailures = 256;
 /**
  * Source cursors in least-recently-used order, so a full store evicts the
  * cursors of sessions nobody reads any more instead of live ones (an
@@ -26,6 +32,8 @@ const maxCursorEntries = 16_384;
  */
 class RecentCursors extends Map {
     #used = new Set();
+    // Values from before `track`, for each cursor changed since.
+    #journal;
     get(key) {
         const value = super.get(key);
         if (value !== undefined)
@@ -33,9 +41,36 @@ class RecentCursors extends Map {
         return value;
     }
     set(key, value) {
+        this.#remember(key);
         super.delete(key);
         return super.set(key, value);
     }
+    delete(key) {
+        this.#remember(key);
+        return super.delete(key);
+    }
+    /** Start remembering the cursors a source changes. */
+    track() {
+        this.#journal = new Map();
+    }
+    /** Stop remembering; with `rollback`, undo every change since `track`. */
+    untrack(rollback) {
+        const journal = this.#journal;
+        this.#journal = undefined;
+        if (!rollback || journal === undefined)
+            return;
+        for (const [key, value] of journal) {
+            if (value === undefined)
+                super.delete(key);
+            else
+                super.set(key, value);
+        }
+    }
+    #remember(key) {
+        if (this.#journal !== undefined && !this.#journal.has(key)) {
+            this.#journal.set(key, super.get(key));
+        }
+    }
     /** Move cursors read since the last call to the recent end, then bound. */
     settle(limit) {
         for (const key of this.#used) {
@@ -67,7 +102,7 @@ export class ActivityCursorStore {
         }
         catch (error) {
             if (isNodeError(error) && error.code === "ENOENT")
-                return new Map();
+                return { cursors: new Map() };
             throw error;
         }
         let value;
@@ -84,7 +119,7 @@ export class ActivityCursorStore {
             const recovered = new Map();
             await this.save(recovered);
             process.emitWarning("Saved activity checkpoints were unreadable. Available source history will be rescanned using stable record IDs.", { code: "PHIGON_CURSOR_RECOVERY" });
-            return recovered;
+            return { cursors: recovered };
         }
         if (typeof value !== "object" ||
             value === null ||
@@ -106,15 +141,24 @@ export class ActivityCursorStore {
                 result.set(key, offset);
             }
         }
-        return result;
+        const scannedAt = "scanned_at" in value && typeof value.scanned_at === "string"
+            ? Date.parse(value.scanned_at)
+            : Number.NaN;
+        return Number.isFinite(scannedAt)
+            ? { cursors: result, scannedAt }
+            : { cursors: result };
     }
-    async save(cursors) {
+    async save(cursors, scannedAt) {
         await mkdir(this.#dataDirectory, { recursive: true, mode: 0o700 });
         await makeDirectoryPrivate(this.#dataDirectory);
         // Keep the map's order (least recently used first) so a restarted
         // reader still knows which cursors are stale.
         const files = Object.fromEntries([...cursors.entries()].slice(-maxCursorEntries));
-        const state = { version: 1, files };
+        const state = {
+            version: 1,
+            files,
+            ...(scannedAt === undefined ? {} : { scanned_at: new Date(scannedAt).toISOString() }),
+        };
         const temporary = `${this.#path}.${process.pid}.${randomUUID()}.tmp`;
         try {
             await writeFile(temporary, `${JSON.stringify(state)}\n`, {
@@ -148,6 +192,14 @@ export class JSONLSourceReader {
     // identical checkpoint every tick; rewriting and flushing it would cost a
     // synchronous disk write per scan interval without changing anything.
     #savedCursors;
+    #savedScannedAt;
+    // When the scan behind the saved cursors started, as loaded at startup.
+    // Undefined on the first start ever. A file without a cursor that began
+    // after it was created while the observer was stopped.
+    #lastRunAt;
+    // The cursors as the previous scan left them, which is where the next
+    // scan starts. Kept so a position costs no copy on every scan.
+    #scanStart;
     #initialScanComplete = false;
     #generation = 0;
     #committedGeneration = 0;
@@ -167,26 +219,40 @@ export class JSONLSourceReader {
     #documentListings = new DirectoryListingCache();
     #gooseConnections = new SQLiteConnections();
     #zedConnections = new SQLiteConnections();
-    constructor(dataDirectory, replayOnFirstStart = false, initialReplayBytes = defaultInitialReplayBytes, initialReplaySessions = 1, mode = "live") {
+    #failures = new Map();
+    // Live sources that were skipped or failed while their first scan would
+    // have baselined them. Their first successful scan baselines instead, so
+    // an old archive is not read as new work once the source recovers.
+    #unbaselinedSources = new Set();
+    #clock;
+    constructor(dataDirectory, replayOnFirstStart = false, initialReplayBytes = defaultInitialReplayBytes, initialReplaySessions = 1, mode = "live", clock = Date.now) {
         this.#cursorStore = new ActivityCursorStore(dataDirectory);
         this.#replayOnFirstStart = replayOnFirstStart;
         this.#initialReplayBytes = Math.min(maxReadBytesPerFile, Math.max(1, initialReplayBytes));
         this.#initialReplaySessions = Math.min(maxSourceFiles, Math.max(1, initialReplaySessions));
         this.#mode = mode;
+        this.#clock = clock;
     }
     async scan(sources, maxRecords) {
         if (this.#cursors === undefined) {
             const loaded = await this.#cursorStore.load();
             this.#cursors = new RecentCursors();
-            for (const [key, offset] of loaded)
+            for (const [key, offset] of loaded.cursors)
                 this.#cursors.set(key, offset);
-            this.#savedCursors = loaded;
+            this.#savedCursors = loaded.cursors;
+            this.#savedScannedAt = loaded.scannedAt;
+            this.#lastRunAt = loaded.scannedAt;
+            this.#scanStart = loaded.cursors;
         }
+        const scannedAt = this.#clock();
         const records = [];
         let fileCount = 0;
         let malformed = 0;
         let oversized = 0;
         let candidates = [];
+        const initialScan = !this.#initialScanComplete;
+        // Files of sources that missed their baseline, baselined on this scan.
+        const baselinePaths = new Set();
         const cacheKey = JSON.stringify(sources);
         if (this.#mode === "history" && this.#sourceCache?.key === cacheKey && Date.now() - this.#sourceCache.at < 5_000) {
             candidates = this.#sourceCache.files;
@@ -196,56 +262,19 @@ export class JSONLSourceReader {
             for (const source of sources) {
                 if (source.format !== undefined && source.format !== "jsonl")
                     continue;
-                const pathPattern = source.pathPattern === undefined
-                    ? undefined
-                    : new RegExp(source.pathPattern, "u");
-                for (const configuredRoot of source.roots) {
-                    const root = expandedActivityRoot(configuredRoot);
-                    const rootFiles = await listJSONLFiles(root);
-                    let rootReal;
-                    for (const file of rootFiles) {
-                        if (pathPattern !== undefined &&
-                            !pathPattern.test(relative(root, file.path).split(sep).join("/"))) {
-                            continue;
-                        }
-                        let canonicalPath;
-                        try {
-                            // The listing never follows links, so a listed file keeps its
-                            // canonical path while its root resolves to the same place and
-                            // the file keeps its identity. Resolve each file only once.
-                            rootReal ??= await realpath(root);
-                            const key = `${rootReal}\u0000${file.path}`;
-                            const identity = fileIdentityKey(file.identity);
-                            const known = this.#canonicalPaths.get(key);
-                            canonicalPath =
-                                known !== undefined && identity !== undefined &&
-                                    known.identity === identity
-                                    ? known.path
-                                    : await realpath(file.path);
-                            if (identity !== undefined) {
-                                canonicalPaths.set(key, { path: canonicalPath, identity });
-                            }
-                        }
-                        catch (error) {
-                            if (isTransientSourceError(error))
-                                continue;
-                            throw error;
-                        }
-                        candidates.push({
-                            adapterId: source.adapterId,
-                            ...(source.sourceDisplayName === undefined
-                                ? {}
-                                : { sourceDisplayName: source.sourceDisplayName }),
-                            root,
-                            path: canonicalPath,
-                            modifiedMs: file.modifiedMs,
-                            identity: file.identity,
-                        });
-                        if (candidates.length > maxCursorEntries * 2) {
-                            candidates = reconciledSourceFiles(candidates);
-                            candidates.sort(recentFileFirst);
-                            candidates.length = maxCursorEntries;
-                        }
+                const key = sourceKey(source);
+                const sourceInitial = initialScan || this.#unbaselinedSources.has(key);
+                const listed = await this.#isolated(key, source.adapterId, sourceInitial, () => this.#listJSONLSource(source, canonicalPaths));
+                if (listed === undefined)
+                    continue;
+                for (const file of listed) {
+                    candidates.push(file);
+                    if (sourceInitial && !initialScan)
+                        baselinePaths.add(file.path);
+                    if (candidates.length > maxCursorEntries * 2) {
+                        candidates = reconciledSourceFiles(candidates);
+                        candidates.sort(recentFileFirst);
+                        candidates.length = maxCursorEntries;
                     }
                 }
             }
@@ -255,22 +284,24 @@ export class JSONLSourceReader {
         const recentCandidates = reconciledSourceFiles(candidates)
             .sort(recentFileFirst)
             .slice(0, maxCursorEntries);
-        const initialScan = !this.#initialScanComplete;
         const selected = this.#mode === "history"
             ? await this.#pendingHistoryFiles(recentCandidates)
             : recentCandidates.slice(0, maxSourceFiles);
         const replayFiles = initialScan && this.#replayOnFirstStart
             ? newestFilesPerAdapter(selected, this.#initialReplaySessions)
             : new Set();
-        if (initialScan && this.#mode === "live") {
+        if (this.#mode === "live" && (initialScan || baselinePaths.size > 0)) {
             for (const file of [...recentCandidates].reverse()) {
+                if (!initialScan && !baselinePaths.has(file.path))
+                    continue;
                 if (!replayFiles.has(file.path)) {
                     try {
                         await this.#baselineFile(file.root, file.path);
                     }
                     catch (error) {
-                        if (!isTransientSourceError(error))
-                            throw error;
+                        if (!isTransientSourceError(error)) {
+                            this.#recordFailure(fileKey(file.path), file.adapterId, error);
+                        }
                     }
                 }
             }
@@ -290,6 +321,8 @@ export class JSONLSourceReader {
                 fileCount++;
                 continue;
             }
+            if (this.#skipping(fileKey(file.path)))
+                continue;
             let result;
             try {
                 const rootReal = rootRealPaths.get(file.root) ??
@@ -298,10 +331,12 @@ export class JSONLSourceReader {
                 result = await this.#readFile(file.adapterId, rootReal, file.path, replayFiles.has(file.path), file.sourceDisplayName, remainingRecords - reservedDatabaseRecords);
             }
             catch (error) {
-                if (isTransientSourceError(error))
-                    continue;
-                throw error;
+                if (!isTransientSourceError(error)) {
+                    this.#recordFailure(fileKey(file.path), file.adapterId, error);
+                }
+                continue;
             }
+            this.#failures.delete(fileKey(file.path));
             fileCount++;
             records.push(...result.records);
             malformed += result.malformed;
@@ -315,25 +350,31 @@ export class JSONLSourceReader {
         }
         let documentsComplete = true;
         let documentFiles = 0;
+        const cursors = this.#cursors;
         for (const source of sources) {
             if (source.format !== "json-messages")
                 continue;
-            const documents = await scanJSONDocumentSource({
+            const key = sourceKey(source);
+            const sourceInitial = initialScan || this.#unbaselinedSources.has(key);
+            const documents = await this.#isolated(key, source.adapterId, sourceInitial, () => scanJSONDocumentSource({
                 adapterId: source.adapterId,
                 roots: source.roots.map(expandedActivityRoot),
                 pathPattern: source.pathPattern ?? "\\.json$",
                 messagesField: source.messagesField ?? "messages",
                 ...source.document,
             }, {
-                cursors: this.#cursors,
+                cursors,
                 listings: this.#documentListings,
-                initialScan,
+                initialScan: sourceInitial,
                 replayOnFirstStart: this.#replayOnFirstStart,
                 replaySessions: this.#initialReplaySessions,
                 maxReplayRecords: Math.max(32, Math.floor(this.#initialReplayBytes / 1_024)),
                 maxRecords: Math.max(0, remainingRecords - reservedDatabaseRecords),
                 historical: this.#mode === "history",
-            });
+                ...(this.#lastRunAt === undefined ? {} : { newSince: this.#lastRunAt }),
+            }));
+            if (documents === undefined)
+                continue;
             records.push(...documents.records);
             fileCount += documents.files;
             malformed += documents.malformed;
@@ -351,17 +392,21 @@ export class JSONLSourceReader {
             }
             // Loaded on demand for the same reason as the OpenCode source below.
             const { scanGooseSQLiteSources } = await import("./goose-sqlite-source.js");
-            const goose = await scanGooseSQLiteSources({
+            const key = sourceKey(source);
+            const sourceInitial = initialScan || this.#unbaselinedSources.has(key);
+            const goose = await this.#isolated(key, source.adapterId, sourceInitial, () => scanGooseSQLiteSources({
                 adapterId: source.adapterId,
                 roots: source.roots.map(expandedActivityRoot),
-                cursors: this.#cursors,
-                initialScan,
+                cursors,
+                initialScan: sourceInitial,
                 replayOnFirstStart: this.#replayOnFirstStart,
                 maxReplayRecords: Math.max(32, Math.floor(this.#initialReplayBytes / 1_024)),
                 maxRecords: remainingRecords - reservedDatabaseRecords,
                 historical: this.#mode === "history",
                 ...(this.#mode === "live" ? { connections: this.#gooseConnections } : { completedHistory: this.#completedHistoryDatabases }),
-            });
+            }));
+            if (goose === undefined)
+                continue;
             records.push(...goose.records);
             fileCount += goose.files;
             malformed += goose.malformed;
@@ -378,18 +423,22 @@ export class JSONLSourceReader {
                 break;
             }
             const { scanZedSQLiteSources } = await import("./zed-sqlite-source.js");
-            const zed = await scanZedSQLiteSources({
+            const key = sourceKey(source);
+            const sourceInitial = initialScan || this.#unbaselinedSources.has(key);
+            const zed = await this.#isolated(key, source.adapterId, sourceInitial, () => scanZedSQLiteSources({
                 adapterId: source.adapterId,
                 roots: source.roots.map(expandedActivityRoot),
-                cursors: this.#cursors,
-                initialScan,
+                cursors,
+                initialScan: sourceInitial,
                 replayOnFirstStart: this.#replayOnFirstStart,
                 replaySessions: this.#initialReplaySessions,
                 maxReplayRecords: Math.max(32, Math.floor(this.#initialReplayBytes / 1_024)),
                 maxRecords: remainingRecords - reservedDatabaseRecords,
                 historical: this.#mode === "history",
                 ...(this.#mode === "live" ? { connections: this.#zedConnections } : { completedHistory: this.#completedHistoryDatabases }),
-            });
+            }));
+            if (zed === undefined)
+                continue;
             records.push(...zed.records);
             fileCount += zed.files;
             malformed += zed.malformed;
@@ -411,16 +460,20 @@ export class JSONLSourceReader {
             // when an OpenCode data source is actually scanned so commands such as
             // pair, doctor, and help stay warning-free.
             const { scanOpenCodeSQLiteSources } = await import("./opencode-sqlite-source.js");
-            const sqlite = await scanOpenCodeSQLiteSources({
+            const key = sourceKey(source);
+            const sourceInitial = initialScan || this.#unbaselinedSources.has(key);
+            const sqlite = await this.#isolated(key, source.adapterId, sourceInitial, () => scanOpenCodeSQLiteSources({
                 adapterId: source.adapterId,
                 roots: source.roots.map(expandedActivityRoot),
-                cursors: this.#cursors,
-                initialScan,
+                cursors,
+                initialScan: sourceInitial,
                 replayOnFirstStart: this.#replayOnFirstStart,
                 maxReplayRecords: Math.min(remainingRecords, Math.max(this.#mode === "history" ? 1 : 32, Math.floor(this.#initialReplayBytes / 1_024))),
                 historical: this.#mode === "history",
                 ...(this.#mode === "live" ? { connections: this.#openCodeConnections } : { completedHistory: this.#completedHistoryDatabases }),
-            });
+            }));
+            if (sqlite === undefined)
+                continue;
             records.push(...sqlite.records);
             fileCount += sqlite.files;
             malformed += sqlite.malformed;
@@ -437,7 +490,9 @@ export class JSONLSourceReader {
         const checkpoint = Object.freeze({
             generation: ++this.#generation,
             cursors: new Map(this.#cursors),
+            scannedAt,
         });
+        this.#scanStart = checkpoint.cursors;
         const jsonlComplete = this.#mode !== "history" || selected.length === 0;
         return {
             records,
@@ -458,13 +513,113 @@ export class JSONLSourceReader {
         if (checkpoint.generation > this.#generation) {
             throw new Error("Activity source checkpoint is invalid.");
         }
+        // Saved once more when the time is missing, so the next start knows
+        // when this one ran. Later idle scans keep the time of the last change:
+        // a file read since then would have changed the cursors.
         if (this.#savedCursors === undefined ||
+            this.#savedScannedAt === undefined ||
             !sameCursors(checkpoint.cursors, this.#savedCursors)) {
-            await this.#cursorStore.save(checkpoint.cursors);
+            await this.#cursorStore.save(checkpoint.cursors, checkpoint.scannedAt);
             this.#savedCursors = checkpoint.cursors;
+            this.#savedScannedAt = checkpoint.scannedAt;
         }
         this.#committedGeneration = checkpoint.generation;
     }
+    /**
+     * Where the next scan starts. A scan moves the cursors past every record
+     * it returns, before the caller delivers them; `rewind` puts the reader
+     * back here when they could not be delivered, so the next scan returns
+     * the same records instead of skipping them.
+     */
+    position() {
+        return {
+            cursors: this.#scanStart,
+            oversizedContinuations: new Set(this.#oversizedContinuations),
+            completedHistoryFiles: new Map(this.#completedHistoryFiles),
+            completedHistoryDatabases: new Map(this.#completedHistoryDatabases),
+            initialScanComplete: this.#initialScanComplete,
+            unbaselinedSources: new Set(this.#unbaselinedSources),
+        };
+    }
+    rewind(position) {
+        if (position.cursors === undefined) {
+            // Taken before the first scan: reload what is durable.
+            this.#cursors = undefined;
+        }
+        else {
+            this.#cursors = new RecentCursors();
+            for (const [key, offset] of position.cursors) {
+                this.#cursors.set(key, offset);
+            }
+        }
+        this.#scanStart = position.cursors;
+        this.#oversizedContinuations = new Set(position.oversizedContinuations);
+        this.#completedHistoryFiles = new Map(position.completedHistoryFiles);
+        this.#completedHistoryDatabases = new Map(position.completedHistoryDatabases);
+        this.#initialScanComplete = position.initialScanComplete;
+        this.#unbaselinedSources = new Set(position.unbaselinedSources);
+    }
+    /** Sources and files that failed and are skipped until `retryAt`. */
+    sourceFailures() {
+        return [...this.#failures.values()];
+    }
+    /**
+     * Run one source's scan so that its failure cannot fail the others. A
+     * failed source is recorded, skipped until its retry time, and its cursor
+     * changes are undone so none of its records is skipped. Returns undefined
+     * when the source was skipped or failed.
+     */
+    async #isolated(key, adapterId, baselines, scan) {
+        const remember = (missed) => {
+            if (this.#mode !== "live")
+                return;
+            if (missed && baselines)
+                this.#unbaselinedSources.add(key);
+            else if (!missed)
+                this.#unbaselinedSources.delete(key);
+        };
+        if (this.#skipping(key)) {
+            remember(true);
+            return undefined;
+        }
+        this.#cursors?.track();
+        try {
+            const result = await scan();
+            this.#cursors?.untrack(false);
+            this.#failures.delete(key);
+            remember(false);
+            return result;
+        }
+        catch (error) {
+            this.#cursors?.untrack(true);
+            this.#recordFailure(key, adapterId, error);
+            remember(true);
+            return undefined;
+        }
+    }
+    #skipping(key) {
+        const failure = this.#failures.get(key);
+        return failure !== undefined && this.#clock() < failure.retryAt.getTime();
+    }
+    #recordFailure(key, adapterId, error) {
+        const failures = (this.#failures.get(key)?.failures ?? 0) + 1;
+        const now = this.#clock();
+        this.#failures.delete(key);
+        this.#failures.set(key, Object.freeze({
+            adapterId,
+            failures,
+            message: (error instanceof Error ? error.message : String(error))
+                .replace(/\s+/gu, " ")
+                .slice(0, 200),
+            failedAt: new Date(now),
+            retryAt: new Date(now + Math.min(failureRetryMaxMs, failureRetryBaseMs * 2 ** Math.min(failures - 1, 16))),
+        }));
+        for (const oldest of this.#failures.keys()) {
+            if (this.#failures.size <= maxTrackedFailures)
+                break;
+            this.#failures.delete(oldest);
+        }
+    }
     /**
      * Close the database connections kept between scans. A later scan opens
      * them again, so this only releases files, for example before shutdown.
@@ -606,11 +761,71 @@ export class JSONLSourceReader {
         this.#setCursor(cursorId, legacyCursorId, offset + processedThrough);
         return { records, malformed, oversized, processedRecords };
     }
+    /** The files of one JSONL source, with their canonical paths. */
+    async #listJSONLSource(source, canonicalPaths) {
+        let files = [];
+        const pathPattern = source.pathPattern === undefined
+            ? undefined
+            : new RegExp(source.pathPattern, "u");
+        for (const configuredRoot of source.roots) {
+            const root = expandedActivityRoot(configuredRoot);
+            const rootFiles = await listJSONLFiles(root);
+            let rootReal;
+            for (const file of rootFiles) {
+                if (pathPattern !== undefined &&
+                    !pathPattern.test(relative(root, file.path).split(sep).join("/"))) {
+                    continue;
+                }
+                let canonicalPath;
+                try {
+                    // The listing never follows links, so a listed file keeps its
+                    // canonical path while its root resolves to the same place and
+                    // the file keeps its identity. Resolve each file only once.
+                    rootReal ??= await realpath(root);
+                    const key = `${rootReal}\u0000${file.path}`;
+                    const identity = fileIdentityKey(file.identity);
+                    const known = this.#canonicalPaths.get(key);
+                    canonicalPath =
+                        known !== undefined && identity !== undefined &&
+                            known.identity === identity
+                            ? known.path
+                            : await realpath(file.path);
+                    if (identity !== undefined) {
+                        canonicalPaths.set(key, { path: canonicalPath, identity });
+                    }
+                }
+                catch (error) {
+                    if (isTransientSourceError(error))
+                        continue;
+                    throw error;
+                }
+                files.push({
+                    adapterId: source.adapterId,
+                    ...(source.sourceDisplayName === undefined
+                        ? {}
+                        : { sourceDisplayName: source.sourceDisplayName }),
+                    root,
+                    path: canonicalPath,
+                    modifiedMs: file.modifiedMs,
+                    identity: file.identity,
+                });
+                if (files.length > maxCursorEntries * 2) {
+                    files = reconciledSourceFiles(files);
+                    files.sort(recentFileFirst);
+                    files.length = maxCursorEntries;
+                }
+            }
+        }
+        return files;
+    }
     async #pendingHistoryFiles(files) {
         const pending = [];
         for (const file of files) {
             if (this.#completedHistoryFiles.get(file.path) === file.modifiedMs)
                 continue;
+            // A failing file is retried later without holding the import open.
+            if (this.#skipping(fileKey(file.path)))
+                continue;
             try {
                 const metadata = await stat(file.path);
                 const offset = this.#cursors?.get(fileCursorId(file.path, metadata)) ??
@@ -625,8 +840,9 @@ export class JSONLSourceReader {
                 }
             }
             catch (error) {
-                if (!isTransientSourceError(error))
-                    throw error;
+                if (!isTransientSourceError(error)) {
+                    this.#recordFailure(fileKey(file.path), file.adapterId, error);
+                }
             }
         }
         return pending;
@@ -655,7 +871,11 @@ export class JSONLSourceReader {
             return;
         if (this.#cursors?.has(fileCursorId(fileReal, metadata)) || this.#cursors?.has(digest(fileReal)))
             return;
-        this.#setCursor(fileCursorId(fileReal, metadata), digest(fileReal), metadata.size);
+        // A session that began while the observer was stopped is read from its
+        // start; any other file without a cursor starts at its end, as before.
+        this.#setCursor(fileCursorId(fileReal, metadata), digest(fileReal), this.#lastRunAt !== undefined && startedAfter(metadata, this.#lastRunAt)
+            ? 0
+            : metadata.size);
     }
 }
 export function expandedActivityRoot(root) {
@@ -820,6 +1040,18 @@ function fileIdentityKey(identity) {
     }
     return `${identity.dev}:${identity.ino}:${identity.birthtimeMs}`;
 }
+/**
+ * Whether saved `cursors` are at the end of the file at `path`, so every
+ * record in it was read. A file that grew or was replaced since does not
+ * count.
+ */
+export async function fileReadThrough(path, cursors) {
+    const fileReal = await realpath(path);
+    const metadata = await stat(fileReal);
+    const offset = cursors.get(fileCursorId(fileReal, metadata)) ??
+        cursors.get(digest(fileReal));
+    return offset === metadata.size;
+}
 function fileCursorId(path, metadata) {
     const identity = metadata.ino > 0 || metadata.dev > 0
         ? `inode:${metadata.dev}:${metadata.ino}:${metadata.birthtimeMs}`
@@ -843,6 +1075,12 @@ function newestFilesPerAdapter(files, limit) {
 function configuredRoots(sources, format) {
     return new Set(sources.flatMap((source) => source.format === format ? source.roots.map(expandedActivityRoot) : []));
 }
+function sourceKey(source) {
+    return JSON.stringify([source.format ?? "jsonl", source.adapterId, source.roots, source.pathPattern ?? ""]);
+}
+function fileKey(path) {
+    return `file\u0000${path}`;
+}
 function sameCursors(left, right) {
     if (left.size !== right.size)
         return false;
@@ -868,6 +1106,7 @@ function isTransientSourceError(error) {
             error.code === "EACCES" ||
             error.code === "EPERM" ||
             error.code === "ENOTDIR" ||
+            error.code === "EBUSY" ||
             error.code === "ESTALE"));
 }
 //# sourceMappingURL=source-reader.js.map