@phigon/observer 8.1.0 → 8.3.0

package/dist/src/activity/collector.js

@@ -1,27 +1,47 @@
 import { createHash } from "node:crypto";
+import { readdir, stat, unlink } from "node:fs/promises";
 import { join } from "node:path";
 import { create, toBinary } from "@bufbuild/protobuf";
 import { timestampFromDate } from "@bufbuild/protobuf/wkt";
+import { abortableDelay } from "../abortable-delay.js";
 import { ActivityKind, AgentKind, AgentState, } from "../generated/phigon/v1/common_pb.js";
 import { AgentIdentitySchema, } from "../generated/phigon/v1/identity_pb.js";
 import { mergeInventoryWithHistoricalIdentities } from "../historical-identity-store.js";
 import { EventEnvelopeSchema, } from "../generated/phigon/v1/telemetry_pb.js";
 import { claudeActivityAdapter } from "./adapters/claude.js";
+import { clineExtensionActivityAdapter, clineTaskFirstMessageField, kiloCodeActivityAdapter, rooCodeActivityAdapter, } from "./adapters/cline-task.js";
+import { clineActivityAdapter } from "./adapters/cline.js";
 import { codexActivityAdapter } from "./adapters/codex.js";
 import { copilotActivityAdapter } from "./adapters/copilot.js";
+import { copilotChatActivityAdapter } from "./adapters/copilot-chat.js";
+import { cursorActivityAdapter } from "./adapters/cursor.js";
+import { factoryDroidActivityAdapter } from "./adapters/factory-droid.js";
+import { geminiActivityAdapter } from "./adapters/gemini.js";
+import { gooseActivityAdapter } from "./adapters/goose.js";
 import { hermesActivityAdapter } from "./adapters/hermes.js";
+import { kiroActivityAdapter } from "./adapters/kiro.js";
 import { openClawActivityAdapter } from "./adapters/openclaw.js";
 import { openCodeActivityAdapter } from "./adapters/opencode.js";
+import { zedActivityAdapter } from "./adapters/zed.js";
 import { createOpenAICompatibleAdapter } from "./adapters/openai-compatible.js";
 import { activityBridgeAgents } from "./bridges.js";
 import { genericActivityAdapter } from "./adapters/generic.js";
 import { LocalPolicyEngine } from "./policy.js";
 import { vendorDraft } from "./normalize.js";
 import { objectValue, recordObservedAt, stringField, } from "./record.js";
-import { JSONLSourceReader, } from "./source-reader.js";
+import { JSONLSourceReader, fileReadThrough, } from "./source-reader.js";
+import { bridgeWritingSuffix, ingestedBridgeFile } from "./ingest.js";
+import { closedJournalSegments } from "../bridge-journal.js";
 import { processKeyCapability } from "../discovery/process-identity.js";
 const maxEnvelopeBytes = 1 << 20;
 const maxPendingCheckpoints = 256;
+// Bridge files are cleaned up this often, and kept at most this long.
+const bridgePruneIntervalMs = 5 * 60_000;
+const bridgeFileRetentionMs = 7 * 86_400_000;
+// A closed journal segment untouched this long has no writer left.
+const closedSegmentQuietMs = 60_000;
+// Saved live checkpoints remembered while their events await acknowledgement.
+const retainedLiveCheckpoints = 8;
 export const historySyncQueueHeadroom = 1_024;
 const maxDraftsPerRecord = 32;
 const adapters = new Map([
@@ -37,8 +57,37 @@ const adapters = new Map([
     // Keep the flexible generic parser after the generated bridge adapters so
     // the generic bridge cannot accidentally replace it with a fixed label.
     ["generic", genericActivityAdapter],
+    // Deliberately replaces the generated OpenCode bridge parser: the native
+    // adapter reads both OpenCode's database events and the structured CLI
+    // records (step_start, tool_use, text, step_finish) that
+    // `ingest --agent opencode` receives, as docs/activity-collection.md says.
     ["opencode", openCodeActivityAdapter],
+    // Native recordings use their own adapter IDs so the OpenAI-compatible
+    // `ingest --agent` bridges for the same products keep their parser.
+    ["gemini-cli", geminiActivityAdapter],
+    ["cline-cli", clineActivityAdapter],
+    ["goose-session", gooseActivityAdapter],
+    ["cursor-transcript", cursorActivityAdapter],
+    ["copilot-chat", copilotChatActivityAdapter],
+    ["kiro-session", kiroActivityAdapter],
+    ["zed-thread", zedActivityAdapter],
+    ["factory-droid", factoryDroidActivityAdapter],
+    ["cline-task", clineExtensionActivityAdapter],
+    ["roo-code-task", rooCodeActivityAdapter],
+    ["kilo-code-task", kiloCodeActivityAdapter],
 ]);
+/**
+ * `emit` refused an event. The reader was put back to where the scan
+ * started, so its records are read again once delivery has room.
+ */
+class DeliveryRefusedError extends Error {
+}
+const sourceIdentityFailureKey = "agent identities";
+/** The kind of an error, never its message, which may quote a record. */
+function errorName(error) {
+    const name = error instanceof Error ? error.name : "";
+    return /^[A-Za-z][A-Za-z0-9]{0,63}$/u.test(name) ? name : "unknown error";
+}
 export class ActivityCollector {
     #config;
     #credential;
@@ -52,6 +101,15 @@ export class ActivityCollector {
     #canCollect;
     #canSyncHistory;
     #onSourceIdentity;
+    #onDeliveryRefused;
+    #onSourceFailure;
+    #reportedFailures = new WeakSet();
+    // Failures in a row of record conversion, per adapter, and of saving
+    // source identities. The keys are a fixed, small set.
+    #collectionFailures = new Map();
+    #registeredSourcesFailing = false;
+    #deliveryRefused = false;
+    #refusedDeliveries = 0;
     #policy;
     #limiter;
     #agentsByKind = new Map();
@@ -65,6 +123,11 @@ export class ActivityCollector {
     #oversizedEvents = 0;
     #acknowledgedSequence;
     #durableDelivery;
+    // Saved live-reader checkpoints, oldest first, with the sequence that
+    // their events end at. Bridge files they read through can be deleted once
+    // the server acknowledges that sequence.
+    #committedLive = [];
+    #flushDelivery;
     #pendingCheckpoints = [];
     #checkpointSave = Promise.resolve();
     #sourceIdentitySaves = [];
@@ -84,6 +147,8 @@ export class ActivityCollector {
         this.#canCollect = options.canCollect ?? (() => true);
         this.#canSyncHistory = options.canSyncHistory ?? this.#canCollect;
         this.#onSourceIdentity = options.onSourceIdentity ?? (() => undefined);
+        this.#onDeliveryRefused = options.onDeliveryRefused ?? (() => undefined);
+        this.#onSourceFailure = options.onSourceFailure ?? (() => undefined);
         this.#policy = new LocalPolicyEngine(options.policy ?? options.config.policy, options.config.redactionPathPatterns);
         this.#limiter = new MinuteRateLimiter(options.config.activityRateLimitPerMinute);
         this.#lastSequence = options.initialSequence;
@@ -94,6 +159,7 @@ export class ActivityCollector {
             throw new Error("Initial activity acknowledgement is invalid.");
         }
         this.#durableDelivery = options.durableDelivery ?? false;
+        this.#flushDelivery = options.flushDelivery ?? (() => undefined);
     }
     updateInventory(inventory) {
         this.#agentsByKind.clear();
@@ -107,6 +173,9 @@ export class ActivityCollector {
         if (acknowledgedSequence < 0n || allocationSequence < 0n) {
             throw new Error("Activity sequence must not be negative.");
         }
+        // The server lost acknowledgements: no saved checkpoint is known to be
+        // delivered any more.
+        this.#committedLive = [];
         this.#acknowledgedSequence = acknowledgedSequence;
         this.#lastSequence =
             allocationSequence < acknowledgedSequence
@@ -140,22 +209,35 @@ export class ActivityCollector {
                 throw new Error("History sync expired before all available records were uploaded.");
             }
             if (!this.#canSyncHistory()) {
-                await abortableDelay(250, signal ?? new AbortController().signal);
+                await abortableDelay(250, signal);
                 continue;
             }
             // Release the collection lock between bounded batches. A large archive
             // must not prevent the live reader from following current work.
-            const pass = await this.#withScanLock(async () => {
-                const before = this.#emittedEvents;
-                const result = await this.#collectOnce(this.#historyReader, true);
-                eventsUploaded += this.#emittedEvents - before;
-                return result;
-            });
+            const started = performance.now();
+            let pass;
+            try {
+                pass = await this.#withScanLock(async () => {
+                    const before = this.#emittedEvents;
+                    const result = await this.#collectOnce(this.#historyReader, true);
+                    eventsUploaded += this.#emittedEvents - before;
+                    return result;
+                });
+            }
+            catch (error) {
+                if (!(error instanceof DeliveryRefusedError))
+                    throw error;
+                await abortableDelay(250, signal);
+                continue;
+            }
             recordsScanned += pass.sourceRecords;
             filesDiscovered = Math.max(filesDiscovered, pass.discoveredFiles);
             complete = pass.complete;
             if (!complete && pass.sourceRecords === 0) {
-                await abortableDelay(100, signal ?? new AbortController().signal);
+                await abortableDelay(100, signal);
+            }
+            else if (!complete) {
+                await abortableDelay(historyRest(performance.now() - started), signal);
             }
         }
         return Object.freeze({
@@ -186,10 +268,12 @@ export class ActivityCollector {
                 discoveredFiles: 0,
             };
         }
+        const start = reader.position();
         const scan = await reader.scan([
             ...this.#sources,
-            ...await this.#additionalSources(),
+            ...await this.#registeredSources(),
         ], Math.max(1, Math.floor(allowance / maxDraftsPerRecord)));
+        this.#reportSourceFailures(reader);
         this.#sourceRecords += scan.metrics.records;
         this.#malformedRecords += scan.metrics.malformed;
         this.#oversizedRecords += scan.metrics.oversized;
@@ -198,61 +282,47 @@ export class ActivityCollector {
             const adapter = adapters.get(record.adapterId);
             if (adapter === undefined)
                 continue;
-            const sessionID = reportedSessionID(record.adapterId, record.value, record.sourceId);
-            const drafts = adapter.parse(record.value, {
-                sessionId: sessionID,
-                observedAt: record.observedAt,
-                policy: this.#policy,
-            });
-            const context = {
-                sessionId: sessionID,
-                observedAt: recordObservedAt(record.value, record.observedAt),
-                policy: this.#policy,
-            };
-            const genericAgent = record.adapterId === "generic"
-                ? reportedGenericAgentIdentity(record.value, record.sourceDisplayName)
-                : undefined;
-            const nativeDraft = vendorDraft(adapter.agentKind, context, genericAgent?.hint !== undefined
-                ? genericAgent.hint
-                : adapter.id, vendorType(record.value), record.value);
-            const candidateDrafts = [nativeDraft, ...drafts.slice(0, 31)];
-            const permittedDrafts = !historical && record.initialReplay === true &&
-                this.#config.activityInitialReplayMode === "messages"
-                ? candidateDrafts.filter((draft) => draft.payload.case === "activity" &&
-                    (draft.payload.value.kind === ActivityKind.PROMPT ||
-                        draft.payload.value.kind === ActivityKind.RESPONSE))
-                : candidateDrafts;
+            let converted;
+            try {
+                converted = this.#convertRecord(record, adapter, historical, now);
+            }
+            catch (error) {
+                // A record this observer cannot convert is skipped like a malformed
+                // one: reading it again would fail the same way, and stopping
+                // would end collection from every source.
+                this.#malformedRecords++;
+                this.#reportCollectionFailure(record.adapterId, `one record could not be converted and was skipped (${errorName(error)})`);
+                continue;
+            }
+            this.#collectionFailures.delete(record.adapterId);
+            this.#oversizedEvents += converted.oversized;
             // A source record is atomic: never advance its cursor after dropping
             // part of its normalized activity. One record may use the final budget
             // plus a bounded burst; the next scan waits for replenishment.
             if (!historical)
-                this.#limiter.consume(now, permittedDrafts.length);
-            for (const [draftIndex, draft] of permittedDrafts.entries()) {
-                const nextSequence = this.#lastSequence + 1n;
-                const event = create(EventEnvelopeSchema, {
-                    tenantId: this.#credential.tenantId,
-                    machineId: this.#credential.machineId,
-                    agentId: this.#agentID(draft.agentKind, record.adapterId, record.sourceId, record.value, record.observedAt, genericAgent?.displayName),
-                    sessionId: draft.sessionId,
-                    sequence: nextSequence,
-                    observedAt: timestampFromDate(clampObservedAt(draft.observedAt, now)),
-                    schemaVersion: 1,
-                    privacyClass: draft.privacyClass,
-                    sourceEventId: createHash("sha256")
-                        .update(`${record.sourceRecordId}\u0000${draftIndex}\u0000${draft.payload.case}`)
-                        .digest("hex"),
-                    payload: draft.payload,
-                });
-                if (toBinary(EventEnvelopeSchema, event).byteLength >
-                    maxEnvelopeBytes) {
-                    this.#oversizedEvents++;
-                    continue;
+                this.#limiter.consume(now, converted.drafts);
+            for (const event of converted.events) {
+                try {
+                    this.#emit(event);
                 }
-                this.#lastSequence = nextSequence;
-                this.#emit(event);
+                catch (error) {
+                    // The reader already moved past this scan's records. Put it back
+                    // so none is skipped; the ones delivered before this event are
+                    // sent again with the same source event IDs, which the server
+                    // deduplicates. The sequence stays unused, leaving no gap.
+                    reader.rewind(start);
+                    this.#refusedDeliveries++;
+                    if (!this.#deliveryRefused) {
+                        this.#deliveryRefused = true;
+                        this.#onDeliveryRefused(error);
+                    }
+                    throw new DeliveryRefusedError("Telemetry delivery refused an activity event.", { cause: error });
+                }
+                this.#lastSequence = event.sequence;
                 this.#emittedEvents++;
             }
         }
+        this.#deliveryRefused = false;
         await this.#flushSourceIdentitySaves();
         const pending = {
             throughSequence: this.#lastSequence,
@@ -274,12 +344,163 @@ export class ActivityCollector {
             discoveredFiles: scan.metrics.discoveredFiles,
         };
     }
+    /**
+     * The events for one source record, numbered from the next sequence, and
+     * how many normalized drafts they came from. Throws when the record cannot
+     * be converted; nothing is emitted for it then.
+     */
+    #convertRecord(record, adapter, historical, now) {
+        const sessionID = reportedSessionID(record.adapterId, record.value, record.sourceId);
+        const drafts = adapter.parse(record.value, {
+            sessionId: sessionID,
+            observedAt: record.observedAt,
+            policy: this.#policy,
+        });
+        const context = {
+            sessionId: sessionID,
+            observedAt: recordObservedAt(record.value, record.observedAt),
+            policy: this.#policy,
+        };
+        const genericAgent = record.adapterId === "generic"
+            ? reportedGenericAgentIdentity(record.value, record.sourceDisplayName)
+            : undefined;
+        const nativeDraft = vendorDraft(adapter.agentKind, context, genericAgent?.hint !== undefined
+            ? genericAgent.hint
+            : adapter.id, vendorType(record.value), record.value);
+        const candidateDrafts = [nativeDraft, ...drafts.slice(0, 31)];
+        const permittedDrafts = !historical && record.initialReplay === true &&
+            this.#config.activityInitialReplayMode === "messages"
+            ? candidateDrafts.filter((draft) => draft.payload.case === "activity" &&
+                (draft.payload.value.kind === ActivityKind.PROMPT ||
+                    draft.payload.value.kind === ActivityKind.RESPONSE))
+            : candidateDrafts;
+        const events = [];
+        let oversized = 0;
+        for (const [draftIndex, draft] of permittedDrafts.entries()) {
+            const event = create(EventEnvelopeSchema, {
+                tenantId: this.#credential.tenantId,
+                machineId: this.#credential.machineId,
+                agentId: this.#agentID(draft.agentKind, record.adapterId, record.sourceId, record.value, record.observedAt, genericAgent?.displayName),
+                sessionId: draft.sessionId,
+                sequence: this.#lastSequence + 1n + BigInt(events.length),
+                observedAt: timestampFromDate(clampObservedAt(draft.observedAt, now)),
+                schemaVersion: 1,
+                privacyClass: draft.privacyClass,
+                sourceEventId: createHash("sha256")
+                    .update(`${record.sourceRecordId}\u0000${draftIndex}\u0000${draft.payload.case}`)
+                    .digest("hex"),
+                payload: draft.payload,
+            });
+            if (toBinary(EventEnvelopeSchema, event).byteLength >
+                maxEnvelopeBytes) {
+                oversized++;
+                continue;
+            }
+            events.push(event);
+        }
+        return { events, drafts: permittedDrafts.length, oversized };
+    }
+    /**
+     * Says that collection work keeps failing, at the 1st, 2nd, 4th, 8th...
+     * failure in a row, so it stays visible without flooding the log. The
+     * message never includes activity content.
+     */
+    #reportCollectionFailure(key, message) {
+        const failures = (this.#collectionFailures.get(key) ?? 0) + 1;
+        this.#collectionFailures.set(key, failures);
+        if ((failures & (failures - 1)) !== 0)
+            return;
+        const now = this.#clock();
+        this.#onSourceFailure(Object.freeze({
+            adapterId: key,
+            failures,
+            message,
+            failedAt: now,
+            retryAt: now,
+        }));
+    }
     async run(signal) {
-        while (!signal.aborted) {
-            await this.scanOnce();
-            await abortableDelay(this.#config.activityScanIntervalMs, signal);
+        let prunedAt = 0;
+        try {
+            while (!signal.aborted) {
+                try {
+                    await this.scanOnce();
+                }
+                catch (error) {
+                    // A full queue or spool waits for acknowledgements like any
+                    // other backpressure; the refused records are read again.
+                    if (!(error instanceof DeliveryRefusedError))
+                        throw error;
+                }
+                if (this.#clock().getTime() - prunedAt >= bridgePruneIntervalMs) {
+                    prunedAt = this.#clock().getTime();
+                    // Cleanup only frees disk space; it retries on the next pass.
+                    await this.pruneBridgeFiles().catch(() => undefined);
+                }
+                await abortableDelay(this.#config.activityScanIntervalMs, signal);
+            }
+        }
+        finally {
+            this.close();
+        }
+    }
+    /**
+     * Deletes bridge files this observer is done with: an `ingest` file or a
+     * closed journal segment whose records were all read under a checkpoint
+     * the server has acknowledged through, or any of them after seven days.
+     * A file an `ingest` run is still writing, and each journal's newest
+     * segment, are never deleted while in use.
+     */
+    async pruneBridgeFiles() {
+        const now = this.#clock().getTime();
+        const acknowledged = this.#acknowledgedSequence;
+        const settled = this.#committedLive.findLast((checkpoint) => checkpoint.throughSequence <= acknowledged);
+        for (const agent of activityBridgeAgents) {
+            const root = join(this.#config.dataDirectory, "bridges", agent.id);
+            let names;
+            try {
+                names = await readdir(root);
+            }
+            catch {
+                continue;
+            }
+            const present = new Set(names);
+            const closedSegments = new Set(closedJournalSegments(names));
+            for (const name of names) {
+                const ingested = ingestedBridgeFile.test(name);
+                if (!ingested && !closedSegments.has(name))
+                    continue;
+                const path = join(root, name);
+                try {
+                    const modifiedMs = (await stat(path)).mtimeMs;
+                    const expired = now - modifiedMs >= bridgeFileRetentionMs;
+                    const inUse = ingested
+                        ? present.has(`${name}${bridgeWritingSuffix}`)
+                        // A segment closes when its journal starts the next one; a
+                        // writer that chose it just before may still be appending.
+                        : now - modifiedMs < closedSegmentQuietMs;
+                    if (expired ||
+                        (!inUse && settled !== undefined &&
+                            await fileReadThrough(path, settled.cursors))) {
+                        await unlink(path);
+                        if (ingested) {
+                            await unlink(`${path}${bridgeWritingSuffix}`).catch(() => undefined);
+                        }
+                    }
+                }
+                catch {
+                    // Gone, or locked by another program: retried on the next pass.
+                }
+            }
         }
     }
+    /**
+     * Release the database connections the live reader keeps open. A later
+     * scan opens them again.
+     */
+    close() {
+        this.#reader.close();
+    }
     metrics() {
         return Object.freeze({
             sourceRecords: this.#sourceRecords,
@@ -291,8 +512,53 @@ export class ActivityCollector {
             lastSequence: this.#lastSequence,
             pendingCheckpoints: this.#pendingCheckpoints.length,
             backpressuredScans: this.#backpressuredScans,
+            refusedDeliveries: this.#refusedDeliveries,
+            failingSources: this.#reader.sourceFailures().length,
         });
     }
+    /**
+     * Sources of registered custom agents. When their list cannot be read,
+     * this scan goes on with the built-in sources.
+     */
+    async #registeredSources() {
+        try {
+            const sources = await this.#additionalSources();
+            this.#registeredSourcesFailing = false;
+            return sources;
+        }
+        catch (error) {
+            if (!this.#registeredSourcesFailing) {
+                this.#registeredSourcesFailing = true;
+                const now = this.#clock();
+                this.#onSourceFailure(Object.freeze({
+                    adapterId: "registered agents",
+                    failures: 1,
+                    message: (error instanceof Error ? error.message : String(error))
+                        .replace(/\s+/gu, " ")
+                        .slice(0, 200),
+                    failedAt: now,
+                    // Read again on the next scan.
+                    retryAt: now,
+                }));
+            }
+            return [];
+        }
+    }
+    #reportSourceFailures(reader) {
+        for (const failure of reader.sourceFailures()) {
+            if (failure.failures !== 1 || this.#reportedFailures.has(failure))
+                continue;
+            this.#reportedFailures.add(failure);
+            this.#onSourceFailure(failure);
+        }
+    }
+    /** Sources and files skipped after a failure, by the live and history readers. */
+    sourceFailures() {
+        return [
+            ...this.#reader.sourceFailures(),
+            ...this.#historyReader.sourceFailures(),
+        ];
+    }
     sourceIdentities() {
         return [...this.#sourceIdentities.values()].sort((left, right) => left.agentId.localeCompare(right.agentId));
     }
@@ -306,15 +572,34 @@ export class ActivityCollector {
         this.#checkpointSave = this.#checkpointSave
             .catch(() => undefined)
             .then(async () => {
-            const currentDurableThrough = this.#durableDelivery
-                ? this.#lastSequence
-                : this.#acknowledgedSequence;
+            let currentDurableThrough = this.#acknowledgedSequence;
+            if (this.#durableDelivery) {
+                const emittedThrough = this.#lastSequence;
+                try {
+                    await this.#flushDelivery();
+                    currentDurableThrough = emittedThrough;
+                }
+                catch {
+                    // Not written: only checkpoints the server acknowledged move.
+                    // The rest stay pending for the next save, and delivery refuses
+                    // new events until the spool can be written again.
+                }
+            }
             const committed = new Set();
             for (const pending of eligible) {
                 if (pending.throughSequence > currentDurableThrough)
                     continue;
                 await pending.reader.commit(pending.checkpoint);
                 committed.add(pending);
+                if (pending.reader === this.#reader) {
+                    this.#committedLive = [
+                        ...this.#committedLive.slice(-(retainedLiveCheckpoints - 1)),
+                        {
+                            cursors: pending.checkpoint.cursors,
+                            throughSequence: pending.throughSequence,
+                        },
+                    ];
+                }
             }
             this.#pendingCheckpoints = this.#pendingCheckpoints.filter((candidate) => !committed.has(candidate));
         });
@@ -366,21 +651,45 @@ export class ActivityCollector {
             capabilities: ["activity", "source-isolated", "unattributed"],
         });
         this.#sourceIdentities.set(agentID, identity);
+        // An identity that could not be saved is forgotten, so the source's
+        // next record saves it again.
+        const forget = (error) => {
+            if (this.#sourceIdentities.get(agentID) === identity) {
+                this.#sourceIdentities.delete(agentID);
+            }
+            throw error;
+        };
+        let save;
         try {
-            this.#sourceIdentitySaves.push(Promise.resolve(this.#onSourceIdentity(identity)));
+            save = Promise.resolve(this.#onSourceIdentity(identity)).catch(forget);
         }
         catch (error) {
-            this.#sourceIdentitySaves.push(Promise.reject(error));
+            save = Promise.reject(error).catch(forget);
         }
+        // Handled here too: a scan that stops early leaves the save for the
+        // next flush, and an unhandled rejection would end the process.
+        save.catch(() => undefined);
+        this.#sourceIdentitySaves.push(save);
     }
+    /**
+     * Waits for the identity saves this scan started. A failed save is
+     * reported and retried with a later record; it never stops collection,
+     * because the events already carry the agent ID.
+     */
     async #flushSourceIdentitySaves() {
         const pending = this.#sourceIdentitySaves.splice(0);
         if (pending.length === 0)
             return;
         const results = await Promise.allSettled(pending);
         const failure = results.find((result) => result.status === "rejected");
-        if (failure !== undefined)
-            throw failure.reason;
+        if (failure === undefined) {
+            this.#collectionFailures.delete(sourceIdentityFailureKey);
+            return;
+        }
+        const reason = failure.reason;
+        this.#reportCollectionFailure(sourceIdentityFailureKey, (reason instanceof Error ? reason.message : String(reason))
+            .replace(/\s+/gu, " ")
+            .slice(0, 200));
     }
 }
 export function mergeInventoryWithSourceIdentities(inventory, sourceIdentities) {
@@ -557,6 +866,99 @@ function configuredSources(config) {
             roots: config.activitySourceDirectories.opencode,
             format: "opencode-sqlite",
         },
+        {
+            adapterId: "gemini-cli",
+            roots: config.activitySourceDirectories.gemini,
+            // <project>/chats/session-*.jsonl and subagent <parent>/<id>.jsonl.
+            pathPattern: "^[^/]+/chats/",
+        },
+        {
+            adapterId: "cline-cli",
+            roots: config.activitySourceDirectories.cline,
+            format: "json-messages",
+            // <session>/<session>.messages.json, rewritten in place per turn.
+            pathPattern: "^[^/]+/[^/]+\\.messages\\.json$",
+            messagesField: "messages",
+        },
+        {
+            adapterId: "goose-session",
+            roots: config.activitySourceDirectories.goose,
+            format: "goose-sqlite",
+        },
+        {
+            adapterId: "cursor-transcript",
+            roots: config.activitySourceDirectories.cursor,
+            format: "json-messages",
+            // <project>/agent-transcripts/<id>/<id>.jsonl and
+            // <id>/subagents/<sub>.jsonl. The final line often has no newline yet.
+            pathPattern: "^[^/]+/agent-transcripts/.+\\.jsonl$",
+            document: {
+                documentFormat: "jsonl",
+                directoryPattern: "^[^/]+(?:/agent-transcripts(?:/[^/]+(?:/subagents)?)?)?$",
+            },
+        },
+        // VS Code chat sessions: <workspace>/chatSessions/<id>.json documents
+        // and, with operation-log storage, <id>.jsonl logs of the same session.
+        ...["json", "jsonl"].map((extension) => ({
+            adapterId: "copilot-chat",
+            roots: config.activitySourceDirectories.copilotChat,
+            format: "json-messages",
+            pathPattern: `^(?:[^/]+/chatSessions/)?[^/]+\\.${extension}$`,
+            messagesField: "requests",
+            document: {
+                documentFormat: extension === "json"
+                    ? "json"
+                    : "mutation-log",
+                directoryPattern: "^[^/]+(?:/chatSessions)?$",
+                messageIdField: "requestId",
+                completeFields: ["result", "isCanceled", "modelState.completedAt"],
+                sessionIdField: "sessionId",
+            },
+        })),
+        {
+            adapterId: "kiro-session",
+            roots: config.activitySourceDirectories.kiro,
+            format: "json-messages",
+            // <workspace>/<session id>.json beside the <workspace>/sessions.json index.
+            pathPattern: "^[^/]+/(?!sessions[.]json$)[^/]+[.]json$",
+            messagesField: "history",
+            document: {
+                directoryPattern: "^[^/]+$",
+                messageIdField: "message.id",
+                sessionIdField: "sessionId",
+            },
+        },
+        {
+            adapterId: "zed-thread",
+            roots: config.activitySourceDirectories.zed,
+            format: "zed-sqlite",
+        },
+        {
+            adapterId: "factory-droid",
+            roots: config.activitySourceDirectories.droid,
+            // <project>/<session id>.jsonl, appended as the session runs, beside
+            // <session id>.settings.json.
+            pathPattern: "^(?:[^/]+/)?[^/]+\\.jsonl$",
+        },
+        // Cline, Roo Code and Kilo Code tasks: <task id>/ui_messages.json, the
+        // list of messages the extension shows, rewritten as the task runs.
+        ...[
+            ["cline-task", config.activitySourceDirectories.clineTasks],
+            ["roo-code-task", config.activitySourceDirectories.rooCode],
+            ["kilo-code-task", config.activitySourceDirectories.kiloCode],
+        ].map(([adapterId, roots]) => ({
+            adapterId,
+            roots,
+            format: "json-messages",
+            pathPattern: "^[^/]+/ui_messages[.]json$",
+            messagesField: "",
+            document: {
+                directoryPattern: "^[^/]+$",
+                messageIdField: "ts",
+                pendingField: "partial",
+                firstMessageField: clineTaskFirstMessageField,
+            },
+        })),
         {
             adapterId: "generic",
             roots: config.activitySourceDirectories.generic,
@@ -602,6 +1004,17 @@ function sourceIdentityDisplayName(adapterID, sourceID, sourceDisplayName) {
             case "hermes": return "Hermes Agent";
             case "openclaw": return "OpenClaw";
             case "opencode": return "OpenCode";
+            case "gemini-cli": return "Gemini CLI";
+            case "cline-cli": return "Cline";
+            case "goose-session": return "Goose";
+            case "cursor-transcript": return "Cursor";
+            case "copilot-chat": return "GitHub Copilot";
+            case "kiro-session": return "Kiro";
+            case "zed-thread": return "Zed";
+            case "factory-droid": return "Factory Droid";
+            case "cline-task": return "Cline";
+            case "roo-code-task": return "Roo Code";
+            case "kilo-code-task": return "Kilo Code";
             default: return "Agent activity";
         }
     })();
@@ -692,15 +1105,15 @@ function clampObservedAt(value, now) {
         return now;
     return value;
 }
-async function abortableDelay(milliseconds, signal) {
-    if (signal.aborted)
-        return;
-    await new Promise((resolve) => {
-        const timeout = setTimeout(resolve, milliseconds);
-        signal.addEventListener("abort", () => {
-            clearTimeout(timeout);
-            resolve();
-        }, { once: true });
-    });
+/**
+ * How long a history import rests after a batch that took `spentMs`. Resting
+ * as long as the batch worked keeps a large first import near half a core,
+ * and leaves the other half of the event loop to live collection; a full
+ * catch-up once held a whole core for minutes. One second at most.
+ */
+export function historyRest(spentMs) {
+    if (!Number.isFinite(spentMs) || spentMs <= 0)
+        return 0;
+    return Math.min(1_000, Math.round(spentMs));
 }
 //# sourceMappingURL=collector.js.map