@phigon/observer 8.2.0 → 8.3.0

package/dist/src/activity/collector.js

@@ -1,7 +1,9 @@
 import { createHash } from "node:crypto";
+import { readdir, stat, unlink } from "node:fs/promises";
 import { join } from "node:path";
 import { create, toBinary } from "@bufbuild/protobuf";
 import { timestampFromDate } from "@bufbuild/protobuf/wkt";
+import { abortableDelay } from "../abortable-delay.js";
 import { ActivityKind, AgentKind, AgentState, } from "../generated/phigon/v1/common_pb.js";
 import { AgentIdentitySchema, } from "../generated/phigon/v1/identity_pb.js";
 import { mergeInventoryWithHistoricalIdentities } from "../historical-identity-store.js";
@@ -27,10 +29,19 @@ import { genericActivityAdapter } from "./adapters/generic.js";
 import { LocalPolicyEngine } from "./policy.js";
 import { vendorDraft } from "./normalize.js";
 import { objectValue, recordObservedAt, stringField, } from "./record.js";
-import { JSONLSourceReader, } from "./source-reader.js";
+import { JSONLSourceReader, fileReadThrough, } from "./source-reader.js";
+import { bridgeWritingSuffix, ingestedBridgeFile } from "./ingest.js";
+import { closedJournalSegments } from "../bridge-journal.js";
 import { processKeyCapability } from "../discovery/process-identity.js";
 const maxEnvelopeBytes = 1 << 20;
 const maxPendingCheckpoints = 256;
+// Bridge files are cleaned up this often, and kept at most this long.
+const bridgePruneIntervalMs = 5 * 60_000;
+const bridgeFileRetentionMs = 7 * 86_400_000;
+// A closed journal segment untouched this long has no writer left.
+const closedSegmentQuietMs = 60_000;
+// Saved live checkpoints remembered while their events await acknowledgement.
+const retainedLiveCheckpoints = 8;
 export const historySyncQueueHeadroom = 1_024;
 const maxDraftsPerRecord = 32;
 const adapters = new Map([
@@ -46,6 +57,10 @@ const adapters = new Map([
     // Keep the flexible generic parser after the generated bridge adapters so
     // the generic bridge cannot accidentally replace it with a fixed label.
     ["generic", genericActivityAdapter],
+    // Deliberately replaces the generated OpenCode bridge parser: the native
+    // adapter reads both OpenCode's database events and the structured CLI
+    // records (step_start, tool_use, text, step_finish) that
+    // `ingest --agent opencode` receives, as docs/activity-collection.md says.
     ["opencode", openCodeActivityAdapter],
     // Native recordings use their own adapter IDs so the OpenAI-compatible
     // `ingest --agent` bridges for the same products keep their parser.
@@ -61,6 +76,18 @@ const adapters = new Map([
     ["roo-code-task", rooCodeActivityAdapter],
     ["kilo-code-task", kiloCodeActivityAdapter],
 ]);
+/**
+ * `emit` refused an event. The reader was put back to where the scan
+ * started, so its records are read again once delivery has room.
+ */
+class DeliveryRefusedError extends Error {
+}
+const sourceIdentityFailureKey = "agent identities";
+/** The kind of an error, never its message, which may quote a record. */
+function errorName(error) {
+    const name = error instanceof Error ? error.name : "";
+    return /^[A-Za-z][A-Za-z0-9]{0,63}$/u.test(name) ? name : "unknown error";
+}
 export class ActivityCollector {
     #config;
     #credential;
@@ -74,6 +101,15 @@ export class ActivityCollector {
     #canCollect;
     #canSyncHistory;
     #onSourceIdentity;
+    #onDeliveryRefused;
+    #onSourceFailure;
+    #reportedFailures = new WeakSet();
+    // Failures in a row of record conversion, per adapter, and of saving
+    // source identities. The keys are a fixed, small set.
+    #collectionFailures = new Map();
+    #registeredSourcesFailing = false;
+    #deliveryRefused = false;
+    #refusedDeliveries = 0;
     #policy;
     #limiter;
     #agentsByKind = new Map();
@@ -87,6 +123,11 @@ export class ActivityCollector {
     #oversizedEvents = 0;
     #acknowledgedSequence;
     #durableDelivery;
+    // Saved live-reader checkpoints, oldest first, with the sequence that
+    // their events end at. Bridge files they read through can be deleted once
+    // the server acknowledges that sequence.
+    #committedLive = [];
+    #flushDelivery;
     #pendingCheckpoints = [];
     #checkpointSave = Promise.resolve();
     #sourceIdentitySaves = [];
@@ -106,6 +147,8 @@ export class ActivityCollector {
         this.#canCollect = options.canCollect ?? (() => true);
         this.#canSyncHistory = options.canSyncHistory ?? this.#canCollect;
         this.#onSourceIdentity = options.onSourceIdentity ?? (() => undefined);
+        this.#onDeliveryRefused = options.onDeliveryRefused ?? (() => undefined);
+        this.#onSourceFailure = options.onSourceFailure ?? (() => undefined);
         this.#policy = new LocalPolicyEngine(options.policy ?? options.config.policy, options.config.redactionPathPatterns);
         this.#limiter = new MinuteRateLimiter(options.config.activityRateLimitPerMinute);
         this.#lastSequence = options.initialSequence;
@@ -116,6 +159,7 @@ export class ActivityCollector {
             throw new Error("Initial activity acknowledgement is invalid.");
         }
         this.#durableDelivery = options.durableDelivery ?? false;
+        this.#flushDelivery = options.flushDelivery ?? (() => undefined);
     }
     updateInventory(inventory) {
         this.#agentsByKind.clear();
@@ -129,6 +173,9 @@ export class ActivityCollector {
         if (acknowledgedSequence < 0n || allocationSequence < 0n) {
             throw new Error("Activity sequence must not be negative.");
         }
+        // The server lost acknowledgements: no saved checkpoint is known to be
+        // delivered any more.
+        this.#committedLive = [];
         this.#acknowledgedSequence = acknowledgedSequence;
         this.#lastSequence =
             allocationSequence < acknowledgedSequence
@@ -162,26 +209,35 @@ export class ActivityCollector {
                 throw new Error("History sync expired before all available records were uploaded.");
             }
             if (!this.#canSyncHistory()) {
-                await abortableDelay(250, signal ?? new AbortController().signal);
+                await abortableDelay(250, signal);
                 continue;
             }
             // Release the collection lock between bounded batches. A large archive
             // must not prevent the live reader from following current work.
             const started = performance.now();
-            const pass = await this.#withScanLock(async () => {
-                const before = this.#emittedEvents;
-                const result = await this.#collectOnce(this.#historyReader, true);
-                eventsUploaded += this.#emittedEvents - before;
-                return result;
-            });
+            let pass;
+            try {
+                pass = await this.#withScanLock(async () => {
+                    const before = this.#emittedEvents;
+                    const result = await this.#collectOnce(this.#historyReader, true);
+                    eventsUploaded += this.#emittedEvents - before;
+                    return result;
+                });
+            }
+            catch (error) {
+                if (!(error instanceof DeliveryRefusedError))
+                    throw error;
+                await abortableDelay(250, signal);
+                continue;
+            }
             recordsScanned += pass.sourceRecords;
             filesDiscovered = Math.max(filesDiscovered, pass.discoveredFiles);
             complete = pass.complete;
             if (!complete && pass.sourceRecords === 0) {
-                await abortableDelay(100, signal ?? new AbortController().signal);
+                await abortableDelay(100, signal);
             }
             else if (!complete) {
-                await abortableDelay(historyRest(performance.now() - started), signal ?? new AbortController().signal);
+                await abortableDelay(historyRest(performance.now() - started), signal);
             }
         }
         return Object.freeze({
@@ -212,10 +268,12 @@ export class ActivityCollector {
                 discoveredFiles: 0,
             };
         }
+        const start = reader.position();
         const scan = await reader.scan([
             ...this.#sources,
-            ...await this.#additionalSources(),
+            ...await this.#registeredSources(),
         ], Math.max(1, Math.floor(allowance / maxDraftsPerRecord)));
+        this.#reportSourceFailures(reader);
         this.#sourceRecords += scan.metrics.records;
         this.#malformedRecords += scan.metrics.malformed;
         this.#oversizedRecords += scan.metrics.oversized;
@@ -224,61 +282,47 @@ export class ActivityCollector {
             const adapter = adapters.get(record.adapterId);
             if (adapter === undefined)
                 continue;
-            const sessionID = reportedSessionID(record.adapterId, record.value, record.sourceId);
-            const drafts = adapter.parse(record.value, {
-                sessionId: sessionID,
-                observedAt: record.observedAt,
-                policy: this.#policy,
-            });
-            const context = {
-                sessionId: sessionID,
-                observedAt: recordObservedAt(record.value, record.observedAt),
-                policy: this.#policy,
-            };
-            const genericAgent = record.adapterId === "generic"
-                ? reportedGenericAgentIdentity(record.value, record.sourceDisplayName)
-                : undefined;
-            const nativeDraft = vendorDraft(adapter.agentKind, context, genericAgent?.hint !== undefined
-                ? genericAgent.hint
-                : adapter.id, vendorType(record.value), record.value);
-            const candidateDrafts = [nativeDraft, ...drafts.slice(0, 31)];
-            const permittedDrafts = !historical && record.initialReplay === true &&
-                this.#config.activityInitialReplayMode === "messages"
-                ? candidateDrafts.filter((draft) => draft.payload.case === "activity" &&
-                    (draft.payload.value.kind === ActivityKind.PROMPT ||
-                        draft.payload.value.kind === ActivityKind.RESPONSE))
-                : candidateDrafts;
+            let converted;
+            try {
+                converted = this.#convertRecord(record, adapter, historical, now);
+            }
+            catch (error) {
+                // A record this observer cannot convert is skipped like a malformed
+                // one: reading it again would fail the same way, and stopping
+                // would end collection from every source.
+                this.#malformedRecords++;
+                this.#reportCollectionFailure(record.adapterId, `one record could not be converted and was skipped (${errorName(error)})`);
+                continue;
+            }
+            this.#collectionFailures.delete(record.adapterId);
+            this.#oversizedEvents += converted.oversized;
             // A source record is atomic: never advance its cursor after dropping
             // part of its normalized activity. One record may use the final budget
             // plus a bounded burst; the next scan waits for replenishment.
             if (!historical)
-                this.#limiter.consume(now, permittedDrafts.length);
-            for (const [draftIndex, draft] of permittedDrafts.entries()) {
-                const nextSequence = this.#lastSequence + 1n;
-                const event = create(EventEnvelopeSchema, {
-                    tenantId: this.#credential.tenantId,
-                    machineId: this.#credential.machineId,
-                    agentId: this.#agentID(draft.agentKind, record.adapterId, record.sourceId, record.value, record.observedAt, genericAgent?.displayName),
-                    sessionId: draft.sessionId,
-                    sequence: nextSequence,
-                    observedAt: timestampFromDate(clampObservedAt(draft.observedAt, now)),
-                    schemaVersion: 1,
-                    privacyClass: draft.privacyClass,
-                    sourceEventId: createHash("sha256")
-                        .update(`${record.sourceRecordId}\u0000${draftIndex}\u0000${draft.payload.case}`)
-                        .digest("hex"),
-                    payload: draft.payload,
-                });
-                if (toBinary(EventEnvelopeSchema, event).byteLength >
-                    maxEnvelopeBytes) {
-                    this.#oversizedEvents++;
-                    continue;
+                this.#limiter.consume(now, converted.drafts);
+            for (const event of converted.events) {
+                try {
+                    this.#emit(event);
                 }
-                this.#lastSequence = nextSequence;
-                this.#emit(event);
+                catch (error) {
+                    // The reader already moved past this scan's records. Put it back
+                    // so none is skipped; the ones delivered before this event are
+                    // sent again with the same source event IDs, which the server
+                    // deduplicates. The sequence stays unused, leaving no gap.
+                    reader.rewind(start);
+                    this.#refusedDeliveries++;
+                    if (!this.#deliveryRefused) {
+                        this.#deliveryRefused = true;
+                        this.#onDeliveryRefused(error);
+                    }
+                    throw new DeliveryRefusedError("Telemetry delivery refused an activity event.", { cause: error });
+                }
+                this.#lastSequence = event.sequence;
                 this.#emittedEvents++;
             }
         }
+        this.#deliveryRefused = false;
         await this.#flushSourceIdentitySaves();
         const pending = {
             throughSequence: this.#lastSequence,
@@ -300,10 +344,99 @@ export class ActivityCollector {
             discoveredFiles: scan.metrics.discoveredFiles,
         };
     }
+    /**
+     * The events for one source record, numbered from the next sequence, and
+     * how many normalized drafts they came from. Throws when the record cannot
+     * be converted; nothing is emitted for it then.
+     */
+    #convertRecord(record, adapter, historical, now) {
+        const sessionID = reportedSessionID(record.adapterId, record.value, record.sourceId);
+        const drafts = adapter.parse(record.value, {
+            sessionId: sessionID,
+            observedAt: record.observedAt,
+            policy: this.#policy,
+        });
+        const context = {
+            sessionId: sessionID,
+            observedAt: recordObservedAt(record.value, record.observedAt),
+            policy: this.#policy,
+        };
+        const genericAgent = record.adapterId === "generic"
+            ? reportedGenericAgentIdentity(record.value, record.sourceDisplayName)
+            : undefined;
+        const nativeDraft = vendorDraft(adapter.agentKind, context, genericAgent?.hint !== undefined
+            ? genericAgent.hint
+            : adapter.id, vendorType(record.value), record.value);
+        const candidateDrafts = [nativeDraft, ...drafts.slice(0, 31)];
+        const permittedDrafts = !historical && record.initialReplay === true &&
+            this.#config.activityInitialReplayMode === "messages"
+            ? candidateDrafts.filter((draft) => draft.payload.case === "activity" &&
+                (draft.payload.value.kind === ActivityKind.PROMPT ||
+                    draft.payload.value.kind === ActivityKind.RESPONSE))
+            : candidateDrafts;
+        const events = [];
+        let oversized = 0;
+        for (const [draftIndex, draft] of permittedDrafts.entries()) {
+            const event = create(EventEnvelopeSchema, {
+                tenantId: this.#credential.tenantId,
+                machineId: this.#credential.machineId,
+                agentId: this.#agentID(draft.agentKind, record.adapterId, record.sourceId, record.value, record.observedAt, genericAgent?.displayName),
+                sessionId: draft.sessionId,
+                sequence: this.#lastSequence + 1n + BigInt(events.length),
+                observedAt: timestampFromDate(clampObservedAt(draft.observedAt, now)),
+                schemaVersion: 1,
+                privacyClass: draft.privacyClass,
+                sourceEventId: createHash("sha256")
+                    .update(`${record.sourceRecordId}\u0000${draftIndex}\u0000${draft.payload.case}`)
+                    .digest("hex"),
+                payload: draft.payload,
+            });
+            if (toBinary(EventEnvelopeSchema, event).byteLength >
+                maxEnvelopeBytes) {
+                oversized++;
+                continue;
+            }
+            events.push(event);
+        }
+        return { events, drafts: permittedDrafts.length, oversized };
+    }
+    /**
+     * Says that collection work keeps failing, at the 1st, 2nd, 4th, 8th...
+     * failure in a row, so it stays visible without flooding the log. The
+     * message never includes activity content.
+     */
+    #reportCollectionFailure(key, message) {
+        const failures = (this.#collectionFailures.get(key) ?? 0) + 1;
+        this.#collectionFailures.set(key, failures);
+        if ((failures & (failures - 1)) !== 0)
+            return;
+        const now = this.#clock();
+        this.#onSourceFailure(Object.freeze({
+            adapterId: key,
+            failures,
+            message,
+            failedAt: now,
+            retryAt: now,
+        }));
+    }
     async run(signal) {
+        let prunedAt = 0;
         try {
             while (!signal.aborted) {
-                await this.scanOnce();
+                try {
+                    await this.scanOnce();
+                }
+                catch (error) {
+                    // A full queue or spool waits for acknowledgements like any
+                    // other backpressure; the refused records are read again.
+                    if (!(error instanceof DeliveryRefusedError))
+                        throw error;
+                }
+                if (this.#clock().getTime() - prunedAt >= bridgePruneIntervalMs) {
+                    prunedAt = this.#clock().getTime();
+                    // Cleanup only frees disk space; it retries on the next pass.
+                    await this.pruneBridgeFiles().catch(() => undefined);
+                }
                 await abortableDelay(this.#config.activityScanIntervalMs, signal);
             }
         }
@@ -311,6 +444,56 @@ export class ActivityCollector {
             this.close();
         }
     }
+    /**
+     * Deletes bridge files this observer is done with: an `ingest` file or a
+     * closed journal segment whose records were all read under a checkpoint
+     * the server has acknowledged through, or any of them after seven days.
+     * A file an `ingest` run is still writing, and each journal's newest
+     * segment, are never deleted while in use.
+     */
+    async pruneBridgeFiles() {
+        const now = this.#clock().getTime();
+        const acknowledged = this.#acknowledgedSequence;
+        const settled = this.#committedLive.findLast((checkpoint) => checkpoint.throughSequence <= acknowledged);
+        for (const agent of activityBridgeAgents) {
+            const root = join(this.#config.dataDirectory, "bridges", agent.id);
+            let names;
+            try {
+                names = await readdir(root);
+            }
+            catch {
+                continue;
+            }
+            const present = new Set(names);
+            const closedSegments = new Set(closedJournalSegments(names));
+            for (const name of names) {
+                const ingested = ingestedBridgeFile.test(name);
+                if (!ingested && !closedSegments.has(name))
+                    continue;
+                const path = join(root, name);
+                try {
+                    const modifiedMs = (await stat(path)).mtimeMs;
+                    const expired = now - modifiedMs >= bridgeFileRetentionMs;
+                    const inUse = ingested
+                        ? present.has(`${name}${bridgeWritingSuffix}`)
+                        // A segment closes when its journal starts the next one; a
+                        // writer that chose it just before may still be appending.
+                        : now - modifiedMs < closedSegmentQuietMs;
+                    if (expired ||
+                        (!inUse && settled !== undefined &&
+                            await fileReadThrough(path, settled.cursors))) {
+                        await unlink(path);
+                        if (ingested) {
+                            await unlink(`${path}${bridgeWritingSuffix}`).catch(() => undefined);
+                        }
+                    }
+                }
+                catch {
+                    // Gone, or locked by another program: retried on the next pass.
+                }
+            }
+        }
+    }
     /**
      * Release the database connections the live reader keeps open. A later
      * scan opens them again.
@@ -329,8 +512,53 @@ export class ActivityCollector {
             lastSequence: this.#lastSequence,
             pendingCheckpoints: this.#pendingCheckpoints.length,
             backpressuredScans: this.#backpressuredScans,
+            refusedDeliveries: this.#refusedDeliveries,
+            failingSources: this.#reader.sourceFailures().length,
         });
     }
+    /**
+     * Sources of registered custom agents. When their list cannot be read,
+     * this scan goes on with the built-in sources.
+     */
+    async #registeredSources() {
+        try {
+            const sources = await this.#additionalSources();
+            this.#registeredSourcesFailing = false;
+            return sources;
+        }
+        catch (error) {
+            if (!this.#registeredSourcesFailing) {
+                this.#registeredSourcesFailing = true;
+                const now = this.#clock();
+                this.#onSourceFailure(Object.freeze({
+                    adapterId: "registered agents",
+                    failures: 1,
+                    message: (error instanceof Error ? error.message : String(error))
+                        .replace(/\s+/gu, " ")
+                        .slice(0, 200),
+                    failedAt: now,
+                    // Read again on the next scan.
+                    retryAt: now,
+                }));
+            }
+            return [];
+        }
+    }
+    #reportSourceFailures(reader) {
+        for (const failure of reader.sourceFailures()) {
+            if (failure.failures !== 1 || this.#reportedFailures.has(failure))
+                continue;
+            this.#reportedFailures.add(failure);
+            this.#onSourceFailure(failure);
+        }
+    }
+    /** Sources and files skipped after a failure, by the live and history readers. */
+    sourceFailures() {
+        return [
+            ...this.#reader.sourceFailures(),
+            ...this.#historyReader.sourceFailures(),
+        ];
+    }
     sourceIdentities() {
         return [...this.#sourceIdentities.values()].sort((left, right) => left.agentId.localeCompare(right.agentId));
     }
@@ -344,15 +572,34 @@ export class ActivityCollector {
         this.#checkpointSave = this.#checkpointSave
             .catch(() => undefined)
             .then(async () => {
-            const currentDurableThrough = this.#durableDelivery
-                ? this.#lastSequence
-                : this.#acknowledgedSequence;
+            let currentDurableThrough = this.#acknowledgedSequence;
+            if (this.#durableDelivery) {
+                const emittedThrough = this.#lastSequence;
+                try {
+                    await this.#flushDelivery();
+                    currentDurableThrough = emittedThrough;
+                }
+                catch {
+                    // Not written: only checkpoints the server acknowledged move.
+                    // The rest stay pending for the next save, and delivery refuses
+                    // new events until the spool can be written again.
+                }
+            }
             const committed = new Set();
             for (const pending of eligible) {
                 if (pending.throughSequence > currentDurableThrough)
                     continue;
                 await pending.reader.commit(pending.checkpoint);
                 committed.add(pending);
+                if (pending.reader === this.#reader) {
+                    this.#committedLive = [
+                        ...this.#committedLive.slice(-(retainedLiveCheckpoints - 1)),
+                        {
+                            cursors: pending.checkpoint.cursors,
+                            throughSequence: pending.throughSequence,
+                        },
+                    ];
+                }
             }
             this.#pendingCheckpoints = this.#pendingCheckpoints.filter((candidate) => !committed.has(candidate));
         });
@@ -404,21 +651,45 @@ export class ActivityCollector {
             capabilities: ["activity", "source-isolated", "unattributed"],
         });
         this.#sourceIdentities.set(agentID, identity);
+        // An identity that could not be saved is forgotten, so the source's
+        // next record saves it again.
+        const forget = (error) => {
+            if (this.#sourceIdentities.get(agentID) === identity) {
+                this.#sourceIdentities.delete(agentID);
+            }
+            throw error;
+        };
+        let save;
         try {
-            this.#sourceIdentitySaves.push(Promise.resolve(this.#onSourceIdentity(identity)));
+            save = Promise.resolve(this.#onSourceIdentity(identity)).catch(forget);
         }
         catch (error) {
-            this.#sourceIdentitySaves.push(Promise.reject(error));
+            save = Promise.reject(error).catch(forget);
         }
+        // Handled here too: a scan that stops early leaves the save for the
+        // next flush, and an unhandled rejection would end the process.
+        save.catch(() => undefined);
+        this.#sourceIdentitySaves.push(save);
     }
+    /**
+     * Waits for the identity saves this scan started. A failed save is
+     * reported and retried with a later record; it never stops collection,
+     * because the events already carry the agent ID.
+     */
     async #flushSourceIdentitySaves() {
         const pending = this.#sourceIdentitySaves.splice(0);
         if (pending.length === 0)
             return;
         const results = await Promise.allSettled(pending);
         const failure = results.find((result) => result.status === "rejected");
-        if (failure !== undefined)
-            throw failure.reason;
+        if (failure === undefined) {
+            this.#collectionFailures.delete(sourceIdentityFailureKey);
+            return;
+        }
+        const reason = failure.reason;
+        this.#reportCollectionFailure(sourceIdentityFailureKey, (reason instanceof Error ? reason.message : String(reason))
+            .replace(/\s+/gu, " ")
+            .slice(0, 200));
     }
 }
 export function mergeInventoryWithSourceIdentities(inventory, sourceIdentities) {
@@ -845,15 +1116,4 @@ export function historyRest(spentMs) {
         return 0;
     return Math.min(1_000, Math.round(spentMs));
 }
-async function abortableDelay(milliseconds, signal) {
-    if (signal.aborted)
-        return;
-    await new Promise((resolve) => {
-        const timeout = setTimeout(resolve, milliseconds);
-        signal.addEventListener("abort", () => {
-            clearTimeout(timeout);
-            resolve();
-        }, { once: true });
-    });
-}
 //# sourceMappingURL=collector.js.map