@phigon/observer 8.1.0 → 8.3.0

package/dist/src/activity/ingest.js

@@ -1,9 +1,21 @@
+import { makeDirectoryPrivate } from "../private-directory.js";
 import { randomUUID } from "node:crypto";
-import { chmod, mkdir, open, unlink, } from "node:fs/promises";
+import { chmod, mkdir, open, readdir, stat, unlink, } from "node:fs/promises";
 import { join } from "node:path";
 import { createInterface } from "node:readline";
-import { activityBridgeAgent } from "./bridges.js";
+import { activityBridgeAgent, activityBridgeAgents } from "./bridges.js";
 const maxLineBytes = 1 << 20;
+// Bridge files wait for the collector, which deletes them once delivered.
+// While no observer runs, agent hooks keep adding files that can hold
+// activity content, so past these limits the oldest are deleted.
+const maxIngestedBytes = 100 << 20;
+const maxIngestedFiles = 2_000;
+// A writing marker this old was left by an ingest run that did not finish.
+const abandonedWritingMs = 3_600_000;
+/** Beside a bridge file that an `ingest` run is still writing. */
+export const bridgeWritingSuffix = ".writing";
+/** A file written by one `ingest` run. */
+export const ingestedBridgeFile = /^\d{1,16}-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.jsonl$/u;
 export async function ingestActivityJSONL(options) {
     const requestedAgent = options.agent.trim().toLowerCase();
     const customAgentName = requestedAgent === "custom"
@@ -19,11 +31,22 @@ export async function ingestActivityJSONL(options) {
     const bridgeRoot = join(options.dataDirectory, "bridges");
     const agentRoot = join(bridgeRoot, agent.id);
     await mkdir(agentRoot, { recursive: true, mode: 0o700 });
-    await chmod(options.dataDirectory, 0o700);
+    await makeDirectoryPrivate(options.dataDirectory);
     await chmod(bridgeRoot, 0o700);
     await chmod(agentRoot, 0o700);
     const target = join(agentRoot, `${Date.now()}-${randomUUID()}.jsonl`);
-    const handle = await open(target, "wx", 0o600);
+    // The collector deletes bridge files it has delivered. This marker, which
+    // it does not read as activity, says the file may still grow.
+    const writing = `${target}${bridgeWritingSuffix}`;
+    await (await open(writing, "wx", 0o600)).close();
+    let handle;
+    try {
+        handle = await open(target, "wx", 0o600);
+    }
+    catch (error) {
+        await unlink(writing);
+        throw error;
+    }
     let records = 0;
     let malformed = 0;
     let oversized = 0;
@@ -73,12 +96,16 @@ export async function ingestActivityJSONL(options) {
     }
     finally {
         await handle.close();
+        await unlink(writing).catch(() => undefined);
     }
     if (records === 0) {
         await unlink(target);
     }
     else {
         await chmod(target, 0o600);
+        // Cleanup only frees disk space; the next run tries again.
+        await pruneIngestedFiles(bridgeRoot, target, options.limits ?? {})
+            .catch(() => undefined);
     }
     return Object.freeze({
         agent: agent.id,
@@ -90,6 +117,69 @@ export async function ingestActivityJSONL(options) {
         oversized,
     });
 }
+/**
+ * Deletes the oldest `ingest` files, across every bridge, until the rest fit
+ * the byte and file limits. The file just written, and files another run is
+ * still writing, stay.
+ */
+async function pruneIngestedFiles(bridgeRoot, keep, limits) {
+    const maxBytes = limits.maxBytes ?? maxIngestedBytes;
+    const maxFiles = limits.maxFiles ?? maxIngestedFiles;
+    const now = Date.now();
+    const files = [];
+    for (const agent of activityBridgeAgents) {
+        const root = join(bridgeRoot, agent.id);
+        let names;
+        try {
+            names = await readdir(root);
+        }
+        catch {
+            continue;
+        }
+        const present = new Set(names);
+        for (const name of names) {
+            if (!ingestedBridgeFile.test(name))
+                continue;
+            const path = join(root, name);
+            let bytes;
+            let modifiedMs;
+            try {
+                ({ size: bytes, mtimeMs: modifiedMs } = await stat(path));
+            }
+            catch {
+                continue;
+            }
+            const writing = present.has(`${name}${bridgeWritingSuffix}`) &&
+                now - modifiedMs < abandonedWritingMs;
+            files.push({
+                path,
+                bytes,
+                // The name starts with the time the run started.
+                createdAt: Number(name.slice(0, name.indexOf("-"))),
+                removable: path !== keep && !writing,
+            });
+        }
+    }
+    let bytes = files.reduce((total, file) => total + file.bytes, 0);
+    let count = files.length;
+    files.sort((left, right) => left.createdAt - right.createdAt);
+    for (const file of files) {
+        if (bytes <= maxBytes && count <= maxFiles)
+            return;
+        if (!file.removable)
+            continue;
+        try {
+            await unlink(file.path);
+        }
+        catch {
+            // Gone already, or locked by another program.
+            continue;
+        }
+        await unlink(`${file.path}${bridgeWritingSuffix}`).catch(() => undefined);
+        bytes -= file.bytes;
+        count--;
+    }
+}
 function validatedAgentName(value) {
     const name = value?.trim() ?? "";
     if (name.length === 0 ||

package/dist/src/activity/ingest.js.map

@@ -1 +1 @@
-{"version":3,"file":"ingest.js","sourceRoot":"","sources":["../../../src/activity/ingest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EACL,KAAK,EACL,KAAK,EACL,IAAI,EACJ,MAAM,GACP,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAEhD,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAEnD,MAAM,YAAY,GAAG,CAAC,IAAI,EAAE,CAAC;AAU7B,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,OAKxC;IACA,MAAM,cAAc,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC1D,MAAM,eAAe,GAAG,cAAc,KAAK,QAAQ;QACjD,CAAC,CAAC,kBAAkB,CAAC,OAAO,CAAC,SAAS,CAAC;QACvC,CAAC,CAAC,SAAS,CAAC;IACd,IAAI,cAAc,KAAK,QAAQ,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QACnE,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;IAC1E,CAAC;IACD,MAAM,KAAK,GAAG,mBAAmB,CAC/B,cAAc,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc,CACzD,CAAC;IACF,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CACb,kEAAkE,CACnE,CAAC;IACJ,CAAC;IACD,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;IAC1D,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC;IAC7C,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACzD,MAAM,KAAK,CAAC,OAAO,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;IAC1C,MAAM,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;IAC/B,MAAM,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAC9B,MAAM,MAAM,GAAG,IAAI,CACjB,SAAS,EACT,GAAG,IAAI,CAAC,GAAG,EAAE,IAAI,UAAU,EAAE,QAAQ,CACtC,CAAC;IACF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;IAC/C,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,eAAe,CAAC;YAC5B,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,SAAS,EAAE,QAAQ;YACnB,QAAQ,EAAE,KAAK;SAChB,CAAC,CAAC;QACH,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YAC/B,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE;gBAAE,SAAS;YACjC,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,YAAY,EAAE,CAAC;gBACnD,SAAS,EAAE,CAAC;gBACZ,SAAS;YACX,CAAC;YACD,IAAI,UAAU,GAAG,IAAI,CAAC;YACtB,IAAI,CAAC;gBACH,MAAM,KAAK,GAAY,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACxC,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;oBAClC,IACE,OAAO,KAAK,KAAK,QAAQ;wBACzB,KAAK,KAAK,IAAI;wBACd,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EACpB,CAAC;wBACD,SAAS,EAAE,CAAC;wBACZ,SAAS;oBACX,CAAC;oBACD,MAAM,YAAY,GAAI,KAAiC,CACrD,mBAAmB,CACpB,CAAC;oBACF,IACE,YAAY,KAAK,SAAS;wBAC1B,YAAY,KAAK,eAAe,EAChC,CAAC;wBACD,SAAS,EAAE,CAAC;wBACZ,SAAS;oBACX,CAAC;oBACD,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC;wBAC1B,GAAG,KAAK;wBACR,iBAAiB,EAAE,eAAe;qBACnC,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS,EAAE,CAAC;gBACZ,SAAS;YACX,CAAC;YACD,MAAM,MAAM,CAAC,KAAK,CAAC,GAAG,UAAU,IAAI,CAAC,CAAC;YACtC,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;IACtB,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;IACD,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;QAClB,MAAM,MAAM,CAAC,MAAM,CAAC,CAAC;IACvB,CAAC;SAAM,CAAC;QACN,MAAM,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,CAAC;QACnB,KAAK,EAAE,KAAK,CAAC,EAAE;QACf,GAAG,CAAC,eAAe,KAAK,SAAS;YAC/B,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,EAAE,SAAS,EAAE,eAAe,EAAE,CAAC;QACnC,OAAO;QACP,SAAS;QACT,SAAS;KACV,CAAC,CAAC;AACL,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAyB;IACnD,MAAM,IAAI,GAAG,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IACjC,IACE,IAAI,CAAC,MAAM,KAAK,CAAC;QACjB,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,GAAG;QACrC,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC;QACnC,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,EAC3B,CAAC;QACD,MAAM,IAAI,KAAK,CACb,wEAAwE,CACzE,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}
+{"version":3,"file":"ingest.js","sourceRoot":"","sources":["../../../src/activity/ingest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EACL,KAAK,EACL,KAAK,EACL,IAAI,EACJ,OAAO,EACP,IAAI,EACJ,MAAM,GACP,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAEhD,OAAO,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAC;AAEzE,MAAM,YAAY,GAAG,CAAC,IAAI,EAAE,CAAC;AAC7B,0EAA0E;AAC1E,sEAAsE;AACtE,iEAAiE;AACjE,MAAM,gBAAgB,GAAG,GAAG,IAAI,EAAE,CAAC;AACnC,MAAM,gBAAgB,GAAG,KAAK,CAAC;AAC/B,2EAA2E;AAC3E,MAAM,kBAAkB,GAAG,SAAS,CAAC;AAIrC,kEAAkE;AAClE,MAAM,CAAC,MAAM,mBAAmB,GAAG,UAAU,CAAC;AAE9C,0CAA0C;AAC1C,MAAM,CAAC,MAAM,kBAAkB,GAC7B,iFAAiF,CAAC;AAUpF,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,OAMxC;IACA,MAAM,cAAc,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC1D,MAAM,eAAe,GAAG,cAAc,KAAK,QAAQ;QACjD,CAAC,CAAC,kBAAkB,CAAC,OAAO,CAAC,SAAS,CAAC;QACvC,CAAC,CAAC,SAAS,CAAC;IACd,IAAI,cAAc,KAAK,QAAQ,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QACnE,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;IAC1E,CAAC;IACD,MAAM,KAAK,GAAG,mBAAmB,CAC/B,cAAc,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc,CACzD,CAAC;IACF,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CACb,kEAAkE,CACnE,CAAC;IACJ,CAAC;IACD,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;IAC1D,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC;IAC7C,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACzD,MAAM,oBAAoB,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;IAClD,MAAM,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;IAC/B,MAAM,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAC9B,MAAM,MAAM,GAAG,IAAI,CACjB,SAAS,EACT,GAAG,IAAI,CAAC,GAAG,EAAE,IAAI,UAAU,EAAE,QAAQ,CACtC,CAAC;IACF,0EAA0E;IAC1E,8DAA8D;IAC9D,MAAM,OAAO,GAAG,GAAG,MAAM,GAAG,mBAAmB,EAAE,CAAC;IAClD,MAAM,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC;IACjD,IAAI,MAAM,CAAC;IACX,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;IAC3C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC;QACtB,MAAM,KAAK,CAAC;IACd,CAAC;IACD,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,eAAe,CAAC;YAC5B,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,SAAS,EAAE,QAAQ;YACnB,QAAQ,EAAE,KAAK;SAChB,CAAC,CAAC;QACH,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YAC/B,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE;gBAAE,SAAS;YACjC,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,YAAY,EAAE,CAAC;gBACnD,SAAS,EAAE,CAAC;gBACZ,SAAS;YACX,CAAC;YACD,IAAI,UAAU,GAAG,IAAI,CAAC;YACtB,IAAI,CAAC;gBACH,MAAM,KAAK,GAAY,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACxC,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;oBAClC,IACE,OAAO,KAAK,KAAK,QAAQ;wBACzB,KAAK,KAAK,IAAI;wBACd,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EACpB,CAAC;wBACD,SAAS,EAAE,CAAC;wBACZ,SAAS;oBACX,CAAC;oBACD,MAAM,YAAY,GAAI,KAAiC,CACrD,mBAAmB,CACpB,CAAC;oBACF,IACE,YAAY,KAAK,SAAS;wBAC1B,YAAY,KAAK,eAAe,EAChC,CAAC;wBACD,SAAS,EAAE,CAAC;wBACZ,SAAS;oBACX,CAAC;oBACD,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC;wBAC1B,GAAG,KAAK;wBACR,iBAAiB,EAAE,eAAe;qBACnC,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS,EAAE,CAAC;gBACZ,SAAS;YACX,CAAC;YACD,MAAM,MAAM,CAAC,KAAK,CAAC,GAAG,UAAU,IAAI,CAAC,CAAC;YACtC,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;IACtB,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACrB,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IAC/C,CAAC;IACD,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;QAClB,MAAM,MAAM,CAAC,MAAM,CAAC,CAAC;IACvB,CAAC;SAAM,CAAC;QACN,MAAM,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QAC3B,2DAA2D;QAC3D,MAAM,kBAAkB,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC;aAC/D,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IAC5B,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,CAAC;QACnB,KAAK,EAAE,KAAK,CAAC,EAAE;QACf,GAAG,CAAC,eAAe,KAAK,SAAS;YAC/B,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,EAAE,SAAS,EAAE,eAAe,EAAE,CAAC;QACnC,OAAO;QACP,SAAS;QACT,SAAS;KACV,CAAC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,kBAAkB,CAC/B,UAAkB,EAClB,IAAY,EACZ,MAAoB;IAEpB,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,gBAAgB,CAAC;IACrD,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,gBAAgB,CAAC;IACrD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,KAAK,GAAkF,EAAE,CAAC;IAChG,KAAK,MAAM,KAAK,IAAI,oBAAoB,EAAE,CAAC;QACzC,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC;QACxC,IAAI,KAAe,CAAC;QACpB,IAAI,CAAC;YACH,KAAK,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;QAC9B,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;QAC/B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,SAAS;YAC7C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YAC9B,IAAI,KAAa,CAAC;YAClB,IAAI,UAAkB,CAAC;YACvB,IAAI,CAAC;gBACH,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;YAC5D,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS;YACX,CAAC;YACD,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,GAAG,mBAAmB,EAAE,CAAC;gBAC1D,GAAG,GAAG,UAAU,GAAG,kBAAkB,CAAC;YACxC,KAAK,CAAC,IAAI,CAAC;gBACT,IAAI;gBACJ,KAAK;gBACL,iDAAiD;gBACjD,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;gBACnD,SAAS,EAAE,IAAI,KAAK,IAAI,IAAI,CAAC,OAAO;aACrC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,IAAI,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IACjE,IAAI,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;IACzB,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,KAAK,IAAI,QAAQ,IAAI,KAAK,IAAI,QAAQ;YAAE,OAAO;QACnD,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,SAAS;QAC9B,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;QAAC,MAAM,CAAC;YACP,8CAA8C;YAC9C,SAAS;QACX,CAAC;QACD,MAAM,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,GAAG,mBAAmB,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC1E,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC;QACpB,KAAK,EAAE,CAAC;IACV,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAyB;IACnD,MAAM,IAAI,GAAG,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IACjC,IACE,IAAI,CAAC,MAAM,KAAK,CAAC;QACjB,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,GAAG;QACrC,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC;QACnC,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,EAC3B,CAAC;QACD,MAAM,IAAI,KAAK,CACb,wEAAwE,CACzE,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/src/activity/json-document-source.d.ts

@@ -0,0 +1,151 @@
+import type { SourceRecord } from "./source-reader.js";
+export type JSONDocumentOptions = Readonly<{
+    /**
+     * `json` (default): one JSON document. `jsonl`: one message per line; a
+     * final line that is still being written is read once it is complete.
+     * `mutation-log`: a VS Code chat session log (an initial state followed
+     * by set, push and delete entries) replayed into its document.
+     */
+    documentFormat?: "json" | "jsonl" | "mutation-log";
+    /**
+     * Regular expression matched against each directory's `/`-separated
+     * relative path. Only matching directories are walked.
+     */
+    directoryPattern?: string;
+    /** Dotted path of a stable id inside each message. */
+    messageIdField?: string;
+    /**
+     * Dotted fields that mark a message as finished. The last message is held
+     * back while it has none of them set, so a reply that is still being
+     * written is delivered once, complete. An earlier message is final even
+     * without them: the agent has moved on from it.
+     */
+    completeFields?: readonly string[];
+    /**
+     * Field that is true while a message is still being written (for example
+     * Cline's `partial`). The last message is held back while it is true.
+     */
+    pendingField?: string;
+    /**
+     * When set, the document's first message is delivered with this field set
+     * to true. Some agents store the user's opening request like any other
+     * message, so only its position tells it apart.
+     */
+    firstMessageField?: string;
+    /**
+     * Top-level field that holds the agent's own session id. When present it
+     * names the session and its records, so two files holding the same
+     * session produce the same record identities.
+     */
+    sessionIdField?: string;
+}>;
+export type JSONDocumentSource = Readonly<{
+    adapterId: string;
+    roots: readonly string[];
+    /** Regular expression matched against the `/`-separated relative path. */
+    pathPattern: string;
+    /**
+     * Top-level array that holds the conversation messages, or "" when the
+     * document itself is the array.
+     */
+    messagesField: string;
+}> & JSONDocumentOptions;
+type ScanOptions = Readonly<{
+    cursors: Map<string, number>;
+    /** Reused folder listings; without one every scan lists every folder. */
+    listings?: DirectoryListingCache;
+    initialScan: boolean;
+    replayOnFirstStart: boolean;
+    replaySessions: number;
+    maxReplayRecords: number;
+    maxRecords: number;
+    historical: boolean;
+    /**
+     * When the previous run last scanned, if there was one. On the first
+     * scan a document without a cursor that began after it is read from its
+     * first message instead of being baselined.
+     */
+    newSince?: number;
+}>;
+/**
+ * Whether a file was created after `time`. It must also have been modified
+ * after it; where the file system records creation times, an older file
+ * that was only written to does not count, so it is not uploaded again.
+ */
+export declare function startedAfter(metadata: Readonly<{
+    birthtimeMs: number;
+    mtimeMs: number;
+}>, time: number): boolean;
+/**
+ * Reads agents that keep each conversation as one JSON document rewritten
+ * in place (for example `{ messages: [...] }`). A document's cursor holds
+ * the number of messages already delivered and a fingerprint of the last
+ * one, so new messages are emitted once although the whole file is
+ * rewritten. A document whose delivered prefix changed (compaction or a
+ * rewind) resumes after the remembered message, or is re-baselined when
+ * that message is gone. Unchanged documents are not parsed again.
+ */
+export declare function scanJSONDocumentSource(source: JSONDocumentSource, options: ScanOptions): Promise<Readonly<{
+    records: readonly SourceRecord[];
+    files: number;
+    malformed: number;
+    oversized: number;
+    complete: boolean;
+}>>;
+export declare function packCursor(messages: readonly unknown[], count: number, idField: string | undefined): number;
+/**
+ * Index of the first undelivered message. The delivered prefix must still
+ * end with the remembered message; if it moved, resume after it; if it is
+ * gone the document was rewritten and everything present is treated as
+ * already delivered.
+ */
+export declare function resumeIndex(messages: readonly unknown[], cursor: number, idField: string | undefined): number;
+type DirectoryEntry = Readonly<{
+    name: string;
+    directory: boolean;
+    file: boolean;
+}>;
+type ListingFileSystem = Readonly<{
+    readdir: (directory: string) => Promise<readonly DirectoryEntry[]>;
+    modifiedMs: (directory: string) => Promise<number>;
+    metadata: (file: string) => Promise<Readonly<{
+        modifiedMs: number;
+        size: number;
+    }>>;
+    now: () => number;
+}>;
+/**
+ * Folder listings reused across scans. Adding, removing or renaming an entry
+ * changes its folder's modified time, so a folder is listed again only when
+ * that time moved. Checking a folder costs about half of listing it, and
+ * most folders under an agent's session store never change. A folder that
+ * changed within the last two seconds is listed again on the next scan,
+ * because coarse file-system clocks can hide a second change in the same
+ * tick, and every folder is listed afresh at least every 30 seconds.
+ *
+ * Folders and documents untouched for ten minutes are cold: they are checked
+ * every ten seconds instead of on every scan, so a conversation resumed in a
+ * quiet project is noticed within ten seconds while active ones stay live.
+ */
+export declare class DirectoryListingCache {
+    #private;
+    static readonly relistAfterMs = 30000;
+    static readonly settleMs = 2000;
+    static readonly coldAfterMs: number;
+    static readonly coldCheckMs = 10000;
+    static readonly maxFolders = 8192;
+    constructor(fileSystem?: ListingFileSystem);
+    /**
+     * The folder's entries. `trustMs` skips even the folder check while the
+     * listing is younger than that; history imports use it, since live
+     * collection already follows new conversations.
+     */
+    list(directory: string, trustMs?: number): Promise<readonly DirectoryEntry[]>;
+    /** A document's time and size; a cold document is checked every ten seconds. */
+    metadata(file: string): Promise<Readonly<{
+        modifiedMs: number;
+        size: number;
+    }>>;
+}
+export {};
+//# sourceMappingURL=json-document-source.d.ts.map

package/dist/src/activity/json-document-source.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"json-document-source.d.ts","sourceRoot":"","sources":["../../../src/activity/json-document-source.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAcvD,MAAM,MAAM,mBAAmB,GAAG,QAAQ,CAAC;IACzC;;;;;OAKG;IACH,cAAc,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,cAAc,CAAC;IACnD;;;OAGG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,sDAAsD;IACtD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;;OAKG;IACH,cAAc,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B;;;;OAIG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,CAAC,CAAC;AAEH,MAAM,MAAM,kBAAkB,GAAG,QAAQ,CAAC;IACxC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,SAAS,MAAM,EAAE,CAAC;IACzB,0EAA0E;IAC1E,WAAW,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,aAAa,EAAE,MAAM,CAAC;CACvB,CAAC,GAAG,mBAAmB,CAAC;AAEzB,KAAK,WAAW,GAAG,QAAQ,CAAC;IAC1B,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7B,yEAAyE;IACzE,QAAQ,CAAC,EAAE,qBAAqB,CAAC;IACjC,WAAW,EAAE,OAAO,CAAC;IACrB,kBAAkB,EAAE,OAAO,CAAC;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,gBAAgB,EAAE,MAAM,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,OAAO,CAAC;IACpB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC,CAAC;AAEH;;;;GAIG;AACH,wBAAgB,YAAY,CAC1B,QAAQ,EAAE,QAAQ,CAAC;IAAE,WAAW,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,EAC5D,IAAI,EAAE,MAAM,GACX,OAAO,CAGT;AASD;;;;;;;;GAQG;AACH,wBAAsB,sBAAsB,CAC1C,MAAM,EAAE,kBAAkB,EAC1B,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC,QAAQ,CAAC;IAClB,OAAO,EAAE,SAAS,YAAY,EAAE,CAAC;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,OAAO,CAAC;CACnB,CAAC,CAAC,CAsJF;AAED,wBAAgB,UAAU,CACxB,QAAQ,EAAE,SAAS,OAAO,EAAE,EAC5B,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,GAAG,SAAS,GAC1B,MAAM,CAKR;AAED;;;;;GAKG;AACH,wBAAgB,WAAW,CACzB,QAAQ,EAAE,SAAS,OAAO,EAAE,EAC5B,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,GAAG,SAAS,GAC1B,MAAM,CAcR;AAmPD,KAAK,cAAc,GAAG,QAAQ,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,OAAO,CAAC;IAAC,IAAI,EAAE,OAAO,CAAA;CAAE,CAAC,CAAC;AAWpF,KAAK,iBAAiB,GAAG,QAAQ,CAAC;IAChC,OAAO,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,OAAO,CAAC,SAAS,cAAc,EAAE,CAAC,CAAC;IACnE,UAAU,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IACnD,QAAQ,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,QAAQ,CAAC;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC,CAAC;IACpF,GAAG,EAAE,MAAM,MAAM,CAAC;CACnB,CAAC,CAAC;AAaH;;;;;;;;;;;;GAYG;AACH,qBAAa,qBAAqB;;IAChC,MAAM,CAAC,QAAQ,CAAC,aAAa,SAAU;IACvC,MAAM,CAAC,QAAQ,CAAC,QAAQ,QAAS;IACjC,MAAM,CAAC,QAAQ,CAAC,WAAW,SAAe;IAC1C,MAAM,CAAC,QAAQ,CAAC,WAAW,SAAU;IACrC,MAAM,CAAC,QAAQ,CAAC,UAAU,QAAS;gBAKvB,UAAU,GAAE,iBAAyC;IAIjE;;;;OAIG;IACG,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,SAAI,GAAG,OAAO,CAAC,SAAS,cAAc,EAAE,CAAC;IAmC9E,gFAAgF;IAC1E,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAiBtF"}