@pdpp/local-collector 0.7.8 → 0.7.9

package/dist/polyfill-connectors/connectors/claude_code/index.js

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 import { createReadStream, statSync } from "node:fs";
-import { readdir, readFile, stat } from "node:fs/promises";
+import { readdir, stat } from "node:fs/promises";
 import { homedir } from "node:os";
 import { basename, join } from "node:path";
 import { createInterface as createFileReader } from "node:readline";
@@ -379,6 +379,10 @@ function markFileMtimeAndShouldSkip(fileMtimes, newMtimes, path, mtime) {
     newMtimes[path] = mtime;
     return fileMtimes[path] === mtime;
 }
+async function readBoundedUtf8(path) {
+    const preview = await readBoundedFilePreview(path);
+    return preview?.buffer.toString("utf8") ?? null;
+}
 async function emitSkills({ claudeHome, requested, emitRecord, fileMtimes, newMtimes }) {
     if (!requested.has("skills")) {
         return;
@@ -411,11 +415,14 @@ async function emitSkills({ claudeHome, requested, emitRecord, fileMtimes, newMt
             continue;
         }
         try {
-            raw = await readFile(skillPath, "utf8");
+            raw = await readBoundedUtf8(skillPath);
         }
         catch {
             continue;
         }
+        if (raw === null) {
+            continue;
+        }
         const { frontmatter, body } = parseFrontmatter(raw);
         await emitRecord("skills", buildSkillRecord({ name: ent.name, frontmatter, body, path: skillPath, mtimeMs: st.mtimeMs }));
     }
@@ -436,11 +443,14 @@ async function processSlashCommandFile(args) {
         return;
     }
     try {
-        raw = await readFile(args.full, "utf8");
+        raw = await readBoundedUtf8(args.full);
     }
     catch {
         return;
     }
+    if (raw === null) {
+        return;
+    }
     const { frontmatter, body } = parseFrontmatter(raw);
     const base = basename(args.name, ".md");
     const idPath = args.prefix ? `${args.prefix}/${base}` : base;
@@ -495,11 +505,14 @@ async function emitProjectMemoryNotes({ emitRecord, fileMtimes, newMtimes, proje
             continue;
         }
         try {
-            raw = await readFile(fullPath, "utf8");
+            raw = await readBoundedUtf8(fullPath);
         }
         catch {
             continue;
         }
+        if (raw === null) {
+            continue;
+        }
         const { frontmatter, body } = parseFrontmatter(raw);
         await emitRecord("memory_notes", buildMemoryNoteRecord({ projectDir, relPath, frontmatter, body, path: fullPath, mtimeMs: st.mtimeMs }));
     }

package/dist/polyfill-connectors/connectors/codex/index.js

@@ -1,11 +1,12 @@
 #!/usr/bin/env node
 import { createHash } from "node:crypto";
-import { createReadStream, existsSync, statSync } from "node:fs";
-import { readdir, readFile, stat } from "node:fs/promises";
+import { createReadStream, statSync } from "node:fs";
+import { readdir, stat } from "node:fs/promises";
 import { homedir } from "node:os";
 import { join } from "node:path";
 import { createInterface } from "node:readline";
 import { DatabaseSync } from "node:sqlite";
+import { readBoundedFilePreview } from "../../src/bounded-file-preview.js";
 import { flushAndExitAfterRuntimeAck } from "../../src/connector-exit.js";
 import { openCarryForwardCursor } from "../../src/fingerprint-cursor.js";
 import { isMainModule } from "../../src/is-main-module.js";
@@ -17,6 +18,7 @@ import { validateRecord } from "./schemas.js";
 const DEFAULT_ACTIVE_ROLLOUT_QUIET_MS = 120_000;
 const ACTIVE_ROLLOUT_QUIET_MS_ENV = "PDPP_CODEX_ACTIVE_ROLLOUT_QUIET_MS";
 const GUARD_PREFIX_BYTES = 64 * 1024;
+const MAX_PENDING_FUNCTION_CALLS = 1024;
 let stdoutDrainPromise = null;
 const emit = (m) => {
     const ok = process.stdout.write(stringifyForJsonl(m));
@@ -270,46 +272,32 @@ function openThreadsDb(dbPath) {
         return null;
     }
 }
-function queryThreadsRows(db) {
+function isThreadRow(row) {
+    return typeof row === "object" && row !== null && typeof row.id === "string";
+}
+function* queryThreadsRows(db) {
     try {
-        const rawRows = db.prepare(THREADS_QUERY).all();
-        return rawRows;
+        for (const row of db.prepare(THREADS_QUERY).iterate()) {
+            if (isThreadRow(row)) {
+                yield row;
+            }
+        }
     }
     catch {
         emit({
             type: "PROGRESS",
             message: "Codex phase=index pass=index state_db_query_failed=true fallback=rollouts_only",
         });
-        return [];
     }
 }
-function loadThreadsMap(dbPath) {
-    if (!existsSync(dbPath)) {
-        return { map: new Map(), present: false };
-    }
-    const db = openThreadsDb(dbPath);
-    if (!db) {
-        return { map: new Map(), present: false };
-    }
-    const map = new Map();
-    try {
-        for (const r of queryThreadsRows(db)) {
-            map.set(r.id, r);
-        }
-    }
-    finally {
-        try {
-            db.close();
-        }
-        catch {
-        }
-    }
-    return { map, present: true };
-}
 async function statAndRead(path) {
     try {
         const st = await stat(path);
-        const text = await readFile(path, "utf8");
+        const preview = await readBoundedFilePreview(path);
+        if (preview === null) {
+            return null;
+        }
+        const text = preview.buffer.toString("utf8");
         return { mtimeMs: Number(st.mtimeMs), size: Number(st.size), text };
     }
     catch {
@@ -428,12 +416,21 @@ function emitMessageRecord(state, payload, ts, emitRecord) {
         timestamp: ts,
     });
 }
-function registerFunctionCall(state, payload, ts) {
+function registerFunctionCall(state, payload, ts, emitRecord) {
     const sessionId = state.sessionId;
     if (!sessionId) {
         return;
     }
     const callId = payload.call_id || `${sessionId}:${state.lineCount}`;
+    while (state.pendingCalls.size >= MAX_PENDING_FUNCTION_CALLS) {
+        const oldestEntry = state.pendingCalls.entries().next();
+        if (oldestEntry.done) {
+            break;
+        }
+        const [oldestCallId, oldest] = oldestEntry.value;
+        emitRecord("function_calls", oldest);
+        state.pendingCalls.delete(oldestCallId);
+    }
     state.pendingCalls.set(callId, {
         id: callId,
         session_id: sessionId,
@@ -485,7 +482,7 @@ export function processResponseItem({ deps, payload, state, ts }) {
     if (payload.type === "function_call") {
         state.functionCallCount++;
         if (deps.requested.has("function_calls")) {
-            registerFunctionCall(state, payload, ts);
+            registerFunctionCall(state, payload, ts, deps.emitRecord);
         }
         return;
     }
@@ -575,6 +572,20 @@ export function emitSessionsFromMaps({ threadsMap, rolloutAggregates, emitRecord
         emitRecord("sessions", buildRolloutOnlySessionRecord(id, agg));
     }
 }
+function emitSessionsFromRows({ threadsRows, rolloutAggregates, emitRecord, cursor }) {
+    for (const t of threadsRows) {
+        const agg = rolloutAggregates.get(t.id);
+        rolloutAggregates.delete(t.id);
+        const prior = cursor?.prior(t.id);
+        if (shouldReemitThreadSession(t, agg, prior)) {
+            emitRecord("sessions", buildThreadSessionRecord(t.id, t, agg, prior));
+        }
+        cursor?.note(t.id, makeThreadFingerprint(t, agg, prior));
+    }
+    for (const [id, agg] of rolloutAggregates) {
+        emitRecord("sessions", buildRolloutOnlySessionRecord(id, agg));
+    }
+}
 async function parseRolloutFile(args) {
     const state = makeRolloutParseState(args.seed);
     const deps = {
@@ -754,13 +765,24 @@ async function scanRollouts(args) {
     return { parsedFiles: parsedRollouts };
 }
 function emitSessions({ stateDbPath, rolloutAggregates, emitRecord, cursor }) {
-    const { map: threadsById } = loadThreadsMap(stateDbPath);
-    emitSessionsFromMaps({
-        threadsMap: threadsById,
-        rolloutAggregates,
-        emitRecord,
-        cursor,
-    });
+    const db = openThreadsDb(stateDbPath);
+    if (!db) {
+        for (const [id, agg] of rolloutAggregates) {
+            emitRecord("sessions", buildRolloutOnlySessionRecord(id, agg));
+        }
+        return;
+    }
+    try {
+        emitSessionsFromRows({
+            threadsRows: queryThreadsRows(db),
+            rolloutAggregates,
+            emitRecord,
+            cursor,
+        });
+    }
+    finally {
+        db.close();
+    }
 }
 async function readStartMessage() {
     const rl = createInterface({ input: process.stdin, terminal: false });

package/dist/polyfill-connectors/src/collector-build-info.js

@@ -1,8 +1,8 @@
 const COLLECTOR_BUILD_SOURCE_SENTINEL = "source";
 const COLLECTOR_BUILD_INFO = {
-    builtAt: "2026-06-17T17:19:47.443Z",
-    revision: "adc321f24092",
-    version: "0.7.8",
+    builtAt: "2026-06-17T18:00:24.313Z",
+    revision: "e5531179a760",
+    version: "0.7.9",
 };
 function buildAgentVersion(info = COLLECTOR_BUILD_INFO) {
     return `${info.version}+${info.revision}`;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pdpp/local-collector",
-  "version": "0.7.8",
+  "version": "0.7.9",
   "description": "Publishable local collector runtime for PDPP: filesystem-class connectors (Claude Code, Codex) plus the device-exporter ingest client.",
   "type": "module",
   "private": false,