@pdpp/local-collector 0.1.0-beta.6 → 0.1.0-beta.8

package/dist/local-collector/src/runner.d.ts

@@ -1,5 +1,5 @@
 import { type RuntimeCapabilityProfile } from "../../polyfill-connectors/src/runner/index.js";
-export { buildCollectorStartMessage, COLLECTOR_PROTOCOL_VERSION, CollectorStateReadError, drainCollectorQueue, emitToStdout, enrollCollector, evaluatePlacement, isMainModule, LocalDeviceClient, LocalDeviceHttpError, LocalDeviceOutbox, LocalDeviceQueue, PROVIDER_RUNTIME_CAPABILITIES, RUNTIME_CAPABILITY_MISMATCH_CODE, RuntimeCapabilityMismatchError, assertPlacementOrThrow, buildLocalDeviceRecordEnvelope, buildLocalDeviceOutboxId, canonicalJson, diffRequiredBindings, hashCanonicalJson, parseJsonlLine, resourceSet, runCollectorConnector, stringifyForJsonl, transformRecordsToCollectorEnvelopes, type CollectorChildContext, type CollectorConnectorSpec, type CollectorEnrollmentConfig, type CollectorRunConfig, type CollectorRunResult, type ConnectorPlacementInput, type ConnectorRuntimeRequirements, type EmittedMessage, type EnrollmentExchangeResponse, type LocalDeviceRecordEnvelope, type BuildLocalDeviceOutboxIdInput, type LocalDeviceOutboxClaimInput, type LocalDeviceOutboxDeadLetterInput, type LocalDeviceOutboxEnqueueInput, type LocalDeviceOutboxFailInput, type LocalDeviceOutboxItem, type LocalDeviceOutboxKind, type LocalDeviceOutboxLeaseInput, type LocalDeviceOutboxOptions, type LocalDeviceOutboxStatus, type LocalDeviceOutboxSummary, type PlacementDecision, type RuntimeBindingName, type RuntimeCapabilityProfile, type StartMessage, type StreamScope, } from "../../polyfill-connectors/src/runner/index.js";
+export { buildCollectorStartMessage, COLLECTOR_COVERAGE_STATUSES, COLLECTOR_PROTOCOL_VERSION, CollectorStateReadError, drainCollectorQueue, emitToStdout, enrollCollector, evaluatePlacement, isMainModule, LocalDeviceClient, LocalDeviceHttpError, LocalDeviceOutbox, LocalDeviceQueue, PROVIDER_RUNTIME_CAPABILITIES, RUNTIME_CAPABILITY_MISMATCH_CODE, RuntimeCapabilityMismatchError, assertPlacementOrThrow, buildLocalDeviceRecordEnvelope, buildLocalDeviceOutboxId, canonicalJson, classifyDeadLetterError, deriveLocalCollectorLifecycleState, diffRequiredBindings, hashCanonicalJson, LOCAL_COLLECTOR_LIFECYCLE_STATES, parseJsonlLine, resourceSet, runCollectorConnector, stringifyForJsonl, summarizeCollectorCompleteness, transformRecordsToCollectorEnvelopes, type CollectorChildContext, type CollectorCompletenessSummary, type CollectorConnectorSpec, type CollectorCoverageStatus, type CollectorEnrollmentConfig, type CollectorRunConfig, type CollectorRunResult, type ConnectorPlacementInput, type ConnectorRuntimeRequirements, type EmittedMessage, type EnrollmentExchangeResponse, type LocalCollectorLifecycleInput, type LocalCollectorLifecycleState, type LocalDeviceRecordEnvelope, type BuildLocalDeviceOutboxIdInput, type LocalDeviceOutboxClaimInput, type LocalDeviceOutboxCompactResult, type LocalDeviceOutboxDeadLetterErrorClass, type LocalDeviceOutboxDeadLetterErrorSummary, type LocalDeviceOutboxDeadLetterErrorSummaryInput, type LocalDeviceOutboxDeadLetterInput, type LocalDeviceOutboxEnqueueInput, type LocalDeviceOutboxFailInput, type LocalDeviceOutboxItem, type LocalDeviceOutboxKind, type LocalDeviceOutboxLeaseInput, type LocalDeviceOutboxOptions, type LocalDeviceOutboxPageStats, type LocalDeviceOutboxPruneSentInput, type LocalDeviceOutboxPruneSentResult, type LocalDeviceOutboxRequeueDeadLettersInput, type LocalDeviceOutboxRequeueDeadLettersResult, type LocalDeviceOutboxStatus, type LocalDeviceOutboxSummary, type PlacementDecision, type RuntimeBindingName, type RuntimeCapabilityProfile, type StartMessage, type StreamScope, } from "../../polyfill-connectors/src/runner/index.js";
 export declare const COLLECTOR_RUNTIME_CAPABILITIES: RuntimeCapabilityProfile;
 export interface BundledConnectorEntry {
     readonly connector_id: string;

package/dist/local-collector/src/runner.js

@@ -2,7 +2,7 @@ import { existsSync } from "node:fs";
 import { extname } from "node:path";
 import { fileURLToPath } from "node:url";
 import { COLLECTOR_PROTOCOL_VERSION as PROTOCOL_VERSION, COLLECTOR_RUNTIME_CAPABILITIES as POLYFILL_COLLECTOR_RUNTIME_CAPABILITIES, } from "../../polyfill-connectors/src/runner/index.js";
-export { buildCollectorStartMessage, COLLECTOR_PROTOCOL_VERSION, CollectorStateReadError, drainCollectorQueue, emitToStdout, enrollCollector, evaluatePlacement, isMainModule, LocalDeviceClient, LocalDeviceHttpError, LocalDeviceOutbox, LocalDeviceQueue, PROVIDER_RUNTIME_CAPABILITIES, RUNTIME_CAPABILITY_MISMATCH_CODE, RuntimeCapabilityMismatchError, assertPlacementOrThrow, buildLocalDeviceRecordEnvelope, buildLocalDeviceOutboxId, canonicalJson, diffRequiredBindings, hashCanonicalJson, parseJsonlLine, resourceSet, runCollectorConnector, stringifyForJsonl, transformRecordsToCollectorEnvelopes, } from "../../polyfill-connectors/src/runner/index.js";
+export { buildCollectorStartMessage, COLLECTOR_COVERAGE_STATUSES, COLLECTOR_PROTOCOL_VERSION, CollectorStateReadError, drainCollectorQueue, emitToStdout, enrollCollector, evaluatePlacement, isMainModule, LocalDeviceClient, LocalDeviceHttpError, LocalDeviceOutbox, LocalDeviceQueue, PROVIDER_RUNTIME_CAPABILITIES, RUNTIME_CAPABILITY_MISMATCH_CODE, RuntimeCapabilityMismatchError, assertPlacementOrThrow, buildLocalDeviceRecordEnvelope, buildLocalDeviceOutboxId, canonicalJson, classifyDeadLetterError, deriveLocalCollectorLifecycleState, diffRequiredBindings, hashCanonicalJson, LOCAL_COLLECTOR_LIFECYCLE_STATES, parseJsonlLine, resourceSet, runCollectorConnector, stringifyForJsonl, summarizeCollectorCompleteness, transformRecordsToCollectorEnvelopes, } from "../../polyfill-connectors/src/runner/index.js";
 export const COLLECTOR_RUNTIME_CAPABILITIES = {
     id: POLYFILL_COLLECTOR_RUNTIME_CAPABILITIES.id,
     bindings: new Set(["network", "filesystem", "local_device"]),
@@ -32,6 +32,13 @@ export const BUNDLED_CONNECTORS = Object.freeze({
             "memory_notes",
             "skills",
             "slash_commands",
+            "file_history",
+            "cache_inventory",
+            "coverage_diagnostics",
+            "debug_artifacts",
+            "downloads",
+            "backup_inventory",
+            "config_inventory",
         ]),
     }),
     codex: Object.freeze({
@@ -46,6 +53,13 @@ export const BUNDLED_CONNECTORS = Object.freeze({
             "rules",
             "prompts",
             "skills",
+            "history",
+            "session_index",
+            "logs",
+            "shell_snapshots",
+            "config_inventory",
+            "cache_inventory",
+            "coverage_diagnostics",
         ]),
     }),
 });

package/dist/polyfill-connectors/connectors/claude_code/index.js

@@ -4,9 +4,10 @@ import { readdir, readFile, stat } from "node:fs/promises";
 import { homedir } from "node:os";
 import { basename, join } from "node:path";
 import { createInterface as createFileReader } from "node:readline";
+import { readBoundedFilePreview } from "../../src/bounded-file-preview.js";
 import { runConnector } from "../../src/connector-runtime.js";
 import { isMainModule } from "../../src/is-main-module.js";
-import { buildLocalSourceInventory, listDirectoryInventory, } from "../../src/local-source-inventory.js";
+import { buildLocalSourceInventory, listDirectoryInventory, openInventoryFingerprintCursor, } from "../../src/local-source-inventory.js";
 import { safeTextPreview } from "../../src/safe-text-preview.js";
 import { ATTACHMENT_PREVIEW_CHARS, applyProjectDirScope, BYTES_PER_MB, buildMemoryNoteRecord, buildSkillRecord, buildSlashCommandRecord, extractContent, LINE_PROGRESS_INTERVAL, MESSAGE_CONTENT_PREVIEW_CHARS, makeEmptySessionAccumulator, mergeSessionObservations, parseCsvEnv, parseFrontmatter, SESSION_DIR_PREFIX_RE, TOOL_RESULT_PREVIEW_CHARS, textPreview, widenSessionTimeRange, } from "./parsers.js";
 import { validateRecord } from "./schemas.js";
@@ -148,6 +149,17 @@ export function observeJsonlFields(obj, obs, forcedSessionId) {
         }
     }
 }
+function updateSessionAccumulatorFromCurrentLine(sessionAccumulators, projectDir, obs, obj, messageCountDelta) {
+    if (!obs.sessionId) {
+        return;
+    }
+    updateSessionAccumulator(sessionAccumulators, projectDir, {
+        ...obs,
+        firstTimestamp: obj.timestamp ?? null,
+        lastTimestamp: obj.timestamp ?? null,
+        messageCount: messageCountDelta,
+    });
+}
 export function isMessageType(type) {
     return type === "user" || type === "assistant";
 }
@@ -211,16 +223,13 @@ export async function emitSessionsFromAccumulators({ emitRecord, requested, sess
         await emitRecord("sessions", { ...session });
     }
 }
-async function emitToolResultFile(args) {
-    let buf;
-    try {
-        buf = await readFile(args.full, "utf8");
-    }
-    catch {
+export async function emitToolResultFile(args) {
+    const bounded = await readBoundedFilePreview(args.full);
+    if (bounded === null) {
         return;
     }
     const rel = args.full.slice(args.toolResultsDir.length + 1);
-    const previewResult = safeTextPreview(buf, TOOL_RESULT_PREVIEW_CHARS);
+    const previewResult = safeTextPreview(bounded.buffer, TOOL_RESULT_PREVIEW_CHARS);
     await args.emitRecord("attachments", {
         id: `tool_result_file:${args.projectDir}/${args.sessionId}/${rel}`,
         session_id: args.sessionId,
@@ -354,14 +363,15 @@ async function parseJsonlFile(args) {
         if (!buildOnly && lineCount % LINE_PROGRESS_INTERVAL === 0) {
             await emit({
                 type: "PROGRESS",
-                message: `  ${path}: ${lineCount} lines parsed`,
+                message: `Claude Code phase=emit pass=emit lines_parsed=${lineCount}`,
             });
         }
+        const messageCountBeforeLine = obs.messageCount;
         observeJsonlFields(obj, obs, forcedSessionId);
         await processJsonlLine({ buildOnly, deps: { emitRecord, requested }, obj, obs });
-    }
-    if (buildOnly) {
-        updateSessionAccumulator(sessionAccumulators, projectDir, obs);
+        if (buildOnly) {
+            updateSessionAccumulatorFromCurrentLine(sessionAccumulators, projectDir, obs, obj, obs.messageCount - messageCountBeforeLine);
+        }
     }
     return obs.sessionId;
 }
@@ -494,7 +504,7 @@ async function emitProjectMemoryNotes({ emitRecord, fileMtimes, newMtimes, proje
         await emitRecord("memory_notes", buildMemoryNoteRecord({ projectDir, relPath, frontmatter, body, path: fullPath, mtimeMs: st.mtimeMs }));
     }
 }
-async function processJsonlFile({ args, forcedSessionId, path, progressLabel, projectDir, }) {
+async function processJsonlFile({ args, forcedSessionId, path, projectDir }) {
     let st;
     try {
         st = statSync(path);
@@ -509,7 +519,7 @@ async function processJsonlFile({ args, forcedSessionId, path, progressLabel, pr
     }
     await args.emit({
         type: "PROGRESS",
-        message: `${args.buildOnly ? "Indexing" : "Emitting"} ${progressLabel} (${(st.size / BYTES_PER_MB).toFixed(1)}MB)`,
+        message: `Claude Code phase=${args.buildOnly ? "index" : "emit"} pass=${args.buildOnly ? "index" : "emit"} file_size_mb=${(st.size / BYTES_PER_MB).toFixed(1)}`,
     });
     await parseJsonlFile({
         buildOnly: args.buildOnly,
@@ -530,7 +540,6 @@ async function processTopLevelJsonl(entries, projectPath, projectDir, args) {
             args,
             forcedSessionId: null,
             path: join(projectPath, f),
-            progressLabel: `${projectDir}/${f}`,
             projectDir,
         });
     }
@@ -549,7 +558,6 @@ async function processSessionDir(sessEnt, projectPath, projectDir, args) {
             args,
             forcedSessionId: sessionId,
             path: join(subagentsDir, f),
-            progressLabel: `${projectDir}/${sessionId}/subagents/${f}`,
             projectDir,
         });
     }
@@ -594,13 +602,12 @@ async function listProjectDirs(baseDir, emit) {
     try {
         projectDirs = (await readdir(baseDir)).filter((name) => !name.startsWith("."));
     }
-    catch (err) {
-        const errMsg = err instanceof Error ? err.message : String(err);
+    catch {
         await emit({
             type: "SKIP_RESULT",
             stream: "sessions",
             reason: "claude_dir_not_found",
-            message: `${baseDir} not readable: ${errMsg}`,
+            message: "Claude Code projects directory not readable",
         });
         return null;
     }
@@ -613,9 +620,10 @@ export async function scanProjectDirs(args) {
     if (projectDirs === null) {
         return;
     }
+    const totalProjectDirs = projectDirs.length;
     await args.emit({
         type: "PROGRESS",
-        message: `${projectDirs.length} project dirs in scope`,
+        message: `Claude Code phase=index pass=index total_project_dirs=${totalProjectDirs}`,
     });
     for (const projectDir of projectDirs) {
         await scanProjectDir(projectDir, args);
@@ -649,15 +657,40 @@ async function assertRequestedClaudeSources(input) {
         throw new Error(`requested Claude Code local source path(s) are missing or unreadable: ${missing.join(", ")}`);
     }
 }
+async function emitCoverageDiagnostics(input) {
+    if (!input.requested.has("coverage_diagnostics")) {
+        return;
+    }
+    for (const record of input.inventory.coverage) {
+        await input.emitRecord("coverage_diagnostics", record);
+    }
+}
+async function emitGatedInventoryStream(input) {
+    const cursor = openInventoryFingerprintCursor(input.priorState);
+    for (const record of input.records) {
+        if (cursor.shouldEmit(record)) {
+            await input.emitRecord(input.stream, record);
+        }
+    }
+    cursor.pruneStale();
+    const inventoryCursor = { fetched_at: nowIso() };
+    if (cursor.size() > 0) {
+        inventoryCursor.fingerprints = cursor.toState();
+    }
+    await input.emit({ type: "STATE", stream: input.stream, cursor: inventoryCursor });
+}
 async function emitLocalInventoryStreams(input) {
-    const inventory = await buildLocalSourceInventory("claude_code", input.claudeHome, CLAUDE_CODE_KNOWN_LOCAL_STORES);
-    for (const [stream, records] of inventory.recordsByStream) {
+    for (const [stream, records] of input.inventory.recordsByStream) {
         if (!input.requested.has(stream)) {
             continue;
         }
-        for (const record of records) {
-            await input.emitRecord(stream, record);
-        }
+        await emitGatedInventoryStream({
+            emit: input.emit,
+            emitRecord: input.emitRecord,
+            priorState: input.state[stream],
+            records,
+            stream,
+        });
     }
     if (input.requested.has("file_history")) {
         const records = await listDirectoryInventory({
@@ -668,14 +701,13 @@ async function emitLocalInventoryStreams(input) {
             stream: "file_history",
             reason: "metadata-only until payload contract is approved",
         });
-        for (const record of records) {
-            await input.emitRecord("file_history", record);
-        }
-    }
-    if (input.requested.has("coverage_diagnostics")) {
-        for (const record of inventory.coverage) {
-            await input.emitRecord("coverage_diagnostics", record);
-        }
+        await emitGatedInventoryStream({
+            emit: input.emit,
+            emitRecord: input.emitRecord,
+            priorState: input.state.file_history,
+            records,
+            stream: "file_history",
+        });
     }
 }
 async function runSkillsAndCommands(claudeHome, requested, emit, emitRecord, state) {
@@ -688,9 +720,8 @@ async function runSkillsAndCommands(claudeHome, requested, emit, emitRecord, sta
             newMtimes: state.newSkillsMtimes,
         });
     }
-    catch (err) {
-        const msg = err instanceof Error ? err.message : String(err);
-        await emit({ type: "PROGRESS", message: `skills scan skipped: ${msg}` });
+    catch {
+        await emit({ type: "PROGRESS", message: "Claude Code phase=index pass=index stream=skills scan_skipped=true" });
     }
     try {
         await emitSlashCommands({
@@ -701,9 +732,11 @@ async function runSkillsAndCommands(claudeHome, requested, emit, emitRecord, sta
             newMtimes: state.newSlashCommandMtimes,
         });
     }
-    catch (err) {
-        const msg = err instanceof Error ? err.message : String(err);
-        await emit({ type: "PROGRESS", message: `slash_commands scan skipped: ${msg}` });
+    catch {
+        await emit({
+            type: "PROGRESS",
+            message: "Claude Code phase=index pass=index stream=slash_commands scan_skipped=true",
+        });
     }
     if (requested.has("skills")) {
         await emit({
@@ -730,16 +763,19 @@ if (isMainModule(import.meta.url)) {
         async collect({ state, requested, emit, emitRecord }) {
             const claudeHome = process.env.CLAUDE_CODE_HOME || join(homedir(), ".claude");
             const baseDir = process.env.CLAUDE_CODE_PROJECTS_DIR || join(claudeHome, "projects");
+            const inventory = await buildLocalSourceInventory("claude_code", claudeHome, CLAUDE_CODE_KNOWN_LOCAL_STORES);
+            await emitCoverageDiagnostics({ emitRecord, inventory, requested });
             await assertRequestedClaudeSources({ baseDir, claudeHome, requested });
             const typedState = state;
-            const fileMtimes = streamFileMtimes(typedState, "messages") ?? typedState.file_mtimes ?? {};
+            const messageFileMtimes = streamFileMtimes(typedState, "messages") ?? typedState.file_mtimes ?? {};
+            const sessionFileMtimes = streamFileMtimes(typedState, "sessions") ?? {};
             const skillsMtimes = streamFileMtimes(typedState, "skills") ?? {};
             const slashCommandMtimes = streamFileMtimes(typedState, "slash_commands") ?? {};
             const memoryNoteMtimes = streamFileMtimes(typedState, "memory_notes") ?? {};
             const newSkillsMtimes = { ...skillsMtimes };
             const newSlashCommandMtimes = { ...slashCommandMtimes };
             const newMemoryNoteMtimes = { ...memoryNoteMtimes };
-            await emitLocalInventoryStreams({ claudeHome, requested, emitRecord });
+            await emitLocalInventoryStreams({ claudeHome, emit, emitRecord, inventory, requested, state: typedState });
             await runSkillsAndCommands(claudeHome, requested, emit, emitRecord, {
                 skillsMtimes,
                 newSkillsMtimes,
@@ -753,15 +789,16 @@ if (isMainModule(import.meta.url)) {
             if (!needsProjects) {
                 return;
             }
-            const newMtimes = { ...fileMtimes };
+            const newMessageFileMtimes = { ...messageFileMtimes };
+            const newSessionFileMtimes = { ...sessionFileMtimes };
             const sessionAccumulators = new Map();
             await scanProjectDirs({
                 baseDir,
                 buildOnly: true,
                 emit,
                 emitRecord,
-                fileMtimes,
-                newMtimes,
+                fileMtimes: sessionFileMtimes,
+                newMtimes: newSessionFileMtimes,
                 memoryNoteMtimes,
                 newMemoryNoteMtimes,
                 requested,
@@ -772,7 +809,7 @@ if (isMainModule(import.meta.url)) {
                 await emit({
                     type: "STATE",
                     stream: "sessions",
-                    cursor: { fetched_at: nowIso() },
+                    cursor: { file_mtimes: newSessionFileMtimes, fetched_at: nowIso() },
                 });
             }
             if (requested.has("memory_notes")) {
@@ -788,8 +825,8 @@ if (isMainModule(import.meta.url)) {
                     buildOnly: false,
                     emit,
                     emitRecord,
-                    fileMtimes,
-                    newMtimes,
+                    fileMtimes: messageFileMtimes,
+                    newMtimes: newMessageFileMtimes,
                     requested,
                     sessionAccumulators,
                 });
@@ -798,7 +835,7 @@ if (isMainModule(import.meta.url)) {
                 await emit({
                     type: "STATE",
                     stream: "messages",
-                    cursor: { file_mtimes: newMtimes, fetched_at: nowIso() },
+                    cursor: { file_mtimes: newMessageFileMtimes, fetched_at: nowIso() },
                 });
             }
         },