@roberttlange/agentlens 0.2.3 → 0.3.0

package/node_modules/@agentlens/core/dist/traceIndex.js

@@ -25,6 +25,24 @@ const ACTIVITY_BIN_COUNT = 12;
 const ACTIVE_IDLE_GAP_MS = 20 * 60_000;
 const MATERIALIZED_TTL_MS = 5 * 60_000;
 const DIRTY_BATCH_LIMIT = 64;
+const DIRTY_REFRESH_DELAY_MS = 25;
+const WATCH_WRITE_STABILITY_MIN_MS = 35;
+const WATCH_WRITE_STABILITY_MAX_MS = 75;
+const WATCH_WRITE_POLL_INTERVAL_MS = 20;
+const STARTUP_RECENT_TRACE_LIMIT = 120;
+const BACKGROUND_HYDRATE_BATCH_SIZE = 25;
+const BACKGROUND_HYDRATE_DELAY_MS = 25;
+function createInitialStartupStatus() {
+    return {
+        phase: "cold",
+        inspectorReady: false,
+        fullReady: false,
+        isPartial: false,
+        discoveredTraceCount: 0,
+        hydratedTraceCount: 0,
+        startupError: "",
+    };
+}
 function emptyEventKindCounts() {
     return {
         system: 0,
@@ -74,6 +92,21 @@ function buildSessionActivityArtifacts(events) {
         activeSegments: buildActiveSegmentsFromEventTimestamps(eventTimestamps),
     };
 }
+function buildSessionUsageArtifacts(events, agent, config) {
+    return {
+        eventCount: events.length,
+        usagePoints: deriveSessionMetrics(events, agent, config).usagePoints,
+    };
+}
+function compactActivityArtifacts(artifacts) {
+    if (artifacts.eventTimestamps.length === 0)
+        return artifacts;
+    return {
+        eventCount: artifacts.eventCount,
+        eventTimestamps: [],
+        activeSegments: artifacts.activeSegments,
+    };
+}
 function normalizeActivityCounts(counts) {
     let maxCount = 0;
     for (const count of counts) {
@@ -479,6 +512,7 @@ export class TraceIndex extends EventEmitter {
     config;
     entries = new Map();
     pathToId = new Map();
+    discoveredFilesById = new Map();
     cursorById = new Map();
     watcher = null;
     timer = null;
@@ -489,6 +523,11 @@ export class TraceIndex extends EventEmitter {
     queuedForceFullRefresh = false;
     dirtyPaths = new Set();
     forceReparsePaths = new Set();
+    pendingHydrationFiles = [];
+    pendingHydrationIds = new Set();
+    pendingHydrationPaths = new Set();
+    pendingHydrationCursor = 0;
+    startupStatus = createInitialStartupStatus();
     adaptiveIntervalMs;
     perf = {
         refreshCount: 0,
@@ -532,8 +571,43 @@ export class TraceIndex extends EventEmitter {
     }
     async start() {
         this.started = true;
-        await this.refresh();
+        this.startupStatus = {
+            phase: "bootstrapping",
+            inspectorReady: false,
+            fullReady: false,
+            isPartial: false,
+            discoveredTraceCount: 0,
+            hydratedTraceCount: 0,
+            startupError: "",
+        };
+        this.pendingHydrationFiles = [];
+        this.pendingHydrationIds.clear();
+        this.pendingHydrationPaths.clear();
+        this.pendingHydrationCursor = 0;
         await this.restartWatcher();
+        if (!this.started) {
+            if (this.watcher) {
+                await this.watcher.close();
+                this.watcher = null;
+            }
+            return;
+        }
+        try {
+            await this.bootstrapRecentTraces();
+        }
+        catch (error) {
+            this.startupStatus = {
+                ...this.startupStatus,
+                phase: "failed",
+                inspectorReady: false,
+                fullReady: false,
+                isPartial: false,
+                startupError: error instanceof Error ? error.message : String(error),
+            };
+            throw error;
+        }
+        if (!this.started)
+            return;
         this.scheduleNextRefresh(this.computeBaseIntervalMs());
     }
     stop() {
@@ -547,6 +621,11 @@ export class TraceIndex extends EventEmitter {
             this.watcher = null;
         }
         this.forceReparsePaths.clear();
+        this.pendingHydrationFiles = [];
+        this.pendingHydrationIds.clear();
+        this.pendingHydrationPaths.clear();
+        this.pendingHydrationCursor = 0;
+        this.discoveredFilesById.clear();
     }
     async refresh() {
         this.queuedForceFullRefresh = true;
@@ -564,6 +643,40 @@ export class TraceIndex extends EventEmitter {
     getStreamVersion() {
         return this.streamVersion;
     }
+    getStartupStatus() {
+        return { ...this.startupStatus };
+    }
+    getStartupState() {
+        return this.getStartupStatus();
+    }
+    getHydrationProgress(windowStartMs) {
+        if (!this.startupStatus.inspectorReady) {
+            return {
+                ready: false,
+                relevantDiscoveredCount: 0,
+                relevantHydratedCount: 0,
+                percent: 0,
+            };
+        }
+        let relevantDiscoveredCount = 0;
+        let relevantHydratedCount = 0;
+        for (const file of this.discoveredFilesById.values()) {
+            if (file.mtimeMs < windowStartMs)
+                continue;
+            relevantDiscoveredCount += 1;
+            if (!this.pendingHydrationIds.has(file.id)) {
+                relevantHydratedCount += 1;
+            }
+        }
+        const ready = relevantHydratedCount >= relevantDiscoveredCount;
+        const percent = relevantDiscoveredCount === 0 ? 100 : Math.round((relevantHydratedCount / relevantDiscoveredCount) * 100);
+        return {
+            ready,
+            relevantDiscoveredCount,
+            relevantHydratedCount,
+            percent,
+        };
+    }
     getTopTools(limit = 12) {
         const counts = new Map();
         for (const entry of this.entries.values()) {
@@ -576,6 +689,91 @@ export class TraceIndex extends EventEmitter {
             .slice(0, Math.max(1, limit))
             .map(([name, count]) => ({ name, count }));
     }
+    async bootstrapRecentTraces() {
+        const bootstrapNowMs = nowMs();
+        const files = await discoverTraceFiles(this.config);
+        this.lastFullRefreshAtMs = bootstrapNowMs;
+        this.forceReparsePaths.clear();
+        const nextPathToId = new Map();
+        const discoveredFilesById = new Map();
+        for (const file of files) {
+            nextPathToId.set(file.path, file.id);
+            discoveredFilesById.set(file.id, file);
+        }
+        this.pathToId = nextPathToId;
+        this.discoveredFilesById = discoveredFilesById;
+        const bootstrapFiles = files.slice(0, STARTUP_RECENT_TRACE_LIMIT);
+        const pendingFiles = files.slice(bootstrapFiles.length);
+        this.pendingHydrationFiles = pendingFiles;
+        this.pendingHydrationIds = new Set(pendingFiles.map((file) => file.id));
+        this.pendingHydrationPaths = new Set(pendingFiles.map((file) => file.path));
+        this.pendingHydrationCursor = 0;
+        this.startupStatus = {
+            phase: bootstrapFiles.length > 0 ? "bootstrapping" : pendingFiles.length > 0 ? "hydrating" : "ready",
+            inspectorReady: false,
+            fullReady: pendingFiles.length === 0,
+            isPartial: pendingFiles.length > 0,
+            discoveredTraceCount: files.length,
+            hydratedTraceCount: 0,
+            startupError: "",
+        };
+        const stats = {
+            parsedFileCount: 0,
+            dirtyPathCount: 0,
+            usedFullRefresh: false,
+            hadFileMutations: false,
+        };
+        for (const file of bootstrapFiles) {
+            const changed = await this.upsertFile(file, bootstrapNowMs);
+            if (changed) {
+                stats.parsedFileCount += 1;
+                stats.hadFileMutations = true;
+            }
+        }
+        this.startupStatus = {
+            ...this.startupStatus,
+            phase: pendingFiles.length > 0 ? "hydrating" : "ready",
+            inspectorReady: true,
+            fullReady: pendingFiles.length === 0,
+            isPartial: pendingFiles.length > 0,
+            hydratedTraceCount: this.entries.size,
+        };
+        this.finishMutationBatch(bootstrapNowMs, stats);
+    }
+    finishMutationBatch(refreshNowMs, stats) {
+        this.applyRetention(refreshNowMs);
+        this.refreshActivityStatus(refreshNowMs, stats);
+        this.perf.trackedFiles = this.entries.size;
+        this.perf.dirtyPathQueue = this.dirtyPaths.size;
+        const retentionStats = this.buildRetentionStats();
+        this.perf.hotTraces = retentionStats.hotTraces;
+        this.perf.warmTraces = retentionStats.warmTraces;
+        this.perf.coldTraces = retentionStats.coldTraces;
+        this.perf.materializedTraces = retentionStats.materializedTraces;
+        this.emitOverviewUpdated();
+    }
+    emitOverviewUpdated() {
+        this.emitStream("overview_updated", {
+            overview: this.getOverview(),
+            startup: this.getStartupStatus(),
+        });
+    }
+    hasPendingHydrationWork() {
+        return this.pendingHydrationIds.size > 0;
+    }
+    updateStartupProgress() {
+        const remainingCount = this.pendingHydrationIds.size;
+        const discoveredTraceCount = this.startupStatus.discoveredTraceCount;
+        this.startupStatus = {
+            ...this.startupStatus,
+            phase: this.startupStatus.startupError ? "failed" : remainingCount > 0 ? "hydrating" : "ready",
+            inspectorReady: true,
+            fullReady: remainingCount === 0,
+            isPartial: remainingCount > 0,
+            discoveredTraceCount,
+            hydratedTraceCount: Math.max(0, discoveredTraceCount - remainingCount),
+        };
+    }
     buildRetentionStats() {
         let hotTraces = 0;
         let warmTraces = 0;
@@ -625,13 +823,14 @@ export class TraceIndex extends EventEmitter {
         if (roots.length === 0)
             return;
         const debounceMs = Math.max(50, this.config.scan.batchDebounceMs);
+        const writeStabilityMs = Math.max(WATCH_WRITE_STABILITY_MIN_MS, Math.min(WATCH_WRITE_STABILITY_MAX_MS, debounceMs));
         this.watcher = chokidar.watch(roots, {
             ignoreInitial: true,
             persistent: true,
             followSymlinks: false,
             awaitWriteFinish: {
-                stabilityThreshold: debounceMs,
-                pollInterval: 40,
+                stabilityThreshold: writeStabilityMs,
+                pollInterval: WATCH_WRITE_POLL_INTERVAL_MS,
             },
         });
         const onDirty = (rawPath) => {
@@ -651,7 +850,7 @@ export class TraceIndex extends EventEmitter {
                 }
             }
             this.perf.dirtyPathQueue = this.dirtyPaths.size;
-            this.scheduleNextRefresh(debounceMs);
+            this.scheduleNextRefresh(DIRTY_REFRESH_DELAY_MS);
         };
         this.watcher.on("add", onDirty);
         this.watcher.on("change", onDirty);
@@ -752,6 +951,8 @@ export class TraceIndex extends EventEmitter {
         return Array.from(matches.values());
     }
     shouldRunFullRefresh(nowMsValue) {
+        if (this.hasPendingHydrationWork())
+            return false;
         if (this.lastFullRefreshAtMs <= 0)
             return true;
         const fullIntervalMs = Math.max(1_000, this.config.scan.fullRescanIntervalMs);
@@ -781,6 +982,9 @@ export class TraceIndex extends EventEmitter {
             if (this.refreshPending || this.queuedForceFullRefresh) {
                 this.scheduleNextRefresh(25);
             }
+            else if (this.hasPendingHydrationWork()) {
+                this.scheduleNextRefresh(BACKGROUND_HYDRATE_DELAY_MS);
+            }
             else if (this.started) {
                 this.scheduleNextRefresh();
             }
@@ -822,31 +1026,45 @@ export class TraceIndex extends EventEmitter {
         if (useFullRefresh) {
             await this.refreshFull(refreshNowMs, stats);
         }
-        else {
+        else if (this.dirtyPaths.size > 0) {
             await this.refreshDirty(refreshNowMs, stats);
         }
-        this.applyRetention(refreshNowMs);
-        this.refreshActivityStatus(refreshNowMs, stats);
-        this.perf.trackedFiles = this.entries.size;
-        this.perf.dirtyPathQueue = this.dirtyPaths.size;
-        const retentionStats = this.buildRetentionStats();
-        this.perf.hotTraces = retentionStats.hotTraces;
-        this.perf.warmTraces = retentionStats.warmTraces;
-        this.perf.coldTraces = retentionStats.coldTraces;
-        this.perf.materializedTraces = retentionStats.materializedTraces;
-        this.emitStream("overview_updated", { overview: this.getOverview() });
+        else if (this.hasPendingHydrationWork()) {
+            await this.hydratePendingBatch(refreshNowMs, stats);
+        }
+        if (this.startupStatus.inspectorReady && this.hasPendingHydrationWork()) {
+            this.updateStartupProgress();
+        }
+        else if (this.startupStatus.inspectorReady && !this.startupStatus.fullReady) {
+            this.startupStatus = {
+                ...this.startupStatus,
+                phase: "ready",
+                fullReady: true,
+                isPartial: false,
+                hydratedTraceCount: this.startupStatus.discoveredTraceCount,
+            };
+        }
+        this.finishMutationBatch(refreshNowMs, stats);
         return stats;
     }
     async refreshFull(refreshNowMs, stats) {
         const files = await discoverTraceFiles(this.config);
         this.lastFullRefreshAtMs = refreshNowMs;
         this.forceReparsePaths.clear();
+        this.pendingHydrationFiles = [];
+        this.pendingHydrationIds.clear();
+        this.pendingHydrationPaths.clear();
+        this.pendingHydrationCursor = 0;
+        this.discoveredFilesById.clear();
         const nextIds = new Set(files.map((file) => file.id));
         const nextPathToId = new Map();
+        const discoveredFilesById = new Map();
         for (const file of files) {
             nextPathToId.set(file.path, file.id);
+            discoveredFilesById.set(file.id, file);
         }
         this.pathToId = nextPathToId;
+        this.discoveredFilesById = discoveredFilesById;
         for (const existingId of this.entries.keys()) {
             if (!nextIds.has(existingId)) {
                 this.entries.delete(existingId);
@@ -861,7 +1079,17 @@ export class TraceIndex extends EventEmitter {
                 stats.parsedFileCount += 1;
                 stats.hadFileMutations = true;
             }
+            this.applyRetention(refreshNowMs);
         }
+        this.startupStatus = {
+            phase: "ready",
+            inspectorReady: true,
+            fullReady: true,
+            isPartial: false,
+            discoveredTraceCount: files.length,
+            hydratedTraceCount: files.length,
+            startupError: "",
+        };
     }
     async refreshDirty(refreshNowMs, stats) {
         if (this.dirtyPaths.size === 0)
@@ -893,14 +1121,20 @@ export class TraceIndex extends EventEmitter {
                     this.emitStream("trace_removed", { id: existingId });
                     stats.hadFileMutations = true;
                 }
+                this.pendingHydrationPaths.delete(normalizedPath);
+                this.pendingHydrationIds.delete(existingId ?? "");
+                if (existingId)
+                    this.discoveredFilesById.delete(existingId);
                 this.pathToId.delete(normalizedPath);
                 continue;
             }
             this.pathToId.set(file.path, file.id);
+            this.discoveredFilesById.set(file.id, file);
             if (existingId && existingId !== file.id && this.entries.delete(existingId)) {
                 this.cursorById.delete(existingId);
                 this.emitStream("trace_removed", { id: existingId });
                 stats.hadFileMutations = true;
+                this.discoveredFilesById.delete(existingId);
             }
             if (processedIds.has(file.id)) {
                 if (!dirtyEntry.forceReparse)
@@ -914,6 +1148,33 @@ export class TraceIndex extends EventEmitter {
                 stats.parsedFileCount += 1;
                 stats.hadFileMutations = true;
             }
+            this.pendingHydrationIds.delete(file.id);
+            this.pendingHydrationPaths.delete(file.path);
+        }
+    }
+    async hydratePendingBatch(refreshNowMs, stats) {
+        if (!this.hasPendingHydrationWork())
+            return;
+        let processedCount = 0;
+        while (this.pendingHydrationCursor < this.pendingHydrationFiles.length && processedCount < BACKGROUND_HYDRATE_BATCH_SIZE) {
+            const file = this.pendingHydrationFiles[this.pendingHydrationCursor];
+            this.pendingHydrationCursor += 1;
+            if (!file)
+                continue;
+            if (!this.pendingHydrationIds.has(file.id) || !this.pendingHydrationPaths.has(file.path))
+                continue;
+            this.pendingHydrationIds.delete(file.id);
+            this.pendingHydrationPaths.delete(file.path);
+            const changed = await this.upsertFile(file, refreshNowMs);
+            if (changed) {
+                stats.parsedFileCount += 1;
+                stats.hadFileMutations = true;
+            }
+            processedCount += 1;
+        }
+        if (this.pendingHydrationCursor >= this.pendingHydrationFiles.length) {
+            this.pendingHydrationFiles = [];
+            this.pendingHydrationCursor = 0;
         }
     }
     inferSourceMetadata(filePath) {
@@ -1034,6 +1295,7 @@ export class TraceIndex extends EventEmitter {
             const parsed = await this.parserRegistry.parseFile(file);
             const summary = summarize(file, parsed.agent, parsed.parser, parsed.sessionId, parsed.events, parsed.parseError, this.config, refreshNowMs);
             const redactedEvents = redactEvents(parsed.events, this.config.redaction);
+            const activityArtifacts = buildSessionActivityArtifacts(redactedEvents);
             const previousCount = current?.summary.eventCount ?? 0;
             this.entries.set(file.id, {
                 file,
@@ -1041,7 +1303,8 @@ export class TraceIndex extends EventEmitter {
                 residentEvents: redactedEvents,
                 cachedFullEvents: redactedEvents,
                 cachedRawEvents: parsed.events,
-                activityArtifacts: null,
+                activityArtifacts,
+                usageArtifacts: buildSessionUsageArtifacts(parsed.events, parsed.agent, this.config),
                 pinnedMaterializedAtMs: current?.pinnedMaterializedAtMs ?? 0,
             });
             this.cursorById.set(file.id, {
@@ -1071,6 +1334,7 @@ export class TraceIndex extends EventEmitter {
                 cachedFullEvents: [],
                 cachedRawEvents: [],
                 activityArtifacts: null,
+                usageArtifacts: null,
                 pinnedMaterializedAtMs: 0,
             });
             this.cursorById.set(file.id, {
@@ -1141,6 +1405,7 @@ export class TraceIndex extends EventEmitter {
         const rebasedRawEvents = rebaseEvents(parsed.events);
         const mergedEvents = current.cachedFullEvents.concat(rebasedEvents);
         const mergedRawEvents = current.cachedRawEvents.concat(rebasedRawEvents);
+        const activityArtifacts = buildSessionActivityArtifacts(mergedEvents);
         const mergedSummary = summarize(file, current.summary.agent, current.summary.parser, current.summary.sessionId || parsed.sessionId, mergedRawEvents, "", this.config, refreshNowMs);
         const summary = {
             ...mergedSummary,
@@ -1154,7 +1419,8 @@ export class TraceIndex extends EventEmitter {
             residentEvents: mergedEvents,
             cachedFullEvents: mergedEvents,
             cachedRawEvents: mergedRawEvents,
-            activityArtifacts: null,
+            activityArtifacts,
+            usageArtifacts: buildSessionUsageArtifacts(mergedRawEvents, current.summary.agent, this.config),
             pinnedMaterializedAtMs: current.pinnedMaterializedAtMs,
         });
         this.cursorById.set(file.id, {
@@ -1214,6 +1480,9 @@ export class TraceIndex extends EventEmitter {
                 entry.cachedFullEvents = null;
                 entry.cachedRawEvents = null;
             }
+            if (this.config.retention.strategy !== "full_memory" && tier !== "hot" && entry.activityArtifacts) {
+                entry.activityArtifacts = compactActivityArtifacts(entry.activityArtifacts);
+            }
             if (this.config.retention.strategy === "full_memory" && !entry.cachedFullEvents) {
                 entry.cachedFullEvents = sourceEvents;
             }
@@ -1299,7 +1568,8 @@ export class TraceIndex extends EventEmitter {
         const refreshedSummary = summarize(entry.file, parsed.agent, parsed.parser, parsed.sessionId, parsed.events, parsed.parseError, this.config, nowMs());
         entry.cachedFullEvents = redactedEvents;
         entry.cachedRawEvents = parsed.events;
-        entry.activityArtifacts = null;
+        entry.activityArtifacts = buildSessionActivityArtifacts(redactedEvents);
+        entry.usageArtifacts = buildSessionUsageArtifacts(parsed.events, parsed.agent, this.config);
         entry.pinnedMaterializedAtMs = nowMs();
         entry.summary = {
             ...refreshedSummary,
@@ -1327,9 +1597,40 @@ export class TraceIndex extends EventEmitter {
         if (found.activityArtifacts && found.activityArtifacts.eventCount === found.summary.eventCount) {
             return found.activityArtifacts;
         }
-        const events = this.hydrateEventsForEntry(found);
-        const artifacts = buildSessionActivityArtifacts(events);
-        found.activityArtifacts = artifacts;
+        const artifacts = (() => {
+            if (found.cachedFullEvents && found.cachedFullEvents.length >= found.summary.eventCount) {
+                return buildSessionActivityArtifacts(found.cachedFullEvents);
+            }
+            if (found.residentEvents.length >= found.summary.eventCount) {
+                return buildSessionActivityArtifacts(found.residentEvents);
+            }
+            const parsed = this.parserRegistry.parseFileSync(found.file, found.summary.parser);
+            const redactedEvents = redactEvents(parsed.events, this.config.redaction);
+            return buildSessionActivityArtifacts(redactedEvents);
+        })();
+        if (this.config.retention.strategy === "full_memory" || found.summary.residentTier === "hot") {
+            found.activityArtifacts = artifacts;
+        }
+        return artifacts;
+    }
+    getSessionUsageArtifacts(id) {
+        const found = this.entries.get(id);
+        if (!found) {
+            throw new Error(`unknown trace id: ${id}`);
+        }
+        if (found.usageArtifacts && found.usageArtifacts.eventCount === found.summary.eventCount) {
+            return found.usageArtifacts;
+        }
+        const artifacts = (() => {
+            if (found.cachedRawEvents && found.cachedRawEvents.length >= found.summary.eventCount) {
+                return buildSessionUsageArtifacts(found.cachedRawEvents, found.summary.agent, this.config);
+            }
+            const parsed = this.parserRegistry.parseFileSync(found.file, found.summary.parser);
+            return buildSessionUsageArtifacts(parsed.events, parsed.agent, this.config);
+        })();
+        if (this.config.retention.strategy === "full_memory" || found.summary.residentTier === "hot") {
+            found.usageArtifacts = artifacts;
+        }
         return artifacts;
     }
     getTracePage(id, options = {}) {