codesesh 1.0.0 → 1.0.2

package/dist/scan-refresh-worker.js

@@ -1,22 +1,86 @@
 #!/usr/bin/env node
 import {
   appLogger
-} from "./chunk-TIHF4T3K.js";
+} from "./chunk-PROMURPM.js";
 import {
   FileSystemSessionSource,
+  SMART_TAG_CLASSIFIER_REVISION,
   attachMissingProjectIdentities,
   buildAgentCacheMeta,
   computeSessionDiff,
   createRegisteredAgents,
-  diffSessionSources,
   ensureSessionTagsSync,
   sessionSignature,
-  sortSessions
-} from "./chunk-BEF6LLRG.js";
+  sortSessions,
+  synchronizePricingGeneration,
+  synchronizeSessionSources
+} from "./chunk-VAUC2W7I.js";
 
 // src/scan-refresh-worker.ts
 import { parentPort, workerData } from "worker_threads";
 import { isDeepStrictEqual } from "util";
+
+// src/monotonic-value-sampler.ts
+var MonotonicValueSampler = class {
+  constructor(intervalMs, emit, now = () => performance.now()) {
+    this.intervalMs = intervalMs;
+    this.emit = emit;
+    this.now = now;
+  }
+  intervalMs;
+  emit;
+  now;
+  phase = null;
+  lastEmittedAt = -Infinity;
+  pending;
+  hasPending = false;
+  push(value, phase) {
+    if (this.phase !== phase) {
+      this.flush();
+      this.phase = phase;
+      this.emitNow(value);
+      return;
+    }
+    if (this.now() - this.lastEmittedAt >= this.intervalMs) {
+      this.pending = void 0;
+      this.hasPending = false;
+      this.emitNow(value);
+      return;
+    }
+    this.pending = value;
+    this.hasPending = true;
+  }
+  flush() {
+    if (!this.hasPending) return;
+    const pending = this.pending;
+    this.pending = void 0;
+    this.hasPending = false;
+    this.emitNow(pending);
+  }
+  cancel() {
+    this.phase = null;
+    this.lastEmittedAt = -Infinity;
+    this.pending = void 0;
+    this.hasPending = false;
+  }
+  emitNow(value) {
+    this.lastEmittedAt = this.now();
+    this.emit(value);
+  }
+};
+
+// src/scan-refresh-operation.ts
+function synchronizesSessionSources(operation) {
+  return operation.kind === "source-refresh" || operation.kind === "source-backfill";
+}
+function isBackfillOperation(operation) {
+  return operation.kind === "full-backfill" || operation.kind === "source-backfill";
+}
+function usesDurableCheckpoints(operation) {
+  return "checkpoint" in operation && operation.checkpoint === "durable";
+}
+
+// src/scan-refresh-worker.ts
 function computeCacheMetaDiff(previous, next) {
   const changes = {};
   const removedIds = [];
@@ -30,7 +94,7 @@ function computeCacheMetaDiff(previous, next) {
 }
 function hasStaleSmartTags(session) {
   const sourceUpdatedAt = session.time_updated ?? session.time_created;
-  return !Array.isArray(session.smart_tags) || session.smart_tags_source_updated_at !== sourceUpdatedAt;
+  return !Array.isArray(session.smart_tags) || session.smart_tags_source_updated_at !== sourceUpdatedAt || session.smart_tags_classifier_revision !== SMART_TAG_CLASSIFIER_REVISION;
 }
 function selectBackfillSessions(sessions, changedIds, cursor) {
   const orderedSessions = sortSessions(sessions);
@@ -47,44 +111,23 @@ function selectBackfillSessions(sessions, changedIds, cursor) {
   }
   return { orderedSessions, finalizeSessionIds, cursorIndex };
 }
-function syncAgentSources(agent, cachedSessions, cachedMeta, windowOptions, onProgress) {
-  const sessionMap = new Map(cachedSessions.map((session) => [session.id, session]));
-  const sourceRefs = agent.listSessionSources(windowOptions);
-  const sourceById = new Map(sourceRefs.map((source) => [source.sessionId, source]));
-  const { changedIds, removedIds } = diffSessionSources(
-    sourceRefs,
-    cachedSessions,
-    cachedMeta,
-    windowOptions
-  );
-  const rescanIds = agent.expandChangedSessionIds?.(
-    [.../* @__PURE__ */ new Set([...changedIds, ...removedIds])],
-    sourceRefs
-  ) ?? [.../* @__PURE__ */ new Set([...changedIds, ...removedIds])];
-  const isWindowed = windowOptions?.from != null || windowOptions?.to != null;
-  const finalizeSessionIds = isWindowed ? [...rescanIds] : sourceRefs.map((source) => source.sessionId);
-  rescanIds.forEach((sessionId, index) => {
-    const source = sourceById.get(sessionId);
-    if (!source) return;
-    const next = agent.scanSessionSource(source.sourcePath);
-    if (next) {
-      sessionMap.set(next.id, next);
-    } else {
-      sessionMap.delete(sessionId);
-    }
-    onProgress?.({ total: rescanIds.length, processed: index + 1, sessions: sessionMap.size });
+function inheritSmartTags(sessions, previousSessions) {
+  const previousById = new Map(previousSessions.map((session) => [session.id, session]));
+  return sessions.map((session) => {
+    if (Array.isArray(session.smart_tags)) return session;
+    const previous = previousById.get(session.id);
+    if (!previous || !Array.isArray(previous.smart_tags)) return session;
+    return {
+      ...session,
+      smart_tags: previous.smart_tags,
+      smart_tags_source_updated_at: previous.smart_tags_source_updated_at,
+      smart_tags_classifier_revision: previous.smart_tags_classifier_revision
+    };
   });
-  for (const sessionId of removedIds) sessionMap.delete(sessionId);
-  return {
-    sessions: [...sessionMap.values()],
-    changedIds: rescanIds,
-    finalizeSessionIds,
-    sourceCount: sourceRefs.length,
-    removedCount: removedIds.length
-  };
 }
 var TAG_SETTLE_MS = 6e4;
 var TAG_CHECKPOINT_SIZE = 32;
+var PROGRESS_INTERVAL_MS = 100;
 function createSessionFinalizationTiming() {
   return {
     batches: 0,
@@ -155,75 +198,150 @@ function finalizeSessions(agent, sessions, onProgress, onCheckpoint, finalizeSes
   }
   return ordered.map((session) => taggedById.get(session.id) ?? session);
 }
-async function run(data) {
-  const startedAt = performance.now();
-  const agent = createRegisteredAgents().find((item) => item.name === data.agentName);
-  if (!agent) {
-    throw new Error(`Unknown agent: ${data.agentName}`);
+var workerBaseline = null;
+function baselineFor(data) {
+  const generation = data.generation ?? 0;
+  const hasSessions = Array.isArray(data.previousSessions);
+  const hasMeta = data.meta != null;
+  if (hasSessions !== hasMeta)
+    throw new Error("Worker baseline requires sessions and meta together");
+  if (hasSessions && hasMeta) {
+    if (workerBaseline) throw new Error("Scan refresh worker baseline is already initialized");
+    const agent = createRegisteredAgents().find((item) => item.name === data.agentName);
+    if (!agent) throw new Error(`Unknown agent: ${data.agentName}`);
+    workerBaseline = {
+      agentName: data.agentName,
+      agent,
+      generation,
+      sessions: data.previousSessions,
+      meta: data.meta,
+      staged: null
+    };
+  }
+  if (!workerBaseline) throw new Error("Scan refresh worker baseline is not initialized");
+  if (workerBaseline.agentName !== data.agentName) {
+    throw new Error(`Worker Agent mismatch: expected ${workerBaseline.agentName}`);
+  }
+  if (workerBaseline.generation !== generation) {
+    throw new Error(
+      `Worker generation mismatch: expected ${workerBaseline.generation}, received ${generation}`
+    );
   }
+  if (workerBaseline.staged) {
+    throw new Error(`Worker result ${workerBaseline.staged.requestId} is awaiting commit`);
+  }
+  workerBaseline.agent.setSessionMetaMap(new Map(Object.entries(workerBaseline.meta)));
+  return workerBaseline;
+}
+async function run(data, progressEmitter) {
+  const startedAt = performance.now();
+  const baseline = baselineFor(data);
+  const { agent } = baseline;
+  const previousSessions = baseline.sessions;
+  const previousMeta = baseline.meta;
+  const operation = data.operation;
+  const sourceSynchronization = synchronizesSessionSources(operation);
+  const backfill = isBackfillOperation(operation);
+  const durableCheckpoints = usesDurableCheckpoints(operation);
+  const backfillCursor = backfill ? operation.cursor : void 0;
   appLogger.debug("scan.refresh_worker.started", {
     agent: data.agentName,
-    source_sync: data.sourceSync ?? false,
-    backfill: data.backfill ?? false,
-    backfill_cursor: data.backfillCursor ?? void 0,
-    changed_ids: data.changedIds?.length ?? 0,
-    previous_sessions: data.previousSessions.length
+    operation: operation.kind,
+    backfill_cursor: backfillCursor ?? void 0,
+    changed_ids: operation.kind === "incremental-scan" ? operation.changedIds.length : 0,
+    previous_sessions: previousSessions.length
   });
   const reportProgress = (progress) => {
-    parentPort?.postMessage({
-      type: "progress",
-      requestId: data.requestId,
-      progress
-    });
+    progressEmitter.push(progress, progress.phase ?? "scanning");
   };
-  agent.setSessionMetaMap(new Map(Object.entries(data.meta)));
   const isAvailable = agent.isAvailable();
   let sessions;
   let changedIds;
+  let sourceFailures = [];
+  let explicitRemovedSessionIds = [];
   let finalizeSessionIds;
   let backfillOrder;
   let backfillCursorIndex = -1;
-  let sourceSyncDetails;
+  let sourceSynchronizationDetails;
   if (!isAvailable) {
     sessions = [];
-  } else if (data.sourceSync && agent instanceof FileSystemSessionSource) {
-    const result = syncAgentSources(
+  } else if (operation.kind === "recompute-derived") {
+    sessions = previousSessions;
+  } else if (sourceSynchronization) {
+    if (!(agent instanceof FileSystemSessionSource)) {
+      throw new Error(`Agent ${agent.name} does not support Session Source synchronization`);
+    }
+    const result = synchronizeSessionSources(
+      agent,
+      { sessions: previousSessions, meta: previousMeta },
+      {
+        kind: "refresh",
+        scanOptions: { ...data.scanOptions, onProgress: reportProgress }
+      }
+    );
+    sessions = result.sessions;
+    changedIds = result.changedSessionIds;
+    finalizeSessionIds = new Set(result.finalizeSessionIds);
+    sourceSynchronizationDetails = {
+      sourceCount: result.sourceCount,
+      removedCount: result.removedSourceCount
+    };
+    sourceFailures = result.sourceFailures;
+    explicitRemovedSessionIds = result.explicitRemovedSessionIds;
+  } else if (operation.kind === "incremental-scan") {
+    changedIds = operation.changedIds;
+    sessions = inheritSmartTags(
+      await Promise.resolve(
+        agent.incrementalScan(previousSessions, operation.changedIds, void 0, {
+          ...data.scanOptions,
+          onProgress: reportProgress
+        })
+      ),
+      previousSessions
+    );
+  } else if (agent instanceof FileSystemSessionSource) {
+    const result = synchronizeSessionSources(
       agent,
-      data.previousSessions,
-      data.meta,
-      data.scanOptions,
-      reportProgress
+      { sessions: previousSessions, meta: previousMeta },
+      {
+        kind: "reload",
+        scanOptions: { ...data.scanOptions, onProgress: reportProgress }
+      }
     );
     sessions = result.sessions;
-    changedIds = result.changedIds;
     finalizeSessionIds = new Set(result.finalizeSessionIds);
-    sourceSyncDetails = result;
-  } else if (data.changedIds) {
-    sessions = await Promise.resolve(agent.incrementalScan(data.previousSessions, data.changedIds));
+    sourceFailures = result.sourceFailures;
+    explicitRemovedSessionIds = result.explicitRemovedSessionIds;
   } else {
-    sessions = await Promise.resolve(
-      agent.scan({
-        ...data.scanOptions,
-        onProgress: reportProgress
-      })
+    sessions = inheritSmartTags(
+      await Promise.resolve(
+        agent.scan({
+          ...data.scanOptions,
+          onProgress: reportProgress
+        })
+      ),
+      previousSessions
     );
   }
   sessions = attachMissingProjectIdentities(sessions);
-  if (data.checkpoint) {
+  const completeness = data.scanOptions.from == null && data.scanOptions.to == null && sourceFailures.length === 0 ? "complete" : "partial";
+  if (durableCheckpoints) {
     const ordered = sortSessions(sessions);
     parentPort?.postMessage({
       type: "checkpoint",
       requestId: data.requestId,
+      generation: baseline.generation,
       checkpoint: {
         stage: "scanned",
         sessions: ordered,
-        meta: buildAgentCacheMeta(agent, new Set(ordered.map((session) => session.id)))
+        meta: buildAgentCacheMeta(agent, new Set(ordered.map((session) => session.id))),
+        completeness
       }
     });
     sessions = ordered;
   }
-  if (data.backfill) {
-    const selection = selectBackfillSessions(sessions, changedIds ?? [], data.backfillCursor);
+  if (backfill) {
+    const selection = selectBackfillSessions(sessions, changedIds ?? [], backfillCursor);
     sessions = selection.orderedSessions;
     finalizeSessionIds = selection.finalizeSessionIds;
     backfillOrder = selection.orderedSessions;
@@ -232,13 +350,13 @@ async function run(data) {
   const scanDuration = performance.now() - startedAt;
   appLogger.debug("scan.refresh_worker.scanned", {
     agent: data.agentName,
-    source_sync: data.sourceSync ?? false,
-    backfill: data.backfill ?? false,
-    backfill_cursor: data.backfillCursor ?? void 0,
+    operation: operation.kind,
+    backfill_cursor: backfillCursor ?? void 0,
     sessions: sessions.length,
     changed_ids: changedIds?.length ?? 0,
-    source_count: sourceSyncDetails?.sourceCount,
-    removed_count: sourceSyncDetails?.removedCount,
+    source_count: sourceSynchronizationDetails?.sourceCount,
+    removed_count: sourceSynchronizationDetails?.removedCount,
+    failed_sources: sourceFailures.length,
     duration_ms: Math.round(scanDuration)
   });
   const finalizeStartedAt = performance.now();
@@ -248,7 +366,7 @@ async function run(data) {
     agent,
     sessions,
     reportProgress,
-    (checkpoint) => {
+    durableCheckpoints ? (checkpoint) => {
       let nextCheckpoint = checkpoint;
       if (checkpoint.stage === "finalizing" && backfillOrder && backfillPositionById) {
         let nextCursorIndex = backfillCursorIndex;
@@ -267,9 +385,10 @@ async function run(data) {
       parentPort?.postMessage({
         type: "checkpoint",
         requestId: data.requestId,
+        generation: baseline.generation,
         checkpoint: nextCheckpoint
       });
-    },
+    } : void 0,
     finalizeSessionIds,
     (batchTiming) => {
       finalizationTiming.batches += 1;
@@ -295,7 +414,7 @@ async function run(data) {
   );
   appLogger.debug("scan.refresh_worker.finalized", {
     agent: data.agentName,
-    backfill: data.backfill ?? false,
+    operation: operation.kind,
     backfill_cursor: backfillOrder?.[backfillCursorIndex]?.id,
     sessions: sessions.length,
     finalized_sessions: finalizationTiming.sessions,
@@ -318,46 +437,106 @@ async function run(data) {
     total_duration_ms: Math.round(performance.now() - startedAt)
   });
   const nextMeta = buildAgentCacheMeta(agent, new Set(sessions.map((session) => session.id)));
-  const metaDiff = computeCacheMetaDiff(data.meta, nextMeta);
+  const metaDiff = computeCacheMetaDiff(previousMeta, nextMeta);
   const diff = computeSessionDiff(
-    data.previousSessions,
+    previousSessions,
     sessions,
     [...changedIds ?? [], ...Object.keys(metaDiff.changes), ...metaDiff.removedIds],
     sessionSignature
   );
+  baseline.staged = {
+    requestId: data.requestId,
+    generation: baseline.generation,
+    sessions,
+    meta: nextMeta
+  };
+  progressEmitter.flush();
   parentPort?.postMessage({
     type: "done",
     requestId: data.requestId,
+    generation: baseline.generation,
     changes: diff.changes,
     removedSessionIds: diff.removedSessionIds,
     meta: metaDiff.changes,
     removedMetaIds: metaDiff.removedIds,
+    sourceFailures,
+    completeness,
+    explicitRemovedSessionIds,
     durationMs: performance.now() - startedAt
   });
 }
 async function handleRequest(data) {
   const startedAt = performance.now();
+  const progressEmitter = new MonotonicValueSampler(
+    PROGRESS_INTERVAL_MS,
+    (progress) => {
+      parentPort?.postMessage({
+        type: "progress",
+        requestId: data.requestId,
+        generation: data.generation ?? 0,
+        progress
+      });
+    }
+  );
   try {
-    await run(data);
+    synchronizePricingGeneration(data.pricingGenerationId);
+    await run(data, progressEmitter);
   } catch (error) {
-    parentPort?.postMessage({
-      type: "error",
-      requestId: data.requestId,
-      error: error instanceof Error ? error.message : String(error),
-      durationMs: performance.now() - startedAt
-    });
+    progressEmitter.flush();
+    postRequestError(data, error, startedAt);
+  } finally {
+    progressEmitter.cancel();
   }
 }
 var requestTail = Promise.resolve();
+function commitBaseline(data) {
+  if (!workerBaseline) throw new Error("Cannot commit an uninitialized worker baseline");
+  const staged = workerBaseline.staged;
+  if (!staged || staged.requestId !== data.requestId) {
+    throw new Error(`Worker result ${data.requestId} is not awaiting commit`);
+  }
+  if (workerBaseline.generation !== data.generation || staged.generation !== data.generation) {
+    throw new Error(
+      `Worker commit generation mismatch: expected ${workerBaseline.generation}, received ${data.generation}`
+    );
+  }
+  workerBaseline.sessions = staged.sessions;
+  workerBaseline.meta = staged.meta;
+  workerBaseline.generation += 1;
+  workerBaseline.staged = null;
+}
+function postRequestError(data, error, startedAt) {
+  parentPort?.postMessage({
+    type: "error",
+    requestId: data.requestId,
+    generation: data.generation,
+    error: error instanceof Error ? error.message : String(error),
+    durationMs: performance.now() - startedAt
+  });
+}
+function handleCommit(data) {
+  const startedAt = performance.now();
+  try {
+    commitBaseline(data);
+  } catch (error) {
+    postRequestError(data, error, startedAt);
+  }
+}
 function enqueueRequest(data) {
-  requestTail = requestTail.then(() => handleRequest(data));
+  requestTail = requestTail.then(() => data.type === "commit" ? handleCommit(data) : handleRequest(data)).catch((error) => {
+    appLogger.error("scan.refresh_worker.request_error", {
+      request_id: data.requestId,
+      request_type: data.type,
+      error
+    });
+  });
 }
 var initialRequest = workerData;
-if (initialRequest?.type === "run" && typeof initialRequest.requestId === "number" && typeof initialRequest.agentName === "string" && Array.isArray(initialRequest.previousSessions)) {
+if (initialRequest?.type === "run" && typeof initialRequest.requestId === "number" && typeof initialRequest.agentName === "string" && typeof initialRequest.pricingGenerationId === "number" && initialRequest.operation != null && Array.isArray(initialRequest.previousSessions) && initialRequest.meta != null) {
   enqueueRequest(initialRequest);
 }
 parentPort?.on("message", (message) => {
-  if (message.type === "run") enqueueRequest(message);
+  enqueueRequest(message);
 });
 export {
   finalizeSessions