modelstat 0.4.1 → 0.5.0

package/dist/cli.mjs

@@ -34544,7 +34544,8 @@ function load() {
       apiUrl: obj.apiUrl ?? DEFAULTS.apiUrl,
       cursor: obj.cursor ?? {},
       segmentsSent: obj.segmentsSent ?? 0,
-      processingVersion: obj.processingVersion ?? null
+      processingVersion: obj.processingVersion ?? null,
+      reconcileCache: obj.reconcileCache ?? {}
     };
   } catch {
     cache2 = { ...DEFAULTS, cursor: {} };
@@ -34566,7 +34567,8 @@ var init_runtime_state = __esm({
       apiUrl: "",
       cursor: {},
       segmentsSent: 0,
-      processingVersion: null
+      processingVersion: null,
+      reconcileCache: {}
     };
     cache2 = null;
     runtimeState = {
@@ -34593,6 +34595,29 @@ var init_runtime_state = __esm({
         s.cursor = {};
         persist(s);
       },
+      /** Drop ONE file's cursor so the next scan re-reads it — the precise lever the
+       * self-healing reconcile pulls for the files of sessions the server is missing. */
+      clearCursor(path) {
+        const s = load();
+        if (path in s.cursor) {
+          delete s.cursor[path];
+          persist(s);
+        }
+      },
+      /** Drop cursors for files no longer present so the map tracks the CURRENT file
+       * set, not every file ever seen. Returns how many were pruned. */
+      pruneCursors(present) {
+        const s = load();
+        let removed = 0;
+        for (const p of Object.keys(s.cursor)) {
+          if (!present.has(p)) {
+            delete s.cursor[p];
+            removed += 1;
+          }
+        }
+        if (removed) persist(s);
+        return removed;
+      },
       getSegmentsSent() {
         return load().segmentsSent;
       },
@@ -34610,6 +34635,15 @@ var init_runtime_state = __esm({
         s.processingVersion = v;
         persist(s);
       },
+      /** Self-healing reconcile cache (see {@link RuntimeState.reconcileCache}). */
+      getReconcileCache() {
+        return load().reconcileCache;
+      },
+      setReconcileCache(c) {
+        const s = load();
+        s.reconcileCache = c;
+        persist(s);
+      },
       /** Test-only: drop the in-memory cache so the next read hits disk. */
       _resetCacheForTests() {
         cache2 = null;
@@ -34884,6 +34918,25 @@ async function uploadBatch(batch) {
     batch_id: result2.response.batch_id
   };
 }
+async function backfillGet(query) {
+  const bearer = state.bearer;
+  if (!bearer) return null;
+  const res = await (0, import_undici.request)(`${state.apiUrl}/v1/backfill/digests${query}`, {
+    method: "GET",
+    headers: { authorization: `Bearer ${bearer}` }
+  });
+  if (res.statusCode >= 300) {
+    await res.body.dump();
+    return null;
+  }
+  return await res.body.json();
+}
+function fetchBackfillDays() {
+  return backfillGet("");
+}
+function fetchBackfillDaySessions(day) {
+  return backfillGet(`?day=${encodeURIComponent(day)}`);
+}
 var import_undici, DeviceMeUnauthorized, _ingest;
 var init_api = __esm({
   "src/api.ts"() {
@@ -36836,9 +36889,7 @@ import { join as join7 } from "path";
 function withNonNullTokens(e) {
   return e.tokens ? e : { ...e, tokens: { ...ZERO_TOKENS } };
 }
-async function scanAll(cb = {}) {
-  const deviceId = state.deviceId;
-  if (!deviceId) throw new Error("daemon not enrolled \u2014 run `register` first");
+async function discoverJobs(deviceId) {
   const jobs = [];
   try {
     const base = join7(homedir5(), ".claude/projects");
@@ -36853,8 +36904,8 @@ async function scanAll(cb = {}) {
         const full = join7(dir, f);
         jobs.push({
           path: full,
-          parse: async (sink2) => {
-            const r = await parseClaudeCodeJsonl({ deviceId, sourceFile: full, onEvents: sink2 });
+          parse: async (sink) => {
+            const r = await parseClaudeCodeJsonl({ deviceId, sourceFile: full, onEvents: sink });
             return { toolCalls: r.toolCalls ?? [], scriptContexts: r.scriptContexts ?? [] };
           }
         });
@@ -36877,8 +36928,8 @@ async function scanAll(cb = {}) {
             const full = join7(base, y, m, d, f);
             jobs.push({
               path: full,
-              parse: async (sink2) => {
-                const r = await parseCodexRollout({ deviceId, sourceFile: full, onEvents: sink2 });
+              parse: async (sink) => {
+                const r = await parseCodexRollout({ deviceId, sourceFile: full, onEvents: sink });
                 return { toolCalls: r.toolCalls ?? [], scriptContexts: r.scriptContexts ?? [] };
               }
             });
@@ -36889,6 +36940,12 @@ async function scanAll(cb = {}) {
   } catch (e) {
     console.warn("codex scan skipped:", e.message);
   }
+  return jobs;
+}
+async function scanAll(cb = {}) {
+  const deviceId = state.deviceId;
+  if (!deviceId) throw new Error("daemon not enrolled \u2014 run `register` first");
+  const jobs = await discoverJobs(deviceId);
   const ordered = (await Promise.all(
     jobs.map(async (j) => ({
       job: j,
@@ -37029,7 +37086,7 @@ var init_scan = __esm({
     init_api();
     init_config2();
     init_pipeline2();
-    DAEMON_VERSION = true ? "daemon-0.4.1" : "daemon-dev";
+    DAEMON_VERSION = true ? "daemon-0.5.0" : "daemon-dev";
     BATCH_MAX_EVENTS = INGEST_BATCH_MAX_EVENTS;
     BATCH_MAX_TOOL_CALLS = 2e4;
     BATCH_BUFFER_HARD_CAP = BATCH_MAX_EVENTS * 2;
@@ -37182,6 +37239,117 @@ var init_lock = __esm({
   }
 });
 
+// src/reconcile.ts
+import { stat as stat3 } from "fs/promises";
+function utcDay(ts) {
+  const d = new Date(ts);
+  return Number.isNaN(d.getTime()) ? "" : d.toISOString().slice(0, 10);
+}
+async function reconcileBackfill(requestScan2) {
+  const deviceId = state.deviceId;
+  if (!deviceId) return null;
+  const jobs = await discoverJobs(deviceId);
+  const present = new Set(jobs.map((j) => j.path));
+  const cache3 = runtimeState.getReconcileCache();
+  let filesParsed = 0;
+  for (const job of jobs) {
+    const mtime = (await stat3(job.path).catch(() => null))?.mtimeMs ?? 0;
+    const hit = cache3[job.path];
+    if (hit && hit.mtime === mtime) continue;
+    const perDaySession = {};
+    try {
+      await job.parse(async (chunk) => {
+        for (const e of chunk) {
+          const bySession = perDaySession[utcDay(e.ts)] ??= {};
+          bySession[e.session_id] = (bySession[e.session_id] ?? 0) + 1;
+        }
+      });
+      cache3[job.path] = { mtime, perDaySession };
+      filesParsed += 1;
+    } catch (e) {
+      console.warn(`  ! reconcile parse failed for ${job.path}:`, e.message);
+    }
+  }
+  for (const p of Object.keys(cache3)) if (!present.has(p)) delete cache3[p];
+  runtimeState.setReconcileCache(cache3);
+  runtimeState.pruneCursors(present);
+  const localDay = /* @__PURE__ */ new Map();
+  const localDaySession = /* @__PURE__ */ new Map();
+  const filesOf = /* @__PURE__ */ new Map();
+  let localEvents = 0;
+  for (const [path, entry] of Object.entries(cache3)) {
+    for (const [day, sessions] of Object.entries(entry.perDaySession)) {
+      let ds = localDaySession.get(day);
+      if (!ds) {
+        ds = /* @__PURE__ */ new Map();
+        localDaySession.set(day, ds);
+      }
+      for (const [sid, n] of Object.entries(sessions)) {
+        localEvents += n;
+        localDay.set(day, (localDay.get(day) ?? 0) + n);
+        ds.set(sid, (ds.get(sid) ?? 0) + n);
+        const key = `${day}\0${sid}`;
+        let fs2 = filesOf.get(key);
+        if (!fs2) {
+          fs2 = /* @__PURE__ */ new Set();
+          filesOf.set(key, fs2);
+        }
+        fs2.add(path);
+      }
+    }
+  }
+  const serverDays = await fetchBackfillDays();
+  if (!serverDays) return null;
+  const base = {
+    inSync: true,
+    localEvents,
+    serverEvents: serverDays.total_events,
+    filesParsed,
+    daysChecked: 0,
+    sessionsShort: 0,
+    filesInvalidated: 0
+  };
+  if (localEvents <= serverDays.total_events) return base;
+  const serverDayMap = new Map(serverDays.days.map((d) => [d.day, d.events]));
+  const filesToReship = /* @__PURE__ */ new Set();
+  let sessionsShort = 0;
+  let daysChecked = 0;
+  for (const [day, localCount] of localDay) {
+    if (localCount <= (serverDayMap.get(day) ?? 0)) continue;
+    daysChecked += 1;
+    const serverSessions = await fetchBackfillDaySessions(day);
+    const have = new Map((serverSessions?.sessions ?? []).map((s) => [s.session_id, s.events]));
+    for (const [sid, n] of localDaySession.get(day) ?? []) {
+      if (n > (have.get(sid) ?? 0)) {
+        sessionsShort += 1;
+        for (const f of filesOf.get(`${day}\0${sid}`) ?? []) filesToReship.add(f);
+      }
+    }
+  }
+  if (filesToReship.size === 0) return { ...base, daysChecked };
+  for (const f of filesToReship) runtimeState.clearCursor(f);
+  console.log(
+    `[modelstat] self-heal: server short ${sessionsShort} session(s) across ${daysChecked} day(s); re-shipping ${filesToReship.size} file(s) from local logs`
+  );
+  await requestScan2("self-heal");
+  return {
+    ...base,
+    inSync: false,
+    daysChecked,
+    sessionsShort,
+    filesInvalidated: filesToReship.size
+  };
+}
+var init_reconcile = __esm({
+  "src/reconcile.ts"() {
+    "use strict";
+    init_api();
+    init_config2();
+    init_runtime_state();
+    init_scan();
+  }
+});
+
 // src/single-flight.ts
 function createCoalescingRunner(task) {
   let running = false;
@@ -37491,7 +37659,7 @@ var init_policies2 = __esm({
 });
 
 // ../../node_modules/.pnpm/readdirp@4.1.2/node_modules/readdirp/esm/index.js
-import { stat as stat3, lstat, readdir as readdir2, realpath } from "fs/promises";
+import { stat as stat4, lstat, readdir as readdir2, realpath } from "fs/promises";
 import { Readable } from "stream";
 import { resolve as presolve, relative as prelative, join as pjoin, sep as psep } from "path";
 function readdirp(root, options = {}) {
@@ -37578,7 +37746,7 @@ var init_esm = __esm({
         const { root, type } = opts;
         this._fileFilter = normalizeFilter(opts.fileFilter);
         this._directoryFilter = normalizeFilter(opts.directoryFilter);
-        const statMethod = opts.lstat ? lstat : stat3;
+        const statMethod = opts.lstat ? lstat : stat4;
         if (wantBigintFsStats) {
           this._stat = (path) => statMethod(path, { bigint: true });
         } else {
@@ -37717,7 +37885,7 @@ var init_esm = __esm({
 
 // ../../node_modules/.pnpm/chokidar@4.0.3/node_modules/chokidar/esm/handler.js
 import { watchFile, unwatchFile, watch as fs_watch } from "fs";
-import { open, stat as stat4, lstat as lstat2, realpath as fsrealpath } from "fs/promises";
+import { open, stat as stat5, lstat as lstat2, realpath as fsrealpath } from "fs/promises";
 import * as sysPath from "path";
 import { type as osType } from "os";
 function createFsWatchInstance(path, options, listener, errHandler, emitRaw) {
@@ -37765,7 +37933,7 @@ var init_handler = __esm({
     };
     EV = EVENTS;
     THROTTLE_MODE_WATCH = "watch";
-    statMethods = { lstat: lstat2, stat: stat4 };
+    statMethods = { lstat: lstat2, stat: stat5 };
     KEY_LISTENERS = "listeners";
     KEY_ERR = "errHandlers";
     KEY_RAW = "rawEmitters";
@@ -38234,7 +38402,7 @@ var init_handler = __esm({
             return;
           if (!newStats || newStats.mtimeMs === 0) {
             try {
-              const newStats2 = await stat4(file);
+              const newStats2 = await stat5(file);
               if (this.fsw.closed)
                 return;
               const at = newStats2.atimeMs;
@@ -38486,7 +38654,7 @@ __export(esm_exports, {
   watch: () => watch
 });
 import { stat as statcb } from "fs";
-import { stat as stat5, readdir as readdir3 } from "fs/promises";
+import { stat as stat6, readdir as readdir3 } from "fs/promises";
 import { EventEmitter } from "events";
 import * as sysPath2 from "path";
 function arrify(item) {
@@ -38957,7 +39125,7 @@ var init_esm2 = __esm({
           const fullPath = opts.cwd ? sysPath2.join(opts.cwd, path) : path;
           let stats2;
           try {
-            stats2 = await stat5(fullPath);
+            stats2 = await stat6(fullPath);
           } catch (err) {
           }
           if (!stats2 || this.closed)
@@ -39084,8 +39252,8 @@ var init_esm2 = __esm({
         }
         return this._userIgnored(path, stats);
       }
-      _isntIgnored(path, stat6) {
-        return !this._isIgnored(path, stat6);
+      _isntIgnored(path, stat7) {
+        return !this._isIgnored(path, stat7);
       }
       /**
        * Provides a set of common helpers and properties relating to symlink handling.
@@ -39300,12 +39468,12 @@ async function sendHeartbeat() {
 async function rotateRunawayLogs() {
   const { homedir: homedir9 } = await import("os");
   const { join: join14 } = await import("path");
-  const { open: open2, stat: stat6, truncate, writeFile: writeFile2 } = await import("fs/promises");
+  const { open: open2, stat: stat7, truncate, writeFile: writeFile2 } = await import("fs/promises");
   const dir = join14(homedir9(), ".modelstat", "logs");
   for (const name of ["out.log", "err.log"]) {
     const p = join14(dir, name);
     try {
-      const st = await stat6(p);
+      const st = await stat7(p);
       if (st.size <= LOG_MAX_BYTES) continue;
       const keep = Math.min(LOG_TAIL_KEEP_BYTES, st.size);
       const fh = await open2(p, "r");
@@ -39510,6 +39678,13 @@ async function runDaemon(opts = {}) {
   backstop.unref();
   const discoveryTimer = setInterval(() => void runDiscovery(), DISCOVERY_INTERVAL_MS);
   discoveryTimer.unref();
+  const RECONCILE_INTERVAL_MS = 30 * 6e4;
+  const reconcileTimer = setInterval(
+    () => void reconcileBackfill(requestScan),
+    RECONCILE_INTERVAL_MS
+  );
+  reconcileTimer.unref();
+  setTimeout(() => void reconcileBackfill(requestScan), 6e4).unref();
   const shutdown = async () => {
     setPhase("offline", "Shutting down");
     await sendHeartbeat();
@@ -39537,9 +39712,10 @@ var init_daemon = __esm({
     init_config2();
     init_lock();
     init_machine_key();
+    init_reconcile();
     init_scan();
     init_single_flight();
-    DAEMON_VERSION2 = true ? "daemon-0.4.1" : "daemon-dev";
+    DAEMON_VERSION2 = true ? "daemon-0.5.0" : "daemon-dev";
     HEARTBEAT_INTERVAL_MS = 1e4;
     SCAN_INTERVAL_MS = 5 * 60 * 1e3;
     DISCOVERY_INTERVAL_MS = 6e4;
@@ -40140,7 +40316,7 @@ function tryOpenBrowser(url) {
     return false;
   }
 }
-var DAEMON_VERSION3 = true ? "daemon-0.4.1" : "daemon-dev";
+var DAEMON_VERSION3 = true ? "daemon-0.5.0" : "daemon-dev";
 function osFamily() {
   const p = platform5();
   if (p === "darwin") return "macos";