@jagilber-org/index-server 1.27.0 → 1.27.2

package/dist/services/indexContext.js

@@ -3,6 +3,9 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+exports._internal = void 0;
+exports._resetIndexContextProcessLatches = _resetIndexContextProcessLatches;
+exports._resetIndexContextStateForTests = _resetIndexContextStateForTests;
 exports.getInvariantRepairSummary = getInvariantRepairSummary;
 exports.clearUsageRateLimit = clearUsageRateLimit;
 exports.loadUsageSnapshot = loadUsageSnapshot;
@@ -95,6 +98,33 @@ function trackInvariantRepair(id, field, source) {
     if (invariantRepairLog.length > MAX_REPAIR_LOG)
         invariantRepairLog.shift();
 }
+// ── Process-scoped latches for noise + work suppression ──────────
+// Symptom that motivated these latches (observed live on dev port 8687,
+// 2026-05-01): every dashboard request triggered ensureLoaded() →
+// migrateJsonToSqlite() because jsonFiles.length > sqliteRowCount was
+// permanently true (a few JSON files failed loader validation), and every
+// /api/admin/stats request emitted hundreds of stack-traced WARN entries
+// from restoreFirstSeenInvariant for entries whose firstSeenTs was
+// genuinely unrecoverable. Both of those are infinite-cost loops once the
+// process is up. We dedupe both per-process here.
+const autoMigrationAttempted = new Set();
+const firstSeenExhaustedReported = new Set();
+/**
+ * Test-only hook — reset process-scoped latches between vitest specs.
+ * @internal Not part of the public API.
+ */
+function _resetIndexContextProcessLatches() {
+    autoMigrationAttempted.clear();
+    firstSeenExhaustedReported.clear();
+}
+/**
+ * Test-only hook — reset the module-scoped index state cache.
+ * @internal Not part of the public API.
+ */
+function _resetIndexContextStateForTests() {
+    state = null;
+    dirty = false;
+}
 /** Returns a summary of invariant repairs for health check visibility. */
 function getInvariantRepairSummary() {
     return { totalRepairs: invariantRepairLog.length, recentRepairs: invariantRepairLog.slice(-20) };
@@ -110,7 +140,7 @@ function restoreFirstSeenInvariant(e) {
         e.firstSeenTs = auth;
         (0, features_1.incrementCounter)('usage:firstSeenAuthorityRepair');
         trackInvariantRepair(e.id, 'firstSeenTs', 'authority');
-        (0, logger_js_1.logWarn)(`[invariant-repair] firstSeenTs restored from authority for ${e.id}`);
+        (0, logger_js_1.logDebug)(`[invariant-repair] firstSeenTs restored from authority for ${e.id}`);
         return;
     }
     const ep = ephemeralFirstSeen[e.id];
@@ -118,7 +148,7 @@ function restoreFirstSeenInvariant(e) {
         e.firstSeenTs = ep;
         (0, features_1.incrementCounter)('usage:firstSeenInvariantRepair');
         trackInvariantRepair(e.id, 'firstSeenTs', 'ephemeral');
-        (0, logger_js_1.logWarn)(`[invariant-repair] firstSeenTs restored from ephemeral cache for ${e.id}`);
+        (0, logger_js_1.logDebug)(`[invariant-repair] firstSeenTs restored from ephemeral cache for ${e.id}`);
         return;
     }
     const snap = lastGoodUsageSnapshot[e.id];
@@ -126,15 +156,40 @@ function restoreFirstSeenInvariant(e) {
         e.firstSeenTs = snap.firstSeenTs;
         (0, features_1.incrementCounter)('usage:firstSeenInvariantRepair');
         trackInvariantRepair(e.id, 'firstSeenTs', 'snapshot');
-        (0, logger_js_1.logWarn)(`[invariant-repair] firstSeenTs restored from snapshot for ${e.id}`);
+        (0, logger_js_1.logDebug)(`[invariant-repair] firstSeenTs restored from snapshot for ${e.id}`);
+        return;
+    }
+    // Final fallback: createdAt. By definition firstSeenTs ≤ createdAt is impossible
+    // (the index can never have observed an entry before it was created). For
+    // freshly-imported / freshly-added entries with no usage history yet, this is
+    // the correct answer; for legacy on-disk entries written before write-path
+    // populated firstSeenTs, it heals them silently. Repaired silently (no WARN)
+    // because this is the documented authoritative semantic, not a defect.
+    if (e.createdAt) {
+        e.firstSeenTs = e.createdAt;
+        firstSeenAuthority[e.id] = e.createdAt;
+        (0, features_1.incrementCounter)('usage:firstSeenCreatedAtFallback');
+        trackInvariantRepair(e.id, 'firstSeenTs', 'createdAt');
+        return;
     }
-    // If still missing after all repair sources, track an exhausted repair attempt (extremely rare diagnostic)
+    // If still missing after all repair sources, track an exhausted repair attempt (extremely rare diagnostic).
+    // Dedup the WARN per-id-per-process: a permanently unrecoverable id otherwise spams hundreds of
+    // stack-traced WARNs per dashboard poll (RCA 2026-05-01, dev port 8687). The counter and audit
+    // trail still increment on every call so health metrics remain accurate.
     if (!e.firstSeenTs) {
         (0, features_1.incrementCounter)('usage:firstSeenRepairExhausted');
         trackInvariantRepair(e.id, 'firstSeenTs', 'exhausted');
-        (0, logger_js_1.logWarn)(`[invariant-repair] firstSeenTs repair exhausted — no source found for ${e.id}`);
+        if (!firstSeenExhaustedReported.has(e.id)) {
+            firstSeenExhaustedReported.add(e.id);
+            (0, logger_js_1.logWarn)(`[invariant-repair] firstSeenTs repair exhausted — no source found for ${e.id}`);
+        }
     }
 }
+/**
+ * Internal handles for unit tests only. Not part of the public API.
+ * @internal
+ */
+exports._internal = { restoreFirstSeenInvariant };
 // Usage invariant repair (mirrors firstSeen invariant strategy). Extremely rare reload races in CI produced
 // states where a freshly re-materialized InstructionEntry temporarily lacked its prior usageCount (observed
 // by usageTracking.spec snapshot reads) even though authority maps retained the correct monotonic value.
@@ -421,14 +476,30 @@ function touchIndexVersion() {
     }
     catch { /* ignore */ }
 }
-function readVersionMTime() { try {
-    const vf = getVersionFile();
-    if (fs_1.default.existsSync(vf)) {
-        const st = fs_1.default.statSync(vf);
+function readVersionMTime() {
+    try {
+        const vf = getVersionFile();
+        if (fs_1.default.existsSync(vf)) {
+            const st = fs_1.default.statSync(vf);
+            return st.mtimeMs || 0;
+        }
+    }
+    catch { /* ignore */ }
+    // Fallback: when no .index-version file exists, use the instructions
+    // directory's own mtime so the ensureLoaded() cache short-circuit can still
+    // recognize an unchanged source. RCA 2026-05-01 (live dev port 8787 vs an
+    // operator-explored repo with no version file): readVersionMTime() returned
+    // 0, the falsy short-circuit `if (currentVersionMTime && ...)` always failed,
+    // every dashboard poll triggered a full disk reload + simple-reload trace,
+    // saturating the event loop and breaking dashboard imports.
+    try {
+        const baseDir = getInstructionsDir();
+        const st = fs_1.default.statSync(baseDir);
         return st.mtimeMs || 0;
     }
+    catch { /* ignore */ }
+    return 0;
 }
-catch { /* ignore */ } return 0; }
 function readVersionToken() { try {
     const vf = getVersionFile();
     if (fs_1.default.existsSync(vf)) {
@@ -503,20 +574,30 @@ function ensureLoaded() {
     const backend = (0, runtimeConfig_1.getRuntimeConfig)().storage?.backend ?? 'json';
     const store = backend === 'sqlite' ? getStoreForDir(baseDir) : null;
     let result = store ? store.load() : new indexLoader_1.IndexLoader(baseDir).load();
-    // Auto-migrate JSON → SQLite when JSON files on disk outnumber SQLite rows
+    // Auto-migrate JSON → SQLite when JSON files on disk outnumber SQLite rows.
+    // Per-process latch (RCA 2026-05-01): without this, mismatched counts caused by
+    // a few unparseable JSON files (jsonFiles.length permanently > sqlite rows) made
+    // ensureLoaded() re-invoke migrateJsonToSqlite() on every reload tick, causing
+    // INSERT-OR-REPLACE storms and unbounded WAL growth (~1.21 GB observed in dev
+    // before the fix). One attempt per (baseDir, dbPath) per process is enough; if
+    // an operator truly needs a re-migrate, they restart the server.
     if (store && (0, runtimeConfig_1.getRuntimeConfig)().storage?.sqliteMigrateOnStart) {
-        const jsonFiles = fs_1.default.existsSync(baseDir) ? fs_1.default.readdirSync(baseDir).filter(f => f.endsWith('.json') && !f.startsWith('_')) : [];
-        if (jsonFiles.length > result.entries.length) {
-            try {
-                const dbPath = (0, runtimeConfig_1.getRuntimeConfig)().storage?.sqlitePath ?? path_1.default.join(process.cwd(), 'data', 'index.db');
-                const mr = (0, migrationEngine_1.migrateJsonToSqlite)(baseDir, dbPath);
-                if (mr.migrated > 0) {
-                    (0, logger_js_1.logInfo)(`[storage] Auto-migrated ${mr.migrated} instruction(s) from JSON → SQLite`);
-                    result = store.load();
+        const dbPath = (0, runtimeConfig_1.getRuntimeConfig)().storage?.sqlitePath ?? path_1.default.join(process.cwd(), 'data', 'index.db');
+        const latchKey = `${baseDir}|${dbPath}`;
+        if (!autoMigrationAttempted.has(latchKey)) {
+            autoMigrationAttempted.add(latchKey);
+            const jsonFiles = fs_1.default.existsSync(baseDir) ? fs_1.default.readdirSync(baseDir).filter(f => f.endsWith('.json') && !f.startsWith('_')) : [];
+            if (jsonFiles.length > result.entries.length) {
+                try {
+                    const mr = (0, migrationEngine_1.migrateJsonToSqlite)(baseDir, dbPath);
+                    if (mr.migrated > 0) {
+                        (0, logger_js_1.logInfo)(`[storage] Auto-migrated ${mr.migrated} instruction(s) from JSON → SQLite`);
+                        result = store.load();
+                    }
+                }
+                catch (err) {
+                    (0, logger_js_1.logWarn)('[storage] Auto-migration failed:', err);
                 }
-            }
-            catch (err) {
-                (0, logger_js_1.logWarn)('[storage] Auto-migration failed:', err);
             }
         }
     }
@@ -525,7 +606,17 @@ function ensureLoaded() {
     // Deduplicate list using byId so two on-disk files with the same id field never produce duplicate
     // search results. byId already uses last-write-wins semantics; list must be consistent with it.
     const deduplicatedList = Array.from(byId.values());
-    state = { loadedAt: new Date().toISOString(), hash: result.hash, byId, list: deduplicatedList, fileCount: deduplicatedList.length, versionMTime: currentVersionMTime, versionToken: currentVersionToken, loadErrors: result.errors, loadDebug: result.debug, loadSummary: result.summary };
+    // RCA 2026-05-01 (live dev port 8787): IndexLoader.load() writes _manifest.json
+    // and _skipped.json into baseDir as a side-effect, which bumps the directory's
+    // mtime. If we cached the pre-load mtime here, the very next ensureLoaded()
+    // call would observe a newer mtime, miss the cache, and reload again — an
+    // unbounded loop that emitted [trace:ensureLoaded:simple-reload] hundreds of
+    // times per second and saturated the event loop (dashboard imports failed).
+    // Re-read the mtime AFTER load so the cached value reflects the post-write
+    // state; subsequent calls without source changes will then short-circuit.
+    const postLoadVersionMTime = readVersionMTime();
+    const postLoadVersionToken = readVersionToken();
+    state = { loadedAt: new Date().toISOString(), hash: result.hash, byId, list: deduplicatedList, fileCount: deduplicatedList.length, versionMTime: postLoadVersionMTime || currentVersionMTime, versionToken: postLoadVersionToken || currentVersionToken, loadErrors: result.errors, loadDebug: result.debug, loadSummary: result.summary };
     dirty = false;
     // Overlay usage snapshot (simplified; no spin/repair loops here—existing invariant repairs still occur in getIndexState)
     try {
@@ -573,7 +664,11 @@ async function ensureLoadedAsync() {
     const byId = new Map();
     result.entries.forEach(e => byId.set(e.id, e));
     const deduplicatedList = Array.from(byId.values());
-    state = { loadedAt: new Date().toISOString(), hash: result.hash, byId, list: deduplicatedList, fileCount: deduplicatedList.length, versionMTime: currentVersionMTime, versionToken: currentVersionToken, loadErrors: result.errors, loadDebug: result.debug, loadSummary: result.summary };
+    // See RCA comment in ensureLoaded() above: re-read mtime AFTER load to absorb
+    // _manifest.json / _skipped.json side-effect writes.
+    const postLoadVersionMTime = readVersionMTime();
+    const postLoadVersionToken = readVersionToken();
+    state = { loadedAt: new Date().toISOString(), hash: result.hash, byId, list: deduplicatedList, fileCount: deduplicatedList.length, versionMTime: postLoadVersionMTime || currentVersionMTime, versionToken: postLoadVersionToken || currentVersionToken, loadErrors: result.errors, loadDebug: result.debug, loadSummary: result.summary };
     dirty = false;
     try {
         const snap = loadUsageSnapshot();
@@ -826,6 +921,15 @@ function isDuplicateInstructionWriteError(error) {
 }
 function writeEntry(entry, opts) {
     const file = path_1.default.join(getInstructionsDir(), `${entry.id}.json`);
+    // Establish firstSeenTs at the write boundary if the caller omitted it.
+    // Semantically firstSeenTs ≤ createdAt always — for fresh entries this is
+    // the correct value, and persisting it on disk avoids spurious
+    // [invariant-repair] WARN spam on every subsequent getIndexState() poll
+    // (RCA 2026-05-01 dev port 8687, third loop in chain after PR #285/#286).
+    if (!entry.firstSeenTs) {
+        entry.firstSeenTs = entry.createdAt || new Date().toISOString();
+        firstSeenAuthority[entry.id] = entry.firstSeenTs;
+    }
     const classifier = new classificationService_1.ClassificationService();
     let record = classifier.normalize(entry);
     if (record.owner === 'unowned') {
@@ -880,6 +984,11 @@ function writeEntry(entry, opts) {
 }
 async function writeEntryAsync(entry, opts) {
     const file = path_1.default.join(getInstructionsDir(), `${entry.id}.json`);
+    // See writeEntry: establish firstSeenTs at the write boundary if missing.
+    if (!entry.firstSeenTs) {
+        entry.firstSeenTs = entry.createdAt || new Date().toISOString();
+        firstSeenAuthority[entry.id] = entry.firstSeenTs;
+    }
     const classifier = new classificationService_1.ClassificationService();
     let record = classifier.normalize(entry);
     if (record.owner === 'unowned') {

package/dist/services/logger.js

@@ -11,6 +11,7 @@ const fs_1 = __importDefault(require("fs"));
 const path_1 = __importDefault(require("path"));
 const runtimeConfig_1 = require("../config/runtimeConfig");
 const mcpLogBridge_1 = require("./mcpLogBridge");
+const eventBuffer_1 = require("./eventBuffer");
 /** Numeric priority for log level filtering (lower = more verbose). */
 const LEVEL_PRIORITY = {
     TRACE: 0,
@@ -231,6 +232,14 @@ function emit(rec) {
         }
         catch { /* ignore file write failures */ }
     }
+    // Surface WARN/ERROR into the in-process events ring buffer so the dashboard
+    // Monitoring panel can display them without log-file tailing (OB-3, OB-5).
+    if (rec.level === 'WARN' || rec.level === 'ERROR') {
+        try {
+            (0, eventBuffer_1.recordEvent)(rec.level, rec.msg, rec.detail, rec.pid);
+        }
+        catch { /* never let buffer failure break logging */ }
+    }
 }
 /**
  * Emit a structured NDJSON log record at the specified level.

package/dist/services/manifestManager.js

@@ -20,7 +20,17 @@ const features_1 = require("./features");
 const logger_1 = require("./logger");
 const runtimeConfig_1 = require("../config/runtimeConfig");
 const MANIFEST_RELATIVE = path_1.default.join('snapshots', 'index-manifest.json');
-function getManifestPath() { return path_1.default.join(process.cwd(), MANIFEST_RELATIVE); }
+function getManifestPath() {
+    // Tests run in parallel forks (vitest pool=forks, maxWorkers=4) and each fork
+    // shares the same process.cwd(); without an override, multiple test files that
+    // enable manifest write race on the same on-disk file. Honor an explicit path
+    // override so test setup can isolate per-spec output. Production leaves this
+    // unset and continues to use the canonical snapshot path.
+    const override = process.env.INDEX_SERVER_MANIFEST_PATH;
+    if (override && override.trim())
+        return path_1.default.isAbsolute(override) ? override : path_1.default.join(process.cwd(), override);
+    return path_1.default.join(process.cwd(), MANIFEST_RELATIVE);
+}
 /**
  * Load the manifest from disk.
  * @returns The parsed {@link IndexManifest}, or `null` if the file is absent or unparseable

package/dist/services/seedBootstrap.js

@@ -223,7 +223,11 @@ function autoSeedBootstrap() {
         // Directory empty OR missing seed triggers creation.
         try {
             const tmp = path_1.default.join(dir, `.${seed.file}.tmp-${process.pid}-${Date.now()}-${Math.random().toString(16).slice(2)}`);
-            fs_1.default.writeFileSync(tmp, JSON.stringify(seed.json, null, 2), { encoding: 'utf8' });
+            // Inject timestamps at write time so loaders never trigger
+            // [invariant-repair] firstSeenTs WARN noise on subsequent reads.
+            const nowIso = new Date().toISOString();
+            const stamped = { createdAt: nowIso, firstSeenTs: nowIso, ...seed.json };
+            fs_1.default.writeFileSync(tmp, JSON.stringify(stamped, null, 2), { encoding: 'utf8' });
             fs_1.default.renameSync(tmp, target);
             summary.created.push(seed.file);
         }

package/dist/services/storage/factory.d.ts

@@ -6,6 +6,8 @@
  */
 import type { IInstructionStore, IEmbeddingStore } from './types.js';
 export type StorageBackend = 'json' | 'sqlite';
+/** Test-only: reset the warn-once latch. */
+export declare function _resetSqliteExperimentalWarning(): void;
 /**
  * Check that the current Node.js version meets the minimum requirement.
  * Throws a clear error if the version is too old.

package/dist/services/storage/factory.js

@@ -6,6 +6,7 @@
  * Default: JsonFileStore (json). Experimental: SqliteStore (sqlite).
  */
 Object.defineProperty(exports, "__esModule", { value: true });
+exports._resetSqliteExperimentalWarning = _resetSqliteExperimentalWarning;
 exports.checkNodeVersion = checkNodeVersion;
 exports.createStore = createStore;
 exports.createEmbeddingStore = createEmbeddingStore;
@@ -14,6 +15,13 @@ const jsonFileStore_js_1 = require("./jsonFileStore.js");
 const jsonEmbeddingStore_js_1 = require("./jsonEmbeddingStore.js");
 const sqliteStore_js_1 = require("./sqliteStore.js");
 const logger_js_1 = require("../logger.js");
+/** Process-scoped flag so the EXPERIMENTAL SQLite warning is emitted once,
+ *  not per-store-creation (which floods the events ring on every request). */
+let sqliteExperimentalWarned = false;
+/** Test-only: reset the warn-once latch. */
+function _resetSqliteExperimentalWarning() {
+    sqliteExperimentalWarned = false;
+}
 /**
  * Check that the current Node.js version meets the minimum requirement.
  * Throws a clear error if the version is too old.
@@ -46,7 +54,10 @@ function createStore(backend, dir, sqlitePath) {
     switch (resolvedBackend) {
         case 'sqlite': {
             checkNodeVersion('22.5.0', 'SQLite storage backend (node:sqlite)');
-            (0, logger_js_1.logWarn)('[storage] ⚠️  EXPERIMENTAL: SQLite backend is enabled. This feature has limited testing and may have data-loss or compatibility issues. Not recommended for production use.');
+            if (!sqliteExperimentalWarned) {
+                sqliteExperimentalWarned = true;
+                (0, logger_js_1.logWarn)('[storage] ⚠️  EXPERIMENTAL: SQLite backend is enabled. This feature has limited testing and may have data-loss or compatibility issues. Not recommended for production use.');
+            }
             const dbPath = sqlitePath ?? config.storage?.sqlitePath ?? 'data/index.db';
             return new sqliteStore_js_1.SqliteStore(dbPath);
         }

package/dist/services/tracing.js

@@ -219,8 +219,10 @@ function emitTrace(label, data, min = 1) {
     if (caller)
         rec.func = caller;
     pushBuffer(rec, tracing.buffer);
+    // Trace records are diagnostic, not errors. Route through DEBUG so they do
+    // not pollute the WARN/ERROR events ring buffer surfaced to the dashboard.
     try {
-        (0, logger_js_1.logError)(label, JSON.stringify(rec));
+        (0, logger_js_1.logDebug)(label, JSON.stringify(rec));
     }
     catch { /* ignore */ }
     if (!(tracing.persist || tracing.file))

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "@jagilber-org/index-server",
-  "version": "1.27.0",
+  "version": "1.27.2",
   "mcpName": "io.github.jagilber-org/index-server",
-  "description": "MCP instruction indexing server for AI assistant governance — search, CRUD, schema validation, usage tracking, and cross-repo knowledge promotion.",
+  "description": "MCP instruction indexing server with search, CRUD, validation, and cross-repo knowledge promotion.",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -62,6 +62,7 @@
     "typecheck": "tsc -p tsconfig.json --noEmit",
     "lint": "eslint .",
     "lint:instructions": "node scripts/lint-instructions.mjs",
+    "lint:logs": "node scripts/crawl-logs.mjs --dir logs --file test-results/test-output.log --allowlist .crawl-logs-allowlist --strict",
     "format": "prettier --write .",
     "prepare": "node scripts/setup-hooks.cjs",
     "audit": "npm audit --omit=dev --audit-level=high",
@@ -134,6 +135,15 @@
     "ws": "^8.19.0",
     "zod": "^3.23.8"
   },
+  "overrides": {
+    "mermaid": {
+      "uuid": "^14.0.0"
+    },
+    "@mermaid-js/layout-elk": {
+      "uuid": "^14.0.0"
+    },
+    "@types/express-serve-static-core": "5.0.7"
+  },
   "optionalScripts": {},
   "promptReviewCriteriaVersion": "1.0.0",
   "license": "MIT",

package/server.json

@@ -1,17 +1,17 @@
 {
   "$schema": "https://static.modelcontextprotocol.io/schemas/2025-12-11/server.schema.json",
   "name": "io.github.jagilber-org/index-server",
-  "description": "MCP instruction indexing server for AI assistant governance — search, CRUD, schema validation, usage tracking, and cross-repo knowledge promotion.",
+  "description": "MCP instruction indexing server with search, CRUD, validation, and cross-repo knowledge promotion.",
   "repository": {
     "url": "https://github.com/jagilber-org/index-server",
     "source": "github"
   },
-  "version": "1.27.0",
+  "version": "1.27.2",
   "packages": [
     {
       "registryType": "npm",
       "identifier": "@jagilber-org/index-server",
-      "version": "1.27.0",
+      "version": "1.27.2",
       "transport": {
         "type": "stdio"
       }