@jagilber-org/index-server 1.22.1 → 1.26.4

package/dist/services/indexContext.js

@@ -3,6 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.getInvariantRepairSummary = getInvariantRepairSummary;
 exports.clearUsageRateLimit = clearUsageRateLimit;
 exports.loadUsageSnapshot = loadUsageSnapshot;
 exports.getInstructionsDir = getInstructionsDir;
@@ -10,15 +11,19 @@ exports.diagnoseInstructionsDir = diagnoseInstructionsDir;
 exports.touchIndexVersion = touchIndexVersion;
 exports.markindexDirty = markindexDirty;
 exports.ensureLoaded = ensureLoaded;
+exports.ensureLoadedAsync = ensureLoadedAsync;
 exports.startIndexVersionPoller = startIndexVersionPoller;
 exports.stopIndexVersionPoller = stopIndexVersionPoller;
 exports.invalidate = invalidate;
 exports.getIndexState = getIndexState;
+exports.getIndexStateAsync = getIndexStateAsync;
 exports.getDebugIndexSnapshot = getDebugIndexSnapshot;
 exports.getIndexDiagnostics = getIndexDiagnostics;
 exports.projectGovernance = projectGovernance;
 exports.computeGovernanceHash = computeGovernanceHash;
+exports.isDuplicateInstructionWriteError = isDuplicateInstructionWriteError;
 exports.writeEntry = writeEntry;
+exports.writeEntryAsync = writeEntryAsync;
 exports.removeEntry = removeEntry;
 exports.scheduleUsagePersist = scheduleUsagePersist;
 exports.incrementUsage = incrementUsage;
@@ -35,6 +40,9 @@ const envUtils_1 = require("../utils/envUtils");
 const runtimeConfig_1 = require("../config/runtimeConfig");
 const factory_1 = require("./storage/factory");
 const migrationEngine_1 = require("./storage/migrationEngine");
+const instructionRecordValidation_1 = require("./instructionRecordValidation");
+const loaderSchemaValidator_1 = require("./loaderSchemaValidator");
+const schemaVersion_1 = require("../versioning/schemaVersion");
 let state = null;
 // Simple reliable invalidation: any mutation sets dirty=true; next ensureLoaded() performs full rescan.
 let dirty = false;
@@ -79,6 +87,19 @@ const firstSeenAuthority = {};
 const usageAuthority = {};
 // Authoritative lastUsedAt map for resilience between reload + snapshot overlay timing.
 const lastUsedAuthority = {};
+// ── Invariant repair tracking (#131) ─────────────────────────────
+// Accumulates repair events so they can be surfaced via health checks.
+const invariantRepairLog = [];
+const MAX_REPAIR_LOG = 200;
+function trackInvariantRepair(id, field, source) {
+    invariantRepairLog.push({ ts: new Date().toISOString(), id, field, source });
+    if (invariantRepairLog.length > MAX_REPAIR_LOG)
+        invariantRepairLog.shift();
+}
+/** Returns a summary of invariant repairs for health check visibility. */
+function getInvariantRepairSummary() {
+    return { totalRepairs: invariantRepairLog.length, recentRepairs: invariantRepairLog.slice(-20) };
+}
 // Defensive invariant repair: if any code path ever observes an InstructionEntry with a missing
 // firstSeenTs after it was previously established (should not happen, but flake indicates a very
 // rare timing or cross-test interaction), we repair it from ephemeral cache or lastGood snapshot.
@@ -89,22 +110,30 @@ function restoreFirstSeenInvariant(e) {
     if (auth) {
         e.firstSeenTs = auth;
         (0, features_1.incrementCounter)('usage:firstSeenAuthorityRepair');
+        trackInvariantRepair(e.id, 'firstSeenTs', 'authority');
+        (0, logger_js_1.logWarn)(`[invariant-repair] firstSeenTs restored from authority for ${e.id}`);
         return;
     }
     const ep = ephemeralFirstSeen[e.id];
     if (ep) {
         e.firstSeenTs = ep;
         (0, features_1.incrementCounter)('usage:firstSeenInvariantRepair');
+        trackInvariantRepair(e.id, 'firstSeenTs', 'ephemeral');
+        (0, logger_js_1.logWarn)(`[invariant-repair] firstSeenTs restored from ephemeral cache for ${e.id}`);
         return;
     }
     const snap = lastGoodUsageSnapshot[e.id];
     if (snap?.firstSeenTs) {
         e.firstSeenTs = snap.firstSeenTs;
         (0, features_1.incrementCounter)('usage:firstSeenInvariantRepair');
+        trackInvariantRepair(e.id, 'firstSeenTs', 'snapshot');
+        (0, logger_js_1.logWarn)(`[invariant-repair] firstSeenTs restored from snapshot for ${e.id}`);
     }
     // If still missing after all repair sources, track an exhausted repair attempt (extremely rare diagnostic)
     if (!e.firstSeenTs) {
         (0, features_1.incrementCounter)('usage:firstSeenRepairExhausted');
+        trackInvariantRepair(e.id, 'firstSeenTs', 'exhausted');
+        (0, logger_js_1.logWarn)(`[invariant-repair] firstSeenTs repair exhausted — no source found for ${e.id}`);
     }
 }
 // Usage invariant repair (mirrors firstSeen invariant strategy). Extremely rare reload races in CI produced
@@ -115,26 +144,33 @@ function restoreFirstSeenInvariant(e) {
 function restoreUsageInvariant(e) {
     if (e.usageCount != null)
         return;
-    // Prefer authoritative value, then observed, then persisted snapshot, else default 0.
     if (usageAuthority[e.id] != null) {
         e.usageCount = usageAuthority[e.id];
         (0, features_1.incrementCounter)('usage:usageInvariantAuthorityRepair');
+        trackInvariantRepair(e.id, 'usageCount', 'authority');
+        (0, logger_js_1.logWarn)(`[invariant-repair] usageCount restored from authority for ${e.id} (value=${usageAuthority[e.id]})`);
         return;
     }
     if (observedUsage[e.id] != null) {
         e.usageCount = observedUsage[e.id];
         (0, features_1.incrementCounter)('usage:usageInvariantObservedRepair');
+        trackInvariantRepair(e.id, 'usageCount', 'observed');
+        (0, logger_js_1.logWarn)(`[invariant-repair] usageCount restored from observed for ${e.id} (value=${observedUsage[e.id]})`);
         return;
     }
     const snap = lastGoodUsageSnapshot[e.id];
     if (snap?.usageCount != null) {
         e.usageCount = snap.usageCount;
         (0, features_1.incrementCounter)('usage:usageInvariantSnapshotRepair');
+        trackInvariantRepair(e.id, 'usageCount', 'snapshot');
+        (0, logger_js_1.logWarn)(`[invariant-repair] usageCount restored from snapshot for ${e.id} (value=${snap.usageCount})`);
         return;
     }
     // Fall back to 0 – deterministic floor; next increment will advance.
     e.usageCount = 0;
     (0, features_1.incrementCounter)('usage:usageInvariantZeroRepair');
+    trackInvariantRepair(e.id, 'usageCount', 'zero-default');
+    (0, logger_js_1.logWarn)(`[invariant-repair] usageCount defaulted to 0 for ${e.id} — no repair source found`);
 }
 // Repair missing lastUsedAt for entries with usage.
 function restoreLastUsedInvariant(e) {
@@ -143,17 +179,23 @@ function restoreLastUsedInvariant(e) {
     if (lastUsedAuthority[e.id]) {
         e.lastUsedAt = lastUsedAuthority[e.id];
         (0, features_1.incrementCounter)('usage:lastUsedAuthorityRepair');
+        trackInvariantRepair(e.id, 'lastUsedAt', 'authority');
+        (0, logger_js_1.logWarn)(`[invariant-repair] lastUsedAt restored from authority for ${e.id}`);
         return;
     }
     const snap = lastGoodUsageSnapshot[e.id];
     if (snap?.lastUsedAt) {
         e.lastUsedAt = snap.lastUsedAt;
         (0, features_1.incrementCounter)('usage:lastUsedSnapshotRepair');
+        trackInvariantRepair(e.id, 'lastUsedAt', 'snapshot');
+        (0, logger_js_1.logWarn)(`[invariant-repair] lastUsedAt restored from snapshot for ${e.id}`);
         return;
     }
     if ((e.usageCount ?? 0) > 0 && e.firstSeenTs) {
         e.lastUsedAt = e.firstSeenTs;
         (0, features_1.incrementCounter)('usage:lastUsedFirstSeenRepair');
+        trackInvariantRepair(e.id, 'lastUsedAt', 'firstSeen-approx');
+        (0, logger_js_1.logWarn)(`[invariant-repair] lastUsedAt approximated from firstSeenTs for ${e.id}`);
     }
 }
 // Rate limiting for usage increments (Phase 1 requirement)
@@ -214,8 +256,9 @@ function loadUsageSnapshot() {
             }
             break; // file not present – exit attempts
         }
-        catch {
-            // swallow and retry (tight loop – extremely rare path)
+        catch (err) {
+            // Log parse/read error and retry (tight loop – extremely rare path)
+            (0, logger_js_1.logWarn)(`[invariant-repair] loadUsageSnapshot attempt ${attempt} failed: ${err.message || String(err)}`);
         }
     }
     // Fallback to last good snapshot (prevents loss of firstSeenTs on rare parse race)
@@ -243,7 +286,7 @@ function flushUsageSnapshot() {
             for (const e of state.list) {
                 const authoritative = e.firstSeenTs || firstSeenAuthority[e.id];
                 if (authoritative && !firstSeenAuthority[e.id])
-                    firstSeenAuthority[e.id] = authoritative;
+                    firstSeenAuthority[e.id] = authoritative; // lgtm[js/remote-property-injection] — id is regex-validated by instruction schema (^[a-z0-9](?:[a-z0-9-_]{0,118}[a-z0-9])?$) before reaching index
                 if (e.usageCount || e.lastUsedAt || authoritative) {
                     const rec = { usageCount: e.usageCount, firstSeenTs: authoritative, lastUsedAt: e.lastUsedAt };
                     // Merge signal/comment/action from in-memory cache (last-write-wins from incrementUsage calls)
@@ -256,18 +299,18 @@ function flushUsageSnapshot() {
                         if (cached.lastComment)
                             rec.lastComment = cached.lastComment;
                     }
-                    obj[e.id] = rec;
+                    obj[e.id] = rec; // lgtm[js/remote-property-injection] — id is schema-validated before reaching index
                 }
             }
             // Atomic write: write to temp then rename to avoid readers seeing partial JSON
             const snapPath = getUsageSnapshotPath();
             const tmp = snapPath + '.tmp';
-            fs_1.default.writeFileSync(tmp, JSON.stringify(obj, null, 2));
+            fs_1.default.writeFileSync(tmp, JSON.stringify(obj, null, 2)); // lgtm[js/http-to-file-access] — snapPath is config-controlled usage snapshot path
             try {
                 fs_1.default.renameSync(tmp, snapPath);
             }
             catch { /* fallback to direct write if rename fails */
-                fs_1.default.writeFileSync(snapPath, JSON.stringify(obj, null, 2));
+                fs_1.default.writeFileSync(snapPath, JSON.stringify(obj, null, 2)); /* lgtm[js/http-to-file-access] — snapPath is config-controlled usage snapshot path */
             }
             lastGoodUsageSnapshot = obj; // update cache
         }
@@ -278,7 +321,7 @@ function flushUsageSnapshot() {
 // The guard ensures cleanup runs exactly once even if multiple signals race.
 try {
     // Import directly from shutdownGuard module (no circular dependency)
-    // eslint-disable-next-line @typescript-eslint/no-var-requires, @typescript-eslint/no-require-imports
+    // eslint-disable-next-line @typescript-eslint/no-require-imports
     const { createShutdownGuard: _createShutdownGuard } = require('../server/shutdownGuard');
     // Get or create a process-wide singleton via a global symbol
     const key = Symbol.for('mcp-shutdown-guard');
@@ -336,6 +379,7 @@ function getInstructionsDir() {
 }
 // Centralized tracing utilities
 const tracing_1 = require("./tracing");
+const logger_js_1 = require("./logger.js");
 // Throttled file trace emission (avoid per-get amplification). We emit per-file decisions only
 // on true reloads AND if file signature changed OR time since last emission > threshold.
 // (legacy file-level trace removed in simplified loader)
@@ -394,6 +438,58 @@ function readVersionToken() { try {
 }
 catch { /* ignore */ } return ''; }
 function markindexDirty() { dirty = true; }
+function syncTouchedVersionIntoState() {
+    try {
+        touchIndexVersion();
+        const vfMTime = (function () { try {
+            const vf = path_1.default.join(getInstructionsDir(), '.index-version');
+            if (fs_1.default.existsSync(vf)) {
+                return fs_1.default.statSync(vf).mtimeMs || 0;
+            }
+        }
+        catch { /* ignore */ } return 0; })();
+        const vfToken = (function () { try {
+            const vf = path_1.default.join(getInstructionsDir(), '.index-version');
+            if (fs_1.default.existsSync(vf)) {
+                return fs_1.default.readFileSync(vf, 'utf8').trim();
+            }
+        }
+        catch { /* ignore */ } return ''; })();
+        if (state) {
+            if (vfMTime && state.versionMTime !== vfMTime) {
+                state.versionMTime = vfMTime;
+            }
+            if (vfToken && state.versionToken !== vfToken) {
+                state.versionToken = vfToken;
+            }
+        }
+    }
+    catch { /* ignore */ }
+}
+function materializeWrittenEntry(record) {
+    if (state) {
+        const existing = state.byId.get(record.id);
+        if (existing) {
+            Object.assign(existing, record);
+            try {
+                (0, features_1.incrementCounter)('index:inMemoryUpdate');
+            }
+            catch { /* ignore */ }
+        }
+        else {
+            state.list.push(record);
+            state.byId.set(record.id, record);
+            try {
+                (0, features_1.incrementCounter)('index:inMemoryMaterialize');
+            }
+            catch { /* ignore */ }
+        }
+        syncTouchedVersionIntoState();
+        return;
+    }
+    markindexDirty();
+    syncTouchedVersionIntoState();
+}
 function ensureLoaded() {
     const baseDir = getInstructionsDir();
     // Always reload if no state or dirty or version file changed.
@@ -416,12 +512,12 @@ function ensureLoaded() {
                 const dbPath = (0, runtimeConfig_1.getRuntimeConfig)().storage?.sqlitePath ?? path_1.default.join(process.cwd(), 'data', 'index.db');
                 const mr = (0, migrationEngine_1.migrateJsonToSqlite)(baseDir, dbPath);
                 if (mr.migrated > 0) {
-                    console.log(`[storage] Auto-migrated ${mr.migrated} instruction(s) from JSON → SQLite`);
+                    (0, logger_js_1.logInfo)(`[storage] Auto-migrated ${mr.migrated} instruction(s) from JSON → SQLite`);
                     result = store.load();
                 }
             }
             catch (err) {
-                console.warn('[storage] Auto-migration failed:', err);
+                (0, logger_js_1.logWarn)('[storage] Auto-migration failed:', err);
             }
         }
     }
@@ -445,7 +541,54 @@ function ensureLoaded() {
                         e.firstSeenTs = rec.firstSeenTs;
                         if (!firstSeenAuthority[e.id])
                             firstSeenAuthority[e.id] = rec.firstSeenTs;
-                    }
+                    } // lgtm[js/remote-property-injection] — id is schema-validated before reaching index
+                    if (!e.lastUsedAt && rec.lastUsedAt)
+                        e.lastUsedAt = rec.lastUsedAt;
+                }
+            }
+        }
+    }
+    catch { /* ignore */ }
+    if ((0, tracing_1.traceEnabled)(1)) {
+        try {
+            (0, tracing_1.emitTrace)('[trace:ensureLoaded:simple-reload]', { dir: baseDir, count: state.list.length });
+        }
+        catch { /* ignore */ }
+    }
+    return state;
+}
+async function ensureLoadedAsync() {
+    const baseDir = getInstructionsDir();
+    const currentVersionMTime = readVersionMTime();
+    const currentVersionToken = readVersionToken();
+    if (state && !dirty) {
+        if (currentVersionMTime && currentVersionMTime === state.versionMTime && currentVersionToken === state.versionToken) {
+            return state;
+        }
+    }
+    const backend = (0, runtimeConfig_1.getRuntimeConfig)().storage?.backend ?? 'json';
+    if (backend === 'sqlite') {
+        return ensureLoaded();
+    }
+    const result = await new indexLoader_1.IndexLoader(baseDir).loadAsync();
+    const byId = new Map();
+    result.entries.forEach(e => byId.set(e.id, e));
+    const deduplicatedList = Array.from(byId.values());
+    state = { loadedAt: new Date().toISOString(), hash: result.hash, byId, list: deduplicatedList, fileCount: deduplicatedList.length, versionMTime: currentVersionMTime, versionToken: currentVersionToken, loadErrors: result.errors, loadDebug: result.debug, loadSummary: result.summary };
+    dirty = false;
+    try {
+        const snap = loadUsageSnapshot();
+        if (snap) {
+            for (const e of state.list) {
+                const rec = snap[e.id];
+                if (rec) {
+                    if (e.usageCount == null && rec.usageCount != null)
+                        e.usageCount = rec.usageCount;
+                    if (!e.firstSeenTs && rec.firstSeenTs) {
+                        e.firstSeenTs = rec.firstSeenTs;
+                        if (!firstSeenAuthority[e.id])
+                            firstSeenAuthority[e.id] = rec.firstSeenTs;
+                    } // lgtm[js/remote-property-injection] — id is schema-validated before reaching index
                     if (!e.lastUsedAt && rec.lastUsedAt)
                         e.lastUsedAt = rec.lastUsedAt;
                 }
@@ -561,6 +704,21 @@ function getIndexState() {
     }
     return st;
 }
+async function getIndexStateAsync() {
+    const st = await ensureLoadedAsync();
+    for (const e of st.list) {
+        if (!e.firstSeenTs) {
+            restoreFirstSeenInvariant(e);
+        }
+        if (e.usageCount == null) {
+            restoreUsageInvariant(e);
+        }
+        if (e.lastUsedAt == null) {
+            restoreLastUsedInvariant(e);
+        }
+    }
+    return st;
+}
 // Lightweight debug snapshot WITHOUT forcing a reload (observes current in-memory view vs disk)
 function getDebugIndexSnapshot() {
     const dir = getInstructionsDir();
@@ -658,10 +816,19 @@ function computeGovernanceHash(entries) {
     return h.digest('hex');
 }
 // Mutation helpers (import/add/remove/groom share)
-function writeEntry(entry) {
+function isDuplicateInstructionWriteError(error) {
+    const code = error?.code;
+    if (code === 'EEXIST')
+        return true;
+    if (!(error instanceof Error))
+        return false;
+    const message = error.message.toLowerCase();
+    return message.includes('unique constraint failed') || message.includes('duplicate key');
+}
+function writeEntry(entry, opts) {
     const file = path_1.default.join(getInstructionsDir(), `${entry.id}.json`);
     const classifier = new classificationService_1.ClassificationService();
-    const record = classifier.normalize(entry);
+    let record = classifier.normalize(entry);
     if (record.owner === 'unowned') {
         const auto = (0, ownershipService_1.resolveOwner)(record.id);
         if (auto) {
@@ -669,77 +836,96 @@ function writeEntry(entry) {
             record.updatedAt = new Date().toISOString();
         }
     }
+    record = (0, instructionRecordValidation_1.assertValidInstructionRecord)(record);
+    // Run the SAME migration the loader runs on read so the write path is
+    // symmetric with the read path. This brings legacy in-memory records
+    // (carrying old schemaVersion or missing v3+ defaults) up to current
+    // schema BEFORE validateForDisk gates them against the loader schema.
+    // Without this, callers passing legacy records would be silently rejected
+    // by the loader-symmetric validator even though the loader itself would
+    // have migrated them transparently.
+    (0, schemaVersion_1.migrateInstructionRecord)(record);
+    // Validate against the SAME JSON schema the loader uses at reload time.
+    // This prevents schema drift from silently dropping entries on reload.
+    const diskCheck = (0, loaderSchemaValidator_1.validateForDisk)(record);
+    if (!diskCheck.valid) {
+        const err = new Error(`Pre-write loader-schema validation failed for '${entry.id}': ${diskCheck.errors?.join('; ')}`);
+        err.validationErrors = diskCheck.errors;
+        err.isInstructionValidation = true;
+        throw err;
+    }
     const store = getStoreForDir(getInstructionsDir());
     if (store) {
-        store.write(record);
+        store.write(record, opts);
+    }
+    else if (opts?.createOnly) {
+        (0, atomicFs_1.atomicCreateJson)(file, record);
     }
     else {
         (0, atomicFs_1.atomicWriteJson)(file, record);
     }
-    // Revised mutation strategy (2025-09-14): Avoid setting dirty=true when we can
-    // apply the change directly to the in-memory index. Previous implementation
-    // marked the index dirty before an immediate getIndexState() call in tests,
-    // forcing a reload that sometimes raced the Windows filesystem directory
-    // visibility of the new file. That produced a flake where the opportunistic
-    // materialization guarantee was lost. We now:
-    //  1. Opportunistically materialize (add or update) the entry in-memory.
-    //  2. Touch the version file so other processes/pollers observe the change.
-    //  3. Only mark dirty if no state is currently loaded (so first subsequent
-    //     access triggers a load). Otherwise we keep the current state hot.
-    if (state) {
-        const existing = state.byId.get(record.id);
-        if (existing) {
-            // Update in-place so references (including any cached projections) see new fields.
-            Object.assign(existing, record);
-            try {
-                (0, features_1.incrementCounter)('index:inMemoryUpdate');
+    // Post-write read-back: verify the file on disk passes the loader schema
+    if (!store) {
+        try {
+            const diskRaw = JSON.parse(fs_1.default.readFileSync(file, 'utf8'));
+            const postCheck = (0, loaderSchemaValidator_1.validateForDisk)(diskRaw);
+            if (!postCheck.valid) {
+                (0, logger_js_1.logWarn)(`[writeEntry] Post-write validation FAILED for '${entry.id}': ${postCheck.errors?.join('; ')}`);
             }
-            catch { /* ignore */ }
         }
-        else {
-            state.list.push(record);
-            state.byId.set(record.id, record);
-            try {
-                (0, features_1.incrementCounter)('index:inMemoryMaterialize');
-            }
-            catch { /* ignore */ }
+        catch (readErr) {
+            (0, logger_js_1.logWarn)(`[writeEntry] Post-write read-back failed for '${entry.id}': ${readErr.message}`);
         }
-        // Signal externally. Then optimistically update in-memory version snapshot so getIndexState()
-        // does NOT trigger an immediate reload (which can race directory enumeration on Windows).
-        try {
-            touchIndexVersion();
-            // After touching, read back token + mtime to align with ensureLoaded's cache validation logic.
-            const vfMTime = (function () { try {
-                const vf = path_1.default.join(getInstructionsDir(), '.index-version');
-                if (fs_1.default.existsSync(vf)) {
-                    return fs_1.default.statSync(vf).mtimeMs || 0;
-                }
-            }
-            catch { /* ignore */ } return 0; })();
-            const vfToken = (function () { try {
-                const vf = path_1.default.join(getInstructionsDir(), '.index-version');
-                if (fs_1.default.existsSync(vf)) {
-                    return fs_1.default.readFileSync(vf, 'utf8').trim();
-                }
-            }
-            catch { /* ignore */ } return ''; })();
-            if (vfMTime && state.versionMTime !== vfMTime) {
-                state.versionMTime = vfMTime;
-            }
-            if (vfToken && state.versionToken !== vfToken) {
-                state.versionToken = vfToken;
-            }
+    }
+    materializeWrittenEntry(record);
+}
+async function writeEntryAsync(entry, opts) {
+    const file = path_1.default.join(getInstructionsDir(), `${entry.id}.json`);
+    const classifier = new classificationService_1.ClassificationService();
+    let record = classifier.normalize(entry);
+    if (record.owner === 'unowned') {
+        const auto = (0, ownershipService_1.resolveOwner)(record.id);
+        if (auto) {
+            record.owner = auto;
+            record.updatedAt = new Date().toISOString();
         }
-        catch { /* ignore */ }
+    }
+    record = (0, instructionRecordValidation_1.assertValidInstructionRecord)(record);
+    // Mirror the loader's migration step before validating against the loader
+    // schema. See writeEntry for full rationale.
+    (0, schemaVersion_1.migrateInstructionRecord)(record);
+    // Validate against the SAME JSON schema the loader uses at reload time.
+    const diskCheck = (0, loaderSchemaValidator_1.validateForDisk)(record);
+    if (!diskCheck.valid) {
+        const err = new Error(`Pre-write loader-schema validation failed for '${entry.id}': ${diskCheck.errors?.join('; ')}`);
+        err.validationErrors = diskCheck.errors;
+        err.isInstructionValidation = true;
+        throw err;
+    }
+    const store = getStoreForDir(getInstructionsDir());
+    if (store) {
+        store.write(record, opts);
+    }
+    else if (opts?.createOnly) {
+        await (0, atomicFs_1.atomicCreateJsonAsync)(file, record);
     }
     else {
-        // No in-memory state yet; next ensureLoaded should pick up new file.
-        markindexDirty();
+        await (0, atomicFs_1.atomicWriteJsonAsync)(file, record);
+    }
+    // Post-write read-back: verify the file on disk passes the loader schema
+    if (!store) {
         try {
-            touchIndexVersion();
+            const diskRaw = JSON.parse(fs_1.default.readFileSync(file, 'utf8'));
+            const postCheck = (0, loaderSchemaValidator_1.validateForDisk)(diskRaw);
+            if (!postCheck.valid) {
+                (0, logger_js_1.logWarn)(`[writeEntryAsync] Post-write validation FAILED for '${entry.id}': ${postCheck.errors?.join('; ')}`);
+            }
+        }
+        catch (readErr) {
+            (0, logger_js_1.logWarn)(`[writeEntryAsync] Post-write read-back failed for '${entry.id}': ${readErr.message}`);
         }
-        catch { /* ignore */ }
     }
+    materializeWrittenEntry(record);
 }
 function removeEntry(id) {
     const store = getStoreForDir(getInstructionsDir());
@@ -866,12 +1052,12 @@ function incrementUsage(id, opts) {
     // Atomically establish firstSeenTs if missing (avoid any window where undefined persists after increment)
     if (!e.firstSeenTs) {
         e.firstSeenTs = nowIso;
-        ephemeralFirstSeen[e.id] = e.firstSeenTs; // track immediately for reload resilience
+        ephemeralFirstSeen[e.id] = e.firstSeenTs; // track immediately for reload resilience  // lgtm[js/remote-property-injection] — id is schema-validated before reaching index
         firstSeenAuthority[e.id] = e.firstSeenTs;
-        (0, features_1.incrementCounter)('usage:firstSeenAuthoritySet');
+        (0, features_1.incrementCounter)('usage:firstSeenAuthoritySet'); // lgtm[js/remote-property-injection] — id is schema-validated before reaching index
     }
     e.lastUsedAt = nowIso; // always advance lastUsedAt on any increment
-    lastUsedAuthority[e.id] = e.lastUsedAt;
+    lastUsedAuthority[e.id] = e.lastUsedAt; // lgtm[js/remote-property-injection] — id is schema-validated before reaching index
     // For the first usage we force a synchronous flush to guarantee persistence of firstSeenTs quickly;
     // subsequent usages can rely on the debounce timer to coalesce writes.
     if (e.usageCount <= 2) {
@@ -892,8 +1078,7 @@ function incrementUsage(id, opts) {
         // Allow tests (or advanced operators) to disable the protective clamp logic for deterministic expectations.
         // Setting INDEX_SERVER_DISABLE_USAGE_CLAMP=1 will let the anomalous >1 initial count pass through for diagnostic visibility.
         if (!(0, runtimeConfig_1.getRuntimeConfig)().index.disableUsageClamp) {
-            // eslint-disable-next-line no-console
-            console.error('[incrementUsage] anomalous initial usageCount', e.usageCount, 'id', id);
+            (0, logger_js_1.logError)('[incrementUsage] anomalous initial usageCount', { usageCount: e.usageCount, id });
             // Clamp to 1 to enforce deterministic semantics for first observed increment. We intentionally
             // retain lastUsedAt/firstSeenTs. This guards rare race producing flaky test expectations while
             // preserving forward progress for subsequent increments (next call will yield 2).
@@ -969,13 +1154,13 @@ function __testResetUsageState() {
     usageRateLimiter.clear();
     lastGoodUsageSnapshot = {};
     for (const k of Object.keys(ephemeralFirstSeen))
-        delete ephemeralFirstSeen[k];
+        delete ephemeralFirstSeen[k]; // lgtm[js/remote-property-injection] — k is own-key from internal object reset (test helper)
     for (const k of Object.keys(firstSeenAuthority))
-        delete firstSeenAuthority[k];
+        delete firstSeenAuthority[k]; // lgtm[js/remote-property-injection] — k is own-key from internal object reset (test helper)
     for (const k of Object.keys(usageAuthority))
-        delete usageAuthority[k];
+        delete usageAuthority[k]; // lgtm[js/remote-property-injection] — k is own-key from internal object reset (test helper)
     for (const k of Object.keys(lastUsedAuthority))
-        delete lastUsedAuthority[k];
+        delete lastUsedAuthority[k]; // lgtm[js/remote-property-injection] — k is own-key from internal object reset (test helper)
     if (state) {
         for (const e of state.list) {
             // Reset optional usage-related fields; preserve object identity.

package/dist/services/indexLoader.d.ts

@@ -39,6 +39,7 @@ export declare class IndexLoader {
      * to momentary locks while another process is atomically renaming/writing.
      */
     private readJsonWithRetry;
+    loadAsync(): Promise<IndexLoadResult>;
     load(): IndexLoadResult;
     private computeindexHash;
 }

package/dist/services/indexLoader.js

@@ -19,6 +19,16 @@ const instruction_schema_json_1 = __importDefault(require("../../schemas/instruc
 const tracing_1 = require("./tracing");
 const runtimeConfig_1 = require("../config/runtimeConfig");
 const autoSplit_1 = require("./autoSplit");
+const logger_js_1 = require("./logger.js");
+function sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}
+function getRetryBackoffMs(baseBackoff, attempt) {
+    return baseBackoff * Math.pow(2, attempt - 1) + Math.floor(Math.random() * baseBackoff);
+}
+function isRetryableLoadError(error) {
+    return /empty file transient|unexpected end of json input|eprem|ebusy|eacces|enoent/i.test(error);
+}
 class IndexLoader {
     baseDir;
     classifier;
@@ -32,9 +42,8 @@ class IndexLoader {
      * to momentary locks while another process is atomically renaming/writing.
      */
     readJsonWithRetry(file) {
-        const { attempts, backoffMs } = (0, runtimeConfig_1.getRuntimeConfig)().index.readRetries;
+        const { attempts } = (0, runtimeConfig_1.getRuntimeConfig)().index.readRetries;
         const maxAttempts = Math.max(1, attempts);
-        const baseBackoff = Math.max(1, backoffMs);
         let lastErr = null;
         for (let attempt = 1; attempt <= maxAttempts; attempt++) {
             try {
@@ -58,15 +67,26 @@ class IndexLoader {
                     break;
                 }
                 lastErr = err;
-                const sleep = baseBackoff * Math.pow(2, attempt - 1) + Math.floor(Math.random() * baseBackoff);
-                const start = Date.now();
-                while (Date.now() - start < sleep) { /* spin tiny backoff (< few ms) */ }
             }
         }
         if (lastErr)
             throw lastErr instanceof Error ? lastErr : new Error('readJsonWithRetry failed');
         return {}; // unreachable but satisfies typing
     }
+    async loadAsync() {
+        const { attempts, backoffMs } = (0, runtimeConfig_1.getRuntimeConfig)().index.readRetries;
+        const maxAttempts = Math.max(1, attempts);
+        const baseBackoff = Math.max(1, backoffMs);
+        for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+            const result = this.load();
+            const retryable = result.errors.length > 0 && result.errors.every(error => isRetryableLoadError(error.error));
+            if (!retryable || attempt === maxAttempts) {
+                return result;
+            }
+            await sleep(getRetryBackoffMs(baseBackoff, attempt));
+        }
+        return this.load();
+    }
     load() {
         const runtimeConfig = (0, runtimeConfig_1.getRuntimeConfig)();
         const IndexConfig = runtimeConfig.index;
@@ -126,7 +146,7 @@ class IndexLoader {
         }
         catch { /* ignore meta-schema registration issues */ }
         // Patch schema body maxLength from config before compiling (allows INDEX_SERVER_BODY_WARN_LENGTH override)
-        const bodyMaxLen = IndexConfig.bodyWarnLength || 100000;
+        const bodyMaxLen = IndexConfig.bodyWarnLength || 50000;
         const autoSplit = IndexConfig.autoSplitOversized === true;
         const schemaCopy = JSON.parse(JSON.stringify(instruction_schema_json_1.default));
         try {
@@ -672,7 +692,7 @@ class IndexLoader {
                         trace.push({ file: f, accepted: false, reason });
                     // Log schema rejections at info level for operational visibility
                     try {
-                        console.error(`[index:skip] ${f}: ${reason}`);
+                        (0, logger_js_1.logError)(`[index:skip] ${f}: ${reason}`);
                     }
                     catch { /* ignore */ }
                     if ((0, tracing_1.traceEnabled)(1)) {
@@ -698,7 +718,7 @@ class IndexLoader {
                     if (trace)
                         trace.push({ file: f, accepted: false, reason });
                     try {
-                        console.error(`[index:skip] ${f}: classification: ${reason}`);
+                        (0, logger_js_1.logError)(`[index:skip] ${f}: classification: ${reason}`);
                     }
                     catch { /* ignore */ }
                     if ((0, tracing_1.traceEnabled)(1)) {