@mnemonik/scanner 5.136.2 → 5.151.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mnemonik/scanner",
-  "version": "5.136.2",
+  "version": "5.151.0",
   "description": "Automatic codebase indexing daemon for Mnemonik",
   "type": "module",
   "bin": {
@@ -20,6 +20,6 @@
   "devDependencies": {
     "typescript": "^5.3.3",
     "@types/node": "^20.19.43",
-    "vitest": "^4.1.9"
+    "vitest": "^4.1.10"
   }
 }

package/src/client.ts

@@ -53,6 +53,16 @@ export interface GitMiningMeta {
 
 const SCAN_PUSH_BATCH_DELAY_MS = 500;
 
+/**
+ * Exponential backoff (base 2000ms × attempt) with ±25% jitter, to avoid a
+ * fleet of scanners retrying in lockstep after a shared 5xx/network blip
+ * (thundering herd). Base values unchanged from the pre-jitter constants.
+ */
+function backoffWithJitter(attempt: number): number {
+  const base = 2000 * (attempt + 1);
+  return Math.round(base * (0.75 + Math.random() * 0.5));
+}
+
 export class MnemonikClient {
   constructor(
     private serverUrl: string,
@@ -83,7 +93,7 @@ export class MnemonikClient {
             continue;
           }
           if (res.status >= 500 && attempt < retries) {
-            await new Promise((r) => setTimeout(r, 2000 * (attempt + 1)));
+            await new Promise((r) => setTimeout(r, backoffWithJitter(attempt)));
             continue;
           }
           throw new Error(`${res.status} ${res.statusText}: ${text}`);
@@ -92,7 +102,7 @@ export class MnemonikClient {
         return res.json() as Promise<T>;
       } catch (err) {
         if (attempt < retries && (err as Error).message?.includes('fetch failed')) {
-          await new Promise((r) => setTimeout(r, 2000 * (attempt + 1)));
+          await new Promise((r) => setTimeout(r, backoffWithJitter(attempt)));
           continue;
         }
         throw err;

package/dist/client.d.ts

@@ -1,99 +0,0 @@
-export interface ScanPushFile {
-    path: string;
-    hash: string;
-    chunks: Array<{
-        content: string;
-        startLine: number;
-        endLine: number;
-        chunkType: string;
-        language: string;
-        contentHash: string;
-        metadata: {
-            fileName: string;
-            extension: string;
-            size: number;
-        };
-    }>;
-    /**
-     * Optional raw file content. Sent by the scanner daemon for ALL files
-     * (code and authority/manifest alike) so the doc-truth worker can run
-     * claim extraction against whole documents without filesystem access.
-     * Omitted when the file exceeds the server's 5 MB schema cap or when
-     * the raw content could not be read (chunk-based fallback path).
-     */
-    content?: string;
-}
-export interface ScanPushCommit {
-    sha: string;
-    author: string;
-    date: string;
-    message: string;
-    files: string[];
-}
-export interface ScanPushPayload {
-    projectId: string;
-    files: ScanPushFile[];
-    commits?: ScanPushCommit[];
-}
-export interface ScanStatusResponse {
-    files: Array<{
-        path: string;
-        hash: string;
-    }>;
-    gitMining?: {
-        enabled: boolean;
-        lastMinedCommit: string | null;
-    };
-}
-export interface GitMiningMeta {
-    enabled: boolean;
-    lastMinedCommit: string | null;
-}
-export declare class MnemonikClient {
-    private serverUrl;
-    private apiKey;
-    constructor(serverUrl: string, apiKey: string);
-    private request;
-    /**
-     * Fetch per-file hashes for dedup, plus git-mining metadata.
-     * The daemon uses `gitMining.enabled` to decide whether to collect commits
-     * this cycle, and `lastMinedCommit` as the lower bound of `git log`.
-     */
-    getStatus(projectId: string): Promise<{
-        fileHashes: Map<string, string>;
-        gitMining: GitMiningMeta;
-    }>;
-    /**
-     * Push file chunks in batches. When `commits` is supplied (non-empty) it is
-     * attached to the first batch only — BullMQ's idempotent jobId means a
-     * duplicate would collapse anyway, but one payload saves bandwidth.
-     *
-     * When `files` is empty and `commits` is non-empty, a single commit-only
-     * push is made — the server accepts `files=[]` since If both are
-     * empty, no request is sent.
-     */
-    pushFiles(projectId: string, files: ScanPushFile[], commits?: ScanPushCommit[]): Promise<{
-        success: boolean;
-    }>;
-    sendHeartbeat(projectId: string, scanner: {
-        scope: 'global';
-        version?: string;
-    }): Promise<void>;
-    /**
-     * Notify the server of files that have been removed since the daemon's
-     * previous scan of this project. Server deprecates exactly those code
-     * memories. Empty arrays are accepted as no-ops so the daemon can call
-     * this every tick regardless of whether anything was removed.
-     *
-     * The caller is responsible for computing the removal set locally — the
-     * old inventory-diff shape that asked the server to derive removals from
-     * a "known files" list has been removed because a small/malformed list
-     * would mass-deprecate. The narrow `removedFiles` shape cannot exhibit
-     * that failure mode by construction.
-     */
-    reportRemovedFiles(projectId: string, removedFiles: string[]): Promise<{
-        deprecated: number;
-        couplingsRemoved: number;
-    }>;
-    healthCheck(): Promise<boolean>;
-}

package/dist/client.js

@@ -1,131 +0,0 @@
-const SCAN_PUSH_BATCH_DELAY_MS = 500;
-export class MnemonikClient {
-    serverUrl;
-    apiKey;
-    constructor(serverUrl, apiKey) {
-        this.serverUrl = serverUrl;
-        this.apiKey = apiKey;
-    }
-    async request(path, body, retries = 3) {
-        const url = `${this.serverUrl}${path}`;
-        for (let attempt = 0; attempt <= retries; attempt++) {
-            try {
-                const res = await fetch(url, {
-                    method: 'POST',
-                    headers: {
-                        'Content-Type': 'application/json',
-                        Authorization: `Bearer ${this.apiKey}`,
-                    },
-                    body: JSON.stringify(body),
-                });
-                if (!res.ok) {
-                    const text = await res.text().catch(() => 'Unknown error');
-                    if (res.status === 401 || res.status === 403) {
-                        throw new Error(`Auth failed (${res.status}). Check your API key.`);
-                    }
-                    if (res.status === 503 && attempt < retries) {
-                        const retryAfter = parseInt(res.headers.get('Retry-After') || '5', 10);
-                        await new Promise((r) => setTimeout(r, retryAfter * 1000));
-                        continue;
-                    }
-                    if (res.status >= 500 && attempt < retries) {
-                        await new Promise((r) => setTimeout(r, 2000 * (attempt + 1)));
-                        continue;
-                    }
-                    throw new Error(`${res.status} ${res.statusText}: ${text}`);
-                }
-                return res.json();
-            }
-            catch (err) {
-                if (attempt < retries && err.message?.includes('fetch failed')) {
-                    await new Promise((r) => setTimeout(r, 2000 * (attempt + 1)));
-                    continue;
-                }
-                throw err;
-            }
-        }
-        throw new Error(`Request to ${path} failed after ${retries} retries`);
-    }
-    /**
-     * Fetch per-file hashes for dedup, plus git-mining metadata.
-     * The daemon uses `gitMining.enabled` to decide whether to collect commits
-     * this cycle, and `lastMinedCommit` as the lower bound of `git log`.
-     */
-    async getStatus(projectId) {
-        const result = await this.request('/api/v1/scan/status', { projectId });
-        return {
-            fileHashes: new Map(result.files.map((f) => [f.path, f.hash])),
-            gitMining: result.gitMining ?? { enabled: false, lastMinedCommit: null },
-        };
-    }
-    /**
-     * Push file chunks in batches. When `commits` is supplied (non-empty) it is
-     * attached to the first batch only — BullMQ's idempotent jobId means a
-     * duplicate would collapse anyway, but one payload saves bandwidth.
-     *
-     * When `files` is empty and `commits` is non-empty, a single commit-only
-     * push is made — the server accepts `files=[]` since If both are
-     * empty, no request is sent.
-     */
-    async pushFiles(projectId, files, commits) {
-        const hasCommits = !!commits && commits.length > 0;
-        if (files.length === 0) {
-            if (hasCommits) {
-                await this.request('/api/v1/scan/push', { projectId, files: [], commits });
-            }
-            return { success: true };
-        }
-        const batchSize = 25;
-        let attachedCommits = false;
-        for (let i = 0; i < files.length; i += batchSize) {
-            const batch = files.slice(i, i + batchSize);
-            const body = { projectId, files: batch };
-            if (!attachedCommits && hasCommits) {
-                body.commits = commits;
-                attachedCommits = true;
-            }
-            await this.request('/api/v1/scan/push', body);
-            if (i + batchSize < files.length) {
-                await sleep(SCAN_PUSH_BATCH_DELAY_MS);
-            }
-        }
-        return { success: true };
-    }
-    async sendHeartbeat(projectId, scanner) {
-        await this.request('/api/v1/scan/heartbeat', { projectId, scanner });
-    }
-    /**
-     * Notify the server of files that have been removed since the daemon's
-     * previous scan of this project. Server deprecates exactly those code
-     * memories. Empty arrays are accepted as no-ops so the daemon can call
-     * this every tick regardless of whether anything was removed.
-     *
-     * The caller is responsible for computing the removal set locally — the
-     * old inventory-diff shape that asked the server to derive removals from
-     * a "known files" list has been removed because a small/malformed list
-     * would mass-deprecate. The narrow `removedFiles` shape cannot exhibit
-     * that failure mode by construction.
-     */
-    async reportRemovedFiles(projectId, removedFiles) {
-        const result = await this.request('/api/v1/scan/reconcile', { projectId, removedFiles });
-        return {
-            deprecated: result.deprecated ?? 0,
-            couplingsRemoved: result.couplingsRemoved ?? 0,
-        };
-    }
-    async healthCheck() {
-        try {
-            const res = await fetch(`${this.serverUrl}/api/v1/health`, {
-                headers: { Authorization: `Bearer ${this.apiKey}` },
-            });
-            return res.ok;
-        }
-        catch {
-            return false;
-        }
-    }
-}
-function sleep(ms) {
-    return new Promise((resolve) => setTimeout(resolve, ms));
-}
-//# sourceMappingURL=client.js.map

package/dist/client.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"client.js","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAqDA,MAAM,wBAAwB,GAAG,GAAG,CAAC;AAErC,MAAM,OAAO,cAAc;IAEf;IACA;IAFV,YACU,SAAiB,EACjB,MAAc;QADd,cAAS,GAAT,SAAS,CAAQ;QACjB,WAAM,GAAN,MAAM,CAAQ;IACrB,CAAC;IAEI,KAAK,CAAC,OAAO,CAAI,IAAY,EAAE,IAAa,EAAE,OAAO,GAAG,CAAC;QAC/D,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,SAAS,GAAG,IAAI,EAAE,CAAC;QACvC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;YACpD,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;oBAC3B,MAAM,EAAE,MAAM;oBACd,OAAO,EAAE;wBACP,cAAc,EAAE,kBAAkB;wBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;qBACvC;oBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;iBAC3B,CAAC,CAAC;gBAEH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;oBACZ,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,eAAe,CAAC,CAAC;oBAC3D,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;wBAC7C,MAAM,IAAI,KAAK,CAAC,gBAAgB,GAAG,CAAC,MAAM,wBAAwB,CAAC,CAAC;oBACtE,CAAC;oBACD,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,IAAI,OAAO,GAAG,OAAO,EAAE,CAAC;wBAC5C,MAAM,UAAU,GAAG,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;wBACvE,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC;wBAC3D,SAAS;oBACX,CAAC;oBACD,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,IAAI,OAAO,GAAG,OAAO,EAAE,CAAC;wBAC3C,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,IAAI,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;wBAC9D,SAAS;oBACX,CAAC;oBACD,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,KAAK,IAAI,EAAE,CAAC,CAAC;gBAC9D,CAAC;gBAED,OAAO,GAAG,CAAC,IAAI,EAAgB,CAAC;YAClC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,OAAO,GAAG,OAAO,IAAK,GAAa,CAAC,OAAO,EAAE,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;oBAC1E,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,IAAI,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC9D,SAAS;gBACX,CAAC;gBACD,MAAM,GAAG,CAAC;YACZ,CAAC;QACH,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,cAAc,IAAI,iBAAiB,OAAO,UAAU,CAAC,CAAC;IACxE,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,SAAS,CACb,SAAiB;QAEjB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAqB,qBAAqB,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QAC5F,OAAO;YACL,UAAU,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;YAC9D,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,eAAe,EAAE,IAAI,EAAE;SACzE,CAAC;IACJ,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,SAAS,CACb,SAAiB,EACjB,KAAqB,EACrB,OAA0B;QAE1B,MAAM,UAAU,GAAG,CAAC,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACnD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,IAAI,UAAU,EAAE,CAAC;gBACf,MAAM,IAAI,CAAC,OAAO,CAAC,mBAAmB,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YAC7E,CAAC;YACD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QAC3B,CAAC;QAED,MAAM,SAAS,GAAG,EAAE,CAAC;QACrB,IAAI,eAAe,GAAG,KAAK,CAAC;QAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YACjD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;YAC5C,MAAM,IAAI,GAAoB,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;YAC1D,IAAI,CAAC,eAAe,IAAI,UAAU,EAAE,CAAC;gBACnC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;gBACvB,eAAe,GAAG,IAAI,CAAC;YACzB,CAAC;YACD,MAAM,IAAI,CAAC,OAAO,CAAC,mBAAmB,EAAE,IAAI,CAAC,CAAC;YAC9C,IAAI,CAAC,GAAG,SAAS,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;gBACjC,MAAM,KAAK,CAAC,wBAAwB,CAAC,CAAC;YACxC,CAAC;QACH,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,aAAa,CACjB,SAAiB,EACjB,OAA8C;QAE9C,MAAM,IAAI,CAAC,OAAO,CAAC,wBAAwB,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC;IACvE,CAAC;IAED;;;;;;;;;;;OAWG;IACH,KAAK,CAAC,kBAAkB,CACtB,SAAiB,EACjB,YAAsB;QAEtB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAI9B,wBAAwB,EAAE,EAAE,SAAS,EAAE,YAAY,EAAE,CAAC,CAAC;QAC1D,OAAO;YACL,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,CAAC;YAClC,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,IAAI,CAAC;SAC/C,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,WAAW;QACf,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,gBAAgB,EAAE;gBACzD,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE,EAAE;aACpD,CAAC,CAAC;YACH,OAAO,GAAG,CAAC,EAAE,CAAC;QAChB,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;CACF;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC"}

package/dist/daemon.d.ts

@@ -1,51 +0,0 @@
-export interface DaemonConfig {
-    serverUrl: string;
-    apiKey: string;
-    roots: string[];
-    refreshIntervalMs?: number;
-    maxConcurrentScans?: number;
-}
-export declare class ScannerDaemon {
-    private config;
-    private client;
-    private scanner;
-    private projects;
-    private refreshTimer;
-    private heartbeatTimer;
-    private discovery;
-    private refreshIntervalMs;
-    private maxConcurrentScans;
-    private scannerVersion;
-    constructor(config: DaemonConfig);
-    start(): Promise<void>;
-    stop(): Promise<void>;
-    private sendHeartbeats;
-    private getScannerVersion;
-    getWatchedProjects(): Array<{
-        projectId: string;
-        path: string;
-        name?: string;
-    }>;
-    /**
-     * Discover projects from configured roots and reconcile with current watch list.
-     */
-    refreshProjects(): Promise<void>;
-    private addProject;
-    private waitForServer;
-    private initialScan;
-    /**
-     * Send the explicit list of paths that vanished since the previous scan.
-     * Empty lists short-circuit so we don't pay an HTTP round-trip when
-     * nothing was removed this tick.
-     */
-    private sendRemovedFiles;
-    /**
-     * Fetch commits since the last mine. Swallows errors — a git failure must
-     * not block the file scan push.
-     */
-    private collectCommits;
-    private handleChanges;
-    private startRetryLoop;
-    private collectAuthorityPushFiles;
-    private groupChunksByFile;
-}