@velaro/cli 1.4.96 → 1.4.98

package/lib/log-search.js

@@ -1,93 +1,200 @@
 // Complete Velaro Logger reads (2026-10-14).
 //
-// WHY: velaro-messaging's /superadmin/logs/search and /internal/logs/search silently under-report text queries.
+// WHY: velaro-messaging's /superadmin/logs/search and /internal/logs/search silently under-reported text queries.
 // Proven live that day: q=[HttpFallback] over 3.9 h returned 0-2 rows while 136 existed, and q=AsrsNegotiateFailed
-// (no level) returned 0 while 4 existed. The AI-search path only sees the newest 1,000 candidates and its answer was
-// final; the exact path only sees the newest 5,000 rows of the window before filtering. Neither said so.
+// (no level) returned 0 while 4 existed. The AI-search path only saw the newest 1,000 candidates and its answer was
+// final; the exact path only saw the newest 5,000 rows of the window before filtering. Neither said so.
 //
-// HOW (no q= ever sent): level/siteId/source/integration/platform/environment/correlationId are pushed to the
-// server (Table Storage OData, cheap and exact); includeTotal=true makes the server report totalCountCapped when a
-// slice hit its scan cap; a capped slice is bisected until none is capped; text (contains/regex/logger) is
-// filtered client side. The result says whether it is COMPLETE, so a 0 is never ambiguous.
+// TWO MODES, picked automatically from the first response:
+//  * CURSOR mode (server with velaro-messaging PR #2510, response carries `truncated`/`scannedRows`): the server scans the
+//    window newest-first applying level/site AND text (as an escaped regex, which also forces the exact path) while it
+//    scans, reports `truncated`, and returns `nextCursor` whenever more rows may exist. We follow the cursor until it is
+//    null, so a 2 h window is a handful of requests instead of hundreds.
+//  * BISECT mode (older server): never send q=; push level/site/etc server side; includeTotal=true reports
+//    totalCountCapped for a slice; a capped slice is bisected until none is capped; text is filtered client side.
+// Either way the result says COMPLETE or INCOMPLETE, so a 0 is never ambiguous. Hard caps: maxRequests and maxSeconds.
 //
 // KEEP IN SYNC: this file is duplicated byte for byte in mcp/log-search.js (the npm packages ship separately);
 // cli/lib/log-search.test.js fails if the two copies drift.
 
 const MIN_SLICE_MS = 2000;
 
+export const escapeRegex = (s) => String(s).replace(/[\\^$.*+?()[\]{}|]/g, '\\$&');
+
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+const tsOf = (e) => { const t = Date.parse(e.timestamp ?? e.LogTimestamp ?? ''); return Number.isNaN(t) ? 0 : t; };
+
+/**
+ * fetch that retries ONLY 429 (honouring Retry-After, seconds, capped at 30 s) for up to `retries` attempts and returns the final Response.
+ * `init` may be a function so each attempt gets a fresh AbortSignal.
+ */
+export async function fetchRetry429(url, init = {}, { retries = 3, fetchImpl = fetch, sleepImpl = sleep } = {}) {
+  for (let attempt = 0; ; attempt++) {
+    const res = await fetchImpl(url, typeof init === 'function' ? init() : init);
+    if (res.status === 429 && attempt < retries) {
+      // Number(null) is 0, which would turn a 429 without Retry-After into an immediate retry: a missing header means the 5 s default.
+      const raw = res.headers?.get?.('retry-after');
+      const ra = raw == null || raw === '' ? NaN : Number(raw);
+      await sleepImpl(Math.min(Number.isFinite(ra) && ra >= 0 ? ra : 5, 30) * 1000);
+      continue;
+    }
+    return res;
+  }
+}
+
+/** fetchRetry429 + JSON. Any non-2xx throws an Error carrying `.status` and the response text. */
+export async function fetchJsonRetry(url, init = {}, opts = {}) {
+  const res = await fetchRetry429(url, init, opts);
+  if (!res.ok) {
+    const err = new Error(`HTTP ${res.status}: ${(await res.text().catch(() => '')).slice(0, 400)}`);
+    err.status = res.status;
+    throw err;
+  }
+  return res.json();
+}
+
 /**
  * @param {object} o
- * @param {(query: string) => Promise<any>} o.fetchJson  GET `...search?<query>` and return parsed JSON (caller adds auth)
+ * @param {(query: string) => Promise<any>} o.fetchJson  GET `...search?<query>` and return parsed JSON (caller adds auth/timeout)
  * @param {Date} o.from
  * @param {Date} o.to
  * @param {object} [o.params]  server-side filters: level, siteId, source, integration, platform, environment, correlationId
- * @param {string} [o.contains] case-insensitive substring of the message (client side)
- * @param {string} [o.regex]    regex on the message (client side)
- * @param {string} [o.logger]   substring of the logger name (client side)
+ * @param {string} [o.contains] case-insensitive substring of the message
+ * @param {string} [o.regex]    regex on the message
+ * @param {string} [o.logger]   substring of the logger name
  * @param {number} [o.maxRequests=200]
+ * @param {number} [o.maxSeconds=120]   wall-clock budget for the whole read
+ * @param {number} [o.wanted=Infinity]  CURSOR mode: stop once this many matches were collected (newest first); the result is then `satisfied`
  */
-export async function completeLogSearch({ fetchJson, from, to, params = {}, contains, regex, logger, maxRequests = 200 }) {
+export async function completeLogSearch({ fetchJson, from, to, params = {}, contains, regex, logger, maxRequests = 200, maxSeconds = 120, wanted = Infinity }) {
   const server = {};
   for (const k of ['level', 'siteId', 'source', 'integration', 'platform', 'environment', 'correlationId']) {
     if (params[k] !== undefined && params[k] !== null && params[k] !== '') server[k] = String(k === 'level' ? String(params[k]).toUpperCase() : params[k]);
   }
   const byId = new Map();
   const unresolved = [];
+  const started = Date.now();
   let requests = 0;
   let slices = 0;
   let scanned = 0;
   let budgetExhausted = false;
+  let timeExhausted = false;
 
-  const call = async (a, b, cursor, total) => {
-    if (requests >= maxRequests) { budgetExhausted = true; return null; }
-    requests++;
-    const q = new URLSearchParams({ from: a.toISOString(), to: b.toISOString(), take: '500', ...server });
-    if (total) q.set('includeTotal', 'true');
+  const canRequest = () => {
+    if (requests >= maxRequests) { budgetExhausted = true; return false; }
+    if ((Date.now() - started) / 1000 >= maxSeconds && requests > 0) { timeExhausted = true; return false; }
+    return true;
+  };
+  const norm = (r) => (Array.isArray(r) ? { entries: r } : (r || {}));
+  const add = (e) => byId.set(e.id ?? `${e.timestamp}|${e.message}`, e);
+
+  // ---- text handling shared by both modes ----
+  const textQ = regex ?? (contains ? escapeRegex(contains) : undefined);
+  const textLogger = logger ? escapeRegex(logger) : undefined;
+
+  // ---- probe: first CURSOR-mode page, doubles as the new-server detector ----
+  const cursorQuery = (cursor) => {
+    const q = new URLSearchParams({ from: from.toISOString(), to: to.toISOString(), take: '500', ...server });
+    if (textQ) q.set('q', textQ);
+    if (textLogger) q.set('logger', textLogger);
+    if (textQ || textLogger) q.set('regex', 'true'); // regex also forces the exact path (the AI-search path is never consulted)
+    q.set('fallthrough', 'false');
     if (cursor) q.set('cursor', cursor);
-    const r = await fetchJson(q.toString());
-    return Array.isArray(r) ? { entries: r } : (r || {});
+    return q.toString();
   };
 
-  const walk = async (a, b) => {
-    const r = await call(a, b, null, true);
-    if (!r) { unresolved.push(`${a.toISOString()}..${b.toISOString()} (request budget)`); return; }
-    if (r.totalCountCapped === true && (b - a) > MIN_SLICE_MS) {
-      const mid = new Date(a.getTime() + Math.floor((b - a) / 2));
-      await walk(a, mid);
-      await walk(mid, b);
-      return;
-    }
-    if (r.totalCountCapped === true) unresolved.push(`${a.toISOString()}..${b.toISOString()} (still capped at <=2s)`);
-    slices++;
-    scanned += Number(r.totalCount ?? (r.entries?.length ?? 0));
-    for (const e of r.entries ?? []) byId.set(e.id ?? `${e.timestamp}|${e.message}`, e);
-    let cursor = r.nextCursor;
-    while (cursor) {
-      const r2 = await call(a, b, cursor, false);
-      if (!r2) { unresolved.push(`${a.toISOString()}..${b.toISOString()} (request budget while paging)`); break; }
-      for (const e of r2.entries ?? []) byId.set(e.id ?? `${e.timestamp}|${e.message}`, e);
-      cursor = r2.nextCursor;
+  let mode = 'bisect';
+  let satisfied = false;
+  let cursorDone = false;
+
+  if (canRequest()) {
+    requests++;
+    const probe = norm(await fetchJson(cursorQuery(null)));
+    if (typeof probe.truncated === 'boolean') {
+      mode = 'cursor';
+      scanned += Number(probe.scannedRows ?? 0);
+      for (const e of probe.entries ?? []) add(e);
+      let page = probe;
+      let cursor = probe.nextCursor || null;
+      for (;;) {
+        // The ONLY way a cursor read is COMPLETE: the last page said truncated === false and carried no cursor. `truncated: true` with no
+        // cursor means the server stopped scanning (deadline/cap) and could not say where to continue: INCOMPLETE, never COMPLETE.
+        if (!cursor) {
+          if (page.truncated === false) cursorDone = true;
+          else unresolved.push('the server stopped scanning (truncated) without a cursor to continue from');
+          break;
+        }
+        if (byId.size >= wanted) { satisfied = true; break; }
+        if (!canRequest()) { unresolved.push(`cursor ${cursor} (${timeExhausted ? 'time budget' : 'request budget'})`); break; }
+        requests++;
+        const next = norm(await fetchJson(cursorQuery(cursor)));
+        scanned += Number(next.scannedRows ?? 0);
+        for (const e of next.entries ?? []) add(e);
+        if (next.nextCursor && next.nextCursor === cursor) {
+          unresolved.push('no progress: the server returned the cursor it was given (its scan budget ended before reaching any row)');
+          break;
+        }
+        page = next;
+        cursor = next.nextCursor || null;
+      }
+      slices = 1;
     }
-  };
+  }
 
-  await walk(from, to);
+  // ---- BISECT mode (older server) ----
+  if (mode === 'bisect' && !budgetExhausted) {
+    const call = async (a, b, cursor, total) => {
+      if (!canRequest()) return null;
+      requests++;
+      const q = new URLSearchParams({ from: a.toISOString(), to: b.toISOString(), take: '500', ...server });
+      if (total) q.set('includeTotal', 'true');
+      if (cursor) q.set('cursor', cursor);
+      return norm(await fetchJson(q.toString()));
+    };
+    const walk = async (a, b) => {
+      const r = await call(a, b, null, true);
+      if (!r) { unresolved.push(`${a.toISOString()}..${b.toISOString()} (${timeExhausted ? 'time' : 'request'} budget)`); return; }
+      if (r.totalCountCapped === true && (b - a) > MIN_SLICE_MS) {
+        const mid = new Date(a.getTime() + Math.floor((b - a) / 2));
+        await walk(a, mid);
+        await walk(mid, b);
+        return;
+      }
+      if (r.totalCountCapped === true) unresolved.push(`${a.toISOString()}..${b.toISOString()} (still capped at <=2s: a log storm; add level/site)`);
+      slices++;
+      scanned += Number(r.totalCount ?? (r.entries?.length ?? 0));
+      for (const e of r.entries ?? []) add(e);
+      let cursor = r.nextCursor;
+      while (cursor) {
+        const r2 = await call(a, b, cursor, false);
+        if (!r2) { unresolved.push(`${a.toISOString()}..${b.toISOString()} (budget while paging)`); break; }
+        for (const e of r2.entries ?? []) add(e);
+        cursor = r2.nextCursor;
+      }
+    };
+    await walk(from, to);
+  }
 
   let rows = [...byId.values()];
   const msg = (e) => String(e.message ?? e.Message ?? '');
-  if (contains) { const c = contains.toLowerCase(); rows = rows.filter((e) => msg(e).toLowerCase().includes(c)); }
-  if (regex) { const re = new RegExp(regex, 'i'); rows = rows.filter((e) => re.test(msg(e))); }
-  if (logger) { const l = logger.toLowerCase(); rows = rows.filter((e) => String(e.logger ?? e.Logger ?? '').toLowerCase().includes(l)); }
-  rows.sort((x, y) => String(y.timestamp ?? '').localeCompare(String(x.timestamp ?? '')));
+  if (mode === 'bisect') {
+    if (contains) { const c = contains.toLowerCase(); rows = rows.filter((e) => msg(e).toLowerCase().includes(c)); }
+    if (regex) { const re = new RegExp(regex, 'i'); rows = rows.filter((e) => re.test(msg(e))); }
+    if (logger) { const l = logger.toLowerCase(); rows = rows.filter((e) => String(e.logger ?? e.Logger ?? '').toLowerCase().includes(l)); }
+  }
+  rows.sort((x, y) => tsOf(y) - tsOf(x)); // compare as instants: fractional-second digit counts differ between rows
 
-  return { entries: rows, fetched: byId.size, scanned, requests, slices, complete: unresolved.length === 0 && !budgetExhausted, unresolved, budgetExhausted };
+  const complete = mode === 'cursor' ? (cursorDone && unresolved.length === 0) : (unresolved.length === 0 && !budgetExhausted && !timeExhausted);
+  return { entries: rows, fetched: byId.size, scanned, requests, slices, mode, complete, satisfied, unresolved, budgetExhausted, timeExhausted };
 }
 
-/** One line a human or an LLM can trust: says COMPLETE or INCOMPLETE and how much was examined. */
+/** One line a human or an LLM can trust: says COMPLETE or INCOMPLETE and how much the SERVER actually scanned. */
 export function describeCompleteness(r) {
-  const base = `${r.entries.length} matched of ${r.fetched} fetched (server scanned ${r.scanned}) in ${r.requests} request(s)/${r.slices} slice(s)`;
+  const base = `${r.entries.length} matched (${r.fetched} fetched; server scanned ${r.scanned} rows) in ${r.requests} request(s), ${r.mode} mode`;
+  if (r.satisfied) return `${base}; showing the newest matches, more may exist beyond what was asked for (stopped early on purpose).`;
   return r.complete
-    ? `${base}; COMPLETE (no slice hit the server scan cap), so 0 matches means nothing was logged.`
-    : `${base}; INCOMPLETE (${r.unresolved.length} slice(s) unresolved${r.budgetExhausted ? ', request budget exhausted' : ''}): narrow the window or add level/site; a short or empty result is NOT proof of absence.`;
+    ? `${base}; COMPLETE (the whole window was covered), so 0 matches means nothing was logged.`
+    : `${base}; INCOMPLETE (${r.unresolved.length} part(s) not covered${r.budgetExhausted ? ', request budget exhausted' : ''}${r.timeExhausted ? ', time budget exhausted' : ''}): narrow the window or add level/site; a short or empty result is NOT proof of absence.`;
 }
 
 /** For raw (server-side q=) responses: surfaces searchPath/scannedRows/truncated so a 0 is never ambiguous. */
@@ -96,5 +203,7 @@ export function describeRawMeta(resp) {
     return 'server did not report scan metadata (older deploy): treat an empty or short result as unverified, use the complete mode.';
   }
   const t = resp.truncated === true;
-  return `searchPath=${resp.searchPath} scannedRows=${resp.scannedRows} truncated=${t}${t ? ' (NOT proof of absence; continue with nextCursor or narrow the window)' : ''}${resp.nextCursor ? ' nextCursor=present' : ''}`;
+  // exact=false: the answer involves the AI-search index, which is best-effort (written behind, can lag, misses text that starts mid-word).
+  const bestEffort = resp.exact === false ? ' exact=false (answered from the best-effort search index: it can lag and misses text that starts mid-word, so treat counts as a floor, not exact)' : '';
+  return `searchPath=${resp.searchPath} scannedRows=${resp.scannedRows} truncated=${t}${t ? ' (NOT proof of absence; continue with nextCursor or narrow the window)' : ''}${bestEffort}${resp.nextCursor ? ' nextCursor=present' : ''}`;
 }