@velaro/cli 1.4.95 → 1.4.96

package/lib/commands/logs.js

@@ -1,5 +1,6 @@
 import { getCredentials }          from '../api.js';
 import { readConfig, ENVS }        from '../config.js';
+import { completeLogSearch, describeCompleteness, describeRawMeta } from '../log-search.js';
 
 // ── helpers ──────────────────────────────────────────────────────────────────
 
@@ -60,39 +61,61 @@ async function searchLogs(argv) {
   await requireAdmin(creds);
   const msgBase = await getMsgBase(argv);
 
-  const from = new Date(Date.now() - parseDuration(argv.last ?? '2h'));
-  const params = new URLSearchParams({ from: from.toISOString(), take: String(argv.take ?? 50) });
-  if (argv.query)         params.set('q',             argv.query);
-  if (argv.level)         params.set('level',         argv.level.toUpperCase());
-  if (argv.integration)   params.set('integration',   argv.integration);
-  if (argv.site)          params.set('siteId',        String(argv.site));
-  if (argv.source)        params.set('source',        argv.source);
-  if (argv.platform)      params.set('platform',      argv.platform);
-  if (argv.environment)   params.set('environment',   argv.environment);
-  if (argv.correlationId) params.set('correlationId', argv.correlationId);
-  if (argv.regex)         params.set('regex',         'true');
-
-  const res = await fetch(`${msgBase}/superadmin/logs/search?${params}`, {
-    headers: {
-      Authorization:       `Bearer ${creds.velaroToken}`,
-      'X-Internal-SiteId': String(creds.siteId),
-    },
-    signal: AbortSignal.timeout(20000),
-  });
-
-  if (!res.ok) {
-    console.error(`HTTP ${res.status}: ${await res.text().catch(() => '')}`);
+  const to   = argv.to   ? new Date(argv.to)   : new Date();
+  const from = argv.from ? new Date(argv.from) : new Date(to.getTime() - parseDuration(argv.last ?? '2h'));
+  if (Number.isNaN(from.getTime()) || Number.isNaN(to.getTime()) || to <= from) {
+    console.error('Invalid time window: --from/--to must be ISO timestamps with --to after --from.');
     process.exit(1);
   }
+  const headers = { Authorization: `Bearer ${creds.velaroToken}`, 'X-Internal-SiteId': String(creds.siteId) };
+  const getJson = async (query) => {
+    const res = await fetch(`${msgBase}/superadmin/logs/search?${query}`, { headers, signal: AbortSignal.timeout(30000) });
+    if (!res.ok) {
+      console.error(`HTTP ${res.status}: ${await res.text().catch(() => '')}`);
+      process.exit(1);
+    }
+    return res.json();
+  };
 
-  const entries = extractLogEntries(await res.json());
-  if (!entries.length) {
-    console.log('No results.');
+  // --raw: the server's own q= search (kept for parity). Proven 2026-10-14 to under-report (q=[HttpFallback]
+  // returned 0-2 rows while 136 existed), so it now always prints the server's searchPath/scannedRows/truncated.
+  if (argv.raw) {
+    const params = new URLSearchParams({ from: from.toISOString(), to: to.toISOString(), take: String(argv.take ?? 50) });
+    if (argv.query)         params.set('q',             argv.query);
+    if (argv.level)         params.set('level',         argv.level.toUpperCase());
+    if (argv.integration)   params.set('integration',   argv.integration);
+    if (argv.site)          params.set('siteId',        String(argv.site));
+    if (argv.source)        params.set('source',        argv.source);
+    if (argv.platform)      params.set('platform',      argv.platform);
+    if (argv.environment)   params.set('environment',   argv.environment);
+    if (argv.correlationId) params.set('correlationId', argv.correlationId);
+    if (argv.regex)         params.set('regex',         'true');
+    const resp = await getJson(params.toString());
+    const entries = extractLogEntries(resp);
+    console.log(`${DIM}${describeRawMeta(resp)}${RESET}`);
+    if (!entries.length) { console.log('No results (raw server search; see the line above before trusting that).'); return; }
+    console.log(`\n${DIM}${entries.length} result(s) from ${from.toISOString().slice(0, 16)} UTC${RESET}\n`);
+    for (const e of entries) printEntry(e);
+    console.log('');
     return;
   }
 
-  console.log(`\n${DIM}${entries.length} result(s) from ${from.toISOString().slice(0, 16)} UTC${RESET}\n`);
-  for (const e of entries) printEntry(e);
+  // Default: COMPLETE read. Never sends q=: filters level/site/source/... server side, bisects any slice the
+  // server reports as capped (includeTotal), filters text client side, and says COMPLETE or INCOMPLETE.
+  const result = await completeLogSearch({
+    fetchJson: getJson,
+    from, to,
+    params: { level: argv.level, siteId: argv.site, source: argv.source, integration: argv.integration, platform: argv.platform, environment: argv.environment, correlationId: argv.correlationId },
+    contains: argv.regex ? undefined : argv.query,
+    regex:    argv.regex ? argv.query : undefined,
+    logger:   argv.logger,
+    maxRequests: argv.maxRequests ?? 200,
+  });
+  console.log(`${result.complete ? DIM : YEL}${describeCompleteness(result)}${RESET}`);
+  if (!result.entries.length) { console.log('No results.'); return; }
+  const shown = result.entries.slice(0, argv.take ?? 50);
+  console.log(`\n${DIM}showing ${shown.length} of ${result.entries.length} (newest first) from ${from.toISOString().slice(0, 16)} UTC${RESET}\n`);
+  for (const e of shown) printEntry(e);
   console.log('');
 }
 
@@ -696,7 +719,12 @@ export const logsCommand = {
         .option('correlationId', { describe: 'Exact correlation/trace ID filter', type: 'string' })
         .option('last',          { describe: 'Time window: 30m, 2h, 1d', default: '2h', type: 'string' })
         .option('regex',         { describe: 'Treat query as regex', boolean: true, default: false })
-        .option('take',          { describe: 'Max results (max 500)', default: 50, type: 'number' })
+        .option('logger',        { describe: 'Logger-name substring (client side)', type: 'string' })
+        .option('from',          { describe: 'ISO start time (overrides --last)', type: 'string' })
+        .option('to',            { describe: 'ISO end time (default now)', type: 'string' })
+        .option('raw',           { describe: 'Use the server q= search (can under-report; prints searchPath/scannedRows/truncated)', boolean: true, default: false })
+        .option('max-requests',  { describe: 'Request budget for the complete read', default: 200, type: 'number' })
+        .option('take',          { describe: 'Max lines to print (complete read) / server page size (--raw, max 500)', default: 50, type: 'number' })
         .option('env',           { describe: 'staging or prod', type: 'string' }),
       handler: searchLogs,
     })

package/lib/log-search.js

@@ -0,0 +1,100 @@
+// Complete Velaro Logger reads (2026-10-14).
+//
+// WHY: velaro-messaging's /superadmin/logs/search and /internal/logs/search silently under-report text queries.
+// Proven live that day: q=[HttpFallback] over 3.9 h returned 0-2 rows while 136 existed, and q=AsrsNegotiateFailed
+// (no level) returned 0 while 4 existed. The AI-search path only sees the newest 1,000 candidates and its answer was
+// final; the exact path only sees the newest 5,000 rows of the window before filtering. Neither said so.
+//
+// HOW (no q= ever sent): level/siteId/source/integration/platform/environment/correlationId are pushed to the
+// server (Table Storage OData, cheap and exact); includeTotal=true makes the server report totalCountCapped when a
+// slice hit its scan cap; a capped slice is bisected until none is capped; text (contains/regex/logger) is
+// filtered client side. The result says whether it is COMPLETE, so a 0 is never ambiguous.
+//
+// KEEP IN SYNC: this file is duplicated byte for byte in mcp/log-search.js (the npm packages ship separately);
+// cli/lib/log-search.test.js fails if the two copies drift.
+
+const MIN_SLICE_MS = 2000;
+
+/**
+ * @param {object} o
+ * @param {(query: string) => Promise<any>} o.fetchJson  GET `...search?<query>` and return parsed JSON (caller adds auth)
+ * @param {Date} o.from
+ * @param {Date} o.to
+ * @param {object} [o.params]  server-side filters: level, siteId, source, integration, platform, environment, correlationId
+ * @param {string} [o.contains] case-insensitive substring of the message (client side)
+ * @param {string} [o.regex]    regex on the message (client side)
+ * @param {string} [o.logger]   substring of the logger name (client side)
+ * @param {number} [o.maxRequests=200]
+ */
+export async function completeLogSearch({ fetchJson, from, to, params = {}, contains, regex, logger, maxRequests = 200 }) {
+  const server = {};
+  for (const k of ['level', 'siteId', 'source', 'integration', 'platform', 'environment', 'correlationId']) {
+    if (params[k] !== undefined && params[k] !== null && params[k] !== '') server[k] = String(k === 'level' ? String(params[k]).toUpperCase() : params[k]);
+  }
+  const byId = new Map();
+  const unresolved = [];
+  let requests = 0;
+  let slices = 0;
+  let scanned = 0;
+  let budgetExhausted = false;
+
+  const call = async (a, b, cursor, total) => {
+    if (requests >= maxRequests) { budgetExhausted = true; return null; }
+    requests++;
+    const q = new URLSearchParams({ from: a.toISOString(), to: b.toISOString(), take: '500', ...server });
+    if (total) q.set('includeTotal', 'true');
+    if (cursor) q.set('cursor', cursor);
+    const r = await fetchJson(q.toString());
+    return Array.isArray(r) ? { entries: r } : (r || {});
+  };
+
+  const walk = async (a, b) => {
+    const r = await call(a, b, null, true);
+    if (!r) { unresolved.push(`${a.toISOString()}..${b.toISOString()} (request budget)`); return; }
+    if (r.totalCountCapped === true && (b - a) > MIN_SLICE_MS) {
+      const mid = new Date(a.getTime() + Math.floor((b - a) / 2));
+      await walk(a, mid);
+      await walk(mid, b);
+      return;
+    }
+    if (r.totalCountCapped === true) unresolved.push(`${a.toISOString()}..${b.toISOString()} (still capped at <=2s)`);
+    slices++;
+    scanned += Number(r.totalCount ?? (r.entries?.length ?? 0));
+    for (const e of r.entries ?? []) byId.set(e.id ?? `${e.timestamp}|${e.message}`, e);
+    let cursor = r.nextCursor;
+    while (cursor) {
+      const r2 = await call(a, b, cursor, false);
+      if (!r2) { unresolved.push(`${a.toISOString()}..${b.toISOString()} (request budget while paging)`); break; }
+      for (const e of r2.entries ?? []) byId.set(e.id ?? `${e.timestamp}|${e.message}`, e);
+      cursor = r2.nextCursor;
+    }
+  };
+
+  await walk(from, to);
+
+  let rows = [...byId.values()];
+  const msg = (e) => String(e.message ?? e.Message ?? '');
+  if (contains) { const c = contains.toLowerCase(); rows = rows.filter((e) => msg(e).toLowerCase().includes(c)); }
+  if (regex) { const re = new RegExp(regex, 'i'); rows = rows.filter((e) => re.test(msg(e))); }
+  if (logger) { const l = logger.toLowerCase(); rows = rows.filter((e) => String(e.logger ?? e.Logger ?? '').toLowerCase().includes(l)); }
+  rows.sort((x, y) => String(y.timestamp ?? '').localeCompare(String(x.timestamp ?? '')));
+
+  return { entries: rows, fetched: byId.size, scanned, requests, slices, complete: unresolved.length === 0 && !budgetExhausted, unresolved, budgetExhausted };
+}
+
+/** One line a human or an LLM can trust: says COMPLETE or INCOMPLETE and how much was examined. */
+export function describeCompleteness(r) {
+  const base = `${r.entries.length} matched of ${r.fetched} fetched (server scanned ${r.scanned}) in ${r.requests} request(s)/${r.slices} slice(s)`;
+  return r.complete
+    ? `${base}; COMPLETE (no slice hit the server scan cap), so 0 matches means nothing was logged.`
+    : `${base}; INCOMPLETE (${r.unresolved.length} slice(s) unresolved${r.budgetExhausted ? ', request budget exhausted' : ''}): narrow the window or add level/site; a short or empty result is NOT proof of absence.`;
+}
+
+/** For raw (server-side q=) responses: surfaces searchPath/scannedRows/truncated so a 0 is never ambiguous. */
+export function describeRawMeta(resp) {
+  if (!resp || Array.isArray(resp) || resp.scannedRows === undefined) {
+    return 'server did not report scan metadata (older deploy): treat an empty or short result as unverified, use the complete mode.';
+  }
+  const t = resp.truncated === true;
+  return `searchPath=${resp.searchPath} scannedRows=${resp.scannedRows} truncated=${t}${t ? ' (NOT proof of absence; continue with nextCursor or narrow the window)' : ''}${resp.nextCursor ? ' nextCursor=present' : ''}`;
+}

package/lib/log-search.test.js

@@ -0,0 +1,129 @@
+// Regression (2026-10-14): velaro-messaging's logs search under-reported text queries (q=[HttpFallback] returned 0-2 rows
+// while 136 existed; q=AsrsNegotiateFailed with no level returned 0 while 4 existed) because each scan only examined the
+// newest N rows of the window and never said so. completeLogSearch must find rows behind an arbitrary pile of newer
+// rows, say COMPLETE only when no slice hit the server cap, and never send q=.
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { completeLogSearch, describeCompleteness, describeRawMeta } from './log-search.js';
+
+const here = path.dirname(fileURLToPath(import.meta.url));
+
+/** Mock of the messaging search endpoint: newest-first scan capped at `cap` rows BEFORE any filter other than level/site. */
+function mockServer(rows, cap = 5000) {
+  const calls = [];
+  const fetchJson = async (query) => {
+    const p = new URLSearchParams(query);
+    calls.push(p);
+    assert.equal(p.get('q'), null, 'the complete read must never send q=');
+    assert.equal(p.get('regex'), null);
+    const from = new Date(p.get('from')).getTime();
+    const to = new Date(p.get('to')).getTime();
+    const lvl = p.get('level');
+    const site = p.get('siteId');
+    // newest first, server-side filters first, then the fetch cap (the real bug), then in-window filter
+    let pool = rows.filter((r) => (!lvl || r.level === lvl) && (!site || String(r.siteId) === site));
+    pool = pool.filter((r) => new Date(r.timestamp).getTime() >= from && new Date(r.timestamp).getTime() <= to);
+    pool.sort((a, b) => b.timestamp.localeCompare(a.timestamp));
+    const capped = pool.length >= cap;
+    const fetched = pool.slice(0, cap);
+    const take = Number(p.get('take'));
+    let start = 0;
+    if (p.get('cursor')) start = fetched.findIndex((r) => r.id === p.get('cursor')) + 1;
+    const page = fetched.slice(start, start + take);
+    const more = start + take < fetched.length;
+    const out = { entries: page, nextCursor: more ? page[page.length - 1].id : null };
+    if (p.get('includeTotal') === 'true') { out.totalCount = fetched.length; out.totalCountCapped = capped; }
+    return out;
+  };
+  return { fetchJson, calls };
+}
+
+function makeRows(n, needles) {
+  const base = Date.UTC(2026, 9, 14, 13, 45, 0);
+  const rows = [];
+  for (let i = 0; i < n; i++) {
+    rows.push({ id: `r${String(i).padStart(7, '0')}`, timestamp: new Date(base - i * 1000).toISOString(), level: 'WARN', siteId: 20519, logger: 'VisitorLoggingService', message: needles.has(i) ? '[AsrsNegotiateFailed] site=20715' : `noise ${i}` });
+  }
+  return rows;
+}
+const W = (rows) => [new Date(rows[rows.length - 1].timestamp), new Date(rows[0].timestamp)];
+
+test('the single raw call misses a needle behind 9,000 newer rows (the live bug shape)', async () => {
+  const rows = makeRows(12000, new Set([9000]));
+  const { fetchJson } = mockServer(rows, 5000);
+  const [from, to] = W(rows);
+  const raw = await fetchJson(new URLSearchParams({ from: from.toISOString(), to: to.toISOString(), take: '500', includeTotal: 'true' }).toString());
+  assert.equal(raw.totalCountCapped, true);
+  assert.equal(raw.entries.some((e) => e.message.includes('AsrsNegotiate')), false);
+});
+
+test('completeLogSearch finds the needle behind 9,000 newer rows and reports COMPLETE', async () => {
+  const rows = makeRows(12000, new Set([9000, 11000]));
+  const { fetchJson } = mockServer(rows, 5000);
+  const [from, to] = W(rows);
+  const r = await completeLogSearch({ fetchJson, from, to, contains: '[AsrsNegotiateFailed]', maxRequests: 200 });
+  assert.equal(r.entries.length, 2);
+  assert.equal(r.complete, true);
+  assert.ok(r.slices > 1, 'capped slices must have been bisected');
+  assert.match(describeCompleteness(r), /COMPLETE/);
+  assert.doesNotMatch(describeCompleteness(r), /INCOMPLETE/);
+});
+
+test('a request budget that runs out is INCOMPLETE, never a silent partial', async () => {
+  const rows = makeRows(12000, new Set([9000]));
+  const { fetchJson } = mockServer(rows, 5000);
+  const [from, to] = W(rows);
+  const r = await completeLogSearch({ fetchJson, from, to, contains: '[AsrsNegotiateFailed]', maxRequests: 2 });
+  assert.equal(r.complete, false);
+  assert.equal(r.budgetExhausted, true);
+  assert.match(describeCompleteness(r), /INCOMPLETE/);
+  assert.match(describeCompleteness(r), /NOT proof of absence/);
+});
+
+test('level and siteId are pushed server side; contains/regex/logger filter client side', async () => {
+  const rows = makeRows(300, new Set([10, 20]));
+  rows[5].logger = 'Other.Logger';
+  const { fetchJson, calls } = mockServer(rows, 5000);
+  const [from, to] = W(rows);
+  const r = await completeLogSearch({ fetchJson, from, to, params: { level: 'warn', siteId: 20519 }, regex: 'Asrs.*Failed', logger: 'visitorlogging' });
+  assert.equal(r.entries.length, 2);
+  assert.equal(calls[0].get('level'), 'WARN');
+  assert.equal(calls[0].get('siteId'), '20519');
+  assert.equal(r.complete, true);
+});
+
+test('a genuine 0 on a complete scan says so (0 means nothing was logged)', async () => {
+  const rows = makeRows(500, new Set());
+  const { fetchJson } = mockServer(rows, 5000);
+  const [from, to] = W(rows);
+  const r = await completeLogSearch({ fetchJson, from, to, contains: 'nope' });
+  assert.equal(r.entries.length, 0);
+  assert.match(describeCompleteness(r), /0 matches means nothing was logged/);
+});
+
+test('describeRawMeta surfaces searchPath/scannedRows/truncated and warns on old deploys', () => {
+  assert.match(describeRawMeta({ entries: [], searchPath: 'ai-search', scannedRows: 1000, truncated: true }), /truncated=true.*NOT proof of absence/);
+  assert.match(describeRawMeta({ entries: [], searchPath: 'table-storage', scannedRows: 911, truncated: false }), /scannedRows=911 truncated=false/);
+  assert.match(describeRawMeta([]), /older deploy/);
+  assert.match(describeRawMeta({ entries: [] }), /older deploy/);
+});
+
+test('cli/lib/log-search.js and mcp/log-search.js are byte-identical (the npm packages ship separately)', () => {
+  const a = fs.readFileSync(path.join(here, 'log-search.js'), 'utf8');
+  const b = fs.readFileSync(path.join(here, '..', '..', 'mcp', 'log-search.js'), 'utf8');
+  assert.equal(a, b, 'copy cli/lib/log-search.js to mcp/log-search.js (or the reverse)');
+});
+
+test('neither the CLI search handler nor the MCP tool sends q= in complete mode', () => {
+  const cli = fs.readFileSync(path.join(here, 'commands', 'logs.js'), 'utf8');
+  const mcp = fs.readFileSync(path.join(here, '..', '..', 'mcp', 'staff-server.js'), 'utf8');
+  for (const [name, src, marker] of [['cli', cli, 'Default: COMPLETE read'], ['mcp', mcp, 'Default: COMPLETE read']]) {
+    const tail = src.slice(src.indexOf(marker));
+    const handler = tail.slice(0, 1200);
+    assert.ok(handler.includes('completeLogSearch('), `${name}: complete mode must call completeLogSearch`);
+    assert.ok(!/params\.set\('q'/.test(handler), `${name}: complete mode must not set q=`);
+  }
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@velaro/cli",
-  "version": "1.4.95",
+  "version": "1.4.96",
   "description": "Velaro Workspace v20 — command-line interface for managing bots, knowledge base ingestion, MCP API keys, search indexes, and ops health.",
   "type": "module",
   "bin": {