@chatpanel/gateway 0.6.43 → 0.6.44

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chatpanel/gateway",
-  "version": "0.6.43",
+  "version": "0.6.44",
   "description": "Local privacy gateway \u2014 redacts PII out of OpenAI/Anthropic API traffic before it reaches a model, then restores it in the reply. Point opencode, codex, aider, Claude Code, etc. at it.",
   "type": "module",
   "bin": {

package/src/mcp.js

@@ -33,7 +33,10 @@ const INSTRUCTIONS = [
   'meeting, call, demo, note, or past conversation ("outcome of the meeting", "what did we',
   'decide", "action items", "notes from yesterday", a person/day/topic in their history),',
   'ALSO consult ChatPanel — it is the source of truth for that personal history:',
-  '  • search_history — search by CONTENT (not the generic meeting title). Supports filters:',
+  '  • smart_search — START HERE: give it the question plus 2-4 of your own keyword',
+  '    phrasings; it runs them all and fuses the rankings, finding what one query misses.',
+  '  • search_history — one exact keyword query, when you already know the terms. Search by',
+  '    CONTENT (not the generic meeting title). Supports filters:',
   '    type (chat|meeting|note), since/before (dates or relative like "7d", "yesterday"),',
   '    and limit/offset paging. Returns compact snippets, not full bodies.',
   '  • get_record — the full text of one result id; use maxChars/offset to page a long',
@@ -103,6 +106,22 @@ async function bridgeJson(path) {
 }
 
 const TOOLS = [
+  {
+    name: 'smart_search',
+    description: 'BEST first choice for a question about the user\'s ChatPanel history (meetings, notes, past chats). Ask it a natural-language QUESTION and it expands that into several complementary keyword queries, runs them all, and fuses the rankings — which finds things a single query misses, in one round trip instead of several probes. You know the domain, so pass 2-4 of your own phrasings in `queries` too (e.g. for "what did we decide in the Ben demo": ["Ben demo decisions", "tooling demo action items", "demo outcome next steps"]). Supports the same filters as search_history (type, since, before) and returns snippets with each result\'s id; follow up with get_record for the full text or find_related to expand around a hit.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        question: { type: 'string', description: 'The user\'s question, in natural language.' },
+        queries: { type: 'array', items: { type: 'string' }, description: 'Your own 2-4 keyword formulations of it — these lead the search.' },
+        type: { type: 'string', enum: ['chat', 'meeting', 'note'], description: 'Only this kind of record.' },
+        since: { type: 'string', description: 'Earliest date: 2026-08-01, or a window like "7d"/"yesterday".' },
+        before: { type: 'string', description: 'Latest date: a date or window like `since`.' },
+        limit: { type: 'number', description: 'Max fused results (default 10).' },
+      },
+      required: ['question'],
+    },
+  },
   {
     name: 'search_history',
     description: 'Search the user\'s ChatPanel history — their past chats, meeting/call transcripts, and notes — by keyword relevance. Consult this (in ADDITION to your other tools) whenever the question touches a meeting, call, demo, note, or past conversation: "outcome of the meeting", "what did we decide", "action items", "what did <person> say", "notes from yesterday". Filters: `type` (chat|meeting|note), `since`/`before` (a date like 2026-08-01 or a relative window like "7d"/"yesterday"), and `limit`/`offset` for paging. Returns compact SNIPPETS (the matching excerpt) with each record\'s id/title/type/date — token-friendly; call get_record for the full text and find_related to follow connections. This is a LOCAL WARM COPY that syncs from ChatPanel; very recent items may not be here yet — results report how current the index is, so if something is missing it likely has not synced. Meeting titles are often generic ("Zoom Meeting"), so search by CONTENT, not the title.',
@@ -230,6 +249,30 @@ function parseWhen(v) {
 const fmtRow = (r, i) => `${i + 1}. [${r.id}] ${r.title || '(untitled)'} · ${r.type}${r.date ? ' · ' + new Date(r.date).toISOString().slice(0, 10) : ''}${r.snippet ? `\n     ${r.snippet}` : ''}`;
 
 async function callTool(name, args = {}) {
+  if (name === 'smart_search') {
+    const body = {
+      question: String(args.question || ''),
+      queries: Array.isArray(args.queries) ? args.queries.map(String) : [],
+      limit: Number(args.limit) || 10,
+    };
+    if (args.type) body.type = String(args.type);
+    const since = parseWhen(args.since); if (since != null) body.since = since;
+    const before = parseWhen(args.before); if (before != null) body.before = before;
+    const data = await gatewayJson('/v1/history/smart-search', {
+      method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify(body),
+    });
+    const rows = data.results || [];
+    const horizon = horizonLine(data.newest, data.size);
+    const asked = (data.queries || []).map((q) => `"${q}"`).join(', ');
+    if (!rows.length) {
+      return `No match for "${args.question}".\nSearched ${data.queries?.length || 0} way(s): ${asked}.\n${horizon}\nIf you expected a recent item it may not have synced yet — check ChatPanel directly, or try different keywords (meeting titles are often generic).`;
+    }
+    return [
+      horizon, '',
+      `${rows.length} result(s) for "${args.question}" — searched ${data.queries.length} way(s): ${asked}`,
+      ...rows.map((r, i) => `${fmtRow(r, i)}${r.foundBy?.length > 1 ? `\n     (matched ${r.foundBy.length} of the queries)` : ''}`),
+    ].join('\n') + '\n\nget_record <id> for full text (maxChars/offset to page) · find_related <id> to follow connections.';
+  }
   if (name === 'search_history') {
     const body = { query: String(args.query || ''), limit: Number(args.limit) || 10, offset: Math.max(0, Number(args.offset) || 0) };
     if (args.type) body.type = String(args.type);

package/src/rrf.js

@@ -0,0 +1,93 @@
+// VENDORED from @chatpanel/events/rrf.js — edit there, then copy over.
+// Same pattern as observability.js: one pure module copied in rather than pulling the whole
+// events package. Source of truth: chatpanel-events/rrf.js.
+//
+// rrf.js — Reciprocal Rank Fusion, and the query planning that feeds it.
+//
+// One question rarely makes one good keyword query. "What was the outcome of the Ben tooling
+// demo?" and "Ben demo decisions action items" retrieve different things, and the answer is
+// usually in the union. RRF merges several ranked lists without needing their scores to be
+// comparable — each list contributes 1/(k+rank) — which is exactly the situation here: BM25
+// scores, vector distances and a hot/warm split are all on different scales.
+//
+// Pure and dependency-free, so the identical fusion runs in the extension (hot+warm), the
+// gateway (multi-query search) and any future client. The extension had this privately; it
+// lives here now so the gateway doesn't grow a second, subtly different copy.
+
+/**
+ * Fuse ranked id lists. `lists` is an array of arrays of ids, each already in rank order.
+ * k dampens the head of each list (60 is the standard default). limit 0 = everything.
+ */
+export function fuseRRF(lists, { k = 60, limit = 0 } = {}) {
+  const score = new Map();
+  for (const list of lists || []) {
+    if (!Array.isArray(list)) continue;
+    list.forEach((id, rank) => {
+      if (id == null) return;
+      score.set(id, (score.get(id) || 0) + 1 / (k + rank));
+    });
+  }
+  const out = [...score.entries()].map(([id, s]) => ({ id, score: s })).sort((a, b) => b.score - a.score);
+  return limit > 0 ? out.slice(0, limit) : out;
+}
+
+// Words that carry no retrieval signal but do dilute BM25 — dropped to build a keyword-only
+// variant of a natural-language question.
+const STOP = new Set(('a an and are as at be been but by can could did do does for from had has have how i if in into is it its me my of on or our ought shall should '
+  + 'so than that the their them then there these they this those to um was we were what when where which who whom why will with would you your about tell show give find get '
+  + 'please could-you was-there did-we').split(/\s+/));
+
+/**
+ * Turn one natural-language question into a small set of complementary queries — the cheap,
+ * deterministic half of query expansion. No model call, so it costs nothing and cannot fail.
+ *
+ * A CALLING AGENT can do better (it understands the domain), which is why the tools accept an
+ * explicit `queries` list; these variants are the floor, not the ceiling.
+ */
+export function planQueries(question, { extra = [], max = 4 } = {}) {
+  const q = String(question || '').trim();
+  const out = [];
+  const seen = new Set();
+  const add = (s) => {
+    const t = String(s || '').trim().replace(/\s+/g, ' ');
+    const key = t.toLowerCase();
+    if (t && !seen.has(key)) { seen.add(key); out.push(t); }
+  };
+
+  add(q);                                    // the question as asked
+  for (const e of extra) add(e);             // whatever the agent proposed — it knows more
+
+  const words = q.toLowerCase().match(/[a-z0-9][a-z0-9'’_+-]*/g) || [];
+  const keywords = words.filter((w) => !STOP.has(w) && w.length > 2);
+  if (keywords.length >= 2) add(keywords.join(' '));          // keyword-only: BM25's best shape
+  // The rarest-looking terms (longest words are a decent proxy for specificity) — helps when
+  // the full question is too broad to rank anything well.
+  if (keywords.length > 3) add([...keywords].sort((a, b) => b.length - a.length).slice(0, 3).join(' '));
+
+  return out.slice(0, Math.max(1, max));
+}
+
+/**
+ * Run several queries through one `search(query, opts)` function and fuse the results by id.
+ * `search` returns arrays of { id, ... }; the fused output keeps the richest record seen for
+ * each id (so snippets survive) and reports which queries found it — the "why is this here"
+ * a reader needs when a multi-query search surfaces something unexpected.
+ */
+export async function multiSearch(queries, search, { limit = 10, k = 60 } = {}) {
+  const lists = [];
+  const byId = new Map();
+  const foundBy = new Map();
+  for (const q of queries || []) {
+    let rows = [];
+    try { rows = (await search(q)) || []; } catch { rows = []; } // one bad query must not sink the rest
+    lists.push(rows.map((r) => r.id));
+    for (const r of rows) {
+      if (!byId.has(r.id) || (!byId.get(r.id).snippet && r.snippet)) byId.set(r.id, r);
+      if (!foundBy.has(r.id)) foundBy.set(r.id, []);
+      foundBy.get(r.id).push(q);
+    }
+  }
+  return fuseRRF(lists, { k, limit }).map(({ id, score }) => ({
+    ...byId.get(id), id, score, foundBy: foundBy.get(id) || [],
+  }));
+}

package/src/server.js

@@ -43,11 +43,12 @@ import { publicConfig, applyConfigPatch, applyNerModelSelection, persistConfig,
 import { resolveDestination, aggregateModelsAsync } from './router.js';
 import { makeAccessEvent } from './observability.js';
 import { createPersistentAccessLog } from './access-log-store.js';
+import { planQueries, multiSearch } from './rrf.js';
 import * as openai from './openai.js';
 import * as responses from './responses.js';
 import * as anthropic from './anthropic.js';
 
-export const VERSION = '0.6.43';
+export const VERSION = '0.6.44';
 
 // WARM search tier — SQLite + FTS5 record store (falls back to an encrypted-JSON
 // store if SQLite can't load), fed by the extension's ingest sync + backup-ingest.
@@ -616,6 +617,38 @@ export function createGateway(cfg = loadConfig()) {
         return sendJson(res, 400, { error: { message: `search failed: ${e.message}`, type: 'search_error' } });
       }
     }
+    // SMART SEARCH — one round trip that expands the question into several complementary
+    // queries, runs them all, and RRF-fuses the results. A natural-language question is a
+    // poor BM25 query; asking three ways and fusing beats asking once, and doing it here
+    // means the agent pays one call instead of probing repeatedly.
+    if (pathname === '/v1/history/smart-search' && req.method === 'POST') {
+      try {
+        const body = JSON.parse((await readBody(req, cfg.maxBodyBytes)).toString('utf8')) || {};
+        const question = String(body.question || body.query || '');
+        // The agent's own formulations lead (it understands the domain); planQueries adds
+        // deterministic variants and dedupes.
+        const queries = planQueries(question, {
+          extra: Array.isArray(body.queries) ? body.queries.map(String) : [],
+          max: Math.min(6, Math.max(1, Number(body.maxQueries) || 4)),
+        });
+        const filters = {
+          type: body.type ? String(body.type) : null,
+          since: body.since != null ? Number(body.since) : null,
+          before: body.before != null ? Number(body.before) : null,
+        };
+        const perQuery = Math.min(30, Math.max(5, Number(body.limit) || 10) * 2);
+        const results = await multiSearch(
+          queries,
+          (q) => historyStore.search(q, { limit: perQuery, ...filters }),
+          { limit: Math.min(50, Math.max(1, Number(body.limit) || 10)) },
+        );
+        return sendJson(res, 200, {
+          ok: true, size: historyStore.size, newest: historyStore.newest, queries, results,
+        });
+      } catch (e) {
+        return sendJson(res, 400, { error: { message: `smart search failed: ${e.message}`, type: 'search_error' } });
+      }
+    }
     // Graph navigation — records most connected to a given one.
     if (pathname === '/v1/history/related' && req.method === 'GET') {
       const id = String(url.searchParams.get('id') || '');