@3sln/trove 0.0.2

package/packages/core/src/search/transformer.js

@@ -0,0 +1,195 @@
+// SearchTransformer — turns a user's raw search string into the structured query we
+// actually dispatch: { semanticText, tagFilters }. The default just parses the
+// `#tag` / `#key:op:value` grammar out of the string (deterministic, offline-safe).
+// Inject a smarter one (e.g. an LLM that maps "photos from my trip last summer" to
+// semantic text + tag/date filters) via config.searchTransformer.
+//
+// Whatever a transformer returns is what the search runs on, and the server reports
+// it back to the client so the UI can honestly show what was searched.
+//
+// It may also say how the results want to be LOOKED AT. "photos from the trip last
+// summer" is a request for a gallery as much as it is a query, and the transformer is
+// the one thing in the stack that has read the sentence — by the time results come back
+// all anyone can do is guess from content types. So `view` rides along on the resolved
+// query, and the client treats it as the suggestion it is.
+
+import { TroveError } from '../errors.js';
+
+/**
+ * @typedef {object} ResolvedQuery
+ * @property {string} semanticText what to search for
+ * @property {Array<object>} tagFilters structured filters to narrow it by
+ * @property {string} [source] how it was resolved — 'parse' or 'llm'
+ * @property {string} [note] why, when it didn't go as planned
+ * @property {string|null} [view] the id of a view the results would suit, from
+ *   `ctx.views`. A suggestion: the client ignores an id it doesn't have, and never
+ *   applies it over a view the user picked for themselves.
+ */
+
+/**
+ * @typedef {object} SearchPrompt
+ * @property {string} placeholder what to show in an empty search box
+ * @property {string} [short] a phone-width version of the same
+ * @property {string} [hint] one line explaining what the box accepts
+ * @property {Array<{query: string, label?: string}>} [examples] queries worth trying
+ */
+
+export class SearchTransformer {
+  /**
+   * @param {string} rawQuery the user's input
+   * @param {{ tagKeys?: string[], views?: Array<{id: string, title?: string}> }} [ctx]
+   *   what the caller can tell us: the tag keys in use, and the views this client can
+   *   draw with (so a `view` hint names one that exists rather than one we invented).
+   * @returns {Promise<ResolvedQuery>}
+   */
+  async transform(rawQuery, ctx) {
+    throw TroveError.unsupported('SearchTransformer.transform');
+  }
+
+  /**
+   * What to tell the user this search box accepts.
+   *
+   * The transformer defines the grammar, so the transformer has to define the prompt.
+   * A hardcoded "# filter by tag" is a lie the moment an LLM transformer is plugged in
+   * and the right thing to type becomes a sentence — and it is the specific kind of lie
+   * that teaches people the search doesn't work, because they type what the box told
+   * them to and get nothing.
+   *
+   * @returns {SearchPrompt}
+   */
+  describe() {
+    return { placeholder: 'Search files', short: 'Search' };
+  }
+}
+
+// The `#tag` grammar (mirrors the client parser in web/src/bl/tagQuery.js):
+//   #tag  present · #key:=v #key:!=v #key:<v #key:<=v #key:>v #key:>=v  · #key:v (= shorthand)
+const TOKEN = /#([\w.-]+)(?::(<=|>=|!=|=|<|>)?("[^"]*"|[^#\s]+)?)?/g;
+
+/** Parse `#tag`/`#key:op:value` filters out of `query`, returning the residual text. */
+export function parseTagFilters(query) {
+  const filters = [];
+  let text = query || '';
+  for (const m of (query || '').matchAll(TOKEN)) {
+    const [, key, op, rawValue] = m;
+    const value = rawValue && rawValue.startsWith('"') ? rawValue.slice(1, -1) : rawValue;
+    filters.push(value == null || value === ''
+      ? { key, present: true }
+      : { key, op: op || '=', value, present: false });
+    text = text.replace(m[0], ' ');
+  }
+  return { text: text.replace(/\s+/g, ' ').trim(), filters };
+}
+
+// Re-exported so existing importers keep working; the semantics live in tagMatch.js,
+// which the sqlite store's SQL is also written to agree with.
+export { matchTagFilters, matchesFilter, tagProps } from './tagMatch.js';
+
+/** The default: deterministic `#tag` parsing, no external calls. */
+export class ParsingSearchTransformer extends SearchTransformer {
+  async transform(rawQuery) {
+    const { text, filters } = parseTagFilters(rawQuery);
+    return { semanticText: text, tagFilters: filters, source: 'parse' };
+  }
+
+  describe() {
+    return {
+      placeholder: 'Search files · # filter by tag',
+      short: 'Search files',
+      hint: 'Words match content and meaning. #tag narrows to items carrying that tag, '
+        + 'and #key:value compares one — with =, !=, <, <=, > or >=.',
+      examples: [
+        { query: '#draft', label: 'everything tagged draft' },
+        { query: '#year:>2023', label: 'a comparison on a tag value' },
+        { query: 'sailing #draft', label: 'meaning and a tag together' },
+      ],
+    };
+  }
+}
+
+/**
+ * The views a client offered, cleaned up enough to put in a prompt.
+ *
+ * This list arrives from the client, so it is bounded before it becomes part of a system
+ * prompt — not because a long one is dangerous, but because it is the caller's text in
+ * our instructions and an unbounded field there is somebody else's budget.
+ */
+function viewChoices(views) {
+  if (!Array.isArray(views)) return [];
+  return views
+    .filter((v) => v && typeof v.id === 'string' && v.id)
+    .slice(0, 12)
+    .map((v) => ({
+      id: v.id.slice(0, 80),
+      title: (typeof v.title === 'string' && v.title ? v.title : v.id).slice(0, 40),
+    }));
+}
+
+// An LLM-assisted transformer for Cloudflare Workers AI (or any compatible runner).
+// `run(model, { messages })` should return the model's text (Workers AI returns
+// `{ response }`). It asks a cheap chat model to convert free text into
+// { semanticText, tagFilters } given the known tag keys, and falls back to plain
+// parsing if the model is unavailable or returns unusable output — so search never
+// breaks because the LLM hiccuped.
+export class WorkersAiSearchTransformer extends SearchTransformer {
+  constructor({ ai, model = '@cf/meta/llama-3.1-8b-instruct', run } = {}) {
+    super();
+    // `ai` is a Workers AI binding ({ run(model, input) }); `run` overrides it.
+    this._run = run || (ai ? (m, input) => ai.run(m, input) : null);
+    this.model = model;
+    this._fallback = new ParsingSearchTransformer();
+  }
+
+  describe() {
+    // With a model in the loop the right thing to type is a sentence, so that is what
+    // the box should ask for. The `#tag` grammar still works — explicit filters are
+    // passed through untouched below — so it is mentioned rather than dropped.
+    return {
+      placeholder: 'Describe what you\'re looking for',
+      short: 'Describe what you want',
+      hint: 'Plain language works — "invoices from last spring". Explicit #tag filters '
+        + 'are still applied exactly as written.',
+      examples: [
+        { query: 'photos from the trip last summer' },
+        { query: 'contracts I haven\'t signed #draft', label: 'a sentence plus an exact filter' },
+      ],
+    };
+  }
+
+  async transform(rawQuery, ctx = {}) {
+    // Explicit `#tag` filters are already structured — respect them and only let the
+    // model reinterpret the residual free text.
+    const parsed = parseTagFilters(rawQuery);
+    if (!this._run || !parsed.text) return { semanticText: parsed.text, tagFilters: parsed.filters, source: 'parse' };
+    try {
+      const keys = (ctx.tagKeys || []).slice(0, 60);
+      const views = viewChoices(ctx.views);
+      const sys = 'You convert a user\'s file-search request into JSON: '
+        + '{"semanticText": string, "tagFilters": [{"key":string,"op":"="|"!="|"<"|"<="|">"|">=","value":string}|{"key":string,"present":true}]'
+        + (views.length ? ', "view": string|null' : '') + '}. '
+        + 'Only use tag keys from this list: ' + JSON.stringify(keys) + '. '
+        + (views.length
+          // Asking for null explicitly, and saying when: a model given a list of options
+          // and no way to decline will pick one every time, and a gallery forced onto a
+          // search for meeting notes is worse than not having asked.
+          ? 'If the request clearly implies how the results should be displayed, set "view" to one of '
+            + JSON.stringify(views.map((v) => v.id)) + ' — ' + views.map((v) => `${v.id} is ${v.title}`).join(', ')
+            + '. Otherwise set "view" to null. '
+          : '')
+        + 'Put the natural-language description of the content into semanticText. Output JSON only.';
+      const out = await this._run(this.model, { messages: [{ role: 'system', content: sys }, { role: 'user', content: parsed.text }] });
+      const textOut = typeof out === 'string' ? out : (out?.response || out?.result?.response || '');
+      const json = JSON.parse((textOut.match(/\{[\s\S]*\}/) || [textOut])[0]);
+      const semanticText = typeof json.semanticText === 'string' ? json.semanticText : parsed.text;
+      const modelFilters = Array.isArray(json.tagFilters) ? json.tagFilters.filter((f) => f && f.key) : [];
+      // Merge explicit user filters with the model's; explicit ones win.
+      const tagFilters = [...parsed.filters, ...modelFilters];
+      // Only a view we offered. A model that invents an id, or reaches for one this
+      // client never mentioned, has said nothing.
+      const view = views.some((v) => v.id === json.view) ? json.view : null;
+      return { semanticText, tagFilters, source: 'llm', view };
+    } catch (err) {
+      return { semanticText: parsed.text, tagFilters: parsed.filters, source: 'parse', note: 'llm-unavailable' };
+    }
+  }
+}

package/packages/core/src/search/vectorStore.js

@@ -0,0 +1,274 @@
+// VectorStore — the pluggable nearest-neighbour index behind semantic search.
+// The contract is fully ASYNC so an external vector database (pgvector, Qdrant,
+// Pinecone, Milvos, LanceDB…) fits behind the same interface as the built-in
+// in-memory store. You construct a concrete VectorStore and pass it into the
+// server (or SearchService) — core never hardcodes one, and stays platform
+// agnostic (the bundled adapters use only fetch + Web Crypto).
+//
+// A "doc" is one embedded chunk:
+//   { id, nodeId, indexerId, vector: number[], fields?: object }
+// `id` is stable per (indexer, node, chunk); `nodeId`/`indexerId` let the store
+// delete a file's or an indexer's vectors wholesale.
+
+import { TroveError, wrapError } from '../errors.js';
+import { withRetry } from '../retry.js';
+
+export class VectorStore {
+  /** Vector dimensionality this store expects (must match the embeddings). */
+  get dimensions() {
+    return this._dimensions ?? 0;
+  }
+
+  /** Upsert documents. @param {Array} docs */
+  async add(docs) {
+    throw TroveError.unsupported('add not implemented');
+  }
+  /** Remove one document by id. */
+  async remove(docId) {
+    throw TroveError.unsupported('remove not implemented');
+  }
+  /** Remove every document belonging to a node. */
+  async removeByNode(nodeId) {
+    throw TroveError.unsupported('removeByNode not implemented');
+  }
+  /** Remove every document contributed by an indexer namespace. */
+  async removeByIndexer(indexerId) {
+    throw TroveError.unsupported('removeByIndexer not implemented');
+  }
+  /** Remove documents matching both a node and an indexer (re-index a file). */
+  async removeByNodeIndexer(nodeId, indexerId) {
+    throw TroveError.unsupported('removeByNodeIndexer not implemented');
+  }
+  /**
+   * Nearest neighbours by cosine similarity.
+   * @param {number[]} vector normalised query vector
+   * @param {{limit?: number, indexers?: string[]}} [opts]
+   * @returns {Promise<Array<{docId, nodeId, indexerId, score, fields}>>}
+   */
+  async query(vector, opts) {
+    throw TroveError.unsupported('query not implemented');
+  }
+  /** Optional: number of stored vectors (diagnostics). */
+  async count() {
+    return null;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// MemoryVectorStore — exact, brute-force, dependency-free. Correct up to ~10^5
+// vectors, and EPHEMERAL: everything in it is gone on restart. That makes it a test
+// double and a last-resort fallback, not a default for a drive with files in it —
+// SqliteVectorStore (search/sqliteStores.js) is the durable local answer.
+// ---------------------------------------------------------------------------
+
+export class MemoryVectorStore extends VectorStore {
+  constructor({ dimensions } = {}) {
+    super();
+    this._dimensions = dimensions;
+    this.docs = new Map(); // docId -> { id, nodeId, indexerId, vector: Float32Array, fields }
+    this.byNode = new Map();
+    this.byIndexer = new Map();
+  }
+
+  async add(docs) {
+    for (const doc of docs) {
+      if (this._dimensions && doc.vector.length !== this._dimensions) {
+        throw TroveError.invalid(`Vector dim ${doc.vector.length} != index dim ${this._dimensions}`);
+      }
+      await this.remove(doc.id);
+      const rec = {
+        id: doc.id, nodeId: doc.nodeId, indexerId: doc.indexerId,
+        vector: doc.vector instanceof Float32Array ? doc.vector : Float32Array.from(doc.vector),
+        fields: doc.fields || {},
+      };
+      this.docs.set(rec.id, rec);
+      index(this.byNode, rec.nodeId, rec.id);
+      index(this.byIndexer, rec.indexerId, rec.id);
+    }
+  }
+
+  async remove(docId) {
+    const rec = this.docs.get(docId);
+    if (!rec) return;
+    this.docs.delete(docId);
+    this.byNode.get(rec.nodeId)?.delete(docId);
+    this.byIndexer.get(rec.indexerId)?.delete(docId);
+  }
+
+  // Both of these dropped the docs and their OWN index entry, but left the docs' ids
+  // sitting in the SIBLING index forever — so a drive that re-indexes repeatedly grew a
+  // per-indexer (or per-node) id list without bound. Queries stayed correct, which is
+  // why it never showed up as anything but memory.
+  async removeByNode(nodeId) {
+    for (const id of [...(this.byNode.get(nodeId) || [])]) this.#drop(id);
+    this.byNode.delete(nodeId);
+  }
+  async removeByIndexer(indexerId) {
+    for (const id of [...(this.byIndexer.get(indexerId) || [])]) this.#drop(id);
+    this.byIndexer.delete(indexerId);
+  }
+  /** Forget one doc from `docs` and from BOTH indexes. */
+  #drop(docId) {
+    const rec = this.docs.get(docId);
+    if (!rec) return;
+    this.docs.delete(docId);
+    const byNode = this.byNode.get(rec.nodeId);
+    if (byNode) { byNode.delete(docId); if (!byNode.size) this.byNode.delete(rec.nodeId); }
+    const byIndexer = this.byIndexer.get(rec.indexerId);
+    if (byIndexer) { byIndexer.delete(docId); if (!byIndexer.size) this.byIndexer.delete(rec.indexerId); }
+  }
+  async removeByNodeIndexer(nodeId, indexerId) {
+    for (const id of this.byNode.get(nodeId) || []) {
+      const rec = this.docs.get(id);
+      if (rec && rec.indexerId === indexerId) await this.remove(id);
+    }
+  }
+
+  async query(vector, opts = {}) {
+    const limit = opts.limit ?? 20;
+    const allow = opts.indexers ? new Set(opts.indexers) : null;
+    const out = [];
+    for (const rec of this.docs.values()) {
+      if (allow && !allow.has(rec.indexerId)) continue;
+      out.push({ docId: rec.id, nodeId: rec.nodeId, indexerId: rec.indexerId, score: dot(vector, rec.vector), fields: rec.fields });
+    }
+    out.sort((a, b) => b.score - a.score);
+    return out.slice(0, limit);
+  }
+
+  async count() {
+    return this.docs.size;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// QdrantVectorStore — a real external adapter, proving the interface fits an
+// async network DB. Uses Qdrant's REST API over fetch (works on Node, Bun,
+// Workers). Point ids are deterministic UUIDs derived from the docId (Qdrant
+// requires uint/uuid ids); the original docId + routing keys live in the
+// payload so node/indexer deletes are filter-based.
+// ---------------------------------------------------------------------------
+
+export class QdrantVectorStore extends VectorStore {
+  /**
+   * @param {object} cfg
+   * @param {string} cfg.url          e.g. http://localhost:6333
+   * @param {string} cfg.collection
+   * @param {number} cfg.dimensions
+   * @param {string} [cfg.apiKey]
+   * @param {string} [cfg.distance]   Cosine (default) | Dot | Euclid
+   */
+  constructor(cfg) {
+    super();
+    for (const k of ['url', 'collection', 'dimensions']) {
+      if (!cfg?.[k]) throw TroveError.invalid(`QdrantVectorStore requires "${k}"`);
+    }
+    this.cfg = cfg;
+    this._dimensions = cfg.dimensions;
+    this._ready = null;
+  }
+
+  async #req(method, path, body) {
+    return withRetry(
+      async () => {
+        let res;
+        try {
+          res = await fetch(this.cfg.url.replace(/\/$/, '') + path, {
+            method,
+            headers: { 'content-type': 'application/json', ...(this.cfg.apiKey ? { 'api-key': this.cfg.apiKey } : {}) },
+            body: body !== undefined ? JSON.stringify(body) : undefined,
+          });
+        } catch (err) {
+          throw wrapError(err);
+        }
+        if (res.status === 429 || res.status >= 500) throw TroveError.transient(`Qdrant ${res.status}`);
+        const json = await res.json().catch(() => ({}));
+        if (!res.ok) throw TroveError.internal(`Qdrant ${method} ${path} failed: ${json?.status?.error || res.status}`);
+        return json;
+      },
+      { retries: 3 },
+    );
+  }
+
+  /** Create the collection on first use (idempotent). */
+  async #ensure() {
+    if (!this._ready) {
+      this._ready = (async () => {
+        const existing = await this.#req('GET', `/collections/${this.cfg.collection}/exists`).catch(() => null);
+        if (existing?.result?.exists) return;
+        await this.#req('PUT', `/collections/${this.cfg.collection}`, {
+          vectors: { size: this.cfg.dimensions, distance: this.cfg.distance || 'Cosine' },
+        }).catch((e) => {
+          // Tolerate a concurrent create (409-ish) but surface real errors.
+          if (!/already exists/i.test(e.message)) throw e;
+        });
+      })();
+    }
+    return this._ready;
+  }
+
+  async add(docs) {
+    if (!docs.length) return;
+    await this.#ensure();
+    const points = await Promise.all(
+      docs.map(async (d) => ({
+        id: await docUuid(d.id),
+        vector: Array.from(d.vector),
+        payload: { docId: d.id, nodeId: d.nodeId, indexerId: d.indexerId, fields: d.fields || {} },
+      })),
+    );
+    await this.#req('PUT', `/collections/${this.cfg.collection}/points?wait=true`, { points });
+  }
+
+  async remove(docId) {
+    await this.#deleteByFilter({ must: [{ key: 'docId', match: { value: docId } }] });
+  }
+  async removeByNode(nodeId) {
+    await this.#deleteByFilter({ must: [{ key: 'nodeId', match: { value: nodeId } }] });
+  }
+  async removeByIndexer(indexerId) {
+    await this.#deleteByFilter({ must: [{ key: 'indexerId', match: { value: indexerId } }] });
+  }
+  async removeByNodeIndexer(nodeId, indexerId) {
+    await this.#deleteByFilter({ must: [{ key: 'nodeId', match: { value: nodeId } }, { key: 'indexerId', match: { value: indexerId } }] });
+  }
+  async #deleteByFilter(filter) {
+    await this.#ensure();
+    await this.#req('POST', `/collections/${this.cfg.collection}/points/delete?wait=true`, { filter });
+  }
+
+  async query(vector, opts = {}) {
+    await this.#ensure();
+    const body = { vector: Array.from(vector), limit: opts.limit ?? 20, with_payload: true };
+    if (opts.indexers?.length) body.filter = { must: [{ key: 'indexerId', match: { any: opts.indexers } }] };
+    const res = await this.#req('POST', `/collections/${this.cfg.collection}/points/search`, body);
+    return (res.result || []).map((r) => ({
+      docId: r.payload?.docId, nodeId: r.payload?.nodeId, indexerId: r.payload?.indexerId,
+      score: r.score, fields: r.payload?.fields || {},
+    }));
+  }
+
+  async count() {
+    await this.#ensure();
+    const res = await this.#req('POST', `/collections/${this.cfg.collection}/points/count`, { exact: true });
+    return res.result?.count ?? null;
+  }
+}
+
+// Deterministic UUIDv5-style id from a string (SHA-256 → formatted as a UUID).
+async function docUuid(str) {
+  const buf = new Uint8Array(await crypto.subtle.digest('SHA-256', new TextEncoder().encode(str)));
+  const h = [...buf.slice(0, 16)].map((b) => b.toString(16).padStart(2, '0')).join('');
+  return `${h.slice(0, 8)}-${h.slice(8, 12)}-5${h.slice(13, 16)}-${((parseInt(h[16], 16) & 0x3) | 0x8).toString(16)}${h.slice(17, 20)}-${h.slice(20, 32)}`;
+}
+
+function index(map, key, id) {
+  let set = map.get(key);
+  if (!set) map.set(key, (set = new Set()));
+  set.add(id);
+}
+function dot(a, b) {
+  let s = 0;
+  for (let i = 0; i < a.length; i++) s += a[i] * b[i];
+  return s;
+}