@volter/twin-algolia 0.1.0

package/src/algolia-filter.ts

@@ -0,0 +1,150 @@
+// Algolia FILTER + FACET evaluator — net-new, PORTED CONCEPTUALLY (not copied) from
+// pinecone-filter.ts's discipline (a real interpreter over real stored data, never a name/
+// keyword match) — the SYNTAX is entirely different because Algolia's is its own: `filters` is
+// a STRING that must be tokenized (Mongo-style JSON it is not), and `facetFilters` arrives as a
+// nested array structure (outer array = AND, an inner array = OR) that needs no string parsing
+// at all — two genuinely different evaluators sharing one file because they gate the same
+// `search`/`browse`/`delete`-style operations together (mirrors pinecone-filter.ts's `matchesFilter`
+// covering both query and delete-by-filter).
+//
+// Modeled `filters` grammar (v1, honest cut — see spec-sources.json): `field OP value` numeric
+// comparisons (`> >= < <= = !=`, value a number literal), `field:value` facet/string equality
+// (quoted or bare value), `NOT field:value` facet negation, and `AND`/`OR` combinators (case-
+// sensitive keywords, exactly like the real product) with AND binding tighter than OR (real
+// Algolia operator precedence) — evaluated LEFT-TO-RIGHT across OR-separated AND-groups.
+// Parenthesized precedence override, `_tags`/`tag:` shorthand, and geo filters are NOT modeled
+// (todo: `filters.parentheses`/`filters.tag_shorthand`/`filters.geo` — an honest v1 cut, not an
+// oversight).
+export type FacetFilterClause = string | string[];
+export type FacetFilters = FacetFilterClause[];
+
+function parseFacetToken(token: string): { field: string; value: string; negate: boolean } | undefined {
+  const idx = token.indexOf(':');
+  if (idx < 0) return undefined;
+  const field = token.slice(0, idx).trim();
+  let value = token.slice(idx + 1).trim();
+  let negate = false;
+  if (value.startsWith('-')) {
+    negate = true;
+    value = value.slice(1);
+  }
+  if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) {
+    value = value.slice(1, -1);
+  }
+  return { field, value, negate };
+}
+
+function facetValueMatches(recordValue: unknown, wanted: string): boolean {
+  if (Array.isArray(recordValue)) return recordValue.some((v) => String(v) === wanted);
+  if (recordValue === undefined || recordValue === null) return false;
+  return String(recordValue) === wanted;
+}
+
+/** Evaluate a single `facetFilters` clause (a bare "field:value" string, or an OR-array of them)
+ *  against a record. An absent/empty facetFilters matches everything. */
+export function matchesFacetFilters(record: Record<string, unknown>, facetFilters: FacetFilters | undefined | null): boolean {
+  if (!facetFilters || facetFilters.length === 0) return true;
+  for (const clause of facetFilters) {
+    const items = Array.isArray(clause) ? clause : [clause];
+    // outer array element: AND with the rest; an array-of-strings ELEMENT is itself an OR group.
+    const orMatch = items.some((token) => {
+      const parsed = parseFacetToken(token);
+      if (!parsed) return false;
+      const matched = facetValueMatches(record[parsed.field], parsed.value);
+      return parsed.negate ? !matched : matched;
+    });
+    if (!orMatch) return false;
+  }
+  return true;
+}
+
+// ── `filters` string grammar ────────────────────────────────────────────────────────────────
+type Comparison = { field: string; op: '>' | '>=' | '<' | '<=' | '=' | '!='; value: number };
+type FacetClause = { field: string; value: string; negate: boolean };
+type Clause = { kind: 'compare'; c: Comparison } | { kind: 'facet'; c: FacetClause };
+
+const COMPARE_RE = /^(\S+)\s*(>=|<=|!=|>|<|=)\s*(-?\d+(?:\.\d+)?)$/;
+
+function parseClause(raw: string): Clause | undefined {
+  const trimmed = raw.trim();
+  if (trimmed.length === 0) return undefined;
+  let text = trimmed;
+  let negate = false;
+  if (text.startsWith('NOT ')) {
+    negate = true;
+    text = text.slice(4).trim();
+  }
+  const cmp = COMPARE_RE.exec(text);
+  if (cmp) {
+    const [, field, op, num] = cmp;
+    return { kind: 'compare', c: { field: field!, op: (negate ? negateOp(op as Comparison['op']) : op) as Comparison['op'], value: Number(num) } };
+  }
+  const facet = parseFacetToken(text);
+  if (facet) return { kind: 'facet', c: { field: facet.field, value: facet.value, negate: negate !== facet.negate } };
+  return undefined;
+}
+
+function negateOp(op: Comparison['op']): Comparison['op'] {
+  const table: Record<Comparison['op'], Comparison['op']> = { '>': '<=', '>=': '<', '<': '>=', '<=': '>', '=': '!=', '!=': '=' };
+  return table[op];
+}
+
+function evalClause(record: Record<string, unknown>, clause: Clause): boolean {
+  if (clause.kind === 'facet') {
+    const matched = facetValueMatches(record[clause.c.field], clause.c.value);
+    return clause.c.negate ? !matched : matched;
+  }
+  const raw = record[clause.c.field];
+  if (typeof raw !== 'number') return false;
+  switch (clause.c.op) {
+    case '>':
+      return raw > clause.c.value;
+    case '>=':
+      return raw >= clause.c.value;
+    case '<':
+      return raw < clause.c.value;
+    case '<=':
+      return raw <= clause.c.value;
+    case '=':
+      return raw === clause.c.value;
+    case '!=':
+      return raw !== clause.c.value;
+  }
+}
+
+/** Evaluate an Algolia `filters` STRING against a stored record: AND binds tighter than OR,
+ *  evaluated as OR-of-AND-groups (no parenthesized precedence override — v1 honest cut). An
+ *  absent/empty/whitespace-only filter string matches everything. */
+export function matchesAlgoliaFilters(record: Record<string, unknown>, filters: string | undefined | null): boolean {
+  if (!filters || filters.trim().length === 0) return true;
+  const orGroups = filters.split(/\s+OR\s+/);
+  return orGroups.some((group) =>
+    group
+      .split(/\s+AND\s+/)
+      .map((raw) => parseClause(raw))
+      .every((clause) => (clause ? evalClause(record, clause) : true)),
+  );
+}
+
+/**
+ * REAL facet-count aggregation over an already-matched-and-filtered record set (conjunctive —
+ * counts reflect the final hit set, not a per-facet disjunctive relaxation — disjunctive facet
+ * counting is `facets.disjunctive_facets`, todo). Multi-value (array) facet attributes count
+ * each element once per record — a genuine tally, not a name check.
+ */
+export function computeFacetCounts(records: ReadonlyArray<Record<string, unknown>>, facetAttributes: readonly string[]): Record<string, Record<string, number>> {
+  const out: Record<string, Record<string, number>> = {};
+  for (const attr of facetAttributes) {
+    const counts: Record<string, number> = {};
+    for (const r of records) {
+      const v = r[attr];
+      const values = Array.isArray(v) ? v : v === undefined || v === null ? [] : [v];
+      for (const one of values) {
+        const key = String(one);
+        counts[key] = (counts[key] ?? 0) + 1;
+      }
+    }
+    out[attr] = counts;
+  }
+  return out;
+}

package/src/algolia-search.ts

@@ -0,0 +1,204 @@
+// Algolia SEARCH ENGINE — net-new, no clone (ADDING_A_TWIN.md's "real computed op as strong
+// done" pattern; the analog of pinecone-similarity.ts's real brute-force top-K, ported
+// CONCEPTUALLY not copied — the shape of the problem differs completely: Algolia ranks TEXT
+// records by a tie-break criteria chain, not vectors by a distance metric). This is REAL
+// compute, not a stub: deterministic ASCII/Unicode tokenization, a real Damerau-Levenshtein
+// edit-distance typo matcher (<=2), and a faithful SUBSET of Algolia's documented default
+// `ranking` tie-break chain (`typo`, `words` (folded into typo-match — AND semantics, see
+// below), `proximity`, `attribute`, `exact`, `custom`) — every criterion below is GENUINELY
+// COMPUTED against the stored record values and searchableAttributes configuration, not a
+// name/keyword match or a marker.
+//
+// WHAT IS NOT YET RANKED (`algolia.search.ranking_criteria_coverage`, todo): the real product
+// ships an additional `geo`/`filters`-as-ranking-criterion slot, and on top a proprietary
+// NeuralSearch/AI re-rank pass. What IS modeled (typo/proximity/attribute/exact/custom) is exact,
+// deterministic, correct math over the modeled criteria — a faithful subset, not a lesser
+// approximation of what it does model (mirrors pinecone-similarity.ts's framing).
+//
+// MATCH SEMANTICS (v1 simplification, documented rather than silently guessed): a record
+// matches a non-empty query iff there EXISTS at least one searchable attribute whose token
+// list satisfies EVERY query word (exact, <=2-edit-distance typo, or same synonym group) —
+// i.e. AND-across-query-words, matched WITHIN one attribute (not unioned across attributes).
+// Real Algolia's default `removeWordsIfNoResults`/optional-words progressive relaxation is
+// NOT modeled (see `search.optional_words` / `search.remove_words_if_no_results`, todo) — this
+// twin's strict-AND match is the honest, simpler default behavior, not the full adaptive
+// algorithm. Attribute VALUES are read top-level only (dot-path nested attribute addressing is
+// not modeled, consistent with the v1 slice's flat-record framing used throughout this pack).
+export type CustomRankingRule = { attribute: string; direction: 'asc' | 'desc' };
+
+export type SearchRecord = Record<string, unknown> & { objectID: string };
+
+/** Deterministic ASCII/Unicode tokenizer: lowercase, split on runs of non letter/number
+ *  characters. This is the line `algolia.search.language_processing` (todo) will move: no
+ *  stemming/plural-folding/CJK segmentation/language normalization yet — just a real,
+ *  deterministic word-boundary split. */
+export function tokenize(text: string): string[] {
+  return text.toLowerCase().match(/[\p{L}\p{N}]+/gu) ?? [];
+}
+
+/** Real Damerau-Levenshtein edit distance (insertions/deletions/substitutions/adjacent
+ *  transpositions) — a genuine DP table, not an approximation. */
+export function damerauLevenshtein(a: string, b: string): number {
+  const al = a.length;
+  const bl = b.length;
+  if (al === 0) return bl;
+  if (bl === 0) return al;
+  const d: number[][] = Array.from({ length: al + 1 }, () => new Array<number>(bl + 1).fill(0));
+  for (let i = 0; i <= al; i++) d[i]![0] = i;
+  for (let j = 0; j <= bl; j++) d[0]![j] = j;
+  for (let i = 1; i <= al; i++) {
+    for (let j = 1; j <= bl; j++) {
+      const cost = a[i - 1] === b[j - 1] ? 0 : 1;
+      let v = Math.min(
+        d[i - 1]![j]! + 1, //      deletion
+        d[i]![j - 1]! + 1, //      insertion
+        d[i - 1]![j - 1]! + cost, // substitution
+      );
+      if (i > 1 && j > 1 && a[i - 1] === b[j - 2] && a[i - 2] === b[j - 1]) {
+        v = Math.min(v, d[i - 2]![j - 2]! + cost); // adjacent transposition
+      }
+      d[i]![j] = v;
+    }
+  }
+  return d[al]![bl]!;
+}
+
+const MAX_TYPOS = 2;
+
+/** All values a group of interchangeable synonym words maps to each other, lowercase. A word's
+ *  group includes itself. Basic bidirectional `type:"synonym"` groups only — one-way/altCorrection/
+ *  placeholder synonyms are `search.optional_words`-adjacent todos (synonyms.one_way etc). */
+export function buildSynonymGroups(synonymRecords: ReadonlyArray<{ type?: string; synonyms?: readonly string[] }>): string[][] {
+  const groups: string[][] = [];
+  for (const s of synonymRecords) {
+    if (s.type !== 'synonym' || !Array.isArray(s.synonyms) || s.synonyms.length < 2) continue;
+    groups.push(s.synonyms.map((w) => w.toLowerCase()));
+  }
+  return groups;
+}
+
+function sameSynonymGroup(a: string, b: string, groups: readonly string[][]): boolean {
+  for (const g of groups) if (g.includes(a) && g.includes(b)) return true;
+  return false;
+}
+
+/** Distance between a query word and a candidate token: 0 for exact or same-synonym-group,
+ *  else the Damerau-Levenshtein edit distance capped at MAX_TYPOS, else `undefined` (no match
+ *  at all — the record does not satisfy this query word via this token). */
+function wordDistance(queryWord: string, token: string, synonymGroups: readonly string[][]): number | undefined {
+  if (queryWord === token) return 0;
+  if (sameSynonymGroup(queryWord, token, synonymGroups)) return 0;
+  const d = damerauLevenshtein(queryWord, token);
+  return d <= MAX_TYPOS ? d : undefined;
+}
+
+/** Flatten one attribute's stored value into a token list (string, array-of-strings/values, or
+ *  a scalar stringified) — non-string/array/number/boolean values (objects, null, undefined)
+ *  contribute no tokens. */
+function tokensForAttributeValue(value: unknown): string[] {
+  if (typeof value === 'string') return tokenize(value);
+  if (typeof value === 'number' || typeof value === 'boolean') return tokenize(String(value));
+  if (Array.isArray(value)) return value.flatMap((v) => tokensForAttributeValue(v));
+  return [];
+}
+
+type AttributeMatch = { attributeIndex: number; typoTotal: number; exactCount: number; proximity: number };
+
+/** Try to match every query word within ONE attribute's token list; undefined if any query word
+ *  finds no token in this attribute within MAX_TYPOS. */
+function matchWithinAttribute(tokens: readonly string[], queryWords: readonly string[], synonymGroups: readonly string[][], attributeIndex: number): AttributeMatch | undefined {
+  if (tokens.length === 0) return undefined;
+  const positions: number[] = [];
+  let typoTotal = 0;
+  let exactCount = 0;
+  for (const qw of queryWords) {
+    let best: { idx: number; typos: number } | undefined;
+    for (let i = 0; i < tokens.length; i++) {
+      const d = wordDistance(qw, tokens[i]!, synonymGroups);
+      if (d !== undefined && (!best || d < best.typos)) best = { idx: i, typos: d };
+    }
+    if (!best) return undefined;
+    positions.push(best.idx);
+    typoTotal += best.typos;
+    if (best.typos === 0) exactCount++;
+  }
+  const proximity = positions.length > 0 ? Math.max(...positions) - Math.min(...positions) : 0;
+  return { attributeIndex, typoTotal, exactCount, proximity };
+}
+
+export type RankedHit<T extends SearchRecord = SearchRecord> = {
+  record: T;
+  typoTotal: number;
+  attributeIndex: number;
+  proximity: number;
+  exactCount: number;
+};
+
+/** Default searchable-attribute order when settings.searchableAttributes is unset: every
+ *  top-level field except objectID, in a deterministic sorted order. Real Algolia treats every
+ *  attribute as equally searchable when unconfigured (no attribute-priority distinction) — this
+ *  twin's sorted-key stand-in only matters for tie-breaking when the caller HAS configured an
+ *  explicit order (search.attributes_to_retrieve / settings.searchable_attributes_affects_search). */
+function defaultSearchableAttributes(records: readonly SearchRecord[]): string[] {
+  const keys = new Set<string>();
+  for (const r of records) for (const k of Object.keys(r)) if (k !== 'objectID') keys.add(k);
+  return [...keys].sort();
+}
+
+function customRankingCompare(a: SearchRecord, b: SearchRecord, rules: readonly CustomRankingRule[]): number {
+  for (const rule of rules) {
+    const av = a[rule.attribute];
+    const bv = b[rule.attribute];
+    const an = typeof av === 'number' ? av : Number.NEGATIVE_INFINITY;
+    const bn = typeof bv === 'number' ? bv : Number.NEGATIVE_INFINITY;
+    if (an === bn) continue;
+    return rule.direction === 'desc' ? bn - an : an - bn;
+  }
+  return 0;
+}
+
+/**
+ * Match + rank records against a query — the pack's signature strong-`done`. Returns every
+ * MATCHING record, best-first, per the faithful-subset tie-break chain: typo -> attribute
+ * (earlier configured attribute wins) -> proximity -> exact -> customRanking -> objectID
+ * (final deterministic tie-break). An empty/whitespace query matches every record (real
+ * Algolia's browse-all-via-empty-query convention) with every criterion tied, so ordering then
+ * falls straight to customRanking / objectID.
+ */
+export function rankRecords<T extends SearchRecord>(
+  records: readonly T[],
+  query: string,
+  opts: { searchableAttributes?: readonly string[]; customRanking?: readonly CustomRankingRule[]; synonymGroups?: readonly string[][] } = {},
+): RankedHit<T>[] {
+  const attrs = opts.searchableAttributes && opts.searchableAttributes.length > 0 ? opts.searchableAttributes : defaultSearchableAttributes(records);
+  const customRanking = opts.customRanking ?? [];
+  const synonymGroups = opts.synonymGroups ?? [];
+  const queryWords = tokenize(query);
+
+  const hits: RankedHit<T>[] = [];
+  for (const record of records) {
+    if (queryWords.length === 0) {
+      hits.push({ record, typoTotal: 0, attributeIndex: 0, proximity: 0, exactCount: 0 });
+      continue;
+    }
+    let best: AttributeMatch | undefined;
+    for (let i = 0; i < attrs.length; i++) {
+      const tokens = tokensForAttributeValue(record[attrs[i]!]);
+      const m = matchWithinAttribute(tokens, queryWords, synonymGroups, i);
+      if (!m) continue;
+      if (!best || m.typoTotal < best.typoTotal || (m.typoTotal === best.typoTotal && m.attributeIndex < best.attributeIndex)) best = m;
+    }
+    if (best) hits.push({ record, typoTotal: best.typoTotal, attributeIndex: best.attributeIndex, proximity: best.proximity, exactCount: best.exactCount });
+  }
+
+  hits.sort((a, b) => {
+    if (a.typoTotal !== b.typoTotal) return a.typoTotal - b.typoTotal;
+    if (a.attributeIndex !== b.attributeIndex) return a.attributeIndex - b.attributeIndex;
+    if (a.proximity !== b.proximity) return a.proximity - b.proximity;
+    if (a.exactCount !== b.exactCount) return b.exactCount - a.exactCount;
+    const custom = customRankingCompare(a.record, b.record, customRanking);
+    if (custom !== 0) return custom;
+    return a.record.objectID < b.record.objectID ? -1 : a.record.objectID > b.record.objectID ? 1 : 0;
+  });
+  return hits;
+}

package/src/algolia-server.ts

@@ -0,0 +1,43 @@
+// Algolia twin HTTP server — ONE process serving every host shape a real `algoliasearch` SDK (or
+// raw REST caller) might target, so an unmodified SDK works end-to-end against a single local
+// process. Unlike pinecone's control/data HOST split, Algolia's REST paths already carry the
+// index name (`/1/indexes/{indexName}/...`), so this server does zero host-based dispatch — the
+// literal HTTP `Host` header (or, for the SDK-integration test's proxy-rewrite pattern, an
+// `x-algolia-target-host` stash) is threaded through purely for `routeAlgoliaSurface`'s host-
+// tolerance proof (see algolia-twin.ts's header); routing itself is 100% path-driven.
+//
+// Writable by default; pass readOnly to reject writes (per-operation — see algolia-twin.ts).
+//
+// FETCH-FIRST (runtime contract R12b): the serve path is the plain fetch below, built from the
+// kernel's ONE adaptation (`createTwinFetchFromHandler`) with the host threading as its per-request
+// `extras`; the server is one line of Bun.serve around that same closure.
+import { serveHttp } from '@volter/world-core';
+import { handleAlgoliaTwinRequest } from './algolia-twin.ts';
+import { createTwinFetchFromHandler, statefulTwinManifest } from '@volter/world-core';
+
+/** Options every Algolia-twin HTTP surface needs, independent of who owns the socket. */
+export interface AlgoliaTwinFetchOptions {
+  root?: string;
+  readOnly?: boolean;
+}
+
+export function createAlgoliaTwinFetch(options: AlgoliaTwinFetchOptions = {}): (request: Request) => Promise<Response> {
+  return createTwinFetchFromHandler(handleAlgoliaTwinRequest, {
+    ...options,
+    manifest: statefulTwinManifest({ vendor: 'algolia', twinOf: 'Algolia hosted search', stores: 'indexes and their records (CRUD + batch), searched by a real local engine' }),
+    // `Headers` iteration already lower-cases every name, so the adapter's header map matches the
+    // one this server used to build by hand.
+    extras: (request, url) => ({
+      host: request.headers.get('x-algolia-target-host') || request.headers.get('host') || url.host,
+    }),
+  });
+}
+
+export async function createAlgoliaTwinServer(options: { root?: string; port?: number; readOnly?: boolean } = {}): Promise<{ port: number; stop: () => void }> {
+  const server = await serveHttp({
+    port: options.port ?? 0,
+    idleTimeout: 60,
+    fetch: createAlgoliaTwinFetch(options),
+  });
+  return { port: server.port ?? options.port ?? 0, stop: () => server.stop(true) };
+}