@volter/twin-turbopuffer 0.1.0

package/src/turbopuffer-connector.ts

@@ -0,0 +1,153 @@
+// Turbopuffer's half of the real state system (protocol 2): the PERFORM adapter that replays one
+// recorded write against the vendor, and the REFRESH adapter that observes a real account into
+// the tree. Both run over the kernel's injected `RemoteExecute` — the kernel applies the sealed
+// credential (the placeholder `authorization` below is replaced), so this pack never holds a key
+// and issues no network call of its own.
+import { observeResource, observeResources, type PerformContext, type PushOutcome, type RemoteExecute, type TwinAction } from '@volter/world-core';
+import { documentSubjectId, SERVICE } from './turbopuffer-store.ts';
+import { DEFAULT_FTS, type FtsConfig } from './turbopuffer-text.ts';
+import type { AttrConfig } from './turbopuffer-filter.ts';
+
+const HEADERS = { accept: 'application/json', 'content-type': 'application/json', authorization: 'Bearer twin' };
+/** Bounds on one refresh, so a huge account cannot turn a sync into an unbounded crawl. */
+export const REFRESH_MAX_NAMESPACE_PAGES = 50;
+export const REFRESH_MAX_DOCUMENTS_PER_NAMESPACE = 100_000;
+const DOC_PAGE = 1000;
+
+async function call(execute: RemoteExecute, method: string, path: string, body?: unknown): Promise<Record<string, unknown>> {
+  const res = await execute({ method, path, headers: { ...HEADERS }, ...(body === undefined ? {} : { body: JSON.stringify(body) }) });
+  // A refusal is thrown, never read as an empty account (adding-a-twin.md §6).
+  if (res.status < 200 || res.status >= 300) throw new Error(`turbopuffer ${method} ${path} refused: HTTP ${res.status} ${res.body.slice(0, 200)}`);
+  try {
+    return JSON.parse(res.body || '{}') as Record<string, unknown>;
+  } catch {
+    throw new Error(`turbopuffer ${method} ${path} answered a body that is not JSON`);
+  }
+}
+
+/**
+ * Perform ONE recorded entry against the vendor. A write request is recorded whole (its body is the
+ * action's `input`), so the perform replays exactly what the caller sent; Turbopuffer ids are
+ * caller-chosen, so there is no local→vendor id to rebind.
+ */
+export async function performTurbopufferAction(execute: RemoteExecute, action: TwinAction, _ctx: PerformContext): Promise<PushOutcome> {
+  const ns = action.subject.id;
+  const path = `/v2/namespaces/${encodeURIComponent(ns)}`;
+  const input = (action.input ?? {}) as Record<string, unknown>;
+  switch (action.operation) {
+    case 'namespace.write':
+      return { externalId: ns, data: await call(execute, 'POST', path, input) };
+    case 'namespace.delete_all':
+      return { externalId: ns, data: await call(execute, 'DELETE', path) };
+    case 'namespace.update_schema':
+      return { externalId: ns, data: await call(execute, 'POST', `/v1/namespaces/${encodeURIComponent(ns)}/schema`, input.schema ?? {}) };
+  }
+  throw new Error(`turbopuffer perform: no vendor operation for ${action.operation ?? '(unnamed)'} on ${action.subject.type}:${ns}`);
+}
+
+/** Map one attribute of the vendor's schema wire into the stored form (lenient: the vendor is authoritative). */
+export function mapSchemaAttr(raw: unknown): AttrConfig {
+  const r = (typeof raw === 'string' ? { type: raw } : raw ?? {}) as Record<string, unknown>;
+  const fts: FtsConfig | false = r.full_text_search === undefined || r.full_text_search === false || r.full_text_search === null
+    ? false
+    : { ...DEFAULT_FTS, ...(typeof r.full_text_search === 'object' ? r.full_text_search as Partial<FtsConfig> : {}) };
+  return {
+    type: typeof r.type === 'string' ? r.type : 'string',
+    filterable: typeof r.filterable === 'boolean' ? r.filterable : fts === false,
+    full_text_search: fts,
+    ...(r.glob === true ? { glob: true } : {}),
+    ...(r.regex === true ? { regex: true } : {}),
+  };
+}
+
+/** The refresh adapter (the descriptor's `stateSystem.refresh`): the D7 pull over the kernel's executor. */
+export async function syncTurbopufferFromRemote(
+  execute: RemoteExecute,
+  opts: { root?: string; origin?: string; occurredAt?: string } = {},
+): Promise<{ observed: number; complete?: string[] }> {
+  return syncTurbopufferFromReal(execute, opts);
+}
+
+/**
+ * D7 consumer-facing pull, through the injected executor (the kernel's in production, a fake in
+ * tests): list the account's namespaces, then for each read its schema, metadata and every document
+ * (paged by id), and OBSERVE them — the kernel diffs and folds. Read-only.
+ */
+export async function syncTurbopufferFromReal(
+  execute: RemoteExecute,
+  opts: { root?: string; occurredAt?: string } = {},
+): Promise<{ observed: number; complete?: string[] }> {
+  const at = opts.occurredAt ?? new Date().toISOString();
+  const target = { ...(opts.root !== undefined ? { root: opts.root } : {}), at };
+  let observed = 0;
+  let truncated = false;
+  const names: string[] = [];
+  let cursor = '';
+  for (let page = 0; page < REFRESH_MAX_NAMESPACE_PAGES; page++) {
+    const listed = await call(execute, 'GET', `/v1/namespaces?page_size=1000${cursor ? `&cursor=${encodeURIComponent(cursor)}` : ''}`);
+    for (const n of (listed.namespaces as Array<{ id?: unknown }> | undefined) ?? []) if (typeof n.id === 'string') names.push(n.id);
+    cursor = typeof listed.next_cursor === 'string' ? listed.next_cursor : '';
+    if (cursor === '') break;
+  }
+  if (cursor !== '') truncated = true;
+  for (const ns of names) {
+    const enc = encodeURIComponent(ns);
+    const meta = await call(execute, 'GET', `/v2/namespaces/${enc}/metadata`);
+    const wireSchema = (meta.schema ?? {}) as Record<string, unknown>;
+    const schema: Record<string, AttrConfig> = {};
+    let distance: string | null = null;
+    let dims: number | null = null;
+    for (const [attr, raw] of Object.entries(wireSchema)) {
+      schema[attr] = mapSchemaAttr(raw);
+      const m = /^\[(\d+)\]f(16|32)$/.exec(schema[attr]!.type);
+      if (attr === 'vector' && m) {
+        dims = Number(m[1]);
+        const ann = (raw as Record<string, unknown> | null)?.ann as Record<string, unknown> | undefined;
+        if (typeof ann?.distance_metric === 'string') distance = ann.distance_metric;
+      }
+    }
+    observeResource(SERVICE, {
+      type: 'namespace',
+      id: ns,
+      fields: {
+        name: ns, schema, distance_metric: distance, vector_dims: dims,
+        created_at: typeof meta.created_at === 'string' ? meta.created_at : at,
+        updated_at: typeof meta.updated_at === 'string' ? meta.updated_at : at,
+        last_write_at: typeof meta.last_write_at === 'string' ? meta.last_write_at : typeof meta.updated_at === 'string' ? meta.updated_at : at,
+        gone: false,
+      },
+    }, target);
+    observed++;
+    let last: string | number | null = null;
+    for (let seen = 0; seen < REFRESH_MAX_DOCUMENTS_PER_NAMESPACE;) {
+      const page = await call(execute, 'POST', `/v2/namespaces/${enc}/query`, {
+        rank_by: ['id', 'asc'],
+        top_k: DOC_PAGE,
+        include_attributes: true,
+        ...(last === null ? {} : { filters: ['id', 'Gt', last] }),
+      });
+      const rows = (page.rows as Array<Record<string, unknown>> | undefined) ?? [];
+      const docs: Array<{ type: string; id: string; fields: Record<string, unknown> }> = [];
+      for (const row of rows) {
+        const { id, vector, $dist: _d, ...attributes } = row;
+        if (typeof id !== 'string' && typeof id !== 'number') continue;
+        for (const k of Object.keys(attributes)) if (attributes[k] === null) delete attributes[k];
+        docs.push({
+          type: 'document',
+          id: documentSubjectId(ns, id),
+          fields: { namespace: ns, doc_id: id, attributes, vector: Array.isArray(vector) ? vector : null },
+        });
+        last = id;
+      }
+      if (docs.length > 0) observeResources(SERVICE, docs, target);
+      observed += docs.length;
+      seen += rows.length;
+      if (rows.length < DOC_PAGE) break;
+      if (seen >= REFRESH_MAX_DOCUMENTS_PER_NAMESPACE) truncated = true;
+    }
+  }
+  // A listing cut short by the bounds is not the whole account, so it never claims completeness
+  // (the kernel removes what a COMPLETE refresh no longer saw).
+  return { observed, ...(truncated ? {} : { complete: ['namespace', 'document'] }) };
+}
+

package/src/turbopuffer-filter.ts

@@ -0,0 +1,277 @@
+// THE FILTER LANGUAGE — Turbopuffer's array-encoded `filters` (a SQL WHERE clause as JSON), compiled
+// into a predicate over the namespace's documents.
+//
+// The operator set is the one the installed SDK declares (`Filter` in
+// @turbopuffer/turbopuffer@2.8.0 src/resources/custom.ts). Modelled: Eq, NotEq, In, NotIn,
+// Contains, NotContains, ContainsAny, NotContainsAny, Lt, Lte, Gt, Gte, AnyLt, AnyLte, AnyGt,
+// AnyGte, Glob, NotGlob, IGlob, NotIGlob, ContainsAllTokens, ContainsAnyToken, And, Or, Not.
+// Refused with a 400 naming the gap: Regex, Fuzzy, ContainsTokenSequence.
+//
+// Semantics a caller depends on (Dub's provider relies on the first one explicitly): an attribute a
+// document does not carry reads as null, so `NotIn` / `NotEq` / `NotContainsAny` MATCH it and
+// `In` / `Eq x` / `ContainsAny` do not. Comparison operators are refused on an attribute whose
+// schema says `filterable: false` (the default for a full-text-search attribute, per the SDK's
+// `AttributeSchemaConfig.full_text_search` doc); the token operators read the BM25 index instead
+// and require `full_text_search` on the attribute.
+import { MODELED_TOKENIZERS, queryTerms, tokenize, tokensMatch, type FtsConfig } from './turbopuffer-text.ts';
+
+/** A vendor-shaped API failure: served as `{ "status": "error", "error": message }`. */
+export class TurbopufferError extends Error {
+  constructor(readonly status: number, message: string) {
+    super(message);
+    this.name = 'TurbopufferError';
+  }
+}
+
+/** 400: a request outside the shape turbopuffer's pages give it (a malformed filter, schema, write or query, a value of
+ *  the wrong type, a parameter out of its documented range). The messages are the twin's; the pages name the rules, not
+ *  the text of their refusals. */
+export function badRequest(message: string): TurbopufferError {
+  return new TurbopufferError(400, message);
+}
+
+export type AttrConfig = {
+  /** Transient: an `ann.distance_metric` on a vector schema entry, moved onto the namespace. */
+  ann_distance_metric?: string;
+  type: string;
+  filterable: boolean;
+  full_text_search: FtsConfig | false;
+  glob?: boolean;
+  regex?: boolean;
+  fuzzy?: boolean;
+  /** a `{}f16` attribute's `sparse_knn` config (`distance_metric: dot_product`) */
+  sparse_knn?: { distance_metric: string };
+  /** a `[][N]f32` attribute's `ann: {late_interaction: true}` (false: stored without the index) */
+  late_interaction?: boolean;
+  /** native embedding: the model, the vector attribute the embeddings are stored in, and its dimensions */
+  embed?: { model: string; attribute: string; dims: number };
+};
+
+export type Doc = {
+  /** `n:<id>` or `s:<id>` — the id's JSON type is part of its identity (7 and "7" are two docs). */
+  key: string;
+  id: string | number;
+  attributes: Record<string, unknown>;
+  vector: number[] | null;
+};
+
+export type FilterContext = {
+  schema: Record<string, AttrConfig>;
+  /** Per (attribute, doc key) token cache, shared across one query. */
+  tokenCache: Map<string, string[]>;
+};
+
+export type Predicate = (doc: Doc) => boolean;
+
+const COMPARISON = new Set(['Eq', 'NotEq', 'In', 'NotIn', 'Contains', 'NotContains', 'ContainsAny', 'NotContainsAny', 'Lt', 'Lte', 'Gt', 'Gte', 'AnyLt', 'AnyLte', 'AnyGt', 'AnyGte', 'Glob', 'NotGlob', 'IGlob', 'NotIGlob']);
+const TOKEN_OPS = new Set(['ContainsAllTokens', 'ContainsAnyToken']);
+const UNMODELED_OPS: Record<string, string> = {
+  ContainsTokenSequence: 'turbopuffer.filters.contains_token_sequence',
+};
+
+/**
+ * A namespace observed from a real account may carry a full-text config the twin does not model
+ * (the vendor default `word_v4`, stemming, another language). Such an attribute is refused at
+ * query time rather than tokenized as if it were `word_v2`.
+ */
+export function assertModelledFts(attr: string, fts: FtsConfig): void {
+  if (!MODELED_TOKENIZERS.includes(fts.tokenizer)) throw badRequest(`turbopuffer twin: attribute "${attr}" uses tokenizer "${fts.tokenizer}", which this twin does not model yet (turbopuffer.fts.tokenizers is the filed gap)`);
+  if (fts.language !== 'english') throw badRequest(`turbopuffer twin: attribute "${attr}" uses language "${fts.language}", which this twin does not model (turbopuffer.fts.languages is the filed gap)`);
+}
+
+export function attributeValue(doc: Doc, attr: string): unknown {
+  if (attr === 'id') return doc.id;
+  if (attr === 'vector') return doc.vector;
+  const v = doc.attributes[attr];
+  return v === undefined ? null : v;
+}
+
+function sameValue(a: unknown, b: unknown): boolean {
+  if (a === b) return true;
+  if (a === null || b === null || typeof a !== 'object' || typeof b !== 'object') return false;
+  return JSON.stringify(a) === JSON.stringify(b);
+}
+
+/** -1/0/1 for two comparable scalars (number↔number, string↔string), NaN otherwise. */
+export function compareScalars(a: unknown, b: unknown): number {
+  if (typeof a === 'number' && typeof b === 'number') return a < b ? -1 : a > b ? 1 : 0;
+  if (typeof a === 'string' && typeof b === 'string') return a < b ? -1 : a > b ? 1 : 0;
+  if (typeof a === 'boolean' && typeof b === 'boolean') return a === b ? 0 : a ? 1 : -1;
+  return Number.NaN;
+}
+
+function globRegex(pattern: string, insensitive: boolean): RegExp {
+  let out = '';
+  for (let i = 0; i < pattern.length; i++) {
+    const c = pattern[i]!;
+    if (c === '*') out += '.*';
+    else if (c === '?') out += '.';
+    else if (c === '[') {
+      const close = pattern.indexOf(']', i + 1);
+      if (close === -1) { out += '\\['; continue; }
+      let body = pattern.slice(i + 1, close);
+      if (body.startsWith('!')) body = `^${body.slice(1)}`;
+      out += `[${body.replace(/\\/g, '\\\\')}]`;
+      i = close;
+    } else out += c.replace(/[.*+?^${}()|[\]\\/]/g, '\\$&');
+  }
+  return new RegExp(`^${out}$`, insensitive ? 'is' : 's');
+}
+
+function describe(f: unknown): string {
+  const s = JSON.stringify(f);
+  return s.length > 120 ? `${s.slice(0, 117)}...` : s;
+}
+
+/** Compile a filter into a predicate, refusing a malformed one with a 400. */
+export function compileFilter(f: unknown, ctx: FilterContext): Predicate {
+  if (!Array.isArray(f) || f.length === 0) throw badRequest(`invalid filter: expected a non-empty array, got ${describe(f)}`);
+  const head = f[0];
+  if (head === 'And' || head === 'Or') {
+    if (f.length !== 2 || !Array.isArray(f[1])) throw badRequest(`invalid filter: ${head} takes one array of filters, got ${describe(f)}`);
+    const parts = (f[1] as unknown[]).map((p) => compileFilter(p, ctx));
+    return head === 'And' ? (d) => parts.every((p) => p(d)) : (d) => parts.some((p) => p(d));
+  }
+  if (head === 'Not') {
+    if (f.length !== 2) throw badRequest(`invalid filter: Not takes one filter, got ${describe(f)}`);
+    const inner = compileFilter(f[1], ctx);
+    return (d) => !inner(d);
+  }
+  if (typeof head !== 'string') throw badRequest(`invalid filter: expected an attribute name, got ${describe(f)}`);
+  const attr = head;
+  const op = f[1];
+  if (typeof op !== 'string') throw badRequest(`invalid filter: expected an operator for attribute "${attr}", got ${describe(f)}`);
+  const gap = UNMODELED_OPS[op];
+  if (gap !== undefined) throw badRequest(`turbopuffer twin: the ${op} filter is real Turbopuffer surface this twin does not model yet (${gap} is the filed gap)`);
+  const config = ctx.schema[attr];
+
+  if (TOKEN_OPS.has(op)) {
+    if (f.length < 3 || f.length > 4) throw badRequest(`invalid filter: ${op} takes a query and optional params, got ${describe(f)}`);
+    if (config === undefined || config.full_text_search === false) throw badRequest(`invalid filter: ${op} requires full-text search to be enabled on attribute "${attr}"`);
+    const fts = config.full_text_search;
+    assertModelledFts(attr, fts);
+    const params = (f[3] ?? {}) as Record<string, unknown>;
+    if (typeof params !== 'object' || params === null || Array.isArray(params)) throw badRequest(`invalid filter: ${op} params must be an object`);
+    for (const k of Object.keys(params)) if (k !== 'last_as_prefix') throw badRequest(`invalid filter: unknown ${op} param "${k}"`);
+    const query = f[2];
+    if (typeof query !== 'string' && !(Array.isArray(query) && query.every((q) => typeof q === 'string'))) throw badRequest(`invalid filter: ${op} query must be a string or an array of strings`);
+    const terms = queryTerms(query, fts, params.last_as_prefix === true);
+    const mode = op === 'ContainsAllTokens' ? 'all' : 'any';
+    return (d) => tokensMatch(docTokens(ctx, attr, fts, d), terms, mode);
+  }
+
+  if (op === 'Regex') return regexFilter(attr, config, f);
+  if (op === 'Fuzzy') return fuzzyFilter(attr, config, f);
+  if (!COMPARISON.has(op)) throw badRequest(`invalid filter: unknown operator "${op}" in ${describe(f)}`);
+  if (f.length !== 3) throw badRequest(`invalid filter: ${op} takes exactly one value, got ${describe(f)}`);
+  // Glob "Requires the glob (or for backwards compatibility, filterable) schema attribute" (https://turbopuffer.com/docs/query)
+  const globbed = /^(Not)?I?Glob$/.test(op) && config?.glob === true;
+  if (config !== undefined && !config.filterable && !globbed && attr !== 'id') {
+    throw badRequest(`invalid filter: attribute "${attr}" is not filterable (set filterable: true in its schema)`);
+  }
+  const value = f[2];
+  const needsArray = op === 'In' || op === 'NotIn' || op === 'ContainsAny' || op === 'NotContainsAny';
+  if (needsArray && !Array.isArray(value)) throw badRequest(`invalid filter: ${op} requires an array value, got ${describe(f)}`);
+  const list = needsArray ? (value as unknown[]) : [];
+  // Ids order numbers before strings (the order rank_by [id, asc] serves), so paging by id never skips a type.
+  const cmp = attr === 'id' ? compareIdValues : compareScalars;
+  switch (op) {
+    case 'Eq': return (d) => sameValue(attributeValue(d, attr), value);
+    case 'NotEq': return (d) => !sameValue(attributeValue(d, attr), value);
+    case 'In': return (d) => { const v = attributeValue(d, attr); return v !== null && list.some((x) => sameValue(v, x)); };
+    case 'NotIn': return (d) => { const v = attributeValue(d, attr); return v === null || !list.some((x) => sameValue(v, x)); };
+    case 'Contains': return (d) => { const v = attributeValue(d, attr); return Array.isArray(v) && v.some((x) => sameValue(x, value)); };
+    case 'NotContains': return (d) => { const v = attributeValue(d, attr); return !(Array.isArray(v) && v.some((x) => sameValue(x, value))); };
+    case 'ContainsAny': return (d) => { const v = attributeValue(d, attr); return Array.isArray(v) && v.some((x) => list.some((y) => sameValue(x, y))); };
+    case 'NotContainsAny': return (d) => { const v = attributeValue(d, attr); return !(Array.isArray(v) && v.some((x) => list.some((y) => sameValue(x, y)))); };
+    case 'Lt': return (d) => cmp(attributeValue(d, attr), value) < 0;
+    case 'Lte': return (d) => cmp(attributeValue(d, attr), value) <= 0;
+    case 'Gt': return (d) => cmp(attributeValue(d, attr), value) > 0;
+    case 'Gte': return (d) => cmp(attributeValue(d, attr), value) >= 0;
+    case 'AnyLt': return (d) => anyOf(attributeValue(d, attr), (x) => compareScalars(x, value) < 0);
+    case 'AnyLte': return (d) => anyOf(attributeValue(d, attr), (x) => compareScalars(x, value) <= 0);
+    case 'AnyGt': return (d) => anyOf(attributeValue(d, attr), (x) => compareScalars(x, value) > 0);
+    case 'AnyGte': return (d) => anyOf(attributeValue(d, attr), (x) => compareScalars(x, value) >= 0);
+    case 'Glob': case 'NotGlob': case 'IGlob': case 'NotIGlob': {
+      if (typeof value !== 'string') throw badRequest(`invalid filter: ${op} requires a string pattern`);
+      let re: RegExp;
+      try { re = globRegex(value, op === 'IGlob' || op === 'NotIGlob'); } catch { throw badRequest(`invalid filter: ${op} pattern ${JSON.stringify(value)} is not a valid glob`); }
+      const negate = op.startsWith('Not');
+      return (d) => { const v = attributeValue(d, attr); const hit = typeof v === 'string' && re.test(v); return negate ? !hit : hit; };
+    }
+  }
+  throw badRequest(`invalid filter: unknown operator "${op}"`);
+}
+
+/** Regex (https://turbopuffer.com/docs/query): "Regular expression match against `string` attribute values. Requires the
+ *  regex schema attribute to be enabled before use"; its example `["text", "Regex", "\\w+fish"]` "matches "swordfish",
+ *  "pufferfish", "clownfish"", so a pattern matches anywhere in the value. Where the docs stop: the twin runs the pattern
+ *  as a JavaScript regular expression (Unicode mode), which agrees with Rust's regex syntax for the common classes. */
+function regexFilter(attr: string, config: AttrConfig | undefined, f: unknown[]): Predicate {
+  if (config === undefined || !config.regex) throw badRequest(`invalid filter: Regex requires regex: true in the schema of attribute "${attr}"`);
+  if (f.length !== 3 || typeof f[2] !== 'string') throw badRequest('invalid filter: Regex takes a pattern string');
+  let re: RegExp;
+  try { re = new RegExp(f[2], 'u'); } catch { throw badRequest(`invalid filter: Regex pattern ${JSON.stringify(f[2])} is not a valid regular expression`); }
+  return (d) => { const v = attributeValue(d, attr); return typeof v === 'string' && re.test(v); };
+}
+
+/** Fuzzy (https://turbopuffer.com/docs/query and /docs/fts, Fuzzy matching): "Fuzzy substring match against `string` or
+ *  `[]string` attribute values. Requires the fuzzy schema attribute". `max_edit_distance` "sets how many edits to
+ *  tolerate by the query length in characters (uses Levenshtein distance). `distance` can be `0`, `1`, or `2`, and
+ *  `min_query_chars` must be at least 3 · (`distance` + 1). Queries shorter than the first `min_query_chars` threshold
+ *  return no matches"; `case_sensitive` "defaults to `true`", and "A missing or added character, incorrect character,
+ *  missing or added diacritic (e.g. ü), or case difference will add 1 to the edit distance". The twin measures the
+ *  fewest edits between the query and any substring of the value, over NFC characters. */
+function fuzzyFilter(attr: string, config: AttrConfig | undefined, f: unknown[]): Predicate {
+  if (config === undefined || !config.fuzzy) throw badRequest(`invalid filter: Fuzzy requires fuzzy: true in the schema of attribute "${attr}"`);
+  if (f.length !== 4 || typeof f[2] !== 'string' || f[3] === null || typeof f[3] !== 'object' || Array.isArray(f[3])) throw badRequest('invalid filter: Fuzzy takes a query string and { max_edit_distance, case_sensitive? }');
+  const opts = f[3] as Record<string, unknown>;
+  const steps = opts.max_edit_distance;
+  if (!Array.isArray(steps) || steps.length === 0) throw badRequest('invalid filter: Fuzzy requires max_edit_distance');
+  const table = steps.map((s) => {
+    const e = s as Record<string, unknown>;
+    if (e === null || typeof e !== 'object' || ![0, 1, 2].includes(e.distance as number) || typeof e.min_query_chars !== 'number' || e.min_query_chars < 3 * ((e.distance as number) + 1)) {
+      throw badRequest('invalid filter: each max_edit_distance entry is {min_query_chars, distance}, distance 0, 1 or 2 and min_query_chars at least 3 · (distance + 1)');
+    }
+    return { min: e.min_query_chars as number, distance: e.distance as number };
+  }).sort((a, b) => a.min - b.min);
+  const sensitive = opts.case_sensitive !== false;
+  const norm = (s: string): string[] => [...(sensitive ? s.normalize('NFC') : s.normalize('NFC').toLowerCase())];
+  const q = norm(f[2]);
+  const allowed = table.filter((t) => q.length >= t.min).at(-1)?.distance;
+  if (allowed === undefined) return () => false;
+  const within = (value: string): boolean => substringDistance(q, norm(value)) <= allowed;
+  return (d) => { const v = attributeValue(d, attr); return typeof v === 'string' ? within(v) : Array.isArray(v) && v.some((x) => typeof x === 'string' && within(x)); };
+}
+
+/** The fewest Levenshtein edits turning the query into some substring of the text (Sellers' algorithm). */
+function substringDistance(q: string[], t: string[]): number {
+  let prev = new Array(t.length + 1).fill(0);
+  for (let i = 1; i <= q.length; i++) {
+    const cur = [i];
+    for (let j = 1; j <= t.length; j++) cur[j] = Math.min(prev[j] + 1, cur[j - 1]! + 1, prev[j - 1] + (q[i - 1] === t[j - 1] ? 0 : 1));
+    prev = cur;
+  }
+  return Math.min(...prev);
+}
+
+function compareIdValues(a: unknown, b: unknown): number {
+  const ta = typeof a; const tb = typeof b;
+  if ((ta !== 'number' && ta !== 'string') || (tb !== 'number' && tb !== 'string')) return Number.NaN;
+  if (ta !== tb) return ta === 'number' ? -1 : 1;
+  return compareScalars(a, b);
+}
+
+function anyOf(v: unknown, test: (x: unknown) => boolean): boolean {
+  return Array.isArray(v) && v.some(test);
+}
+
+export function docTokens(ctx: FilterContext, attr: string, fts: FtsConfig, d: Doc): string[] {
+  const key = `${attr}\u0000${d.key}`;
+  let tokens = ctx.tokenCache.get(key);
+  if (tokens === undefined) {
+    tokens = tokenize(attributeValue(d, attr), fts);
+    ctx.tokenCache.set(key, tokens);
+  }
+  return tokens;
+}

package/src/turbopuffer-server.ts

@@ -0,0 +1,81 @@
+// FETCH-FIRST (runtime contract R12b): the pack's HTTP surface is a plain `(Request) => Promise<Response>`, and the
+// local server is the kernel's serve seam around the SAME closure. The SDK never gzips a request body unless the
+// caller opts into `compression: true` (client.ts: `options.compression === undefined ? false`), so the handler's
+// text read of the body is faithful.
+import { compileSurface, createDerivedFetch, createTwinFetchFromHandler, matchOperation, semanticsContext, serveHttp, statefulTwinManifest, type DerivedFetch, type DerivedOperation, type DerivedSurface } from '@volter/world-core';
+import surface from './generated/surface.gen.json' with { type: 'json' };
+import { keyGate } from './key-gate.ts';
+import { manifest } from './manifest.ts';
+import { DASHBOARD_HOST, dashboard } from './screens/dashboard.tsx';
+import { turbopufferHandlers } from './semantics/namespaces.ts';
+import { handleTurbopufferTwinRequest, normalizeTurbopufferPath } from './turbopuffer-twin.ts';
+
+export interface TurbopufferTwinFetchOptions {
+  root?: string;
+  readOnly?: boolean;
+  /** The API key the twin demands. Omit to accept any non-empty key (a missing one still 401s). */
+  token?: string;
+}
+
+// the operation a context is opened under for the dashboard's pages, which answer no operation of the spec
+const DASHBOARD: DerivedOperation = { id: 'TurbopufferDashboard', method: 'GET', path: '/dashboard', class: 'action' };
+const routes = compileSurface(surface as DerivedSurface);
+
+/**
+ * The pack's wire (docs/contributing/architecture.md, "Protocol 3"): the twin's discovery door (`GET /twin`) in
+ * front, turbopuffer's dashboard on turbopuffer.com (screens/dashboard.tsx: sign-in and API keys), then the key gate
+ * (key-gate.ts) and the derived dispatch over turbopuffer's spec. A leading region segment (`/aws-us-east-1/v2/...`) is
+ * dropped first: it is how a World points an unmodified client here through the SDK's own `TURBOPUFFER_BASE_URL`,
+ * whose `{region}` placeholder the client fills in (the pack descriptor's `endpointEnv`). The operations the twin
+ * serves go to their handlers (semantics/namespaces.ts); every other one, and a path the spec does not have, is the
+ * gap.
+ */
+export function createTurbopufferTwinFetch(options: TurbopufferTwinFetchOptions = {}): DerivedFetch {
+  const answer = createTwinFetchFromHandler(handleTurbopufferTwinRequest, {
+    ...(options.root !== undefined ? { root: options.root } : {}),
+    ...(options.readOnly !== undefined ? { readOnly: options.readOnly } : {}),
+    ...(options.token !== undefined ? { handlerOptions: { token: options.token } } : {}),
+    manifest: statefulTwinManifest({
+      vendor: 'turbopuffer',
+      twinOf: 'the Turbopuffer API (v2 namespaces: write, query and multi-query, deleteAll, metadata, schema, namespace listing, cache warm hint)',
+      stores: 'namespaces (schema, distance metric) and their documents; queries filter, BM25-rank, order and count them deterministically',
+      identity: 'Send `Authorization: Bearer <any non-empty key>`, as the SDK does from TURBOPUFFER_API_KEY. A leading region path segment (/aws-us-east-1/v2/...) is accepted, which is how TURBOPUFFER_BASE_URL=<twin>/{region} reaches the twin.',
+    }),
+  });
+  const derived = createDerivedFetch({
+    surface: surface as DerivedSurface,
+    handlers: turbopufferHandlers(answer),
+    gap: (request) => turbopufferGap(request),
+  });
+  const scope = options.root !== undefined ? { root: options.root } : {};
+  const contextFor = (request: Request, operation: DerivedOperation = DASHBOARD) => semanticsContext(manifest, request, operation, scope);
+  return Object.assign(async (request: Request): Promise<Response> => {
+    const url = new URL(request.url);
+    // the vendor host a redirected request names (the injector's fetch path, a hosted World), else its own
+    const host = (request.headers.get('x-volter-twin-original-host') ?? url.host).split(':')[0]!.toLowerCase();
+    if (host === DASHBOARD_HOST) return (await dashboard(request, (r) => contextFor(r))) ?? new Response('Not Found', { status: 404, headers: { 'content-type': 'text/plain' } });
+    if (request.method === 'GET' && url.pathname.replace(/\/+$/, '') === '/twin') return answer(request);
+    const path = normalizeTurbopufferPath(url.pathname);
+    const routed = path === url.pathname ? request : new Request(Object.assign(url, { pathname: path }), request);
+    const matched = matchOperation(routes, routed.method, new URL(routed.url).pathname, new URL(routed.url).searchParams, routed.headers);
+    const refused = matched ? keyGate(await contextFor(new Request(routed.url), matched.operation), routed, matched.operation.id) : undefined;
+    return refused ?? derived(routed);
+  }, { owners: () => derived.owners() });
+}
+
+/** What the twin answers a request it has no operation for, in turbopuffer's error body
+ *  (https://turbopuffer.com/docs/auth, "Error responses"). Where the documentation stops: no page prints the answer
+ *  to an unknown path or an operation the twin does not model; 404 is the twin's. */
+function turbopufferGap(request: Request): Response {
+  const url = new URL(request.url);
+  return Response.json({ status: 'error', error: `not found: ${request.method} ${url.pathname}` }, { status: 404 });
+}
+
+export async function createTurbopufferTwinServer(options: TurbopufferTwinFetchOptions & { port?: number } = {}): Promise<{ port: number; stop: () => void }> {
+  const server = await serveHttp({
+    port: options.port ?? 0,
+    idleTimeout: 60,
+    fetch: createTurbopufferTwinFetch(options),
+  });
+  return { port: server.port ?? options.port ?? 0, stop: () => server.stop(true) };
+}

package/src/turbopuffer-stem.ts

@@ -0,0 +1,104 @@
+// ENGLISH STEMMING — `full_text_search: {stemming: true}` with `language: "english"`: "Language-specific stemming for
+// the text" (https://turbopuffer.com/docs/write, full_text_search). turbopuffer's analyzer (alyze, which "Includes a
+// complete analyzer implementation, with support for lowercasing, ASCII case folding, stemming & stopword removal",
+// github.com/turbopuffer/alyze src/analyze/mod.rs) stems with the `rust_stemmers` crate's Snowball algorithms; for
+// English that is the Snowball English ("Porter2") stemmer, written out here from its published definition
+// (https://snowballstem.org/algorithms/english/stemmer.html). Other languages' stemmers are not modelled.
+
+const VOWELS = new Set(['a', 'e', 'i', 'o', 'u', 'y']);
+const isVowel = (w: string, i: number): boolean => VOWELS.has(w[i]!);
+const DOUBLES = ['bb', 'dd', 'ff', 'gg', 'mm', 'nn', 'pp', 'rr', 'tt'];
+const LI_ENDINGS = new Set(['c', 'd', 'e', 'g', 'h', 'k', 'm', 'n', 'r', 't']);
+
+const EXCEPTIONS: Record<string, string> = {
+  skis: 'ski', skies: 'sky', dying: 'die', lying: 'lie', tying: 'tie', idly: 'idl', gently: 'gentl', ugly: 'ugli',
+  early: 'earli', only: 'onli', singly: 'singl', sky: 'sky', news: 'news', howe: 'howe', atlas: 'atlas', cosmos: 'cosmos',
+  bias: 'bias', andes: 'andes',
+};
+const AFTER_1A = new Set(['inning', 'outing', 'canning', 'herring', 'earring', 'proceed', 'exceed', 'succeed']);
+
+/** The start of the region after the first non-vowel following a vowel, from `from`. */
+function regionAfter(w: string, from: number): number {
+  for (let i = from + 1; i < w.length; i++) if (!isVowel(w, i) && isVowel(w, i - 1)) return i + 1;
+  return w.length;
+}
+
+/** A short syllable ending at index `i` (inclusive). */
+function shortSyllableAt(w: string, i: number): boolean {
+  if (i === 1) return isVowel(w, 0) && !isVowel(w, 1);
+  if (i < 2) return false;
+  return !isVowel(w, i - 2) && isVowel(w, i - 1) && !isVowel(w, i) && !['w', 'x', 'Y'].includes(w[i]!);
+}
+
+/** Step 2's rewrite of a suffix in R1: ogi to og after l, li deleted after a valid li-ending, the rest by the table. */
+function step2(w: string, s2: string, table: Record<string, string>): string {
+  if (s2 === 'ogi') return w[w.length - 4] === 'l' ? `${w.slice(0, -3)}og` : w;
+  if (s2 === 'li') return LI_ENDINGS.has(w[w.length - 3] ?? '') ? w.slice(0, -2) : w;
+  return w.slice(0, -s2.length) + table[s2]!;
+}
+
+/** Step 3's rewrite of a suffix in R1 by the table (ative, which needs R2, is the caller's). */
+function step3(w: string, s3: string, table: Record<string, string>): string {
+  return w.slice(0, -s3.length) + table[s3]!;
+}
+
+export function stemEnglish(word: string): string {
+  if (word.length <= 2) return word;
+  if (EXCEPTIONS[word] !== undefined) return EXCEPTIONS[word]!;
+  let w = word.startsWith("'") ? word.slice(1) : word;
+  // initial y, and y after a vowel, are consonants: marked Y
+  w = w.replace(/^y/, 'Y').replace(/([aeiouy])y/g, '$1Y');
+  let r1 = /^(gener|commun|arsen)/.test(w) ? /^(gener|commun|arsen)/.exec(w)![0].length : regionAfter(w, 0);
+  const r2 = (): number => regionAfter(w, r1);
+  const inR1 = (suffix: string): boolean => w.length - suffix.length >= r1;
+  const inR2 = (suffix: string): boolean => w.length - suffix.length >= r2();
+  const ends = (s: string): boolean => w.endsWith(s);
+  const cut = (n: number): void => { w = w.slice(0, w.length - n); };
+  const longest = (list: string[]): string | undefined => list.filter(ends).sort((a, b) => b.length - a.length)[0];
+  const hasVowelBefore = (end: number): boolean => { for (let i = 0; i < end; i++) if (isVowel(w, i)) return true; return false; };
+
+  // Step 0
+  const s0 = longest(["'s'", "'s", "'"]);
+  if (s0) cut(s0.length);
+  // Step 1a
+  const s1a = longest(['sses', 'ied', 'ies', 'us', 'ss', 's']);
+  if (s1a === 'sses') cut(2);
+  else if (s1a === 'ied' || s1a === 'ies') { cut(3); w += w.length > 1 ? 'i' : 'ie'; }
+  else if (s1a === 's') { if (hasVowelBefore(w.length - 2)) cut(1); }
+  if (AFTER_1A.has(w)) return w;
+  // Step 1b
+  const s1b = longest(['eedly', 'eed', 'ingly', 'edly', 'ing', 'ed']);
+  if (s1b === 'eed' || s1b === 'eedly') { if (inR1(s1b)) { cut(s1b.length); w += 'ee'; } }
+  else if (s1b && hasVowelBefore(w.length - s1b.length)) {
+    cut(s1b.length);
+    if (ends('at') || ends('bl') || ends('iz')) w += 'e';
+    else if (DOUBLES.some(ends)) cut(1);
+    else if (r1 >= w.length && shortSyllableAt(w, w.length - 1)) w += 'e';
+  }
+  // Step 1c
+  if ((ends('y') || ends('Y')) && w.length > 2 && !isVowel(w, w.length - 2)) w = `${w.slice(0, -1)}i`;
+  // Step 2
+  const STEP2: Record<string, string> = {
+    tional: 'tion', enci: 'ence', anci: 'ance', abli: 'able', entli: 'ent', izer: 'ize', ization: 'ize', ational: 'ate',
+    ation: 'ate', ator: 'ate', alism: 'al', aliti: 'al', alli: 'al', fulness: 'ful', ousli: 'ous', ousness: 'ous',
+    iveness: 'ive', iviti: 'ive', biliti: 'ble', bli: 'ble', ogi: 'og', fulli: 'ful', lessli: 'less', li: '',
+  };
+  const s2 = longest(Object.keys(STEP2));
+  if (s2 && inR1(s2)) w = step2(w, s2, STEP2);
+  // Step 3
+  const STEP3: Record<string, string> = { tional: 'tion', ational: 'ate', alize: 'al', icate: 'ic', iciti: 'ic', ical: 'ic', ful: '', ness: '', ative: '' };
+  const s3 = longest(Object.keys(STEP3));
+  if (s3 && inR1(s3)) w = s3 === 'ative' ? (inR2(s3) ? w.slice(0, -5) : w) : step3(w, s3, STEP3);
+  // Step 4
+  const s4 = longest(['al', 'ance', 'ence', 'er', 'ic', 'able', 'ible', 'ant', 'ement', 'ment', 'ent', 'ism', 'ate', 'iti', 'ous', 'ive', 'ize', 'ion']);
+  if (s4 && inR2(s4)) {
+    if (s4 === 'ion') { if (['s', 't'].includes(w[w.length - 4] ?? '')) cut(3); }
+    else cut(s4.length);
+  }
+  // Step 5
+  if (ends('e')) {
+    if (inR2('e') || (inR1('e') && !shortSyllableAt(w, w.length - 2))) cut(1);
+  } else if (ends('l') && inR2('l') && w[w.length - 2] === 'l') cut(1);
+  r1 = 0;
+  return w.replace(/Y/g, 'y');
+}