@volter/twin-turbopuffer 0.1.0

package/dist/src/turbopuffer-connector.js

@@ -0,0 +1,152 @@
+// Turbopuffer's half of the real state system (protocol 2): the PERFORM adapter that replays one
+// recorded write against the vendor, and the REFRESH adapter that observes a real account into
+// the tree. Both run over the kernel's injected `RemoteExecute` — the kernel applies the sealed
+// credential (the placeholder `authorization` below is replaced), so this pack never holds a key
+// and issues no network call of its own.
+import { observeResource, observeResources } from '@volter/world-core';
+import { documentSubjectId, SERVICE } from "./turbopuffer-store.js";
+import { DEFAULT_FTS } from "./turbopuffer-text.js";
+const HEADERS = { accept: 'application/json', 'content-type': 'application/json', authorization: 'Bearer twin' };
+/** Bounds on one refresh, so a huge account cannot turn a sync into an unbounded crawl. */
+export const REFRESH_MAX_NAMESPACE_PAGES = 50;
+export const REFRESH_MAX_DOCUMENTS_PER_NAMESPACE = 100_000;
+const DOC_PAGE = 1000;
+async function call(execute, method, path, body) {
+    const res = await execute({ method, path, headers: { ...HEADERS }, ...(body === undefined ? {} : { body: JSON.stringify(body) }) });
+    // A refusal is thrown, never read as an empty account (adding-a-twin.md §6).
+    if (res.status < 200 || res.status >= 300)
+        throw new Error(`turbopuffer ${method} ${path} refused: HTTP ${res.status} ${res.body.slice(0, 200)}`);
+    try {
+        return JSON.parse(res.body || '{}');
+    }
+    catch {
+        throw new Error(`turbopuffer ${method} ${path} answered a body that is not JSON`);
+    }
+}
+/**
+ * Perform ONE recorded entry against the vendor. A write request is recorded whole (its body is the
+ * action's `input`), so the perform replays exactly what the caller sent; Turbopuffer ids are
+ * caller-chosen, so there is no local→vendor id to rebind.
+ */
+export async function performTurbopufferAction(execute, action, _ctx) {
+    const ns = action.subject.id;
+    const path = `/v2/namespaces/${encodeURIComponent(ns)}`;
+    const input = (action.input ?? {});
+    switch (action.operation) {
+        case 'namespace.write':
+            return { externalId: ns, data: await call(execute, 'POST', path, input) };
+        case 'namespace.delete_all':
+            return { externalId: ns, data: await call(execute, 'DELETE', path) };
+        case 'namespace.update_schema':
+            return { externalId: ns, data: await call(execute, 'POST', `/v1/namespaces/${encodeURIComponent(ns)}/schema`, input.schema ?? {}) };
+    }
+    throw new Error(`turbopuffer perform: no vendor operation for ${action.operation ?? '(unnamed)'} on ${action.subject.type}:${ns}`);
+}
+/** Map one attribute of the vendor's schema wire into the stored form (lenient: the vendor is authoritative). */
+export function mapSchemaAttr(raw) {
+    const r = (typeof raw === 'string' ? { type: raw } : raw ?? {});
+    const fts = r.full_text_search === undefined || r.full_text_search === false || r.full_text_search === null
+        ? false
+        : { ...DEFAULT_FTS, ...(typeof r.full_text_search === 'object' ? r.full_text_search : {}) };
+    return {
+        type: typeof r.type === 'string' ? r.type : 'string',
+        filterable: typeof r.filterable === 'boolean' ? r.filterable : fts === false,
+        full_text_search: fts,
+        ...(r.glob === true ? { glob: true } : {}),
+        ...(r.regex === true ? { regex: true } : {}),
+    };
+}
+/** The refresh adapter (the descriptor's `stateSystem.refresh`): the D7 pull over the kernel's executor. */
+export async function syncTurbopufferFromRemote(execute, opts = {}) {
+    return syncTurbopufferFromReal(execute, opts);
+}
+/**
+ * D7 consumer-facing pull, through the injected executor (the kernel's in production, a fake in
+ * tests): list the account's namespaces, then for each read its schema, metadata and every document
+ * (paged by id), and OBSERVE them — the kernel diffs and folds. Read-only.
+ */
+export async function syncTurbopufferFromReal(execute, opts = {}) {
+    const at = opts.occurredAt ?? new Date().toISOString();
+    const target = { ...(opts.root !== undefined ? { root: opts.root } : {}), at };
+    let observed = 0;
+    let truncated = false;
+    const names = [];
+    let cursor = '';
+    for (let page = 0; page < REFRESH_MAX_NAMESPACE_PAGES; page++) {
+        const listed = await call(execute, 'GET', `/v1/namespaces?page_size=1000${cursor ? `&cursor=${encodeURIComponent(cursor)}` : ''}`);
+        for (const n of listed.namespaces ?? [])
+            if (typeof n.id === 'string')
+                names.push(n.id);
+        cursor = typeof listed.next_cursor === 'string' ? listed.next_cursor : '';
+        if (cursor === '')
+            break;
+    }
+    if (cursor !== '')
+        truncated = true;
+    for (const ns of names) {
+        const enc = encodeURIComponent(ns);
+        const meta = await call(execute, 'GET', `/v2/namespaces/${enc}/metadata`);
+        const wireSchema = (meta.schema ?? {});
+        const schema = {};
+        let distance = null;
+        let dims = null;
+        for (const [attr, raw] of Object.entries(wireSchema)) {
+            schema[attr] = mapSchemaAttr(raw);
+            const m = /^\[(\d+)\]f(16|32)$/.exec(schema[attr].type);
+            if (attr === 'vector' && m) {
+                dims = Number(m[1]);
+                const ann = raw?.ann;
+                if (typeof ann?.distance_metric === 'string')
+                    distance = ann.distance_metric;
+            }
+        }
+        observeResource(SERVICE, {
+            type: 'namespace',
+            id: ns,
+            fields: {
+                name: ns, schema, distance_metric: distance, vector_dims: dims,
+                created_at: typeof meta.created_at === 'string' ? meta.created_at : at,
+                updated_at: typeof meta.updated_at === 'string' ? meta.updated_at : at,
+                last_write_at: typeof meta.last_write_at === 'string' ? meta.last_write_at : typeof meta.updated_at === 'string' ? meta.updated_at : at,
+                gone: false,
+            },
+        }, target);
+        observed++;
+        let last = null;
+        for (let seen = 0; seen < REFRESH_MAX_DOCUMENTS_PER_NAMESPACE;) {
+            const page = await call(execute, 'POST', `/v2/namespaces/${enc}/query`, {
+                rank_by: ['id', 'asc'],
+                top_k: DOC_PAGE,
+                include_attributes: true,
+                ...(last === null ? {} : { filters: ['id', 'Gt', last] }),
+            });
+            const rows = page.rows ?? [];
+            const docs = [];
+            for (const row of rows) {
+                const { id, vector, $dist: _d, ...attributes } = row;
+                if (typeof id !== 'string' && typeof id !== 'number')
+                    continue;
+                for (const k of Object.keys(attributes))
+                    if (attributes[k] === null)
+                        delete attributes[k];
+                docs.push({
+                    type: 'document',
+                    id: documentSubjectId(ns, id),
+                    fields: { namespace: ns, doc_id: id, attributes, vector: Array.isArray(vector) ? vector : null },
+                });
+                last = id;
+            }
+            if (docs.length > 0)
+                observeResources(SERVICE, docs, target);
+            observed += docs.length;
+            seen += rows.length;
+            if (rows.length < DOC_PAGE)
+                break;
+            if (seen >= REFRESH_MAX_DOCUMENTS_PER_NAMESPACE)
+                truncated = true;
+        }
+    }
+    // A listing cut short by the bounds is not the whole account, so it never claims completeness
+    // (the kernel removes what a COMPLETE refresh no longer saw).
+    return { observed, ...(truncated ? {} : { complete: ['namespace', 'document'] }) };
+}

package/dist/src/turbopuffer-filter.d.ts

@@ -0,0 +1,57 @@
+import { type FtsConfig } from './turbopuffer-text.js';
+/** A vendor-shaped API failure: served as `{ "status": "error", "error": message }`. */
+export declare class TurbopufferError extends Error {
+    readonly status: number;
+    constructor(status: number, message: string);
+}
+/** 400: a request outside the shape turbopuffer's pages give it (a malformed filter, schema, write or query, a value of
+ *  the wrong type, a parameter out of its documented range). The messages are the twin's; the pages name the rules, not
+ *  the text of their refusals. */
+export declare function badRequest(message: string): TurbopufferError;
+export type AttrConfig = {
+    /** Transient: an `ann.distance_metric` on a vector schema entry, moved onto the namespace. */
+    ann_distance_metric?: string;
+    type: string;
+    filterable: boolean;
+    full_text_search: FtsConfig | false;
+    glob?: boolean;
+    regex?: boolean;
+    fuzzy?: boolean;
+    /** a `{}f16` attribute's `sparse_knn` config (`distance_metric: dot_product`) */
+    sparse_knn?: {
+        distance_metric: string;
+    };
+    /** a `[][N]f32` attribute's `ann: {late_interaction: true}` (false: stored without the index) */
+    late_interaction?: boolean;
+    /** native embedding: the model, the vector attribute the embeddings are stored in, and its dimensions */
+    embed?: {
+        model: string;
+        attribute: string;
+        dims: number;
+    };
+};
+export type Doc = {
+    /** `n:<id>` or `s:<id>` — the id's JSON type is part of its identity (7 and "7" are two docs). */
+    key: string;
+    id: string | number;
+    attributes: Record<string, unknown>;
+    vector: number[] | null;
+};
+export type FilterContext = {
+    schema: Record<string, AttrConfig>;
+    /** Per (attribute, doc key) token cache, shared across one query. */
+    tokenCache: Map<string, string[]>;
+};
+export type Predicate = (doc: Doc) => boolean;
+/**
+ * A namespace observed from a real account may carry a full-text config the twin does not model
+ * (the vendor default `word_v4`, stemming, another language). Such an attribute is refused at
+ * query time rather than tokenized as if it were `word_v2`.
+ */
+export declare function assertModelledFts(attr: string, fts: FtsConfig): void;
+export declare function attributeValue(doc: Doc, attr: string): unknown;
+/** -1/0/1 for two comparable scalars (number↔number, string↔string), NaN otherwise. */
+export declare function compareScalars(a: unknown, b: unknown): number;
+/** Compile a filter into a predicate, refusing a malformed one with a 400. */
+export declare function compileFilter(f: unknown, ctx: FilterContext): Predicate;
+export declare function docTokens(ctx: FilterContext, attr: string, fts: FtsConfig, d: Doc): string[];

package/dist/src/turbopuffer-filter.js

@@ -0,0 +1,286 @@
+// THE FILTER LANGUAGE — Turbopuffer's array-encoded `filters` (a SQL WHERE clause as JSON), compiled
+// into a predicate over the namespace's documents.
+//
+// The operator set is the one the installed SDK declares (`Filter` in
+// @turbopuffer/turbopuffer@2.8.0 src/resources/custom.ts). Modelled: Eq, NotEq, In, NotIn,
+// Contains, NotContains, ContainsAny, NotContainsAny, Lt, Lte, Gt, Gte, AnyLt, AnyLte, AnyGt,
+// AnyGte, Glob, NotGlob, IGlob, NotIGlob, ContainsAllTokens, ContainsAnyToken, And, Or, Not.
+// Refused with a 400 naming the gap: Regex, Fuzzy, ContainsTokenSequence.
+//
+// Semantics a caller depends on (Dub's provider relies on the first one explicitly): an attribute a
+// document does not carry reads as null, so `NotIn` / `NotEq` / `NotContainsAny` MATCH it and
+// `In` / `Eq x` / `ContainsAny` do not. Comparison operators are refused on an attribute whose
+// schema says `filterable: false` (the default for a full-text-search attribute, per the SDK's
+// `AttributeSchemaConfig.full_text_search` doc); the token operators read the BM25 index instead
+// and require `full_text_search` on the attribute.
+import { MODELED_TOKENIZERS, queryTerms, tokenize, tokensMatch } from "./turbopuffer-text.js";
+/** A vendor-shaped API failure: served as `{ "status": "error", "error": message }`. */
+export class TurbopufferError extends Error {
+    status;
+    constructor(status, message) {
+        super(message);
+        this.status = status;
+        this.name = 'TurbopufferError';
+    }
+}
+/** 400: a request outside the shape turbopuffer's pages give it (a malformed filter, schema, write or query, a value of
+ *  the wrong type, a parameter out of its documented range). The messages are the twin's; the pages name the rules, not
+ *  the text of their refusals. */
+export function badRequest(message) {
+    return new TurbopufferError(400, message);
+}
+const COMPARISON = new Set(['Eq', 'NotEq', 'In', 'NotIn', 'Contains', 'NotContains', 'ContainsAny', 'NotContainsAny', 'Lt', 'Lte', 'Gt', 'Gte', 'AnyLt', 'AnyLte', 'AnyGt', 'AnyGte', 'Glob', 'NotGlob', 'IGlob', 'NotIGlob']);
+const TOKEN_OPS = new Set(['ContainsAllTokens', 'ContainsAnyToken']);
+const UNMODELED_OPS = {
+    ContainsTokenSequence: 'turbopuffer.filters.contains_token_sequence',
+};
+/**
+ * A namespace observed from a real account may carry a full-text config the twin does not model
+ * (the vendor default `word_v4`, stemming, another language). Such an attribute is refused at
+ * query time rather than tokenized as if it were `word_v2`.
+ */
+export function assertModelledFts(attr, fts) {
+    if (!MODELED_TOKENIZERS.includes(fts.tokenizer))
+        throw badRequest(`turbopuffer twin: attribute "${attr}" uses tokenizer "${fts.tokenizer}", which this twin does not model yet (turbopuffer.fts.tokenizers is the filed gap)`);
+    if (fts.language !== 'english')
+        throw badRequest(`turbopuffer twin: attribute "${attr}" uses language "${fts.language}", which this twin does not model (turbopuffer.fts.languages is the filed gap)`);
+}
+export function attributeValue(doc, attr) {
+    if (attr === 'id')
+        return doc.id;
+    if (attr === 'vector')
+        return doc.vector;
+    const v = doc.attributes[attr];
+    return v === undefined ? null : v;
+}
+function sameValue(a, b) {
+    if (a === b)
+        return true;
+    if (a === null || b === null || typeof a !== 'object' || typeof b !== 'object')
+        return false;
+    return JSON.stringify(a) === JSON.stringify(b);
+}
+/** -1/0/1 for two comparable scalars (number↔number, string↔string), NaN otherwise. */
+export function compareScalars(a, b) {
+    if (typeof a === 'number' && typeof b === 'number')
+        return a < b ? -1 : a > b ? 1 : 0;
+    if (typeof a === 'string' && typeof b === 'string')
+        return a < b ? -1 : a > b ? 1 : 0;
+    if (typeof a === 'boolean' && typeof b === 'boolean')
+        return a === b ? 0 : a ? 1 : -1;
+    return Number.NaN;
+}
+function globRegex(pattern, insensitive) {
+    let out = '';
+    for (let i = 0; i < pattern.length; i++) {
+        const c = pattern[i];
+        if (c === '*')
+            out += '.*';
+        else if (c === '?')
+            out += '.';
+        else if (c === '[') {
+            const close = pattern.indexOf(']', i + 1);
+            if (close === -1) {
+                out += '\\[';
+                continue;
+            }
+            let body = pattern.slice(i + 1, close);
+            if (body.startsWith('!'))
+                body = `^${body.slice(1)}`;
+            out += `[${body.replace(/\\/g, '\\\\')}]`;
+            i = close;
+        }
+        else
+            out += c.replace(/[.*+?^${}()|[\]\\/]/g, '\\$&');
+    }
+    return new RegExp(`^${out}$`, insensitive ? 'is' : 's');
+}
+function describe(f) {
+    const s = JSON.stringify(f);
+    return s.length > 120 ? `${s.slice(0, 117)}...` : s;
+}
+/** Compile a filter into a predicate, refusing a malformed one with a 400. */
+export function compileFilter(f, ctx) {
+    if (!Array.isArray(f) || f.length === 0)
+        throw badRequest(`invalid filter: expected a non-empty array, got ${describe(f)}`);
+    const head = f[0];
+    if (head === 'And' || head === 'Or') {
+        if (f.length !== 2 || !Array.isArray(f[1]))
+            throw badRequest(`invalid filter: ${head} takes one array of filters, got ${describe(f)}`);
+        const parts = f[1].map((p) => compileFilter(p, ctx));
+        return head === 'And' ? (d) => parts.every((p) => p(d)) : (d) => parts.some((p) => p(d));
+    }
+    if (head === 'Not') {
+        if (f.length !== 2)
+            throw badRequest(`invalid filter: Not takes one filter, got ${describe(f)}`);
+        const inner = compileFilter(f[1], ctx);
+        return (d) => !inner(d);
+    }
+    if (typeof head !== 'string')
+        throw badRequest(`invalid filter: expected an attribute name, got ${describe(f)}`);
+    const attr = head;
+    const op = f[1];
+    if (typeof op !== 'string')
+        throw badRequest(`invalid filter: expected an operator for attribute "${attr}", got ${describe(f)}`);
+    const gap = UNMODELED_OPS[op];
+    if (gap !== undefined)
+        throw badRequest(`turbopuffer twin: the ${op} filter is real Turbopuffer surface this twin does not model yet (${gap} is the filed gap)`);
+    const config = ctx.schema[attr];
+    if (TOKEN_OPS.has(op)) {
+        if (f.length < 3 || f.length > 4)
+            throw badRequest(`invalid filter: ${op} takes a query and optional params, got ${describe(f)}`);
+        if (config === undefined || config.full_text_search === false)
+            throw badRequest(`invalid filter: ${op} requires full-text search to be enabled on attribute "${attr}"`);
+        const fts = config.full_text_search;
+        assertModelledFts(attr, fts);
+        const params = (f[3] ?? {});
+        if (typeof params !== 'object' || params === null || Array.isArray(params))
+            throw badRequest(`invalid filter: ${op} params must be an object`);
+        for (const k of Object.keys(params))
+            if (k !== 'last_as_prefix')
+                throw badRequest(`invalid filter: unknown ${op} param "${k}"`);
+        const query = f[2];
+        if (typeof query !== 'string' && !(Array.isArray(query) && query.every((q) => typeof q === 'string')))
+            throw badRequest(`invalid filter: ${op} query must be a string or an array of strings`);
+        const terms = queryTerms(query, fts, params.last_as_prefix === true);
+        const mode = op === 'ContainsAllTokens' ? 'all' : 'any';
+        return (d) => tokensMatch(docTokens(ctx, attr, fts, d), terms, mode);
+    }
+    if (op === 'Regex')
+        return regexFilter(attr, config, f);
+    if (op === 'Fuzzy')
+        return fuzzyFilter(attr, config, f);
+    if (!COMPARISON.has(op))
+        throw badRequest(`invalid filter: unknown operator "${op}" in ${describe(f)}`);
+    if (f.length !== 3)
+        throw badRequest(`invalid filter: ${op} takes exactly one value, got ${describe(f)}`);
+    // Glob "Requires the glob (or for backwards compatibility, filterable) schema attribute" (https://turbopuffer.com/docs/query)
+    const globbed = /^(Not)?I?Glob$/.test(op) && config?.glob === true;
+    if (config !== undefined && !config.filterable && !globbed && attr !== 'id') {
+        throw badRequest(`invalid filter: attribute "${attr}" is not filterable (set filterable: true in its schema)`);
+    }
+    const value = f[2];
+    const needsArray = op === 'In' || op === 'NotIn' || op === 'ContainsAny' || op === 'NotContainsAny';
+    if (needsArray && !Array.isArray(value))
+        throw badRequest(`invalid filter: ${op} requires an array value, got ${describe(f)}`);
+    const list = needsArray ? value : [];
+    // Ids order numbers before strings (the order rank_by [id, asc] serves), so paging by id never skips a type.
+    const cmp = attr === 'id' ? compareIdValues : compareScalars;
+    switch (op) {
+        case 'Eq': return (d) => sameValue(attributeValue(d, attr), value);
+        case 'NotEq': return (d) => !sameValue(attributeValue(d, attr), value);
+        case 'In': return (d) => { const v = attributeValue(d, attr); return v !== null && list.some((x) => sameValue(v, x)); };
+        case 'NotIn': return (d) => { const v = attributeValue(d, attr); return v === null || !list.some((x) => sameValue(v, x)); };
+        case 'Contains': return (d) => { const v = attributeValue(d, attr); return Array.isArray(v) && v.some((x) => sameValue(x, value)); };
+        case 'NotContains': return (d) => { const v = attributeValue(d, attr); return !(Array.isArray(v) && v.some((x) => sameValue(x, value))); };
+        case 'ContainsAny': return (d) => { const v = attributeValue(d, attr); return Array.isArray(v) && v.some((x) => list.some((y) => sameValue(x, y))); };
+        case 'NotContainsAny': return (d) => { const v = attributeValue(d, attr); return !(Array.isArray(v) && v.some((x) => list.some((y) => sameValue(x, y)))); };
+        case 'Lt': return (d) => cmp(attributeValue(d, attr), value) < 0;
+        case 'Lte': return (d) => cmp(attributeValue(d, attr), value) <= 0;
+        case 'Gt': return (d) => cmp(attributeValue(d, attr), value) > 0;
+        case 'Gte': return (d) => cmp(attributeValue(d, attr), value) >= 0;
+        case 'AnyLt': return (d) => anyOf(attributeValue(d, attr), (x) => compareScalars(x, value) < 0);
+        case 'AnyLte': return (d) => anyOf(attributeValue(d, attr), (x) => compareScalars(x, value) <= 0);
+        case 'AnyGt': return (d) => anyOf(attributeValue(d, attr), (x) => compareScalars(x, value) > 0);
+        case 'AnyGte': return (d) => anyOf(attributeValue(d, attr), (x) => compareScalars(x, value) >= 0);
+        case 'Glob':
+        case 'NotGlob':
+        case 'IGlob':
+        case 'NotIGlob': {
+            if (typeof value !== 'string')
+                throw badRequest(`invalid filter: ${op} requires a string pattern`);
+            let re;
+            try {
+                re = globRegex(value, op === 'IGlob' || op === 'NotIGlob');
+            }
+            catch {
+                throw badRequest(`invalid filter: ${op} pattern ${JSON.stringify(value)} is not a valid glob`);
+            }
+            const negate = op.startsWith('Not');
+            return (d) => { const v = attributeValue(d, attr); const hit = typeof v === 'string' && re.test(v); return negate ? !hit : hit; };
+        }
+    }
+    throw badRequest(`invalid filter: unknown operator "${op}"`);
+}
+/** Regex (https://turbopuffer.com/docs/query): "Regular expression match against `string` attribute values. Requires the
+ *  regex schema attribute to be enabled before use"; its example `["text", "Regex", "\\w+fish"]` "matches "swordfish",
+ *  "pufferfish", "clownfish"", so a pattern matches anywhere in the value. Where the docs stop: the twin runs the pattern
+ *  as a JavaScript regular expression (Unicode mode), which agrees with Rust's regex syntax for the common classes. */
+function regexFilter(attr, config, f) {
+    if (config === undefined || !config.regex)
+        throw badRequest(`invalid filter: Regex requires regex: true in the schema of attribute "${attr}"`);
+    if (f.length !== 3 || typeof f[2] !== 'string')
+        throw badRequest('invalid filter: Regex takes a pattern string');
+    let re;
+    try {
+        re = new RegExp(f[2], 'u');
+    }
+    catch {
+        throw badRequest(`invalid filter: Regex pattern ${JSON.stringify(f[2])} is not a valid regular expression`);
+    }
+    return (d) => { const v = attributeValue(d, attr); return typeof v === 'string' && re.test(v); };
+}
+/** Fuzzy (https://turbopuffer.com/docs/query and /docs/fts, Fuzzy matching): "Fuzzy substring match against `string` or
+ *  `[]string` attribute values. Requires the fuzzy schema attribute". `max_edit_distance` "sets how many edits to
+ *  tolerate by the query length in characters (uses Levenshtein distance). `distance` can be `0`, `1`, or `2`, and
+ *  `min_query_chars` must be at least 3 · (`distance` + 1). Queries shorter than the first `min_query_chars` threshold
+ *  return no matches"; `case_sensitive` "defaults to `true`", and "A missing or added character, incorrect character,
+ *  missing or added diacritic (e.g. ü), or case difference will add 1 to the edit distance". The twin measures the
+ *  fewest edits between the query and any substring of the value, over NFC characters. */
+function fuzzyFilter(attr, config, f) {
+    if (config === undefined || !config.fuzzy)
+        throw badRequest(`invalid filter: Fuzzy requires fuzzy: true in the schema of attribute "${attr}"`);
+    if (f.length !== 4 || typeof f[2] !== 'string' || f[3] === null || typeof f[3] !== 'object' || Array.isArray(f[3]))
+        throw badRequest('invalid filter: Fuzzy takes a query string and { max_edit_distance, case_sensitive? }');
+    const opts = f[3];
+    const steps = opts.max_edit_distance;
+    if (!Array.isArray(steps) || steps.length === 0)
+        throw badRequest('invalid filter: Fuzzy requires max_edit_distance');
+    const table = steps.map((s) => {
+        const e = s;
+        if (e === null || typeof e !== 'object' || ![0, 1, 2].includes(e.distance) || typeof e.min_query_chars !== 'number' || e.min_query_chars < 3 * (e.distance + 1)) {
+            throw badRequest('invalid filter: each max_edit_distance entry is {min_query_chars, distance}, distance 0, 1 or 2 and min_query_chars at least 3 · (distance + 1)');
+        }
+        return { min: e.min_query_chars, distance: e.distance };
+    }).sort((a, b) => a.min - b.min);
+    const sensitive = opts.case_sensitive !== false;
+    const norm = (s) => [...(sensitive ? s.normalize('NFC') : s.normalize('NFC').toLowerCase())];
+    const q = norm(f[2]);
+    const allowed = table.filter((t) => q.length >= t.min).at(-1)?.distance;
+    if (allowed === undefined)
+        return () => false;
+    const within = (value) => substringDistance(q, norm(value)) <= allowed;
+    return (d) => { const v = attributeValue(d, attr); return typeof v === 'string' ? within(v) : Array.isArray(v) && v.some((x) => typeof x === 'string' && within(x)); };
+}
+/** The fewest Levenshtein edits turning the query into some substring of the text (Sellers' algorithm). */
+function substringDistance(q, t) {
+    let prev = new Array(t.length + 1).fill(0);
+    for (let i = 1; i <= q.length; i++) {
+        const cur = [i];
+        for (let j = 1; j <= t.length; j++)
+            cur[j] = Math.min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (q[i - 1] === t[j - 1] ? 0 : 1));
+        prev = cur;
+    }
+    return Math.min(...prev);
+}
+function compareIdValues(a, b) {
+    const ta = typeof a;
+    const tb = typeof b;
+    if ((ta !== 'number' && ta !== 'string') || (tb !== 'number' && tb !== 'string'))
+        return Number.NaN;
+    if (ta !== tb)
+        return ta === 'number' ? -1 : 1;
+    return compareScalars(a, b);
+}
+function anyOf(v, test) {
+    return Array.isArray(v) && v.some(test);
+}
+export function docTokens(ctx, attr, fts, d) {
+    const key = `${attr}\u0000${d.key}`;
+    let tokens = ctx.tokenCache.get(key);
+    if (tokens === undefined) {
+        tokens = tokenize(attributeValue(d, attr), fts);
+        ctx.tokenCache.set(key, tokens);
+    }
+    return tokens;
+}

package/dist/src/turbopuffer-server.d.ts

@@ -0,0 +1,23 @@
+import { type DerivedFetch } from '@volter/world-core';
+export interface TurbopufferTwinFetchOptions {
+    root?: string;
+    readOnly?: boolean;
+    /** The API key the twin demands. Omit to accept any non-empty key (a missing one still 401s). */
+    token?: string;
+}
+/**
+ * The pack's wire (docs/contributing/architecture.md, "Protocol 3"): the twin's discovery door (`GET /twin`) in
+ * front, turbopuffer's dashboard on turbopuffer.com (screens/dashboard.tsx: sign-in and API keys), then the key gate
+ * (key-gate.ts) and the derived dispatch over turbopuffer's spec. A leading region segment (`/aws-us-east-1/v2/...`) is
+ * dropped first: it is how a World points an unmodified client here through the SDK's own `TURBOPUFFER_BASE_URL`,
+ * whose `{region}` placeholder the client fills in (the pack descriptor's `endpointEnv`). The operations the twin
+ * serves go to their handlers (semantics/namespaces.ts); every other one, and a path the spec does not have, is the
+ * gap.
+ */
+export declare function createTurbopufferTwinFetch(options?: TurbopufferTwinFetchOptions): DerivedFetch;
+export declare function createTurbopufferTwinServer(options?: TurbopufferTwinFetchOptions & {
+    port?: number;
+}): Promise<{
+    port: number;
+    stop: () => void;
+}>;

package/dist/src/turbopuffer-server.js

@@ -0,0 +1,72 @@
+// FETCH-FIRST (runtime contract R12b): the pack's HTTP surface is a plain `(Request) => Promise<Response>`, and the
+// local server is the kernel's serve seam around the SAME closure. The SDK never gzips a request body unless the
+// caller opts into `compression: true` (client.ts: `options.compression === undefined ? false`), so the handler's
+// text read of the body is faithful.
+import { compileSurface, createDerivedFetch, createTwinFetchFromHandler, matchOperation, semanticsContext, serveHttp, statefulTwinManifest } from '@volter/world-core';
+import surface from './generated/surface.gen.json' with { type: 'json' };
+import { keyGate } from "./key-gate.js";
+import { manifest } from "./manifest.js";
+import { DASHBOARD_HOST, dashboard } from "./screens/dashboard.js";
+import { turbopufferHandlers } from "./semantics/namespaces.js";
+import { handleTurbopufferTwinRequest, normalizeTurbopufferPath } from "./turbopuffer-twin.js";
+// the operation a context is opened under for the dashboard's pages, which answer no operation of the spec
+const DASHBOARD = { id: 'TurbopufferDashboard', method: 'GET', path: '/dashboard', class: 'action' };
+const routes = compileSurface(surface);
+/**
+ * The pack's wire (docs/contributing/architecture.md, "Protocol 3"): the twin's discovery door (`GET /twin`) in
+ * front, turbopuffer's dashboard on turbopuffer.com (screens/dashboard.tsx: sign-in and API keys), then the key gate
+ * (key-gate.ts) and the derived dispatch over turbopuffer's spec. A leading region segment (`/aws-us-east-1/v2/...`) is
+ * dropped first: it is how a World points an unmodified client here through the SDK's own `TURBOPUFFER_BASE_URL`,
+ * whose `{region}` placeholder the client fills in (the pack descriptor's `endpointEnv`). The operations the twin
+ * serves go to their handlers (semantics/namespaces.ts); every other one, and a path the spec does not have, is the
+ * gap.
+ */
+export function createTurbopufferTwinFetch(options = {}) {
+    const answer = createTwinFetchFromHandler(handleTurbopufferTwinRequest, {
+        ...(options.root !== undefined ? { root: options.root } : {}),
+        ...(options.readOnly !== undefined ? { readOnly: options.readOnly } : {}),
+        ...(options.token !== undefined ? { handlerOptions: { token: options.token } } : {}),
+        manifest: statefulTwinManifest({
+            vendor: 'turbopuffer',
+            twinOf: 'the Turbopuffer API (v2 namespaces: write, query and multi-query, deleteAll, metadata, schema, namespace listing, cache warm hint)',
+            stores: 'namespaces (schema, distance metric) and their documents; queries filter, BM25-rank, order and count them deterministically',
+            identity: 'Send `Authorization: Bearer <any non-empty key>`, as the SDK does from TURBOPUFFER_API_KEY. A leading region path segment (/aws-us-east-1/v2/...) is accepted, which is how TURBOPUFFER_BASE_URL=<twin>/{region} reaches the twin.',
+        }),
+    });
+    const derived = createDerivedFetch({
+        surface: surface,
+        handlers: turbopufferHandlers(answer),
+        gap: (request) => turbopufferGap(request),
+    });
+    const scope = options.root !== undefined ? { root: options.root } : {};
+    const contextFor = (request, operation = DASHBOARD) => semanticsContext(manifest, request, operation, scope);
+    return Object.assign(async (request) => {
+        const url = new URL(request.url);
+        // the vendor host a redirected request names (the injector's fetch path, a hosted World), else its own
+        const host = (request.headers.get('x-volter-twin-original-host') ?? url.host).split(':')[0].toLowerCase();
+        if (host === DASHBOARD_HOST)
+            return (await dashboard(request, (r) => contextFor(r))) ?? new Response('Not Found', { status: 404, headers: { 'content-type': 'text/plain' } });
+        if (request.method === 'GET' && url.pathname.replace(/\/+$/, '') === '/twin')
+            return answer(request);
+        const path = normalizeTurbopufferPath(url.pathname);
+        const routed = path === url.pathname ? request : new Request(Object.assign(url, { pathname: path }), request);
+        const matched = matchOperation(routes, routed.method, new URL(routed.url).pathname, new URL(routed.url).searchParams, routed.headers);
+        const refused = matched ? keyGate(await contextFor(new Request(routed.url), matched.operation), routed, matched.operation.id) : undefined;
+        return refused ?? derived(routed);
+    }, { owners: () => derived.owners() });
+}
+/** What the twin answers a request it has no operation for, in turbopuffer's error body
+ *  (https://turbopuffer.com/docs/auth, "Error responses"). Where the documentation stops: no page prints the answer
+ *  to an unknown path or an operation the twin does not model; 404 is the twin's. */
+function turbopufferGap(request) {
+    const url = new URL(request.url);
+    return Response.json({ status: 'error', error: `not found: ${request.method} ${url.pathname}` }, { status: 404 });
+}
+export async function createTurbopufferTwinServer(options = {}) {
+    const server = await serveHttp({
+        port: options.port ?? 0,
+        idleTimeout: 60,
+        fetch: createTurbopufferTwinFetch(options),
+    });
+    return { port: server.port ?? options.port ?? 0, stop: () => server.stop(true) };
+}

package/dist/src/turbopuffer-stem.d.ts

@@ -0,0 +1 @@
+export declare function stemEnglish(word: string): string;