@volter/twin-turbopuffer 0.1.0

package/dist/src/turbopuffer-text.d.ts

@@ -0,0 +1,44 @@
+export declare const TOKENIZERS: readonly ["pre_tokenized_array", "word_v0", "word_v1", "word_v2", "word_v3", "word_v4"];
+/** The tokenizers this twin models. Every other one in `TOKENIZERS` is refused with a 400. */
+export declare const MODELED_TOKENIZERS: readonly string[];
+/** The vendor's default tokenizer (namespaces.ts: "Defaults to `word_v4`"). */
+export declare const DEFAULT_TOKENIZER = "word_v4";
+export type FtsConfig = {
+    tokenizer: string;
+    case_sensitive: boolean;
+    remove_stopwords: boolean;
+    stemming: boolean;
+    ascii_folding: boolean;
+    language: string;
+    max_token_length: number;
+    k1: number;
+    b: number;
+    k3?: number;
+};
+export declare const DEFAULT_FTS: FtsConfig;
+/**
+ * Tokenize one value (a string, or a `[]string`, whose elements are tokenized and concatenated).
+ * `keepStopwords` is used for the final prefix token of a `last_as_prefix` query, which must not
+ * vanish because the user has so far typed only `an` of `anna`.
+ */
+export declare function tokenize(value: unknown, cfg: FtsConfig, opts?: {
+    keepStopwords?: boolean;
+}): string[];
+/** A query's terms, split into the exact terms and the optional trailing prefix. */
+export type QueryTerms = {
+    exact: string[];
+    prefix: string | null;
+};
+export declare function queryTerms(query: unknown, cfg: FtsConfig, lastAsPrefix: boolean): QueryTerms;
+/** Does a document's token list satisfy the terms? `all` for ContainsAllTokens, else any. */
+export declare function tokensMatch(docTokens: readonly string[], terms: QueryTerms, mode: 'all' | 'any'): boolean;
+/** The per-attribute corpus statistics BM25 needs, computed once per query over the namespace. */
+export type Corpus = {
+    n: number;
+    avgdl: number;
+    df: Map<string, number>;
+    docs: Map<string, string[]>;
+};
+export declare function buildCorpus(entries: Iterable<[string, unknown]>, cfg: FtsConfig): Corpus;
+/** BM25 of one document against the query terms; 0 means "no term matched". */
+export declare function bm25Score(key: string, corpus: Corpus, terms: QueryTerms, cfg: FtsConfig): number;

package/dist/src/turbopuffer-text.js

@@ -0,0 +1,189 @@
+// FULL-TEXT SEARCH — the tokenizer and the BM25 scorer behind `rank_by: [attr, 'BM25', q]` and the
+// `ContainsAllTokens` / `ContainsAnyToken` filters.
+//
+// ── WHAT IS GROUNDED, AND WHERE THE EVIDENCE STOPS ──────────────────────────────────────────
+// Grounded in the installed `@turbopuffer/turbopuffer@2.8.0` (src/resources/namespaces.ts,
+// `FullTextSearchConfig`): the defaults — `k1` 1.2, `b` 0.75, case-insensitive, `language`
+// english, `stemming` false, `ascii_folding` false, `max_token_length` 39
+// BYTES (tokens longer are filtered out), and "by default, BM25-enabled attributes are not
+// filterable". `remove_stopwords` defaults to false: "Defaults to false (i.e. keep common words)"
+// (https://turbopuffer.com/docs/write, full_text_search) since "`remove_stopwords` now defaults to `false`"
+// (https://turbopuffer.com/docs/roadmap, January 2026); the spec's description, and the client's doc comment, still
+// say true. Grounded in Dub's provider (apps/web/lib/api/partners/search/providers/
+// turbopuffer.ts, measured against the real service): `word_v2` splits a URL into its
+// components (so `scottdigital` matches `https://www.scottdigital-42.techcorp.io`), and "a
+// single-token `last_as_prefix` query scores every match at exactly 1".
+//
+// EXTRAPOLATED (deterministic, not the vendor's exact algorithm): `word_v2` is modelled as
+// maximal runs of Unicode letters, digits, marks and `_`; the English stopword list is the
+// classic Lucene/tantivy 33-word list; BM25 is the textbook Okapi form with the Lucene IDF
+// `ln(1 + (N − n + 0.5)/(n + 0.5))` over the namespace's documents that carry the attribute, and a
+// `last_as_prefix` final token contributes a constant 1 per matching document. Scores are
+// therefore deterministic and ordered sensibly, but they are not byte-equal to Turbopuffer's.
+// Tokenizers other than `word_v2` and `pre_tokenized_array`, stemming, and non-English languages
+// are refused (400) rather than silently approximated.
+import { stemEnglish } from "./turbopuffer-stem.js";
+export const TOKENIZERS = ['pre_tokenized_array', 'word_v0', 'word_v1', 'word_v2', 'word_v3', 'word_v4'];
+/** The tokenizers this twin models. Every other one in `TOKENIZERS` is refused with a 400. */
+export const MODELED_TOKENIZERS = ['word_v4', 'word_v3', 'word_v2', 'pre_tokenized_array'];
+/** The vendor's default tokenizer (namespaces.ts: "Defaults to `word_v4`"). */
+export const DEFAULT_TOKENIZER = 'word_v4';
+export const DEFAULT_FTS = {
+    tokenizer: DEFAULT_TOKENIZER,
+    case_sensitive: false,
+    remove_stopwords: false,
+    stemming: false,
+    ascii_folding: false,
+    language: 'english',
+    max_token_length: 39,
+    k1: 1.2,
+    b: 0.75,
+};
+const ENGLISH_STOPWORDS = new Set([
+    'a', 'an', 'and', 'are', 'as', 'at', 'be', 'but', 'by', 'for', 'if', 'in', 'into', 'is', 'it', 'no', 'not',
+    'of', 'on', 'or', 'such', 'that', 'the', 'their', 'then', 'there', 'these', 'they', 'this', 'to', 'was',
+    'will', 'with',
+]);
+const WORD = /[\p{L}\p{N}\p{M}_]+/gu;
+const encoder = new TextEncoder();
+function normalizeToken(token, cfg) {
+    let t = cfg.case_sensitive ? token : token.toLowerCase();
+    if (cfg.ascii_folding)
+        t = t.normalize('NFD').replace(/\p{M}+/gu, '');
+    return t;
+}
+/**
+ * Tokenize one value (a string, or a `[]string`, whose elements are tokenized and concatenated).
+ * `keepStopwords` is used for the final prefix token of a `last_as_prefix` query, which must not
+ * vanish because the user has so far typed only `an` of `anna`.
+ */
+export function tokenize(value, cfg, opts = {}) {
+    const parts = Array.isArray(value) ? value.filter((v) => typeof v === 'string') : typeof value === 'string' ? [value] : [];
+    if (cfg.tokenizer === 'word_v4' || cfg.tokenizer === 'word_v3')
+        return parts.flatMap((part) => uax29Tokens(part, cfg, opts));
+    const out = [];
+    for (const part of parts) {
+        const raw = cfg.tokenizer === 'pre_tokenized_array' ? [part] : (part.match(WORD) ?? []);
+        for (const r of raw) {
+            const t = normalizeToken(r, cfg);
+            if (t === '' || encoder.encode(t).length > cfg.max_token_length)
+                continue;
+            if (cfg.remove_stopwords && !opts.keepStopwords && ENGLISH_STOPWORDS.has(t))
+                continue;
+            out.push(cfg.stemming ? stemEnglish(t) : t);
+        }
+    }
+    return out;
+}
+// ── word_v4 and word_v3 ─────────────────────────────────────────────────────────────────────
+// "The `word_v4` and `word_v3` tokenizers use Unicode v17.0 text segmentation rules (UAX #29) for accurate
+// segmentation across most languages, scripts, and emojis. `word_v4` is the current default for new namespaces; it
+// behaves like `word_v3`, but is roughly 3x faster and fixes a few tokenization edge cases. It's powered by our
+// open-source alyze library" (https://turbopuffer.com/docs/fts, Tokenizers). Modelled on alyze at
+// 1de437c8604b751f26e7061070b9d2b35aebdc73 (github.com/turbopuffer/alyze, src/analyze/mod.rs, src/uax29/word/mod.rs):
+// - word boundaries by UAX #29, and only "word-like" segments become tokens: a segment "is 'word-like' if it contains
+//   any char that is ALetter, HebrewLetter, or Numeric, Ideographic or Extended_Pictographic, Other_Number general
+//   category, [or] a character whose Script is something meaningful … as opposed to Script=Common/Inherited/Unknown";
+// - then, in alyze's order: the maximum token length (alyze admits a token within the limit in bytes or in
+//   characters, `s.len() <= max || s.chars().nth(max).is_none()`), lowercasing unless case_sensitive, stopword removal
+//   (the Lucene English list, as word_v2's), and ASCII folding, lowercased again.
+// Where the evidence stops: the twin segments with the host's Intl.Segmenter (ICU's UAX #29, with runs of ideographs
+// and hiragana split back into characters as UAX #29's rules leave them; ICU's dictionary segmentation of Thai, Lao,
+// Khmer and Burmese is kept, which alyze may not do), whose Unicode version is
+// the runtime's, not 17.0; alyze's own DFA may differ from ICU on edge cases, and the "few tokenization edge cases"
+// word_v4 fixes over word_v3 are not documented, so the two are one tokenizer here; ASCII folding strips combining
+// marks after NFD, where alyze's `ascii_fold` may map more characters (ß, æ).
+const segmenter = new Intl.Segmenter('und', { granularity: 'word' });
+const WORD_LIKE = /[\p{L}\p{Nd}\p{No}\p{Extended_Pictographic}\p{Ideographic}]|[^\p{Script=Common}\p{Script=Inherited}\p{Script=Unknown}]/u;
+function uax29Tokens(text, cfg, opts) {
+    const out = [];
+    // ICU joins runs of ideographs and hiragana by dictionary; UAX #29's rules break between them (neither is ALetter,
+    // and no rule joins them), so such a run is split back into its characters
+    const segments = [...segmenter.segment(text)].flatMap(({ segment }) => (/^[\p{Ideographic}\p{Script=Hiragana}]+$/u.test(segment) ? [...segment] : [segment]));
+    for (const segment of segments) {
+        if (!WORD_LIKE.test(segment))
+            continue;
+        if (!(encoder.encode(segment).length <= cfg.max_token_length || [...segment].length <= cfg.max_token_length))
+            continue;
+        let t = cfg.case_sensitive ? segment : segment.toLowerCase();
+        if (cfg.remove_stopwords && !opts.keepStopwords && ENGLISH_STOPWORDS.has(t))
+            continue;
+        if (cfg.stemming)
+            t = stemEnglish(t);
+        out.push(cfg.ascii_folding && /[^\x00-\x7f]/.test(t) ? foldToken(t, cfg) : t);
+    }
+    return out;
+}
+/** alyze's ASCII folding of a non-ASCII token, lowercased again unless case_sensitive ("ASCII folding can produce
+ *  uppercase ASCII characters, so we'll lowercase again if case folding is enabled", alyze src/analyze/mod.rs). */
+function foldToken(t, cfg) {
+    const folded = t.normalize('NFD').replace(/\p{M}+/gu, '');
+    return cfg.case_sensitive ? folded : folded.toLowerCase();
+}
+export function queryTerms(query, cfg, lastAsPrefix) {
+    if (!lastAsPrefix)
+        return { exact: [...new Set(tokenize(query, cfg))], prefix: null };
+    // The prefix is the LAST token of the raw input, kept even when it is a stopword.
+    const all = tokenize(query, cfg, { keepStopwords: true });
+    if (all.length === 0)
+        return { exact: [], prefix: null };
+    const prefix = all[all.length - 1];
+    const head = cfg.remove_stopwords ? all.slice(0, -1).filter((t) => !ENGLISH_STOPWORDS.has(t)) : all.slice(0, -1);
+    return { exact: [...new Set(head)], prefix };
+}
+/** Does a document's token list satisfy the terms? `all` for ContainsAllTokens, else any. */
+export function tokensMatch(docTokens, terms, mode) {
+    const set = new Set(docTokens);
+    const checks = terms.exact.map((t) => set.has(t));
+    if (terms.prefix !== null) {
+        const p = terms.prefix;
+        checks.push(docTokens.some((t) => t.startsWith(p)));
+    }
+    // A query whose every token is dropped (all stopwords, all over-length) matches nothing, in
+    // either mode — the same documents BM25 would score above zero. The vendor's answer here is
+    // unverified; this keeps the twin's count and its ranked search in agreement.
+    if (checks.length === 0)
+        return false;
+    return mode === 'all' ? checks.every(Boolean) : checks.some(Boolean);
+}
+export function buildCorpus(entries, cfg) {
+    const docs = new Map();
+    const df = new Map();
+    let total = 0;
+    for (const [key, value] of entries) {
+        if (value === null || value === undefined)
+            continue;
+        const tokens = tokenize(value, cfg);
+        docs.set(key, tokens);
+        total += tokens.length;
+        for (const t of new Set(tokens))
+            df.set(t, (df.get(t) ?? 0) + 1);
+    }
+    return { n: docs.size, avgdl: docs.size === 0 ? 0 : total / docs.size, df, docs };
+}
+/** BM25 of one document against the query terms; 0 means "no term matched". */
+export function bm25Score(key, corpus, terms, cfg) {
+    const tokens = corpus.docs.get(key);
+    if (tokens === undefined || tokens.length === 0)
+        return 0;
+    let score = 0;
+    const dl = tokens.length;
+    for (const term of terms.exact) {
+        let tf = 0;
+        for (const t of tokens)
+            if (t === term)
+                tf++;
+        if (tf === 0)
+            continue;
+        const n = corpus.df.get(term) ?? 0;
+        const idf = Math.log(1 + (corpus.n - n + 0.5) / (n + 0.5));
+        const norm = corpus.avgdl === 0 ? 1 : 1 - cfg.b + cfg.b * (dl / corpus.avgdl);
+        score += idf * ((tf * (cfg.k1 + 1)) / (tf + cfg.k1 * norm));
+    }
+    if (terms.prefix !== null) {
+        const p = terms.prefix;
+        if (tokens.some((t) => t.startsWith(p)))
+            score += 1;
+    }
+    return score;
+}

package/dist/src/turbopuffer-twin.d.ts

@@ -0,0 +1,25 @@
+export type TurbopufferTwinRequest = {
+    method: string;
+    path: string;
+    body?: string;
+    headers?: Record<string, string>;
+    root?: string;
+    occurredAt?: string;
+    readOnly?: boolean;
+    /** The API key the twin demands. Omit to accept any non-empty key (a missing one still 401s). */
+    token?: string;
+};
+export type TurbopufferTwinResponse = {
+    status: number;
+    body: unknown;
+    headers?: Record<string, string>;
+};
+/** `Authorization: Bearer <key>` (client.ts `authHeaders`). An empty key is no key. */
+export declare function extractTurbopufferApiKey(headers: Record<string, string> | undefined): string | null;
+/** Collapse duplicate slashes and drop a leading region segment (see the header). */
+export declare function normalizeTurbopufferPath(rawPath: string): string;
+export declare function handleTurbopufferTwinRequest(req: TurbopufferTwinRequest): Promise<TurbopufferTwinResponse>;
+export declare function turbopufferTwinSnapshot(): {
+    resourceTypes: readonly string[];
+    implementedEndpoints: readonly string[];
+};