@volter/twin-turbopuffer 0.1.0

package/dist/src/turbopuffer-stem.js

@@ -0,0 +1,133 @@
+// ENGLISH STEMMING — `full_text_search: {stemming: true}` with `language: "english"`: "Language-specific stemming for
+// the text" (https://turbopuffer.com/docs/write, full_text_search). turbopuffer's analyzer (alyze, which "Includes a
+// complete analyzer implementation, with support for lowercasing, ASCII case folding, stemming & stopword removal",
+// github.com/turbopuffer/alyze src/analyze/mod.rs) stems with the `rust_stemmers` crate's Snowball algorithms; for
+// English that is the Snowball English ("Porter2") stemmer, written out here from its published definition
+// (https://snowballstem.org/algorithms/english/stemmer.html). Other languages' stemmers are not modelled.
+const VOWELS = new Set(['a', 'e', 'i', 'o', 'u', 'y']);
+const isVowel = (w, i) => VOWELS.has(w[i]);
+const DOUBLES = ['bb', 'dd', 'ff', 'gg', 'mm', 'nn', 'pp', 'rr', 'tt'];
+const LI_ENDINGS = new Set(['c', 'd', 'e', 'g', 'h', 'k', 'm', 'n', 'r', 't']);
+const EXCEPTIONS = {
+    skis: 'ski', skies: 'sky', dying: 'die', lying: 'lie', tying: 'tie', idly: 'idl', gently: 'gentl', ugly: 'ugli',
+    early: 'earli', only: 'onli', singly: 'singl', sky: 'sky', news: 'news', howe: 'howe', atlas: 'atlas', cosmos: 'cosmos',
+    bias: 'bias', andes: 'andes',
+};
+const AFTER_1A = new Set(['inning', 'outing', 'canning', 'herring', 'earring', 'proceed', 'exceed', 'succeed']);
+/** The start of the region after the first non-vowel following a vowel, from `from`. */
+function regionAfter(w, from) {
+    for (let i = from + 1; i < w.length; i++)
+        if (!isVowel(w, i) && isVowel(w, i - 1))
+            return i + 1;
+    return w.length;
+}
+/** A short syllable ending at index `i` (inclusive). */
+function shortSyllableAt(w, i) {
+    if (i === 1)
+        return isVowel(w, 0) && !isVowel(w, 1);
+    if (i < 2)
+        return false;
+    return !isVowel(w, i - 2) && isVowel(w, i - 1) && !isVowel(w, i) && !['w', 'x', 'Y'].includes(w[i]);
+}
+/** Step 2's rewrite of a suffix in R1: ogi to og after l, li deleted after a valid li-ending, the rest by the table. */
+function step2(w, s2, table) {
+    if (s2 === 'ogi')
+        return w[w.length - 4] === 'l' ? `${w.slice(0, -3)}og` : w;
+    if (s2 === 'li')
+        return LI_ENDINGS.has(w[w.length - 3] ?? '') ? w.slice(0, -2) : w;
+    return w.slice(0, -s2.length) + table[s2];
+}
+/** Step 3's rewrite of a suffix in R1 by the table (ative, which needs R2, is the caller's). */
+function step3(w, s3, table) {
+    return w.slice(0, -s3.length) + table[s3];
+}
+export function stemEnglish(word) {
+    if (word.length <= 2)
+        return word;
+    if (EXCEPTIONS[word] !== undefined)
+        return EXCEPTIONS[word];
+    let w = word.startsWith("'") ? word.slice(1) : word;
+    // initial y, and y after a vowel, are consonants: marked Y
+    w = w.replace(/^y/, 'Y').replace(/([aeiouy])y/g, '$1Y');
+    let r1 = /^(gener|commun|arsen)/.test(w) ? /^(gener|commun|arsen)/.exec(w)[0].length : regionAfter(w, 0);
+    const r2 = () => regionAfter(w, r1);
+    const inR1 = (suffix) => w.length - suffix.length >= r1;
+    const inR2 = (suffix) => w.length - suffix.length >= r2();
+    const ends = (s) => w.endsWith(s);
+    const cut = (n) => { w = w.slice(0, w.length - n); };
+    const longest = (list) => list.filter(ends).sort((a, b) => b.length - a.length)[0];
+    const hasVowelBefore = (end) => { for (let i = 0; i < end; i++)
+        if (isVowel(w, i))
+            return true; return false; };
+    // Step 0
+    const s0 = longest(["'s'", "'s", "'"]);
+    if (s0)
+        cut(s0.length);
+    // Step 1a
+    const s1a = longest(['sses', 'ied', 'ies', 'us', 'ss', 's']);
+    if (s1a === 'sses')
+        cut(2);
+    else if (s1a === 'ied' || s1a === 'ies') {
+        cut(3);
+        w += w.length > 1 ? 'i' : 'ie';
+    }
+    else if (s1a === 's') {
+        if (hasVowelBefore(w.length - 2))
+            cut(1);
+    }
+    if (AFTER_1A.has(w))
+        return w;
+    // Step 1b
+    const s1b = longest(['eedly', 'eed', 'ingly', 'edly', 'ing', 'ed']);
+    if (s1b === 'eed' || s1b === 'eedly') {
+        if (inR1(s1b)) {
+            cut(s1b.length);
+            w += 'ee';
+        }
+    }
+    else if (s1b && hasVowelBefore(w.length - s1b.length)) {
+        cut(s1b.length);
+        if (ends('at') || ends('bl') || ends('iz'))
+            w += 'e';
+        else if (DOUBLES.some(ends))
+            cut(1);
+        else if (r1 >= w.length && shortSyllableAt(w, w.length - 1))
+            w += 'e';
+    }
+    // Step 1c
+    if ((ends('y') || ends('Y')) && w.length > 2 && !isVowel(w, w.length - 2))
+        w = `${w.slice(0, -1)}i`;
+    // Step 2
+    const STEP2 = {
+        tional: 'tion', enci: 'ence', anci: 'ance', abli: 'able', entli: 'ent', izer: 'ize', ization: 'ize', ational: 'ate',
+        ation: 'ate', ator: 'ate', alism: 'al', aliti: 'al', alli: 'al', fulness: 'ful', ousli: 'ous', ousness: 'ous',
+        iveness: 'ive', iviti: 'ive', biliti: 'ble', bli: 'ble', ogi: 'og', fulli: 'ful', lessli: 'less', li: '',
+    };
+    const s2 = longest(Object.keys(STEP2));
+    if (s2 && inR1(s2))
+        w = step2(w, s2, STEP2);
+    // Step 3
+    const STEP3 = { tional: 'tion', ational: 'ate', alize: 'al', icate: 'ic', iciti: 'ic', ical: 'ic', ful: '', ness: '', ative: '' };
+    const s3 = longest(Object.keys(STEP3));
+    if (s3 && inR1(s3))
+        w = s3 === 'ative' ? (inR2(s3) ? w.slice(0, -5) : w) : step3(w, s3, STEP3);
+    // Step 4
+    const s4 = longest(['al', 'ance', 'ence', 'er', 'ic', 'able', 'ible', 'ant', 'ement', 'ment', 'ent', 'ism', 'ate', 'iti', 'ous', 'ive', 'ize', 'ion']);
+    if (s4 && inR2(s4)) {
+        if (s4 === 'ion') {
+            if (['s', 't'].includes(w[w.length - 4] ?? ''))
+                cut(3);
+        }
+        else
+            cut(s4.length);
+    }
+    // Step 5
+    if (ends('e')) {
+        if (inR2('e') || (inR1('e') && !shortSyllableAt(w, w.length - 2)))
+            cut(1);
+    }
+    else if (ends('l') && inR2('l') && w[w.length - 2] === 'l')
+        cut(1);
+    r1 = 0;
+    return w.replace(/Y/g, 'y');
+}

package/dist/src/turbopuffer-store.d.ts

@@ -0,0 +1,80 @@
+import { type TwinResource } from '@volter/world-core';
+import { TurbopufferError, type AttrConfig, type Doc } from './turbopuffer-filter.js';
+export { badRequest, TurbopufferError } from './turbopuffer-filter.js';
+export type { AttrConfig, Doc } from './turbopuffer-filter.js';
+export declare const SERVICE = "turbopuffer";
+export declare const TURBOPUFFER_RESOURCE_TYPES: readonly ["namespace", "document"];
+export declare const DISTANCE_METRICS: readonly ["cosine_distance", "euclidean_squared"];
+export type NamespaceState = {
+    name: string;
+    schema: Record<string, AttrConfig>;
+    distance_metric: string | null;
+    vector_dims: number | null;
+    created_at: string;
+    updated_at: string;
+    /** when its documents last changed (a schema-only write moves `updated_at` alone) */
+    last_write_at: string;
+    /** the namespace a branch was made from (metadata's `branching.parent`) */
+    parent?: string;
+    docs: Map<string, Doc>;
+};
+export declare function assertNamespaceName(name: string): void;
+export declare function docKey(id: string | number): string;
+export declare function documentSubjectId(ns: string, id: string | number): string;
+/** Fold the tree's `namespace` / `document` subjects into live namespaces. */
+export declare function loadNamespaces(resources: readonly TwinResource[]): Map<string, NamespaceState>;
+export declare function namespaceFields(ns: NamespaceState): Record<string, unknown>;
+export declare function documentFields(ns: string, d: Doc): Record<string, unknown>;
+/** "Vectors are attributes with a vector type (`[N]f32`, `[N]f16`, or `[N]i8` where N is the number of dimensions)"
+ *  (https://turbopuffer.com/docs/write, Vectors). */
+export declare function isVectorType(t: string): boolean;
+/** "`[][N]f32`: Variable-length array of `N` dimensional f32 vectors, used for late-interaction (multi-vector) search"
+ *  (https://turbopuffer.com/docs/write, type). */
+export declare function isVectorArrayType(t: string): boolean;
+/** Normalize one schema entry (`"string"` or a config object) into the stored form. */
+export declare function normalizeAttrSchema(attr: string, raw: unknown): AttrConfig;
+export type WriteOutcome = {
+    next: NamespaceState;
+    created: boolean;
+    upserts: Doc[];
+    deletes: string[];
+    response: Record<string, unknown>;
+};
+export declare function approxLogicalBytes(ns: NamespaceState): number;
+/** `copy_from_namespace` and `branch_from_namespace` (https://turbopuffer.com/docs/write): the destination is made as a
+ *  copy of the source's documents, schema and distance metric. "The destination namespace you are copying into must be
+ *  empty. The initial request currently cannot make schema changes or contain documents"; a branch is "an instant
+ *  copy-on-write clone of the source namespace. The destination namespace must be empty", and "reads, writes, queries,
+ *  and deletes on one namespace do not affect the other". A copy answers "namespace cloned successfully" (the async
+ *  copy's result, https://turbopuffer.com/docs/api-overview), and a destination that exists "destination namespace
+ *  already exists" (the same page's failed result).
+ *
+ *  Where the docs stop and the twin decides: a World holds one organization's namespaces in one store whatever region a
+ *  request names, so a copy's `source_region` and `source_api_key` read the same store; `rows_affected` counts the
+ *  documents cloned; a copy bills the logical bytes copied and a branch none ("billed at a flat rate"); a branch's
+ *  answer carries the copy's message. Pure. */
+export declare function applyClone(spaces: Map<string, NamespaceState>, name: string, rawBody: unknown, at: string): WriteOutcome;
+/** Apply one write request body to a namespace (or create it). Pure. */
+export declare function applyWrite(existing: NamespaceState | undefined, name: string, rawBody: unknown, at: string): WriteOutcome;
+export declare const EMBEDDING_MODELS: string[];
+/** The placeholder embedding of a text: a unit vector of `dims` derived from the model's name and the text. */
+export declare function placeholderEmbedding(model: string, text: string, dims: number): number[];
+export type QueryResult = {
+    rows?: Record<string, unknown>[];
+    aggregations?: Record<string, unknown>;
+    aggregation_groups?: Record<string, unknown>[];
+};
+/** `vector_encoding` other than float: "base64" is documented ("The encoding to use for vectors in the response", the
+ *  spec's VectorEncoding) and not modelled; anything else is refused. */
+export declare function vectorEncodingRefusal(v: unknown): TurbopufferError;
+/** Answer one query (the body of `POST /v2/namespaces/{ns}/query`, or one of `queries[]`). Pure. */
+export declare function runQuery(ns: NamespaceState, rawQuery: unknown, opts?: {
+    multi?: boolean;
+}): QueryResult;
+export declare function checkConsistency(c: unknown): void;
+export declare function cosineDistance(a: readonly number[], b: readonly number[]): number;
+export declare function euclideanSquared(a: readonly number[], b: readonly number[]): number;
+/** The wire form of a namespace's schema: every attribute with its full resolved config. */
+export declare function schemaWire(ns: NamespaceState): Record<string, unknown>;
+export declare function queryBilling(ns: NamespaceState, returned: unknown): Record<string, number>;
+export declare function queryPerformance(ns: NamespaceState): Record<string, unknown>;