@kernl-sdk/turbopuffer 0.1.0

package/dist/convert.js

@@ -0,0 +1,333 @@
+/**
+ * Codecs for converting between kernl and Turbopuffer types.
+ */
+/**
+ * Codec for converting kernl scalar types to Turbopuffer attribute types.
+ */
+export const SCALAR_TYPE = {
+    encode: (type) => {
+        switch (type) {
+            case "string":
+                return "string";
+            case "int":
+                return "int";
+            case "float":
+                return "int"; // tpuf doesn't have float
+            case "boolean":
+                return "bool";
+            case "date":
+                return "datetime";
+            case "string[]":
+                return "[]string";
+            case "int[]":
+                return "[]int";
+            case "date[]":
+                return "[]datetime";
+            default:
+                return "string";
+        }
+    },
+    decode: (type) => {
+        switch (type) {
+            case "string":
+                return "string";
+            case "int":
+                return "int";
+            case "bool":
+                return "boolean";
+            case "datetime":
+                return "date";
+            case "[]string":
+                return "string[]";
+            case "[]int":
+                return "int[]";
+            case "[]datetime":
+                return "date[]";
+            default:
+                return "string";
+        }
+    },
+};
+/**
+ * Codec for converting similarity metric to Turbopuffer distance metric.
+ *
+ * Turbopuffer supports: cosine_distance, euclidean_squared
+ * We support: cosine, euclidean, dot_product
+ */
+export const SIMILARITY = {
+    encode: (similarity) => {
+        switch (similarity) {
+            case "euclidean":
+                return "euclidean_squared";
+            case "cosine":
+            case "dot_product":
+            default:
+                return "cosine_distance";
+        }
+    },
+    decode: (metric) => {
+        switch (metric) {
+            case "euclidean_squared":
+                return "euclidean";
+            case "cosine_distance":
+            default:
+                return "cosine";
+        }
+    },
+};
+/**
+ * Codec-like converter for FieldSchema to Turbopuffer AttributeSchema.
+ *
+ * Takes the field name as context since Turbopuffer requires `ann: true`
+ * only on the special `vector` attribute.
+ */
+export const FIELD_SCHEMA = {
+    encode: (field, name) => {
+        // Vector fields
+        if (field.type === "vector" || field.type === "sparse-vector") {
+            const vf = field;
+            const precision = vf.quantization === "f16" ? "f16" : "f32";
+            return {
+                type: `[${vf.dimensions}]${precision}`,
+                ann: name === "vector",
+            };
+        }
+        // Scalar fields
+        const config = {
+            type: SCALAR_TYPE.encode(field.type),
+        };
+        if (field.filterable) {
+            config.filterable = true;
+        }
+        if (field.fts) {
+            config.full_text_search =
+                typeof field.fts === "object"
+                    ? { language: field.fts.language }
+                    : true;
+        }
+        return config;
+    },
+    decode: () => {
+        throw new Error("FIELD_SCHEMA.decode: not implemented");
+    },
+};
+/**
+ * Codec for converting a full schema record.
+ *
+ * Validates that vector fields are named `vector` since Turbopuffer only
+ * supports ANN indexing on that specific attribute name.
+ */
+export const INDEX_SCHEMA = {
+    encode: (schema) => {
+        const result = {};
+        for (const [name, field] of Object.entries(schema)) {
+            const isVector = field.type === "vector" || field.type === "sparse-vector";
+            // Enforce vector field naming
+            if (isVector && name !== "vector") {
+                throw new Error(`Turbopuffer requires vector fields to be named "vector", got "${name}". ` +
+                    `Rename your field or use a different search provider.`);
+            }
+            result[name] = FIELD_SCHEMA.encode(field, name);
+        }
+        return result;
+    },
+    decode: () => {
+        throw new Error("INDEX_SCHEMA.decode: not implemented");
+    },
+};
+/**
+ * Check if a value is a DenseVector.
+ */
+function isDenseVector(val) {
+    return (typeof val === "object" &&
+        val !== null &&
+        "kind" in val &&
+        val.kind === "vector");
+}
+/**
+ * Convert a FieldValue to Turbopuffer attribute value.
+ * Extracts vector values from DenseVector wrapper.
+ */
+function encodeFieldValue(val) {
+    if (isDenseVector(val)) {
+        return val.values;
+    }
+    return val;
+}
+/**
+ * Codec for converting SearchDocument to Turbopuffer Row.
+ */
+export const DOCUMENT = {
+    encode: (doc) => {
+        const row = { id: doc.id };
+        for (const [key, val] of Object.entries(doc.fields)) {
+            row[key] = encodeFieldValue(val);
+        }
+        return row;
+    },
+    decode: (_row) => {
+        throw new Error("DOCUMENT.decode: not implemented");
+    },
+};
+const FILTER_OP_MAP = {
+    eq: "Eq",
+    neq: "NotEq",
+    gt: "Gt",
+    gte: "Gte",
+    lt: "Lt",
+    lte: "Lte",
+    in: "In",
+    nin: "NotIn",
+    contains: "Contains",
+    starts_with: "Glob", // will add * suffix
+    ends_with: "Glob", // will add * prefix
+    contains_all: "ContainsAny", // closest match
+    contains_any: "ContainsAny",
+};
+/**
+ * Type guards for filter expressions.
+ */
+function isFieldFilter(f) {
+    return "field" in f && "op" in f && "value" in f;
+}
+function isExistsFilter(f) {
+    return ("field" in f && "op" in f && (f.op === "exists" || f.op === "not_exists"));
+}
+function isAndFilter(f) {
+    return "and" in f;
+}
+function isOrFilter(f) {
+    return "or" in f;
+}
+function isNotFilter(f) {
+    return "not" in f;
+}
+/**
+ * Codec for converting FilterExpression to Turbopuffer Filter.
+ */
+export const FILTER = {
+    encode: (filter) => {
+        if (isAndFilter(filter)) {
+            return ["And", filter.and.map(FILTER.encode)];
+        }
+        if (isOrFilter(filter)) {
+            return ["Or", filter.or.map(FILTER.encode)];
+        }
+        if (isNotFilter(filter)) {
+            return ["Not", FILTER.encode(filter.not)];
+        }
+        if (isExistsFilter(filter)) {
+            // exists → NotEq null, not_exists → Eq null
+            return filter.op === "exists"
+                ? [filter.field, "NotEq", null]
+                : [filter.field, "Eq", null];
+        }
+        if (isFieldFilter(filter)) {
+            const { field, op, value } = filter;
+            // Handle glob patterns for starts_with/ends_with
+            if (op === "starts_with") {
+                return [field, "Glob", `${value}*`];
+            }
+            if (op === "ends_with") {
+                return [field, "Glob", `*${value}`];
+            }
+            const tpufOp = FILTER_OP_MAP[op];
+            return [field, tpufOp, value];
+        }
+        throw new Error(`Unknown filter type: ${JSON.stringify(filter)}`);
+    },
+    decode: (_filter) => {
+        throw new Error("FILTER.decode: not implemented");
+    },
+};
+/**
+ * Build rank_by for vector search.
+ */
+function buildVectorRankBy(vector) {
+    return ["vector", "ANN", vector];
+}
+/**
+ * Build rank_by for full-text search.
+ */
+function buildTextRankBy(text, fields) {
+    if (!fields || fields.length === 0) {
+        throw new Error("textFields required for full-text search");
+    }
+    if (fields.length === 1) {
+        return [fields[0], "BM25", text];
+    }
+    // Multiple fields: combine with Sum
+    const subQueries = fields.map((field) => [field, "BM25", text]);
+    return ["Sum", subQueries];
+}
+/**
+ * Codec for converting SearchQuery to Turbopuffer NamespaceQueryParams.
+ *
+ * Note: Hybrid search (vector + text) requires multi-query which is
+ * handled separately.
+ */
+export const QUERY = {
+    encode: (query) => {
+        const params = {};
+        // Determine ranking method
+        if (query.vector) {
+            params.rank_by = buildVectorRankBy(query.vector);
+        }
+        else if (query.text) {
+            params.rank_by = buildTextRankBy(query.text, query.textFields);
+        }
+        // Top K
+        if (query.topK !== undefined) {
+            params.top_k = query.topK;
+        }
+        // Filters
+        if (query.filter) {
+            params.filters = FILTER.encode(query.filter);
+        }
+        // Include attributes
+        if (query.includeFields !== undefined) {
+            if (typeof query.includeFields === "boolean") {
+                params.include_attributes = query.includeFields;
+            }
+            else {
+                const attrs = [...query.includeFields];
+                if (query.includeVectors && !attrs.includes("vector")) {
+                    attrs.push("vector");
+                }
+                params.include_attributes = attrs;
+            }
+        }
+        else if (query.includeVectors) {
+            params.include_attributes = true; // include all to get vector
+        }
+        return params;
+    },
+    decode: (_params) => {
+        throw new Error("QUERY.decode: not implemented");
+    },
+};
+/**
+ * Codec for converting Turbopuffer Row to SearchHit.
+ */
+export const SEARCH_HIT = {
+    encode: (_hit) => {
+        throw new Error("SEARCH_HIT.encode: not implemented");
+    },
+    decode: (row, index) => {
+        const { id, $dist, vector, ...rest } = row;
+        const hit = {
+            id: String(id),
+            index,
+            score: typeof $dist === "number" ? $dist : 0,
+        };
+        // Include vector if present
+        if (vector !== undefined) {
+            hit.vector = vector;
+        }
+        // Include other fields
+        if (Object.keys(rest).length > 0) {
+            hit.fields = rest;
+        }
+        return hit;
+    },
+};

package/dist/handle.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Turbopuffer index handle implementation.
+ */
+import type Turbopuffer from "@turbopuffer/turbopuffer";
+import type { IndexHandle, DocumentPatch, UpsertResult, PatchResult, DeleteResult, QueryInput, SearchHit, FieldSchema, UnknownDocument } from "@kernl-sdk/retrieval";
+/**
+ * Handle for operating on a specific Turbopuffer namespace (index).
+ */
+export declare class TurbopufferIndexHandle<TDocument = UnknownDocument> implements IndexHandle<TDocument> {
+    readonly id: string;
+    private client;
+    constructor(client: Turbopuffer, id: string);
+    /**
+     * Query the index.
+     */
+    query(query: QueryInput): Promise<SearchHit<TDocument>[]>;
+    /**
+     * Upsert one or more documents.
+     */
+    upsert(docs: TDocument | TDocument[]): Promise<UpsertResult>;
+    /**
+     * Patch one or more documents.
+     */
+    patch(patches: DocumentPatch<TDocument> | DocumentPatch<TDocument>[]): Promise<PatchResult>;
+    /**
+     * Delete one or more documents by ID.
+     */
+    delete(ids: string | string[]): Promise<DeleteResult>;
+    /**
+     * Add a field to the index schema.
+     */
+    addField(_field: string, _schema: FieldSchema): Promise<void>;
+}
+//# sourceMappingURL=handle.d.ts.map

package/dist/handle.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"handle.d.ts","sourceRoot":"","sources":["../src/handle.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,WAAW,MAAM,0BAA0B,CAAC;AACxD,OAAO,KAAK,EACV,WAAW,EACX,aAAa,EACb,YAAY,EACZ,WAAW,EACX,YAAY,EACZ,UAAU,EACV,SAAS,EACT,WAAW,EACX,eAAe,EAChB,MAAM,sBAAsB,CAAC;AAK9B;;GAEG;AACH,qBAAa,sBAAsB,CAAC,SAAS,GAAG,eAAe,CAC7D,YAAW,WAAW,CAAC,SAAS,CAAC;IAEjC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,MAAM,CAAc;gBAEhB,MAAM,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM;IAK3C;;OAEG;IACG,KAAK,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,CAAC;IAY/D;;OAEG;IACG,MAAM,CAAC,IAAI,EAAE,SAAS,GAAG,SAAS,EAAE,GAAG,OAAO,CAAC,YAAY,CAAC;IAalE;;OAEG;IACG,KAAK,CACT,OAAO,EAAE,aAAa,CAAC,SAAS,CAAC,GAAG,aAAa,CAAC,SAAS,CAAC,EAAE,GAC7D,OAAO,CAAC,WAAW,CAAC;IAavB;;OAEG;IACG,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,OAAO,CAAC,YAAY,CAAC;IAW3D;;OAEG;IACG,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;CAGpE"}

package/dist/handle.js

@@ -0,0 +1,72 @@
+/**
+ * Turbopuffer index handle implementation.
+ */
+import { normalizeQuery } from "@kernl-sdk/retrieval";
+import { DOCUMENT, PATCH, QUERY, SEARCH_HIT } from "./convert.js";
+/**
+ * Handle for operating on a specific Turbopuffer namespace (index).
+ */
+export class TurbopufferIndexHandle {
+    id;
+    client;
+    constructor(client, id) {
+        this.client = client;
+        this.id = id;
+    }
+    /**
+     * Query the index.
+     */
+    async query(query) {
+        const q = normalizeQuery(query);
+        const ns = this.client.namespace(this.id);
+        const res = await ns.query(QUERY.encode(q));
+        if (!res.rows) {
+            return [];
+        }
+        return res.rows.map((row) => SEARCH_HIT.decode(row, this.id));
+    }
+    /**
+     * Upsert one or more documents.
+     */
+    async upsert(docs) {
+        const arr = Array.isArray(docs) ? docs : [docs];
+        if (arr.length === 0) {
+            return { count: 0 };
+        }
+        const ns = this.client.namespace(this.id);
+        const rows = arr.map((doc) => DOCUMENT.encode(doc));
+        await ns.write({ upsert_rows: rows });
+        return { count: arr.length };
+    }
+    /**
+     * Patch one or more documents.
+     */
+    async patch(patches) {
+        const arr = Array.isArray(patches) ? patches : [patches];
+        if (arr.length === 0) {
+            return { count: 0 };
+        }
+        const ns = this.client.namespace(this.id);
+        const rows = arr.map((p) => PATCH.encode(p));
+        await ns.write({ patch_rows: rows });
+        return { count: arr.length };
+    }
+    /**
+     * Delete one or more documents by ID.
+     */
+    async delete(ids) {
+        const arr = Array.isArray(ids) ? ids : [ids];
+        if (arr.length === 0) {
+            return { count: 0 };
+        }
+        const ns = this.client.namespace(this.id);
+        await ns.write({ deletes: arr });
+        return { count: arr.length };
+    }
+    /**
+     * Add a field to the index schema.
+     */
+    async addField(_field, _schema) {
+        throw new Error("addField not supported by Turbopuffer");
+    }
+}

package/dist/index.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Turbopuffer search index adapter.
+ */
+import { TurbopufferSearchIndex } from "./search.js";
+import type { TurbopufferConfig } from "./types.js";
+export { TurbopufferSearchIndex } from "./search.js";
+export { TurbopufferIndexHandle } from "./handle.js";
+export type { TurbopufferConfig } from "./types.js";
+/**
+ * Create a Turbopuffer search index.
+ *
+ * @param config - Turbopuffer API key and region
+ * @returns SearchIndex instance backed by Turbopuffer
+ *
+ * @example
+ * ```ts
+ * const tpuf = turbopuffer({
+ *   apiKey: "your-api-key",
+ *   region: "us-east-1",
+ * });
+ *
+ * const docs = tpuf.index("my-index");
+ * await docs.upsert({ id: "doc-1", fields: { text: "Hello" } });
+ * ```
+ */
+export declare function turbopuffer(config: TurbopufferConfig): TurbopufferSearchIndex;
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAC;AAClD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAEjD,OAAO,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAC;AAClD,OAAO,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAC;AAClD,YAAY,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAEjD;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,iBAAiB,0BAEpD"}

package/dist/index.js

@@ -0,0 +1,26 @@
+/**
+ * Turbopuffer search index adapter.
+ */
+import { TurbopufferSearchIndex } from "./search.js";
+export { TurbopufferSearchIndex } from "./search.js";
+export { TurbopufferIndexHandle } from "./handle.js";
+/**
+ * Create a Turbopuffer search index.
+ *
+ * @param config - Turbopuffer API key and region
+ * @returns SearchIndex instance backed by Turbopuffer
+ *
+ * @example
+ * ```ts
+ * const tpuf = turbopuffer({
+ *   apiKey: "your-api-key",
+ *   region: "us-east-1",
+ * });
+ *
+ * const docs = tpuf.index("my-index");
+ * await docs.upsert({ id: "doc-1", fields: { text: "Hello" } });
+ * ```
+ */
+export function turbopuffer(config) {
+    return new TurbopufferSearchIndex(config);
+}

package/dist/search.d.ts

@@ -0,0 +1,85 @@
+/**
+ * Turbopuffer SearchIndex implementation.
+ *
+ * Adapts the Turbopuffer vector database to the kernl SearchIndex interface.
+ * Turbopuffer uses "namespaces" as the equivalent of our "indexes".
+ *
+ * @see https://turbopuffer.com/docs
+ */
+import type { SearchIndex, IndexHandle, NewIndexParams, ListIndexesParams, IndexSummary, IndexStats, UnknownDocument, SearchCapabilities } from "@kernl-sdk/retrieval";
+import { CursorPage } from "@kernl-sdk/shared";
+import { TurbopufferConfig } from "./types.js";
+/**
+ * Turbopuffer search index adapter.
+ *
+ * Implements the kernl SearchIndex interface backed by Turbopuffer's
+ * vector database. Supports vector search, full-text search (BM25),
+ * and hybrid queries.
+ *
+ * @example
+ * ```ts
+ * const tpuf = turbopuffer({
+ *   apiKey: "your-api-key",
+ *   region: "us-east-1",
+ * });
+ *
+ * const docs = tpuf.index("my-index");
+ *
+ * await docs.upsert({
+ *   id: "doc-1",
+ *   text: "Hello world",
+ *   vector: [0.1, 0.2, ...],
+ * });
+ *
+ * const hits = await docs.query({
+ *   query: [{ vector: [0.1, 0.2, ...] }],
+ *   topK: 10,
+ * });
+ * ```
+ */
+export declare class TurbopufferSearchIndex implements SearchIndex {
+    readonly id = "turbopuffer";
+    private readonly client;
+    /**
+     * Create a new Turbopuffer search index.
+     */
+    constructor(config: TurbopufferConfig);
+    /**
+     * Create a new index (namespace) in Turbopuffer.
+     */
+    createIndex(params: NewIndexParams): Promise<void>;
+    /**
+     * List indexes (namespaces) with optional pagination and prefix filtering.
+     */
+    listIndexes(params?: ListIndexesParams): Promise<CursorPage<IndexSummary, ListIndexesParams>>;
+    /**
+     * Get metadata and statistics about an index (namespace).
+     */
+    describeIndex(id: string): Promise<IndexStats>;
+    /**
+     * Delete an index (namespace) and all its documents.
+     */
+    deleteIndex(id: string): Promise<void>;
+    /**
+     * Get a handle for operating on a specific index.
+     */
+    index<TDocument = UnknownDocument>(id: string): IndexHandle<TDocument>;
+    /**
+     * Bind an existing resource as an index.
+     *
+     * Not supported by Turbopuffer - indexes are created implicitly.
+     */
+    bindIndex(_id: string, _config: unknown): Promise<void>;
+    /**
+     * Warm/preload an index for faster queries.
+     */
+    warm(id: string): Promise<void>;
+    /**
+     * Turbopuffer capabilities.
+     *
+     * Note: Turbopuffer supports text (BM25) and vector (ANN) separately,
+     * but not hybrid fusion (text + vector in the same query).
+     */
+    capabilities(): SearchCapabilities;
+}
+//# sourceMappingURL=search.d.ts.map

package/dist/search.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../src/search.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH,OAAO,KAAK,EACV,WAAW,EACX,WAAW,EACX,cAAc,EACd,iBAAiB,EACjB,YAAY,EACZ,UAAU,EACV,eAAe,EACf,kBAAkB,EACnB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,UAAU,EAA2B,MAAM,mBAAmB,CAAC;AAExE,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAI5C;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,qBAAa,sBAAuB,YAAW,WAAW;IACxD,QAAQ,CAAC,EAAE,iBAAiB;IAE5B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;IAErC;;OAEG;gBACS,MAAM,EAAE,iBAAiB;IASrC;;OAEG;IACG,WAAW,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAyBxD;;OAEG;IACG,WAAW,CACf,MAAM,GAAE,iBAAsB,GAC7B,OAAO,CAAC,UAAU,CAAC,YAAY,EAAE,iBAAiB,CAAC,CAAC;IAkCvD;;OAEG;IACG,aAAa,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC;IAYpD;;OAEG;IACG,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAO5C;;OAEG;IACH,KAAK,CAAC,SAAS,GAAG,eAAe,EAAE,EAAE,EAAE,MAAM,GAAG,WAAW,CAAC,SAAS,CAAC;IAItE;;;;OAIG;IACG,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAM7D;;OAEG;IACG,IAAI,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKrC;;;;;OAKG;IACH,YAAY,IAAI,kBAAkB;CAUnC"}