@kernl-sdk/turbopuffer 0.1.0

package/dist/convert/filter.js

@@ -0,0 +1,109 @@
+/**
+ * Filter conversion codecs.
+ *
+ * Converts MongoDB-style filters to Turbopuffer filter format.
+ */
+/**
+ * Codec for converting Filter to Turbopuffer Filter.
+ */
+export const FILTER = {
+    encode: (filter) => {
+        const conditions = [];
+        for (const [key, value] of Object.entries(filter)) {
+            if (value === undefined)
+                continue;
+            // Logical operators
+            if (key === "$and" && Array.isArray(value)) {
+                const sub = value.map(FILTER.encode);
+                if (sub.length === 1) {
+                    conditions.push(sub[0]);
+                }
+                else {
+                    conditions.push(["And", sub]);
+                }
+                continue;
+            }
+            if (key === "$or" && Array.isArray(value)) {
+                const sub = value.map(FILTER.encode);
+                conditions.push(["Or", sub]);
+                continue;
+            }
+            if (key === "$not") {
+                conditions.push(["Not", FILTER.encode(value)]);
+                continue;
+            }
+            // Field-level filter
+            if (isFieldOps(value)) {
+                conditions.push(...encodeFieldOps(key, value));
+            }
+            else {
+                // Simple equality: { field: value }
+                conditions.push([key, "Eq", value]);
+            }
+        }
+        if (conditions.length === 0) {
+            throw new Error("Empty filter");
+        }
+        if (conditions.length === 1) {
+            return conditions[0];
+        }
+        return ["And", conditions];
+    },
+    decode: (_filter) => {
+        throw new Error("FILTER.decode: not implemented");
+    },
+};
+/**
+ * Check if a value is a FieldOps object (has operator keys like $eq, $gt, etc.)
+ */
+function isFieldOps(value) {
+    if (typeof value !== "object" || value === null || Array.isArray(value)) {
+        return false;
+    }
+    const keys = Object.keys(value);
+    return keys.some((k) => k.startsWith("$"));
+}
+/**
+ * Encode field-level operators to Turbopuffer filters.
+ */
+function encodeFieldOps(field, ops) {
+    const conditions = [];
+    if (ops.$eq !== undefined) {
+        conditions.push([field, "Eq", ops.$eq]);
+    }
+    if (ops.$neq !== undefined) {
+        conditions.push([field, "NotEq", ops.$neq]);
+    }
+    if (ops.$gt !== undefined) {
+        conditions.push([field, "Gt", ops.$gt]);
+    }
+    if (ops.$gte !== undefined) {
+        conditions.push([field, "Gte", ops.$gte]);
+    }
+    if (ops.$lt !== undefined) {
+        conditions.push([field, "Lt", ops.$lt]);
+    }
+    if (ops.$lte !== undefined) {
+        conditions.push([field, "Lte", ops.$lte]);
+    }
+    if (ops.$in !== undefined) {
+        conditions.push([field, "In", ops.$in]);
+    }
+    if (ops.$nin !== undefined) {
+        conditions.push([field, "NotIn", ops.$nin]);
+    }
+    if (ops.$contains !== undefined) {
+        conditions.push([field, "Contains", ops.$contains]);
+    }
+    if (ops.$startsWith !== undefined) {
+        conditions.push([field, "Glob", `${ops.$startsWith}*`]);
+    }
+    if (ops.$endsWith !== undefined) {
+        conditions.push([field, "Glob", `*${ops.$endsWith}`]);
+    }
+    if (ops.$exists !== undefined) {
+        // exists: true → NotEq null, exists: false → Eq null
+        conditions.push([field, ops.$exists ? "NotEq" : "Eq", null]);
+    }
+    return conditions;
+}

package/dist/convert/index.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Type conversion codecs for Turbopuffer.
+ */
+export { SCALAR_TYPE, SIMILARITY, FIELD_SCHEMA, INDEX_SCHEMA } from "./schema.js";
+export { DOCUMENT, PATCH } from "./document.js";
+export { FILTER } from "./filter.js";
+export { QUERY, SEARCH_HIT } from "./query.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/convert/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/convert/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAC/E,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAClC,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC"}

package/dist/convert/index.js

@@ -0,0 +1,7 @@
+/**
+ * Type conversion codecs for Turbopuffer.
+ */
+export { SCALAR_TYPE, SIMILARITY, FIELD_SCHEMA, INDEX_SCHEMA } from "./schema.js";
+export { DOCUMENT, PATCH } from "./document.js";
+export { FILTER } from "./filter.js";
+export { QUERY, SEARCH_HIT } from "./query.js";

package/dist/convert/query.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Query conversion codecs.
+ *
+ * Converts the new RankingSignal-based query format to Turbopuffer params.
+ */
+import type { SearchQuery, SearchHit, UnknownDocument } from "@kernl-sdk/retrieval";
+import type { Row, NamespaceQueryParams } from "@turbopuffer/turbopuffer/resources/namespaces";
+/**
+ * Codec for converting SearchQuery to Turbopuffer NamespaceQueryParams.
+ */
+export declare const QUERY: {
+    encode: (query: SearchQuery) => NamespaceQueryParams;
+    decode: (_params: NamespaceQueryParams) => SearchQuery;
+};
+/**
+ * Codec for converting Turbopuffer Row to SearchHit.
+ */
+export declare const SEARCH_HIT: {
+    encode: <TDocument = UnknownDocument>(_hit: SearchHit<TDocument>) => Row;
+    decode: <TDocument = UnknownDocument>(row: Row, index: string) => SearchHit<TDocument>;
+};
+//# sourceMappingURL=query.d.ts.map

package/dist/convert/query.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"query.d.ts","sourceRoot":"","sources":["../../src/convert/query.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EACV,WAAW,EACX,SAAS,EAET,eAAe,EAChB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,KAAK,EACV,GAAG,EACH,oBAAoB,EACrB,MAAM,+CAA+C,CAAC;AAKvD;;GAEG;AACH,eAAO,MAAM,KAAK;oBACA,WAAW,KAAG,oBAAoB;sBA+BhC,oBAAoB,KAAG,WAAW;CAGrD,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,UAAU;aACZ,SAAS,0BAA0B,SAAS,CAAC,SAAS,CAAC,KAAG,GAAG;aAI7D,SAAS,yBACX,GAAG,SACD,MAAM,KACZ,SAAS,CAAC,SAAS,CAAC;CAgBxB,CAAC"}

package/dist/convert/query.js

@@ -0,0 +1,111 @@
+/**
+ * Query conversion codecs.
+ *
+ * Converts the new RankingSignal-based query format to Turbopuffer params.
+ */
+import { FILTER } from "./filter.js";
+/**
+ * Codec for converting SearchQuery to Turbopuffer NamespaceQueryParams.
+ */
+export const QUERY = {
+    encode: (query) => {
+        const params = {};
+        // Build rank_by from query signals
+        const signals = query.query ?? query.max;
+        if (signals && signals.length > 0) {
+            params.rank_by = buildRankBy(signals, query.max !== undefined);
+        }
+        // top K
+        if (query.topK !== undefined) {
+            params.top_k = query.topK;
+        }
+        // filters
+        if (query.filter) {
+            params.filters = FILTER.encode(query.filter);
+        }
+        // include attributes
+        if (query.include !== undefined) {
+            if (typeof query.include === "boolean") {
+                params.include_attributes = query.include;
+            }
+            else {
+                params.include_attributes = [...query.include];
+            }
+        }
+        return params;
+    },
+    decode: (_params) => {
+        throw new Error("QUERY.decode: not implemented");
+    },
+};
+/**
+ * Codec for converting Turbopuffer Row to SearchHit.
+ */
+export const SEARCH_HIT = {
+    encode: (_hit) => {
+        throw new Error("SEARCH_HIT.encode: not implemented");
+    },
+    decode: (row, index) => {
+        const { id, $dist, ...rest } = row;
+        const dist = typeof $dist === "number" ? $dist : 0;
+        const hit = {
+            id: String(id),
+            index,
+            score: dist === 0 ? 0 : -dist, // convert distance to similarity (negate so higher = better)
+        };
+        // include document fields with id
+        hit.document = { id, ...rest };
+        return hit;
+    },
+};
+/**
+ * Build rank_by from ranking signals.
+ *
+ * Turbopuffer constraints:
+ * - Sum/Max fusion only works with BM25 (text) signals
+ * - Vector search must be a single ANN query
+ * - Hybrid (text + vector) fusion is not supported in a single query
+ */
+function buildRankBy(signals, useMax) {
+    const textRankBys = [];
+    const vectorRankBys = [];
+    for (const signal of signals) {
+        const { weight, ...fields } = signal;
+        for (const [field, value] of Object.entries(fields)) {
+            if (value === undefined)
+                continue;
+            if (Array.isArray(value)) {
+                vectorRankBys.push(["vector", "ANN", value]);
+            }
+            else if (typeof value === "string") {
+                textRankBys.push([field, "BM25", value]);
+            }
+        }
+    }
+    const hasVector = vectorRankBys.length > 0;
+    const hasText = textRankBys.length > 0;
+    if (!hasVector && !hasText) {
+        throw new Error("No ranking signals provided");
+    }
+    // hybrid fusion not supported
+    if (hasVector && hasText) {
+        throw new Error("Turbopuffer does not support hybrid (vector + text) fusion in a single query. " +
+            "Use separate queries and merge results client-side.");
+    }
+    // multi-vector fusion not supported
+    if (vectorRankBys.length > 1) {
+        throw new Error("Turbopuffer does not support multi-vector fusion. " +
+            "Use separate queries and merge results client-side.");
+    }
+    // single vector query
+    if (hasVector) {
+        return vectorRankBys[0];
+    }
+    // single text query
+    if (textRankBys.length === 1) {
+        return textRankBys[0];
+    }
+    // multiple text signals: use Sum or Max fusion
+    const fusion = useMax ? "Max" : "Sum";
+    return [fusion, textRankBys];
+}

package/dist/convert/schema.d.ts

@@ -0,0 +1,39 @@
+/**
+ * Schema conversion codecs.
+ */
+import type { Codec } from "@kernl-sdk/shared/lib";
+import type { FieldSchema, VectorFieldSchema, ScalarFieldSchema } from "@kernl-sdk/retrieval";
+import type { AttributeSchema, DistanceMetric } from "@turbopuffer/turbopuffer/resources/namespaces";
+type Similarity = VectorFieldSchema["similarity"];
+type ScalarType = ScalarFieldSchema["type"];
+type TpufType = string;
+/**
+ * Codec for converting kernl scalar types to Turbopuffer attribute types.
+ */
+export declare const SCALAR_TYPE: Codec<ScalarType, TpufType>;
+/**
+ * Codec for converting similarity metric to Turbopuffer distance metric.
+ *
+ * Turbopuffer supports: cosine_distance, euclidean_squared
+ * We support: cosine, euclidean, dot_product
+ */
+export declare const SIMILARITY: Codec<Similarity, DistanceMetric>;
+/**
+ * Codec-like converter for FieldSchema to Turbopuffer AttributeSchema.
+ *
+ * Takes the field name as context since Turbopuffer requires `ann: true`
+ * only on the special `vector` attribute.
+ */
+export declare const FIELD_SCHEMA: {
+    encode: (field: FieldSchema, name: string) => AttributeSchema;
+    decode: () => never;
+};
+/**
+ * Codec for converting a full schema record.
+ *
+ * Validates that vector fields are named `vector` since Turbopuffer only
+ * supports ANN indexing on that specific attribute name.
+ */
+export declare const INDEX_SCHEMA: Codec<Record<string, FieldSchema>, Record<string, AttributeSchema>>;
+export {};
+//# sourceMappingURL=schema.d.ts.map

package/dist/convert/schema.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/convert/schema.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,KAAK,EACV,WAAW,EACX,iBAAiB,EACjB,iBAAiB,EAClB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,KAAK,EACV,eAAe,EAEf,cAAc,EACf,MAAM,+CAA+C,CAAC;AAEvD,KAAK,UAAU,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAC;AAClD,KAAK,UAAU,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC;AAC5C,KAAK,QAAQ,GAAG,MAAM,CAAC;AA+BvB;;GAEG;AACH,eAAO,MAAM,WAAW,EAAE,KAAK,CAAC,UAAU,EAAE,QAAQ,CAGnD,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,cAAc,CAqBxD,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,YAAY;oBACP,WAAW,QAAQ,MAAM,KAAG,eAAe;;CAiC5D,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,YAAY,EAAE,KAAK,CAC9B,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,EAC3B,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CA0BhC,CAAC"}

package/dist/convert/schema.js

@@ -0,0 +1,124 @@
+/**
+ * Schema conversion codecs.
+ */
+/**
+ * Mapping from kernl scalar types to Turbopuffer attribute types.
+ */
+const SCALAR_TO_TPUF = {
+    string: "string",
+    int: "int",
+    bigint: "uint", // Unix epoch timestamps in ms
+    float: "int", // tpuf doesn't have float
+    boolean: "bool",
+    date: "datetime",
+    "string[]": "[]string",
+    "int[]": "[]int",
+    "date[]": "[]datetime",
+};
+/**
+ * Mapping from Turbopuffer attribute types to kernl scalar types.
+ */
+const TPUF_TO_SCALAR = {
+    string: "string",
+    int: "int",
+    uint: "bigint",
+    bool: "boolean",
+    datetime: "date",
+    "[]string": "string[]",
+    "[]int": "int[]",
+    "[]datetime": "date[]",
+};
+/**
+ * Codec for converting kernl scalar types to Turbopuffer attribute types.
+ */
+export const SCALAR_TYPE = {
+    encode: (type) => SCALAR_TO_TPUF[type] ?? "string",
+    decode: (type) => TPUF_TO_SCALAR[type] ?? "string",
+};
+/**
+ * Codec for converting similarity metric to Turbopuffer distance metric.
+ *
+ * Turbopuffer supports: cosine_distance, euclidean_squared
+ * We support: cosine, euclidean, dot_product
+ */
+export const SIMILARITY = {
+    encode: (similarity) => {
+        switch (similarity) {
+            case "euclidean":
+                return "euclidean_squared";
+            case "cosine":
+            case "dot_product":
+            default:
+                return "cosine_distance";
+        }
+    },
+    decode: (metric) => {
+        switch (metric) {
+            case "euclidean_squared":
+                return "euclidean";
+            case "cosine_distance":
+            default:
+                return "cosine";
+        }
+    },
+};
+/**
+ * Codec-like converter for FieldSchema to Turbopuffer AttributeSchema.
+ *
+ * Takes the field name as context since Turbopuffer requires `ann: true`
+ * only on the special `vector` attribute.
+ */
+export const FIELD_SCHEMA = {
+    encode: (field, name) => {
+        // Vector fields
+        if (field.type === "vector" || field.type === "sparse-vector") {
+            const vf = field;
+            const precision = vf.quantization === "f16" ? "f16" : "f32";
+            return {
+                type: `[${vf.dimensions}]${precision}`,
+                ann: name === "vector",
+            };
+        }
+        // Scalar fields
+        const config = {
+            type: SCALAR_TYPE.encode(field.type),
+        };
+        if (field.filterable) {
+            config.filterable = true;
+        }
+        if (field.fts) {
+            config.full_text_search =
+                typeof field.fts === "object"
+                    ? { language: field.fts.language }
+                    : true;
+        }
+        return config;
+    },
+    decode: () => {
+        throw new Error("FIELD_SCHEMA.decode: not implemented");
+    },
+};
+/**
+ * Codec for converting a full schema record.
+ *
+ * Validates that vector fields are named `vector` since Turbopuffer only
+ * supports ANN indexing on that specific attribute name.
+ */
+export const INDEX_SCHEMA = {
+    encode: (schema) => {
+        const result = {};
+        for (const [name, field] of Object.entries(schema)) {
+            const isVector = field.type === "vector" || field.type === "sparse-vector";
+            // Enforce vector field naming
+            if (isVector && name !== "vector") {
+                throw new Error(`Turbopuffer requires vector fields to be named "vector", got "${name}". ` +
+                    `Rename your field or use a different search provider.`);
+            }
+            result[name] = FIELD_SCHEMA.encode(field, name);
+        }
+        return result;
+    },
+    decode: () => {
+        throw new Error("INDEX_SCHEMA.decode: not implemented");
+    },
+};

package/dist/convert.d.ts

@@ -0,0 +1,68 @@
+/**
+ * Codecs for converting between kernl and Turbopuffer types.
+ */
+import type { Codec } from "@kernl-sdk/shared/lib";
+import type { FieldSchema, VectorFieldSchema, ScalarFieldSchema, SearchDocument, SearchQuery, SearchHit, FilterExpression } from "@kernl-sdk/retrieval";
+import type { AttributeSchema, DistanceMetric, Row, NamespaceQueryParams } from "@turbopuffer/turbopuffer/resources/namespaces";
+import type { Filter } from "@turbopuffer/turbopuffer/resources/custom";
+type Similarity = VectorFieldSchema["similarity"];
+type ScalarType = ScalarFieldSchema["type"];
+type TpufType = string;
+/**
+ * Codec for converting kernl scalar types to Turbopuffer attribute types.
+ */
+export declare const SCALAR_TYPE: Codec<ScalarType, TpufType>;
+/**
+ * Codec for converting similarity metric to Turbopuffer distance metric.
+ *
+ * Turbopuffer supports: cosine_distance, euclidean_squared
+ * We support: cosine, euclidean, dot_product
+ */
+export declare const SIMILARITY: Codec<Similarity, DistanceMetric>;
+/**
+ * Codec-like converter for FieldSchema to Turbopuffer AttributeSchema.
+ *
+ * Takes the field name as context since Turbopuffer requires `ann: true`
+ * only on the special `vector` attribute.
+ */
+export declare const FIELD_SCHEMA: {
+    encode: (field: FieldSchema, name: string) => AttributeSchema;
+    decode: () => never;
+};
+/**
+ * Codec for converting a full schema record.
+ *
+ * Validates that vector fields are named `vector` since Turbopuffer only
+ * supports ANN indexing on that specific attribute name.
+ */
+export declare const INDEX_SCHEMA: Codec<Record<string, FieldSchema>, Record<string, AttributeSchema>>;
+/**
+ * Codec for converting SearchDocument to Turbopuffer Row.
+ */
+export declare const DOCUMENT: Codec<SearchDocument, Row>;
+/**
+ * Codec for converting FilterExpression to Turbopuffer Filter.
+ */
+export declare const FILTER: {
+    encode: (filter: FilterExpression) => Filter;
+    decode: (_filter: Filter) => FilterExpression;
+};
+/**
+ * Codec for converting SearchQuery to Turbopuffer NamespaceQueryParams.
+ *
+ * Note: Hybrid search (vector + text) requires multi-query which is
+ * handled separately.
+ */
+export declare const QUERY: {
+    encode: (query: SearchQuery) => NamespaceQueryParams;
+    decode: (_params: NamespaceQueryParams) => SearchQuery;
+};
+/**
+ * Codec for converting Turbopuffer Row to SearchHit.
+ */
+export declare const SEARCH_HIT: {
+    encode: (_hit: SearchHit) => Row;
+    decode: (row: Row, index: string) => SearchHit;
+};
+export {};
+//# sourceMappingURL=convert.d.ts.map

package/dist/convert.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"convert.d.ts","sourceRoot":"","sources":["../src/convert.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,KAAK,EACV,WAAW,EACX,iBAAiB,EACjB,iBAAiB,EAGjB,cAAc,EACd,WAAW,EACX,SAAS,EACT,gBAAgB,EAIjB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,KAAK,EACV,eAAe,EAEf,cAAc,EACd,GAAG,EACH,oBAAoB,EACrB,MAAM,+CAA+C,CAAC;AACvD,OAAO,KAAK,EAAE,MAAM,EAAU,MAAM,2CAA2C,CAAC;AAEhF,KAAK,UAAU,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAC;AAClD,KAAK,UAAU,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC;AAC5C,KAAK,QAAQ,GAAG,MAAM,CAAC;AAEvB;;GAEG;AACH,eAAO,MAAM,WAAW,EAAE,KAAK,CAAC,UAAU,EAAE,QAAQ,CA4CnD,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,cAAc,CAqBxD,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,YAAY;oBACP,WAAW,QAAQ,MAAM,KAAG,eAAe;;CAiC5D,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,YAAY,EAAE,KAAK,CAC9B,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,EAC3B,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CA0BhC,CAAC;AAyBF;;GAEG;AACH,eAAO,MAAM,QAAQ,EAAE,KAAK,CAAC,cAAc,EAAE,GAAG,CAc/C,CAAC;AAkEF;;GAEG;AACH,eAAO,MAAM,MAAM;qBACA,gBAAgB,KAAG,MAAM;sBAsCxB,MAAM,KAAG,gBAAgB;CAG5C,CAAC;AA4BF;;;;;GAKG;AACH,eAAO,MAAM,KAAK;oBACA,WAAW,KAAG,oBAAoB;sBAsChC,oBAAoB,KAAG,WAAW;CAGrD,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,UAAU;mBACN,SAAS,KAAG,GAAG;kBAIhB,GAAG,SAAS,MAAM,KAAG,SAAS;CAqB7C,CAAC"}