@kernl-sdk/turbopuffer 0.1.3 → 0.1.5

package/dist/__tests__/query.integration.test.js

@@ -1,423 +0,0 @@
-/**
- * Comprehensive query modes integration tests.
- *
- * Tests vector search, BM25 text search, hybrid queries, fusion modes,
- * topK behavior, include semantics, and rank ordering against real Turbopuffer API.
- */
-import { describe, it, expect, beforeAll, afterAll } from "vitest";
-import { TurbopufferSearchIndex } from "../search.js";
-const TURBOPUFFER_API_KEY = process.env.TURBOPUFFER_API_KEY;
-const TURBOPUFFER_REGION = process.env.TURBOPUFFER_REGION ?? "api";
-/**
- * Helper to create a DenseVector.
- */
-function vec(values) {
-    return { kind: "vector", values };
-}
-/**
- * Deterministic test dataset for query testing.
- *
- * Documents are designed to have predictable vector similarity and text relevance:
- * - Vectors are orthogonal basis vectors for predictable ANN results
- * - Text content has specific keywords for BM25 testing
- */
-const TEST_DOCS = [
-    {
-        id: "vec-1",
-        fields: {
-            title: "Machine Learning Basics",
-            content: "Introduction to neural networks and deep learning fundamentals",
-            category: "ml",
-            priority: 1,
-            vector: vec([1.0, 0.0, 0.0, 0.0]), // Basis vector 1
-        },
-    },
-    {
-        id: "vec-2",
-        fields: {
-            title: "Advanced Neural Networks",
-            content: "Deep dive into transformer architectures and attention mechanisms",
-            category: "ml",
-            priority: 2,
-            vector: vec([0.0, 1.0, 0.0, 0.0]), // Basis vector 2
-        },
-    },
-    {
-        id: "vec-3",
-        fields: {
-            title: "Database Fundamentals",
-            content: "SQL queries and relational database design patterns",
-            category: "db",
-            priority: 3,
-            vector: vec([0.0, 0.0, 1.0, 0.0]), // Basis vector 3
-        },
-    },
-    {
-        id: "vec-4",
-        fields: {
-            title: "Vector Databases",
-            content: "Introduction to vector search and similarity matching",
-            category: "db",
-            priority: 4,
-            vector: vec([0.0, 0.0, 0.0, 1.0]), // Basis vector 4
-        },
-    },
-    {
-        id: "vec-5",
-        fields: {
-            title: "Search Engine Optimization",
-            content: "BM25 ranking and full text search algorithms",
-            category: "search",
-            priority: 5,
-            vector: vec([0.5, 0.5, 0.0, 0.0]), // Mix of 1 and 2
-        },
-    },
-    {
-        id: "vec-6",
-        fields: {
-            title: "Hybrid Search Systems",
-            content: "Combining vector and keyword search for better results",
-            category: "search",
-            priority: 6,
-            vector: vec([0.0, 0.5, 0.5, 0.0]), // Mix of 2 and 3
-        },
-    },
-];
-describe("Query modes integration tests", () => {
-    if (!TURBOPUFFER_API_KEY) {
-        it.skip("requires TURBOPUFFER_API_KEY to be set", () => { });
-        return;
-    }
-    let tpuf;
-    let index;
-    const testIndexId = `kernl-query-test-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
-    beforeAll(async () => {
-        tpuf = new TurbopufferSearchIndex({
-            apiKey: TURBOPUFFER_API_KEY,
-            region: TURBOPUFFER_REGION,
-        });
-        // Create index with FTS-enabled fields
-        await tpuf.createIndex({
-            id: testIndexId,
-            schema: {
-                title: { type: "string", fts: true, filterable: true },
-                content: { type: "string", fts: true },
-                category: { type: "string", filterable: true },
-                priority: { type: "int", filterable: true },
-                vector: { type: "vector", dimensions: 4 },
-            },
-        });
-        index = tpuf.index(testIndexId);
-        // Insert test documents
-        await index.upsert(TEST_DOCS);
-        // Wait for indexing
-        await new Promise((r) => setTimeout(r, 2000));
-    }, 30000);
-    afterAll(async () => {
-        try {
-            await tpuf.deleteIndex(testIndexId);
-        }
-        catch {
-            // Ignore cleanup errors
-        }
-    });
-    // ============================================================
-    // VECTOR SEARCH
-    // ============================================================
-    describe("vector search", () => {
-        it("returns exact match as top result", async () => {
-            // Query with basis vector 1 - should match vec-1 best
-            const hits = await index.query({
-                query: [{ vector: [1.0, 0.0, 0.0, 0.0] }],
-                topK: 10,
-            });
-            expect(hits.length).toBeGreaterThan(0);
-            expect(hits[0].id).toBe("vec-1");
-        });
-        it("returns results in similarity order", async () => {
-            // Query with basis vector 2
-            const hits = await index.query({
-                query: [{ vector: [0.0, 1.0, 0.0, 0.0] }],
-                topK: 10,
-            });
-            expect(hits.length).toBeGreaterThan(0);
-            expect(hits[0].id).toBe("vec-2");
-            // Scores should be in descending order (or ascending distance)
-            for (let i = 1; i < hits.length; i++) {
-                expect(hits[i].score).toBeLessThanOrEqual(hits[i - 1].score);
-            }
-        });
-        it("mixed vector query finds composite matches", async () => {
-            // Query with mix of basis 1 and 2 - should prefer vec-5 which has [0.5, 0.5, 0, 0]
-            const hits = await index.query({
-                query: [{ vector: [0.5, 0.5, 0.0, 0.0] }],
-                topK: 10,
-            });
-            expect(hits.length).toBeGreaterThan(0);
-            expect(hits[0].id).toBe("vec-5");
-        });
-    });
-    // ============================================================
-    // TOPK BEHAVIOR
-    // ============================================================
-    describe("topK behavior", () => {
-        it("topK smaller than doc count returns exactly topK", async () => {
-            const hits = await index.query({
-                query: [{ vector: [0.5, 0.5, 0.5, 0.5] }],
-                topK: 3,
-            });
-            expect(hits.length).toBe(3);
-        });
-        it("topK larger than doc count returns all docs", async () => {
-            const hits = await index.query({
-                query: [{ vector: [0.5, 0.5, 0.5, 0.5] }],
-                topK: 100,
-            });
-            // We have 6 docs
-            expect(hits.length).toBe(6);
-        });
-        it("topK of 1 returns single best match", async () => {
-            const hits = await index.query({
-                query: [{ vector: [1.0, 0.0, 0.0, 0.0] }],
-                topK: 1,
-            });
-            expect(hits.length).toBe(1);
-            expect(hits[0].id).toBe("vec-1");
-        });
-    });
-    // ============================================================
-    // TEXT SEARCH (BM25)
-    // ============================================================
-    describe("text search (BM25)", () => {
-        it("single field text query finds matching docs", async () => {
-            const hits = await index.query({
-                query: [{ title: "neural" }],
-                topK: 10,
-                include: ["title"],
-            });
-            expect(hits.length).toBeGreaterThan(0);
-            // Should find docs with "neural" in title
-            const titles = hits.map((h) => h.document?.title);
-            expect(titles.some((t) => t?.toLowerCase().includes("neural"))).toBe(true);
-        });
-        it("content field text query finds matching docs", async () => {
-            const hits = await index.query({
-                query: [{ content: "transformer" }],
-                topK: 10,
-                include: ["content"],
-            });
-            expect(hits.length).toBeGreaterThan(0);
-            // vec-2 has "transformer" in content
-            expect(hits.some((h) => h.id === "vec-2")).toBe(true);
-        });
-        it("multi-field text query searches both fields", async () => {
-            const hits = await index.query({
-                query: [{ title: "database" }, { content: "database" }],
-                topK: 10,
-                include: ["title", "content"],
-            });
-            expect(hits.length).toBeGreaterThan(0);
-            // vec-3 and vec-4 have "database" related content
-            const ids = hits.map((h) => h.id);
-            expect(ids.some((id) => id === "vec-3" || id === "vec-4")).toBe(true);
-        });
-        it("text query with no matches returns empty", async () => {
-            const hits = await index.query({
-                query: [{ content: "xyznonexistentkeyword123" }],
-                topK: 10,
-            });
-            expect(hits.length).toBe(0);
-        });
-    });
-    // ============================================================
-    // HYBRID QUERIES (VECTOR + TEXT)
-    // ============================================================
-    describe("hybrid queries", () => {
-        it("combines vector and text signals", async () => {
-            // Vector points to vec-1, text "search" appears in vec-5, vec-6
-            const hits = await index.query({
-                query: [
-                    { vector: [1.0, 0.0, 0.0, 0.0] },
-                    { content: "search" },
-                ],
-                topK: 10,
-            });
-            expect(hits.length).toBeGreaterThan(0);
-            // Results should include docs matching either signal
-            const ids = hits.map((h) => h.id);
-            // vec-1 should be present (vector match)
-            expect(ids).toContain("vec-1");
-        });
-        it("hybrid with filter narrows results", async () => {
-            const hits = await index.query({
-                query: [
-                    { vector: [0.5, 0.5, 0.5, 0.5] },
-                    { content: "vector" },
-                ],
-                topK: 10,
-                filter: { category: "db" },
-                include: ["category"],
-            });
-            expect(hits.length).toBeGreaterThan(0);
-            for (const hit of hits) {
-                expect(hit.document?.category).toBe("db");
-            }
-        });
-    });
-    // ============================================================
-    // FUSION MODES
-    // ============================================================
-    describe("fusion modes", () => {
-        it("Sum fusion combines multiple signals", async () => {
-            const hits = await index.query({
-                query: [
-                    { vector: [1.0, 0.0, 0.0, 0.0] },
-                    { vector: [0.0, 1.0, 0.0, 0.0] },
-                ],
-                topK: 10,
-            });
-            expect(hits.length).toBeGreaterThan(0);
-            // With Sum fusion, vec-5 [0.5, 0.5, 0, 0] should score well
-            // as it has some similarity to both query vectors
-        });
-        it("Max fusion takes best signal per doc", async () => {
-            const hits = await index.query({
-                max: [
-                    { vector: [1.0, 0.0, 0.0, 0.0] },
-                    { vector: [0.0, 1.0, 0.0, 0.0] },
-                ],
-                topK: 10,
-            });
-            expect(hits.length).toBeGreaterThan(0);
-            // With Max fusion, vec-1 and vec-2 should be top as they
-            // exactly match one of the query vectors
-            const topIds = hits.slice(0, 2).map((h) => h.id);
-            expect(topIds).toContain("vec-1");
-            expect(topIds).toContain("vec-2");
-        });
-    });
-    // ============================================================
-    // INCLUDE SEMANTICS
-    // ============================================================
-    describe("include semantics", () => {
-        it("include: true returns all attributes", async () => {
-            const hits = await index.query({
-                query: [{ vector: [1.0, 0.0, 0.0, 0.0] }],
-                topK: 1,
-                include: true,
-            });
-            expect(hits.length).toBe(1);
-            expect(hits[0].document).toBeDefined();
-            expect(hits[0].document).toHaveProperty("title");
-            expect(hits[0].document).toHaveProperty("content");
-            expect(hits[0].document).toHaveProperty("category");
-            expect(hits[0].document).toHaveProperty("priority");
-        });
-        it("include: false returns no attributes", async () => {
-            const hits = await index.query({
-                query: [{ vector: [1.0, 0.0, 0.0, 0.0] }],
-                topK: 1,
-                include: false,
-            });
-            expect(hits.length).toBe(1);
-            // document should be undefined or empty
-            expect(hits[0].document).toBeUndefined();
-        });
-        it("include: [fields] returns only specified fields", async () => {
-            const hits = await index.query({
-                query: [{ vector: [1.0, 0.0, 0.0, 0.0] }],
-                topK: 1,
-                include: ["title", "category"],
-            });
-            expect(hits.length).toBe(1);
-            expect(hits[0].document).toBeDefined();
-            expect(hits[0].document).toHaveProperty("title");
-            expect(hits[0].document).toHaveProperty("category");
-            // Should NOT have content or priority
-            expect(hits[0].document).not.toHaveProperty("content");
-            expect(hits[0].document).not.toHaveProperty("priority");
-        });
-        it("include: [] returns no attributes", async () => {
-            const hits = await index.query({
-                query: [{ vector: [1.0, 0.0, 0.0, 0.0] }],
-                topK: 1,
-                include: [],
-            });
-            expect(hits.length).toBe(1);
-            // Empty include array should return no fields
-            expect(hits[0].document === undefined ||
-                Object.keys(hits[0].document).length === 0).toBe(true);
-        });
-    });
-    // ============================================================
-    // QUERY WITH FILTERS
-    // ============================================================
-    describe("query with filters", () => {
-        it("filter by category", async () => {
-            const hits = await index.query({
-                query: [{ vector: [0.5, 0.5, 0.5, 0.5] }],
-                topK: 10,
-                filter: { category: "ml" },
-                include: ["category"],
-            });
-            expect(hits.length).toBe(2); // vec-1, vec-2
-            for (const hit of hits) {
-                expect(hit.document?.category).toBe("ml");
-            }
-        });
-        it("filter by priority range", async () => {
-            const hits = await index.query({
-                query: [{ vector: [0.5, 0.5, 0.5, 0.5] }],
-                topK: 10,
-                filter: { priority: { $gte: 3, $lte: 5 } },
-                include: ["priority"],
-            });
-            expect(hits.length).toBe(3); // priority 3, 4, 5
-            for (const hit of hits) {
-                expect(hit.document?.priority).toBeGreaterThanOrEqual(3);
-                expect(hit.document?.priority).toBeLessThanOrEqual(5);
-            }
-        });
-        it("filter with $or", async () => {
-            const hits = await index.query({
-                query: [{ vector: [0.5, 0.5, 0.5, 0.5] }],
-                topK: 10,
-                filter: {
-                    $or: [{ category: "ml" }, { category: "search" }],
-                },
-                include: ["category"],
-            });
-            expect(hits.length).toBe(4); // ml: 2, search: 2
-            for (const hit of hits) {
-                expect(["ml", "search"]).toContain(hit.document?.category);
-            }
-        });
-    });
-    // ============================================================
-    // RESULT STRUCTURE
-    // ============================================================
-    describe("result structure", () => {
-        it("results have required fields", async () => {
-            const hits = await index.query({
-                query: [{ vector: [1.0, 0.0, 0.0, 0.0] }],
-                topK: 1,
-            });
-            expect(hits.length).toBe(1);
-            expect(hits[0]).toHaveProperty("id");
-            expect(hits[0]).toHaveProperty("index", testIndexId);
-            expect(hits[0]).toHaveProperty("score");
-            expect(typeof hits[0].id).toBe("string");
-            expect(typeof hits[0].score).toBe("number");
-        });
-        it("score is a valid number", async () => {
-            const hits = await index.query({
-                query: [{ vector: [1.0, 0.0, 0.0, 0.0] }],
-                topK: 5,
-            });
-            for (const hit of hits) {
-                expect(typeof hit.score).toBe("number");
-                expect(Number.isFinite(hit.score)).toBe(true);
-            }
-        });
-    });
-});

package/dist/convert.d.ts

@@ -1,68 +0,0 @@
-/**
- * Codecs for converting between kernl and Turbopuffer types.
- */
-import type { Codec } from "@kernl-sdk/shared/lib";
-import type { FieldSchema, VectorFieldSchema, ScalarFieldSchema, SearchDocument, SearchQuery, SearchHit, FilterExpression } from "@kernl-sdk/retrieval";
-import type { AttributeSchema, DistanceMetric, Row, NamespaceQueryParams } from "@turbopuffer/turbopuffer/resources/namespaces";
-import type { Filter } from "@turbopuffer/turbopuffer/resources/custom";
-type Similarity = VectorFieldSchema["similarity"];
-type ScalarType = ScalarFieldSchema["type"];
-type TpufType = string;
-/**
- * Codec for converting kernl scalar types to Turbopuffer attribute types.
- */
-export declare const SCALAR_TYPE: Codec<ScalarType, TpufType>;
-/**
- * Codec for converting similarity metric to Turbopuffer distance metric.
- *
- * Turbopuffer supports: cosine_distance, euclidean_squared
- * We support: cosine, euclidean, dot_product
- */
-export declare const SIMILARITY: Codec<Similarity, DistanceMetric>;
-/**
- * Codec-like converter for FieldSchema to Turbopuffer AttributeSchema.
- *
- * Takes the field name as context since Turbopuffer requires `ann: true`
- * only on the special `vector` attribute.
- */
-export declare const FIELD_SCHEMA: {
-    encode: (field: FieldSchema, name: string) => AttributeSchema;
-    decode: () => never;
-};
-/**
- * Codec for converting a full schema record.
- *
- * Validates that vector fields are named `vector` since Turbopuffer only
- * supports ANN indexing on that specific attribute name.
- */
-export declare const INDEX_SCHEMA: Codec<Record<string, FieldSchema>, Record<string, AttributeSchema>>;
-/**
- * Codec for converting SearchDocument to Turbopuffer Row.
- */
-export declare const DOCUMENT: Codec<SearchDocument, Row>;
-/**
- * Codec for converting FilterExpression to Turbopuffer Filter.
- */
-export declare const FILTER: {
-    encode: (filter: FilterExpression) => Filter;
-    decode: (_filter: Filter) => FilterExpression;
-};
-/**
- * Codec for converting SearchQuery to Turbopuffer NamespaceQueryParams.
- *
- * Note: Hybrid search (vector + text) requires multi-query which is
- * handled separately.
- */
-export declare const QUERY: {
-    encode: (query: SearchQuery) => NamespaceQueryParams;
-    decode: (_params: NamespaceQueryParams) => SearchQuery;
-};
-/**
- * Codec for converting Turbopuffer Row to SearchHit.
- */
-export declare const SEARCH_HIT: {
-    encode: (_hit: SearchHit) => Row;
-    decode: (row: Row, index: string) => SearchHit;
-};
-export {};
-//# sourceMappingURL=convert.d.ts.map

package/dist/convert.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"convert.d.ts","sourceRoot":"","sources":["../src/convert.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,KAAK,EACV,WAAW,EACX,iBAAiB,EACjB,iBAAiB,EAGjB,cAAc,EACd,WAAW,EACX,SAAS,EACT,gBAAgB,EAIjB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,KAAK,EACV,eAAe,EAEf,cAAc,EACd,GAAG,EACH,oBAAoB,EACrB,MAAM,+CAA+C,CAAC;AACvD,OAAO,KAAK,EAAE,MAAM,EAAU,MAAM,2CAA2C,CAAC;AAEhF,KAAK,UAAU,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAC;AAClD,KAAK,UAAU,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC;AAC5C,KAAK,QAAQ,GAAG,MAAM,CAAC;AAEvB;;GAEG;AACH,eAAO,MAAM,WAAW,EAAE,KAAK,CAAC,UAAU,EAAE,QAAQ,CA4CnD,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,cAAc,CAqBxD,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,YAAY;oBACP,WAAW,QAAQ,MAAM,KAAG,eAAe;;CAiC5D,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,YAAY,EAAE,KAAK,CAC9B,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,EAC3B,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CA0BhC,CAAC;AAyBF;;GAEG;AACH,eAAO,MAAM,QAAQ,EAAE,KAAK,CAAC,cAAc,EAAE,GAAG,CAc/C,CAAC;AAkEF;;GAEG;AACH,eAAO,MAAM,MAAM;qBACA,gBAAgB,KAAG,MAAM;sBAsCxB,MAAM,KAAG,gBAAgB;CAG5C,CAAC;AA4BF;;;;;GAKG;AACH,eAAO,MAAM,KAAK;oBACA,WAAW,KAAG,oBAAoB;sBAsChC,oBAAoB,KAAG,WAAW;CAGrD,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,UAAU;mBACN,SAAS,KAAG,GAAG;kBAIhB,GAAG,SAAS,MAAM,KAAG,SAAS;CAqB7C,CAAC"}