@kernl-sdk/turbopuffer 0.1.0

package/dist/__tests__/integration/lifecycle.integration.test.js

@@ -0,0 +1,370 @@
+/**
+ * Index and handle lifecycle edge case integration tests.
+ *
+ * Tests edge cases for index lifecycle operations and document handling
+ * against real Turbopuffer API.
+ */
+import { describe, it, expect, beforeAll, afterAll } from "vitest";
+import { TurbopufferSearchIndex } from "../../search.js";
+const TURBOPUFFER_API_KEY = process.env.TURBOPUFFER_API_KEY;
+const TURBOPUFFER_REGION = process.env.TURBOPUFFER_REGION ?? "api";
+/**
+ * Helper to create a vector array.
+ */
+function vec(dim, fill = 0.1) {
+    return new Array(dim).fill(fill);
+}
+describe("Lifecycle edge cases integration tests", () => {
+    if (!TURBOPUFFER_API_KEY) {
+        it.skip("requires TURBOPUFFER_API_KEY to be set", () => { });
+        return;
+    }
+    let tpuf;
+    const testPrefix = `kernl-lifecycle-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+    beforeAll(() => {
+        tpuf = new TurbopufferSearchIndex({
+            apiKey: TURBOPUFFER_API_KEY,
+            region: TURBOPUFFER_REGION,
+        });
+    });
+    afterAll(async () => {
+        // Clean up any test indexes that might have been created
+        try {
+            const page = await tpuf.listIndexes({ prefix: testPrefix });
+            for (const idx of page.items) {
+                try {
+                    await tpuf.deleteIndex(idx.id);
+                }
+                catch {
+                    // Ignore individual cleanup errors
+                }
+            }
+        }
+        catch {
+            // Ignore cleanup errors
+        }
+    });
+    // ============================================================
+    // INDEX LIFECYCLE EDGE CASES
+    // ============================================================
+    describe("index lifecycle edge cases", () => {
+        it("describes index immediately after creation", async () => {
+            const indexId = `${testPrefix}-describe-after-create`;
+            await tpuf.createIndex({
+                id: indexId,
+                schema: {
+                    text: { type: "string" },
+                    vector: { type: "vector", dimensions: 4 },
+                },
+            });
+            const stats = await tpuf.describeIndex(indexId);
+            expect(stats.id).toBe(indexId);
+            expect(stats.count).toBe(0);
+            await tpuf.deleteIndex(indexId);
+        });
+        it("describe after delete throws", async () => {
+            const indexId = `${testPrefix}-describe-after-delete`;
+            await tpuf.createIndex({
+                id: indexId,
+                schema: { text: { type: "string" } },
+            });
+            await tpuf.deleteIndex(indexId);
+            await expect(tpuf.describeIndex(indexId)).rejects.toThrow();
+        });
+        it("delete non-existent index throws", async () => {
+            const indexId = `${testPrefix}-nonexistent-${Date.now()}`;
+            await expect(tpuf.deleteIndex(indexId)).rejects.toThrow();
+        });
+        it("create index with same id twice is idempotent", async () => {
+            const indexId = `${testPrefix}-duplicate`;
+            await tpuf.createIndex({
+                id: indexId,
+                schema: { text: { type: "string" } },
+            });
+            // Second create should succeed (idempotent) - Turbopuffer creates indexes implicitly
+            await tpuf.createIndex({
+                id: indexId,
+                schema: { text: { type: "string" } },
+            });
+            // Verify index still exists and is accessible
+            const stats = await tpuf.describeIndex(indexId);
+            expect(stats.id).toBe(indexId);
+            await tpuf.deleteIndex(indexId);
+        });
+        it("listIndexes returns empty page for non-matching prefix", async () => {
+            const page = await tpuf.listIndexes({
+                prefix: `nonexistent-prefix-${Date.now()}`,
+            });
+            expect(page.items).toEqual([]);
+        });
+    });
+    // ============================================================
+    // DOCUMENT UPSERT EDGE CASES
+    // ============================================================
+    describe("document upsert edge cases", () => {
+        const indexId = `${testPrefix}-upsert-edge`;
+        beforeAll(async () => {
+            await tpuf.createIndex({
+                id: indexId,
+                schema: {
+                    content: { type: "string" },
+                    count: { type: "int" },
+                    vector: { type: "vector", dimensions: 4 },
+                },
+            });
+        });
+        afterAll(async () => {
+            try {
+                await tpuf.deleteIndex(indexId);
+            }
+            catch {
+                // Ignore
+            }
+        });
+        it("multiple upserts same id keeps last write", async () => {
+            const index = tpuf.index(indexId);
+            // First write
+            await index.upsert({
+                id: "overwrite-test",
+                content: "First version",
+                count: 1,
+                vector: vec(4, 0.1),
+            });
+            // Second write
+            await index.upsert({
+                id: "overwrite-test",
+                content: "Second version",
+                count: 2,
+                vector: vec(4, 0.2),
+            });
+            // Third write
+            await index.upsert({
+                id: "overwrite-test",
+                content: "Final version",
+                count: 3,
+                vector: vec(4, 0.3),
+            });
+            // Wait for indexing
+            await new Promise((r) => setTimeout(r, 1000));
+            // Query and verify final state
+            const hits = await index.query({
+                query: [{ vector: [0.3, 0.3, 0.3, 0.3] }],
+                topK: 10,
+                filter: { id: "overwrite-test" },
+                include: ["content", "count"],
+            });
+            expect(hits.length).toBe(1);
+            expect(hits[0].document?.content).toBe("Final version");
+            expect(hits[0].document?.count).toBe(3);
+        });
+        it("upsert with empty fields object", async () => {
+            const index = tpuf.index(indexId);
+            // This should work - document with just id and vector
+            await index.upsert({
+                id: "minimal-doc",
+                vector: vec(4),
+            });
+            await new Promise((r) => setTimeout(r, 500));
+            const hits = await index.query({
+                query: [{ vector: [0.1, 0.1, 0.1, 0.1] }],
+                topK: 10,
+                filter: { id: "minimal-doc" },
+            });
+            expect(hits.some((h) => h.id === "minimal-doc")).toBe(true);
+        });
+        it("batch upsert with same id in batch throws", async () => {
+            const index = tpuf.index(indexId);
+            // Turbopuffer rejects batches with duplicate IDs
+            await expect(index.upsert([
+                {
+                    id: "batch-dup",
+                    content: "First",
+                    count: 1,
+                    vector: vec(4, 0.1),
+                },
+                {
+                    id: "batch-dup",
+                    content: "Second",
+                    count: 2,
+                    vector: vec(4, 0.2),
+                },
+                {
+                    id: "batch-dup",
+                    content: "Third",
+                    count: 3,
+                    vector: vec(4, 0.3),
+                },
+            ])).rejects.toThrow(/duplicate document IDs/i);
+        });
+    });
+    // ============================================================
+    // DOCUMENT DELETE EDGE CASES
+    // ============================================================
+    describe("document delete edge cases", () => {
+        const indexId = `${testPrefix}-delete-edge`;
+        beforeAll(async () => {
+            await tpuf.createIndex({
+                id: indexId,
+                schema: {
+                    content: { type: "string" },
+                    vector: { type: "vector", dimensions: 4 },
+                },
+            });
+        });
+        afterAll(async () => {
+            try {
+                await tpuf.deleteIndex(indexId);
+            }
+            catch {
+                // Ignore
+            }
+        });
+        it("delete non-existent id does not throw", async () => {
+            const index = tpuf.index(indexId);
+            // Should not throw - just a no-op
+            const result = await index.delete("nonexistent-doc-id-12345");
+            // count reflects input, not actual deletions
+            expect(result.count).toBe(1);
+        });
+        it("batch delete with non-existent ids does not throw", async () => {
+            const index = tpuf.index(indexId);
+            const result = await index.delete([
+                "nonexistent-1",
+                "nonexistent-2",
+                "nonexistent-3",
+            ]);
+            expect(result.count).toBe(3);
+        });
+        it("delete empty array returns count 0", async () => {
+            const index = tpuf.index(indexId);
+            const result = await index.delete([]);
+            expect(result.count).toBe(0);
+        });
+        it("delete then query returns empty", async () => {
+            const index = tpuf.index(indexId);
+            // Insert
+            await index.upsert({
+                id: "to-delete",
+                content: "Will be deleted",
+                vector: vec(4, 0.9),
+            });
+            await new Promise((r) => setTimeout(r, 500));
+            // Verify exists
+            let hits = await index.query({
+                query: [{ vector: [0.9, 0.9, 0.9, 0.9] }],
+                topK: 10,
+                filter: { id: "to-delete" },
+            });
+            expect(hits.length).toBe(1);
+            // Delete
+            await index.delete("to-delete");
+            await new Promise((r) => setTimeout(r, 500));
+            // Verify gone
+            hits = await index.query({
+                query: [{ vector: [0.9, 0.9, 0.9, 0.9] }],
+                topK: 10,
+                filter: { id: "to-delete" },
+            });
+            expect(hits.length).toBe(0);
+        });
+    });
+    // ============================================================
+    // QUERY EDGE CASES
+    // ============================================================
+    describe("query edge cases", () => {
+        const indexId = `${testPrefix}-query-edge`;
+        beforeAll(async () => {
+            await tpuf.createIndex({
+                id: indexId,
+                schema: {
+                    content: { type: "string", fts: true },
+                    vector: { type: "vector", dimensions: 4 },
+                },
+            });
+        });
+        afterAll(async () => {
+            try {
+                await tpuf.deleteIndex(indexId);
+            }
+            catch {
+                // Ignore
+            }
+        });
+        it("query on empty index returns empty array", async () => {
+            const index = tpuf.index(indexId);
+            const hits = await index.query({
+                query: [{ vector: [0.1, 0.2, 0.3, 0.4] }],
+                topK: 10,
+            });
+            expect(hits).toEqual([]);
+        });
+        it("query with filter matching nothing returns empty", async () => {
+            const index = tpuf.index(indexId);
+            // Add a doc first
+            await index.upsert({
+                id: "query-edge-doc",
+                content: "Test content",
+                vector: vec(4),
+            });
+            await new Promise((r) => setTimeout(r, 500));
+            // Query with filter that matches nothing
+            const hits = await index.query({
+                query: [{ vector: [0.1, 0.1, 0.1, 0.1] }],
+                topK: 10,
+                filter: { id: "nonexistent-id" },
+            });
+            expect(hits).toEqual([]);
+        });
+        it("topK of 0 throws", async () => {
+            const index = tpuf.index(indexId);
+            // Turbopuffer requires topK between 1 and 1200
+            await expect(index.query({
+                query: [{ vector: [0.1, 0.1, 0.1, 0.1] }],
+                topK: 0,
+            })).rejects.toThrow(/top_k must be between 1 and 1200/i);
+        });
+    });
+    // ============================================================
+    // VECTOR DIMENSION EDGE CASES
+    // ============================================================
+    describe("vector dimension edge cases", () => {
+        it("upsert with wrong vector dimension throws", async () => {
+            const indexId = `${testPrefix}-wrong-dim`;
+            await tpuf.createIndex({
+                id: indexId,
+                schema: {
+                    vector: { type: "vector", dimensions: 4 },
+                },
+            });
+            const index = tpuf.index(indexId);
+            // Try to upsert with wrong dimension (8 instead of 4)
+            await expect(index.upsert({
+                id: "wrong-dim-doc",
+                vector: vec(8), // Wrong dimension!
+            })).rejects.toThrow();
+            await tpuf.deleteIndex(indexId);
+        });
+        it("query with wrong vector dimension throws", async () => {
+            const indexId = `${testPrefix}-wrong-query-dim`;
+            await tpuf.createIndex({
+                id: indexId,
+                schema: {
+                    vector: { type: "vector", dimensions: 4 },
+                },
+            });
+            const index = tpuf.index(indexId);
+            // Insert valid doc
+            await index.upsert({
+                id: "valid-doc",
+                vector: vec(4),
+            });
+            await new Promise((r) => setTimeout(r, 500));
+            // Query with wrong dimension
+            await expect(index.query({
+                query: [{ vector: [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8] }], // 8 dims
+                topK: 10,
+            })).rejects.toThrow();
+            await tpuf.deleteIndex(indexId);
+        });
+    });
+});

package/dist/__tests__/integration/memory.integration.test.d.ts

@@ -0,0 +1,2 @@
+import "@kernl-sdk/ai/openai";
+//# sourceMappingURL=memory.integration.test.d.ts.map

package/dist/__tests__/integration/memory.integration.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"memory.integration.test.d.ts","sourceRoot":"","sources":["../../../src/__tests__/integration/memory.integration.test.ts"],"names":[],"mappings":"AAIA,OAAO,sBAAsB,CAAC"}

package/dist/__tests__/integration/memory.integration.test.js

@@ -0,0 +1,287 @@
+import { describe, it, expect, beforeAll, afterAll, beforeEach } from "vitest";
+import { Pool } from "pg";
+import { Kernl, Agent } from "kernl";
+import "@kernl-sdk/ai/openai"; // Register OpenAI embedding provider
+import { postgres } from "@kernl-sdk/pg";
+import { turbopuffer } from "../../index.js";
+const TEST_DB_URL = process.env.KERNL_PG_TEST_URL;
+const TURBOPUFFER_API_KEY = process.env.TURBOPUFFER_API_KEY;
+const TURBOPUFFER_REGION = process.env.TURBOPUFFER_REGION ?? "api";
+describe.sequential("Memory Integration with Turbopuffer", { timeout: 60000 }, () => {
+    if (!TEST_DB_URL) {
+        it.skip("requires KERNL_PG_TEST_URL environment variable", () => { });
+        return;
+    }
+    if (!TURBOPUFFER_API_KEY) {
+        it.skip("requires TURBOPUFFER_API_KEY environment variable", () => { });
+        return;
+    }
+    if (!process.env.OPENAI_API_KEY) {
+        it.skip("requires OPENAI_API_KEY environment variable", () => { });
+        return;
+    }
+    let pool;
+    let kernl;
+    let agent;
+    const vectorIndexId = `kernl-test-memories-${Date.now()}`;
+    // Unique ID generator to avoid stale data between tests
+    let testCounter = 0;
+    const uid = (base) => `${base}-${Date.now()}-${++testCounter}`;
+    beforeAll(async () => {
+        pool = new Pool({ connectionString: TEST_DB_URL });
+        // Clean slate for postgres
+        await pool.query('DROP SCHEMA IF EXISTS "kernl" CASCADE');
+        // Create Kernl with PG for relational + Turbopuffer for vector
+        kernl = new Kernl({
+            storage: {
+                db: postgres({ pool }),
+                vector: turbopuffer({
+                    apiKey: TURBOPUFFER_API_KEY,
+                    region: TURBOPUFFER_REGION,
+                }),
+            },
+            memory: {
+                embeddingModel: "openai/text-embedding-3-small",
+                dimensions: 1536,
+                indexId: vectorIndexId,
+            },
+        });
+        // Register a dummy agent for test scope
+        const model = {
+            spec: "1.0",
+            provider: "test",
+            modelId: "test-model",
+        };
+        agent = new Agent({
+            id: "test-agent",
+            name: "Test Agent",
+            instructions: () => "test instructions",
+            model,
+        });
+        kernl.register(agent);
+        // Initialize storage (creates "kernl" schema and tables)
+        await kernl.storage.init();
+    });
+    afterAll(async () => {
+        // Clean up Turbopuffer namespace
+        try {
+            const tpuf = turbopuffer({
+                apiKey: TURBOPUFFER_API_KEY,
+                region: TURBOPUFFER_REGION,
+            });
+            await tpuf.deleteIndex(vectorIndexId);
+        }
+        catch {
+            // Ignore if already deleted
+        }
+        if (kernl) {
+            await kernl.storage.close();
+        }
+    });
+    beforeEach(async () => {
+        // Clean memories in postgres
+        await pool.query('DELETE FROM "kernl"."memories"');
+        // Note: We don't delete the Turbopuffer namespace between tests because
+        // MemoryIndexHandle caches its init state. The namespace is cleaned up
+        // in afterAll. Tests use unique memory IDs so they don't conflict.
+    });
+    it("creates memory and indexes it in Turbopuffer", async () => {
+        const id = uid("m");
+        const memory = await agent.memories.create({
+            id,
+            namespace: "test",
+            collection: "facts",
+            content: { text: "The user loves TypeScript programming" },
+        });
+        expect(memory.id).toBe(id);
+        expect(memory.content.text).toBe("The user loves TypeScript programming");
+        // Verify memory exists in postgres
+        const dbResult = await pool.query('SELECT * FROM "kernl"."memories" WHERE id = $1', [id]);
+        expect(dbResult.rows).toHaveLength(1);
+        // Verify memory is searchable in Turbopuffer
+        const results = await agent.memories.search({
+            query: "TypeScript",
+            limit: 10,
+        });
+        expect(results.length).toBeGreaterThan(0);
+        expect(results[0].document?.id).toBe(id);
+    });
+    it("searches memories using vector search", async () => {
+        // Create several memories with unique IDs
+        const id1 = uid("m");
+        const id2 = uid("m");
+        const id3 = uid("m");
+        await agent.memories.create({
+            id: id1,
+            namespace: "test",
+            collection: "facts",
+            content: { text: "The user loves TypeScript programming" },
+        });
+        await agent.memories.create({
+            id: id2,
+            namespace: "test",
+            collection: "facts",
+            content: { text: "The user enjoys cooking Italian food" },
+        });
+        await agent.memories.create({
+            id: id3,
+            namespace: "test",
+            collection: "facts",
+            content: { text: "TypeScript has excellent type safety" },
+        });
+        // Search for TypeScript-related memories (vector-only since hybrid not supported)
+        const results = await agent.memories.search({
+            query: "programming languages",
+            limit: 10,
+        });
+        expect(results.length).toBeGreaterThan(0);
+        // Should find TypeScript-related memories with higher scores
+        const ids = results.map((r) => r.document?.id);
+        expect(ids).toContain(id1); // Direct match
+        expect(ids).toContain(id3); // Related to TypeScript
+    });
+    it("returns no results when filters exclude all matches", async () => {
+        const id = uid("m");
+        const ns = uid("ns");
+        await agent.memories.create({
+            id,
+            namespace: ns,
+            collection: "facts",
+            content: { text: "User likes hiking" },
+        });
+        // Filter for a different namespace that has no memories
+        const results = await agent.memories.search({
+            query: "hiking",
+            filter: { scope: { namespace: "nonexistent-ns" } },
+            limit: 10,
+        });
+        expect(results.length).toBe(0);
+    });
+    it("filters search results by scope", async () => {
+        const id1 = uid("m");
+        const id2 = uid("m");
+        const ns1 = uid("user");
+        const ns2 = uid("user");
+        await agent.memories.create({
+            id: id1,
+            namespace: ns1,
+            collection: "facts",
+            content: { text: "User 1 likes cats" },
+        });
+        await agent.memories.create({
+            id: id2,
+            namespace: ns2,
+            collection: "facts",
+            content: { text: "User 2 likes cats" },
+        });
+        // Search only in ns1 namespace
+        const results = await agent.memories.search({
+            query: "cats",
+            filter: { scope: { namespace: ns1 } },
+            limit: 10,
+        });
+        expect(results.length).toBe(1);
+        expect(results[0].document?.id).toBe(id1);
+    });
+    it("respects topK limit", async () => {
+        const ns = uid("ns");
+        await agent.memories.create({
+            namespace: ns,
+            collection: "facts",
+            content: { text: "The user likes TypeScript" },
+        });
+        await agent.memories.create({
+            namespace: ns,
+            collection: "facts",
+            content: { text: "The user likes JavaScript" },
+        });
+        await agent.memories.create({
+            namespace: ns,
+            collection: "facts",
+            content: { text: "The user likes Rust" },
+        });
+        // Filter by namespace to only get our test's memories
+        const results = await agent.memories.search({
+            query: "programming languages",
+            filter: { scope: { namespace: ns } },
+            limit: 1,
+        });
+        expect(results.length).toBe(1);
+    });
+    it("updates memory content and re-indexes", async () => {
+        const id = uid("m");
+        const ns = uid("ns");
+        await agent.memories.create({
+            id,
+            namespace: ns,
+            collection: "facts",
+            content: { text: "Original content about dogs" },
+        });
+        // Update content (still use kernl.memories for update - not yet on agent API)
+        await kernl.memories.update({
+            id,
+            content: { text: "Updated content about cats" },
+        });
+        // Search should find updated content
+        const results = await agent.memories.search({
+            query: "cats",
+            filter: { scope: { namespace: ns } },
+            limit: 10,
+        });
+        expect(results.length).toBeGreaterThan(0);
+        const match = results.find((r) => r.document?.id === id);
+        expect(match).toBeDefined();
+        expect(match?.document?.text).toBe("Updated content about cats");
+    });
+    it("creates memories with multimodal content", async () => {
+        const id = uid("m");
+        const ns = uid("ns");
+        await agent.memories.create({
+            id,
+            namespace: ns,
+            collection: "media",
+            content: {
+                text: "A beautiful sunset",
+                image: {
+                    data: "base64encodedimage",
+                    mime: "image/png",
+                    alt: "Sunset over the ocean",
+                },
+            },
+        });
+        // Should be searchable by text
+        const results = await agent.memories.search({
+            query: "sunset",
+            filter: { scope: { namespace: ns } },
+            limit: 10,
+        });
+        expect(results.length).toBeGreaterThan(0);
+        const match = results.find((r) => r.document?.id === id);
+        expect(match).toBeDefined();
+    });
+    it("filters by collection", async () => {
+        const id1 = uid("m");
+        const id2 = uid("m");
+        const ns = uid("ns");
+        await agent.memories.create({
+            id: id1,
+            namespace: ns,
+            collection: "facts",
+            content: { text: "This is a fact about programming" },
+        });
+        await agent.memories.create({
+            id: id2,
+            namespace: ns,
+            collection: "preferences",
+            content: { text: "User prefers programming in TypeScript" },
+        });
+        // Filter by collection and namespace
+        const results = await agent.memories.search({
+            query: "programming",
+            filter: { scope: { namespace: ns }, collections: ["facts"] },
+            limit: 10,
+        });
+        expect(results.length).toBe(1);
+        expect(results[0].document?.id).toBe(id1);
+    });
+});

package/dist/__tests__/integration/query.integration.test.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Comprehensive query modes integration tests.
+ *
+ * Tests vector search, BM25 text search, hybrid queries, fusion modes,
+ * topK behavior, include semantics, and rank ordering against real Turbopuffer API.
+ */
+export {};
+//# sourceMappingURL=query.integration.test.d.ts.map

package/dist/__tests__/integration/query.integration.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"query.integration.test.d.ts","sourceRoot":"","sources":["../../../src/__tests__/integration/query.integration.test.ts"],"names":[],"mappings":"AAAA;;;;;GAKG"}