@kernl-sdk/pg 0.1.10 → 0.1.12

package/src/memory/store.ts

@@ -0,0 +1,166 @@
+/**
+ * PostgreSQL Memory store implementation.
+ */
+
+import type { Pool, PoolClient } from "pg";
+
+import type {
+  MemoryStore,
+  MemoryRecord,
+  NewMemory,
+  MemoryRecordUpdate,
+  MemoryListOptions,
+} from "kernl";
+import {
+  KERNL_SCHEMA_NAME,
+  MemoryRecordCodec,
+  NewMemoryCodec,
+  type MemoryDBRecord,
+} from "@kernl-sdk/storage";
+
+import { SQL_WHERE, ORDER, SQL_UPDATE } from "./sql";
+
+/**
+ * PostgreSQL memory store implementation.
+ *
+ * All async methods call `ensureInit()` before database operations
+ * to ensure schema/tables exist.
+ */
+export class PGMemoryStore implements MemoryStore {
+  private db: Pool | PoolClient;
+  private ensureInit: () => Promise<void>;
+
+  constructor(db: Pool | PoolClient, ensureInit: () => Promise<void>) {
+    this.db = db;
+    this.ensureInit = ensureInit;
+  }
+
+  /**
+   * Get a memory by ID.
+   */
+  async get(id: string): Promise<MemoryRecord | null> {
+    await this.ensureInit();
+
+    const result = await this.db.query<MemoryDBRecord>(
+      `SELECT * FROM ${KERNL_SCHEMA_NAME}.memories WHERE id = $1`,
+      [id],
+    );
+
+    if (result.rows.length === 0) {
+      return null;
+    }
+
+    return MemoryRecordCodec.decode(result.rows[0]);
+  }
+
+  /**
+   * List memories matching optional filter criteria.
+   */
+  async list(options?: MemoryListOptions): Promise<MemoryRecord[]> {
+    await this.ensureInit();
+
+    const { sql: where, params } = SQL_WHERE.encode({
+      filter: options?.filter,
+      startIdx: 1,
+    });
+
+    let idx = params.length + 1;
+    let query = `SELECT * FROM ${KERNL_SCHEMA_NAME}.memories`;
+
+    // build where + order by
+    if (where) query += ` WHERE ${where}`;
+    query += ` ORDER BY ${ORDER.encode({ order: options?.order })}`;
+
+    // add limit + offset
+    if (options?.limit) {
+      query += ` LIMIT $${idx++}`;
+      params.push(options.limit);
+    }
+    if (options?.offset) {
+      query += ` OFFSET $${idx++}`;
+      params.push(options.offset);
+    }
+
+    const result = await this.db.query<MemoryDBRecord>(query, params);
+    return result.rows.map((row) => MemoryRecordCodec.decode(row));
+  }
+
+  /**
+   * Create a new memory record.
+   */
+  async create(memory: NewMemory): Promise<MemoryRecord> {
+    await this.ensureInit();
+
+    const row = NewMemoryCodec.encode(memory);
+
+    const result = await this.db.query<MemoryDBRecord>(
+      `INSERT INTO ${KERNL_SCHEMA_NAME}.memories
+       (id, namespace, entity_id, agent_id, kind, collection, content, wmem, smem_expires_at, timestamp, created_at, updated_at, metadata)
+       VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb, $8, $9, $10, $11, $12, $13::jsonb)
+       RETURNING *`,
+      [
+        row.id,
+        row.namespace,
+        row.entity_id,
+        row.agent_id,
+        row.kind,
+        row.collection,
+        JSON.stringify(row.content), // ??
+        row.wmem,
+        row.smem_expires_at,
+        row.timestamp,
+        row.created_at,
+        row.updated_at,
+        row.metadata ? JSON.stringify(row.metadata) : null,
+      ],
+    );
+
+    return MemoryRecordCodec.decode(result.rows[0]);
+  }
+
+  /**
+   * Update a memory record.
+   */
+  async update(id: string, patch: MemoryRecordUpdate): Promise<MemoryRecord> {
+    await this.ensureInit();
+
+    const { sql: updates, params } = SQL_UPDATE.encode({ patch, startIdx: 1 });
+    const idx = params.length + 1;
+    params.push(id);
+
+    // (TODO): might we not want to return the whole record sometimes?
+    const result = await this.db.query<MemoryDBRecord>(
+      `UPDATE ${KERNL_SCHEMA_NAME}.memories SET ${updates} WHERE id = $${idx} RETURNING *`,
+      params,
+    );
+
+    if (result.rows.length === 0) {
+      throw new Error(`memory not found: ${id}`);
+    }
+
+    return MemoryRecordCodec.decode(result.rows[0]);
+  }
+
+  /**
+   * Delete a memory by ID.
+   */
+  async delete(id: string): Promise<void> {
+    await this.ensureInit();
+    await this.db.query(
+      `DELETE FROM ${KERNL_SCHEMA_NAME}.memories WHERE id = $1`,
+      [id],
+    );
+  }
+
+  /**
+   * Delete multiple memories by ID.
+   */
+  async mdelete(ids: string[]): Promise<void> {
+    if (ids.length === 0) return;
+    await this.ensureInit();
+    await this.db.query(
+      `DELETE FROM ${KERNL_SCHEMA_NAME}.memories WHERE id = ANY($1)`,
+      [ids],
+    );
+  }
+}

package/src/migrations.ts

@@ -4,7 +4,11 @@
 
 import type { PoolClient } from "pg";
 import type { Table, Column } from "@kernl-sdk/storage";
-import { TABLE_THREADS, TABLE_THREAD_EVENTS, SCHEMA_NAME } from "@kernl-sdk/storage";
+import {
+  TABLE_THREADS,
+  TABLE_THREAD_EVENTS,
+  TABLE_MEMORIES,
+} from "@kernl-sdk/storage";
 
 /**
  * Migration context with helpers.
@@ -22,14 +26,20 @@ export interface Migration {
 /**
  * List of all migrations in order.
  */
-export const migrations: Migration[] = [
+export const MIGRATIONS: Migration[] = [
   {
-    id: "0001_initial",
+    id: "001_threads",
     async up(ctx) {
       await ctx.createTable(TABLE_THREADS);
       await ctx.createTable(TABLE_THREAD_EVENTS);
     },
   },
+  {
+    id: "002_memories",
+    async up(ctx) {
+      await ctx.createTable(TABLE_MEMORIES);
+    },
+  },
 ];
 
 /**

package/src/pgvector/README.md

@@ -0,0 +1,50 @@
+# kernl :: pgvector
+
+## Index conventions
+
+kernl follows simple conventions so most indexes “just work” without extra configuration:
+
+```ts
+const pgvec = pgvector({ pool });
+const docs = pgvec.index<Doc>("docs"); // "public.docs"
+await docs.upsert({ id: "doc-1", title: "Hello", embedding: [/* ... */] });
+await docs.query({ title: "Hello" });
+```
+
+### Index id = table name
+
+By default, `index(name)` refers to the "public" schema and the name would be the table name. So:
+
+  - `search.index("docs")` refers to the table `public.docs`.
+  - `search.index("analytics.events")` refers to the table `analytics.events`.
+
+
+### Field conventions
+
+  - field names map directly to column names,
+      - `title` → `"title"`,
+      - `content` → `"content"`,
+      - `embedding` → `"embedding"`, etc.
+  - any field you pass a `number[]` for is used as a pgvector `vector` column with the same name.
+
+
+### Primary key column
+
+  - kernl assumes PK column is `id` by default,
+  - Upserts use `INSERT ... ON CONFLICT ("id") DO UPDATE ...`.
+  - If your table uses a different key name, you must explicitly bind the index:
+
+```ts
+const pgvec = pgvector({ pool });
+
+pgvec.bindIndex("docs", {
+  schema: "public",
+  table: "articles", // ← table name differs from passed schema name (atypical)
+  pkey: "article_id", // ← primary key is not "id"
+  fields: {
+    embedding: { column: "embed_vec", type: "vector", dimensions: 1536, similarity: "cosine" },
+    title:     { column: "article_title", type: "string" },
+    // ...
+  },
+});
+```

package/src/pgvector/__tests__/handle.test.ts

@@ -0,0 +1,335 @@
+import { describe, it, expect, beforeAll, afterAll, beforeEach } from "vitest";
+import { Pool } from "pg";
+import { PGSearchIndex } from "../search";
+
+const TEST_DB_URL = process.env.KERNL_PG_TEST_URL;
+const SCHEMA = "kernl_search_test";
+
+describe.sequential("PGIndexHandle", () => {
+  if (!TEST_DB_URL) {
+    it.skip("requires KERNL_PG_TEST_URL environment variable", () => {});
+    return;
+  }
+
+  let pool: Pool;
+  let search: PGSearchIndex;
+  let initialized = false;
+
+  const ensureInit = async () => {
+    if (initialized) return;
+    await pool.query(`CREATE EXTENSION IF NOT EXISTS vector`);
+    await pool.query(`CREATE SCHEMA IF NOT EXISTS "${SCHEMA}"`);
+    await pool.query(`
+      CREATE TABLE IF NOT EXISTS "${SCHEMA}"."documents" (
+        id TEXT PRIMARY KEY,
+        title TEXT,
+        content TEXT,
+        status TEXT,
+        views INTEGER DEFAULT 0,
+        embedding vector(3)
+      )
+    `);
+    await pool.query(`
+      CREATE INDEX IF NOT EXISTS documents_embedding_idx
+      ON "${SCHEMA}"."documents"
+      USING hnsw (embedding vector_cosine_ops)
+    `);
+    initialized = true;
+  };
+
+  beforeAll(async () => {
+    pool = new Pool({ connectionString: TEST_DB_URL });
+    search = new PGSearchIndex({ pool, ensureInit });
+
+    // Clean slate
+    await pool.query(`DROP SCHEMA IF EXISTS "${SCHEMA}" CASCADE`);
+  });
+
+  afterAll(async () => {
+    await pool.query(`DROP SCHEMA IF EXISTS "${SCHEMA}" CASCADE`);
+    await pool.end();
+  });
+
+  beforeEach(async () => {
+    // Clear table between tests
+    if (initialized) {
+      await pool.query(`DELETE FROM "${SCHEMA}"."documents"`);
+    }
+  });
+
+  describe("bindIndex", () => {
+    it("registers binding for later use", async () => {
+      await search.bindIndex("docs", {
+        schema: SCHEMA,
+        table: "documents",
+        pkey: "id",
+        fields: {
+          title: { column: "title", type: "string" },
+          content: { column: "content", type: "string" },
+          embedding: {
+            column: "embedding",
+            type: "vector",
+            dimensions: 3,
+            similarity: "cosine",
+          },
+        },
+      });
+
+      // Can get handle without error
+      const handle = search.index("docs");
+      expect(handle.id).toBe("docs");
+    });
+  });
+
+  describe("index().query()", () => {
+    beforeAll(async () => {
+      await search.bindIndex("docs", {
+        schema: SCHEMA,
+        table: "documents",
+        pkey: "id",
+        fields: {
+          title: { column: "title", type: "string" },
+          content: { column: "content", type: "string" },
+          status: { column: "status", type: "string" },
+          views: { column: "views", type: "int" },
+          embedding: {
+            column: "embedding",
+            type: "vector",
+            dimensions: 3,
+            similarity: "cosine",
+          },
+        },
+      });
+    });
+
+    async function insertDocs() {
+      await ensureInit();
+      await pool.query(`
+        INSERT INTO "${SCHEMA}"."documents" (id, title, content, status, views, embedding)
+        VALUES
+          ('doc1', 'Hello World', 'First document', 'active', 100, '[0.1, 0.2, 0.3]'),
+          ('doc2', 'Goodbye World', 'Second document', 'active', 200, '[0.4, 0.5, 0.6]'),
+          ('doc3', 'Hello Again', 'Third document', 'draft', 50, '[0.15, 0.25, 0.35]'),
+          ('doc4', 'Final Doc', 'Fourth document', 'archived', 500, '[0.9, 0.8, 0.7]')
+      `);
+    }
+
+    it("queries with vector search", async () => {
+      await insertDocs();
+
+      const handle = search.index("docs");
+      const results = await handle.query({
+        query: [{ embedding: [0.1, 0.2, 0.3] }],
+        topK: 2,
+      });
+
+      expect(results).toHaveLength(2);
+      expect(results[0].id).toBe("doc1"); // closest match
+      expect(results[0].score).toBeGreaterThan(0.9); // high similarity
+      expect(results[0].index).toBe("docs");
+    });
+
+    it("queries with filter", async () => {
+      await insertDocs();
+
+      const handle = search.index("docs");
+      const results = await handle.query({
+        query: [{ embedding: [0.1, 0.2, 0.3] }],
+        filter: { status: "active" },
+        topK: 10,
+      });
+
+      expect(results).toHaveLength(2);
+      results.forEach((r) => {
+        expect(r.document?.status).toBe("active");
+      });
+    });
+
+    it("queries with comparison filter", async () => {
+      await insertDocs();
+
+      const handle = search.index("docs");
+      const results = await handle.query({
+        query: [{ embedding: [0.1, 0.2, 0.3] }],
+        filter: { views: { $gte: 100 } },
+        topK: 10,
+      });
+
+      expect(results.length).toBeGreaterThanOrEqual(2);
+      results.forEach((r) => {
+        expect(r.document?.views).toBeGreaterThanOrEqual(100);
+      });
+    });
+
+    it("queries with $or filter", async () => {
+      await insertDocs();
+
+      const handle = search.index("docs");
+      const results = await handle.query({
+        query: [{ embedding: [0.1, 0.2, 0.3] }],
+        filter: {
+          $or: [{ status: "draft" }, { status: "archived" }],
+        },
+        topK: 10,
+      });
+
+      expect(results).toHaveLength(2);
+      results.forEach((r) => {
+        expect(["draft", "archived"]).toContain(r.document?.status);
+      });
+    });
+
+    it("queries with $in filter", async () => {
+      await insertDocs();
+
+      const handle = search.index("docs");
+      const results = await handle.query({
+        query: [{ embedding: [0.1, 0.2, 0.3] }],
+        filter: { status: { $in: ["active", "draft"] } },
+        topK: 10,
+      });
+
+      expect(results).toHaveLength(3);
+    });
+
+    it("respects topK limit", async () => {
+      await insertDocs();
+
+      const handle = search.index("docs");
+      const results = await handle.query({
+        query: [{ embedding: [0.5, 0.5, 0.5] }],
+        topK: 2,
+      });
+
+      expect(results).toHaveLength(2);
+    });
+
+    it("respects offset for pagination", async () => {
+      await insertDocs();
+
+      const handle = search.index("docs");
+
+      // Get first 2
+      const page1 = await handle.query({
+        query: [{ embedding: [0.5, 0.5, 0.5] }],
+        topK: 2,
+        offset: 0,
+      });
+
+      // Get next 2
+      const page2 = await handle.query({
+        query: [{ embedding: [0.5, 0.5, 0.5] }],
+        topK: 2,
+        offset: 2,
+      });
+
+      expect(page1).toHaveLength(2);
+      expect(page2).toHaveLength(2);
+      expect(page1[0].id).not.toBe(page2[0].id);
+    });
+
+    it("queries with orderBy (non-vector)", async () => {
+      await insertDocs();
+
+      const handle = search.index("docs");
+      const results = await handle.query({
+        filter: { status: "active" },
+        orderBy: { field: "views", direction: "desc" },
+        topK: 10,
+      });
+
+      expect(results).toHaveLength(2);
+      expect(results[0].document?.views).toBe(200);
+      expect(results[1].document?.views).toBe(100);
+    });
+
+    it("returns documents with mapped field names", async () => {
+      await insertDocs();
+
+      const handle = search.index("docs");
+      const results = await handle.query({
+        query: [{ embedding: [0.1, 0.2, 0.3] }],
+        topK: 1,
+      });
+
+      expect(results[0].document).toHaveProperty("title");
+      expect(results[0].document).toHaveProperty("content");
+      expect(results[0].document).toHaveProperty("status");
+      expect(results[0].document).toHaveProperty("views");
+      expect(results[0].document).toHaveProperty("embedding");
+    });
+
+    it("returns empty array when no matches", async () => {
+      await insertDocs();
+
+      const handle = search.index("docs");
+      const results = await handle.query({
+        query: [{ embedding: [0.1, 0.2, 0.3] }],
+        filter: { status: "nonexistent" },
+        topK: 10,
+      });
+
+      expect(results).toEqual([]);
+    });
+  });
+
+  describe("index() generic type", () => {
+    interface DocFields {
+      title: string;
+      content: string;
+      status: string;
+      views: number;
+    }
+
+    beforeAll(async () => {
+      await search.bindIndex("typed-docs", {
+        schema: SCHEMA,
+        table: "documents",
+        pkey: "id",
+        fields: {
+          title: { column: "title", type: "string" },
+          content: { column: "content", type: "string" },
+          status: { column: "status", type: "string" },
+          views: { column: "views", type: "int" },
+          embedding: {
+            column: "embedding",
+            type: "vector",
+            dimensions: 3,
+            similarity: "cosine",
+          },
+        },
+      });
+    });
+
+    it("provides typed document access", async () => {
+      await ensureInit();
+      await pool.query(`
+        INSERT INTO "${SCHEMA}"."documents" (id, title, content, status, views, embedding)
+        VALUES ('typed1', 'Typed Doc', 'Typed content', 'active', 42, '[0.1, 0.2, 0.3]')
+      `);
+
+      const handle = search.index<DocFields>("typed-docs");
+      const results = await handle.query({
+        query: [{ embedding: [0.1, 0.2, 0.3] }],
+        topK: 1,
+      });
+
+      // TypeScript should allow these without errors
+      const doc = results[0].document;
+      expect(doc?.title).toBe("Typed Doc");
+      expect(doc?.views).toBe(42);
+    });
+  });
+
+  describe("error handling", () => {
+    it("throws when table does not exist", async () => {
+      const handle = search.index("unbound-index");
+
+      // Convention-based indexing tries to use "public.unbound-index"
+      // which doesn't exist, so PostgreSQL throws an error
+      await expect(
+        handle.query({ query: [{ embedding: [0.1, 0.2, 0.3] }] }),
+      ).rejects.toThrow('relation "public.unbound-index" does not exist');
+    });
+  });
+});