@kernl-sdk/libsql 0.1.38 → 0.1.39

package/src/migrations.ts

@@ -0,0 +1,206 @@
+/**
+ * Database migrations for LibSQL.
+ */
+
+import type { Client, Transaction } from "@libsql/client";
+import type { Table, Column } from "@kernl-sdk/storage";
+import {
+  KERNL_SCHEMA_NAME,
+  TABLE_THREADS,
+  TABLE_THREAD_EVENTS,
+  TABLE_MEMORIES,
+} from "@kernl-sdk/storage";
+
+import { SQL_IDENTIFIER_REGEX } from "./sql";
+
+/**
+ * Migration context with helpers.
+ */
+export interface MigrationContext {
+  client: Client | Transaction;
+  createTable: (table: Table<string, Record<string, Column>>) => Promise<void>;
+}
+
+export interface Migration {
+  id: string;
+  up: (ctx: MigrationContext) => Promise<void>;
+}
+
+/**
+ * List of all migrations in order.
+ */
+export const MIGRATIONS: Migration[] = [
+  {
+    id: "001_threads",
+    async up(ctx) {
+      await ctx.createTable(TABLE_THREADS);
+      await ctx.createTable(TABLE_THREAD_EVENTS);
+    },
+  },
+  {
+    id: "002_memories",
+    async up(ctx) {
+      await ctx.createTable(TABLE_MEMORIES);
+    },
+  },
+];
+
+/**
+ * Minimum schema version required by this version of @kernl/libsql.
+ */
+export const REQUIRED_SCHEMA_VERSION = "0001_initial";
+
+/**
+ * Map PostgreSQL types to SQLite types.
+ */
+function mapColumnType(type: string): string {
+  switch (type.toLowerCase()) {
+    case "jsonb":
+      return "TEXT"; // Store JSON as TEXT
+    case "bigint":
+      return "INTEGER"; // SQLite INTEGER can hold 64-bit
+    case "boolean":
+      return "INTEGER"; // SQLite uses 0/1 for boolean
+    default:
+      return type.toUpperCase();
+  }
+}
+
+/**
+ * Encode a value for use in a DEFAULT clause.
+ */
+function encodeDefault(col: Column, value: unknown): string {
+  if (value === null || value === undefined) return "NULL";
+
+  const type = col.type.toLowerCase();
+  switch (type) {
+    case "text":
+      return `'${String(value).replace(/'/g, "''")}'`;
+    case "integer":
+    case "bigint":
+      return String(value);
+    case "boolean":
+      return value ? "1" : "0";
+    case "jsonb":
+      return `'${JSON.stringify(value)}'`;
+    default:
+      return String(value);
+  }
+}
+
+/**
+ * Create a table from its definition.
+ *
+ * This function handles the conversion from the generic table definition
+ * to SQLite-compatible DDL.
+ */
+export async function createTable(
+  client: Client | Transaction,
+  table: Table<string, Record<string, Column>>,
+): Promise<void> {
+  if (!SQL_IDENTIFIER_REGEX.test(table.name)) {
+    throw new Error(`Invalid table name: ${table.name}`);
+  }
+
+  const columns: string[] = [];
+  const tableConstraints: string[] = [];
+  const indexes: { name: string; columns: string[]; unique?: boolean }[] = [];
+
+  // build column definitions
+  for (const name in table.columns) {
+    const col = table.columns[name];
+    const sqlType = mapColumnType(col.type);
+
+    const constraints: string[] = [];
+    if (col._pk) constraints.push("PRIMARY KEY");
+    if (col._unique) constraints.push("UNIQUE");
+    if (!col._nullable && !col._pk) constraints.push("NOT NULL");
+    if (col._default !== undefined) {
+      constraints.push(`DEFAULT ${encodeDefault(col, col._default)}`);
+    }
+
+    // foreign key reference
+    if (col._fk) {
+      let ref = `REFERENCES "${KERNL_SCHEMA_NAME}_${col._fk.table}" ("${col._fk.column}")`;
+      if (col._onDelete) {
+        ref += ` ON DELETE ${col._onDelete}`;
+      }
+      constraints.push(ref);
+    }
+
+    columns.push(
+      `"${name}" ${sqlType} ${constraints.join(" ")}`.trim(),
+    );
+  }
+
+  // table-level constraints
+  if (table.constraints) {
+    for (const constraint of table.constraints) {
+      switch (constraint.kind) {
+        case "unique": {
+          const name =
+            constraint.name ??
+            `${table.name}_${constraint.columns.join("_")}_unique`;
+          const cols = constraint.columns.map((c) => `"${c}"`).join(", ");
+          tableConstraints.push(`CONSTRAINT "${name}" UNIQUE (${cols})`);
+          break;
+        }
+
+        case "pkey": {
+          const name = constraint.name ?? `${table.name}_pkey`;
+          const cols = constraint.columns.map((c) => `"${c}"`).join(", ");
+          tableConstraints.push(`CONSTRAINT "${name}" PRIMARY KEY (${cols})`);
+          break;
+        }
+
+        case "fkey": {
+          throw new Error(
+            "Composite foreign keys not yet supported. Use column-level .references() for single-column FKs.",
+          );
+        }
+
+        case "check": {
+          const name = constraint.name ?? `${table.name}_check`;
+          tableConstraints.push(
+            `CONSTRAINT "${name}" CHECK (${constraint.expression})`,
+          );
+          break;
+        }
+
+        case "index": {
+          // collect indexes to create after table
+          indexes.push({
+            name: `idx_${table.name}_${constraint.columns.join("_")}`,
+            columns: constraint.columns,
+            unique: constraint.unique,
+          });
+          break;
+        }
+      }
+    }
+  }
+
+  const allConstraints = [...columns, ...tableConstraints];
+
+  // Use schema prefix in table name (SQLite doesn't have schemas)
+  const fullTableName = `${KERNL_SCHEMA_NAME}_${table.name}`;
+
+  const sql = `
+    CREATE TABLE IF NOT EXISTS "${fullTableName}" (
+      ${allConstraints.join(",\n      ")}
+    )
+  `.trim();
+
+  await client.execute(sql);
+
+  // create indexes
+  for (const index of indexes) {
+    const uniqueKeyword = index.unique ? "UNIQUE " : "";
+    const cols = index.columns.map((c) => `"${c}"`).join(", ");
+    const indexSql = `
+      CREATE ${uniqueKeyword}INDEX IF NOT EXISTS "${index.name}"
+      ON "${fullTableName}" (${cols})
+    `.trim();
+    await client.execute(indexSql);
+  }
+}

package/src/sql.ts

@@ -0,0 +1,35 @@
+/**
+ * SQL utilities for safe query construction.
+ */
+
+/**
+ * SQL identifier regex - alphanumeric + underscore, must start with letter/underscore.
+ */
+export const SQL_IDENTIFIER_REGEX = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
+
+/**
+ * SQL clause with placeholder parameters.
+ */
+export interface SQLClause {
+  sql: string;
+  params: unknown[];
+}
+
+/**
+ * Expand an array into individual ? placeholders for IN clause.
+ *
+ * LibSQL/SQLite doesn't support array parameters like PostgreSQL's ANY($1).
+ * This helper expands arrays into individual placeholders.
+ *
+ * @example
+ * expandArray(['a', 'b', 'c']) => { placeholders: '?, ?, ?', params: ['a', 'b', 'c'] }
+ */
+export function expandarray(arr: unknown[]): {
+  placeholders: string;
+  params: unknown[];
+} {
+  return {
+    placeholders: arr.map(() => "?").join(", "),
+    params: arr,
+  };
+}

package/src/storage.ts

@@ -0,0 +1,170 @@
+/**
+ * LibSQL storage adapter.
+ */
+
+import type { Client } from "@libsql/client";
+
+import type {
+  IAgentRegistry,
+  IModelRegistry,
+  KernlStorage,
+  Transaction,
+} from "kernl";
+import type { Table, Column } from "@kernl-sdk/storage";
+import { KERNL_SCHEMA_NAME, TABLE_MIGRATIONS } from "@kernl-sdk/storage";
+import { UnimplementedError } from "@kernl-sdk/shared/lib";
+
+import { LibSQLThreadStore } from "./thread/store";
+import { LibSQLMemoryStore } from "./memory/store";
+import { MIGRATIONS, createTable } from "./migrations";
+
+/**
+ * LibSQL storage configuration.
+ */
+export interface LibSQLStorageConfig {
+  /**
+   * LibSQL client instance.
+   */
+  client: Client;
+
+  /**
+   * Original connection URL (optional).
+   *
+   * Used to detect local file databases for setting SQLite PRAGMAs.
+   * Not needed if using a pre-configured client.
+   */
+  url?: string;
+}
+
+/**
+ * LibSQL storage adapter.
+ *
+ * Storage is lazily initialized on first use via `ensureInit()`. This means
+ * callers don't need to explicitly call `init()` - it happens automatically.
+ */
+export class LibSQLStorage implements KernlStorage {
+  private client: Client;
+  private url?: string;
+  private initPromise: Promise<void> | null = null;
+
+  threads: LibSQLThreadStore;
+  memories: LibSQLMemoryStore;
+
+  constructor(config: LibSQLStorageConfig) {
+    this.client = config.client;
+    this.url = config.url;
+    this.threads = new LibSQLThreadStore(this.client, () => this.ensureInit());
+    this.memories = new LibSQLMemoryStore(this.client, () => this.ensureInit());
+  }
+
+  /**
+   * Ensure storage is initialized before any operation.
+   *
+   * Safe to call multiple times - initialization only runs once.
+   */
+  private async ensureInit(): Promise<void> {
+    if (!this.initPromise) {
+      this.initPromise = this.init().catch((err) => {
+        this.initPromise = null;
+        throw err;
+      });
+    }
+    return this.initPromise;
+  }
+
+  /**
+   * Bind runtime registries to storage.
+   */
+  bind(registries: { agents: IAgentRegistry; models: IModelRegistry }): void {
+    this.threads.bind(registries);
+  }
+
+  /**
+   * Execute a function within a transaction.
+   */
+  async transaction<T>(fn: (tx: Transaction) => Promise<T>): Promise<T> {
+    throw new UnimplementedError();
+  }
+
+  /**
+   * Initialize the storage backend.
+   */
+  async init(): Promise<void> {
+    // Set SQLite PRAGMAs for local file databases
+    // (Turso handles these automatically for remote connections)
+    if (this.isLocal()) {
+      await this.client.execute("PRAGMA journal_mode = WAL");
+      await this.client.execute("PRAGMA busy_timeout = 5000");
+    }
+
+    // Create migrations table first
+    await this.createTable(TABLE_MIGRATIONS);
+    await this.migrate();
+  }
+
+  /**
+   * Check if this is a local file or in-memory database.
+   */
+  private isLocal(): boolean {
+    if (!this.url) return false;
+    return this.url.startsWith("file:") || this.url === ":memory:";
+  }
+
+  /**
+   * Close the storage backend and cleanup resources.
+   */
+  async close(): Promise<void> {
+    this.client.close();
+  }
+
+  /**
+   * Run migrations to ensure all required tables exist.
+   */
+  async migrate(): Promise<void> {
+    const tx = await this.client.transaction("write");
+
+    try {
+      // read applied migration IDs
+      const migrationsTable = `${KERNL_SCHEMA_NAME}_migrations`;
+      const result = await tx.execute(
+        `SELECT id FROM "${migrationsTable}" ORDER BY applied_at ASC`,
+      );
+      const applied = new Set(result.rows.map((row) => row.id as string));
+
+      // filter pending migrations
+      const pending = MIGRATIONS.filter((m) => !applied.has(m.id));
+      if (pending.length === 0) {
+        await tx.commit();
+        return;
+      }
+
+      // run pending migrations + insert into migrations table
+      for (const migration of pending) {
+        await migration.up({
+          client: tx,
+          createTable: async (table: Table<string, Record<string, Column>>) => {
+            await createTable(tx, table);
+          },
+        });
+        await tx.execute({
+          sql: `INSERT INTO "${migrationsTable}" (id, applied_at) VALUES (?, ?)`,
+          args: [migration.id, Date.now()],
+        });
+      }
+
+      await tx.commit();
+    } catch (error) {
+      await tx.rollback();
+      throw error;
+    }
+  }
+
+  /**
+   * Create a table from its definition.
+   */
+  private async createTable(
+    table: Table<string, Record<string, Column>>,
+  ): Promise<void> {
+    await createTable(this.client, table);
+  }
+}

package/src/thread/__tests__/append.test.ts

@@ -0,0 +1,201 @@
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import type { Client } from "@libsql/client";
+import { message, IN_PROGRESS, COMPLETED } from "@kernl-sdk/protocol";
+import type { ThreadEvent } from "kernl/internal";
+
+import {
+  create_client,
+  create_storage,
+  create_mock_registries,
+  testid,
+} from "../../__tests__/helpers";
+import { LibSQLStorage } from "../../storage";
+
+/** Create a ThreadEvent from a message */
+function evt(
+  id: string,
+  tid: string,
+  seq: number,
+  timestamp: Date,
+  role: "user" | "assistant" = "user",
+  text: string = `msg-${seq}`,
+  metadata: Record<string, unknown> = {},
+): ThreadEvent {
+  return {
+    ...message({ role, text }),
+    id,
+    tid,
+    seq,
+    timestamp,
+    metadata,
+  } as ThreadEvent;
+}
+
+/** Create a tool.call ThreadEvent */
+function toolCallEvt(
+  id: string,
+  tid: string,
+  seq: number,
+  timestamp: Date,
+): ThreadEvent {
+  return {
+    kind: "tool.call",
+    id,
+    tid,
+    seq,
+    timestamp,
+    callId: `call-${seq}`,
+    toolId: "test-tool",
+    state: IN_PROGRESS,
+    arguments: "{}",
+    metadata: {},
+  } as ThreadEvent;
+}
+
+/** Create a tool.result ThreadEvent */
+function toolResultEvt(
+  id: string,
+  tid: string,
+  seq: number,
+  timestamp: Date,
+): ThreadEvent {
+  return {
+    kind: "tool.result",
+    id,
+    tid,
+    seq,
+    timestamp,
+    callId: `call-${seq - 1}`,
+    toolId: "test-tool",
+    state: COMPLETED,
+    result: null,
+    error: null,
+    metadata: {},
+  } as ThreadEvent;
+}
+
+describe("LibSQLThreadStore append", () => {
+  let client: Client;
+  let storage: LibSQLStorage;
+  let tid: string;
+
+  beforeEach(async () => {
+    client = create_client();
+    storage = create_storage(client);
+    storage.bind(create_mock_registries());
+    await storage.memories.list(); // init
+
+    tid = testid("thread");
+    await storage.threads.insert({
+      id: tid,
+      namespace: "default",
+      agentId: "test-agent",
+      model: "test/model",
+    });
+  });
+
+  afterEach(() => {
+    client.close();
+  });
+
+  it("appends events in order", async () => {
+    const now = Date.now();
+
+    await storage.threads.append([
+      evt("e1", tid, 1, new Date(now), "user"),
+      evt("e2", tid, 2, new Date(now + 1), "assistant"),
+    ]);
+
+    const history = await storage.threads.history(tid);
+
+    expect(history.length).toBe(2);
+    expect(history[0].id).toBe("e1");
+    expect(history[0].seq).toBe(1);
+    expect(history[1].id).toBe("e2");
+    expect(history[1].seq).toBe(2);
+  });
+
+  it("preserves event data and metadata", async () => {
+    const now = Date.now();
+    const metadata = { source: "api", version: 2 };
+
+    await storage.threads.append([
+      evt("e1", tid, 1, new Date(now), "user", "Hello, world!", metadata),
+    ]);
+
+    const history = await storage.threads.history(tid);
+    const event = history[0];
+
+    expect(event.kind).toBe("message");
+    if (event.kind === "message") {
+      expect(event.role).toBe("user");
+      expect(event.content).toEqual([{ kind: "text", text: "Hello, world!" }]);
+    }
+    expect(event.metadata).toEqual(metadata);
+    expect(event.timestamp.getTime()).toBe(now);
+  });
+
+  it("is idempotent on duplicate ids", async () => {
+    const now = Date.now();
+
+    // First append
+    await storage.threads.append([evt("e1", tid, 1, new Date(now))]);
+
+    // Duplicate append (same id)
+    await storage.threads.append([evt("e1", tid, 1, new Date(now + 100))]);
+
+    const history = await storage.threads.history(tid);
+
+    // Should only have one event
+    expect(history.length).toBe(1);
+    // Original timestamp should be preserved
+    expect(history[0].timestamp.getTime()).toBe(now);
+  });
+
+  it("handles empty event list", async () => {
+    await storage.threads.append([]);
+
+    const history = await storage.threads.history(tid);
+    expect(history.length).toBe(0);
+  });
+
+  it("handles multiple appends", async () => {
+    const now = Date.now();
+
+    await storage.threads.append([evt("e1", tid, 1, new Date(now))]);
+
+    await storage.threads.append([
+      toolCallEvt("e2", tid, 2, new Date(now + 1)),
+      toolResultEvt("e3", tid, 3, new Date(now + 2)),
+    ]);
+
+    await storage.threads.append([evt("e4", tid, 4, new Date(now + 3))]);
+
+    const history = await storage.threads.history(tid);
+
+    expect(history.length).toBe(4);
+    expect(history.map((e) => e.id)).toEqual(["e1", "e2", "e3", "e4"]);
+  });
+
+  it("handles mixed duplicate and new events", async () => {
+    const now = Date.now();
+
+    await storage.threads.append([
+      evt("e1", tid, 1, new Date(now)),
+      evt("e2", tid, 2, new Date(now + 1)),
+    ]);
+
+    // Append with one duplicate and one new
+    await storage.threads.append([
+      evt("e2", tid, 2, new Date(now + 100)), // duplicate
+      evt("e3", tid, 3, new Date(now + 2)), // new
+    ]);
+
+    const history = await storage.threads.history(tid);
+
+    expect(history.length).toBe(3);
+    expect(history.map((e) => e.id)).toEqual(["e1", "e2", "e3"]);
+    // e2 should have original timestamp
+    expect(history[1].timestamp.getTime()).toBe(now + 1);
+  });
+});