@pikku/kysely 0.13.3 → 0.13.5

package/src/schema/pikku-schema.types.ts

@@ -0,0 +1,94 @@
+import { sql, type CompiledQuery, type Kysely, type RawBuilder } from 'kysely'
+
+/**
+ * One DDL statement, bound to a database.
+ *
+ * Bound rather than free-standing because the same declaration serves two
+ * purposes that need the same object: applied to a throwaway database it yields
+ * the schema to introspect, and compiled against a dialect it yields the SQL a
+ * migration is written from.
+ */
+export interface SchemaStatement {
+  execute(): Promise<unknown>
+  compile(): CompiledQuery
+}
+
+export type SchemaStatementFactory = (
+  db: Kysely<any>,
+  types: RequiredTypes
+) => SchemaStatement
+
+/**
+ * The physical types of the columns named in `requires`, keyed `table.column`.
+ *
+ * Introspected from the database the schemas are being applied to, because a
+ * foreign key column has to be declared with the type of the column it points
+ * at and that column belongs to somebody else.
+ */
+export type RequiredTypes = Record<string, string>
+
+/**
+ * The declared type for a foreign key onto a column another source owns.
+ *
+ * Falls back to `text` when nothing has been introspected — the case when the
+ * declaration is compiled without a database, where the real type is unknowable
+ * and only the shape of the statement is being inspected.
+ */
+export const requiredType = (
+  types: RequiredTypes,
+  table: string,
+  column: string
+): RawBuilder<unknown> => sql.raw(types[`${table}.${column}`] ?? 'text')
+
+/**
+ * The tables one part of the pikku runtime needs, declared rather than created.
+ *
+ * `name` labels the generated migration and the drift report, so it is part of
+ * the contract — changing it orphans the migration that was written under the
+ * old one.
+ */
+export interface PikkuSchema {
+  name: string
+  statements: SchemaStatementFactory[]
+  /**
+   * Tables this schema references but does not create, which something else
+   * must have created first.
+   *
+   * Declared rather than discovered so the failure arrives as a sentence naming
+   * what is missing and who owns it, instead of a foreign key error from the
+   * database — or, worse, a stub table conjured up to make the error go away.
+   */
+  requires?: SchemaRequirement[]
+}
+
+/**
+ * A column another source owns, which one of these schemas depends on.
+ *
+ * Named down to the column, not just the table, because that is the granularity
+ * the dependency actually has: a foreign key needs the referenced column to
+ * exist *and* needs its type in order to declare a matching one.
+ *
+ * `owner` is carried in the declaration rather than composed into the error at
+ * the point it is thrown, so adding a second prerequisite cannot leave behind a
+ * message that still names the first one's owner.
+ */
+export interface SchemaRequirement {
+  table: string
+  column: string
+  owner: string
+}
+
+/**
+ * Wrap a raw SQL fragment as a statement.
+ *
+ * For the DDL kysely's schema builder cannot express — a unique index over an
+ * expression, say. Note that raw SQL is *not* rewritten by `withSchema`, so a
+ * fragment naming a table unqualified will resolve against the connection's
+ * search path rather than the schema the rest of the declaration lands in.
+ */
+export const rawStatement =
+  (fragment: RawBuilder<unknown>): SchemaStatementFactory =>
+  (db) => ({
+    execute: () => fragment.execute(db),
+    compile: () => fragment.compile(db),
+  })

package/src/schema/schema.test.ts

@@ -0,0 +1,456 @@
+import { test, describe } from 'node:test'
+import assert from 'node:assert/strict'
+import {
+  Kysely,
+  PostgresAdapter,
+  PostgresIntrospector,
+  PostgresQueryCompiler,
+  SqliteAdapter,
+  SqliteIntrospector,
+  SqliteQueryCompiler,
+  SqliteDialect,
+  DummyDriver,
+} from 'kysely'
+import Database from 'better-sqlite3'
+import {
+  pikkuSchemas,
+  compilePikkuSchemas,
+  applyPikkuSchemas,
+  ensurePikkuSchema,
+  declaredTables,
+  unsatisfiedRequirements,
+  type PikkuSchema,
+} from './index.js'
+import { workflowSchema } from './workflow.schema.js'
+
+const dialect = (kind: 'postgres' | 'sqlite') =>
+  kind === 'postgres'
+    ? {
+        createAdapter: () => new PostgresAdapter(),
+        createDriver: () => new DummyDriver(),
+        createIntrospector: (db: Kysely<any>) => new PostgresIntrospector(db),
+        createQueryCompiler: () => new PostgresQueryCompiler(),
+      }
+    : {
+        createAdapter: () => new SqliteAdapter(),
+        createDriver: () => new DummyDriver(),
+        createIntrospector: (db: Kysely<any>) => new SqliteIntrospector(db),
+        createQueryCompiler: () => new SqliteQueryCompiler(),
+      }
+
+/** A driver whose every query fails with `error` — for testing the failure path. */
+class ThrowingDriver extends DummyDriver {
+  constructor(private readonly error: Error) {
+    super()
+  }
+  override async acquireConnection() {
+    return {
+      executeQuery: async (): Promise<never> => {
+        throw this.error
+      },
+      streamQuery: async function* () {},
+    }
+  }
+}
+
+const compileFor = (kind: 'postgres' | 'sqlite') =>
+  compilePikkuSchemas(new Kysely<any>({ dialect: dialect(kind) }))
+
+describe('pikku runtime schema', () => {
+  test('declares logical names and compiles them to physical ones', () => {
+    const sql = compileFor('postgres')
+
+    // The declaration says `aiThreads.resourceId`, matching how every query in
+    // the package addresses it; the plugin is what makes the table `ai_threads`.
+    assert.match(sql, /create table "ai_threads"/)
+    assert.match(sql, /"resource_id" varchar\(255\) not null/)
+    assert.doesNotMatch(
+      sql,
+      /"aiThreads"/,
+      'a camelCase table name means the plugin was not applied'
+    )
+  })
+
+  test('rewrites foreign key targets too', () => {
+    assert.match(
+      compileFor('postgres'),
+      /references "ai_threads" \("id"\)/,
+      'an unrewritten reference target points at a table that does not exist'
+    )
+  })
+
+  test('compiles for every dialect from the one declaration', () => {
+    for (const kind of ['postgres', 'sqlite'] as const) {
+      const sql = compileFor(kind)
+      assert.match(sql, /create table "workflow_runs"/, `${kind} workflow_runs`)
+      assert.match(sql, /create table "pikku_scopes"/, `${kind} pikku_scopes`)
+    }
+  })
+
+  test('carries the ai_run column the two old declarations disagreed on', () => {
+    assert.match(compileFor('postgres'), /"pending_approvals" text/)
+  })
+
+  test('creates tables outright rather than if-not-exists', () => {
+    // A migration runs once. `if not exists` would let a table that already
+    // exists in a different shape pass silently, which is the drift the
+    // migration is supposed to surface.
+    assert.doesNotMatch(compileFor('postgres'), /create table if not exists/)
+  })
+
+  test('drops the frozen uuid default from workflow primary keys', () => {
+    // It was `sql.raw("'" + crypto.randomUUID() + "'")`, evaluated once when the
+    // statement was built, so every row shared one default.
+    assert.match(
+      compileFor('postgres'),
+      /"workflow_run_id" text primary key,/,
+      'the primary key should carry no default at all'
+    )
+    assert.match(compileFor('postgres'), /"workflow_step_id" text primary key,/)
+    assert.match(compileFor('postgres'), /"history_id" text primary key,/)
+  })
+
+  test('every schema is named, for the migration it will be written into', () => {
+    const names = pikkuSchemas.map((s) => s.name)
+    assert.equal(new Set(names).size, names.length, 'names must be unique')
+    assert.ok(names.every((n) => /^[a-z][a-z-]*$/.test(n)))
+  })
+})
+
+describe('pikku runtime schema — prerequisites', () => {
+  const memoryDb = () =>
+    new Kysely<any>({
+      dialect: new SqliteDialect({ database: new Database(':memory:') }),
+    })
+
+  test('names the missing table and who owns it, before creating anything', async () => {
+    const db = memoryDb()
+    try {
+      await assert.rejects(
+        applyPikkuSchemas(db),
+        /The 'scope' schema requires 'user\.id', which nothing has created\. Better Auth owns it/
+      )
+
+      const created = await db.introspection.getTables()
+      assert.deepEqual(
+        created.map((t) => t.name),
+        [],
+        'a failed prerequisite must not leave a half-applied database'
+      )
+    } finally {
+      await db.destroy()
+    }
+  })
+
+  test('applies in full once the prerequisite is there', async () => {
+    const db = memoryDb()
+    try {
+      await db.schema
+        .createTable('user')
+        .addColumn('id', 'text', (col) => col.primaryKey())
+        .execute()
+
+      assert.deepEqual(await unsatisfiedRequirements(db), [])
+      await applyPikkuSchemas(db)
+
+      const created = new Set(
+        (await db.introspection.getTables()).map((t) => t.name)
+      )
+      assert.ok(created.has('pikku_user_role'))
+      assert.ok(created.has('workflow_runs'))
+    } finally {
+      await db.destroy()
+    }
+  })
+
+  test('takes the grant column type from the user id it references', () => {
+    // The hand-written DDL said `text`. Better Auth generates a `uuid` primary
+    // key on postgres, and postgres rejects a text column referencing it —
+    // which is why projects ended up writing these tables by hand.
+    const sql = compilePikkuSchemas(
+      new Kysely<any>({ dialect: dialect('postgres') }),
+      pikkuSchemas,
+      { 'user.id': 'uuid' }
+    )
+    assert.match(
+      sql,
+      /create table "pikku_user_role" \("user_id" uuid not null references "user" \("id"\)/
+    )
+    assert.match(
+      sql,
+      /create table "pikku_user_scope" \("user_id" uuid not null references "user" \("id"\)/
+    )
+  })
+
+  test('reports the gap as data, so a caller can describe it instead of dying', async () => {
+    const db = memoryDb()
+    try {
+      const unmet = await unsatisfiedRequirements(db)
+      assert.deepEqual(
+        unmet.map(({ schema, requirement }) => ({
+          schema: schema.name,
+          ...requirement,
+        })),
+        [{ schema: 'scope', table: 'user', column: 'id', owner: 'Better Auth' }]
+      )
+
+      // The rest of the declaration does not depend on auth, so it still applies.
+      const satisfiable = pikkuSchemas.filter((s) => s.name !== 'scope')
+      await applyPikkuSchemas(db, satisfiable)
+      const created = new Set(
+        (await db.introspection.getTables()).map((t) => t.name)
+      )
+      assert.ok(created.has('workflow_runs'))
+      assert.equal(created.has('pikku_user_role'), false)
+    } finally {
+      await db.destroy()
+    }
+  })
+
+  test('a statement failing part way takes the ones before it with it', async () => {
+    const db = memoryDb()
+    try {
+      await db.schema.createTable('taken').addColumn('id', 'text').execute()
+
+      // The prerequisite check cannot see this coming: nothing is missing, the
+      // second statement simply collides with a table the database already has.
+      const collides: PikkuSchema = {
+        name: 'collides',
+        statements: [
+          (db) => db.schema.createTable('first').addColumn('id', 'text'),
+          (db) => db.schema.createTable('taken').addColumn('id', 'text'),
+        ],
+      }
+
+      await assert.rejects(applyPikkuSchemas(db, [collides]))
+
+      const created = new Set(
+        (await db.introspection.getTables()).map((t) => t.name)
+      )
+      assert.equal(
+        created.has('first'),
+        false,
+        'the statement that succeeded must roll back with the one that did not'
+      )
+    } finally {
+      await db.destroy()
+    }
+  })
+})
+
+describe('ensurePikkuSchema — what a service does at boot', () => {
+  const memoryDb = () =>
+    new Kysely<any>({
+      dialect: new SqliteDialect({ database: new Database(':memory:') }),
+    })
+
+  test('creates the tables the first time and finds them the second', async () => {
+    const db = memoryDb()
+    try {
+      assert.equal(await ensurePikkuSchema(db, workflowSchema), 'created')
+      assert.equal(await ensurePikkuSchema(db, workflowSchema), 'present')
+    } finally {
+      await db.destroy()
+    }
+  })
+
+  test('two services declaring the same schema do not fight over it', async () => {
+    // The workflow service and the workflow mirror both own these tables, and
+    // both call init(). Whoever runs second must find them and stop, rather
+    // than issuing DDL the database will reject.
+    const db = memoryDb()
+    try {
+      assert.equal(await ensurePikkuSchema(db, workflowSchema), 'created')
+      assert.equal(await ensurePikkuSchema(db, workflowSchema), 'present')
+    } finally {
+      await db.destroy()
+    }
+  })
+
+  test('refuses a half-applied schema rather than filling in the rest', async () => {
+    // Something else already owns part of it — a migration, an older release,
+    // a hand-run script. Creating the remainder at boot leaves two authorities
+    // over one set of tables, which is the condition all of this exists to end.
+    const db = memoryDb()
+    try {
+      await db.schema
+        .createTable('workflow_runs')
+        .addColumn('workflow_run_id', 'text', (col) => col.primaryKey())
+        .execute()
+
+      await assert.rejects(
+        ensurePikkuSchema(db, workflowSchema),
+        /The 'workflow' schema is half applied/
+      )
+    } finally {
+      await db.destroy()
+    }
+  })
+})
+
+describe('ensurePikkuSchema — a connection bound to a schema', () => {
+  const WORKFLOW_TABLES = [
+    'workflow_runs',
+    'workflow_step',
+    'workflow_step_history',
+    'workflow_versions',
+  ]
+
+  /**
+   * A postgres connection whose introspection answers with `tables`.
+   *
+   * Stubbed rather than run against a real database because sqlite has one
+   * schema and nothing else here speaks postgres — and introspection is the
+   * only thing `ensurePikkuSchema` reads before it decides. DummyDriver takes
+   * whatever DDL follows.
+   */
+  const introspecting = (
+    tables: Array<{ schema: string | undefined; name: string }>
+  ) =>
+    new Kysely<any>({
+      dialect: {
+        ...dialect('postgres'),
+        createIntrospector: () => ({
+          getSchemas: async () => [],
+          getMetadata: async () => ({ tables: [] }),
+          getTables: async () =>
+            tables.map(({ schema, name }) => ({
+              schema,
+              name,
+              isView: false,
+              columns: [],
+            })),
+        }),
+      },
+    })
+
+  test('reads the schema out of its own DDL, not the table name', () => {
+    const db = new Kysely<any>({ dialect: dialect('postgres') })
+
+    assert.deepEqual(declaredTables(workflowSchema, db.withSchema('app')), [
+      { schema: 'app', name: 'workflow_runs' },
+      { schema: 'app', name: 'workflow_step' },
+      { schema: 'app', name: 'workflow_step_history' },
+      { schema: 'app', name: 'workflow_versions' },
+    ])
+
+    // Unqualified DDL resolves against the connection's search_path, which is
+    // not knowable from here — so there is no schema to report.
+    assert.deepEqual(
+      declaredTables(workflowSchema, db),
+      WORKFLOW_TABLES.map((name) => ({ schema: undefined, name }))
+    )
+  })
+
+  test('finds tables that already exist under that schema', async () => {
+    // The regression this exists for: a `withSchema('app')` connection reading
+    // its own `create table "app"."workflow_runs"` as a table called `app`,
+    // concluding every table was missing, and issuing a bare CREATE that the
+    // database rejects with `relation "workflow_runs" already exists` — on
+    // every boot, forever.
+    const db = introspecting(
+      WORKFLOW_TABLES.map((name) => ({ schema: 'app', name }))
+    )
+
+    assert.equal(
+      await ensurePikkuSchema(db.withSchema('app'), workflowSchema),
+      'present'
+    )
+  })
+
+  test('does not accept the same names in a different schema', async () => {
+    // `withSchema('app')` reads and writes `app.workflow_runs`. A
+    // `public.workflow_runs` is a different table and answers nothing.
+    const db = introspecting(
+      WORKFLOW_TABLES.map((name) => ({ schema: 'public', name }))
+    )
+
+    assert.equal(
+      await ensurePikkuSchema(db.withSchema('app'), workflowSchema),
+      'created'
+    )
+  })
+
+  test('still matches on the bare name when the connection is unbound', async () => {
+    const db = introspecting(
+      WORKFLOW_TABLES.map((name) => ({ schema: 'public', name }))
+    )
+
+    assert.equal(await ensurePikkuSchema(db, workflowSchema), 'present')
+  })
+
+  test('matches on the bare name when the engine reports no schemas', async () => {
+    // sqlite and mysql leave `TableMetadata.schema` unset. Holding out for the
+    // qualified pair there would read every existing table as missing — the
+    // same false negative this suite exists for, arrived at from the other side.
+    const db = introspecting(
+      WORKFLOW_TABLES.map((name) => ({ schema: undefined, name }))
+    )
+
+    assert.equal(
+      await ensurePikkuSchema(db.withSchema('main'), workflowSchema),
+      'present'
+    )
+  })
+
+  test('leaves a syntax error that is not sqlite refusing a qualifier alone', async () => {
+    // Postgres compiles the same qualified reference legitimately, and can
+    // raise a syntax error of its own on this DDL — a column type resolved from
+    // `requires` goes in verbatim. Answering for it in sqlite's name would be
+    // the misleading-error bug, reintroduced.
+    const pgError = Object.assign(
+      new Error('syntax error at or near "not"'),
+      { code: '42601' }
+    )
+    const db = new Kysely<any>({
+      dialect: {
+        ...dialect('postgres'),
+        createDriver: () => new ThrowingDriver(pgError),
+        createIntrospector: () => ({
+          getSchemas: async () => [],
+          getMetadata: async () => ({ tables: [] }),
+          getTables: async () => [],
+        }),
+      },
+    })
+
+    await assert.rejects(
+      ensurePikkuSchema(db.withSchema('app'), workflowSchema),
+      (error: Error) => {
+        assert.equal(error, pgError, 'the engine error must pass through as-is')
+        return true
+      }
+    )
+  })
+
+  test('explains the qualified foreign key sqlite cannot express', async () => {
+    // `withSchema` qualifies foreign key targets too. Postgres needs that;
+    // sqlite refuses it, and says only `near ".": syntax error` — which names
+    // neither the schema nor the connection that produced it.
+    const db = new Kysely<any>({
+      dialect: new SqliteDialect({ database: new Database(':memory:') }),
+    })
+    try {
+      await assert.rejects(
+        ensurePikkuSchema(db.withSchema('main'), workflowSchema),
+        (error: Error) => {
+          assert.match(error.message, /bound to a schema/)
+          assert.match(error.message, /takes a bare table name/i)
+          assert.ok(error.cause, 'the engine error must survive as the cause')
+          return true
+        }
+      )
+    } finally {
+      await db.destroy()
+    }
+  })
+
+  test('names the schema when it reports a half-applied one', async () => {
+    const db = introspecting([{ schema: 'app', name: 'workflow_runs' }])
+
+    await assert.rejects(
+      ensurePikkuSchema(db.withSchema('app'), workflowSchema),
+      /app\.workflow_step.*missing.*app\.workflow_runs already there/s
+    )
+  })
+})

package/src/schema/scope.schema.ts

@@ -0,0 +1,95 @@
+import { sql } from 'kysely'
+import { requiredType, type PikkuSchema } from './pikku-schema.types.js'
+
+/**
+ * Scopes, roles and the grants that bind them to users.
+ *
+ * `pikkuUserRole` and `pikkuUserScope` reference `user.id`, which Better Auth
+ * owns. Auth is a prerequisite rather than an optional companion: a grant is
+ * made to a user, so without the table the user lives in there is nothing to
+ * grant to, and the cascade that revokes grants when a user is deleted has
+ * nothing to hang off.
+ *
+ * `userId` takes its type from `user.id` rather than declaring one, because
+ * that type is Better Auth's to decide: `text` by default, `uuid` under
+ * `generateId: 'uuid'`, an identity `integer` under `'serial'`. The old
+ * hand-written DDL said `text`, and postgres rejects a `text` column
+ * referencing either of the others — so the whole statement failed and projects
+ * wrote these tables by hand instead.
+ */
+export const scopeSchema: PikkuSchema = {
+  name: 'scope',
+  requires: [{ table: 'user', column: 'id', owner: 'Better Auth' }],
+  statements: [
+    (db) =>
+      db.schema
+        .createTable('pikkuScopes')
+        .addColumn('name', 'text', (col) => col.primaryKey())
+        .addColumn('description', 'text')
+        .addColumn('declared', 'boolean', (col) =>
+          col.defaultTo(true).notNull()
+        ),
+
+    (db) =>
+      db.schema
+        .createTable('pikkuRoles')
+        .addColumn('name', 'text', (col) => col.primaryKey())
+        .addColumn('description', 'text')
+        .addColumn('createdAt', 'timestamp', (col) =>
+          col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
+        ),
+
+    (db) =>
+      db.schema
+        .createTable('pikkuRoleScopes')
+        .addColumn('role', 'text', (col) =>
+          col.notNull().references('pikkuRoles.name').onDelete('cascade')
+        )
+        .addColumn('scope', 'text', (col) =>
+          col.notNull().references('pikkuScopes.name').onDelete('cascade')
+        )
+        .addPrimaryKeyConstraint('pikku_role_scopes_pk', ['role', 'scope']),
+
+    (db, types) =>
+      db.schema
+        .createTable('pikkuUserRole')
+        .addColumn('userId', requiredType(types, 'user', 'id'), (col) =>
+          col.notNull().references('user.id').onDelete('cascade')
+        )
+        .addColumn('role', 'text', (col) =>
+          col.notNull().references('pikkuRoles.name').onDelete('cascade')
+        )
+        .addColumn('grantedBy', 'text')
+        .addColumn('grantedAt', 'timestamp', (col) =>
+          col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
+        )
+        .addPrimaryKeyConstraint('pikku_user_role_pk', ['userId', 'role']),
+
+    (db, types) =>
+      db.schema
+        .createTable('pikkuUserScope')
+        .addColumn('userId', requiredType(types, 'user', 'id'), (col) =>
+          col.notNull().references('user.id').onDelete('cascade')
+        )
+        .addColumn('scope', 'text', (col) =>
+          col.notNull().references('pikkuScopes.name').onDelete('cascade')
+        )
+        .addColumn('grantedBy', 'text')
+        .addColumn('grantedAt', 'timestamp', (col) =>
+          col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
+        )
+        .addPrimaryKeyConstraint('pikku_user_scope_pk', ['userId', 'scope']),
+
+    (db) =>
+      db.schema
+        .createIndex('pikku_role_scopes_scope_idx')
+        .on('pikkuRoleScopes')
+        .column('scope'),
+
+    (db) =>
+      db.schema
+        .createIndex('pikku_user_role_role_idx')
+        .on('pikkuUserRole')
+        .column('role'),
+  ],
+}

package/src/schema/secret.schema.ts

@@ -0,0 +1,40 @@
+import { sql } from 'kysely'
+import type { PikkuSchema } from './pikku-schema.types.js'
+
+/**
+ * Encrypted secrets and their audit trail.
+ *
+ * The audit table is declared unconditionally, where `init()` created it only
+ * when the service was constructed with auditing on. A migration cannot depend
+ * on runtime configuration, and the asymmetry favours declaring it: an unused
+ * table costs nothing, while turning auditing on against a database that never
+ * got the table fails on the first write.
+ */
+export const secretSchema: PikkuSchema = {
+  name: 'secret',
+  statements: [
+    (db) =>
+      db.schema
+        .createTable('secrets')
+        .addColumn('key', 'varchar(255)', (col) => col.primaryKey())
+        .addColumn('ciphertext', 'text', (col) => col.notNull())
+        .addColumn('wrappedDek', 'text', (col) => col.notNull())
+        .addColumn('keyVersion', 'integer', (col) => col.notNull())
+        .addColumn('createdAt', 'timestamp', (col) =>
+          col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
+        )
+        .addColumn('updatedAt', 'timestamp', (col) =>
+          col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
+        ),
+
+    (db) =>
+      db.schema
+        .createTable('secretsAudit')
+        .addColumn('id', 'varchar(36)', (col) => col.primaryKey())
+        .addColumn('secretKey', 'varchar(255)', (col) => col.notNull())
+        .addColumn('action', 'varchar(20)', (col) => col.notNull())
+        .addColumn('performedAt', 'timestamp', (col) =>
+          col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
+        ),
+  ],
+}