@pikku/kysely 0.13.7 → 0.13.11

package/src/schema/index.ts

@@ -1,8 +1,10 @@
 import { CamelCasePlugin, type Kysely } from 'kysely'
-import type {
-  PikkuSchema,
-  RequiredTypes,
-  SchemaRequirement,
+import {
+  schemaContext,
+  type PikkuSchema,
+  type RequiredTypes,
+  type SchemaContext,
+  type SchemaRequirement,
 } from './pikku-schema.types.js'
 import { aiSchema } from './ai.schema.js'
 import { channelSchema } from './channel.schema.js'
@@ -17,11 +19,24 @@ import { workflowSchema } from './workflow.schema.js'
 export type {
   PikkuSchema,
   RequiredTypes,
+  SchemaContext,
   SchemaRequirement,
   SchemaStatement,
   SchemaStatementFactory,
 } from './pikku-schema.types.js'
-export { rawStatement, requiredType } from './pikku-schema.types.js'
+export {
+  rawStatement,
+  requiredType,
+  schemaContext,
+  unqualifiedContext,
+} from './pikku-schema.types.js'
+
+/**
+ * Opt-in, so it is named here rather than folded into `pikkuSchemas` — the
+ * runtime works without an audit table, and `KyselyAuditService.init()` applies
+ * this for the projects that wire the sink.
+ */
+export { auditSchema } from './audit.schema.js'
 
 /**
  * Every table the pikku runtime needs, in dependency order.
@@ -53,6 +68,29 @@ export const pikkuSchemas: PikkuSchema[] = [
  */
 const bind = (db: Kysely<any>) => db.withPlugin(new CamelCasePlugin())
 
+/**
+ * The schema a connection is bound to, or `undefined` for one that is not.
+ *
+ * Read back out of compiled DDL rather than off the connection, because kysely
+ * keeps the `withSchema(...)` setting inside its executor and exposes no way to
+ * ask. Compiling a throwaway `create table` is the same trick `declaredTables`
+ * uses on the real statements, against the same regex — nothing is executed and
+ * the probe never leaves this function.
+ *
+ * Derived rather than passed in so that raw statements follow whatever
+ * connection they are handed. A caller that had to remember to say the schema
+ * twice would eventually say it once.
+ */
+const boundSchema = (db: Kysely<any>): string | undefined =>
+  CREATE_TABLE.exec(
+    db.schema.createTable('pikkuSchemaProbe').addColumn('id', 'text').compile()
+      .sql
+  )?.[1]
+
+/** The context for statements compiled against `db`, matching its binding. */
+const contextFor = (db: Kysely<any>): SchemaContext =>
+  schemaContext(boundSchema(db))
+
 export interface UnmetRequirement {
   schema: PikkuSchema
   requirement: SchemaRequirement
@@ -141,9 +179,10 @@ export const applyPikkuSchemas = async (
 
   await db.transaction().execute(async (trx) => {
     const bound = bind(trx)
+    const ctx = contextFor(bound)
     for (const schema of schemas) {
       for (const statement of schema.statements) {
-        const query = statement(bound, types)
+        const query = statement(bound, types, ctx)
         try {
           await query.execute()
         } catch (error) {
@@ -239,9 +278,10 @@ export const declaredTables = (
   db: Kysely<any>
 ): DeclaredTable[] => {
   const bound = bind(db)
+  const ctx = contextFor(bound)
   const tables: DeclaredTable[] = []
   for (const statement of schema.statements) {
-    const match = CREATE_TABLE.exec(statement(bound, {}).compile().sql)
+    const match = CREATE_TABLE.exec(statement(bound, {}, ctx).compile().sql)
     if (match) tables.push({ schema: match[1], name: match[2]! })
   }
   return tables
@@ -357,17 +397,26 @@ export const ensurePikkuSchema = async (
  * `types` comes from `resolveRequirements` against the database the migration
  * is being written for. Omit it and foreign keys onto another source's columns
  * compile as `text`, which is a shape, not a migration.
+ *
+ * `schema` qualifies every table the migration creates, for a project that
+ * keeps the runtime tables somewhere other than the default search path. It is
+ * applied here and not to `db` itself because the caller's connection is a
+ * throwaway the declaration was just *applied* to, in whatever schema that
+ * database actually has — qualifying that would create tables in a schema the
+ * scratch database has never heard of. Only the rendered SQL is bound.
  */
 export const compilePikkuSchemas = (
   db: Kysely<any>,
   schemas: PikkuSchema[] = pikkuSchemas,
-  types: RequiredTypes = {}
+  types: RequiredTypes = {},
+  schemaName?: string
 ): string => {
-  const bound = bind(db)
+  const bound = schemaName ? bind(db).withSchema(schemaName) : bind(db)
+  const ctx = contextFor(bound)
   return schemas
     .flatMap((schema) =>
       schema.statements.map(
-        (statement) => `${statement(bound, types).compile().sql};`
+        (statement) => `${statement(bound, types, ctx).compile().sql};`
       )
     )
     .join('\n\n')

package/src/schema/pikku-schema.types.ts

@@ -15,9 +15,51 @@ export interface SchemaStatement {
 
 export type SchemaStatementFactory = (
   db: Kysely<any>,
-  types: RequiredTypes
+  types: RequiredTypes,
+  ctx: SchemaContext
 ) => SchemaStatement
 
+/**
+ * What a statement needs to know about the schema its DDL lands in.
+ *
+ * Only raw SQL needs this. The schema builder learns the schema from the
+ * connection — `withSchema('app')` qualifies every table it names, foreign key
+ * targets included — but a raw fragment is passed through verbatim, so a
+ * statement that spells its own SQL has to spell the qualifier too.
+ */
+export interface SchemaContext {
+  /**
+   * A table as the surrounding DDL addresses it: `"app"."credentials"` on a
+   * schema-bound connection, `"credentials"` otherwise.
+   *
+   * Takes the physical name, not the camelCase one the schema builder accepts.
+   * Raw SQL does not go through `CamelCasePlugin`, so it is already written in
+   * physical names throughout and this is no exception.
+   */
+  table: (name: string) => RawBuilder<unknown>
+}
+
+/**
+ * The `SchemaContext` for DDL that names no schema.
+ *
+ * The default everywhere a schema has not been asked for, which is every engine
+ * but postgres and most postgres projects too.
+ */
+export const unqualifiedContext: SchemaContext = {
+  table: (name) => sql.raw(`"${name}"`),
+}
+
+/**
+ * A `SchemaContext` qualifying every table into `schema`.
+ *
+ * Identifiers are quoted rather than interpolated bare so a schema named after
+ * a reserved word still compiles.
+ */
+export const schemaContext = (schema: string | undefined): SchemaContext =>
+  schema
+    ? { table: (name) => sql.raw(`"${schema}"."${name}"`) }
+    : unqualifiedContext
+
 /**
  * The physical types of the columns named in `requires`, keyed `table.column`.
  *
@@ -82,13 +124,26 @@ export interface SchemaRequirement {
  * Wrap a raw SQL fragment as a statement.
  *
  * For the DDL kysely's schema builder cannot express — a unique index over an
- * expression, say. Note that raw SQL is *not* rewritten by `withSchema`, so a
- * fragment naming a table unqualified will resolve against the connection's
- * search path rather than the schema the rest of the declaration lands in.
+ * expression, say. Raw SQL is *not* rewritten by `withSchema`, so a fragment
+ * naming a table unqualified resolves against the connection's search path
+ * rather than the schema the rest of the declaration lands in. Take the
+ * function form and name tables through `ctx.table` to stay with them; the
+ * bare-fragment form is for SQL that names no table at all.
  */
-export const rawStatement =
-  (fragment: RawBuilder<unknown>): SchemaStatementFactory =>
-  (db) => ({
-    execute: () => fragment.execute(db),
-    compile: () => fragment.compile(db),
-  })
+export function rawStatement(
+  fragment: RawBuilder<unknown>
+): SchemaStatementFactory
+export function rawStatement(
+  build: (ctx: SchemaContext) => RawBuilder<unknown>
+): SchemaStatementFactory
+export function rawStatement(
+  source: RawBuilder<unknown> | ((ctx: SchemaContext) => RawBuilder<unknown>)
+): SchemaStatementFactory {
+  return (db, _types, ctx = unqualifiedContext) => {
+    const fragment = typeof source === 'function' ? source(ctx) : source
+    return {
+      execute: () => fragment.execute(db),
+      compile: () => fragment.compile(db),
+    }
+  }
+}

package/src/schema/schema.test.ts

@@ -398,10 +398,9 @@ describe('ensurePikkuSchema — a connection bound to a schema', () => {
     // raise a syntax error of its own on this DDL — a column type resolved from
     // `requires` goes in verbatim. Answering for it in sqlite's name would be
     // the misleading-error bug, reintroduced.
-    const pgError = Object.assign(
-      new Error('syntax error at or near "not"'),
-      { code: '42601' }
-    )
+    const pgError = Object.assign(new Error('syntax error at or near "not"'), {
+      code: '42601',
+    })
     const db = new Kysely<any>({
       dialect: {
         ...dialect('postgres'),
@@ -454,3 +453,82 @@ describe('ensurePikkuSchema — a connection bound to a schema', () => {
     )
   })
 })
+
+describe('compiling into a schema', () => {
+  const compilePg = (schemaName?: string) =>
+    compilePikkuSchemas(
+      new Kysely<any>({ dialect: dialect('postgres') }),
+      pikkuSchemas,
+      {},
+      schemaName
+    )
+
+  test('qualifies every table it creates', () => {
+    const sql = compilePg('app')
+    const creates = sql.match(/create table[^(]*/gi) ?? []
+
+    assert.ok(creates.length > 0, 'nothing was compiled')
+    for (const create of creates) {
+      assert.match(
+        create,
+        /"app"\./,
+        `unqualified create table in a schema-bound compile: ${create.trim()}`
+      )
+    }
+  })
+
+  test('qualifies foreign key targets, so they do not resolve by search path', () => {
+    const sql = compilePg('app')
+
+    assert.match(sql, /references "app"\./)
+    assert.doesNotMatch(
+      sql,
+      /references "(?!app")[^"]+" \(/,
+      'a foreign key onto an unqualified table would find whatever search_path points at'
+    )
+  })
+
+  /**
+   * The regression this option exists for.
+   *
+   * Raw SQL is passed through verbatim, so the expression index on `credentials`
+   * is the one statement `withSchema` cannot reach. Left unqualified it creates
+   * the uniqueness rule on a different table than the `create table` above it.
+   */
+  test('qualifies the raw expression index too', () => {
+    assert.match(
+      compilePg('app'),
+      /CREATE UNIQUE INDEX credentials_name_user_id_unique ON "app"\."credentials"/
+    )
+  })
+
+  test('leaves the index name itself unqualified', () => {
+    // An index belongs to the schema of the table it is on; postgres rejects a
+    // CREATE INDEX that says otherwise.
+    assert.doesNotMatch(compilePg('app'), /INDEX "?app"?\."?credentials_name/i)
+  })
+
+  test('compiles unqualified when no schema is asked for', () => {
+    const sql = compilePg()
+
+    assert.doesNotMatch(sql, /"app"\./)
+    assert.match(sql, /create table "credentials"/i)
+    assert.match(
+      sql,
+      /CREATE UNIQUE INDEX credentials_name_user_id_unique ON "credentials"/
+    )
+  })
+
+  test('does not qualify the connection it was handed', () => {
+    // The caller's connection is a scratch database the declaration was just
+    // applied to. Binding it would create tables in a schema that database has
+    // never heard of — so the schema must reach the SQL and nothing else.
+    const db = new Kysely<any>({ dialect: dialect('postgres') })
+    compilePikkuSchemas(db, pikkuSchemas, {}, 'app')
+
+    assert.match(
+      db.schema.createTable('after').addColumn('id', 'text').compile().sql,
+      /create table "after"/
+    )
+  })
+})

package/src/schema/virtual-user.schema.ts

@@ -0,0 +1,65 @@
+import { sql } from 'kysely'
+import type { PikkuSchema } from './pikku-schema.types.js'
+
+/**
+ * The `virtualUserRun` table {@link KyselyVirtualUserRunStore} writes to.
+ *
+ * Deliberately not in `pikkuSchemas`: the runtime never needs this table, only
+ * a project that runs virtual users does, and creating it for everyone would
+ * put an empty table in every database. `KyselyVirtualUserRunStore.init()`
+ * applies it, so it arrives exactly when the thing that fills it does.
+ *
+ * `persona` is text rather than a foreign key: personas are declared in code
+ * with `definePersonas()`, not rows, and a run has to outlive the deletion of
+ * the persona it ran as.
+ *
+ * The JSON columns are text on every engine, like the audit table's, because
+ * the store serialises them itself — a project on SQLite and one on Postgres
+ * then hold the same bytes, and comparing this week's findings against last
+ * week's does not depend on which database they were found in.
+ */
+export const virtualUserSchema: PikkuSchema = {
+  name: 'virtualUser',
+  statements: [
+    (db) =>
+      db.schema
+        .createTable('virtualUserRun')
+        .addColumn('runId', 'varchar(36)', (col) => col.primaryKey())
+        .addColumn('persona', 'varchar(255)', (col) => col.notNull())
+        .addColumn('disposition', 'varchar(50)', (col) => col.notNull())
+        .addColumn('seed', 'bigint', (col) => col.notNull())
+        // running | completed | failed. Not derived from `finishedAt` being
+        // null: a crashed run has no finish time either, and "crashed" and
+        // "still going" are not the same answer.
+        .addColumn('status', 'varchar(50)', (col) =>
+          col.defaultTo('running').notNull()
+        )
+        .addColumn('goals', 'text', (col) => col.defaultTo('[]').notNull())
+        .addColumn('memory', 'text', (col) => col.defaultTo('{}').notNull())
+        .addColumn('findings', 'text', (col) => col.defaultTo('[]').notNull())
+        // Counts the engine kept: steps, calls, mutations, tokens, elapsed.
+        .addColumn('tally', 'text')
+        .addColumn('stoppedBy', 'varchar(255)')
+        // Why the run itself failed, as opposed to what it found.
+        .addColumn('error', 'text')
+        .addColumn('startedBy', 'varchar(255)')
+        .addColumn('createdAt', 'timestamp', (col) =>
+          col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
+        )
+        .addColumn('finishedAt', 'timestamp'),
+
+    // The two reads: the run list newest-first, and one persona's history when
+    // a finding needs comparing against what that persona found before.
+    (db) =>
+      db.schema
+        .createIndex('idx_virtual_user_run_created')
+        .on('virtualUserRun')
+        .column('createdAt'),
+
+    (db) =>
+      db.schema
+        .createIndex('idx_virtual_user_run_persona')
+        .on('virtualUserRun')
+        .columns(['persona', 'createdAt']),
+  ],
+}