@pikku/kysely 0.13.3 → 0.13.4

package/src/schema/channel.schema.ts

@@ -0,0 +1,37 @@
+import { sql } from 'kysely'
+import type { PikkuSchema } from './pikku-schema.types.js'
+
+/** Channels and their topic subscriptions, also read by the event-hub store. */
+export const channelSchema: PikkuSchema = {
+  name: 'channel',
+  statements: [
+    (db) =>
+      db.schema
+        .createTable('channels')
+        .addColumn('channelId', 'text', (col) => col.primaryKey())
+        .addColumn('channelName', 'text', (col) => col.notNull())
+        .addColumn('createdAt', 'timestamp', (col) =>
+          col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
+        )
+        .addColumn('openingData', 'text', (col) =>
+          col.notNull().defaultTo('{}')
+        )
+        .addColumn('pikkuUserId', 'text')
+        .addColumn('state', 'text')
+        .addColumn('lastWire', 'timestamp', (col) =>
+          col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
+        ),
+
+    (db) =>
+      db.schema
+        .createTable('channelSubscriptions')
+        .addColumn('channelId', 'text', (col) =>
+          col.notNull().references('channels.channelId').onDelete('cascade')
+        )
+        .addColumn('topic', 'text', (col) => col.notNull())
+        .addPrimaryKeyConstraint('channel_subscriptions_pk', [
+          'channelId',
+          'topic',
+        ]),
+  ],
+}

package/src/schema/credential.schema.ts

@@ -0,0 +1,46 @@
+import { sql } from 'kysely'
+import { rawStatement, type PikkuSchema } from './pikku-schema.types.js'
+
+/**
+ * Encrypted per-user credentials and their audit trail.
+ *
+ * The uniqueness rule is an expression index — `user_id` is nullable and two
+ * rows with a null user must still collide on name — which the schema builder
+ * cannot express, so it stays raw. Raw SQL is not rewritten by `withSchema`,
+ * which is why the table is named explicitly rather than left to the search
+ * path.
+ */
+export const credentialSchema: PikkuSchema = {
+  name: 'credential',
+  statements: [
+    (db) =>
+      db.schema
+        .createTable('credentials')
+        .addColumn('name', 'varchar(255)', (col) => col.notNull())
+        .addColumn('userId', 'varchar(255)')
+        .addColumn('ciphertext', 'text', (col) => col.notNull())
+        .addColumn('wrappedDek', 'text', (col) => col.notNull())
+        .addColumn('keyVersion', 'integer', (col) => col.notNull())
+        .addColumn('createdAt', 'timestamp', (col) =>
+          col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
+        )
+        .addColumn('updatedAt', 'timestamp', (col) =>
+          col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
+        ),
+
+    rawStatement(
+      sql`CREATE UNIQUE INDEX credentials_name_user_id_unique ON credentials (name, COALESCE(user_id, ''))`
+    ),
+
+    (db) =>
+      db.schema
+        .createTable('credentialsAudit')
+        .addColumn('id', 'varchar(36)', (col) => col.primaryKey())
+        .addColumn('credentialName', 'varchar(255)', (col) => col.notNull())
+        .addColumn('userId', 'varchar(255)')
+        .addColumn('action', 'varchar(20)', (col) => col.notNull())
+        .addColumn('performedAt', 'timestamp', (col) =>
+          col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
+        ),
+  ],
+}

package/src/schema/deployment.schema.ts

@@ -0,0 +1,47 @@
+import { sql } from 'kysely'
+import type { PikkuSchema } from './pikku-schema.types.js'
+
+/** Registered deployments and the functions each one serves. */
+export const deploymentSchema: PikkuSchema = {
+  name: 'deployment',
+  statements: [
+    (db) =>
+      db.schema
+        .createTable('pikkuDeployments')
+        .addColumn('deploymentId', 'text', (col) => col.primaryKey())
+        .addColumn('endpoint', 'text', (col) => col.notNull())
+        .addColumn('lastHeartbeat', 'timestamp', (col) =>
+          col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
+        )
+        .addColumn('createdAt', 'timestamp', (col) =>
+          col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
+        ),
+
+    (db) =>
+      db.schema
+        .createTable('pikkuDeploymentFunctions')
+        .addColumn('deploymentId', 'text', (col) =>
+          col
+            .notNull()
+            .references('pikkuDeployments.deploymentId')
+            .onDelete('cascade')
+        )
+        .addColumn('functionName', 'text', (col) => col.notNull())
+        .addPrimaryKeyConstraint('pikku_deployment_functions_pk', [
+          'deploymentId',
+          'functionName',
+        ]),
+
+    (db) =>
+      db.schema
+        .createIndex('idx_pikku_deployments_heartbeat')
+        .on('pikkuDeployments')
+        .column('lastHeartbeat'),
+
+    (db) =>
+      db.schema
+        .createIndex('idx_pikku_deployment_functions_name')
+        .on('pikkuDeploymentFunctions')
+        .column('functionName'),
+  ],
+}

package/src/schema/index.ts

@@ -0,0 +1,249 @@
+import { CamelCasePlugin, type Kysely } from 'kysely'
+import type {
+  PikkuSchema,
+  RequiredTypes,
+  SchemaRequirement,
+} from './pikku-schema.types.js'
+import { aiSchema } from './ai.schema.js'
+import { channelSchema } from './channel.schema.js'
+import { credentialSchema } from './credential.schema.js'
+import { deploymentSchema } from './deployment.schema.js'
+import { scopeSchema } from './scope.schema.js'
+import { secretSchema } from './secret.schema.js'
+import { sessionSchema } from './session.schema.js'
+import { webhookSchema } from './webhook.schema.js'
+import { workflowSchema } from './workflow.schema.js'
+
+export type {
+  PikkuSchema,
+  RequiredTypes,
+  SchemaRequirement,
+  SchemaStatement,
+  SchemaStatementFactory,
+} from './pikku-schema.types.js'
+export { rawStatement, requiredType } from './pikku-schema.types.js'
+
+/**
+ * Every table the pikku runtime needs, in dependency order.
+ *
+ * Order is load-bearing, not cosmetic: `scope` has foreign keys onto Better
+ * Auth's `user`, and the tables within each schema reference each other. Sort
+ * this list alphabetically and it stops applying.
+ */
+export const pikkuSchemas: PikkuSchema[] = [
+  channelSchema,
+  sessionSchema,
+  secretSchema,
+  credentialSchema,
+  deploymentSchema,
+  webhookSchema,
+  workflowSchema,
+  aiSchema,
+  scopeSchema,
+]
+
+/**
+ * Bind a declaration to a database.
+ *
+ * The plugin is applied here rather than trusted to the caller. Declarations
+ * name tables and columns the way the rest of the codebase queries them
+ * (`aiThreads.resourceId`), and it is `CamelCasePlugin` that turns those into
+ * the physical `ai_threads.resource_id`. Without it the same declaration
+ * silently creates quoted camelCase tables that nothing can read.
+ */
+const bind = (db: Kysely<any>) => db.withPlugin(new CamelCasePlugin())
+
+export interface UnmetRequirement {
+  schema: PikkuSchema
+  requirement: SchemaRequirement
+}
+
+/**
+ * Look up every prerequisite in `db`, reporting the gaps and the types found.
+ *
+ * Both halves come from one introspection because they are one question. The
+ * gaps say whether a schema can be applied at all; the types say how — a
+ * foreign key onto a `uuid` primary key has to be declared `uuid`, and guessing
+ * `text` produces DDL the database refuses.
+ *
+ * Introspection returns physical names, so `requires` is written in physical
+ * names too — this is the one place in the declaration that is not camelCase.
+ */
+export const resolveRequirements = async (
+  db: Kysely<any>,
+  schemas: PikkuSchema[] = pikkuSchemas
+): Promise<{ types: RequiredTypes; unmet: UnmetRequirement[] }> => {
+  const tables = new Map(
+    (await bind(db).introspection.getTables()).map((table) => [
+      table.name,
+      table,
+    ])
+  )
+
+  const types: RequiredTypes = {}
+  const unmet: UnmetRequirement[] = []
+  for (const schema of schemas) {
+    for (const requirement of schema.requires ?? []) {
+      const column = tables
+        .get(requirement.table)
+        ?.columns.find((c) => c.name === requirement.column)
+      if (!column) {
+        unmet.push({ schema, requirement })
+      } else {
+        types[`${requirement.table}.${requirement.column}`] = column.dataType
+      }
+    }
+  }
+  return { types, unmet }
+}
+
+/**
+ * The prerequisites `db` does not satisfy, as a list rather than a throw.
+ *
+ * Separate from `applyPikkuSchemas` because the two callers want opposite
+ * things from the same answer: a service booting cannot proceed without its
+ * tables, while a tool asking what a project's schema *would* be needs to
+ * describe the gap rather than die on it.
+ */
+export const unsatisfiedRequirements = async (
+  db: Kysely<any>,
+  schemas: PikkuSchema[] = pikkuSchemas
+): Promise<UnmetRequirement[]> => (await resolveRequirements(db, schemas)).unmet
+
+/**
+ * Create the tables, in order, on a database that has whatever they require.
+ *
+ * Prerequisites are checked before anything is created, so a project missing
+ * one is told which schema wanted what — rather than being handed a foreign key
+ * error, or a half-applied database to clean up.
+ *
+ * The statements then run in one transaction, so the same promise holds for a
+ * failure the check cannot foresee: a statement that dies mid-list takes the
+ * ones before it with it. On an engine without transactional DDL the rollback
+ * is the engine's to give, not ours.
+ */
+export const applyPikkuSchemas = async (
+  db: Kysely<any>,
+  schemas: PikkuSchema[] = pikkuSchemas
+): Promise<void> => {
+  const { types, unmet } = await resolveRequirements(db, schemas)
+  if (unmet.length > 0) {
+    throw new Error(
+      unmet
+        .map(
+          ({ schema, requirement }) =>
+            `The '${schema.name}' schema requires '${requirement.table}.${requirement.column}', which nothing has created. ` +
+            `${requirement.owner} owns it — apply its schema first.`
+        )
+        .join('\n')
+    )
+  }
+
+  await db.transaction().execute(async (trx) => {
+    const bound = bind(trx)
+    for (const schema of schemas) {
+      for (const statement of schema.statements) {
+        await statement(bound, types).execute()
+      }
+    }
+  })
+}
+
+/**
+ * The tables a schema creates, read back out of its own compiled SQL.
+ *
+ * Derived rather than declared alongside the statements, because a hand-kept
+ * list of names is a second source of truth: rename a table in a statement and
+ * the list keeps answering for the old one.
+ */
+const declaredTables = (schema: PikkuSchema, db: Kysely<any>): string[] => {
+  const bound = bind(db)
+  const tables: string[] = []
+  for (const statement of schema.statements) {
+    const match = /^\s*create table "([^"]+)"/i.exec(
+      statement(bound, {}).compile().sql
+    )
+    if (match) tables.push(match[1]!)
+  }
+  return tables
+}
+
+/** What `ensurePikkuSchema` found when it looked. */
+export type EnsureOutcome = 'present' | 'created'
+
+const tablesOf = async (db: Kysely<any>): Promise<Set<string>> =>
+  new Set((await bind(db).introspection.getTables()).map((table) => table.name))
+
+/**
+ * Make sure one schema's tables are there, creating them only if none are.
+ *
+ * What a service calls at boot. It looks before it creates, which is the whole
+ * difference from the `.ifNotExists()` DDL this replaces: that spelling turned
+ * every failure into a silent no-op, and hid a foreign key onto `user.id` that
+ * postgres had been rejecting since the day it was written.
+ *
+ * A partially-present schema is refused rather than completed. Half a schema
+ * means something else already applied part of it — a migration, an older
+ * version, a hand-run script — and creating the remainder at boot would leave
+ * two authorities over one set of tables, which is the condition all of this
+ * exists to end.
+ *
+ * Creating at boot is the fallback, not the intent. `pikku db generate` writes
+ * the declaration down as a migration; a project that has done so takes the
+ * `present` path and this never issues DDL at all.
+ */
+export const ensurePikkuSchema = async (
+  db: Kysely<any>,
+  schema: PikkuSchema
+): Promise<EnsureOutcome> => {
+  const existing = await tablesOf(db)
+  const declared = declaredTables(schema, db)
+  const missing = declared.filter((table) => !existing.has(table))
+
+  if (missing.length === 0) return 'present'
+
+  if (missing.length < declared.length) {
+    throw new Error(
+      `The '${schema.name}' schema is half applied: ${missing.join(', ')} missing, ` +
+        `${declared.filter((table) => existing.has(table)).join(', ')} already there. ` +
+        'Something else owns part of it. Reconcile it with a migration — ' +
+        '`pikku db generate` writes the declaration down — rather than creating the rest at boot.'
+    )
+  }
+
+  try {
+    await applyPikkuSchemas(db, [schema])
+    return 'created'
+  } catch (error) {
+    // Two instances booting cold against one database can both see it empty
+    // before either has finished creating. If the other one got there, say so;
+    // anything else is a real failure and still throws. This is a courtesy, not
+    // a guarantee — a boot race is only fully answered by having run the
+    // migration, which is what `pikku db generate` is for.
+    const after = await tablesOf(db)
+    if (declared.every((table) => after.has(table))) return 'present'
+    throw error
+  }
+}
+
+/**
+ * Render the declaration as SQL for the dialect `db` was built with.
+ *
+ * `types` comes from `resolveRequirements` against the database the migration
+ * is being written for. Omit it and foreign keys onto another source's columns
+ * compile as `text`, which is a shape, not a migration.
+ */
+export const compilePikkuSchemas = (
+  db: Kysely<any>,
+  schemas: PikkuSchema[] = pikkuSchemas,
+  types: RequiredTypes = {}
+): string => {
+  const bound = bind(db)
+  return schemas
+    .flatMap((schema) =>
+      schema.statements.map(
+        (statement) => `${statement(bound, types).compile().sql};`
+      )
+    )
+    .join('\n\n')
+}

package/src/schema/pikku-schema.types.ts

@@ -0,0 +1,94 @@
+import { sql, type CompiledQuery, type Kysely, type RawBuilder } from 'kysely'
+
+/**
+ * One DDL statement, bound to a database.
+ *
+ * Bound rather than free-standing because the same declaration serves two
+ * purposes that need the same object: applied to a throwaway database it yields
+ * the schema to introspect, and compiled against a dialect it yields the SQL a
+ * migration is written from.
+ */
+export interface SchemaStatement {
+  execute(): Promise<unknown>
+  compile(): CompiledQuery
+}
+
+export type SchemaStatementFactory = (
+  db: Kysely<any>,
+  types: RequiredTypes
+) => SchemaStatement
+
+/**
+ * The physical types of the columns named in `requires`, keyed `table.column`.
+ *
+ * Introspected from the database the schemas are being applied to, because a
+ * foreign key column has to be declared with the type of the column it points
+ * at and that column belongs to somebody else.
+ */
+export type RequiredTypes = Record<string, string>
+
+/**
+ * The declared type for a foreign key onto a column another source owns.
+ *
+ * Falls back to `text` when nothing has been introspected — the case when the
+ * declaration is compiled without a database, where the real type is unknowable
+ * and only the shape of the statement is being inspected.
+ */
+export const requiredType = (
+  types: RequiredTypes,
+  table: string,
+  column: string
+): RawBuilder<unknown> => sql.raw(types[`${table}.${column}`] ?? 'text')
+
+/**
+ * The tables one part of the pikku runtime needs, declared rather than created.
+ *
+ * `name` labels the generated migration and the drift report, so it is part of
+ * the contract — changing it orphans the migration that was written under the
+ * old one.
+ */
+export interface PikkuSchema {
+  name: string
+  statements: SchemaStatementFactory[]
+  /**
+   * Tables this schema references but does not create, which something else
+   * must have created first.
+   *
+   * Declared rather than discovered so the failure arrives as a sentence naming
+   * what is missing and who owns it, instead of a foreign key error from the
+   * database — or, worse, a stub table conjured up to make the error go away.
+   */
+  requires?: SchemaRequirement[]
+}
+
+/**
+ * A column another source owns, which one of these schemas depends on.
+ *
+ * Named down to the column, not just the table, because that is the granularity
+ * the dependency actually has: a foreign key needs the referenced column to
+ * exist *and* needs its type in order to declare a matching one.
+ *
+ * `owner` is carried in the declaration rather than composed into the error at
+ * the point it is thrown, so adding a second prerequisite cannot leave behind a
+ * message that still names the first one's owner.
+ */
+export interface SchemaRequirement {
+  table: string
+  column: string
+  owner: string
+}
+
+/**
+ * Wrap a raw SQL fragment as a statement.
+ *
+ * For the DDL kysely's schema builder cannot express — a unique index over an
+ * expression, say. Note that raw SQL is *not* rewritten by `withSchema`, so a
+ * fragment naming a table unqualified will resolve against the connection's
+ * search path rather than the schema the rest of the declaration lands in.
+ */
+export const rawStatement =
+  (fragment: RawBuilder<unknown>): SchemaStatementFactory =>
+  (db) => ({
+    execute: () => fragment.execute(db),
+    compile: () => fragment.compile(db),
+  })