@pikku/kysely 0.13.19 → 0.13.21

package/src/kysely-virtual-user-schedule-store.ts

@@ -0,0 +1,187 @@
+import type { Kysely, Selectable } from 'kysely'
+import type {
+  VirtualUserBudget,
+  VirtualUserDisposition,
+  VirtualUserScheduleInput,
+  VirtualUserScheduleRecord,
+  VirtualUserScheduleStore,
+} from '@pikku/core/virtual-user'
+import {
+  DEFAULT_MAX_INTERVAL_MS,
+  DEFAULT_MIN_INTERVAL_MS,
+} from '@pikku/core/virtual-user'
+import type {
+  KyselyPikkuDB,
+  VirtualUserScheduleTable,
+} from './kysely-tables.js'
+import { parseJson } from './kysely-json.js'
+import { requirePikkuSchema } from './schema/index.js'
+import { virtualUserScheduleSchema } from './schema/virtual-user-schedule.schema.js'
+
+/**
+ * Keeps each persona's cadence in a `virtualUserSchedule` table.
+ *
+ * Wiring this is what turns virtual users from something a person triggers into
+ * something that happens: without it, {@link KyselyVirtualUserRunStore} still
+ * records every run, and every run still had someone behind it. Two services
+ * rather than one so that an app can want the history without wanting the bill.
+ *
+ * Reads and writes the same way the run store does — JSON as text, timestamps
+ * as ISO strings, flags as 0/1 — so a project on SQLite and one on Postgres
+ * hold the same bytes and `SerializePlugin` is optional on both.
+ */
+export class KyselyVirtualUserScheduleStore implements VirtualUserScheduleStore {
+  private initialized = false
+
+  constructor(private db: Kysely<KyselyPikkuDB>) {}
+
+  public async init(): Promise<void> {
+    if (this.initialized) return
+    await requirePikkuSchema(this.db, virtualUserScheduleSchema)
+    this.initialized = true
+  }
+
+  async set(
+    schedule: VirtualUserScheduleInput
+  ): Promise<VirtualUserScheduleRecord> {
+    await this.init()
+    // Read-then-write rather than a partial upsert: a schedule is configuration
+    // written by a person, and the alternative is two dialects of "leave the
+    // columns I did not mention alone".
+    const existing = await this.get(schedule.persona)
+    const merged: VirtualUserScheduleRecord = {
+      persona: schedule.persona,
+      enabled: schedule.enabled ?? existing?.enabled ?? false,
+      disposition: schedule.disposition ?? existing?.disposition ?? 'realistic',
+      goals: [...(schedule.goals ?? existing?.goals ?? [])],
+      budget:
+        schedule.budget === undefined
+          ? (existing?.budget ?? null)
+          : schedule.budget,
+      minIntervalMs:
+        schedule.minIntervalMs ??
+        existing?.minIntervalMs ??
+        DEFAULT_MIN_INTERVAL_MS,
+      maxIntervalMs:
+        schedule.maxIntervalMs ??
+        existing?.maxIntervalMs ??
+        DEFAULT_MAX_INTERVAL_MS,
+      // A new schedule is due immediately. Enabling one is a deliberate act, and
+      // waiting a random half-day to find out whether it works is not a feature.
+      nextRunAt: schedule.nextRunAt ?? existing?.nextRunAt ?? new Date(),
+      lastRunId: existing?.lastRunId ?? null,
+      lastRunAt: existing?.lastRunAt ?? null,
+    }
+
+    const values = {
+      persona: merged.persona,
+      enabled: merged.enabled ? 1 : 0,
+      disposition: merged.disposition,
+      goals: JSON.stringify(merged.goals),
+      budget: merged.budget ? JSON.stringify(merged.budget) : null,
+      minIntervalMs: String(merged.minIntervalMs),
+      maxIntervalMs: String(merged.maxIntervalMs),
+      nextRunAt: merged.nextRunAt.toISOString(),
+    }
+
+    await this.db
+      .insertInto('virtualUserSchedule')
+      .values(values)
+      .onConflict((oc) =>
+        oc.column('persona').doUpdateSet({
+          enabled: values.enabled,
+          disposition: values.disposition,
+          goals: values.goals,
+          budget: values.budget,
+          minIntervalMs: values.minIntervalMs,
+          maxIntervalMs: values.maxIntervalMs,
+          nextRunAt: values.nextRunAt,
+        })
+      )
+      .execute()
+
+    return merged
+  }
+
+  async get(persona: string): Promise<VirtualUserScheduleRecord | null> {
+    await this.init()
+    const row = await this.db
+      .selectFrom('virtualUserSchedule')
+      .selectAll()
+      .where('persona', '=', persona)
+      .executeTakeFirst()
+    return row ? this.toRecord(row) : null
+  }
+
+  async list(): Promise<VirtualUserScheduleRecord[]> {
+    await this.init()
+    const rows = await this.db
+      .selectFrom('virtualUserSchedule')
+      .selectAll()
+      .orderBy('persona', 'asc')
+      .execute()
+    return rows.map((row) => this.toRecord(row))
+  }
+
+  async due(now: Date): Promise<VirtualUserScheduleRecord[]> {
+    await this.init()
+    const rows = await this.db
+      .selectFrom('virtualUserSchedule')
+      .selectAll()
+      .where('enabled', '=', 1)
+      .where('nextRunAt', '<=', now.toISOString())
+      .orderBy('nextRunAt', 'asc')
+      .execute()
+    return rows.map((row) => this.toRecord(row))
+  }
+
+  async claim(
+    persona: string,
+    claim: { from: Date; nextRunAt: Date; runId: string | null; at: Date }
+  ): Promise<boolean> {
+    await this.init()
+    const result = await this.db
+      .updateTable('virtualUserSchedule')
+      .set({
+        nextRunAt: claim.nextRunAt.toISOString(),
+        // A claim with no run id is the one written before dispatch, and must
+        // not blank out which run this persona last managed to start.
+        ...(claim.runId
+          ? { lastRunId: claim.runId, lastRunAt: claim.at.toISOString() }
+          : {}),
+      })
+      .where('persona', '=', persona)
+      // The compare-and-set: whoever writes first moves `nextRunAt`, and every
+      // other tick holding the value they read matches nothing.
+      .where('nextRunAt', '=', claim.from.toISOString())
+      .executeTakeFirst()
+    return Number(result.numUpdatedRows ?? 0) > 0
+  }
+
+  async remove(persona: string): Promise<void> {
+    await this.init()
+    await this.db
+      .deleteFrom('virtualUserSchedule')
+      .where('persona', '=', persona)
+      .execute()
+  }
+
+  private toRecord(
+    row: Selectable<VirtualUserScheduleTable>
+  ): VirtualUserScheduleRecord {
+    return {
+      persona: row.persona,
+      enabled: Boolean(row.enabled),
+      disposition: row.disposition as VirtualUserDisposition,
+      goals: parseJson(row.goals) ?? [],
+      budget: row.budget
+        ? ((parseJson(row.budget) ?? null) as VirtualUserBudget | null)
+        : null,
+      minIntervalMs: Number(row.minIntervalMs),
+      maxIntervalMs: Number(row.maxIntervalMs),
+      nextRunAt: new Date(row.nextRunAt),
+      lastRunId: row.lastRunId,
+      lastRunAt: row.lastRunAt ? new Date(row.lastRunAt) : null,
+    }
+  }
+}

package/src/kysely-webhook-service.test.ts

@@ -7,6 +7,7 @@ import { setSingletonServices } from '@pikku/core/state'
 import type { KyselyPikkuDB } from './kysely-tables.js'
 import { SerializePlugin } from './serialize-plugin.js'
 import { KyselyWebhookService } from './kysely-webhook-service.js'
+import { applyPikkuSchemas, webhookSchema } from './schema/index.js'
 
 const noopLogger = { error() {}, info() {}, warn() {}, debug() {} }
 
@@ -50,6 +51,7 @@ describe('KyselyWebhookService', () => {
   test('send() inserts a pending delivery and enqueues with deliveryId === jobId', async () => {
     const db = createDb()
     const { added, queue } = createQueue()
+    await applyPikkuSchemas(db, [webhookSchema])
     const service = new KyselyWebhookService(queue as any, db)
     await service.init()
 
@@ -76,6 +78,7 @@ describe('KyselyWebhookService', () => {
   test('recordAttempt() appends attempts and rolls status forward', async () => {
     const db = createDb()
     const { queue } = createQueue()
+    await applyPikkuSchemas(db, [webhookSchema])
     const service = new KyselyWebhookService(queue as any, db)
     await service.init()
 
@@ -104,6 +107,7 @@ describe('KyselyWebhookService', () => {
   test('listDeliveries() filters by organization', async () => {
     const db = createDb()
     const { queue } = createQueue()
+    await applyPikkuSchemas(db, [webhookSchema])
     const service = new KyselyWebhookService(queue as any, db)
     await service.init()
 

package/src/kysely-webhook-service.ts

@@ -10,7 +10,7 @@ import {
 } from '@pikku/core/services'
 import type { Kysely } from 'kysely'
 import type { KyselyPikkuDB } from './kysely-tables.js'
-import { ensurePikkuSchema } from './schema/index.js'
+import { requirePikkuSchema } from './schema/index.js'
 import { webhookSchema } from './schema/webhook.schema.js'
 
 /**
@@ -32,7 +32,7 @@ export class KyselyWebhookService extends QueueWebhookService {
 
   public async init(): Promise<void> {
     if (this.initialized) return
-    await ensurePikkuSchema(this.db, webhookSchema)
+    await requirePikkuSchema(this.db, webhookSchema)
     this.initialized = true
   }
 

package/src/kysely-workflow-mirror.test.ts

@@ -7,6 +7,7 @@ import Database from 'better-sqlite3'
 import type { KyselyPikkuDB } from './kysely-tables.js'
 import { KyselyWorkflowMirror } from './kysely-workflow-mirror.js'
 import { KyselyWorkflowRunService } from './kysely-workflow-run-service.js'
+import { applyPikkuSchemas, workflowSchema } from './schema/index.js'
 
 let db: Kysely<KyselyPikkuDB>
 let mirror: KyselyWorkflowMirror
@@ -20,6 +21,7 @@ beforeEach(async () => {
     dialect: new SqliteDialect({ database: new Database(':memory:') }),
     plugins: [new CamelCasePlugin(), new SerializePlugin()],
   })
+  await applyPikkuSchemas(db, [workflowSchema])
   mirror = new KyselyWorkflowMirror(db)
   await mirror.init()
   runService = new KyselyWorkflowRunService(db)

package/src/kysely-workflow-mirror.ts

@@ -9,7 +9,7 @@ import type {
 } from '@pikku/core/workflow'
 import type { Kysely } from 'kysely'
 import type { KyselyPikkuDB } from './kysely-tables.js'
-import { ensurePikkuSchema } from './schema/index.js'
+import { requirePikkuSchema } from './schema/index.js'
 import { workflowSchema } from './schema/workflow.schema.js'
 
 /**
@@ -33,14 +33,14 @@ export class KyselyWorkflowMirror implements WorkflowRunMirror {
    * Create the underlying tables if none of them exist yet.
    *
    * Safe to call from either the mirror or `KyselyWorkflowService.init()` —
-   * both apply the same `workflowSchema`, and `ensurePikkuSchema` is a no-op
+   * both apply the same `workflowSchema`, and `requirePikkuSchema` is a no-op
    * once the tables are there. A database holding only *part* of the schema
    * throws rather than filling in the rest: something else owns those tables,
    * and boot is not where that gets reconciled.
    */
   public async init(): Promise<void> {
     if (this.initialized) return
-    await ensurePikkuSchema(this.db, workflowSchema)
+    await requirePikkuSchema(this.db, workflowSchema)
     this.initialized = true
   }
 

package/src/kysely-workflow-service.test.ts

@@ -3,10 +3,11 @@ import assert from 'node:assert/strict'
 import { CamelCasePlugin, Kysely, SqliteDialect, sql } from 'kysely'
 import Database from 'better-sqlite3'
 
-import type { StepState } from '@pikku/core/ecosystem/workflow'
+import type { StepState } from '@pikku/core/workflow'
 import type { KyselyPikkuDB } from './kysely-tables.js'
 import { SerializePlugin } from './serialize-plugin.js'
 import { KyselyWorkflowService } from './kysely-workflow-service.js'
+import { applyPikkuSchemas, workflowSchema } from './schema/index.js'
 
 let db: Kysely<KyselyPikkuDB>
 let service: KyselyWorkflowService
@@ -31,6 +32,7 @@ const createDb = () => {
 
 beforeEach(async () => {
   db = createDb()
+  await applyPikkuSchemas(db, [workflowSchema])
   service = new KyselyWorkflowService(db, { wireQueues: false } as any)
   await service.init()
 })

package/src/kysely-workflow-service.ts

@@ -17,7 +17,7 @@ import { sql, type Kysely } from 'kysely'
 import type { KyselyPikkuDB } from './kysely-tables.js'
 import { KyselyWorkflowRunService } from './kysely-workflow-run-service.js'
 import { parseJson } from './kysely-json.js'
-import { ensurePikkuSchema } from './schema/index.js'
+import { requirePikkuSchema } from './schema/index.js'
 import { workflowSchema } from './schema/workflow.schema.js'
 
 export class KyselyWorkflowService extends PikkuWorkflowService {
@@ -34,7 +34,7 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
 
   public async init(): Promise<void> {
     if (this.initialized) return
-    await ensurePikkuSchema(this.db, workflowSchema)
+    await requirePikkuSchema(this.db, workflowSchema)
     this.initialized = true
   }
 

package/src/schema/agent.schema.ts

@@ -13,6 +13,7 @@ import type { PikkuSchema } from './pikku-schema.types.js'
  */
 export const agentSchema: PikkuSchema = {
   name: 'agent',
+  ownedBy: ['agentStorage', 'agentRunState', 'agentRunService'],
   statements: [
     (db) =>
       db.schema

package/src/schema/audit.schema.ts

@@ -3,10 +3,11 @@ import type { PikkuSchema } from './pikku-schema.types.js'
 /**
  * The `audit` table {@link KyselyAuditService} writes to and reads back.
  *
- * Deliberately not in `pikkuSchemas`: the runtime never needs this table, only
- * a project that has wired a durable audit sink does, and creating it for
- * everyone would put an empty table in every database. `KyselyAuditService.init()`
- * applies it, so it arrives exactly when the sink that fills it does.
+ * Gated by `ownedBy` rather than written for everyone: only a project that has
+ * wired a durable audit sink needs this table, and generating it for the rest
+ * would put an empty table in every database. `KyselyAuditService.init()`
+ * requires it and never creates it, so it arrives with the migration `pikku db
+ * generate` writes for the sink that fills it.
  *
  * Every column is text on every engine, matching the platform audit-queue
  * consumer's row shape — a locally-run project and a deployed stage write rows
@@ -16,6 +17,7 @@ import type { PikkuSchema } from './pikku-schema.types.js'
  */
 export const auditSchema: PikkuSchema = {
   name: 'audit',
+  ownedBy: ['audit', 'auditLog'],
   statements: [
     (db) =>
       db.schema

package/src/schema/channel.schema.ts

@@ -4,6 +4,7 @@ import type { PikkuSchema } from './pikku-schema.types.js'
 /** Channels and their topic subscriptions, also read by the event-hub store. */
 export const channelSchema: PikkuSchema = {
   name: 'channel',
+  ownedBy: ['channelStore', 'eventHub'],
   statements: [
     (db) =>
       db.schema

package/src/schema/credential.schema.ts

@@ -13,6 +13,7 @@ import { rawStatement, type PikkuSchema } from './pikku-schema.types.js'
  */
 export const credentialSchema: PikkuSchema = {
   name: 'credential',
+  ownedBy: ['credentialService'],
   statements: [
     (db) =>
       db.schema

package/src/schema/index.ts

@@ -7,6 +7,7 @@ import {
   type SchemaRequirement,
 } from './pikku-schema.types.js'
 import { agentSchema } from './agent.schema.js'
+import { auditSchema } from './audit.schema.js'
 import { channelSchema } from './channel.schema.js'
 import { credentialSchema } from './credential.schema.js'
 import { deploymentSchema } from './deployment.schema.js'
@@ -15,6 +16,8 @@ import { secretSchema } from './secret.schema.js'
 import { sessionSchema } from './session.schema.js'
 import { webhookSchema } from './webhook.schema.js'
 import { workflowSchema } from './workflow.schema.js'
+import { virtualUserSchema } from './virtual-user.schema.js'
+import { virtualUserScheduleSchema } from './virtual-user-schedule.schema.js'
 
 export type {
   PikkuSchema,
@@ -32,18 +35,18 @@ export {
 } from './pikku-schema.types.js'
 
 /**
- * Opt-in, so it is named here rather than folded into `pikkuSchemas` — the
- * runtime works without an audit table, and `KyselyAuditService.init()` applies
- * this for the projects that wire the sink.
- */
-export { auditSchema } from './audit.schema.js'
-
-/**
- * Every table the pikku runtime needs, in dependency order.
+ * Every table any part of the pikku runtime can need, in dependency order.
  *
  * Order is load-bearing, not cosmetic: `scope` has foreign keys onto Better
  * Auth's `user`, and the tables within each schema reference each other. Sort
  * this list alphabetically and it stops applying.
+ *
+ * `audit` and `virtualUser` are in the list now. They used to be left out and
+ * created by their own services at boot, which was the only way they ever came
+ * into existence — so with the runtime no longer creating anything, leaving
+ * them out would mean a wired audit sink starting up against a table that
+ * nothing writes a migration for. Both are gated by `ownedBy`, so a project
+ * that wires neither still carries neither.
  */
 export const pikkuSchemas: PikkuSchema[] = [
   channelSchema,
@@ -55,8 +58,42 @@ export const pikkuSchemas: PikkuSchema[] = [
   workflowSchema,
   agentSchema,
   scopeSchema,
+  auditSchema,
+  virtualUserSchema,
+  virtualUserScheduleSchema,
 ]
 
+export { agentSchema } from './agent.schema.js'
+export { auditSchema } from './audit.schema.js'
+export { channelSchema } from './channel.schema.js'
+export { credentialSchema } from './credential.schema.js'
+export { deploymentSchema } from './deployment.schema.js'
+export { scopeSchema } from './scope.schema.js'
+export { secretSchema } from './secret.schema.js'
+export { sessionSchema } from './session.schema.js'
+export { virtualUserSchema } from './virtual-user.schema.js'
+export { virtualUserScheduleSchema } from './virtual-user-schedule.schema.js'
+export { webhookSchema } from './webhook.schema.js'
+export { workflowSchema } from './workflow.schema.js'
+
+/**
+ * The schemas a project needs, given the services its code actually reaches.
+ *
+ * `services` is the inspector's `requiredServices` — the same set that drives
+ * service tree-shaking, so a schema is written for exactly the projects whose
+ * functions, middleware or wirings reach the service that owns its tables.
+ *
+ * The gate is one-sided: a schema declaring no `ownedBy` is always in, because
+ * nothing a project writes implies it and leaving it out would be a guess.
+ */
+export const requiredPikkuSchemas = (
+  services: ReadonlySet<string>
+): PikkuSchema[] =>
+  pikkuSchemas.filter(
+    (schema) =>
+      !schema.ownedBy || schema.ownedBy.some((name) => services.has(name))
+  )
+
 /**
  * Bind a declaration to a database.
  *
@@ -287,9 +324,6 @@ export const declaredTables = (
   return tables
 }
 
-/** What `ensurePikkuSchema` found when it looked. */
-export type EnsureOutcome = 'present' | 'created'
-
 /**
  * What `db` already has, indexed both ways.
  *
@@ -337,58 +371,42 @@ const isPresent = (existing: ExistingTables, table: DeclaredTable): boolean =>
     : existing.bare.has(table.name)
 
 /**
- * Make sure one schema's tables are there, creating them only if none are.
+ * Refuse to start unless one schema's tables are already there.
  *
- * What a service calls at boot. It looks before it creates, which is the whole
- * difference from the `.ifNotExists()` DDL this replaces: that spelling turned
- * every failure into a silent no-op, and hid a foreign key onto `user.id` that
- * postgres had been rejecting since the day it was written.
+ * What a service calls at boot, and it never issues DDL. The runtime is not an
+ * author of the schema: `pikku db generate` writes the declaration down as a
+ * migration and `pikku db migrate` applies it, and those two are the only way
+ * these tables come into existence. A service that finds them missing says so
+ * and stops.
  *
- * A partially-present schema is refused rather than completed. Half a schema
- * means something else already applied part of it — a migration, an older
- * version, a hand-run script — and creating the remainder at boot would leave
- * two authorities over one set of tables, which is the condition all of this
- * exists to end.
+ * This is the whole point of gating what `db generate` writes. While boot could
+ * still create, a schema that generation left out was created silently at
+ * startup instead — two authorities over one set of tables, which is the
+ * condition all of this exists to end. With creation gone, a schema generation
+ * missed is a sentence at startup naming the command that fixes it.
  *
- * Creating at boot is the fallback, not the intent. `pikku db generate` writes
- * the declaration down as a migration; a project that has done so takes the
- * `present` path and this never issues DDL at all.
+ * Half-present is not a distinct case any more. Every missing table is reported
+ * the same way, because the remedy is the same one either way.
  */
-export const ensurePikkuSchema = async (
+export const requirePikkuSchema = async (
   db: Kysely<any>,
   schema: PikkuSchema
-): Promise<EnsureOutcome> => {
+): Promise<void> => {
   const existing = await tablesOf(db)
   const declared = declaredTables(schema, db)
   const missing = declared.filter((table) => !isPresent(existing, table))
 
-  if (missing.length === 0) return 'present'
-
-  if (missing.length < declared.length) {
-    throw new Error(
-      `The '${schema.name}' schema is half applied: ${missing.map(displayName).join(', ')} missing, ` +
-        `${declared
-          .filter((table) => isPresent(existing, table))
-          .map(displayName)
-          .join(', ')} already there. ` +
-        'Something else owns part of it. Reconcile it with a migration — ' +
-        '`pikku db generate` writes the declaration down — rather than creating the rest at boot.'
-    )
-  }
+  if (missing.length === 0) return
 
-  try {
-    await applyPikkuSchemas(db, [schema])
-    return 'created'
-  } catch (error) {
-    // Two instances booting cold against one database can both see it empty
-    // before either has finished creating. If the other one got there, say so;
-    // anything else is a real failure and still throws. This is a courtesy, not
-    // a guarantee — a boot race is only fully answered by having run the
-    // migration, which is what `pikku db generate` is for.
-    const after = await tablesOf(db)
-    if (declared.every((table) => isPresent(after, table))) return 'present'
-    throw error
-  }
+  const present = declared.filter((table) => isPresent(existing, table))
+  throw new Error(
+    `The '${schema.name}' schema is not in this database: ${missing.map(displayName).join(', ')} missing` +
+      (present.length > 0
+        ? `, ${present.map(displayName).join(', ')} already there`
+        : '') +
+      '. The runtime does not create these — run `pikku db generate` to write the ' +
+      'migration, then `pikku db migrate` to apply it.'
+  )
 }
 
 /**

package/src/schema/pikku-schema.types.ts

@@ -101,6 +101,26 @@ export interface PikkuSchema {
    * database — or, worse, a stub table conjured up to make the error go away.
    */
   requires?: SchemaRequirement[]
+  /**
+   * The singleton services whose `init()` requires these tables, or absent for
+   * a schema every project needs.
+   *
+   * Named as services rather than as wirings because a service is what the
+   * tables actually belong to: one `requirePikkuSchema` call site per service,
+   * and `pikku db generate` writes the schema for a project whose functions
+   * reach any of them. A wiring is a poor proxy — a project can construct
+   * `KyselyWorkflowService` and wire no workflows at all.
+   *
+   * Absent means unconditional. `secrets` is on every project and the session
+   * and deployment stores are reached by generated code rather than by
+   * functions, so nothing a project writes implies them and their absence
+   * cannot be inferred.
+   *
+   * Under-naming is safe now that the runtime creates nothing: a schema left
+   * out of the migration is a sentence at startup, not a table conjured behind
+   * your back. Over-naming costs an unused table. Prefer over-naming.
+   */
+  ownedBy?: string[]
 }
 
 /**