@pikku/kysely 0.13.28 → 0.13.30

package/CHANGELOG.md

@@ -1,3 +1,28 @@
+## 0.13.30
+
+### Patch Changes
+
+- 658f047: A declared webhook source is a registered one. `reconcileTriggerSources` sets up every declared source with its provider (check, then setup where missing or drifted) and `teardownTriggerSources` removes named ones, recording what was registered in a `TriggerSourceStore` (in-memory, or `KyselyTriggerSourceStore` on `pikku_trigger_source`). The admin addon exposes list, reconcile, teardown and forget under the new `admin:triggers` scopes.
+- Updated dependencies [658f047]
+- Updated dependencies [de63ab2]
+- Updated dependencies [658f047]
+- Updated dependencies [658f047]
+- Updated dependencies [658f047]
+- Updated dependencies [658f047]
+  - @pikku/core@0.12.128
+
+## 0.13.29
+
+### Patch Changes
+
+- 5e93f30: Add `wireTriggerWebhookSource({ name, method?, route?, secret?, events, receive?, check?, setup?, teardown? })` in `#pikku/trigger`. Each source becomes a `POST /webhooks/<name>` route whose events are validated against their schemas and queued on `pikku-incoming-webhooks` through `IncomingWebhookService`; a generated worker runs the matching `wireTrigger({ name: '<source>:<event>' })` and the queue retries it on failure. `pikku webhooks status | setup | teardown --url --labelPrefix [--previous]` registers the routes with the provider and prints one JSON line per source.
+
+  `KyselyIncomingWebhookService` (with the `incoming-webhook` schema) records a receipt per event, drops a provider's redelivery of an event it already accepted, and keeps each dispatch's attempts and last error. `pikku dev` and `pikku serve` use it when a Kysely database is configured.
+
+- Updated dependencies [5e93f30]
+- Updated dependencies [5e93f30]
+  - @pikku/core@0.12.125
+
 ## 0.13.28
 
 ### Patch Changes

package/dist/src/index.d.ts

@@ -14,6 +14,8 @@ export { KyselyScopeService } from './kysely-scope-service.js';
 export { KyselyFeatureFlagStore } from './kysely-feature-flag-store.js';
 export type { KyselyFeatureFlagStoreOptions } from './kysely-feature-flag-store.js';
 export { KyselyWebhookService } from './kysely-webhook-service.js';
+export { KyselyTriggerSourceStore } from './kysely-trigger-source-store.js';
+export { KyselyIncomingWebhookService } from './kysely-incoming-webhook-service.js';
 export { createAuditedKysely, type CreateAuditedKyselyOptions, } from './create-audited-kysely.js';
 export { KyselyAuditService } from './kysely-audit-service.js';
 export { KyselyAnalyticsService } from './kysely-analytics-service.js';
@@ -23,7 +25,7 @@ export { SerializePlugin, BaseSerializePlugin, type Serializer, type Deserialize
 export { createCoercionPlugin, type ColumnKind, type CoercionMap, type CreateCoercionPluginOptions, } from './coercion-plugin.js';
 export { createClassificationPlugin, type CreateClassificationPluginOptions, } from './classification-plugin.js';
 export { ClassificationCrypto, DEFAULT_KEY_ID, isColumnEnvelope, parseColumnEnvelope, type ClassificationCryptoOptions, type ColumnEnvelope, type KEKResolver, type ResolvedKEK, } from './classification-crypto.js';
-export { agentSchema, analyticsSchema, auditSchema, channelSchema, credentialSchema, deploymentSchema, flagSchema, scopeSchema, secretSchema, sessionSchema, virtualUserSchema, webhookSchema, workflowSchema, pikkuSchemas, requiredPikkuSchemas, applyPikkuSchemas, compilePikkuSchemas, requirePikkuSchema, resolveRequirements, type PikkuSchema, type RequiredTypes, type UnmetRequirement, type SchemaStatementFactory, } from './schema/index.js';
+export { agentSchema, analyticsSchema, auditSchema, channelSchema, credentialSchema, deploymentSchema, flagSchema, incomingWebhookSchema, scopeSchema, secretSchema, sessionSchema, virtualUserSchema, webhookSchema, workflowSchema, pikkuSchemas, requiredPikkuSchemas, applyPikkuSchemas, compilePikkuSchemas, requirePikkuSchema, resolveRequirements, type PikkuSchema, type RequiredTypes, type UnmetRequirement, type SchemaStatementFactory, } from './schema/index.js';
 export type { KyselyPikkuDB } from './kysely-tables.js';
 export type { WorkflowRunService } from '@pikku/core/workflow';
 export type { AgentRunService, AgentRunRow } from '@pikku/core/agent';

package/dist/src/index.js

@@ -13,6 +13,8 @@ export { KyselySessionStore } from './kysely-session-store.js';
 export { KyselyScopeService } from './kysely-scope-service.js';
 export { KyselyFeatureFlagStore } from './kysely-feature-flag-store.js';
 export { KyselyWebhookService } from './kysely-webhook-service.js';
+export { KyselyTriggerSourceStore } from './kysely-trigger-source-store.js';
+export { KyselyIncomingWebhookService } from './kysely-incoming-webhook-service.js';
 export { createAuditedKysely, } from './create-audited-kysely.js';
 export { KyselyAuditService } from './kysely-audit-service.js';
 export { KyselyAnalyticsService } from './kysely-analytics-service.js';
@@ -22,4 +24,4 @@ export { SerializePlugin, BaseSerializePlugin, } from './serialize-plugin.js';
 export { createCoercionPlugin, } from './coercion-plugin.js';
 export { createClassificationPlugin, } from './classification-plugin.js';
 export { ClassificationCrypto, DEFAULT_KEY_ID, isColumnEnvelope, parseColumnEnvelope, } from './classification-crypto.js';
-export { agentSchema, analyticsSchema, auditSchema, channelSchema, credentialSchema, deploymentSchema, flagSchema, scopeSchema, secretSchema, sessionSchema, virtualUserSchema, webhookSchema, workflowSchema, pikkuSchemas, requiredPikkuSchemas, applyPikkuSchemas, compilePikkuSchemas, requirePikkuSchema, resolveRequirements, } from './schema/index.js';
+export { agentSchema, analyticsSchema, auditSchema, channelSchema, credentialSchema, deploymentSchema, flagSchema, incomingWebhookSchema, scopeSchema, secretSchema, sessionSchema, virtualUserSchema, webhookSchema, workflowSchema, pikkuSchemas, requiredPikkuSchemas, applyPikkuSchemas, compilePikkuSchemas, requirePikkuSchema, resolveRequirements, } from './schema/index.js';

package/dist/src/kysely-incoming-webhook-service.d.ts

@@ -0,0 +1,29 @@
+import { IncomingWebhookService, type IncomingWebhookAttempt, type IncomingWebhookReceiptRecord } from '@pikku/core/services';
+import type { QueueService } from '@pikku/core/queue';
+import type { TriggerEvent, WebhookRequest } from '@pikku/core/trigger';
+import type { Kysely } from 'kysely';
+import type { KyselyPikkuDB } from './kysely-tables.js';
+/**
+ * Durable {@link IncomingWebhookService}: records a `webhook_receipt` row per
+ * accepted event before queueing it, so a provider's redelivery of an event id
+ * it has already sent is dropped, and each trigger attempt rolls the receipt's
+ * status forward.
+ */
+export declare class KyselyIncomingWebhookService extends IncomingWebhookService {
+    private db;
+    private initialized;
+    constructor(queueService: QueueService, db: Kysely<KyselyPikkuDB>, retries?: number);
+    init(): Promise<void>;
+    accept({ source, events, }: {
+        source: string;
+        request: WebhookRequest;
+        events: TriggerEvent[];
+    }): Promise<number>;
+    /** The new receipt's id, or undefined when this provider event id was already received. */
+    private record;
+    recordAttempt(receiptId: string, { error }: IncomingWebhookAttempt): Promise<void>;
+    listReceipts(opts?: {
+        source?: string;
+        limit?: number;
+    }): Promise<IncomingWebhookReceiptRecord[]>;
+}

package/dist/src/kysely-incoming-webhook-service.js

@@ -0,0 +1,94 @@
+import { IncomingWebhookService, } from '@pikku/core/services';
+import { requirePikkuSchema } from './schema/index.js';
+import { incomingWebhookSchema } from './schema/incoming-webhook.schema.js';
+/**
+ * Durable {@link IncomingWebhookService}: records a `webhook_receipt` row per
+ * accepted event before queueing it, so a provider's redelivery of an event id
+ * it has already sent is dropped, and each trigger attempt rolls the receipt's
+ * status forward.
+ */
+export class KyselyIncomingWebhookService extends IncomingWebhookService {
+    db;
+    initialized = false;
+    constructor(queueService, db, retries) {
+        super(queueService, retries);
+        this.db = db;
+    }
+    async init() {
+        if (this.initialized)
+            return;
+        await requirePikkuSchema(this.db, incomingWebhookSchema);
+        this.initialized = true;
+    }
+    async accept({ source, events, }) {
+        let accepted = 0;
+        for (const event of events) {
+            const receiptId = await this.record(source, event);
+            if (!receiptId)
+                continue;
+            try {
+                await this.enqueue({ source, event, receiptId });
+            }
+            catch (error) {
+                // Without the receipt the provider's retry is accepted again rather
+                // than dropped as a duplicate of an event that was never queued.
+                await this.db
+                    .deleteFrom('webhookReceipt')
+                    .where('receiptId', '=', receiptId)
+                    .execute();
+                throw error;
+            }
+            accepted++;
+        }
+        return accepted;
+    }
+    /** The new receipt's id, or undefined when this provider event id was already received. */
+    async record(source, event) {
+        const receiptId = globalThis.crypto.randomUUID();
+        const providerEventId = event.id ?? null;
+        try {
+            await this.db
+                .insertInto('webhookReceipt')
+                .values({ receiptId, source, event: event.name, providerEventId })
+                .execute();
+            return receiptId;
+        }
+        catch (error) {
+            if (providerEventId === null)
+                throw error;
+            const existing = await this.db
+                .selectFrom('webhookReceipt')
+                .select('receiptId')
+                .where('source', '=', source)
+                .where('event', '=', event.name)
+                .where('providerEventId', '=', providerEventId)
+                .executeTakeFirst();
+            if (existing)
+                return undefined;
+            throw error;
+        }
+    }
+    async recordAttempt(receiptId, { error }) {
+        await this.db
+            .updateTable('webhookReceipt')
+            .set((eb) => ({
+            attempts: eb('attempts', '+', 1),
+            status: error ? 'failed' : 'delivered',
+            lastError: error ?? null,
+            ...(error ? {} : { deliveredAt: new Date() }),
+        }))
+            .where('receiptId', '=', receiptId)
+            .execute();
+    }
+    async listReceipts(opts) {
+        let query = this.db
+            .selectFrom('webhookReceipt')
+            .selectAll()
+            .orderBy('createdAt', 'desc')
+            .limit(opts?.limit ?? 100);
+        if (opts?.source) {
+            query = query.where('source', '=', opts.source);
+        }
+        return query.execute();
+    }
+}

package/dist/src/kysely-tables.d.ts

@@ -236,6 +236,17 @@ export interface PikkuFeatureFlagsTable {
     note: string | null;
     updatedAt: Generated<Date>;
 }
+/** What a declared trigger source registered with its provider. */
+export interface PikkuTriggerSourceTable {
+    name: string;
+    kind: string;
+    declared: Generated<boolean>;
+    status: string | null;
+    /** The last setup's provider identifiers as JSON; never a secret. */
+    state: string | null;
+    detail: string | null;
+    updatedAt: Generated<Date>;
+}
 /** One subject's short-circuit, in either direction, over a flag's rollout. */
 export interface PikkuFeatureFlagOverridesTable {
     flag: string;
@@ -282,6 +293,17 @@ export interface WebhookDeliveryAttemptTable {
     error: string | null;
     createdAt: Generated<Date>;
 }
+export interface WebhookReceiptTable {
+    receiptId: string;
+    source: string;
+    event: string;
+    providerEventId: string | null;
+    status: Generated<'pending' | 'delivered' | 'failed'>;
+    attempts: Generated<number>;
+    lastError: string | null;
+    createdAt: Generated<Date>;
+    deliveredAt: Date | null;
+}
 /**
  * One virtual-user run. The JSON columns (`goals`, `memory`, `findings`,
  * `tally`) are text the store serialises itself, so the row is byte-identical
@@ -410,6 +432,7 @@ export interface KyselyPikkuDB {
     pikkuUserScope: PikkuUserScopeTable;
     pikkuFeatureFlags: PikkuFeatureFlagsTable;
     pikkuFeatureFlagOverrides: PikkuFeatureFlagOverridesTable;
+    pikkuTriggerSource: PikkuTriggerSourceTable;
     channels: ChannelsTable;
     channelSubscriptions: ChannelSubscriptionsTable;
     workflowRuns: WorkflowRunsTable;
@@ -433,6 +456,7 @@ export interface KyselyPikkuDB {
     pikkuUserSessions: UserSessionsTable;
     webhookDelivery: WebhookDeliveryTable;
     webhookDeliveryAttempt: WebhookDeliveryAttemptTable;
+    webhookReceipt: WebhookReceiptTable;
     virtualUserRun: VirtualUserRunTable;
     virtualUserRunStep: VirtualUserRunStepTable;
     virtualUserSchedule: VirtualUserScheduleTable;

package/dist/src/kysely-trigger-source-store.d.ts

@@ -0,0 +1,14 @@
+import type { DeclaredTriggerSource, TriggerSourceResult, TriggerSourceRow, TriggerSourceStore } from '@pikku/core/services';
+import type { Kysely } from 'kysely';
+import type { KyselyPikkuDB } from './kysely-tables.js';
+export declare class KyselyTriggerSourceStore implements TriggerSourceStore {
+    private db;
+    private initialized;
+    constructor(db: Kysely<KyselyPikkuDB>);
+    init(): Promise<void>;
+    syncTriggerSources(sources: DeclaredTriggerSource[]): Promise<void>;
+    listTriggerSources(): Promise<TriggerSourceRow[]>;
+    getTriggerSource(name: string): Promise<TriggerSourceRow | null>;
+    recordTriggerSource(name: string, result: TriggerSourceResult): Promise<void>;
+    deleteTriggerSource(name: string): Promise<void>;
+}

package/dist/src/kysely-trigger-source-store.js

@@ -0,0 +1,92 @@
+import { requirePikkuSchema } from './schema/index.js';
+import { triggerSourceSchema } from './schema/trigger-source.schema.js';
+const parseState = (value) => {
+    if (!value)
+        return null;
+    if (typeof value !== 'string')
+        return value;
+    try {
+        return JSON.parse(value);
+    }
+    catch {
+        return null;
+    }
+};
+const toRow = (row) => ({
+    name: row.name,
+    kind: row.kind,
+    declared: !!row.declared,
+    status: row.status,
+    state: parseState(row.state),
+    detail: row.detail,
+    updatedAt: row.updatedAt ? new Date(row.updatedAt).toISOString() : null,
+});
+export class KyselyTriggerSourceStore {
+    db;
+    initialized = false;
+    constructor(db) {
+        this.db = db;
+    }
+    async init() {
+        if (this.initialized)
+            return;
+        await requirePikkuSchema(this.db, triggerSourceSchema);
+        this.initialized = true;
+    }
+    async syncTriggerSources(sources) {
+        await this.db.transaction().execute(async (trx) => {
+            for (const { name, kind } of sources) {
+                await trx
+                    .insertInto('pikkuTriggerSource')
+                    .values({ name, kind, declared: true })
+                    .onConflict((oc) => oc.column('name').doUpdateSet({ kind, declared: true }))
+                    .execute();
+            }
+            const markStale = trx
+                .updateTable('pikkuTriggerSource')
+                .set({ declared: false });
+            await (sources.length > 0
+                ? markStale.where('name', 'not in', sources.map((s) => s.name))
+                : markStale).execute();
+        });
+    }
+    async listTriggerSources() {
+        const rows = await this.db
+            .selectFrom('pikkuTriggerSource')
+            .selectAll()
+            .orderBy('name')
+            .execute();
+        return rows.map(toRow);
+    }
+    async getTriggerSource(name) {
+        const row = await this.db
+            .selectFrom('pikkuTriggerSource')
+            .selectAll()
+            .where('name', '=', name)
+            .executeTakeFirst();
+        return row ? toRow(row) : null;
+    }
+    async recordTriggerSource(name, result) {
+        const { numUpdatedRows } = await this.db
+            .updateTable('pikkuTriggerSource')
+            .set({
+            status: result.status,
+            ...(result.state !== undefined
+                ? { state: result.state ? JSON.stringify(result.state) : null }
+                : {}),
+            detail: result.detail ?? null,
+            updatedAt: new Date(),
+        })
+            .where('name', '=', name)
+            .executeTakeFirst();
+        if (!numUpdatedRows) {
+            throw new Error(`Unknown trigger source: ${name}`);
+        }
+    }
+    async deleteTriggerSource(name) {
+        await this.db
+            .deleteFrom('pikkuTriggerSource')
+            .where('name', '=', name)
+            .execute();
+    }
+}

package/dist/src/schema/incoming-webhook.schema.d.ts

@@ -0,0 +1,3 @@
+import type { PikkuSchema } from './pikku-schema.types.js';
+/** One row per event a webhook source accepted, keyed for provider de-duplication. */
+export declare const incomingWebhookSchema: PikkuSchema;

package/dist/src/schema/incoming-webhook.schema.js

@@ -0,0 +1,28 @@
+import { sql } from 'kysely';
+/** One row per event a webhook source accepted, keyed for provider de-duplication. */
+export const incomingWebhookSchema = {
+    name: 'incoming-webhook',
+    ownedBy: ['incomingWebhookService'],
+    statements: [
+        (db) => db.schema
+            .createTable('webhookReceipt')
+            .addColumn('receiptId', 'text', (col) => col.primaryKey())
+            .addColumn('source', 'text', (col) => col.notNull())
+            .addColumn('event', 'text', (col) => col.notNull())
+            .addColumn('providerEventId', 'text')
+            .addColumn('status', 'text', (col) => col.defaultTo('pending').notNull())
+            .addColumn('attempts', 'integer', (col) => col.defaultTo(0).notNull())
+            .addColumn('lastError', 'text')
+            .addColumn('createdAt', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
+            .addColumn('deliveredAt', 'timestamp'),
+        (db) => db.schema
+            .createIndex('idx_webhook_receipt_provider_event')
+            .unique()
+            .on('webhookReceipt')
+            .columns(['source', 'event', 'providerEventId']),
+        (db) => db.schema
+            .createIndex('idx_webhook_receipt_source_created')
+            .on('webhookReceipt')
+            .columns(['source', 'createdAt']),
+    ],
+};

package/dist/src/schema/index.d.ts

@@ -24,6 +24,8 @@ export { channelSchema } from './channel.schema.js';
 export { credentialSchema } from './credential.schema.js';
 export { deploymentSchema } from './deployment.schema.js';
 export { flagSchema } from './flag.schema.js';
+export { triggerSourceSchema } from './trigger-source.schema.js';
+export { incomingWebhookSchema } from './incoming-webhook.schema.js';
 export { scopeSchema } from './scope.schema.js';
 export { secretSchema } from './secret.schema.js';
 export { sessionSchema } from './session.schema.js';

package/dist/src/schema/index.js

@@ -8,6 +8,8 @@ import { credentialSchema } from './credential.schema.js';
 import { deploymentSchema } from './deployment.schema.js';
 import { scopeSchema } from './scope.schema.js';
 import { flagSchema } from './flag.schema.js';
+import { triggerSourceSchema } from './trigger-source.schema.js';
+import { incomingWebhookSchema } from './incoming-webhook.schema.js';
 import { secretSchema } from './secret.schema.js';
 import { sessionSchema } from './session.schema.js';
 import { webhookSchema } from './webhook.schema.js';
@@ -36,10 +38,12 @@ export const pikkuSchemas = [
     credentialSchema,
     deploymentSchema,
     webhookSchema,
+    incomingWebhookSchema,
     workflowSchema,
     agentSchema,
     scopeSchema,
     flagSchema,
+    triggerSourceSchema,
     auditSchema,
     analyticsSchema,
     virtualUserSchema,
@@ -52,6 +56,8 @@ export { channelSchema } from './channel.schema.js';
 export { credentialSchema } from './credential.schema.js';
 export { deploymentSchema } from './deployment.schema.js';
 export { flagSchema } from './flag.schema.js';
+export { triggerSourceSchema } from './trigger-source.schema.js';
+export { incomingWebhookSchema } from './incoming-webhook.schema.js';
 export { scopeSchema } from './scope.schema.js';
 export { secretSchema } from './secret.schema.js';
 export { sessionSchema } from './session.schema.js';

package/dist/src/schema/trigger-source.schema.d.ts

@@ -0,0 +1,6 @@
+import type { PikkuSchema } from './pikku-schema.types.js';
+/**
+ * What each declared trigger source registered with its provider. `state` holds provider identifiers only: a signing secret a
+ * provider issues goes to the credential service.
+ */
+export declare const triggerSourceSchema: PikkuSchema;

package/dist/src/schema/trigger-source.schema.js

@@ -0,0 +1,20 @@
+import { sql } from 'kysely';
+/**
+ * What each declared trigger source registered with its provider. `state` holds provider identifiers only: a signing secret a
+ * provider issues goes to the credential service.
+ */
+export const triggerSourceSchema = {
+    name: 'trigger-source',
+    ownedBy: ['triggerSourceStore'],
+    statements: [
+        (db) => db.schema
+            .createTable('pikkuTriggerSource')
+            .addColumn('name', 'text', (col) => col.primaryKey())
+            .addColumn('kind', 'text', (col) => col.notNull())
+            .addColumn('declared', 'boolean', (col) => col.defaultTo(true).notNull())
+            .addColumn('status', 'text')
+            .addColumn('state', 'text')
+            .addColumn('detail', 'text')
+            .addColumn('updatedAt', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull()),
+    ],
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pikku/kysely",
-  "version": "0.13.28",
+  "version": "0.13.30",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/pikkujs/pikku.git",
@@ -26,7 +26,7 @@
     "prepublishOnly": "bun run build"
   },
   "peerDependencies": {
-    "@pikku/core": "^0.12.120"
+    "@pikku/core": "^0.12.128"
   },
   "dependencies": {
     "kysely": "^0.29.5"