@pikku/kysely 0.13.27 → 0.13.29

package/CHANGELOG.md

@@ -1,3 +1,51 @@
+## 0.13.29
+
+### Patch Changes
+
+- 5e93f30: Add `wireTriggerWebhookSource({ name, method?, route?, secret?, events, receive?, check?, setup?, teardown? })` in `#pikku/trigger`. Each source becomes a `POST /webhooks/<name>` route whose events are validated against their schemas and queued on `pikku-incoming-webhooks` through `IncomingWebhookService`; a generated worker runs the matching `wireTrigger({ name: '<source>:<event>' })` and the queue retries it on failure. `pikku webhooks status | setup | teardown --url --labelPrefix [--previous]` registers the routes with the provider and prints one JSON line per source.
+
+  `KyselyIncomingWebhookService` (with the `incoming-webhook` schema) records a receipt per event, drops a provider's redelivery of an event it already accepted, and keeps each dispatch's attempts and last error. `pikku dev` and `pikku serve` use it when a Kysely database is configured.
+
+- Updated dependencies [5e93f30]
+- Updated dependencies [5e93f30]
+  - @pikku/core@0.12.125
+
+## 0.13.28
+
+### Patch Changes
+
+- 4a9dcd2: `admin:listUsers` now pages, counts and can carry roles.
+
+  `ListUsersInput` gains `offset` and `includeRoles`; `ListUsersOutput` gains `total`, and each `User` gains `roles` and `fields`. `total` is how many users match `search`, which is what a pager counts against — `users.length` never was, because it is capped by `limit`.
+
+  ```typescript
+  const { users, total } = await rpc.invoke('admin:listUsers', {
+    search: 'example.com',
+    limit: 50,
+    offset: 50,
+    includeRoles: true,
+  })
+  ```
+
+  Paging only means something over a stable order, so the query now sorts newest first rather than however the database felt like returning rows.
+
+  Synthetic principals — the platform credential owner, Fabric service users, scenario actors — are excluded by the query instead of dropped from the page afterwards. Filtering after the fact broke both halves of paging: `limit` had already counted the rows it then discarded, so a page came back short, and `offset` skipped synthetic rows as though they were people, so the same person could appear on two pages or on none.
+
+  `includeRoles` is refused without `admin:scopes:read`. `admin:users:list` says who may see the directory; it does not say who may see what each of those users can do.
+
+  `ScopeService` gains `listRolesForUsers(userIds)`, implemented in `@pikku/kysely`. It answers for every id asked for — an empty array for a user holding no roles, so a caller cannot read a missing key as "holds nothing" — and chunks its `in` list to stay inside the bound-parameter cap. A page of users used to cost one query per row, which on a database reached over the network is a round trip per row.
+
+  ```typescript
+  listRolesForUsers(userIds: string[]): Promise<Record<string, string[]>>
+  ```
+
+  Anything implementing `ScopeService` outside this repository has to add it.
+
+- Updated dependencies [4a9dcd2]
+- Updated dependencies [5ab24ad]
+- Updated dependencies [42b7ac3]
+  - @pikku/core@0.12.120
+
 ## 0.13.27
 
 ### Patch Changes

package/dist/src/index.d.ts

@@ -14,6 +14,7 @@ export { KyselyScopeService } from './kysely-scope-service.js';
 export { KyselyFeatureFlagStore } from './kysely-feature-flag-store.js';
 export type { KyselyFeatureFlagStoreOptions } from './kysely-feature-flag-store.js';
 export { KyselyWebhookService } from './kysely-webhook-service.js';
+export { KyselyIncomingWebhookService } from './kysely-incoming-webhook-service.js';
 export { createAuditedKysely, type CreateAuditedKyselyOptions, } from './create-audited-kysely.js';
 export { KyselyAuditService } from './kysely-audit-service.js';
 export { KyselyAnalyticsService } from './kysely-analytics-service.js';
@@ -23,7 +24,7 @@ export { SerializePlugin, BaseSerializePlugin, type Serializer, type Deserialize
 export { createCoercionPlugin, type ColumnKind, type CoercionMap, type CreateCoercionPluginOptions, } from './coercion-plugin.js';
 export { createClassificationPlugin, type CreateClassificationPluginOptions, } from './classification-plugin.js';
 export { ClassificationCrypto, DEFAULT_KEY_ID, isColumnEnvelope, parseColumnEnvelope, type ClassificationCryptoOptions, type ColumnEnvelope, type KEKResolver, type ResolvedKEK, } from './classification-crypto.js';
-export { agentSchema, analyticsSchema, auditSchema, channelSchema, credentialSchema, deploymentSchema, flagSchema, scopeSchema, secretSchema, sessionSchema, virtualUserSchema, webhookSchema, workflowSchema, pikkuSchemas, requiredPikkuSchemas, applyPikkuSchemas, compilePikkuSchemas, requirePikkuSchema, resolveRequirements, type PikkuSchema, type RequiredTypes, type UnmetRequirement, type SchemaStatementFactory, } from './schema/index.js';
+export { agentSchema, analyticsSchema, auditSchema, channelSchema, credentialSchema, deploymentSchema, flagSchema, incomingWebhookSchema, scopeSchema, secretSchema, sessionSchema, virtualUserSchema, webhookSchema, workflowSchema, pikkuSchemas, requiredPikkuSchemas, applyPikkuSchemas, compilePikkuSchemas, requirePikkuSchema, resolveRequirements, type PikkuSchema, type RequiredTypes, type UnmetRequirement, type SchemaStatementFactory, } from './schema/index.js';
 export type { KyselyPikkuDB } from './kysely-tables.js';
 export type { WorkflowRunService } from '@pikku/core/workflow';
 export type { AgentRunService, AgentRunRow } from '@pikku/core/agent';

package/dist/src/index.js

@@ -13,6 +13,7 @@ export { KyselySessionStore } from './kysely-session-store.js';
 export { KyselyScopeService } from './kysely-scope-service.js';
 export { KyselyFeatureFlagStore } from './kysely-feature-flag-store.js';
 export { KyselyWebhookService } from './kysely-webhook-service.js';
+export { KyselyIncomingWebhookService } from './kysely-incoming-webhook-service.js';
 export { createAuditedKysely, } from './create-audited-kysely.js';
 export { KyselyAuditService } from './kysely-audit-service.js';
 export { KyselyAnalyticsService } from './kysely-analytics-service.js';
@@ -22,4 +23,4 @@ export { SerializePlugin, BaseSerializePlugin, } from './serialize-plugin.js';
 export { createCoercionPlugin, } from './coercion-plugin.js';
 export { createClassificationPlugin, } from './classification-plugin.js';
 export { ClassificationCrypto, DEFAULT_KEY_ID, isColumnEnvelope, parseColumnEnvelope, } from './classification-crypto.js';
-export { agentSchema, analyticsSchema, auditSchema, channelSchema, credentialSchema, deploymentSchema, flagSchema, scopeSchema, secretSchema, sessionSchema, virtualUserSchema, webhookSchema, workflowSchema, pikkuSchemas, requiredPikkuSchemas, applyPikkuSchemas, compilePikkuSchemas, requirePikkuSchema, resolveRequirements, } from './schema/index.js';
+export { agentSchema, analyticsSchema, auditSchema, channelSchema, credentialSchema, deploymentSchema, flagSchema, incomingWebhookSchema, scopeSchema, secretSchema, sessionSchema, virtualUserSchema, webhookSchema, workflowSchema, pikkuSchemas, requiredPikkuSchemas, applyPikkuSchemas, compilePikkuSchemas, requirePikkuSchema, resolveRequirements, } from './schema/index.js';

package/dist/src/kysely-incoming-webhook-service.d.ts

@@ -0,0 +1,29 @@
+import { IncomingWebhookService, type IncomingWebhookAttempt, type IncomingWebhookReceiptRecord } from '@pikku/core/services';
+import type { QueueService } from '@pikku/core/queue';
+import type { TriggerEvent, WebhookRequest } from '@pikku/core/trigger';
+import type { Kysely } from 'kysely';
+import type { KyselyPikkuDB } from './kysely-tables.js';
+/**
+ * Durable {@link IncomingWebhookService}: records a `webhook_receipt` row per
+ * accepted event before queueing it, so a provider's redelivery of an event id
+ * it has already sent is dropped, and each trigger attempt rolls the receipt's
+ * status forward.
+ */
+export declare class KyselyIncomingWebhookService extends IncomingWebhookService {
+    private db;
+    private initialized;
+    constructor(queueService: QueueService, db: Kysely<KyselyPikkuDB>, retries?: number);
+    init(): Promise<void>;
+    accept({ source, events, }: {
+        source: string;
+        request: WebhookRequest;
+        events: TriggerEvent[];
+    }): Promise<number>;
+    /** The new receipt's id, or undefined when this provider event id was already received. */
+    private record;
+    recordAttempt(receiptId: string, { error }: IncomingWebhookAttempt): Promise<void>;
+    listReceipts(opts?: {
+        source?: string;
+        limit?: number;
+    }): Promise<IncomingWebhookReceiptRecord[]>;
+}

package/dist/src/kysely-incoming-webhook-service.js

@@ -0,0 +1,94 @@
+import { IncomingWebhookService, } from '@pikku/core/services';
+import { requirePikkuSchema } from './schema/index.js';
+import { incomingWebhookSchema } from './schema/incoming-webhook.schema.js';
+/**
+ * Durable {@link IncomingWebhookService}: records a `webhook_receipt` row per
+ * accepted event before queueing it, so a provider's redelivery of an event id
+ * it has already sent is dropped, and each trigger attempt rolls the receipt's
+ * status forward.
+ */
+export class KyselyIncomingWebhookService extends IncomingWebhookService {
+    db;
+    initialized = false;
+    constructor(queueService, db, retries) {
+        super(queueService, retries);
+        this.db = db;
+    }
+    async init() {
+        if (this.initialized)
+            return;
+        await requirePikkuSchema(this.db, incomingWebhookSchema);
+        this.initialized = true;
+    }
+    async accept({ source, events, }) {
+        let accepted = 0;
+        for (const event of events) {
+            const receiptId = await this.record(source, event);
+            if (!receiptId)
+                continue;
+            try {
+                await this.enqueue({ source, event, receiptId });
+            }
+            catch (error) {
+                // Without the receipt the provider's retry is accepted again rather
+                // than dropped as a duplicate of an event that was never queued.
+                await this.db
+                    .deleteFrom('webhookReceipt')
+                    .where('receiptId', '=', receiptId)
+                    .execute();
+                throw error;
+            }
+            accepted++;
+        }
+        return accepted;
+    }
+    /** The new receipt's id, or undefined when this provider event id was already received. */
+    async record(source, event) {
+        const receiptId = globalThis.crypto.randomUUID();
+        const providerEventId = event.id ?? null;
+        try {
+            await this.db
+                .insertInto('webhookReceipt')
+                .values({ receiptId, source, event: event.name, providerEventId })
+                .execute();
+            return receiptId;
+        }
+        catch (error) {
+            if (providerEventId === null)
+                throw error;
+            const existing = await this.db
+                .selectFrom('webhookReceipt')
+                .select('receiptId')
+                .where('source', '=', source)
+                .where('event', '=', event.name)
+                .where('providerEventId', '=', providerEventId)
+                .executeTakeFirst();
+            if (existing)
+                return undefined;
+            throw error;
+        }
+    }
+    async recordAttempt(receiptId, { error }) {
+        await this.db
+            .updateTable('webhookReceipt')
+            .set((eb) => ({
+            attempts: eb('attempts', '+', 1),
+            status: error ? 'failed' : 'delivered',
+            lastError: error ?? null,
+            ...(error ? {} : { deliveredAt: new Date() }),
+        }))
+            .where('receiptId', '=', receiptId)
+            .execute();
+    }
+    async listReceipts(opts) {
+        let query = this.db
+            .selectFrom('webhookReceipt')
+            .selectAll()
+            .orderBy('createdAt', 'desc')
+            .limit(opts?.limit ?? 100);
+        if (opts?.source) {
+            query = query.where('source', '=', opts.source);
+        }
+        return query.execute();
+    }
+}

package/dist/src/kysely-scope-service.d.ts

@@ -61,6 +61,7 @@ export declare class KyselyScopeService implements ScopeService {
     addUserToRole(userId: string, role: string, grantedBy?: string): Promise<void>;
     removeUserFromRole(userId: string, role: string): Promise<void>;
     listUserRoles(userId: string): Promise<string[]>;
+    listRolesForUsers(userIds: string[]): Promise<Record<string, string[]>>;
     addScopeToUser(userId: string, scope: string, grantedBy?: string): Promise<void>;
     removeScopeFromUser(userId: string, scope: string): Promise<void>;
     listUserScopes(userId: string): Promise<string[]>;

package/dist/src/kysely-scope-service.js

@@ -236,6 +236,35 @@ export class KyselyScopeService {
             .execute();
         return rows.map((row) => row.role);
     }
+    async listRolesForUsers(userIds) {
+        // Seeded with every id asked for, so a user holding no roles is an empty
+        // array rather than a missing key.
+        const byUser = {};
+        for (const userId of userIds) {
+            byUser[userId] = [];
+        }
+        const distinct = Object.keys(byUser);
+        if (distinct.length === 0) {
+            return byUser;
+        }
+        // Chunked because an `in` list is one bound parameter per id, and every
+        // engine caps those — SQLite at 999 on older builds, Postgres at 65535 for
+        // the whole statement. A caller asking for a page of users stays well
+        // inside one chunk; one asking for thousands still gets an answer instead
+        // of a driver error.
+        for (let i = 0; i < distinct.length; i += 500) {
+            const chunk = distinct.slice(i, i + 500);
+            const rows = await this.db
+                .selectFrom('pikkuUserRole')
+                .select(['userId', 'role'])
+                .where('userId', 'in', chunk)
+                .execute();
+            for (const row of rows) {
+                byUser[row.userId].push(row.role);
+            }
+        }
+        return byUser;
+    }
     async addScopeToUser(userId, scope, grantedBy) {
         await this.db
             .insertInto('pikkuUserScope')

package/dist/src/kysely-tables.d.ts

@@ -282,6 +282,17 @@ export interface WebhookDeliveryAttemptTable {
     error: string | null;
     createdAt: Generated<Date>;
 }
+export interface WebhookReceiptTable {
+    receiptId: string;
+    source: string;
+    event: string;
+    providerEventId: string | null;
+    status: Generated<'pending' | 'delivered' | 'failed'>;
+    attempts: Generated<number>;
+    lastError: string | null;
+    createdAt: Generated<Date>;
+    deliveredAt: Date | null;
+}
 /**
  * One virtual-user run. The JSON columns (`goals`, `memory`, `findings`,
  * `tally`) are text the store serialises itself, so the row is byte-identical
@@ -433,6 +444,7 @@ export interface KyselyPikkuDB {
     pikkuUserSessions: UserSessionsTable;
     webhookDelivery: WebhookDeliveryTable;
     webhookDeliveryAttempt: WebhookDeliveryAttemptTable;
+    webhookReceipt: WebhookReceiptTable;
     virtualUserRun: VirtualUserRunTable;
     virtualUserRunStep: VirtualUserRunStepTable;
     virtualUserSchedule: VirtualUserScheduleTable;

package/dist/src/schema/incoming-webhook.schema.d.ts

@@ -0,0 +1,3 @@
+import type { PikkuSchema } from './pikku-schema.types.js';
+/** One row per event a webhook source accepted, keyed for provider de-duplication. */
+export declare const incomingWebhookSchema: PikkuSchema;

package/dist/src/schema/incoming-webhook.schema.js

@@ -0,0 +1,28 @@
+import { sql } from 'kysely';
+/** One row per event a webhook source accepted, keyed for provider de-duplication. */
+export const incomingWebhookSchema = {
+    name: 'incoming-webhook',
+    ownedBy: ['incomingWebhookService'],
+    statements: [
+        (db) => db.schema
+            .createTable('webhookReceipt')
+            .addColumn('receiptId', 'text', (col) => col.primaryKey())
+            .addColumn('source', 'text', (col) => col.notNull())
+            .addColumn('event', 'text', (col) => col.notNull())
+            .addColumn('providerEventId', 'text')
+            .addColumn('status', 'text', (col) => col.defaultTo('pending').notNull())
+            .addColumn('attempts', 'integer', (col) => col.defaultTo(0).notNull())
+            .addColumn('lastError', 'text')
+            .addColumn('createdAt', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
+            .addColumn('deliveredAt', 'timestamp'),
+        (db) => db.schema
+            .createIndex('idx_webhook_receipt_provider_event')
+            .unique()
+            .on('webhookReceipt')
+            .columns(['source', 'event', 'providerEventId']),
+        (db) => db.schema
+            .createIndex('idx_webhook_receipt_source_created')
+            .on('webhookReceipt')
+            .columns(['source', 'createdAt']),
+    ],
+};

package/dist/src/schema/index.d.ts

@@ -24,6 +24,7 @@ export { channelSchema } from './channel.schema.js';
 export { credentialSchema } from './credential.schema.js';
 export { deploymentSchema } from './deployment.schema.js';
 export { flagSchema } from './flag.schema.js';
+export { incomingWebhookSchema } from './incoming-webhook.schema.js';
 export { scopeSchema } from './scope.schema.js';
 export { secretSchema } from './secret.schema.js';
 export { sessionSchema } from './session.schema.js';

package/dist/src/schema/index.js

@@ -8,6 +8,7 @@ import { credentialSchema } from './credential.schema.js';
 import { deploymentSchema } from './deployment.schema.js';
 import { scopeSchema } from './scope.schema.js';
 import { flagSchema } from './flag.schema.js';
+import { incomingWebhookSchema } from './incoming-webhook.schema.js';
 import { secretSchema } from './secret.schema.js';
 import { sessionSchema } from './session.schema.js';
 import { webhookSchema } from './webhook.schema.js';
@@ -36,6 +37,7 @@ export const pikkuSchemas = [
     credentialSchema,
     deploymentSchema,
     webhookSchema,
+    incomingWebhookSchema,
     workflowSchema,
     agentSchema,
     scopeSchema,
@@ -52,6 +54,7 @@ export { channelSchema } from './channel.schema.js';
 export { credentialSchema } from './credential.schema.js';
 export { deploymentSchema } from './deployment.schema.js';
 export { flagSchema } from './flag.schema.js';
+export { incomingWebhookSchema } from './incoming-webhook.schema.js';
 export { scopeSchema } from './scope.schema.js';
 export { secretSchema } from './secret.schema.js';
 export { sessionSchema } from './session.schema.js';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pikku/kysely",
-  "version": "0.13.27",
+  "version": "0.13.29",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/pikkujs/pikku.git",
@@ -26,7 +26,7 @@
     "prepublishOnly": "bun run build"
   },
   "peerDependencies": {
-    "@pikku/core": "^0.12.111"
+    "@pikku/core": "^0.12.125"
   },
   "dependencies": {
     "kysely": "^0.29.5"