@antasphere/chassis-server 1.1.0 → 1.2.0

package/src/webhooks/catalog.ts

@@ -0,0 +1,205 @@
+import { z } from 'zod';
+import type { WebhookCatalog } from '@antasphere/chassis-contract';
+
+/**
+ * The event catalog (PRDCT-3482): what an instance can deliver to a webhook
+ * endpoint. The TOOL declares its events through the `webhooks` slot of its
+ * definition (its subjects, its event types, the zod schema of each event's
+ * `data`); the chassis adds its own events, checks the whole at boot, matches
+ * subscriptions against it and serves it on `GET /webhooks/events` with the
+ * JSON Schema of every payload, so an agent discovers the types and the
+ * shapes without reading a page of documentation.
+ *
+ * A SUBJECT is a filterable key the tool exposes (`thing`, `project`): an
+ * endpoint narrows its subscription to a list of ids per subject, and the
+ * envelope carries each subject's value under the FIELD the tool names for it
+ * (`thingId`, `projectIds`). An event declares which subjects it carries. A
+ * subject's value in the envelope is one id, or always a list of ids when
+ * the subject is declared `list` (the shape never varies per event).
+ */
+
+/** One filterable key of the tool, as the slot declares it. */
+export interface WebhookSubjectDeclaration {
+  /** The envelope field the subject's value is written under (`thingId`, `projectIds`). */
+  field: string;
+  description: string;
+  /**
+   * The subject's value is a LIST of ids (a thing in several projects): the
+   * envelope always writes an array under the field, a single id included.
+   * Absent: one id, always a string; a list is refused at emit.
+   */
+  list?: boolean;
+}
+
+/** One event type of the tool, as the slot declares it. */
+export interface WebhookEventDeclaration {
+  /** `thing.created`: a lowercase family, a dot, a lowercase verb. */
+  type: string;
+  description: string;
+  /** The subjects this event carries, by key; every one must be declared in `subjects`. */
+  subjects: readonly string[];
+  /** The schema of the event's `data`, validated at emit (a mismatch is a bug, logged and skipped). */
+  data: z.ZodType;
+}
+
+/** The `webhooks` slot of a tool definition. */
+export interface WebhookCatalogDeclaration {
+  subjects: Readonly<Record<string, WebhookSubjectDeclaration>>;
+  events: readonly WebhookEventDeclaration[];
+}
+
+/** The value one emit gives a subject: one id, or a list (a thing in several projects). */
+export type WebhookSubjectValue = string | readonly string[];
+
+/** An event type: `family.verb`, lowercase, digits and underscores allowed after the first letter. */
+export const WEBHOOK_EVENT_TYPE = /^[a-z][a-z0-9_]*\.[a-z][a-z0-9_]*$/;
+const SUBJECT_KEY = /^[a-z][a-z0-9_]*$/;
+
+/** The test ping every endpoint can be sent (`POST /webhooks/{id}/test`). */
+export const WEBHOOK_PING_EVENT = 'webhook.ping';
+
+/**
+ * The chassis's own events: what the generic half knows happened, bridged
+ * from the in-process bus (member, workspace, key events) or emitted inside
+ * the write's transaction where the chassis owns it (projects, the ping).
+ * None names a tool. A tool may not re-declare one of these types.
+ */
+export const CHASSIS_WEBHOOK_EVENTS: readonly WebhookEventDeclaration[] = [
+  {
+    type: WEBHOOK_PING_EVENT,
+    description: 'A test delivery, sent on request from the endpoint page or the API.',
+    subjects: [],
+    data: z.object({ endpointId: z.string(), sentBy: z.string().nullable() })
+  },
+  {
+    type: 'member.joined',
+    description: 'A person joined the workspace by accepting an invitation.',
+    subjects: [],
+    data: z.object({ userId: z.string(), role: z.string() })
+  },
+  {
+    type: 'member.removed',
+    description: 'A person was removed from the workspace by an owner or an admin.',
+    subjects: [],
+    data: z.object({ userId: z.string() })
+  },
+  {
+    type: 'workspace.created',
+    description: 'A workspace was created from inside the product.',
+    subjects: [],
+    data: z.object({ ownerUserId: z.string() })
+  },
+  {
+    type: 'apikey.created',
+    description:
+      'An API key was created on the API keys page or through the API (not the command line’s own sign-in key).',
+    subjects: [],
+    data: z.object({ apiKeyId: z.string() })
+  },
+  {
+    type: 'apikey.revoked',
+    description:
+      'An API key of the workspace was revoked from the API keys page or through the API (not a key revoking itself at logout).',
+    subjects: [],
+    data: z.object({ apiKeyId: z.string() })
+  },
+  {
+    type: 'project.created',
+    description: 'A project was created in the workspace.',
+    subjects: ['project'],
+    data: z.object({ projectId: z.string(), name: z.string(), createdBy: z.string() })
+  },
+  {
+    type: 'project.archived',
+    description: 'A project of the workspace was archived.',
+    subjects: ['project'],
+    data: z.object({ projectId: z.string(), name: z.string() })
+  }
+];
+
+/**
+ * The chassis's own subject: a project, which every tool has (the chassis
+ * owns projects). A tool that declares `project` itself may name another
+ * field for it (`projectIds`, a list); the chassis's project events follow.
+ */
+export const CHASSIS_WEBHOOK_SUBJECTS: Readonly<Record<string, WebhookSubjectDeclaration>> = {
+  project: { field: 'projectId', description: 'A project of the workspace, by id' }
+};
+
+/** The merged, checked catalog an instance runs on. */
+export interface WebhookCatalogIndex {
+  subjects: Readonly<Record<string, WebhookSubjectDeclaration>>;
+  events: ReadonlyMap<string, WebhookEventDeclaration>;
+}
+
+/**
+ * Merge the chassis's events with the tool's and check the whole, at boot:
+ * a type that is not `family.verb`, a type declared twice (or re-declaring a
+ * chassis type), a subject key that is not a plain identifier, an event
+ * carrying a subject nobody declared, or the chassis's `project` subject
+ * declared under another field, stops the boot naming the fault.
+ */
+export function buildWebhookCatalog(tool: WebhookCatalogDeclaration | undefined): WebhookCatalogIndex {
+  const subjects: Record<string, WebhookSubjectDeclaration> = { ...CHASSIS_WEBHOOK_SUBJECTS };
+  for (const [key, subject] of Object.entries(tool?.subjects ?? {})) {
+    if (!SUBJECT_KEY.test(key)) {
+      throw new Error(`tool definition: webhooks.subjects key "${key}" must match ${SUBJECT_KEY}`);
+    }
+    if (typeof subject.field !== 'string' || subject.field === '') {
+      throw new Error(`tool definition: webhooks.subjects.${key}.field is required (the envelope field)`);
+    }
+    // A tool may re-declare the chassis's `project` subject under its own
+    // field (a tool whose things belong to several projects writes a list
+    // under `projectIds`); the chassis's own project events then write
+    // their one id under that field too. A subject's value is one id or a
+    // list of ids, and the catalog says so.
+    subjects[key] = subject;
+  }
+  const events = new Map<string, WebhookEventDeclaration>();
+  for (const event of [...CHASSIS_WEBHOOK_EVENTS, ...(tool?.events ?? [])]) {
+    if (!WEBHOOK_EVENT_TYPE.test(event.type)) {
+      throw new Error(
+        `tool definition: webhooks event type "${event.type}" must be family.verb (${WEBHOOK_EVENT_TYPE})`
+      );
+    }
+    if (events.has(event.type)) {
+      throw new Error(`tool definition: webhooks event type "${event.type}" is declared twice`);
+    }
+    if (event.type === '*' || event.type.endsWith('.*')) {
+      throw new Error(`tool definition: webhooks event type "${event.type}" is a pattern, not a type`);
+    }
+    for (const key of event.subjects) {
+      if (!(key in subjects)) {
+        throw new Error(
+          `tool definition: webhooks event "${event.type}" carries the subject "${key}", which webhooks.subjects does not declare`
+        );
+      }
+    }
+    if (typeof event.description !== 'string' || event.description === '') {
+      throw new Error(`tool definition: webhooks event "${event.type}" needs a description`);
+    }
+    events.set(event.type, event);
+  }
+  return { subjects, events };
+}
+
+/** The catalog as `GET /webhooks/events` answers it: every event with the JSON Schema of its data. */
+export function catalogToWire(catalog: WebhookCatalogIndex): WebhookCatalog {
+  return {
+    subjects: Object.entries(catalog.subjects).map(([key, s]) => ({
+      key,
+      field: s.field,
+      description: s.description,
+      list: s.list === true
+    })),
+    events: [...catalog.events.values()].map((e) => ({
+      type: e.type,
+      description: e.description,
+      subjects: [...e.subjects],
+      schema: z.toJSONSchema(e.data, { target: 'draft-2020-12', unrepresentable: 'any' }) as Record<
+        string,
+        unknown
+      >
+    }))
+  };
+}

package/src/webhooks/index.ts

@@ -0,0 +1,58 @@
+export {
+  buildWebhookCatalog,
+  catalogToWire,
+  CHASSIS_WEBHOOK_EVENTS,
+  CHASSIS_WEBHOOK_SUBJECTS,
+  WEBHOOK_EVENT_TYPE,
+  WEBHOOK_PING_EVENT,
+  type WebhookCatalogDeclaration,
+  type WebhookCatalogIndex,
+  type WebhookEventDeclaration,
+  type WebhookSubjectDeclaration,
+  type WebhookSubjectValue
+} from './catalog.js';
+export { endpointWants, matchesEventType, matchesFilters } from './match.js';
+export {
+  checkWebhookUrl,
+  isPublicAddress,
+  safePost,
+  WebhookUrlRefusedError,
+  type SafeFetchOptions,
+  type SafeFetchPolicy,
+  type SafeFetchResult,
+  type SafeLookup,
+  type SafeTransport,
+  type WebhookUrlRefusal
+} from './safe-fetch.js';
+export {
+  generateWebhookSecret,
+  signWebhook,
+  SigningError,
+  verifyWebhookSignature,
+  WEBHOOK_SECRET_PREFIX,
+  webhookHeaders,
+  webhookSecretKey
+} from './signing.js';
+export {
+  DEFAULT_WEBHOOK_DIALS,
+  deliveryToWire,
+  endpointToWire,
+  WEBHOOK_DELIVERIES_HELD_QUEUE,
+  WEBHOOK_DELIVERIES_QUEUE,
+  WEBHOOK_EVENTS_QUEUE,
+  WEBHOOK_PURGE_QUEUE,
+  WEBHOOK_SECRET_BOX_INFO,
+  MAX_IN_FLIGHT_PER_ENDPOINT,
+  WebhookDeliveryDeferred,
+  WebhookDeliveryFailed,
+  WebhookRefusedError,
+  WebhookService,
+  type WebhookDials,
+  type WebhookEmit,
+  type WebhookEmitter,
+  type WebhookEventJob,
+  type WebhookServiceDeps
+} from './service.js';
+export { webhookJobs } from './jobs.js';
+export { buildWebhookDisabledEmail, type WebhookDisabledEmailParams } from './mail.js';
+export { connectionOf, drizzleExecutor, sendInTx } from './tx-send.js';

package/src/webhooks/jobs.ts

@@ -0,0 +1,92 @@
+import type { JobDeclaration } from '../jobs/pgboss.js';
+import type { Logger } from '../logger.js';
+import {
+  WEBHOOK_DELIVERIES_HELD_QUEUE,
+  WEBHOOK_DELIVERIES_QUEUE,
+  WEBHOOK_EVENTS_QUEUE,
+  WEBHOOK_PURGE_QUEUE,
+  WebhookDeliveryDeferred,
+  WebhookDeliveryFailed,
+  type WebhookDeliveryJob,
+  type WebhookEventJob,
+  type WebhookService
+} from './service.js';
+
+/**
+ * The four queues of the webhooks, declared to the job runtime like a tool's
+ * own (`createJobs` installs them in the same lock section, in this order):
+ * the held queue FIRST so the delivery queue can name it as its dead letter
+ * (pg-boss's job row holds a foreign key on it), then the fan-out, the
+ * delivery with its retry budget, and the nightly purge.
+ */
+export function webhookJobs(
+  service: WebhookService,
+  logger: Logger,
+  retentionDays: number
+): JobDeclaration[] {
+  const { dials } = service;
+  return [
+    {
+      queue: WEBHOOK_DELIVERIES_HELD_QUEUE,
+      options: { retryLimit: 10, retryDelaySeconds: 60, retryBackoff: true },
+      handler: async (data) => {
+        const job = data as WebhookDeliveryJob | null;
+        if (!job?.deliveryId) return;
+        await service.hold(job.deliveryId);
+      }
+    },
+    {
+      queue: WEBHOOK_EVENTS_QUEUE,
+      concurrency: 2,
+      pollingIntervalSeconds: 0.5,
+      handler: async (data) => {
+        const job = data as WebhookEventJob | null;
+        if (!job?.id) return;
+        const count = await service.fanOut(job);
+        logger.debug({ eventId: job.id, type: job.type, deliveries: count }, 'webhooks: event fanned out');
+      }
+    },
+    {
+      queue: WEBHOOK_DELIVERIES_QUEUE,
+      options: {
+        retryLimit: dials.retryLimit,
+        retryDelaySeconds: dials.retryDelaySeconds,
+        retryBackoff: true,
+        deadLetter: WEBHOOK_DELIVERIES_HELD_QUEUE
+      },
+      // Eight deliveries in flight per replica: a receiver that answers
+      // nothing holds one slot for its timeout, not the instance.
+      concurrency: 8,
+      pollingIntervalSeconds: 0.5,
+      handler: async (data) => {
+        const job = data as WebhookDeliveryJob | null;
+        if (!job?.deliveryId) return;
+        try {
+          await service.attempt(job.deliveryId, { retryable: true });
+        } catch (e) {
+          if (e instanceof WebhookDeliveryDeferred) {
+            // The endpoint already holds its share of the pollers: back in the
+            // queue in two seconds, its retry budget untouched.
+            await service.requeue(job.deliveryId, 2);
+            return;
+          }
+          if (e instanceof WebhookDeliveryFailed) {
+            logger.warn(
+              { deliveryId: e.deliveryId, reason: e.message },
+              'webhooks: a delivery attempt failed, will retry'
+            );
+          }
+          throw e;
+        }
+      }
+    },
+    {
+      queue: WEBHOOK_PURGE_QUEUE,
+      schedule: { cron: '0 3 * * *', enabled: retentionDays > 0 },
+      handler: async () => {
+        const deleted = await service.purge(retentionDays);
+        logger.info({ retentionDays, deleted }, 'webhook deliveries retention purge ran');
+      }
+    }
+  ];
+}

package/src/webhooks/mail.ts

@@ -0,0 +1,48 @@
+import { button, esc, fine, fmtDate, makeShell, para, spelledLink, type MailBrand } from '../email/shell.js';
+
+/**
+ * The one mail of the webhooks: an endpoint was switched off after failing
+ * for days, sent once to the person who registered it. Env-free like the
+ * account mails (`email/templates.ts`): every value arrives as a parameter,
+ * so a tool's mail wall renders it standalone.
+ */
+export interface WebhookDisabledEmailParams {
+  brand: MailBrand;
+  recipientName: string;
+  url: string;
+  description: string;
+  workspaceName: string;
+  consecutiveFailures: number;
+  firstFailureAt: Date;
+  /** The endpoint on its page in the dashboard: where it is re-enabled. */
+  manageUrl: string;
+}
+
+export function buildWebhookDisabledEmail(p: WebhookDisabledEmailParams): {
+  subject: string;
+  html: string;
+  text: string;
+} {
+  const name = p.brand.name;
+  const what = p.description ? `${p.description} (${p.url})` : p.url;
+  const subject = `A webhook of ${p.workspaceName} was switched off`;
+  const html = makeShell(p.brand)({
+    preheader: `${what} has not accepted a delivery since ${fmtDate(p.firstFailureAt)}.`,
+    eyebrow: 'Webhooks',
+    title: 'A webhook was switched off',
+    body:
+      para(
+        `${esc(name)} has stopped delivering to <strong>${esc(what)}</strong>, a webhook of
+         <strong>${esc(p.workspaceName)}</strong>: every delivery since ${fmtDate(p.firstFailureAt)}
+         failed, ${p.consecutiveFailures} in a row.`
+      ) +
+      para(
+        'The deliveries it missed are kept on its page. Once the receiver answers again, switch the webhook back on there and redeliver what matters.'
+      ) +
+      button(p.manageUrl, 'Open the webhook') +
+      fine(`You receive this because you registered the webhook, ${esc(p.recipientName)}.`) +
+      spelledLink(p.manageUrl)
+  });
+  const text = `${name} has stopped delivering to ${what}, a webhook of ${p.workspaceName}: every delivery since ${fmtDate(p.firstFailureAt)} failed, ${p.consecutiveFailures} in a row.\n\nOpen the webhook: ${p.manageUrl}\n\nThe deliveries it missed are kept on its page. Once the receiver answers again, switch the webhook back on there and redeliver what matters.`;
+  return { subject, html, text };
+}

package/src/webhooks/match.ts

@@ -0,0 +1,52 @@
+import type { WebhookSubjectValue } from './catalog.js';
+
+/**
+ * The subscription rule, pure and unit-tested (`webhook-match.test.ts`): an
+ * endpoint's `events` are matched against an event's type, its `filters`
+ * against the event's subjects. The chassis never knows what a subject IS,
+ * only that it is a key with one or several ids.
+ */
+
+/** `*` matches every type; `family.*` matches the family; a type matches itself. */
+export function matchesEventType(patterns: readonly string[], type: string): boolean {
+  const family = type.slice(0, type.indexOf('.'));
+  for (const pattern of patterns) {
+    if (pattern === '*' || pattern === type) return true;
+    if (pattern.endsWith('.*') && pattern.slice(0, -2) === family) return true;
+  }
+  return false;
+}
+
+/**
+ * AND across the filter's keys, OR within a key's ids. An event that does not
+ * carry a filtered subject does not match (a filter on `project` excludes an
+ * event with no project). An empty filter matches everything.
+ */
+export function matchesFilters(
+  filters: Readonly<Record<string, readonly string[]>>,
+  subjects: Readonly<Record<string, WebhookSubjectValue>>
+): boolean {
+  for (const [key, ids] of Object.entries(filters)) {
+    if (ids.length === 0) continue;
+    const value = subjects[key];
+    if (value === undefined) return false;
+    const values = typeof value === 'string' ? [value] : value;
+    if (!values.some((v) => ids.includes(v))) return false;
+  }
+  return true;
+}
+
+/** Whether an endpoint takes this event: enabled, the type subscribed, the filters satisfied. */
+export function endpointWants(
+  endpoint: {
+    enabled: boolean;
+    events: readonly string[];
+    filters: Readonly<Record<string, readonly string[]>>;
+  },
+  type: string,
+  subjects: Readonly<Record<string, WebhookSubjectValue>>
+): boolean {
+  return (
+    endpoint.enabled && matchesEventType(endpoint.events, type) && matchesFilters(endpoint.filters, subjects)
+  );
+}