@kelpie/server 0.1.0

package/src/modules/search/tools.ts

@@ -0,0 +1,60 @@
+import { SEARCH_COLLECTIONS } from '@kelpie/schemas'
+import { z } from 'zod'
+
+import { MAX_PAGE_SIZE } from '../../lib/pagination.ts'
+import type { McpToolRegistry } from '../../runtime/module.ts'
+import { MAX_SEARCH_TERM_LENGTH } from './query.ts'
+import { searchResponse } from './routes.ts'
+import type { SearchService } from './service.ts'
+
+/**
+ * `search_query`, and nothing else. Nothing here writes.
+ *
+ * Named for what it does rather than for the REST verb. `api.md` builds a tool
+ * name from the path segment and the verb, which would give `search_get`; that
+ * reads as fetching a saved search, and a tool an agent picks from a list of
+ * dozens is chosen by its name before its description.
+ *
+ * This is the tool an agent reaches for before any of the `*_list` ones: those
+ * need to know which resource holds the answer, and a question rarely does.
+ */
+const queryArgs = z.strictObject({
+  q: z
+    .string()
+    .min(1)
+    .max(MAX_SEARCH_TERM_LENGTH)
+    .describe('What to search for. Every word is matched as a prefix, so partial words work.'),
+  type: z
+    .array(z.enum(SEARCH_COLLECTIONS))
+    .nonempty()
+    .optional()
+    .describe('Which collections to look through. Omit to search all nine.'),
+  limit: z
+    .number()
+    .int()
+    .min(1)
+    .max(MAX_PAGE_SIZE)
+    .optional()
+    .describe('How many results each group carries. The total beside each group is exact.'),
+})
+
+export function registerSearchTools(mcp: McpToolRegistry, service: SearchService): void {
+  mcp.tool({
+    name: 'search_query',
+    description:
+      'Search every CRM record and handbook page at once: People (including the job titles on ' +
+      'their Positions), Companies, Deals, Opportunities, Raises, Partnerships, Roles, Decisions, ' +
+      'and handbook bodies. Deals, Opportunities and Raises also match the titles of their Plan ' +
+      'items. Results come back grouped by collection, each group ranked and carrying an exact ' +
+      'total, with a snippet centred on the match. Mirrors GET /v1/search.',
+    inputSchema: queryArgs,
+    invoke: async (args, actor) =>
+      searchResponse(
+        await service.find(actor, {
+          term: args.q,
+          limit: args.limit,
+          collections: args.type,
+        }),
+      ),
+  })
+}

package/src/modules/stagedRecords.ts

@@ -0,0 +1,91 @@
+import { and, eq } from 'drizzle-orm'
+
+import type { Queryable } from '../runtime/transaction.ts'
+import { deals } from './deals/schema.ts'
+import { opportunities } from './opportunities/schema.ts'
+import { partnerships } from './partnerships/schema.ts'
+import type { PipelineKind } from './pipelines/schema.ts'
+import { raises } from './raises/schema.ts'
+
+/**
+ * The four tables whose rows sit in a pipeline stage, for the one operation that
+ * spans them all: removing a stage moves whatever still references it.
+ *
+ * Touching four modules' tables here rather than composing four services follows
+ * `attachedRecords.ts`: the alternative is a stage removal that imports every
+ * staged service to ask each for a bulk move it exists for nowhere else.
+ *
+ * Four written-out branches rather than a table-of-tables, because Drizzle's
+ * `update().set()` needs the concrete table type to check the column names.
+ */
+
+export interface StagedRecordMove {
+  readonly workspaceId: string
+  readonly fromStageId: string
+  readonly toStageId: string
+  readonly movedAt: Date
+}
+
+/**
+ * Moves every record of one kind out of a stage.
+ *
+ * @param db Must be the caller's transaction: the move only makes sense alongside
+ *   the stage delete that forced it.
+ * @returns The ids that moved, for the caller's activity trail and events.
+ */
+export async function reassignStagedRecords(
+  db: Queryable,
+  kind: PipelineKind,
+  move: StagedRecordMove,
+): Promise<readonly string[]> {
+  const changes = { stageId: move.toStageId, updatedAt: move.movedAt }
+
+  switch (kind) {
+    case 'deal': {
+      const moved = await db
+        .update(deals)
+        .set(changes)
+        .where(and(eq(deals.workspaceId, move.workspaceId), eq(deals.stageId, move.fromStageId)))
+        .returning({ id: deals.id })
+
+      return moved.map((row) => row.id)
+    }
+    case 'opportunity': {
+      const moved = await db
+        .update(opportunities)
+        .set(changes)
+        .where(
+          and(
+            eq(opportunities.workspaceId, move.workspaceId),
+            eq(opportunities.stageId, move.fromStageId),
+          ),
+        )
+        .returning({ id: opportunities.id })
+
+      return moved.map((row) => row.id)
+    }
+    case 'raise': {
+      const moved = await db
+        .update(raises)
+        .set(changes)
+        .where(and(eq(raises.workspaceId, move.workspaceId), eq(raises.stageId, move.fromStageId)))
+        .returning({ id: raises.id })
+
+      return moved.map((row) => row.id)
+    }
+    case 'partnership': {
+      const moved = await db
+        .update(partnerships)
+        .set(changes)
+        .where(
+          and(
+            eq(partnerships.workspaceId, move.workspaceId),
+            eq(partnerships.stageId, move.fromStageId),
+          ),
+        )
+        .returning({ id: partnerships.id })
+
+      return moved.map((row) => row.id)
+    }
+  }
+}

package/src/modules/webhooks/delivery.ts

@@ -0,0 +1,349 @@
+import { z } from 'zod'
+
+import type { Database } from '../../lib/database.ts'
+import { describeThrown } from '../../lib/errors.ts'
+import type { IdFactory } from '../../lib/ids.ts'
+import type { Logger } from '../../lib/logger.ts'
+import type { SecretCipher } from '../../lib/secrets.ts'
+import type { TransactionScope } from '../../runtime/transaction.ts'
+import type { WebhookEventPayload } from './payloads.ts'
+import * as repository from './repository.ts'
+import type { WebhookRecord } from './repository.ts'
+import { deliveryBody, deliveryHeaders, renderDeliveryBody, signDeliveryBody } from './signing.ts'
+import type { DeliveryEnvelope } from './signing.ts'
+
+/**
+ * The delivery engine: a consumer of the internal event bus that turns one
+ * domain event into one signed HTTP request per subscribed webhook.
+ *
+ * It runs after the emitting transaction commits and is not awaited by the
+ * request that caused it (`runtime/transaction.ts`), so an unreachable endpoint
+ * slows nobody down.
+ *
+ * Delivery is at-least-once and there is no durable queue, the same caveat the
+ * bus itself carries. A crash between the commit and the last attempt loses the
+ * delivery; the retry budget below is deliberately small so a graceful shutdown
+ * is not held open for minutes waiting for an endpoint that is already gone.
+ */
+
+/** How long one attempt may take before it counts as failed. */
+export const DELIVERY_TIMEOUT_MS = 10_000
+
+/** Waits between attempts. Its length plus one is the attempt budget. */
+export const RETRY_DELAYS_MS: readonly number[] = [1_000, 5_000, 15_000]
+
+export const MAX_DELIVERY_ATTEMPTS = RETRY_DELAYS_MS.length + 1
+
+/**
+ * @param attempts How many attempts have already been made.
+ * @returns How long to wait before the next one, or undefined when the budget
+ *   is spent and the delivery has failed for good.
+ */
+export function retryDelayAfter(attempts: number): number | undefined {
+  return RETRY_DELAYS_MS[attempts - 1]
+}
+
+const DAY_MS = 86_400_000
+
+/** What an unset `WEBHOOK_DELIVERY_RETENTION_DAYS` means. */
+export const DEFAULT_DELIVERY_RETENTION_DAYS = 30
+
+/**
+ * The environment slice for the delivery log's retention window. Validated at
+ * boot through `context.config`, so a malformed value stops the service with
+ * the module named rather than quietly pruning by the wrong window.
+ *
+ * Optional with a stated default, unlike most of the environment: absence is
+ * the normal state for an operator who does not care how long the log is, and
+ * the README's configuration table carries the default so it is not silent.
+ * Blank counts as absent, the `SECRET_ENCRYPTION_KEY_PREVIOUS` rule: operators
+ * empty a line far more often than they delete it.
+ */
+export const deliveryRetentionConfigSchema = z.object({
+  WEBHOOK_DELIVERY_RETENTION_DAYS: z
+    .string()
+    .optional()
+    .transform((value) => (value === undefined || value.trim().length === 0 ? undefined : value))
+    .refine(
+      (value) => value === undefined || (Number.isInteger(Number(value)) && Number(value) >= 1),
+      { message: 'must be a whole number of days, at least 1' },
+    )
+    .transform((value) =>
+      value === undefined ? DEFAULT_DELIVERY_RETENTION_DAYS : Number(value),
+    ),
+})
+
+export type DeliveryRetentionConfig = z.infer<typeof deliveryRetentionConfigSchema>
+
+/** The moment a log row written before has outlived its retention. */
+export function retentionCutoff(at: Date, retentionDays: number): Date {
+  return new Date(at.getTime() - retentionDays * DAY_MS)
+}
+
+export interface DeliveryRequest {
+  readonly url: string
+  readonly body: string
+  readonly headers: Readonly<Record<string, string>>
+}
+
+export interface AttemptOutcome {
+  readonly delivered: boolean
+  /** The response status, or null when no response arrived at all. */
+  readonly status: number | null
+  /** Why it failed, for the log. Null on success. */
+  readonly reason: string | null
+}
+
+/** The outbound port. Injected so no test makes a network call. */
+export type SendDelivery = (request: DeliveryRequest) => Promise<AttemptOutcome>
+
+export type Sleep = (milliseconds: number) => Promise<void>
+
+export function sleepFor(milliseconds: number): Promise<void> {
+  return new Promise((resolve) => {
+    setTimeout(resolve, milliseconds)
+  })
+}
+
+/**
+ * The real sender.
+ *
+ * Redirects are not followed. A customer whose endpoint moved should see the
+ * `301` and update the registration, rather than have Kelpie quietly post
+ * workspace data to whichever host the old one now points at.
+ *
+ * The catch is broad because this is the process boundary: `fetch` rejects with
+ * anything from a DNS failure to an abort, the set is open, and turning all of
+ * it into one outcome is the port's whole job. Nothing is swallowed — the
+ * reason reaches the log and the delivery is recorded as failed.
+ */
+export function createHttpSender(fetchImplementation: typeof fetch = fetch): SendDelivery {
+  return async (request) => {
+    try {
+      const response = await fetchImplementation(request.url, {
+        method: 'POST',
+        body: request.body,
+        headers: request.headers,
+        redirect: 'manual',
+        signal: AbortSignal.timeout(DELIVERY_TIMEOUT_MS),
+      })
+
+      return response.ok
+        ? { delivered: true, status: response.status, reason: null }
+        : {
+            delivered: false,
+            status: response.status,
+            reason: `endpoint answered ${String(response.status)}`,
+          }
+    } catch (error: unknown) {
+      return { delivered: false, status: null, reason: describeThrown(error) }
+    }
+  }
+}
+
+export interface DeliveryDependencies {
+  readonly db: Database
+  readonly transaction: TransactionScope
+  readonly createId: IdFactory
+  readonly now: () => Date
+  readonly cipher: SecretCipher
+  readonly send: SendDelivery
+  /** Injected so tests do not spend the retry budget in real time. */
+  readonly sleep: Sleep
+  /** From `deliveryRetentionConfigSchema`, validated at boot. */
+  readonly retentionDays: number
+  readonly log: Logger
+}
+
+export interface DeliveryEngine {
+  /** Fans one event out to every subscribed webhook in its workspace. */
+  deliver(payload: WebhookEventPayload): Promise<void>
+}
+
+export function createDeliveryEngine(dependencies: DeliveryDependencies): DeliveryEngine {
+  /**
+   * Writes the outcome: one log row, the expired rows gone, and the webhook's
+   * status when it moved.
+   *
+   * One transaction, so a reader never sees a hook marked failing with no
+   * failure recorded against it.
+   *
+   * Retention is enforced here rather than by a schedule because there is no
+   * scheduler in the service, and the log only grows through this function —
+   * pruning on every append caps the table exactly where the growth happens.
+   * The cost is the documented residue: a hook that stops delivering keeps its
+   * last window of rows until the hook or the workspace is deleted.
+   */
+  async function record(
+    webhook: WebhookRecord,
+    payload: WebhookEventPayload,
+    delivery: {
+      id: string
+      sentAt: Date
+      body: Record<string, unknown>
+      attempts: number
+      outcome: AttemptOutcome
+    },
+  ): Promise<void> {
+    const nextStatus = delivery.outcome.delivered ? 'active' : 'failing'
+
+    const pruned = await dependencies.transaction(async ({ tx }) => {
+      await repository.insertDelivery(tx, {
+        id: delivery.id,
+        workspaceId: webhook.workspaceId,
+        webhookId: webhook.id,
+        event: payload.event,
+        payload: delivery.body,
+        status: delivery.outcome.delivered ? 'success' : 'failed',
+        attempts: delivery.attempts,
+        // `created_at` is when the delivery was built, which is the `created_at`
+        // inside the payload too, so the log row and what the receiver holds
+        // agree. `delivered_at` is when it actually landed, up to the whole
+        // retry budget later.
+        deliveredAt: delivery.outcome.delivered ? dependencies.now() : null,
+        createdAt: delivery.sentAt,
+      })
+
+      // The cutoff is later than the row above's `created_at` for any positive
+      // window, so a prune can never take the delivery it arrived with, and
+      // `last_delivery_*` always has a row to read.
+      const expired = await repository.deleteExpiredDeliveries(
+        tx,
+        webhook.id,
+        retentionCutoff(dependencies.now(), dependencies.retentionDays),
+      )
+
+      // A paused webhook is never selected for delivery, so the only statuses
+      // reachable here are the two the engine owns. Leaving an unchanged one
+      // alone keeps a healthy endpoint from being rewritten on every event.
+      if (webhook.status !== nextStatus) {
+        await repository.setWebhookStatus(tx, webhook.id, nextStatus)
+      }
+
+      return expired
+    })
+
+    // After the transaction settles: a rollback would make this line a lie.
+    if (pruned > 0) {
+      dependencies.log.debug('pruned expired webhook deliveries', {
+        webhookId: webhook.id,
+        pruned,
+      })
+    }
+  }
+
+  async function attemptUntilDelivered(request: DeliveryRequest): Promise<{
+    attempts: number
+    outcome: AttemptOutcome
+  }> {
+    let attempts = 0
+
+    for (;;) {
+      attempts += 1
+
+      const outcome = await dependencies.send(request)
+
+      if (outcome.delivered) {
+        return { attempts, outcome }
+      }
+
+      const delay = retryDelayAfter(attempts)
+
+      if (delay === undefined) {
+        return { attempts, outcome }
+      }
+
+      await dependencies.sleep(delay)
+    }
+  }
+
+  /**
+   * Every secret this delivery should be signed under, newest first.
+   *
+   * A secret that will not decrypt throws, and the caller logs it against this
+   * webhook. Deliberately not recorded as a failed delivery: the endpoint did
+   * nothing wrong, Kelpie's key did, and marking the registration `failing`
+   * would send the customer debugging their own server while the real fault is
+   * `SECRET_ENCRYPTION_KEY`.
+   *
+   * The expiry is read here rather than swept by a job. There is no scheduler in
+   * the service, and an overlap that has run out simply stops being signed with;
+   * the column is cleared by the next rotation or by deleting the registration.
+   */
+  function signaturesFor(webhook: WebhookRecord, text: string, at: Date): readonly string[] {
+    const signatures = [signDeliveryBody(dependencies.cipher.open(webhook.secretEncrypted), text)]
+    const expiresAt = webhook.previousSecretExpiresAt
+
+    if (webhook.previousSecretEncrypted !== null && expiresAt !== null && expiresAt > at) {
+      signatures.push(
+        signDeliveryBody(dependencies.cipher.open(webhook.previousSecretEncrypted), text),
+      )
+    }
+
+    return signatures
+  }
+
+  async function deliverTo(webhook: WebhookRecord, payload: WebhookEventPayload): Promise<void> {
+    const deliveryId = dependencies.createId('webhookDelivery')
+    const envelope: DeliveryEnvelope = {
+      deliveryId,
+      event: payload.event,
+      sentAt: dependencies.now(),
+      workspaceId: payload.workspaceId,
+      data: payload.data,
+    }
+    const body = deliveryBody(envelope)
+    const text = renderDeliveryBody(body)
+    const { attempts, outcome } = await attemptUntilDelivered({
+      url: webhook.url,
+      body: text,
+      headers: deliveryHeaders(envelope, signaturesFor(webhook, text, envelope.sentAt)),
+    })
+
+    if (!outcome.delivered) {
+      dependencies.log.warn('webhook delivery failed', {
+        webhookId: webhook.id,
+        deliveryId,
+        event: payload.event,
+        attempts,
+        status: outcome.status,
+        reason: outcome.reason,
+      })
+    }
+
+    // The body is stored as it was sent, so the log answers "what did they
+    // actually receive" rather than "what would we build for this event today".
+    await record(webhook, payload, { id: deliveryId, sentAt: envelope.sentAt, body, attempts, outcome })
+  }
+
+  return {
+    async deliver(payload) {
+      const subscribed = await repository.listSubscribed(
+        dependencies.db,
+        payload.workspaceId,
+        payload.event,
+      )
+
+      if (subscribed.length === 0) {
+        return
+      }
+
+      // One unreachable endpoint, or one row whose secret will not decrypt,
+      // must not stop the others. Same reason the bus settles its handlers
+      // rather than racing them.
+      const outcomes = await Promise.allSettled(
+        subscribed.map((webhook) => deliverTo(webhook, payload)),
+      )
+
+      for (const [index, outcome] of outcomes.entries()) {
+        if (outcome.status === 'rejected') {
+          dependencies.log.error('webhook delivery could not be attempted', {
+            webhookId: subscribed[index]?.id,
+            event: payload.event,
+            error: describeThrown(outcome.reason),
+          })
+        }
+      }
+    },
+  }
+}

package/src/modules/webhooks/index.ts

@@ -0,0 +1,85 @@
+import { createSecretCipher, secretEncryptionConfigSchema } from '../../lib/secrets.ts'
+import type { KelpieModule } from '../../runtime/module.ts'
+import {
+  createDeliveryEngine,
+  createHttpSender,
+  deliveryRetentionConfigSchema,
+  sleepFor,
+} from './delivery.ts'
+import type { SendDelivery, Sleep } from './delivery.ts'
+import { subscribeDeliverableEvents } from './payloads.ts'
+import { mountWebhooksRoutes } from './routes.ts'
+import * as schema from './schema.ts'
+import { createWebhooksService } from './service.ts'
+import { registerWebhooksTools } from './tools.ts'
+
+/**
+ * Webhooks: registration, and the engine that bridges the internal event bus to
+ * outbound HTTP.
+ *
+ * `requires` is `workspace` alone. The engine consumes events by name and never
+ * reads the tables behind them, so it does not depend on the modules that emit
+ * them — a payload carries the record's type and id and nothing else, which is
+ * what keeps this module from growing a dependency on every CRM module in turn.
+ *
+ * Subscription happens during `register`, and `architecture.md` fixes this
+ * module near the end of the core order, so every emitting module is already in
+ * place. Nothing depends on that: the bus resolves handlers at publication.
+ */
+export interface WebhooksModuleOptions {
+  /** Injected by tests so no suite makes a network call. */
+  readonly send?: SendDelivery
+  /** Injected by tests so no suite spends the retry budget in real time. */
+  readonly sleep?: Sleep
+}
+
+export function createWebhooksModule(
+  migrationsDirectory: string,
+  options: WebhooksModuleOptions = {},
+): KelpieModule {
+  return {
+    id: 'webhooks',
+    requires: ['workspace'],
+
+    register(context) {
+      // Validated at boot rather than on the first delivery: a missing or
+      // malformed key means no webhook can ever be signed, and finding that out
+      // when a customer's endpoint goes quiet is far too late. The retention
+      // window is validated the same way for the same reason — a bad value
+      // should stop boot, not prune by the wrong window.
+      const cipher = createSecretCipher(context.config(secretEncryptionConfigSchema))
+      const retention = context.config(deliveryRetentionConfigSchema)
+
+      const service = createWebhooksService({
+        db: context.db,
+        transaction: context.transaction,
+        createId: context.createId,
+        now: context.now,
+        cipher,
+      })
+
+      const engine = createDeliveryEngine({
+        db: context.db,
+        transaction: context.transaction,
+        createId: context.createId,
+        now: context.now,
+        cipher,
+        send: options.send ?? createHttpSender(),
+        sleep: options.sleep ?? sleepFor,
+        retentionDays: retention.WEBHOOK_DELIVERY_RETENTION_DAYS,
+        log: context.log,
+      })
+
+      context.schema(schema, migrationsDirectory)
+      subscribeDeliverableEvents(context.events, (payload) => engine.deliver(payload))
+
+      context.routes((router) => {
+        mountWebhooksRoutes(router, { db: context.db, now: context.now, service })
+      })
+
+      registerWebhooksTools(context.mcp, service)
+
+      return Promise.resolve()
+    },
+  }
+}

package/src/modules/webhooks/payloads.ts

@@ -0,0 +1,84 @@
+import type { WebhookEvent } from '@kelpie/schemas'
+
+import type { DomainEvents, EventBus } from '../../runtime/events.ts'
+
+/**
+ * The bridge from the internal event catalog to what a webhook receiver sees.
+ *
+ * `modules.md` makes the webhooks engine a consumer of the bus, so this is the
+ * only place that knows both vocabularies. Everything downstream works with a
+ * `WebhookEventPayload` and never sees a `DomainEvents` key.
+ *
+ * Payload keys are `snake_case`, matching `api.md`, because a receiver has no
+ * reason to expect a different convention from a webhook than from the REST
+ * surface it also calls.
+ */
+
+export type { WebhookEvent }
+
+/** One event, resolved to the workspace it belongs to and the body to send. */
+export interface WebhookEventPayload {
+  readonly workspaceId: string
+  readonly event: WebhookEvent
+  /** The `data` object of the delivery body. */
+  readonly data: Record<string, unknown>
+}
+
+/**
+ * Payload builders, one per deliverable event.
+ *
+ * `satisfies` is what keeps this honest: adding a name to `WEBHOOK_EVENTS`
+ * without a builder here is a compile error rather than an event that resolves
+ * to nothing at runtime.
+ */
+const builders = {
+  'record.created': (payload: DomainEvents['record.created']): WebhookEventPayload => ({
+    workspaceId: payload.workspaceId,
+    event: 'record.created',
+    data: { object_type: payload.objectType, record_id: payload.recordId },
+  }),
+
+  'record.updated': (payload: DomainEvents['record.updated']): WebhookEventPayload => ({
+    workspaceId: payload.workspaceId,
+    event: 'record.updated',
+    data: {
+      object_type: payload.objectType,
+      record_id: payload.recordId,
+      // The values stay as the emitting service named them, which is camelCase
+      // (`parentId`, `sortOrder`) even though the keys around them are not.
+      // Translating them to the wire spelling would mean a second copy of every
+      // module's column naming, kept in step by hand.
+      changed_fields: [...payload.changedFields],
+    },
+  }),
+
+  'record.deleted': (payload: DomainEvents['record.deleted']): WebhookEventPayload => ({
+    workspaceId: payload.workspaceId,
+    event: 'record.deleted',
+    data: { object_type: payload.objectType, record_id: payload.recordId },
+  }),
+
+  'form.submitted': (payload: DomainEvents['form.submitted']): WebhookEventPayload => ({
+    workspaceId: payload.workspaceId,
+    event: 'form.submitted',
+    data: { form_id: payload.formId, submission_id: payload.submissionId },
+  }),
+} satisfies { [Name in WebhookEvent]: (payload: DomainEvents[Name]) => WebhookEventPayload }
+
+/**
+ * Subscribes the engine to every deliverable event.
+ *
+ * Written out one call at a time rather than looped over `WEBHOOK_EVENTS`,
+ * because TypeScript cannot correlate a payload type with a generic key: a loop
+ * would need a cast, and the cast is exactly what would let a wrong payload
+ * through. A literal key indexes `builders` at its precise signature.
+ */
+export function subscribeDeliverableEvents(
+  bus: EventBus,
+  deliver: (payload: WebhookEventPayload) => Promise<void>,
+): void {
+  bus.subscribe('record.created', (payload) => deliver(builders['record.created'](payload)))
+  bus.subscribe('record.updated', (payload) => deliver(builders['record.updated'](payload)))
+  bus.subscribe('record.deleted', (payload) => deliver(builders['record.deleted'](payload)))
+  bus.subscribe('form.submitted', (payload) => deliver(builders['form.submitted'](payload)))
+}