@kelpie/server 0.1.0

package/src/modules/references.ts

@@ -0,0 +1,44 @@
+import { referenceViolationTable } from '../lib/database.ts'
+import { AppError } from '../lib/errors.ts'
+
+/**
+ * Roadmap decision 2: a delete blocked by an independent reference is a 409 that
+ * names what is still pointing at the record, so the caller knows what to detach.
+ *
+ * The database refuses first. This turns its refusal into the wire shape rather
+ * than trying to predict it: a pre-flight count would still race a concurrent
+ * insert, and would need updating every time a table gains a foreign key.
+ */
+
+/** Referencing tables, as the domain names a user of the API would recognise. */
+const REFERENCING_TYPE_BY_TABLE: Readonly<Record<string, string>> = {
+  candidates: 'candidate',
+  deal_people: 'deal',
+  deals: 'deal',
+  form_submissions: 'form submission',
+  forms: 'form',
+  opportunities: 'opportunity',
+  partnership_people: 'partnership',
+  partnerships: 'partnership',
+  raise_people: 'raise',
+  raises: 'raise',
+  workspace_members: 'team member',
+}
+
+/**
+ * @param subject The record being deleted, e.g. `person`. Appears in the message.
+ * @returns A 409. The referencing type is omitted when the driver did not report
+ *   a table, which is a worse message but still the right status.
+ */
+export function referencedElsewhere(error: unknown, subject: string): AppError {
+  const table = referenceViolationTable(error)
+  const type = table === undefined ? undefined : (REFERENCING_TYPE_BY_TABLE[table] ?? table)
+
+  if (type === undefined) {
+    return AppError.conflict(`This ${subject} is still referenced by another record`)
+  }
+
+  return AppError.conflict(`This ${subject} is still referenced by a ${type}`, [
+    { field: 'id', message: `Referenced by ${type}` },
+  ])
+}

package/src/modules/reseal.ts

@@ -0,0 +1,178 @@
+import { eq } from 'drizzle-orm'
+
+import type { Database } from '../lib/database.ts'
+import { SecretDecryptionError } from '../lib/secrets.ts'
+import type { SecretCipher } from '../lib/secrets.ts'
+import { agentRegistrations } from './agent-tasks/schema.ts'
+import { integrationConnections } from './integrations/schema.ts'
+import { webhooks } from './webhooks/schema.ts'
+
+/**
+ * Re-seals every stored secret under the current `SECRET_ENCRYPTION_KEY`.
+ *
+ * Rotating that key makes everything sealed under the old one unreadable, and
+ * the failure is loud but unactionable: the delivery engine logs a
+ * `SecretDecryptionError` naming the webhook and stops signing for it. The way
+ * out is to set `SECRET_ENCRYPTION_KEY_PREVIOUS` to the old key, deploy, and run
+ * this. Deliveries keep working throughout, because `open` falls back to the
+ * previous key while this pass catches up.
+ *
+ * Lives beside `attachedRecords.ts` and `recordTargets.ts` rather than in `lib/`
+ * for the reason those do: it has to read module tables, and `lib` is below
+ * `modules` in the dependency order.
+ *
+ * Idempotent, and deliberately not one transaction. Each row is independent, so
+ * a run that dies halfway leaves the rows it finished already current and the
+ * rest still readable under the previous key. Re-running finishes the job.
+ * Wrapping fifty thousand rows in one transaction would buy nothing and hold a
+ * write lock across the lot.
+ */
+
+/** One sealed column, and how to read and rewrite it. */
+interface SealedColumn {
+  /** `table.column`, for a report an operator can act on. */
+  readonly label: string
+  readonly read: (db: Database) => Promise<readonly { id: string; sealed: string | null }[]>
+  readonly write: (db: Database, id: string, sealed: string) => Promise<unknown>
+}
+
+/**
+ * Every column holding a value sealed by `lib/secrets.ts`.
+ *
+ * All three that exist are here, including the two nothing writes yet. Covering
+ * a column before its first write is the cheap half of this: the expensive half
+ * is noticing, a year later, that a rotation reported success while stranding a
+ * customer's stored OAuth credential with no way back. `resealTest` asserts this
+ * list against the schema, so a fourth `_encrypted` column fails a test the day
+ * it is added rather than at the next rotation.
+ *
+ * There is no registry for modules to declare these through, on purpose. One
+ * caller does not need an extension point, and a sealed column that nothing
+ * re-seals is a bug whichever way it was registered.
+ */
+const SEALED_COLUMNS: readonly SealedColumn[] = [
+  {
+    label: 'webhooks.secret_encrypted',
+    read: async (db) => db.select({ id: webhooks.id, sealed: webhooks.secretEncrypted }).from(webhooks),
+    write: async (db, id, sealed) => {
+      await db.update(webhooks).set({ secretEncrypted: sealed }).where(eq(webhooks.id, id))
+    },
+  },
+  {
+    label: 'webhooks.previous_secret_encrypted',
+    read: async (db) =>
+      db.select({ id: webhooks.id, sealed: webhooks.previousSecretEncrypted }).from(webhooks),
+    write: async (db, id, sealed) => {
+      await db.update(webhooks).set({ previousSecretEncrypted: sealed }).where(eq(webhooks.id, id))
+    },
+  },
+  {
+    label: 'agent_registrations.auth_header_encrypted',
+    read: async (db) =>
+      db
+        .select({ id: agentRegistrations.id, sealed: agentRegistrations.authHeaderEncrypted })
+        .from(agentRegistrations),
+    write: async (db, id, sealed) => {
+      await db
+        .update(agentRegistrations)
+        .set({ authHeaderEncrypted: sealed })
+        .where(eq(agentRegistrations.id, id))
+    },
+  },
+  {
+    label: 'integration_connections.secrets_encrypted',
+    read: async (db) =>
+      db
+        .select({ id: integrationConnections.id, sealed: integrationConnections.secretsEncrypted })
+        .from(integrationConnections),
+    write: async (db, id, sealed) => {
+      await db
+        .update(integrationConnections)
+        .set({ secretsEncrypted: sealed })
+        .where(eq(integrationConnections.id, id))
+    },
+  },
+]
+
+/** The columns this pass covers. Exported for the test that checks none is missing. */
+export const RESEALED_COLUMNS: readonly string[] = SEALED_COLUMNS.map((column) => column.label)
+
+/** What one column's pass did. */
+export interface ResealColumnOutcome {
+  /** `table.column`, for a report an operator can act on. */
+  readonly label: string
+  readonly examined: number
+  readonly resealed: number
+  /**
+   * Rows that opened under neither key. Sealed under a third key, or altered.
+   * Named by id rather than counted: nothing here can recover them, so the only
+   * useful output is which rows an operator has to deal with by hand.
+   */
+  readonly unreadable: readonly string[]
+}
+
+export interface ResealOutcome {
+  readonly columns: readonly ResealColumnOutcome[]
+  readonly examined: number
+  readonly resealed: number
+  readonly unreadable: number
+}
+
+/**
+ * Reads every sealed column, and rewrites anything not already under the
+ * current key.
+ *
+ * Does not throw for a row it cannot open. One unreadable row must not stop the
+ * pass from fixing every other one, so it is reported and the caller decides
+ * what a non-empty `unreadable` means.
+ */
+export async function resealStoredSecrets(
+  db: Database,
+  cipher: SecretCipher,
+): Promise<ResealOutcome> {
+  const columns: ResealColumnOutcome[] = []
+
+  for (const column of SEALED_COLUMNS) {
+    // Null means the row holds no secret, which is every row of a column whose
+    // module has not started writing one. Not counted as examined: a report
+    // saying it looked at forty rows and re-sealed none would read as a problem.
+    const rows = (await column.read(db)).filter(
+      (row): row is { id: string; sealed: string } => row.sealed !== null,
+    )
+    const unreadable: string[] = []
+    let resealed = 0
+
+    for (const row of rows) {
+      let next: string | undefined
+
+      try {
+        next = cipher.reseal(row.sealed)
+      } catch (error: unknown) {
+        if (!(error instanceof SecretDecryptionError)) {
+          throw error
+        }
+
+        unreadable.push(row.id)
+        continue
+      }
+
+      // Undefined means it is already sealed under the current key, which is
+      // every row on a second run and most rows on the first.
+      if (next === undefined) {
+        continue
+      }
+
+      await column.write(db, row.id, next)
+      resealed += 1
+    }
+
+    columns.push({ label: column.label, examined: rows.length, resealed, unreadable })
+  }
+
+  return {
+    columns,
+    examined: columns.reduce((total, column) => total + column.examined, 0),
+    resealed: columns.reduce((total, column) => total + column.resealed, 0),
+    unreadable: columns.reduce((total, column) => total + column.unreadable.length, 0),
+  }
+}

package/src/modules/search/index.ts

@@ -0,0 +1,55 @@
+import type { KelpieModule } from '../../runtime/module.ts'
+import { mountSearchRoutes } from './routes.ts'
+import { createSearchService } from './service.ts'
+import { registerSearchTools } from './tools.ts'
+
+/**
+ * One box across nine collections.
+ *
+ * It contributes no tables and calls no `context.schema`. The `search_vector`
+ * column each searchable table carries belongs to that table's own module, which
+ * is the only place that knows which of its columns are worth indexing.
+ *
+ * `requires` names every module holding one of those columns, including the two
+ * that are never results themselves: `positions` carries the job titles People are
+ * found by, and `plans` the step titles Deals, Opportunities and Raises are found
+ * by. Nothing here runs at registration, so this is not about ordering. It is
+ * about an assembly built without one of them, which would boot and then answer
+ * this endpoint with a query against a table that has no migration.
+ *
+ * `structural`, like `dashboard`: the search box is in the application shell on
+ * every page, so a workspace that switched this off would be left with a control
+ * that answers 404.
+ */
+export function createSearchModule(): KelpieModule {
+  return {
+    id: 'search',
+    requires: [
+      'workspace',
+      'people',
+      'positions',
+      'companies',
+      'deals',
+      'opportunities',
+      'partnerships',
+      'raises',
+      'hiring',
+      'plans',
+      'decisions',
+      'handbook',
+    ],
+    structural: true,
+
+    register(context) {
+      const service = createSearchService({ db: context.db })
+
+      context.routes((router) => {
+        mountSearchRoutes(router, { db: context.db, now: context.now, service })
+      })
+
+      registerSearchTools(context.mcp, service)
+
+      return Promise.resolve()
+    },
+  }
+}

package/src/modules/search/query.ts

@@ -0,0 +1,104 @@
+/**
+ * Turning what someone typed into a `tsquery`, and lifting a readable fragment
+ * out of what matched.
+ *
+ * Pure: no database, no actor, no clock. Both halves are the parts of search most
+ * likely to be wrong in a way no integration test would notice, so both are unit
+ * tested against their own cases.
+ */
+
+/**
+ * How long a `?q=` may be before the boundary refuses it.
+ *
+ * Every word becomes a clause of an `&`-joined `tsquery`, so an unbounded string
+ * is an unbounded query plan. Nobody searches a CRM with a paragraph.
+ */
+export const MAX_SEARCH_TERM_LENGTH = 200
+
+/** Anything that is not a letter or a digit separates one word from the next. */
+const SEPARATORS = /[^\p{L}\p{N}]+/gu
+
+/**
+ * The typed string as a `tsquery`, or null when it holds nothing to search for.
+ *
+ * Every word is a prefix clause. Someone typing `acm` into a search box is part
+ * way through `acme` and expects to see it; `to_tsquery('acm')` alone finds
+ * nothing, because full-text matches whole lexemes. Stemming still applies on top,
+ * so `meetings` finds a record that says `meeting`.
+ *
+ * Words are joined with `&`: every word has to appear somewhere in the record.
+ * `|` would rank a record matching one word of four alongside one matching all of
+ * them, which reads as a search box that ignores what you typed.
+ *
+ * @returns null when the input holds no letters or digits at all, which is a
+ *   query with no clauses rather than a syntax error to hand to Postgres.
+ */
+export function toTsQuery(raw: string): string | null {
+  const words = raw.split(SEPARATORS).filter((word) => word.length > 0)
+
+  if (words.length === 0) {
+    return null
+  }
+
+  return words.map((word) => `${word}:*`).join(' & ')
+}
+
+/** The words `toTsQuery` searched for, which is what `snippet` centres on. */
+export function searchWords(raw: string): readonly string[] {
+  return raw.split(SEPARATORS).filter((word) => word.length > 0)
+}
+
+/** How much of the source text a snippet carries, either side of the match. */
+const LEAD = 24
+const TRAIL = 56
+
+/** Markdown punctuation, which is noise once a page body is one line of preview. */
+const MARKUP = /[#*`|[\]_>]/gu
+
+function flatten(body: string): string {
+  return body.replace(MARKUP, ' ').replace(/\s+/gu, ' ').trim()
+}
+
+/**
+ * Finds the earliest position any search word appears at, case-insensitively.
+ *
+ * @returns -1 when none of them appear literally. That is not a contradiction of
+ *   the match: the row matched a stemmed lexeme (`meeting` for `meetings`) or
+ *   matched on a different field than the one being excerpted.
+ */
+function firstMatch(haystack: string, words: readonly string[]): number {
+  const lowered = haystack.toLowerCase()
+
+  return words
+    .map((word) => lowered.indexOf(word.toLowerCase()))
+    .filter((index) => index >= 0)
+    .reduce((earliest, index) => (earliest < 0 ? index : Math.min(earliest, index)), -1)
+}
+
+/**
+ * A one-line fragment of `body`, centred on the first search word it contains.
+ *
+ * Falls back to the opening of the text when no word appears literally, because a
+ * result with no preview at all reads as a result with no content.
+ *
+ * @param body The record's own prose. Markdown is flattened first.
+ * @param words What was searched for, from `searchWords`.
+ */
+export function snippet(body: string, words: readonly string[]): string {
+  const plain = flatten(body)
+
+  if (plain.length === 0) {
+    return ''
+  }
+
+  const found = firstMatch(plain, words)
+
+  if (found < 0) {
+    return plain.length > LEAD + TRAIL ? `${plain.slice(0, LEAD + TRAIL)}…` : plain
+  }
+
+  const start = Math.max(0, found - LEAD)
+  const end = Math.min(plain.length, found + TRAIL)
+
+  return `${start > 0 ? '…' : ''}${plain.slice(start, end)}${end < plain.length ? '…' : ''}`
+}