@crowdsource.you/core 1.2.0

package/src/outbox/intake.ts

@@ -0,0 +1,175 @@
+import type { SubjectRegistry } from './evidence.js';
+import { reportSubmitEventId, type OutboxService } from './outbox/service.js';
+import type {
+  ModerationReportStore,
+  ModerationTransactionRunner,
+} from './store/types.js';
+import type {
+  CreateReportInput,
+  CreateReportResult,
+  ModerationReportFields,
+} from './types.js';
+
+/**
+ * Storing a report and, when there is somewhere to send it, the promise to
+ * deliver it — in one operation.
+ *
+ * This is the only thing in the integration that a user waits for. A 201 from an
+ * application's report route means the report row and its outbox event committed
+ * together. It does NOT mean CrowdSource accepted anything — CrowdSource may be
+ * unreachable, mid-deploy or not yet configured, and the reporter is told their
+ * report was received either way, because it was.
+ *
+ * The transaction is the whole mechanism. Two writes outside one would give two
+ * failure modes that are both silent: a report with no delivery event (the
+ * report exists, nothing will ever send it, and nobody finds out until somebody
+ * asks why a case never opened) or a delivery event with no report (a delivery
+ * worker looking up an id that was rolled back). Neither surfaces as an error at
+ * the moment it happens, which is exactly why this has to be atomic rather than
+ * carefully ordered.
+ *
+ * The one report with NO delivery event is the one whose type has no subject
+ * provider, and that is a different claim entirely: not "delivery failed" but
+ * "there was never a route out of this application for this kind of object".
+ * Those two must not be conflated, which is why they are different `localStatus`
+ * values and why the absent route is written down as a reason rather than
+ * inferred from a missing row.
+ */
+
+export class DuplicateReportError<TReport> extends Error {
+  readonly existing: TReport;
+
+  constructor(existing: TReport) {
+    super('This item has already been reported by this reporter.');
+    this.name = 'DuplicateReportError';
+    this.existing = existing;
+  }
+}
+
+/**
+ * Refuses an identifier that is not a string, before it reaches the store.
+ *
+ * The input type says these are strings, but a type is erased at runtime and a
+ * truthiness check passes anything non-empty — including an object. What that
+ * costs depends on the backend, and the difference is worth stating rather than
+ * flattening: on Mongo a `{ $ne: null }` becomes a query OPERATOR, so the
+ * duplicate lookup matches an UNRELATED report and answers "you already reported
+ * this" about somebody else's row; on Postgres a bound parameter cannot become
+ * an operator, so that particular failure class does not exist there.
+ *
+ * The guard is not Mongo's, though. A non-string still reaches the insert and
+ * stores something that is not an id where an id belongs, on any backend — and
+ * the check lives here rather than at an application's route because this
+ * function is exported: a queue worker, a reconciliation script or a future
+ * admin path is under no obligation to have passed a route's validation, and a
+ * guard that only exists at one caller is a guard that holds until the second
+ * one arrives.
+ */
+function requireIdentifier(value: unknown, field: string): string {
+  /**
+   * `trim()` and not `length === 0`, because `'   '` is not an identifier.
+   *
+   * A whitespace-only reporter or reported id is not a string somebody meant:
+   * it comes from a form field, a trimmed-elsewhere value, or a client that
+   * sends `' '` for absent. Stored, it is an id that matches nothing and reads
+   * as present — the duplicate check finds no earlier report, the delivery
+   * carries it as `externalReportId`, and nothing fails until a human asks why
+   * a case names nobody.
+   */
+  if (typeof value !== 'string' || value.trim() === '') {
+    throw new TypeError(`createReport: ${field} must be a non-empty string.`);
+  }
+  return value;
+}
+
+/**
+ * Why a report is not going anywhere, in words an operator can read.
+ *
+ * Stored on the row rather than left to be inferred from a missing outbox event.
+ * A missing row is also what a lost write looks like, and the two need to be
+ * distinguishable months later without re-deriving which types had providers at
+ * the time. Bounded by the schema's 300-character limit.
+ */
+function localOnlyReason(reportedType: string): string {
+  return (
+    `There is no moderation subject provider for '${reportedType}', so this report is ` +
+    'recorded locally and is not sent for community review.'
+  );
+}
+
+/**
+ * Store the report, and queue its delivery in the same transaction.
+ *
+ * Delivery is queued when — and only when — the reported type has a subject
+ * provider. A type without one is stored at `received` with the reason recorded,
+ * which is the behaviour the application had before CrowdSource existed: the
+ * report is a receipt and a local record, and nothing else ever happens to it.
+ *
+ * That branch is the reason the two writes stay in one transaction rather than
+ * being ordered carefully. The condition is read BEFORE the transaction body
+ * decides anything, so `localStatus` and the presence of an outbox row are
+ * decided together from one fact — a report can never commit as `queued` with
+ * nothing to deliver it, nor as `received` with a delivery event that will try
+ * anyway.
+ *
+ * Intake deliberately does not read whether the integration is enabled. A report
+ * taken while it is off still gets its delivery event, so turning the flag on
+ * delivers the backlog instead of stranding it — the dispatcher is what is
+ * gated, not the durable record. Nothing here is conditional on a third party's
+ * state; only on whether this application knows how to describe the object at
+ * all.
+ */
+export function createIntake<TReport extends ModerationReportFields, TTx>(input: {
+  transaction: ModerationTransactionRunner<TTx>;
+  reports: ModerationReportStore<TReport, TTx>;
+  registry: SubjectRegistry;
+  outbox: OutboxService<TTx>;
+}): (report: CreateReportInput) => Promise<CreateReportResult<TReport>> {
+  return async (report) => {
+    const reporter = requireIdentifier(report.reporter, 'reporter');
+    const reportedId = requireIdentifier(report.reportedId, 'reportedId');
+    const reportedType = requireIdentifier(report.reportedType, 'reportedType');
+    if (!Array.isArray(report.categories) || report.categories.length === 0) {
+      throw new TypeError('createReport: categories must be a non-empty array.');
+    }
+    for (const category of report.categories) {
+      requireIdentifier(category, 'categories[]');
+    }
+    const deliverable = input.registry.providerFor(reportedType) !== undefined;
+
+    return await input.transaction.run(async (tx) => {
+      const existing = await input.reports.findDuplicate(
+        { reporter, reportedId, reportedType },
+        tx,
+      );
+      if (existing) throw new DuplicateReportError(existing);
+
+      const created = await input.reports.insert(
+        {
+          reportedType,
+          reportedId,
+          reporter,
+          categories: [...report.categories],
+          ...(report.details === undefined ? {} : { details: report.details }),
+          localStatus: deliverable ? 'queued' : 'received',
+          ...(deliverable ? {} : { localStatusReason: localOnlyReason(reportedType) }),
+          ...(report.extra === undefined ? {} : { extra: report.extra }),
+        },
+        tx,
+      );
+
+      if (!deliverable) return { report: created };
+
+      const outboxEventId = await input.outbox.enqueue(
+        {
+          eventId: reportSubmitEventId(created.id),
+          kind: 'report.submit',
+          payload: { reportId: created.id },
+        },
+        tx,
+      );
+
+      return { report: created, outboxEventId };
+    });
+  };
+}

package/src/outbox/integration.ts

@@ -0,0 +1,205 @@
+import type { Router } from 'express';
+import { createClientProvider, type CrowdSourceClientProvider } from './client.js';
+import { createDecisionWorker } from './decision.js';
+import { createDeliveryWorker } from './delivery.js';
+import { createEnforcementExecutor, type EnforcementExecutor } from './enforcement/executor.js';
+import { assertRestoreDirection } from './enforcement/planner.js';
+import { createSubjectRegistry, type SubjectRegistry } from './evidence.js';
+import { createInboundService, createProcessedEventStore } from './inbound.js';
+import { ModerationOutboxDispatcher, createOutboxRouter } from './outbox/dispatcher.js';
+import { createOutboxService } from './outbox/service.js';
+import { createIntake } from './intake.js';
+import {
+  ModerationReconciliationJob,
+  createReconciliation,
+  type ReconcileModerationReports,
+} from './reconciliation.js';
+import { createWebhookRouter } from './webhook.js';
+import type {
+  CreateReportInput,
+  CreateReportResult,
+  ModerationIntegrationConfig,
+  ModerationReportFields,
+} from './types.js';
+
+/**
+ * Everything wired together, from one object.
+ *
+ * A factory rather than a set of module-level singletons, and that is not
+ * stylistic: a module-level client, dispatcher or registry cannot be built
+ * twice, which makes two integrations in one test process impossible and makes
+ * test isolation depend on module-registry surgery. The store is built OUTSIDE
+ * and passed in, so the same wiring serves either backend and neither is
+ * reachable from here.
+ *
+ * `TTx` is deliberately absent from the returned interface. It is inferred from
+ * the config's store, used only inside this factory, and never surfaces — so a
+ * caller holding a `ModerationIntegration` cannot tell which backend built it,
+ * which is what lets one test suite run against both.
+ */
+export interface ModerationIntegration<
+  TReport extends ModerationReportFields,
+  TAction extends string,
+> {
+  /**
+   * Store a report and, when there is somewhere to send it, the promise to
+   * deliver it — in ONE transaction.
+   *
+   * Throws `DuplicateReportError` when this reporter already reported this
+   * object, and `TypeError` for an identifier that is not a non-empty string.
+   */
+  createReport(input: CreateReportInput): Promise<CreateReportResult<TReport>>;
+
+  /**
+   * The webhook receiver. **Mount this BEFORE `express.json()`** — the signature
+   * covers the bytes that arrived, and a parser destroys them.
+   *
+   * Returns an empty router when no webhook secret is configured: an
+   * unconfigured deployment 404s, which is indistinguishable from not having the
+   * feature.
+   */
+  webhookRouter(options?: { path?: string }): Router;
+
+  /**
+   * The outbox loop. Safe on every task — every event is claimed under a lease
+   * with an owner check, so N tasks share the work.
+   */
+  readonly dispatcher: ModerationOutboxDispatcher;
+
+  /**
+   * The reconciliation sweep on a timer. Start it from a LEADER-elected
+   * scheduler only.
+   */
+  readonly reconciliationJob: ModerationReconciliationJob;
+
+  /** One reconciliation sweep, on demand. */
+  readonly reconcile: ReconcileModerationReports;
+
+  /** The reported types that have a subject provider, so a test can pin the set. */
+  deliverableTypes(): string[];
+
+  readonly registry: SubjectRegistry;
+  readonly enforcement: EnforcementExecutor<TAction>;
+  readonly client: CrowdSourceClientProvider;
+}
+
+export function createModerationIntegration<
+  TReport extends ModerationReportFields,
+  TAction extends string,
+  TTx,
+>(
+  config: ModerationIntegrationConfig<TReport, TAction, TTx>,
+): ModerationIntegration<TReport, TAction> {
+  /**
+   * Refuse an inverted `restoreAction` before anything is wired. It cannot be
+   * caught by the type — both directions are `TAction[]` — and it does not fail
+   * at runtime; it applies a punishment on an accepted appeal.
+   */
+  assertRestoreDirection(config.enforcement);
+
+  const store = config.store;
+  const outbox = createOutboxService({ store: store.outbox, logger: config.logger });
+  const registry = createSubjectRegistry(config.subjects);
+  const client = createClientProvider({
+    config: config.crowdSource,
+    logger: config.logger,
+  });
+
+  const enforcement = createEnforcementExecutor<TAction>({
+    enforcement: store.enforcement,
+    config: config.enforcement,
+    defaultMode: config.crowdSource.enforcementMode,
+    logger: config.logger,
+    ...(config.metrics === undefined ? {} : { metrics: config.metrics }),
+  });
+
+  const deliverReport = createDeliveryWorker({
+    reports: store.reports,
+    registry,
+    taxonomy: config.taxonomy,
+    client,
+    logger: config.logger,
+    ...(config.metrics === undefined ? {} : { metrics: config.metrics }),
+  });
+
+  const applyDecision = createDecisionWorker({
+    reports: store.reports,
+    executor: enforcement,
+    enforcement: config.enforcement,
+    logger: config.logger,
+    ...(config.reportDecisionExtraFields === undefined
+      ? {}
+      : { reportDecisionExtraFields: config.reportDecisionExtraFields }),
+  });
+
+  const inbound = createInboundService({
+    transaction: store.transaction,
+    events: store.events,
+    outbox,
+  });
+
+  const reconcile = createReconciliation({
+    transaction: store.transaction,
+    reports: store.reports,
+    outbox,
+    logger: config.logger,
+    ...(config.crowdSource.staleSubmittedHours === undefined
+      ? {}
+      : { staleSubmittedHours: config.crowdSource.staleSubmittedHours }),
+  });
+
+  const dispatcher = new ModerationOutboxDispatcher({
+    outbox,
+    handler: createOutboxRouter({ deliverReport, applyDecision }),
+    logger: config.logger,
+    enabled: config.crowdSource.enabled,
+    ...(config.crowdSource.outboxPollIntervalMs === undefined
+      ? {}
+      : { pollIntervalMs: config.crowdSource.outboxPollIntervalMs }),
+    ...(config.crowdSource.outboxBatchSize === undefined
+      ? {}
+      : { batchSize: config.crowdSource.outboxBatchSize }),
+  });
+
+  const reconciliationJob = new ModerationReconciliationJob({
+    reconcile,
+    logger: config.logger,
+    enabled: config.crowdSource.enabled,
+    ...(config.crowdSource.reconciliationIntervalMs === undefined
+      ? {}
+      : { intervalMs: config.crowdSource.reconciliationIntervalMs }),
+  });
+
+  return {
+    createReport: createIntake({
+      transaction: store.transaction,
+      reports: store.reports,
+      registry,
+      outbox,
+    }),
+
+    webhookRouter(options = {}) {
+      return createWebhookRouter({
+        inbound,
+        store: createProcessedEventStore(store.events),
+        ...(config.crowdSource.webhookSecret === undefined
+          ? {}
+          : { secret: config.crowdSource.webhookSecret }),
+        ...(config.crowdSource.webhookPreviousSecret === undefined
+          ? {}
+          : { previousSecret: config.crowdSource.webhookPreviousSecret }),
+        logger: config.logger,
+        ...(config.metrics === undefined ? {} : { metrics: config.metrics }),
+        ...(options.path === undefined ? {} : { path: options.path }),
+      });
+    },
+
+    dispatcher,
+    reconciliationJob,
+    reconcile,
+    deliverableTypes: () => registry.deliverableTypes(),
+    registry,
+    enforcement,
+    client,
+  };
+}

package/src/outbox/outbox/dispatcher.ts

@@ -0,0 +1,131 @@
+import type { ModerationLogger, ModerationOutboxEvent } from '../types.js';
+import type { ModerationOutboxHandler, OutboxDrain } from './service.js';
+
+/**
+ * The loop that drains the moderation outbox.
+ *
+ * A bounded interval, one batch in flight at a time, an abort signal that stops
+ * claiming new work but lets the event already being handled reach a durable
+ * state.
+ *
+ * It is NOT leader-gated, and that is a property of the claim rather than an
+ * oversight. Every event is taken under a lease with an owner check, so N tasks
+ * draining the same collection simply share the work — and a task dying
+ * mid-delivery has its lease expire and its event reclaimed, which a single
+ * leader would not give us.
+ *
+ * `enabled` gates the LOOP, never the durable record. Reports taken while the
+ * integration is off keep their outbox rows and deliver when it is switched on;
+ * running the loop instead would count attempts against a deployment that has
+ * nowhere to send anything and dead-letter the backlog it was supposed to
+ * preserve.
+ */
+
+const DEFAULT_POLL_INTERVAL_MS = 5_000;
+
+export class ModerationOutboxDispatcher {
+  private timer: ReturnType<typeof setInterval> | null = null;
+  private inFlight: Promise<void> | null = null;
+  private abortController: AbortController | null = null;
+  private running = false;
+
+  constructor(
+    private readonly options: {
+      outbox: OutboxDrain;
+      handler: ModerationOutboxHandler;
+      logger: ModerationLogger;
+      enabled: boolean;
+      pollIntervalMs?: number;
+      batchSize?: number;
+    },
+  ) {}
+
+  start(): void {
+    if (this.running) return;
+    if (!this.options.enabled) {
+      this.options.logger.info(
+        '[CrowdSource] outbox dispatcher not started: the integration is disabled',
+      );
+      return;
+    }
+    const intervalMs = this.options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
+    this.running = true;
+    this.abortController = new AbortController();
+    void this.tick();
+    this.timer = setInterval(() => {
+      void this.tick();
+    }, intervalMs);
+    this.timer.unref?.();
+    this.options.logger.info('[CrowdSource] outbox dispatcher started', {
+      intervalMs,
+      batchSize: this.options.batchSize,
+    });
+  }
+
+  async stop(): Promise<void> {
+    this.running = false;
+    const controller = this.abortController;
+    controller?.abort();
+    if (this.timer) {
+      clearInterval(this.timer);
+      this.timer = null;
+    }
+    await this.inFlight;
+    if (this.abortController === controller) {
+      this.abortController = null;
+    }
+  }
+
+  private async tick(): Promise<void> {
+    if (!this.running) return;
+    if (this.inFlight) return this.inFlight;
+    const work = this.options.outbox
+      .dispatch({
+        handler: this.options.handler,
+        ...(this.options.batchSize === undefined
+          ? {}
+          : { batchSize: this.options.batchSize }),
+        ...(this.abortController === null
+          ? {}
+          : { signal: this.abortController.signal }),
+      })
+      .then(({ processed, failed, deadLettered }) => {
+        if (processed > 0 || failed > 0) {
+          this.options.logger.info('[CrowdSource] outbox batch complete', {
+            processed,
+            failed,
+            deadLettered,
+          });
+        }
+      })
+      .catch((error: unknown) => {
+        // Claim/database failures happen outside the per-event retry block. Keep
+        // the interval alive and avoid an unhandled rejection.
+        this.options.logger.error('[CrowdSource] outbox tick failed', {
+          error: error instanceof Error ? error.message : String(error),
+        });
+      })
+      .finally(() => {
+        if (this.inFlight === work) this.inFlight = null;
+      });
+    this.inFlight = work;
+    return work;
+  }
+}
+
+/** Route an event to the worker that owns its kind. */
+export function createOutboxRouter(handlers: {
+  deliverReport: (event: ModerationOutboxEvent) => Promise<void>;
+  applyDecision: (event: ModerationOutboxEvent) => Promise<void>;
+}): ModerationOutboxHandler {
+  return async (event: ModerationOutboxEvent): Promise<void> => {
+    switch (event.kind) {
+      case 'report.submit':
+        await handlers.deliverReport(event);
+        return;
+      case 'decision.apply':
+        await handlers.applyDecision(event);
+        return;
+    }
+  };
+}