@ultimat3/jobs 30.8.0 → 31.0.0

package/src/events.ts

@@ -3,9 +3,11 @@
 // 12:00:10, so a fire-and-forget emitter would silently strand every waiting run.
 
 import type { Clock } from '@ultimat3/core';
-import { finiteOption, logger, systemClock, uuidV7 } from '@ultimat3/core';
+import { finiteOption, liveClock, logger, uuidV7 } from '@ultimat3/core';
+import { adapterSlot } from '@ultimat3/core/adapter';
 import type { DurationInput } from './clock';
 import { finiteDurationMs, nowMs } from './clock';
+import { eventBefore, pastCursor } from './events-cursor';
 import type { PurgeTarget } from './purge';
 import type { EventLookup } from './steps';
 
@@ -36,7 +38,8 @@ export interface EventBus extends EventLookup {
   list(name?: string): Promise<readonly JobEvent[]>;
   /**
    * Delete every event past its expiry and answer how many went. Measured on the bus's own clock,
-   * like every other instant it holds. The memory bus also sweeps on each publish; the stored one
+   * like every other instant it holds. The memory bus also sweeps on a publish, at most once a
+   * second and only once something has expired; the stored one
    * has no caller but the retention sweep (`eventsPurgeTarget`).
    */
   purgeExpired(): Promise<number>;
@@ -49,8 +52,11 @@ export interface MemoryEventBusOptions {
   readonly maxEvents?: number;
 }
 
+/** The least gap between two publish-time sweeps, on the bus's clock. */
+const SWEEP_EVERY_MS = 1_000;
+
 export function memoryEventBus(options: MemoryEventBusOptions = {}): EventBus {
-  const clock = options.clock ?? systemClock;
+  const clock = options.clock ?? liveClock;
   // TWO screens, because these are two knobs: the default is declared at construction and belongs
   // to whoever built the bus, `ttl` rides the publish CALL. One screen over `ttl ?? defaultTtl`
   // told a caller who wrote `{ ttl: NaN }` to "pass a finite defaultTtl" — an instruction naming an
@@ -63,16 +69,31 @@ export function memoryEventBus(options: MemoryEventBusOptions = {}): EventBus {
   );
   const maxEvents = finiteOption('the memory event bus', 'maxEvents', options.maxEvents ?? 10_000);
   const events = new Map<string, JobEvent>();
+  // Per name, so `find` reads one event's rows and never the whole bus (10k events made every
+  // find a 10k scan). Sets keep insertion order and delete in O(1), which the eviction needs.
+  const byName = new Map<string, Set<JobEvent>>();
+  /** No stored event expires before this, so a publish before it has nothing to sweep. */
+  let nextExpiry = Number.POSITIVE_INFINITY;
+  let lastSweptAt = Number.NEGATIVE_INFINITY;
+
+  const forget = (event: JobEvent): void => {
+    events.delete(event.id);
+    const named = byName.get(event.name);
+    named?.delete(event);
+    if (named?.size === 0) byName.delete(event.name);
+  };
 
   const purgeExpired = (): number => {
     const at = nowMs(clock);
     let removed = 0;
-    for (const [id, event] of events) {
+    nextExpiry = Number.POSITIVE_INFINITY;
+    for (const event of events.values()) {
       if (event.expiresAt <= at) {
-        events.delete(id);
+        forget(event);
         removed += 1;
-      }
+      } else if (event.expiresAt < nextExpiry) nextExpiry = event.expiresAt;
     }
+    lastSweptAt = at;
     return removed;
   };
 
@@ -80,8 +101,11 @@ export function memoryEventBus(options: MemoryEventBusOptions = {}): EventBus {
     stored: false,
     now: () => Promise.resolve(nowMs(clock)),
     publish(name, payload, publishOptions = {}) {
-      purgeExpired();
       const at = nowMs(clock);
+      // A sweep is a pass over every event, so a publish pays for one only when something has
+      // expired AND the last one is a second old: a sweep per publish made a 10k fill 916 ms.
+      // `find` filters the expired either way; this only bounds the heap.
+      if (at >= nextExpiry && at - lastSweptAt >= SWEEP_EVERY_MS) purgeExpired();
       const event: JobEvent = {
         id: uuidV7(),
         name,
@@ -97,10 +121,14 @@ export function memoryEventBus(options: MemoryEventBusOptions = {}): EventBus {
           : { correlationKey: publishOptions.correlationKey }),
       };
       events.set(event.id, event);
+      const named = byName.get(name);
+      if (named === undefined) byName.set(name, new Set([event]));
+      else named.add(event);
+      if (event.expiresAt < nextExpiry) nextExpiry = event.expiresAt;
       while (events.size > maxEvents) {
-        const oldest = events.keys().next();
+        const oldest = events.values().next();
         if (oldest.done === true) break;
-        events.delete(oldest.value);
+        forget(oldest.value);
       }
       logger.debug('jobs.event.published', {
         event: name,
@@ -110,27 +138,23 @@ export function memoryEventBus(options: MemoryEventBusOptions = {}): EventBus {
     },
 
     /**
-     * Earliest matching event at or after `afterMs`, so a resumed step consumes events in
-     * publication order rather than jumping to the newest one. Ties in one ms go to the earlier
-     * publish (the Map's insertion order), and `afterId` seeks past that event in the same order.
+     * Earliest matching event past the cursor, in `(publishedAt, id)` order (`events-cursor.ts`),
+     * so a resumed step consumes events in publication order rather than jumping to the newest
+     * one — the rule `SQL_EVENT_FIND` states. The cursor is compared, never looked up: a refused
+     * event that has since been purged still excludes everything at or before it.
      */
-    find(name, correlationKey, afterMs, afterId) {
+    find(name, correlationKey, afterMs, afterId, _horizon, since) {
       const at = nowMs(clock);
-      const cursor = afterId === undefined ? undefined : events.get(afterId);
-      // Until the cursor's own event is passed, an event in its millisecond came before it.
-      let passed = cursor === undefined;
+      // The wait's own line; no horizon here — a memory publish is visible as it is stamped.
+      const from = since?.at ?? afterMs;
+      const cursor = afterId === undefined ? { afterMs } : { afterMs, afterId };
       let best: JobEvent | undefined;
-      for (const event of events.values()) {
-        if (event === cursor) {
-          passed = true;
-          continue;
-        }
-        if (!passed && event.publishedAt <= (cursor?.publishedAt ?? 0)) continue;
-        if (event.name !== name) continue;
+      for (const event of byName.get(name) ?? []) {
         if (event.expiresAt <= at) continue;
-        if (event.publishedAt < afterMs) continue;
+        if (event.publishedAt < from || !pastCursor(event, cursor)) continue;
+        if (since?.refused?.includes(event.id) === true) continue;
         if (correlationKey !== undefined && event.correlationKey !== correlationKey) continue;
-        if (best === undefined || event.publishedAt < best.publishedAt) best = event;
+        if (best === undefined || eventBefore(event, best)) best = event;
       }
       return Promise.resolve(
         best === undefined
@@ -163,21 +187,24 @@ export function eventsPurgeTarget(bus: EventBus): PurgeTarget {
   return { name: EVENTS_PURGE_TARGET, purgeExpired: () => bus.purgeExpired() };
 }
 
-let ambientBus: EventBus = memoryEventBus();
+/** The process default under every installed bus — replaced only by `freshDefaultEventBus()`. */
+let base: EventBus = memoryEventBus();
+/** The installed bus: a stack over `base`, so boots and tests release in any order. */
+const slot = adapterSlot<EventBus>(() => base);
 
 /**
- * **Install `postgresEventBus({ executor })` here in any deployment with more than one process.**
- * The default above is one process's heap: the pod that publishes and the pod that resumes are
- * never the same one, so a webhook landing on web-3 strands a run on worker-7 until its 24h
- * timeout dead-letters it, with nothing logged before then. This line used to promise a
- * "NATS/Redis-streams bus swapped at boot"; no such bus existed and boot installed the memory one.
+ * The boot's installer (its release removes THIS install only, in any order — `adapterSlot`), on `@ultimat3/jobs/adapter` and never the `.` barrel: the boot installs
+ * `postgresEventBus({ executor })` over its pool, and an app swaps it with `eventBus:` in
+ * `apps/<app>/runtime.ts`. The default above is one process's heap: the pod that publishes and the
+ * pod that resumes are never the same one, so a webhook landing on web-3 strands a run on worker-7
+ * until its 24h timeout dead-letters it, with nothing logged before then.
  */
-export function setEventBus(bus: EventBus): void {
-  ambientBus = bus;
+export function installEventBus(bus: EventBus): () => void {
+  return slot.install(bus);
 }
 
 export function eventBus(): EventBus {
-  return ambientBus;
+  return slot.current();
 }
 
 /**
@@ -187,7 +214,16 @@ export function eventBus(): EventBus {
  * wait before that test has asked anything — call this between tests.
  */
 export function resetEventBus(): void {
-  ambientBus = memoryEventBus();
+  slot.reset();
+  base = memoryEventBus();
+}
+
+/**
+ * The per-test reset: a FRESH default bus under whatever is installed, which stays installed — a
+ * `beforeAll` swap survives every test while a test on the default still starts with no events.
+ */
+export function freshDefaultEventBus(): void {
+  base = memoryEventBus();
 }
 
 /** The one function app code calls to unblock a waiting step. */
@@ -196,5 +232,5 @@ export function publishEvent(
   payload: unknown,
   options?: PublishOptions,
 ): Promise<JobEvent> {
-  return ambientBus.publish(name, payload, options);
+  return slot.current().publish(name, payload, options);
 }

package/src/export.ts

@@ -16,7 +16,7 @@
 // and `backfill-pass.ts` already have.
 
 import type { Clock, Ctx } from '@ultimat3/core';
-import { assert, systemClock } from '@ultimat3/core';
+import { assert, liveClock } from '@ultimat3/core';
 import type { ReadBuilder } from '@ultimat3/entity';
 import type { StandardSchemaV1 } from '@ultimat3/schema';
 import type { Pacer } from './backfill-rate';
@@ -171,7 +171,7 @@ export function exportRows<Row, I>(definition: ExportDefinition<Row, I>): JobHan
     `export "${definition.name}" declares ${definition.columns.length} column(s) and ${new Set(definition.columns).size} distinct one(s)`,
     `declare at least one column on exportRows("${definition.name}") and no duplicates — the list is the csv header and the ndjson key order, and a repeat writes the same cell twice`,
   );
-  const clock = definition.clock ?? systemClock;
+  const clock = definition.clock ?? liveClock;
   // Built at declaration for the reason `backfill()` builds its own there: the interval belongs to
   // the table and the pool, not to whichever attempt holds the run.
   const pace =

package/src/gdpr-audit.ts

@@ -0,0 +1,75 @@
+// The erase of a person is a sensitive mutation, so `subjectErase()` writes it to the `auditSink`
+// port when one is installed: an `allowed` record per completed erase and a `failed` one per
+// attempt that threw. The input is the subject's entity and the run id — never the key, which an
+// app may make an email. At least once: `idempotencyKey` (`<job>:<runId>`) is what a sink dedupes on.
+
+import type { AuditRecord, Ctx } from '@ultimat3/core';
+import { AUDIT_SINK_FAILED_EVENT, auditFailureOf, getAuditSink, logger } from '@ultimat3/core';
+import type { StepApi } from './steps-api';
+
+export interface EraseAuditInput {
+  /** The job's durable name: the record's `name`. */
+  readonly job: string;
+  /** The subject's ENTITY name — which kind of person, never which one. */
+  readonly subject: string;
+  /** The run: what an auditor joins to `x jobs show`, and the sink's dedupe key. */
+  readonly runId: string;
+  readonly ctx: Ctx;
+}
+
+/**
+ * A job ran the erase — no action did — so `primitive: 'job'` on `surface: 'job'`. An `agentJob`
+ * write differs on purpose: there an action ran, so it records `'action'` on surface `'job'`.
+ */
+const recordOf = (input: EraseAuditInput, error: unknown, failed: boolean): AuditRecord => ({
+  at: input.ctx.now(),
+  name: input.job,
+  primitive: 'job',
+  mutator: false,
+  surface: 'job',
+  ctx: input.ctx,
+  input: { subject: input.subject, run: input.runId },
+  idempotencyKey: `${input.job}:${input.runId}`,
+  replayed: false,
+  outcome: failed ? 'failed' : 'allowed',
+  failure: failed ? auditFailureOf(error) : null,
+});
+
+/** Run `erase`, then record it: `allowed` once it completed, `failed` for an attempt that threw. */
+export async function auditedErase<R>(
+  input: EraseAuditInput,
+  step: StepApi,
+  erase: () => Promise<R>,
+): Promise<R> {
+  let report: R;
+  try {
+    report = await erase();
+  } catch (error: unknown) {
+    // The erase's own error wins; a sink that also refuses is logged, as action's gate logs it.
+    const record = recordOf(input, error, true);
+    await Promise.resolve()
+      .then(() => getAuditSink()?.write(record))
+      .catch((sinkError: unknown) => {
+        logger.error(AUDIT_SINK_FAILED_EVENT, {
+          action: record.name,
+          outcome: record.outcome,
+          error: sinkError,
+        });
+      });
+    throw error;
+  }
+  const sink = getAuditSink();
+  // Its own step, so a replay after it landed writes nothing; a write that landed and then lost its
+  // checkpoint is written again under the same `idempotencyKey`. A sink that refuses on EVERY
+  // attempt can hold no record at all — the dead-lettered run (`x jobs show`) is the trail, and the
+  // refusal is logged here as action's audit gate logs it.
+  if (sink !== null) {
+    await step
+      .run('audit', async () => sink.write(recordOf(input, null, false)))
+      .catch((error: unknown) => {
+        logger.error(AUDIT_SINK_FAILED_EVENT, { action: input.job, outcome: 'allowed', error });
+        throw error;
+      });
+  }
+  return report;
+}

package/src/gdpr-coverage.ts

@@ -0,0 +1,94 @@
+// What `personalData()` (entity's marker) adds to `defineSubject()`: proof the declared subjects
+// cover every marked column. A subject's plan says WHICH tables it reaches and how each is treated;
+// this says whether that treatment clears the marked columns, and whether every marked entity is
+// reached at all. Pure — plans and marked entities in, refusals out — so a run and a check agree.
+
+import type { PersonalDataEntity } from '@ultimat3/entity';
+import { personalDataEntities, registeredEntities } from '@ultimat3/entity';
+import { PersonalDataUnerasedError, PersonalDataUnreachedError } from './gdpr-errors';
+import type { Subject } from './subject';
+import { registeredSubjects } from './subject';
+import type { SubjectPlan } from './subject-plan';
+import { planSubject } from './subject-plan';
+
+const markedOf = (
+  marked: readonly PersonalDataEntity[],
+  table: string,
+): PersonalDataEntity | undefined => marked.find((each) => each.entity.$name === table);
+
+/**
+ * Every reached table whose treatment would leave a marked column: `'delete'` clears the row,
+ * `{ retain }` is a stated reason (printed in the export's index), and `{ anonymize }` must name
+ * EVERY marked property — a value it overwrites is no longer the person's.
+ */
+export function unerasedPersonalData(
+  plan: SubjectPlan,
+  marked: readonly PersonalDataEntity[] = personalDataEntities(),
+): readonly PersonalDataUnerasedError[] {
+  return plan.steps.flatMap((step) => {
+    const entry = markedOf(marked, step.entity);
+    const { treatment } = step;
+    if (entry === undefined || typeof treatment !== 'object' || !('anonymize' in treatment)) {
+      return [];
+    }
+    const left = entry.columns
+      .map((column) => column.property)
+      .filter((property) => !Object.hasOwn(treatment.anonymize, property));
+    return left.length === 0
+      ? []
+      : [
+          new PersonalDataUnerasedError({
+            subject: plan.subject,
+            table: step.entity,
+            columns: left,
+          }),
+        ];
+  });
+}
+
+/** Every marked entity no plan reaches, in the registry's name order. */
+export function unreachedPersonalData(
+  plans: readonly SubjectPlan[],
+  marked: readonly PersonalDataEntity[] = personalDataEntities(),
+): readonly PersonalDataUnreachedError[] {
+  const reached = new Set(plans.flatMap((plan) => plan.steps.map((step) => step.entity)));
+  const subjects = plans.map((plan) => plan.subject);
+  return marked
+    .filter((entry) => !reached.has(entry.entity.$name))
+    .map(
+      (entry) =>
+        new PersonalDataUnreachedError({
+          table: entry.entity.$name,
+          columns: entry.columns.map((column) => column.property),
+          subjects,
+        }),
+    );
+}
+
+/**
+ * The whole check over this process's declarations — what `x exercise` lists, once every entity
+ * module has loaded. A subject whose own plan is refused (`X_SUBJECT_*`) raises that refusal first:
+ * coverage of an unplannable subject is not a question with an answer.
+ */
+export function personalDataProblems(
+  subjects: readonly Subject<never>[] = registeredSubjects(),
+  marked: readonly PersonalDataEntity[] = personalDataEntities(),
+): readonly (PersonalDataUnerasedError | PersonalDataUnreachedError)[] {
+  // `planSubject` and not `subject-jobs.ts`'s `plannedSubject`: that one is where a run asks
+  // `unerasedPersonalData`, and importing it back would be a cycle.
+  const plans = subjects.map((subject) => {
+    const { plan, problems } = planSubject({
+      subject: subject.entity,
+      tables: subject.init.tables,
+      columns: subject.init.columns,
+      entries: registeredEntities(),
+    });
+    const [first] = problems;
+    if (first !== undefined) throw first;
+    return plan;
+  });
+  return [
+    ...plans.flatMap((plan) => unerasedPersonalData(plan, marked)),
+    ...unreachedPersonalData(plans, marked),
+  ];
+}

package/src/gdpr-errors.ts

@@ -0,0 +1,35 @@
+// The two `X_PERSONAL_DATA_*` refusals: a subject's erase that would leave a `personalData()` column
+// standing, and a marked entity no subject reaches. Both are refusals of DECLARATIONS — the values
+// printed are entity and column names, never a person's data.
+
+import { UltimateError } from '@ultimat3/core';
+import './error-codes';
+
+/** A reached table's `{ anonymize }` does not name every marked column of it. */
+export class PersonalDataUnerasedError extends UltimateError {
+  constructor(input: { subject: string; table: string; columns: readonly string[] }) {
+    const named = input.columns.map((column) => `${column}: null`).join(', ');
+    super({
+      code: 'X_PERSONAL_DATA_UNERASED',
+      cause: `defineSubject(${input.subject}) anonymizes ${input.table} without ${input.columns.join(', ')}, marked personalData() — an erase would leave the person's value in them`,
+      fix: `tables: { ${input.table}: { anonymize: { ${named} } } }   # in defineSubject(${input.subject}, …), merged into the existing anonymize — or 'delete', or { retain: '<why it is kept>' }`,
+      meta: { subject: input.subject, table: input.table, columns: [...input.columns] },
+    });
+  }
+}
+
+/** No declared subject's walk reaches an entity carrying marked columns. */
+export class PersonalDataUnreachedError extends UltimateError {
+  constructor(input: { table: string; columns: readonly string[]; subjects: readonly string[] }) {
+    const by =
+      input.subjects.length === 0
+        ? 'no subject is declared'
+        : `declared: ${input.subjects.join(', ')}`;
+    super({
+      code: 'X_PERSONAL_DATA_UNREACHED',
+      cause: `${input.table} marks ${input.columns.join(', ')} personalData() and no defineSubject() walk reaches it (${by}) — no export or erase would ever see them`,
+      fix: `defineSubject(${input.table}, { driver, key, tables: { ${input.table}: 'delete' } })   # in the module declaring subjects — or a reference to a subject's entity, or columns: { ${input.table}: '<key column>' } on an existing defineSubject`,
+      meta: { table: input.table, columns: [...input.columns], subjects: [...input.subjects] },
+    });
+  }
+}

package/src/import.ts

@@ -9,7 +9,7 @@
 // what, and who may import, never do. The pass lives in `import-pass.ts`.
 
 import type { Clock, Ctx } from '@ultimat3/core';
-import { assert, systemClock } from '@ultimat3/core';
+import { assert, liveClock } from '@ultimat3/core';
 import type { StandardSchemaV1 } from '@ultimat3/schema';
 import type { Pacer } from './backfill-rate';
 import { backfillPacer } from './backfill-rate';
@@ -146,7 +146,7 @@ export function importRows<Rec, I>(definition: ImportDefinition<Rec, I>): JobHan
     `import "${name}" declares ${definition.columns.length} column(s) and ${new Set(definition.columns).size} distinct one(s)`,
     `declare at least one column on importRows("${name}") and no duplicates — the list is what a csv header must name`,
   );
-  const clock = definition.clock ?? systemClock;
+  const clock = definition.clock ?? liveClock;
   const pace =
     definition.rate === undefined
       ? undefined