@ultimat3/jobs 29.3.2 → 30.0.0

package/src/import.ts

@@ -0,0 +1,175 @@
+// `importRows()` — a file in object storage read a page at a time into the app's tables, declared
+// as a `job` and NOT as a ninth primitive: the twin of `exportRows()`, a FACTORY over `job()` as
+// `backfill()`, `purge()` and `webhook()` are. That is what gives an import `.enqueue()`, the
+// worker's cancellation, the dead-letter path, `x jobs show` and a manifest row.
+//
+// WHY IT IS THE FRAMEWORK'S: the memory bound and the resume. `parse(await disk.get(key))` holds
+// the whole upload, and a retried job re-applies every row from zero. What ships is the quote-aware
+// record boundary, the byte-offset checkpoint and the per-row error report; which columns mean
+// what, and who may import, never do. The pass lives in `import-pass.ts`.
+
+import type { Clock, Ctx } from '@ultimat3/core';
+import { assert, systemClock } from '@ultimat3/core';
+import type { StandardSchemaV1 } from '@ultimat3/schema';
+import type { Pacer } from './backfill-rate';
+import { backfillPacer } from './backfill-rate';
+import type { DurationInput } from './clock';
+import type { ExportFormat } from './export-format';
+import { EXPORT_FORMATS } from './export-format';
+import type { ExportSink } from './export-sink';
+import { importPass } from './import-pass';
+import type { ImportSource } from './import-source';
+import type { JobHandle } from './job';
+import { job } from './job';
+import type { RetryPolicy } from './retry';
+import { DEFAULT_RETRY } from './retry';
+import type { JobTenant } from './tenant';
+
+/** Records per `handle` call: one statement's worth. A page hands over its whole read in these. */
+export const DEFAULT_IMPORT_BATCH = 500;
+/** Bytes one page reads. The heap bound: a window is held, never the file. */
+export const DEFAULT_IMPORT_PAGE_BYTES = 1_024 * 1_024;
+/** The longest one record may be. Must stay below `pageBytes`, so every window holds a record. */
+export const DEFAULT_IMPORT_MAX_RECORD_BYTES = 64 * 1_024;
+/** Refused rows past which the import ends failed. `0` stops at the first bad page. */
+export const DEFAULT_IMPORT_MAX_ROW_ERRORS = 1_000;
+
+export interface ImportDefinition<Rec, I> {
+  /** REQUIRED, as on `exportRows()`: the queue row, the step trace and every page carry it. */
+  readonly name: string;
+  /** The payload: a pointer to the file (`{ orgId, uploadKey }`), never its rows. */
+  readonly input: StandardSchemaV1<unknown, I>;
+  /**
+   * REQUIRED, exactly as on `job()`. `handle` runs under it, so a scoped write in a `'none'` import
+   * fails closed with `X_TENANCY_ACTOR_ORG_REQUIRED` — deliberately not relaxed: an import writes
+   * one file's rows, and the file names no tenant the app did not put in `input`.
+   */
+  readonly tenant: JobTenant<NoInfer<I>>;
+  /** Where the file is — a THUNK, read per page: `source: () => disk('imports')`. */
+  readonly source: () => ImportSource;
+  /**
+   * The file's object key, from the input. `I` and `Rec` are inferred from `input` and `schema`
+   * ALONE (`NoInfer` everywhere else), so a helper typed for half the input cannot narrow it.
+   */
+  key(input: NoInfer<I>): string;
+  readonly format: ExportFormat;
+  /** The declared columns. A csv header must name exactly these; ndjson keys must be among them. */
+  readonly columns: readonly string[];
+  /**
+   * Each record, keyed by column, through this. A csv record is text, coerced as an HTTP form is
+   * (`coerceInput`): an unquoted empty cell is `null`, a quoted `""` is `''`. No formula guard is
+   * undone: an import cannot know a leading `'` was the exporter's.
+   */
+  readonly schema: StandardSchemaV1<unknown, Rec>;
+  /**
+   * One page of accepted rows. AT LEAST ONCE per page: a page that dies after `handle` and before
+   * its checkpoint runs again. `upsertAll` is the idempotent spelling, as for `backfill()`.
+   */
+  handle(args: {
+    readonly rows: readonly NoInfer<Rec>[];
+    readonly ctx: Ctx;
+    readonly input: NoInfer<I>;
+  }): unknown;
+  /** Where refused rows are reported: `<prefix>/<runId>/errors-00000.ndjson`, one part per page. */
+  readonly errors: { readonly sink: () => ExportSink; prefix(input: NoInfer<I>): string };
+  /** Records per `handle` call, never per read. Defaults to `DEFAULT_IMPORT_BATCH`. */
+  readonly batch?: number;
+  /** Defaults to `DEFAULT_IMPORT_PAGE_BYTES`. */
+  readonly pageBytes?: number;
+  /** Defaults to `DEFAULT_IMPORT_MAX_RECORD_BYTES`; below `pageBytes`. */
+  readonly maxRecordBytes?: number;
+  /** Defaults to `DEFAULT_IMPORT_MAX_ROW_ERRORS`. */
+  readonly maxRowErrors?: number;
+  /** Pages per second. No default, as an export's: omit it for unthrottled. */
+  readonly rate?: number;
+  readonly clock?: Clock;
+  readonly queue?: string;
+  readonly retry?: RetryPolicy;
+  /** Per attempt: a resumed attempt picks up at the last checkpointed byte offset. */
+  readonly timeout?: DurationInput;
+}
+
+/** What one completed pass reports — bounded, so `x jobs show` can print it. */
+export interface ImportReport {
+  readonly name: string;
+  readonly key: string;
+  /** Where this run's refused rows are: `<errors.prefix>/<runId>`, the parts under it. */
+  readonly errorPrefix: string;
+  readonly rows: number;
+  readonly rejected: number;
+  /** Error parts written. */
+  readonly parts: number;
+  readonly bytes: number;
+}
+
+/** Everything fixed at declaration, validated once. Read by `importPass`. */
+export interface ImportPlan<Rec, I> {
+  readonly definition: ImportDefinition<Rec, I>;
+  readonly batch: number;
+  readonly pageBytes: number;
+  readonly maxRecordBytes: number;
+  readonly maxRowErrors: number;
+  readonly pace: Pacer | undefined;
+}
+
+export function importRows<Rec, I>(definition: ImportDefinition<Rec, I>): JobHandle<I> {
+  const name = definition.name;
+  const batch = definition.batch ?? DEFAULT_IMPORT_BATCH;
+  const pageBytes = definition.pageBytes ?? DEFAULT_IMPORT_PAGE_BYTES;
+  const maxRecordBytes = definition.maxRecordBytes ?? DEFAULT_IMPORT_MAX_RECORD_BYTES;
+  const maxRowErrors = definition.maxRowErrors ?? DEFAULT_IMPORT_MAX_ROW_ERRORS;
+  assert(
+    Number.isSafeInteger(batch) && batch >= 1,
+    `import "${name}" declares batch: ${String(batch)} — a batch is a whole number of records, at least one`,
+    `set batch: ${DEFAULT_IMPORT_BATCH} on importRows("${name}") — the records one page hands handle()`,
+  );
+  assert(
+    Number.isSafeInteger(maxRecordBytes) &&
+      Number.isSafeInteger(pageBytes) &&
+      maxRecordBytes >= 1 &&
+      maxRecordBytes < pageBytes,
+    `import "${name}" declares pageBytes: ${String(pageBytes)} and maxRecordBytes: ${String(maxRecordBytes)} — a page must be longer than the longest record, or a record can fill a window with no end in it`,
+    `set pageBytes: ${DEFAULT_IMPORT_PAGE_BYTES} and maxRecordBytes: ${DEFAULT_IMPORT_MAX_RECORD_BYTES} on importRows("${name}"), or leave both out`,
+  );
+  assert(
+    Number.isSafeInteger(maxRowErrors) && maxRowErrors >= 0,
+    `import "${name}" declares maxRowErrors: ${String(maxRowErrors)}`,
+    `set maxRowErrors: ${DEFAULT_IMPORT_MAX_ROW_ERRORS} on importRows("${name}") — a whole number, 0 to stop at the first bad page`,
+  );
+  assert(
+    (EXPORT_FORMATS as readonly string[]).includes(definition.format),
+    `import "${name}" declares format: ${String(definition.format)}`,
+    `set format to one of ${EXPORT_FORMATS.join(' | ')} on importRows("${name}")`,
+  );
+  assert(
+    definition.columns.length > 0 && new Set(definition.columns).size === definition.columns.length,
+    `import "${name}" declares ${definition.columns.length} column(s) and ${new Set(definition.columns).size} distinct one(s)`,
+    `declare at least one column on importRows("${name}") and no duplicates — the list is what a csv header must name`,
+  );
+  const clock = definition.clock ?? systemClock;
+  const pace =
+    definition.rate === undefined
+      ? undefined
+      : backfillPacer({ rate: definition.rate, job: name, clock });
+  const plan: ImportPlan<Rec, I> = {
+    definition,
+    batch,
+    pageBytes,
+    maxRecordBytes,
+    maxRowErrors,
+    pace,
+  };
+
+  return job<I>({
+    name,
+    input: definition.input,
+    // ONE live pass per file, derived rather than declared, as an export's per prefix: two passes
+    // over one file would hand every row to `handle` twice and race over every error part.
+    idempotencyKey: (input) => `${name}:${definition.key(input)}`,
+    tenant: definition.tenant,
+    retry: definition.retry ?? DEFAULT_RETRY,
+    ...(definition.queue === undefined ? {} : { queue: definition.queue }),
+    ...(definition.timeout === undefined ? {} : { timeout: definition.timeout }),
+    run: (args) => importPass<Rec, I>(plan, args),
+  });
+}

package/src/index.ts

@@ -18,7 +18,11 @@ export type {
   BackfillBatch,
   BackfillDefinition,
   BackfillInput,
+  BackfillPage,
+  BackfillPages,
   BackfillReport,
+  ScopedBackfillDefinition,
+  ScopedBackfillInput,
 } from './backfill';
 export { backfill, DEFAULT_BACKFILL_BATCH } from './backfill';
 export {
@@ -27,6 +31,7 @@ export {
   BackfillMigrationPendingError,
   BackfillPendingError,
   BackfillRunningError,
+  BackfillSourceInvalidError,
   BackfillStalledError,
   BackfillUnknownError,
 } from './backfill-errors';
@@ -75,6 +80,29 @@ export type { AnnounceExhaustedOptions, ExhaustedCounts } from './claim-exhauste
 export { announceExhausted } from './claim-exhausted';
 export type { JobConcurrency, KeyedConcurrency, WhenBusy } from './concurrency';
 export { MAX_CONCURRENCY_KEY_LENGTH, WHEN_BUSY } from './concurrency';
+export { consumer } from './consumer';
+export { ConsumerInvalidError, ConsumerSelfEnqueueError } from './consumer-errors';
+export {
+  CONSUMER_LINGER_MS,
+  CONSUMER_PROGRESS_NOTE,
+  DEFAULT_CONSUMER_BATCH,
+} from './consumer-loop';
+// `stampConsumer` is deliberately absent, for the reason `stampBackfill` is.
+export type { ConsumerDeclaration, ConsumerRow } from './consumer-registry';
+export {
+  consumerDeclarationOf,
+  consumerRow,
+  isConsumer,
+  registeredConsumers,
+} from './consumer-registry';
+export type {
+  ConsumerBatch,
+  ConsumerCursor,
+  ConsumerDefinition,
+  ConsumerHandle,
+  ConsumerInput,
+  ConsumerReport,
+} from './consumer-types';
 export type { JobConcurrencyDescriptor, JobDescriptor } from './describe';
 export type {
   AckOptions,
@@ -89,6 +117,7 @@ export type {
   JobRecord,
   JobState,
   NackOptions,
+  QueueDemand,
   QueueStats,
   SettleBy,
 } from './driver';
@@ -116,18 +145,18 @@ export { postgresJobDriver, postgresLeader } from './driver-pg';
 // too — the array-parameter proofs are `driver-pg-array.live.test.ts` (25.0.0).
 export { SQL_JOBS_TABLE } from './driver-pg-sql';
 export { signalEnqueued, signalStaged } from './enqueue-signal';
-export type { JobErrorCode } from './errors';
+export type { JobErrorCode } from './error-codes';
+export { JOB_ERROR_CODES, JOB_ERROR_TITLES } from './error-codes';
 export {
   ActionJobUnbridgedError,
   CancelUnsupportedError,
   ClaimQueuesEmptyError,
   DriverUnavailableError,
   IdempotencyRequiredError,
-  JOB_ERROR_CODES,
-  JOB_ERROR_TITLES,
   JobAbortedError,
   JobDrainedError,
   JobDuplicateError,
+  JobEnqueueBatchTooLargeError,
   JobMaxAttemptsError,
   JobNameTakenError,
   JobNotCancellableError,
@@ -153,6 +182,16 @@ export {
   JobPageInvalidError,
 } from './errors-operator';
 export { JobNotFoundError, JobNotRequeueableError } from './errors-requeue';
+export {
+  COMPENSATE_PREFIX,
+  JobCompensationFailedError,
+  StepCompensateUndeclaredError,
+  StepNameReservedError,
+  StepNotResolvableError,
+  StepOutcomeUnknownError,
+  UNWINDING_STEP,
+} from './errors-saga';
+export { JobEventInvalidError, StepOutputUnserializableError } from './errors-steps';
 export { JobTenantMismatchError } from './errors-tenant';
 export type { EventBus, JobEvent, MemoryEventBusOptions, PublishOptions } from './events';
 export {
@@ -180,7 +219,28 @@ export {
   exportPartKey,
   memoryExportSink,
 } from './export-sink';
+export type { CallerFailure } from './failure-row';
+export { callerFailure } from './failure-row';
 export { IDLE_POLL_CEILING_MS } from './idle-backoff';
+export type { ImportDefinition, ImportReport } from './import';
+export {
+  DEFAULT_IMPORT_BATCH,
+  DEFAULT_IMPORT_MAX_RECORD_BYTES,
+  DEFAULT_IMPORT_MAX_ROW_ERRORS,
+  DEFAULT_IMPORT_PAGE_BYTES,
+  importRows,
+} from './import';
+export {
+  ImportHeaderMismatchError,
+  ImportRecordTooLargeError,
+  ImportTooManyErrorsError,
+} from './import-errors';
+export type { ImportRejection } from './import-format';
+export { IMPORT_REJECTIONS } from './import-format';
+export type { ImportRowError } from './import-pass';
+export { importErrorKey } from './import-pass';
+export type { ImportObject, ImportSource, MemoryImportSource } from './import-source';
+export { memoryImportSource } from './import-source';
 export type {
   DeadLetterEntry,
   JobsManifest,
@@ -197,7 +257,13 @@ export {
   inspectQueues,
   retryFromStep,
 } from './inspect';
-export { pauseQueue, promoteJob, removeJob, resumeQueue } from './inspect-operator';
+export {
+  pauseQueue,
+  promoteJob,
+  removeJob,
+  resolveJobStep,
+  resumeQueue,
+} from './inspect-operator';
 export type {
   BulkFilter,
   BulkResult,
@@ -209,6 +275,7 @@ export type {
   JobIntrospection,
   JobProgress,
   PausedName,
+  StepResolution,
   TaskFire,
   WorkerAnnouncement,
   WorkerRecord,
@@ -235,6 +302,8 @@ export type {
   JobRunArgs,
 } from './job';
 export { describeJobs, getJob, isJobHandle, job, registeredJobs, resetJobs } from './job';
+export type { EnqueueManyOptions } from './job-enqueue-many';
+export { MAX_ENQUEUE_BATCH } from './job-enqueue-many';
 export type { JobsRetentionOptions } from './jobs-retention';
 export { jobsPurgeTarget } from './jobs-retention';
 export type { HeldLease, LeaseStore, MemoryLeaseStore, MemoryLeaseStoreOptions } from './leases';
@@ -251,8 +320,10 @@ export type {
 export { concurrencyLimiter, NO_TENANT, tenantKeyFrom } from './limits';
 export {
   queueDeadJobs,
+  queueDemand,
   queueOldestReady,
   recordQueueDeadJobs,
+  recordQueueDemand,
   recordQueueOldestReady,
 } from './metrics';
 export type {
@@ -265,7 +336,6 @@ export type {
   OutboxStore,
 } from './outbox';
 export {
-  enqueueInTx,
   jobsFacade,
   memoryOutboxStore,
   outboxJobsFacade,
@@ -291,13 +361,40 @@ export type {
 } from './purge';
 export { DEFAULT_PURGE_CRON, purge } from './purge';
 export type { PurgeBatching } from './purge-batches';
+export type { PurgeEntities } from './purge-entities';
+export {
+  resetPurgeCoverage,
+  retainedEntities,
+  retainTarget,
+  unsweptEntities,
+} from './purge-entities';
 export type { PgListener, QueueWake, QueueWakeOptions } from './queue-wake';
 export { startQueueWake } from './queue-wake';
+export type { RefreshViewDefinition, RefreshViewReport } from './refresh-view';
+export { refreshView } from './refresh-view';
 export type { IntervalScheduler } from './renewal-timer';
 export type { BackoffStrategy, Random, RetryDecision, RetryPolicy } from './retry';
-export { backoffDelayMs, DEFAULT_RETRY, isFinalAttempt, nextRetry, retrySchedule } from './retry';
+export {
+  backoffDelayMs,
+  DEFAULT_RETRY,
+  enqueuedAttempts,
+  isFinalAttempt,
+  isUnwindAttempt,
+  nextRetry,
+  retrySchedule,
+  UNWIND_MARGIN,
+} from './retry';
 export type { JobRetryDecision, JobStopReason } from './retry-classification';
 export { failureForRow, nextRetryForError } from './retry-classification';
+export type {
+  RollupBucket,
+  RollupDefinition,
+  RollupInput,
+  RollupReport,
+  RollupTarget,
+} from './rollup';
+export { rollup } from './rollup';
+export { RefreshViewInvalidError, RollupInvalidError } from './rollup-errors';
 export type {
   CronResolver,
   DispatchedOccurrence,
@@ -323,27 +420,65 @@ export {
 export type { ScheduledFire, SchedulerState } from './scheduler-state';
 export { fireThroughDriver, memorySchedulerState } from './scheduler-state';
 export type { JobCompleted, JobFailed, JobSettled } from './settled';
-export { ON_SETTLED_ATTEMPTS } from './settled';
+export { ON_SETTLED_ATTEMPTS, ON_SETTLED_TIMEOUT_MS } from './settled';
 export type {
+  CompensateContext,
   EventLookup,
   StepApi,
   StepFence,
   StepRecord,
   StepRunner,
   StepRunnerOptions,
+  StepRunOptions,
   StepStatus,
   StepStore,
   WaitForEventOptions,
 } from './steps';
 export {
+  DEFAULT_EVENT_POLL_MS,
   isStepStatus,
   isStepSuspension,
   MAX_TRACE_NAMES,
   STEP_STATUSES,
   StepSuspension,
   stepRunner,
+  WAKING_EVENT_POLL_MS,
 } from './steps';
 export { memoryStepStore } from './steps-memory';
+export type { StepOutput } from './steps-output';
+export type {
+  RetainedTarget,
+  Subject,
+  SubjectInit,
+  SubjectRef,
+  SubjectTarget,
+  SubjectTargetEntry,
+  SubjectTargetPage,
+  SubjectTreatment,
+} from './subject';
+export { DEFAULT_SUBJECT_BATCH, defineSubject, registeredSubjects, resetSubjects } from './subject';
+export type { SubjectEraseReport, SubjectTally } from './subject-erase-pass';
+export {
+  SubjectCycleError,
+  SubjectTableUndeclaredError,
+  SubjectTreatmentInvalidError,
+} from './subject-errors';
+export type { SubjectExportReport, SubjectIndexEntry } from './subject-export-pass';
+export type { SubjectExportDefinition, SubjectJobDefinition } from './subject-jobs';
+export {
+  plannedSubject,
+  subjectErase,
+  subjectEraseTables,
+  subjectExport,
+} from './subject-jobs';
+export type {
+  SubjectPlan,
+  SubjectPlanInput,
+  SubjectStep,
+  SubjectUnlink,
+  SubjectVia,
+} from './subject-plan';
+export { planSubject } from './subject-plan';
 export type {
   CatchUpPolicy,
   TaskDefinition,
@@ -384,5 +519,26 @@ export {
 } from './webhook-errors';
 export type { MemoryWebhookLedger, WebhookAttempt, WebhookLedger } from './webhook-ledger';
 export { DEFAULT_MAX_WEBHOOK_ATTEMPTS, memoryWebhookLedger } from './webhook-ledger';
+export { webhookReceiver, webhookReceiverRoutes } from './webhook-receiver';
+export {
+  WEBHOOK_EVENT_ID_PATTERN,
+  WEBHOOK_RECEIVER_RETRY_AFTER_SECONDS,
+  WebhookEventIdInvalidError,
+  WebhookReceiverInvalidError,
+  WebhookReceiverUnavailableError,
+} from './webhook-receiver-errors';
+export type {
+  ReceivedEvent,
+  ReceivedEventFacts,
+  WebhookAnswer,
+  WebhookHandleArgs,
+  WebhookInbound,
+  WebhookReceiver,
+  WebhookReceiverDefinition,
+  WebhookReceiverInput,
+  WebhookReceiverTenant,
+} from './webhook-receiver-types';
 export type { Worker, WorkerOptions, WorkerStats } from './worker';
 export { jobWorker } from './worker';
+export type { CutoffTimes } from './worker-drain-cutoff';
+export { cutoffTimes, DRAIN_CANCEL_MARGIN_MS } from './worker-drain-cutoff';

package/src/inspect-operator.ts

@@ -1,12 +1,13 @@
-// The operator verbs `x jobs` and a dashboard bind to: remove, promote, pause, resume. Each one
-// refuses LOUDLY rather than answering "nothing happened" — an operator clearing a queue has to
+// The operator verbs `x jobs` and a dashboard bind to: remove, promote, pause, resume, resolve a
+// step. Each one refuses LOUDLY rather than answering "nothing happened" — an operator has to
 // know whether the job is gone or was never theirs to remove. Apart from `inspect.ts`, which is
 // the read side.
 
 import { NotImplementedError } from '@ultimat3/core';
 import type { JobDriver, JobRecord } from './driver';
 import { JobNotPromotableError } from './errors-operator';
-import type { JobIntrospection, PausedName } from './introspection';
+import type { JobIntrospection, PausedName, StepResolution } from './introspection';
+import type { StepRecord } from './steps-store';
 
 function operatorOf(driver: JobDriver): JobIntrospection {
   if (driver.introspect === undefined) {
@@ -47,3 +48,23 @@ export async function resumeQueue(
   await operator.resumeQueue(queue);
   return operator.pausedQueues();
 }
+
+/**
+ * A person's answer to `X_STEP_OUTCOME_UNKNOWN`: the resolved record. Then `x jobs retry <id>`.
+ * A driver with no `resolveStep` is refused as one with no operator surface at all.
+ */
+export function resolveJobStep(
+  driver: JobDriver,
+  jobId: string,
+  step: string,
+  resolution: StepResolution,
+): Promise<StepRecord> {
+  const operator = operatorOf(driver);
+  if (operator.resolveStep === undefined) {
+    throw new NotImplementedError({
+      cause: `the "${driver.name}" jobs driver keeps no step records an operator can resolve: its introspect has no resolveStep`,
+      fix: 'setJobDriver(postgresJobDriver())   # at boot: the pg and memory drivers implement resolveStep — then x jobs resolve-step <id> <step> --ran --output <json>',
+    });
+  }
+  return operator.resolveStep(jobId, step, resolution);
+}

package/src/inspect.ts

@@ -2,7 +2,8 @@
 // object so `x jobs ... --json` and the MCP tool share one shape — an agent debugging a stuck
 // queue reads exactly what the dashboard renders.
 
-import { NotImplementedError } from '@ultimat3/core';
+import { NotImplementedError, REDACTED } from '@ultimat3/core';
+import { maskSensitive } from '@ultimat3/schema';
 import type { BackfillProgress } from './backfill-inspect';
 import { backfillForRun } from './backfill-inspect';
 import type { WhenBusy } from './concurrency';
@@ -78,8 +79,10 @@ export interface JobTrace {
   /** The thrown value's stack for `lastError`, bounded. `null` when the failure carried none. */
   readonly stack: string | null;
   /**
-   * The stored payload, with every key the app declared secret (`redactKeys`) replaced. App data
-   * all the same: a surface that shows it gates it like any other read of a tenant's rows.
+   * The stored payload, with every `.sensitive()` field of the job's schema and every key the app
+   * declared secret (`redactKeys`) replaced — `[redacted]` whole for a job whose declaration is
+   * gone. App data all the same: a surface that shows it gates it like any other read of a
+   * tenant's rows.
    */
   readonly input: unknown;
   /** What the body last reported through `progress()`; `null` until it reports. */
@@ -129,6 +132,16 @@ function concurrencyKeyOf(handle: AnyJobHandle | undefined, record: JobRecord):
   }
 }
 
+/**
+ * The stored payload as an operator may read it: the job's own `.sensitive()` fields masked by its
+ * schema, THEN `redactKeys`. A job this process has not registered has no schema to say which keys
+ * are sensitive, so its input is hidden whole — never shown on the hope that none were.
+ */
+function operatorInput(handle: AnyJobHandle | undefined, record: JobRecord): unknown {
+  if (handle === undefined) return REDACTED;
+  return redactInput(maskSensitive(handle.input, record.input, REDACTED));
+}
+
 function toStepTrace(record: StepRecord): StepTrace {
   return {
     name: record.name,
@@ -160,7 +173,7 @@ export async function inspectJob(driver: JobDriver, jobId: string): Promise<JobT
     runAt: new Date(record.runAt).toISOString(),
     lastError: record.lastError ?? null,
     stack: record.lastErrorStack ?? null,
-    input: redactInput(record.input),
+    input: operatorInput(handle, record),
     progress: record.progress ?? null,
     tenantId: record.tenantId ?? null,
     traceparent: record.traceparent ?? null,

package/src/introspection.ts

@@ -7,6 +7,7 @@
 import { finiteCount } from '@ultimat3/core';
 import type { ClaimIdentity, JobRecord, JobState } from './driver';
 import { JobPageInvalidError } from './errors-operator';
+import type { StepRecord } from './steps-store';
 
 /** Rows one `list()` call may answer. A page, never a table: a dashboard walks with `after`. */
 export const MAX_JOB_PAGE = 200;
@@ -99,6 +100,14 @@ export function pageLimit(subject: string, limit: number | undefined): number {
   return asked;
 }
 
+/** What a person found out about a `once` step: it ran (and answered `output`), or it did not. */
+export type StepResolution =
+  | { readonly ran: true; readonly output: unknown }
+  | { readonly ran: false };
+
+/** The step statuses an operator may resolve: a `once` step that began and never completed. */
+export const RESOLVABLE_STEP_STATUSES: ReadonlySet<string> = new Set(['started', 'failed']);
+
 /** Which rows a bulk call addresses. `state` is REQUIRED: "every job" is never one call. */
 export interface BulkFilter {
   readonly state: JobState;
@@ -144,6 +153,12 @@ export interface WorkerAnnouncement {
    * what it holds. Absent: serving.
    */
   readonly draining?: boolean;
+  /**
+   * The build this process runs: `BUILD_ID`, the id `x-ultimate-build` answers with. Optional so a
+   * custom driver still compiles, and absent on a row a worker from before 30.0.0 wrote. What
+   * `x deploy verify` tells a stalled worker roll by: the old pods still claim the queue.
+   */
+  readonly build?: string;
 }
 
 export interface WorkerRecord extends WorkerAnnouncement {
@@ -151,7 +166,7 @@ export interface WorkerRecord extends WorkerAnnouncement {
   readonly heartbeatAt: number;
   /**
    * Epoch ms of the FIRST announcement that said `draining` — kept by every later one. Absent while
-   * the worker serves. What `SQL_SCALER_DEPTH` excludes a worker's held jobs by.
+   * the worker serves. What `SQL_SCALER_DEMAND` excludes a worker's held jobs by.
    */
   readonly drainingAt?: number;
 }
@@ -241,6 +256,14 @@ export interface JobIntrospection {
    * `runAt` is due now. `filter.state` must be `ready` (a retry backing off) or `delayed`.
    */
   promoteMany(filter: BulkFilter): Promise<BulkResult>;
+  /**
+   * Answer a `once` step whose outcome is unknown (`X_STEP_OUTCOME_UNKNOWN`): `ran` writes it
+   * `completed` with `output`, so a replay returns it; not `ran` writes `cleared`, so the next run
+   * calls `fn` again. ONE write, fenced to a job no worker holds and a step that `started` or
+   * `failed`; anything else is `X_STEP_NOT_RESOLVABLE`, an unknown id `X_JOB_NOT_FOUND`. Optional,
+   * as `cancel` is: a driver may keep no step records an operator can reach.
+   */
+  resolveStep?(jobId: string, step: string, resolution: StepResolution): Promise<StepRecord>;
   /** A paused queue is never claimed, by any worker; enqueues still land. Idempotent. */
   pauseQueue(queue: string): Promise<void>;
   resumeQueue(queue: string): Promise<void>;