@jwilleke/ngdpbase-types 0.0.0 → 5.19.0

package/lib/src/managers/BackgroundJobManager.d.ts

@@ -0,0 +1,434 @@
+import BaseManager from './BaseManager.js';
+import type { WikiEngine } from '../types/WikiEngine.js';
+import { type JobContext } from '../context/JobContext.js';
+import { type ScheduleInput } from '../utils/schedule.js';
+import { type CatchUp, type Overlap } from '../utils/jobSlots.js';
+import { type JobRunRecord, type JobStateProvider } from '../providers/jobState.js';
+/**
+ * Callback supplied to job run functions so they can push live progress
+ * messages that the client can display while polling.
+ */
+export type ReportProgress = (message: string) => void;
+/**
+ * What a run is handed besides progress (#1715): who asked, as {@link JobContext}
+ * — which stays flat and serialisable, so it is extended rather than changed —
+ * and what only a running job needs.
+ */
+export interface JobRunContext extends JobContext {
+    /** Aborted when the run times out. A long job checks it and stops. */
+    signal: AbortSignal;
+    /** The slot this run is for (ISO instant); null for a run nobody scheduled. */
+    slot: string | null;
+    /**
+     * Set when this run picks up a slot an earlier attempt did not finish
+     * (#1716): which attempt this is, the last checkpoint that attempt saved
+     * (null if none), and why it ended. Null on a first attempt.
+     */
+    resume: {
+        attempt: number;
+        checkpoint: unknown;
+        reason: string;
+    } | null;
+    /**
+     * Save a bookmark the next attempt receives as `resume.checkpoint`, e.g.
+     * `{ doneThrough: 311 }`. JSON only; larger than
+     * `ngdpbase.jobs.max-checkpoint-bytes` throws. Written at most once every
+     * 5 s, the latest winning; deleted when the run succeeds, kept when it fails.
+     * Kept only for a persisted scheduled job; elsewhere it is checked and dropped.
+     */
+    checkpoint(data: unknown): void;
+}
+/** Why an administrator's action on a job was refused (#1718); `status` is the HTTP answer. */
+export declare class JobActionError extends Error {
+    readonly status: 400 | 404 | 409;
+    constructor(message: string, status: 400 | 404 | 409);
+}
+/** A scheduled job as the admin page shows it (#1718). */
+export interface ScheduledJobView {
+    id: string;
+    displayName: string;
+    /** The schedule in force, as written (a shorthand or an RRULE), and what it compiled to. */
+    schedule: string;
+    rrule: string;
+    timeZone: string;
+    catchUp: CatchUp;
+    overlap: Overlap;
+    maxAttempts: number;
+    persist: boolean;
+    /** False while `ngdpbase.jobs.<id>.enabled` is false: paused. */
+    enabled: boolean;
+    running: boolean;
+    nextSlot: string | null;
+    lastSlotDone: string | null;
+    /** The run the state names as current: running, waiting for a retry, or interrupted. */
+    current: JobRunRecord | null;
+    /** History, newest first. A run with no slot was started by hand. */
+    runs: JobRunRecord[];
+}
+/**
+ * A job type that can be registered with the BackgroundJobManager.
+ */
+export interface JobDefinition {
+    /** Unique job type ID, e.g. 'pages.reindex' */
+    id: string;
+    /** Human-readable name shown in UI and notifications */
+    displayName: string;
+    /**
+     * When to run by itself (#1715): an RRULE, or a shorthand such as
+     * `end-of-month 06:00` (see `utils/schedule.ts`). Without one the job runs
+     * only when enqueued. `ngdpbase.jobs.<id>.schedule` overrides it, and
+     * `ngdpbase.jobs.<id>.enabled: false` stops it.
+     */
+    schedule?: ScheduleInput;
+    /** Slots missed while the server was down or the job idle: `none`, `latest` (default) or `all` (newest 12). */
+    catchUp?: CatchUp;
+    /** Slots that came due while the job was still running: `queue` (default, one waits) or `skip`. */
+    overlap?: Overlap;
+    /**
+     * Milliseconds a run may take before it is aborted and failed. Scheduled
+     * jobs default to one hour; `0` is no limit, and must be asked for.
+     * `ngdpbase.jobs.max-timeout-ms` caps it when set. A job that is only ever
+     * enqueued has no limit unless it declares one.
+     */
+    timeout?: number;
+    /**
+     * For a scheduled job: keep slots, history and locks where a restart finds
+     * them (default `true`). `false` keeps them in memory: no catch-up after a
+     * restart and no lock, so only for harmless maintenance ticks.
+     *
+     * For a run started by hand (`enqueue`, Run now) (#1746): `true` declares
+     * the job safe to run again, so a run a restart cut off is restarted, as the
+     * person who asked. Default `false` for a job that is only enqueued — a run
+     * of it that a restart cut off is reported, never repeated, because running
+     * it twice could do harm (an import posting twice). A scheduled job's Run
+     * now follows its `persist`.
+     */
+    persist?: boolean;
+    /**
+     * The permission a person needs to start this job (#1746), as the route
+     * that enqueues it checks. Before a cut-off run is restarted as them, they
+     * must still hold it; if they do not, the run is reported, not restarted.
+     */
+    permission?: string;
+    /**
+     * Attempts a scheduled slot gets (default 3, #1716). A failed attempt — an
+     * error, a timeout, a failure result, or the server stopping mid-run — is
+     * tried again after 1, 5, then 15 minutes, with the same run id and slot.
+     * After the last, the slot is failed and the next slot runs as usual.
+     */
+    maxAttempts?: number;
+    /**
+     * A routine maintenance tick (#1721): a success is neither audited nor
+     * notified — an every-minute sweep would bury both — while a failure is
+     * both, as for any job. Set by {@link BackgroundJobManager.registerMaintenance}.
+     */
+    routine?: boolean;
+    /**
+     * The work to perform. Resolves with a JobResult.
+     *
+     * __`ctx` is mandatory (#631).__ Without it the actor this manager captured
+     * at `enqueue` reached the audit record and stopped there: `job-started`
+     * named who asked for a reindex while the reindex itself ran as nobody. That
+     * is the exact shape P1 in docs/security-posture.md warns about — a
+     * parameter that cannot carry provenance guarantees provenance is lost — and
+     * it was true of this signature for the whole first version of #631.
+     *
+     * A handler that needs to make a permission decision or write a record uses
+     * this; one that does neither still receives it, because a handler which can
+     * quietly grow into taking an action must not be able to do so anonymously.
+     */
+    run: (reportProgress: ReportProgress, ctx: JobRunContext) => Promise<JobResult>;
+}
+/**
+ * Result returned by a completed job run.
+ */
+export interface JobResult {
+    success: boolean;
+    /** e.g. "Scanned 14 327 files, added 12, updated 3" */
+    summary?: string;
+    error?: string;
+}
+/**
+ * State of a single job run instance.
+ */
+export interface JobRun {
+    runId: string;
+    jobId: string;
+    displayName: string;
+    /** Who asked for this work, and from where (#631). */
+    requestedBy: JobContext;
+    status: 'pending' | 'running' | 'completed' | 'failed';
+    /** Live progress message set by the job via reportProgress(); cleared on completion */
+    progress?: string;
+    /** For a scheduled run: its slot, its attempt, and why the attempt before it ended (#1716). */
+    slot?: string;
+    attempt?: number;
+    resumeReason?: string;
+    /** Set at shutdown (#1717): the run was stopped to be handed to the next process, not failed. */
+    interrupted?: boolean;
+    startedAt: Date;
+    completedAt?: Date;
+    result?: JobResult;
+}
+/**
+ * BackgroundJobManager — async long-running admin operations.
+ *
+ * Managers and plugins register job types at startup via registerJob().
+ * Callers enqueue a job by ID; the job runs in the background and the
+ * caller can poll getStatus(runId) for progress.
+ *
+ * Only one instance of a given jobId runs at a time — duplicate enqueue
+ * returns the existing runId.
+ *
+ * On completion, a system notification is posted via NotificationManager.
+ */
+declare class BackgroundJobManager extends BaseManager {
+    /** Registered job types, keyed by jobId */
+    private jobs;
+    /** All run records (completed runs are kept for status polling) */
+    private runs;
+    /** Maps jobId → runId for currently active (pending/running) runs */
+    private activeByJobId;
+    /** Jobs with a schedule (#1715), keyed by jobId. */
+    private scheduled;
+    private stateProvider;
+    private readonly clock;
+    private timer;
+    private ticking;
+    /** Scheduled runs in progress, so a caller can wait for them. */
+    private drains;
+    /** Jobs whose runs a restart cut off have been looked for, since this start (#1746). */
+    private recovered;
+    /** Set once shutdown begins: no slot starts after it (#1717). */
+    private stopping;
+    private handedOff;
+    /** Each running job's abort, by run id. */
+    private controllers;
+    /** For each running scheduled run: write it as interrupted and let go of its slot. */
+    private handoffs;
+    /**
+     * @param options - for tests: a state provider in place of the configured
+     *   one, and the clock.
+     */
+    private readonly checkpointThrottleMs;
+    constructor(engine: WikiEngine, options?: {
+        stateProvider?: JobStateProvider;
+        now?: () => Date;
+        checkpointThrottleMs?: number;
+    });
+    initialize(config?: Record<string, unknown>): Promise<void>;
+    private checkpointMaxBytes;
+    private configManager;
+    private setting;
+    /**
+     * Register a job type. Called at startup by managers and plugins.
+     *
+     * A job with a `schedule` is also scheduled (#1715). Its schedule, catch-up,
+     * overlap and timeout are checked here, so a mistake is the author's error
+     * at startup rather than a job that silently never runs.
+     *
+     * @throws Error naming what is wrong with a scheduled job's declaration
+     */
+    registerJob(def: JobDefinition): void;
+    /**
+     * A routine maintenance tick (#1721): work whose next run does the same
+     * job, so a missed run costs nothing — a sweep, a retention purge, a folder
+     * rescan. One declaration for all of them, in place of a hand-rolled
+     * `setInterval`: it runs every `everyMs` (in whole minutes, at least one),
+     * catches nothing up, keeps nothing across a restart, never overlaps
+     * itself, and reports only failures. It is listed on Admin → Scheduled Jobs
+     * like any job, and can be run, paused or rescheduled from there.
+     *
+     * `everyMs` 0 or less registers the job without a schedule: it stops
+     * recurring, and can still be run by hand.
+     */
+    registerMaintenance(spec: {
+        id: string;
+        displayName: string;
+        everyMs: number;
+        run: (ctx: JobRunContext) => Promise<string | void>;
+    }): void;
+    private compileScheduled;
+    /** A run's time limit: the job's own, else `fallback`, capped by `ngdpbase.jobs.max-timeout-ms` when set. */
+    private timeoutFor;
+    private compile;
+    /**
+     * `ngdpbase.jobs.<id>.schedule` in place of the registered schedule. One
+     * that does not compile is refused, reported once, and the schedule in force
+     * stays.
+     */
+    private applyOverride;
+    /** The configured schedule and switch, read each look so an admin change applies without a restart. */
+    private refreshFromConfig;
+    /**
+     * Start looking for due slots (#1715). Called once the engine and its
+     * add-ons are up, so a catch-up run never meets a manager still starting.
+     */
+    startScheduler(): void;
+    /**
+     * One look at every scheduled job: record the slots passed over, and start
+     * the ones chosen. Public so tests can drive the clock.
+     */
+    tick(): Promise<void>;
+    private look;
+    /**
+     * A run the state still names as current, which this process is not
+     * running (#1716). True when it takes this look: it is held by another
+     * server, waiting out its backoff, or being resumed now.
+     */
+    private handleUnfinished;
+    /**
+     * An attempt failed. With attempts left, the slot waits out its backoff and
+     * is tried again (true: the job waits). After the last, the slot is failed
+     * for good, audited, and the job goes on to its next slot (false).
+     */
+    private afterFailedAttempt;
+    /** The job's next slot once `slot` is handled: never earlier than the one already recorded. */
+    private nextSlotAfter;
+    private startDrain;
+    /** Every scheduled job, its state and its recent history. */
+    listScheduledJobs(): Promise<ScheduledJobView[]>;
+    private scheduledOrThrow;
+    /** Refused while the job runs, waits for a retry, or the server is stopping: one run of a job at a time. */
+    private assertIdle;
+    /**
+     * Run a scheduled job now, outside its schedule (#1611 §9). As the person
+     * who asked, with their permissions. It is no slot: it marks no slot done
+     * and catches nothing up. Slots that come due while it runs follow the job's
+     * overlap rule, as for any run. A restart that cuts it off restarts it, as
+     * for any run started by hand (#1746). Returns the run id.
+     */
+    runNow(jobId: string, requestedBy: JobContext): Promise<string>;
+    /**
+     * Run one slot again, under its own key (#1611 §9): one the scheduler
+     * skipped, or one that failed for good. As the person who asked. Success
+     * marks that slot done; the job's next slot does not move. Returns the run id.
+     */
+    rerunSlot(jobId: string, slot: string, requestedBy: JobContext): Promise<string>;
+    /**
+     * Try a run that is waiting out its backoff now, rather than at its retry
+     * time (#1611 §9). Counts as its next attempt, like any retry.
+     */
+    retryNow(jobId: string, requestedBy: JobContext): Promise<void>;
+    /** Resolves once every scheduled run started so far has finished. */
+    whenIdle(): Promise<void>;
+    /**
+     * Run the chosen slots one at a time, oldest first. A slot that fails with
+     * attempts left stops the drain: the job waits for its retry, and the slots
+     * after it are looked at again once it is done. `after` is the next slot
+     * once all are handled; null leaves it to each run.
+     */
+    private drain;
+    /** One attempt at one slot. `retry` when it failed with attempts left. */
+    private runSlot;
+    /**
+     * A run's `ctx.checkpoint` (#1716). The size is checked at once, so an
+     * oversize checkpoint throws to the job. Writes are at most one per
+     * throttle interval, the latest winning, and `flush` writes what is still
+     * waiting. Without a provider (not persisted) a checkpoint is checked and dropped.
+     */
+    private checkpointWriter;
+    private loadState;
+    private saveState;
+    /**
+     * Slots passed over are recorded, never dropped: one audit event for the
+     * lot, and the newest of them in the job's history (#1611 §2).
+     */
+    private recordSkipped;
+    /** Who the scheduler acts as when nobody asked: the system principal, origin schedule. */
+    private schedulerContext;
+    private recordScheduleChange;
+    private recordSchedulerEvent;
+    /**
+     * Enqueue a job by ID. Returns the runId immediately.
+     * If the job is already pending/running, returns the existing runId.
+     *
+     * __`requestedBy` is mandatory, and positional rather than an option (#631).__
+     * This took a job id alone, so the identity of whoever triggered the work was
+     * discarded here — the route logged the username on the line above and threw
+     * it away on this one. Every defect this codebase has removed lately came
+     * from forgetting being safe; an optional context would be that shape again,
+     * and the first job added without one would go unattributed with nothing
+     * going red. Omitting it is now a compile error.
+     *
+     * Build it with `jobContextFromRequest(req.userContext)` when a person asked,
+     * or `jobContextFromSystem(pip.systemPrincipalName(), reason)` (the PolicyInformationPoint's) when
+     * nothing did (#631: the principal is a name from .env, never a constant).
+     *
+     * @throws Error if jobId is not registered
+     */
+    enqueue(jobId: string, requestedBy: JobContext): Promise<string>;
+    /** Whether a cut-off run of this job may be run again: its declaration (see `persist`). */
+    private resumable;
+    /** Start a run by hand, or a cut-off one again. Returns its run id at once; the work runs in the background. */
+    private startHandRun;
+    /**
+     * One run started by hand. Its record — who asked, when, attempt, running —
+     * is written before the work starts and held under a lock with a heartbeat,
+     * so the next start can tell a run that a restart cut off from one a live
+     * server is still running. With no job state open it runs unrecorded, as
+     * before.
+     */
+    private runByHand;
+    /**
+     * Look once, per job, for runs started by hand that a restart cut off
+     * (#1746). Called from every look until each registered job has been seen,
+     * so an add-on's job is looked at once it has registered. A job running
+     * here, or whose run another live server still holds, is looked at again
+     * next time.
+     */
+    private recoverHandRuns;
+    /** False when a live server still holds one of the job's runs: look again later. */
+    private recoverHandRunsOf;
+    private recoverOne;
+    /** Whether the person who started a run still holds the job's permission — by policy, as at the time it is asked (P2). */
+    private stillHolds;
+    /** A run a restart cut off and that is not run again: recorded as failed, audited, and told to the admins. Never silent. */
+    private reportCutOff;
+    /**
+     * Get the current state of a run by runId.
+     * Returns null if the runId is unknown.
+     */
+    getStatus(runId: string): JobRun | null;
+    /**
+     * Get all currently pending or running jobs.
+     */
+    getActiveJobs(): JobRun[];
+    /**
+     * Get all registered job IDs.
+     */
+    getRegisteredJobIds(): string[];
+    private executeJob;
+    /**
+     * Record who asked for this work, and how it ended (#631).
+     *
+     * Lazily resolved, following ConfigurationManager: AuditManager reads
+     * configuration, so holding a reference would invert the boot order, and it
+     * is absent during early boot — which `recordAuditEvent` already treats as a
+     * configuration state rather than a failure.
+     *
+     * on-failure: continue deliberately. A reindex must not be refused because its
+     * audit record could not be written; the drop is counted and surfaced by
+     * `recordAuditEvent` rather than being fatal.
+     */
+    private recordJobEvent;
+    private sendNotification;
+    /**
+     * Hand running work to the next process (#1717, #1611 §7). No slot starts
+     * after this. Every running job's `ctx.signal` is aborted, and the jobs get
+     * `ngdpbase.jobs.shutdown-grace-ms` (default 5 s) to stop. A scheduled run
+     * that did not finish is written as `interrupted` with its last checkpoint
+     * and its slot lock released, so the next start resumes it at once, without
+     * counting an attempt. A job that ignores the signal is handed off anyway
+     * when the grace ends.
+     *
+     * Depends only on being called: the signal handlers in app.ts (SIGTERM,
+     * SIGINT) reach it through `engine.shutdown()`, which calls it before any
+     * other manager stops. A hard kill never gets here; its lock expires and the
+     * resume counts as an attempt (#1716).
+     */
+    handOff(): Promise<void>;
+    private runHandOff;
+    shutdown(): Promise<void>;
+}
+export default BackgroundJobManager;