@qlever-llc/trellis 0.7.0 → 0.8.0-rc.2

package/esm/npm/src/server/internal_jobs/projection.js

@@ -0,0 +1,37 @@
+const TERMINAL_STATES = new Set([
+    "completed",
+    "failed",
+    "cancelled",
+    "expired",
+    "dead",
+    "dismissed",
+]);
+export function isTerminal(state) {
+    return TERMINAL_STATES.has(state);
+}
+export function jobFromWorkEvent(event) {
+    switch (event.eventType) {
+        case "created":
+        case "retried":
+            return seedJobFromEvent(event);
+        default:
+            return undefined;
+    }
+}
+function seedJobFromEvent(event) {
+    if (event.payload === undefined) {
+        return undefined;
+    }
+    return {
+        id: event.jobId,
+        service: event.service,
+        type: event.jobType,
+        state: event.state,
+        payload: event.payload,
+        createdAt: event.timestamp,
+        updatedAt: event.timestamp,
+        tries: event.tries,
+        maxTries: event.maxTries ?? 1,
+        ...(event.deadline !== undefined ? { deadline: event.deadline } : {}),
+    };
+}

package/esm/npm/src/server/internal_jobs/runtime-worker.d.ts

@@ -0,0 +1,147 @@
+import type { NatsConnection } from "@nats-io/nats-core";
+import type { JobsRuntimeBinding } from "./bindings.js";
+import type { ActiveJob, JobProcessOutcome } from "./job-manager.js";
+import { JobCancellationToken, type JobManager } from "./job-manager.js";
+import type { Job } from "./types.js";
+export type WorkerAckAction = "ack" | "nak";
+export type ProjectedWorkDecision = "process" | "skip-ack";
+export type SchemaRef = {
+    schema: string;
+};
+export type PayloadValidationArgs<TResult> = {
+    schema?: SchemaRef;
+    job: Job<unknown, TResult>;
+};
+export type ResultValidationArgs<TResult> = {
+    schema?: SchemaRef;
+    job: Job<unknown, TResult>;
+    result: TResult;
+};
+export declare class WorkerLoopStopError extends AggregateError {
+    constructor(errors: unknown[]);
+}
+export declare class WorkerHostStopError extends AggregateError {
+    constructor(errors: unknown[]);
+}
+export declare class JobsInfrastructureMissingError extends Error {
+    constructor(stream: string, queueType: string);
+}
+type WorkMessageLike = {
+    data: Uint8Array;
+    subject: string;
+    info?: {
+        redeliveryCount?: number;
+    };
+    ack(): void | Promise<void>;
+    nak(delay?: number): void | Promise<void>;
+    inProgress(): void | Promise<void>;
+};
+type ConsumerMessagesLike = AsyncIterable<WorkMessageLike> & {
+    stop?: () => void;
+    close?: () => Promise<void | Error> | void;
+};
+type WorkerConsumerLike = {
+    consume(): Promise<ConsumerMessagesLike>;
+};
+type CancelMessageLike = {
+    subject: string;
+    data: Uint8Array;
+};
+type CancelSubscriptionLike = AsyncIterable<CancelMessageLike> & {
+    unsubscribe(): void;
+};
+type WorkerStopHandle = {
+    stop(): Promise<void>;
+} | void;
+type ConsumerInfoLike = unknown;
+type StartNatsConsumerDeps = {
+    nats: Pick<NatsConnection, "subscribe">;
+    jsm: {
+        consumers: {
+            add(stream: string, config: Record<string, unknown>): Promise<ConsumerInfoLike>;
+            info(stream: string, consumer: string): Promise<ConsumerInfoLike>;
+        };
+    };
+    js: {
+        consumers: {
+            getConsumerFromInfo(info: ConsumerInfoLike): WorkerConsumerLike;
+        };
+    };
+};
+type StartNatsConnectionDeps = {
+    nats: NatsConnection;
+    jsm?: undefined;
+    js?: undefined;
+};
+type StartNatsRuntimeDeps = StartNatsConsumerDeps | StartNatsConnectionDeps;
+type StartWorkerArgs = {
+    queueType: string;
+    workerIndex: number;
+    cancellation: JobCancellationToken;
+};
+type StartWorkerHostOptions = {
+    instanceId: string;
+    queueTypes?: string[];
+    heartbeatPublisher?: {
+        publish(subject: string, payload: Uint8Array): void | Promise<void>;
+    };
+    heartbeatIntervalMs?: number;
+    version?: string;
+    nowIso?: () => string;
+    startWorker: (args: StartWorkerArgs) => Promise<WorkerStopHandle>;
+};
+type StartNatsWorkerHostOptions<TResult> = Omit<StartWorkerHostOptions, "startWorker"> & StartNatsRuntimeDeps & {
+    manager: JobManager<unknown, TResult>;
+    getProjectedJob?: (job: Job<unknown, TResult>) => Promise<Job<unknown, TResult> | undefined>;
+    validatePayload?: (args: PayloadValidationArgs<TResult>) => Promise<void> | void;
+    validateResult?: (args: ResultValidationArgs<TResult>) => Promise<void> | void;
+    handler: (job: ActiveJob<unknown, TResult>) => Promise<TResult>;
+};
+type StartQueueWorkerLoopOptions<TResult> = {
+    manager: JobManager<unknown, TResult>;
+    consumer: WorkerConsumerLike;
+    cancelSubscription: CancelSubscriptionLike;
+    hostCancellation?: JobCancellationToken;
+    getProjectedJob?: (job: Job<unknown, TResult>) => Promise<Job<unknown, TResult> | undefined>;
+    payloadSchema?: SchemaRef;
+    validatePayload?: (args: PayloadValidationArgs<TResult>) => Promise<void> | void;
+    resultSchema?: SchemaRef;
+    validateResult?: (args: ResultValidationArgs<TResult>) => Promise<void> | void;
+    handler: (job: ActiveJob<unknown, TResult>) => Promise<TResult>;
+};
+type StartNatsQueueWorkerOptions<TResult> = StartNatsRuntimeDeps & {
+    manager: JobManager<unknown, TResult>;
+    binding: JobsRuntimeBinding;
+    queueType: string;
+    hostCancellation?: JobCancellationToken;
+    getProjectedJob?: (job: Job<unknown, TResult>) => Promise<Job<unknown, TResult> | undefined>;
+    validatePayload?: (args: PayloadValidationArgs<TResult>) => Promise<void> | void;
+    validateResult?: (args: ResultValidationArgs<TResult>) => Promise<void> | void;
+    handler: (job: ActiveJob<unknown, TResult>) => Promise<TResult>;
+};
+export declare function processWorkPayloadWithContextAndHeartbeat<TResult>(manager: JobManager<unknown, TResult>, payload: Uint8Array, cancellation: JobCancellationToken, heartbeat: () => Promise<void>, handler: (job: ActiveJob<unknown, TResult>) => Promise<TResult>, validation?: {
+    payloadSchema?: SchemaRef;
+    validatePayload?: (args: PayloadValidationArgs<TResult>) => Promise<void> | void;
+    resultSchema?: SchemaRef;
+    validateResult?: (args: ResultValidationArgs<TResult>) => Promise<void> | void;
+}, runtime?: {
+    redeliveryCount?: number;
+}): Promise<JobProcessOutcome<TResult> | undefined>;
+export declare function projectedWorkDecision(projected: Job | undefined, _work: Job): ProjectedWorkDecision;
+export declare function ackActionForOutcome(outcome: JobProcessOutcome<unknown> | undefined): WorkerAckAction;
+export declare function startQueueWorkerLoop<TResult>(options: StartQueueWorkerLoopOptions<TResult>): Promise<{
+    stop(): Promise<void>;
+}>;
+export declare function startNatsQueueWorker<TResult>(options: StartNatsQueueWorkerOptions<TResult>): Promise<{
+    stop(): Promise<void>;
+}>;
+export declare function startWorkerHostFromBinding(binding: JobsRuntimeBinding, options: StartWorkerHostOptions): Promise<{
+    workerCount(): number;
+    stop(): Promise<void>;
+}>;
+export declare function startNatsWorkerHostFromBinding<TResult>(binding: JobsRuntimeBinding, options: StartNatsWorkerHostOptions<TResult>): Promise<{
+    workerCount(): number;
+    stop(): Promise<void>;
+}>;
+export {};
+//# sourceMappingURL=runtime-worker.d.ts.map

package/esm/npm/src/server/internal_jobs/runtime-worker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runtime-worker.d.ts","sourceRoot":"","sources":["../../../../../src/server/internal_jobs/runtime-worker.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAgB,MAAM,oBAAoB,CAAC;AACvE,OAAO,KAAK,EAAoB,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAG1E,OAAO,KAAK,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrE,OAAO,EACL,oBAAoB,EACpB,KAAK,UAAU,EAEhB,MAAM,kBAAkB,CAAC;AAE1B,OAAO,KAAK,EAAE,GAAG,EAAY,MAAM,YAAY,CAAC;AAEhD,MAAM,MAAM,eAAe,GAAG,KAAK,GAAG,KAAK,CAAC;AAC5C,MAAM,MAAM,qBAAqB,GAAG,SAAS,GAAG,UAAU,CAAC;AAC3D,MAAM,MAAM,SAAS,GAAG;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAC3C,MAAM,MAAM,qBAAqB,CAAC,OAAO,IAAI;IAC3C,MAAM,CAAC,EAAE,SAAS,CAAC;IACnB,GAAG,EAAE,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;CAC5B,CAAC;AAEF,MAAM,MAAM,oBAAoB,CAAC,OAAO,IAAI;IAC1C,MAAM,CAAC,EAAE,SAAS,CAAC;IACnB,GAAG,EAAE,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC3B,MAAM,EAAE,OAAO,CAAC;CACjB,CAAC;AAEF,qBAAa,mBAAoB,SAAQ,cAAc;gBACzC,MAAM,EAAE,OAAO,EAAE;CAI9B;AAED,qBAAa,mBAAoB,SAAQ,cAAc;gBACzC,MAAM,EAAE,OAAO,EAAE;CAI9B;AAED,qBAAa,8BAA+B,SAAQ,KAAK;gBAC3C,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM;CAQ9C;AAED,KAAK,eAAe,GAAG;IACrB,IAAI,EAAE,UAAU,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE;QAAE,eAAe,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACpC,GAAG,IAAI,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5B,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1C,UAAU,IAAI,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACpC,CAAC;AAEF,KAAK,oBAAoB,GAAG,aAAa,CAAC,eAAe,CAAC,GAAG;IAC3D,IAAI,CAAC,EAAE,MAAM,IAAI,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,GAAG,KAAK,CAAC,GAAG,IAAI,CAAC;CAC5C,CAAC;AAEF,KAAK,kBAAkB,GAAG;IACxB,OAAO,IAAI,OAAO,CAAC,oBAAoB,CAAC,CAAC;CAC1C,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,UAAU,CAAC;CAClB,CAAC;AAEF,KAAK,sBAAsB,GAAG,aAAa,CAAC,iBAAiB,CAAC,GAAG;IAC/D,WAAW,IAAI,IAAI,CAAC;CACrB,CAAC;AAEF,KAAK,gBAAgB,GAAG;IAAE,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;CAAE,GAAG,IAAI,CAAC;AAEzD,KAAK,gBAAgB,GAAG,OAAO,CAAC;AAEhC,KAAK,qBAAqB,GAAG;IAC3B,IAAI,EAAE,IAAI,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC;IACxC,GAAG,EAAE;QACH,SAAS,EAAE;YACT,GAAG,CACD,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC9B,OAAO,CAAC,gBAAgB,CAAC,CAAC;YAC7B,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;SACnE,CAAC;KACH,CAAC;IACF,EAAE,EAAE;QACF,SAAS,EAAE;YACT,mBAAmB,CAAC,IAAI,EAAE,gBAAgB,GAAG,kBAAkB,CAAC;SACjE,CAAC;KACH,CAAC;CACH,CAAC;AAEF,KAAK,uBAAuB,GAAG;IAC7B,IAAI,EAAE,cAAc,CAAC;IACrB,GAAG,CAAC,EAAE,SAAS,CAAC;IAChB,EAAE,CAAC,EAAE,SAAS,CAAC;CAChB,CAAC;AAEF,KAAK,oBAAoB,GAAG,qBAAqB,GAAG,uBAAuB,CAAC;AAQ5E,KAAK,eAAe,GAAG;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,oBAAoB,CAAC;CACpC,CAAC;AAEF,KAAK,sBAAsB,GAAG;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,kBAAkB,CAAC,EAAE;QACnB,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;KACrE,CAAC;IACF,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,MAAM,CAAC;IACtB,WAAW,EAAE,CAAC,IAAI,EAAE,eAAe,KAAK,OAAO,CAAC,gBAAgB,CAAC,CAAC;CACnE,CAAC;AAEF,KAAK,0BAA0B,CAAC,OAAO,IACnC,IAAI,CAAC,sBAAsB,EAAE,aAAa,CAAC,GAC3C,oBAAoB,GACpB;IACA,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACtC,eAAe,CAAC,EAAE,CAChB,GAAG,EAAE,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,KACvB,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC,CAAC;IAChD,eAAe,CAAC,EAAE,CAChB,IAAI,EAAE,qBAAqB,CAAC,OAAO,CAAC,KACjC,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAC1B,cAAc,CAAC,EAAE,CACf,IAAI,EAAE,oBAAoB,CAAC,OAAO,CAAC,KAChC,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAC1B,OAAO,EAAE,CAAC,GAAG,EAAE,SAAS,CAAC,OAAO,EAAE,OAAO,CAAC,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;CACjE,CAAC;AAEJ,KAAK,2BAA2B,CAAC,OAAO,IAAI;IAC1C,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACtC,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,kBAAkB,EAAE,sBAAsB,CAAC;IAC3C,gBAAgB,CAAC,EAAE,oBAAoB,CAAC;IACxC,eAAe,CAAC,EAAE,CAChB,GAAG,EAAE,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,KACvB,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC,CAAC;IAChD,aAAa,CAAC,EAAE,SAAS,CAAC;IAC1B,eAAe,CAAC,EAAE,CAChB,IAAI,EAAE,qBAAqB,CAAC,OAAO,CAAC,KACjC,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAC1B,YAAY,CAAC,EAAE,SAAS,CAAC;IACzB,cAAc,CAAC,EAAE,CACf,IAAI,EAAE,oBAAoB,CAAC,OAAO,CAAC,KAChC,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAC1B,OAAO,EAAE,CAAC,GAAG,EAAE,SAAS,CAAC,OAAO,EAAE,OAAO,CAAC,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;CACjE,CAAC;AAEF,KAAK,2BAA2B,CAAC,OAAO,IACpC,oBAAoB,GACpB;IACA,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACtC,OAAO,EAAE,kBAAkB,CAAC;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,CAAC,EAAE,oBAAoB,CAAC;IACxC,eAAe,CAAC,EAAE,CAChB,GAAG,EAAE,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,KACvB,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC,CAAC;IAChD,eAAe,CAAC,EAAE,CAChB,IAAI,EAAE,qBAAqB,CAAC,OAAO,CAAC,KACjC,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAC1B,cAAc,CAAC,EAAE,CACf,IAAI,EAAE,oBAAoB,CAAC,OAAO,CAAC,KAChC,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAC1B,OAAO,EAAE,CAAC,GAAG,EAAE,SAAS,CAAC,OAAO,EAAE,OAAO,CAAC,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;CACjE,CAAC;AAqCJ,wBAAsB,yCAAyC,CAAC,OAAO,EACrE,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,EACrC,OAAO,EAAE,UAAU,EACnB,YAAY,EAAE,oBAAoB,EAClC,SAAS,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,EAC9B,OAAO,EAAE,CAAC,GAAG,EAAE,SAAS,CAAC,OAAO,EAAE,OAAO,CAAC,KAAK,OAAO,CAAC,OAAO,CAAC,EAC/D,UAAU,CAAC,EAAE;IACX,aAAa,CAAC,EAAE,SAAS,CAAC;IAC1B,eAAe,CAAC,EAAE,CAChB,IAAI,EAAE,qBAAqB,CAAC,OAAO,CAAC,KACjC,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAC1B,YAAY,CAAC,EAAE,SAAS,CAAC;IACzB,cAAc,CAAC,EAAE,CACf,IAAI,EAAE,oBAAoB,CAAC,OAAO,CAAC,KAChC,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;CAC3B,EACD,OAAO,CAAC,EAAE;IAAE,eAAe,CAAC,EAAE,MAAM,CAAA;CAAE,GACrC,OAAO,CAAC,iBAAiB,CAAC,OAAO,CAAC,GAAG,SAAS,CAAC,CAwCjD;AAED,wBAAgB,qBAAqB,CACnC,SAAS,EAAE,GAAG,GAAG,SAAS,EAC1B,KAAK,EAAE,GAAG,GACT,qBAAqB,CAKvB;AAED,wBAAgB,mBAAmB,CACjC,OAAO,EAAE,iBAAiB,CAAC,OAAO,CAAC,GAAG,SAAS,GAC9C,eAAe,CAWjB;AAED,wBAAsB,oBAAoB,CAAC,OAAO,EAChD,OAAO,EAAE,2BAA2B,CAAC,OAAO,CAAC,GAC5C,OAAO,CAAC;IAAE,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;CAAE,CAAC,CA6HpC;AAED,wBAAsB,oBAAoB,CAAC,OAAO,EAChD,OAAO,EAAE,2BAA2B,CAAC,OAAO,CAAC,GAC5C,OAAO,CAAC;IAAE,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;CAAE,CAAC,CAkCpC;AAED,wBAAsB,0BAA0B,CAC9C,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,sBAAsB,GAC9B,OAAO,CAAC;IAAE,WAAW,IAAI,MAAM,CAAC;IAAC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;CAAE,CAAC,CA2E3D;AAED,wBAAsB,8BAA8B,CAAC,OAAO,EAC1D,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,0BAA0B,CAAC,OAAO,CAAC,GAC3C,OAAO,CAAC;IAAE,WAAW,IAAI,MAAM,CAAC;IAAC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;CAAE,CAAC,CA6B3D"}

package/esm/npm/src/server/internal_jobs/runtime-worker.js

@@ -0,0 +1,380 @@
+import { jetstream, jetstreamManager } from "@nats-io/jetstream";
+import { ActiveJobCancellationRegistry } from "./cancellation-registry.js";
+import { startWorkerHeartbeatLoop } from "./heartbeat.js";
+import { JobCancellationToken, JobProcessError, } from "./job-manager.js";
+import { isTerminal, jobFromWorkEvent } from "./projection.js";
+export class WorkerLoopStopError extends AggregateError {
+    constructor(errors) {
+        super(errors, "queue worker loop failed");
+        this.name = "WorkerLoopStopError";
+    }
+}
+export class WorkerHostStopError extends AggregateError {
+    constructor(errors) {
+        super(errors, "worker host stop failed");
+        this.name = "WorkerHostStopError";
+    }
+}
+export class JobsInfrastructureMissingError extends Error {
+    constructor(stream, queueType) {
+        super(`Jobs work stream '${stream}' was not found while starting queue '${queueType}'. ` +
+            "The built-in Trellis jobs infrastructure is missing or not provisioned for this environment. " +
+            `Start Trellis and bootstrap the service again so '${stream}' exists before workers start.`);
+        this.name = "JobsInfrastructureMissingError";
+    }
+}
+function isCustomNatsRuntimeDeps(args) {
+    return args.jsm !== undefined && args.js !== undefined;
+}
+function toWorkerConsumer(consumer) {
+    return {
+        async consume() {
+            const messages = await consumer.consume();
+            return {
+                stop: messages.stop?.bind(messages),
+                close: messages.close?.bind(messages),
+                async *[Symbol.asyncIterator]() {
+                    for await (const msg of messages) {
+                        yield {
+                            data: msg.data,
+                            subject: msg.subject,
+                            info: {
+                                redeliveryCount: Math.max(0, msg.info.deliveryCount - 1),
+                            },
+                            ack: msg.ack.bind(msg),
+                            nak: msg.nak.bind(msg),
+                            inProgress: msg.working.bind(msg),
+                        };
+                    }
+                },
+            };
+        },
+    };
+}
+export async function processWorkPayloadWithContextAndHeartbeat(manager, payload, cancellation, heartbeat, handler, validation, runtime) {
+    const event = parseWorkPayloadEvent(payload);
+    if (!event) {
+        return undefined;
+    }
+    const job = jobFromWorkEvent(event);
+    if (!job) {
+        return undefined;
+    }
+    return await manager.processWithHeartbeat(job, cancellation, heartbeat, async (activeJob) => {
+        try {
+            await validation?.validatePayload?.({
+                schema: validation.payloadSchema,
+                job: activeJob.job(),
+            });
+        }
+        catch (error) {
+            throw JobProcessError.failed(error instanceof Error ? error.message : String(error));
+        }
+        return await handler(activeJob);
+    }, {
+        redeliveryCount: runtime?.redeliveryCount,
+    }, {
+        validateResult: validation?.validateResult
+            ? (result, resultJob) => validation.validateResult({
+                schema: validation.resultSchema,
+                result,
+                job: resultJob,
+            })
+            : undefined,
+    });
+}
+export function projectedWorkDecision(projected, _work) {
+    if (!projected) {
+        return "process";
+    }
+    return isTerminal(projected.state) ? "skip-ack" : "process";
+}
+export function ackActionForOutcome(outcome) {
+    if (!outcome) {
+        return "ack";
+    }
+    switch (outcome.outcome) {
+        case "retry":
+        case "interrupted":
+            return "nak";
+        default:
+            return "ack";
+    }
+}
+export async function startQueueWorkerLoop(options) {
+    const registry = new ActiveJobCancellationRegistry();
+    const activeTokens = new Set();
+    const messages = await options.consumer.consume();
+    const hostCancellation = options.hostCancellation;
+    const stopConsuming = () => {
+        if (typeof messages.stop === "function") {
+            messages.stop();
+        }
+        if (typeof messages.close === "function") {
+            void messages.close();
+        }
+    };
+    const cancelActiveForShutdown = () => {
+        for (const token of activeTokens) {
+            token.cancelForShutdown();
+        }
+    };
+    const hostAbortHandler = () => {
+        cancelActiveForShutdown();
+        stopConsuming();
+    };
+    hostCancellation?.signal.addEventListener("abort", hostAbortHandler);
+    if (hostCancellation?.signal.aborted) {
+        hostAbortHandler();
+    }
+    const workTask = (async () => {
+        for await (const msg of messages) {
+            const event = parseWorkPayloadEvent(msg.data);
+            const job = event
+                ? jobFromWorkEvent(event)
+                : undefined;
+            if (!job) {
+                await msg.ack();
+                continue;
+            }
+            const key = `${job.service}.${job.type}.${job.id}`;
+            if (hostCancellation?.isHostShutdown()) {
+                await msg.nak();
+                continue;
+            }
+            const projected = options.getProjectedJob
+                ? await options.getProjectedJob(job)
+                : undefined;
+            if (hostCancellation?.isHostShutdown()) {
+                await msg.nak();
+                continue;
+            }
+            if (projectedWorkDecision(projected, job) === "skip-ack") {
+                registry.clearPending(key);
+                await msg.ack();
+                continue;
+            }
+            const token = new JobCancellationToken();
+            if (hostCancellation?.isHostShutdown()) {
+                token.cancelForShutdown();
+            }
+            activeTokens.add(token);
+            const guard = registry.register(key, token);
+            try {
+                const outcome = await processWorkPayloadWithContextAndHeartbeat(options.manager, msg.data, token, async () => {
+                    await msg.inProgress();
+                }, options.handler, {
+                    payloadSchema: options.payloadSchema,
+                    validatePayload: options.validatePayload,
+                    resultSchema: options.resultSchema,
+                    validateResult: options.validateResult,
+                }, {
+                    redeliveryCount: msg.info?.redeliveryCount,
+                });
+                if (ackActionForOutcome(outcome) === "ack") {
+                    await msg.ack();
+                }
+                else {
+                    await msg.nak();
+                }
+            }
+            finally {
+                guard.dispose();
+                activeTokens.delete(token);
+            }
+        }
+    })();
+    let workFailure;
+    const observedWorkTask = workTask.catch((error) => {
+        workFailure = error;
+    });
+    const cancelTask = (async () => {
+        for await (const msg of options.cancelSubscription) {
+            const event = parseWorkPayloadEvent(msg.data);
+            if (!event || event.eventType !== "cancelled") {
+                continue;
+            }
+            registry.cancel(`${event.service}.${event.jobType}.${event.jobId}`);
+        }
+    })();
+    let cancelFailure;
+    const observedCancelTask = cancelTask.catch((error) => {
+        cancelFailure = error;
+    });
+    return {
+        async stop() {
+            options.cancelSubscription.unsubscribe();
+            cancelActiveForShutdown();
+            stopConsuming();
+            await Promise.all([observedWorkTask, observedCancelTask]);
+            hostCancellation?.signal.removeEventListener("abort", hostAbortHandler);
+            const failures = [workFailure, cancelFailure].filter((error) => error !== undefined);
+            if (failures.length > 0) {
+                throw new WorkerLoopStopError(failures);
+            }
+        },
+    };
+}
+export async function startNatsQueueWorker(options) {
+    const queue = getQueueBinding(options.binding, options.queueType);
+    const jsm = isCustomNatsRuntimeDeps(options)
+        ? options.jsm
+        : await jetstreamManager(options.nats);
+    const js = isCustomNatsRuntimeDeps(options) ? options.js : {
+        consumers: {
+            getConsumerFromInfo(info) {
+                return toWorkerConsumer(jetstream(options.nats).consumers.getConsumerFromInfo(info));
+            },
+        },
+    };
+    const info = await ensureConsumerInfo(jsm, options.binding.workStream, queue);
+    const consumer = js.consumers.getConsumerFromInfo(info);
+    const cancelSubscription = options.nats.subscribe(`${queue.publishPrefix}.*.cancelled`);
+    return await startQueueWorkerLoop({
+        manager: options.manager,
+        consumer,
+        cancelSubscription,
+        hostCancellation: options.hostCancellation,
+        getProjectedJob: options.getProjectedJob,
+        payloadSchema: queue.payload,
+        validatePayload: options.validatePayload,
+        resultSchema: queue.result,
+        validateResult: options.validateResult,
+        handler: options.handler,
+    });
+}
+export async function startWorkerHostFromBinding(binding, options) {
+    const queueTypes = options.queueTypes ??
+        Object.keys(binding.jobs.queues).sort();
+    for (const queueType of queueTypes) {
+        const queue = binding.jobs.queues[queueType];
+        if (!queue) {
+            throw new Error(`Requested worker queue binding '${queueType}' is missing`);
+        }
+        if (queue.concurrency < 1) {
+            throw new Error(`Worker queue '${queueType}' has invalid concurrency ${queue.concurrency}; expected >= 1`);
+        }
+    }
+    const cancellation = new JobCancellationToken();
+    const heartbeatLoops = options.heartbeatPublisher
+        ? await Promise.all(queueTypes.map((queueType) => startWorkerHeartbeatLoop({
+            publisher: options.heartbeatPublisher,
+            service: binding.jobs.namespace,
+            jobType: queueType,
+            instanceId: options.instanceId,
+            concurrency: binding.jobs.queues[queueType].concurrency,
+            version: options.version,
+            intervalMs: options.heartbeatIntervalMs,
+            nowIso: options.nowIso,
+        })))
+        : [];
+    const workers = [];
+    for (const queueType of queueTypes) {
+        const queue = binding.jobs.queues[queueType];
+        for (let workerIndex = 0; workerIndex < queue.concurrency; workerIndex += 1) {
+            const handle = await options.startWorker({
+                queueType,
+                workerIndex,
+                cancellation,
+            });
+            if (handle && typeof handle === "object" && "stop" in handle &&
+                typeof handle.stop === "function") {
+                workers.push(handle);
+            }
+        }
+    }
+    return {
+        workerCount() {
+            return workers.length;
+        },
+        async stop() {
+            cancellation.cancelForShutdown();
+            const results = await Promise.allSettled([
+                ...workers.map((worker) => worker.stop()),
+                ...heartbeatLoops.map((loop) => loop.stop()),
+            ]);
+            const failures = results
+                .filter((result) => result.status === "rejected")
+                .map((result) => result.reason);
+            if (failures.length > 0) {
+                throw new WorkerHostStopError(failures);
+            }
+        },
+    };
+}
+export async function startNatsWorkerHostFromBinding(binding, options) {
+    return await startWorkerHostFromBinding(binding, {
+        instanceId: options.instanceId,
+        queueTypes: options.queueTypes,
+        heartbeatPublisher: options.heartbeatPublisher,
+        heartbeatIntervalMs: options.heartbeatIntervalMs,
+        version: options.version,
+        nowIso: options.nowIso,
+        startWorker: async ({ queueType, cancellation }) => {
+            const common = {
+                manager: options.manager,
+                binding,
+                queueType,
+                hostCancellation: cancellation,
+                getProjectedJob: options.getProjectedJob,
+                validatePayload: options.validatePayload,
+                validateResult: options.validateResult,
+                handler: options.handler,
+            };
+            return await (isCustomNatsRuntimeDeps(options)
+                ? startNatsQueueWorker({
+                    ...common,
+                    nats: options.nats,
+                    jsm: options.jsm,
+                    js: options.js,
+                })
+                : startNatsQueueWorker({ ...common, nats: options.nats }));
+        },
+    });
+}
+async function ensureConsumerInfo(jsm, stream, queue) {
+    const config = {
+        durable_name: queue.consumerName,
+        ack_policy: "explicit",
+        filter_subject: queue.workSubject,
+        ack_wait: queue.ackWaitMs * 1_000_000,
+        max_deliver: queue.maxDeliver,
+        backoff: queue.backoffMs.map((delay) => delay * 1_000_000),
+    };
+    try {
+        return await jsm.consumers.add(stream, config);
+    }
+    catch (addError) {
+        try {
+            return await jsm.consumers.info(stream, queue.consumerName);
+        }
+        catch (infoError) {
+            if (isStreamNotFoundError(addError) || isStreamNotFoundError(infoError)) {
+                throw new JobsInfrastructureMissingError(stream, queue.queueType);
+            }
+            throw infoError;
+        }
+    }
+}
+function isStreamNotFoundError(error) {
+    return error instanceof Error && (error.name === "StreamNotFoundError" ||
+        error.message.includes("stream not found"));
+}
+function getQueueBinding(binding, queueType) {
+    const queue = binding.jobs.queues[queueType];
+    if (!queue) {
+        throw new Error(`Requested worker queue binding '${queueType}' is missing`);
+    }
+    if (queue.concurrency < 1) {
+        throw new Error(`Worker queue '${queueType}' has invalid concurrency ${queue.concurrency}; expected >= 1`);
+    }
+    return queue;
+}
+function parseWorkPayloadEvent(payload) {
+    try {
+        return JSON.parse(new TextDecoder().decode(payload));
+    }
+    catch {
+        return undefined;
+    }
+}

package/esm/npm/src/server/internal_jobs/types.d.ts

@@ -0,0 +1,87 @@
+import { type StaticDecode, Type } from "typebox";
+export declare const JobStateSchema: Type.TUnion<[Type.TLiteral<"pending">, Type.TLiteral<"active">, Type.TLiteral<"retry">, Type.TLiteral<"completed">, Type.TLiteral<"failed">, Type.TLiteral<"cancelled">, Type.TLiteral<"expired">, Type.TLiteral<"dead">, Type.TLiteral<"dismissed">]>;
+export type JobState = StaticDecode<typeof JobStateSchema>;
+export declare const JobLogEntrySchema: Type.TObject<{
+    timestamp: Type.TString;
+    level: Type.TUnion<[Type.TLiteral<"info">, Type.TLiteral<"warn">, Type.TLiteral<"error">]>;
+    message: Type.TString;
+}>;
+export type JobLogEntry = StaticDecode<typeof JobLogEntrySchema>;
+export declare const JobProgressSchema: Type.TObject<{
+    step: Type.TOptional<Type.TString>;
+    message: Type.TOptional<Type.TString>;
+    current: Type.TOptional<Type.TInteger>;
+    total: Type.TOptional<Type.TInteger>;
+}>;
+export type JobProgress = StaticDecode<typeof JobProgressSchema>;
+export declare const JobSchema: Type.TObject<{
+    id: Type.TString;
+    service: Type.TString;
+    type: Type.TString;
+    state: Type.TUnion<[Type.TLiteral<"pending">, Type.TLiteral<"active">, Type.TLiteral<"retry">, Type.TLiteral<"completed">, Type.TLiteral<"failed">, Type.TLiteral<"cancelled">, Type.TLiteral<"expired">, Type.TLiteral<"dead">, Type.TLiteral<"dismissed">]>;
+    payload: Type.TUnknown;
+    result: Type.TOptional<Type.TUnknown>;
+    createdAt: Type.TString;
+    updatedAt: Type.TString;
+    startedAt: Type.TOptional<Type.TString>;
+    completedAt: Type.TOptional<Type.TString>;
+    tries: Type.TInteger;
+    maxTries: Type.TInteger;
+    lastError: Type.TOptional<Type.TString>;
+    deadline: Type.TOptional<Type.TString>;
+    progress: Type.TOptional<Type.TObject<{
+        step: Type.TOptional<Type.TString>;
+        message: Type.TOptional<Type.TString>;
+        current: Type.TOptional<Type.TInteger>;
+        total: Type.TOptional<Type.TInteger>;
+    }>>;
+    logs: Type.TOptional<Type.TArray<Type.TObject<{
+        timestamp: Type.TString;
+        level: Type.TUnion<[Type.TLiteral<"info">, Type.TLiteral<"warn">, Type.TLiteral<"error">]>;
+        message: Type.TString;
+    }>>>;
+}>;
+export type Job<TPayload = unknown, TResult = unknown> = Omit<StaticDecode<typeof JobSchema>, "payload" | "result"> & {
+    payload: TPayload;
+    result?: TResult;
+};
+export declare const JobEventSchema: Type.TObject<{
+    jobId: Type.TString;
+    service: Type.TString;
+    jobType: Type.TString;
+    eventType: Type.TUnion<[Type.TLiteral<"created">, Type.TLiteral<"started">, Type.TLiteral<"retry">, Type.TLiteral<"progress">, Type.TLiteral<"logged">, Type.TLiteral<"completed">, Type.TLiteral<"failed">, Type.TLiteral<"cancelled">, Type.TLiteral<"expired">, Type.TLiteral<"retried">, Type.TLiteral<"dead">, Type.TLiteral<"dismissed">]>;
+    state: Type.TUnion<[Type.TLiteral<"pending">, Type.TLiteral<"active">, Type.TLiteral<"retry">, Type.TLiteral<"completed">, Type.TLiteral<"failed">, Type.TLiteral<"cancelled">, Type.TLiteral<"expired">, Type.TLiteral<"dead">, Type.TLiteral<"dismissed">]>;
+    previousState: Type.TOptional<Type.TUnion<[Type.TLiteral<"pending">, Type.TLiteral<"active">, Type.TLiteral<"retry">, Type.TLiteral<"completed">, Type.TLiteral<"failed">, Type.TLiteral<"cancelled">, Type.TLiteral<"expired">, Type.TLiteral<"dead">, Type.TLiteral<"dismissed">]>>;
+    tries: Type.TInteger;
+    maxTries: Type.TOptional<Type.TInteger>;
+    error: Type.TOptional<Type.TString>;
+    progress: Type.TOptional<Type.TObject<{
+        step: Type.TOptional<Type.TString>;
+        message: Type.TOptional<Type.TString>;
+        current: Type.TOptional<Type.TInteger>;
+        total: Type.TOptional<Type.TInteger>;
+    }>>;
+    logs: Type.TOptional<Type.TArray<Type.TObject<{
+        timestamp: Type.TString;
+        level: Type.TUnion<[Type.TLiteral<"info">, Type.TLiteral<"warn">, Type.TLiteral<"error">]>;
+        message: Type.TString;
+    }>>>;
+    payload: Type.TOptional<Type.TUnknown>;
+    result: Type.TOptional<Type.TUnknown>;
+    deadline: Type.TOptional<Type.TString>;
+    timestamp: Type.TString;
+}>;
+export type JobEvent<TPayload = unknown, TResult = unknown> = Omit<StaticDecode<typeof JobEventSchema>, "payload" | "result"> & {
+    payload?: TPayload;
+    result?: TResult;
+};
+export declare const WorkerHeartbeatSchema: Type.TObject<{
+    service: Type.TString;
+    jobType: Type.TString;
+    instanceId: Type.TString;
+    concurrency: Type.TOptional<Type.TInteger>;
+    version: Type.TOptional<Type.TString>;
+    timestamp: Type.TString;
+}>;
+export type WorkerHeartbeat = StaticDecode<typeof WorkerHeartbeatSchema>;
+//# sourceMappingURL=types.d.ts.map

package/esm/npm/src/server/internal_jobs/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../../../src/server/internal_jobs/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,YAAY,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAElD,eAAO,MAAM,cAAc,wPAUzB,CAAC;AAEH,MAAM,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,cAAc,CAAC,CAAC;AAE3D,eAAO,MAAM,iBAAiB;;;;EAQ5B,CAAC;AAEH,MAAM,MAAM,WAAW,GAAG,YAAY,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAEjE,eAAO,MAAM,iBAAiB;;;;;EAK5B,CAAC;AAEH,MAAM,MAAM,WAAW,GAAG,YAAY,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAEjE,eAAO,MAAM,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;;EAiBpB,CAAC;AAEH,MAAM,MAAM,GAAG,CAAC,QAAQ,GAAG,OAAO,EAAE,OAAO,GAAG,OAAO,IACjD,IAAI,CAAC,YAAY,CAAC,OAAO,SAAS,CAAC,EAAE,SAAS,GAAG,QAAQ,CAAC,GAC1D;IACA,OAAO,EAAE,QAAQ,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB,CAAC;AAEJ,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;EA6BzB,CAAC;AAEH,MAAM,MAAM,QAAQ,CAAC,QAAQ,GAAG,OAAO,EAAE,OAAO,GAAG,OAAO,IACtD,IAAI,CAAC,YAAY,CAAC,OAAO,cAAc,CAAC,EAAE,SAAS,GAAG,QAAQ,CAAC,GAC/D;IACA,OAAO,CAAC,EAAE,QAAQ,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB,CAAC;AAEJ,eAAO,MAAM,qBAAqB;;;;;;;EAOhC,CAAC;AAEH,MAAM,MAAM,eAAe,GAAG,YAAY,CAAC,OAAO,qBAAqB,CAAC,CAAC"}