@qlever-llc/trellis 0.10.17 → 0.19.0-rc.1

package/src/server/internal_jobs/projection.ts

@@ -5,6 +5,8 @@ const TERMINAL_STATES = new Set<JobState>([
   "failed",
   "cancelled",
   "expired",
+  "skipped",
+  "stale",
   "dead",
   "dismissed",
 ]);

package/src/server/internal_jobs/runtime-worker.ts

@@ -177,6 +177,8 @@ type StartQueueWorkerLoopOptions<TResult> = {
     args: ResultValidationArgs<TResult>,
   ) => Promise<void> | void;
   handler: (job: ActiveJob<unknown, TResult>) => Promise<TResult>;
+  instanceId?: string;
+  deferralBackoffMs?: number;
 };
 
 type StartNatsQueueWorkerOptions<TResult> =
@@ -199,6 +201,7 @@ type StartNatsQueueWorkerOptions<TResult> =
       args: ResultValidationArgs<TResult>,
     ) => Promise<void> | void;
     handler: (job: ActiveJob<unknown, TResult>) => Promise<TResult>;
+    instanceId?: string;
   };
 
 function toWorkerConsumer(
@@ -252,7 +255,8 @@ export async function processWorkPayloadWithContextAndHeartbeat<TResult>(
       args: ResultValidationArgs<TResult>,
     ) => Promise<void> | void;
   },
-  runtime?: { redeliveryCount?: number },
+  runtime?: { latestState?: Job["state"]; redeliveryCount?: number },
+  instanceId?: string,
 ): Promise<JobProcessOutcome<TResult> | undefined> {
   const event = parseWorkPayloadEvent(payload);
   if (!event) {
@@ -280,7 +284,9 @@ export async function processWorkPayloadWithContextAndHeartbeat<TResult>(
       return await handler(activeJob);
     },
     {
+      latestState: runtime?.latestState,
       redeliveryCount: runtime?.redeliveryCount,
+      instanceId,
     },
     {
       validateResult: validation?.validateResult
@@ -323,6 +329,7 @@ export function ackActionForOutcome(
   switch (outcome.outcome) {
     case "retry":
     case "interrupted":
+    case "deferred":
       return "nak";
     default:
       return "ack";
@@ -381,6 +388,7 @@ export async function startQueueWorkerLoop<TResult>(
         continue;
       }
       if (lifecycleWorkDecision(latestLifecycle) === "skip-ack") {
+        await options.manager.cleanupQueuedKeyedJob(job);
         registry.clearPending(key);
         await msg.ack();
         continue;
@@ -394,6 +402,7 @@ export async function startQueueWorkerLoop<TResult>(
           continue;
         }
         if (projectedWorkDecision(projected, job) === "skip-ack") {
+          await options.manager.cleanupQueuedKeyedJob(job);
           registry.clearPending(key);
           await msg.ack();
           continue;
@@ -422,11 +431,15 @@ export async function startQueueWorkerLoop<TResult>(
             validateResult: options.validateResult,
           },
           {
+            latestState: latestLifecycle?.state,
             redeliveryCount: msg.info?.redeliveryCount,
           },
+          options.instanceId,
         );
         if (ackActionForOutcome(outcome) === "ack") {
           await msg.ack();
+        } else if (outcome?.outcome === "deferred") {
+          await msg.nak(options.deferralBackoffMs ?? 1_000);
         } else {
           await msg.nak();
         }
@@ -512,6 +525,8 @@ export async function startNatsQueueWorker<TResult>(
     resultSchema: queue.result,
     validateResult: options.validateResult,
     handler: options.handler,
+    instanceId: options.instanceId,
+    deferralBackoffMs: queue.backoffMs[0] ?? 1_000,
   });
 }
 
@@ -624,6 +639,7 @@ export async function startNatsWorkerHostFromBinding<TResult>(
         validatePayload: options.validatePayload,
         validateResult: options.validateResult,
         handler: options.handler,
+        instanceId: options.instanceId,
       };
       return await (isCustomNatsRuntimeDeps(options)
         ? startNatsQueueWorker({

package/src/server/internal_jobs/types.ts

@@ -19,6 +19,8 @@ export const JobStateSchema = Type.Union([
   Type.Literal("failed"),
   Type.Literal("cancelled"),
   Type.Literal("expired"),
+  Type.Literal("skipped"),
+  Type.Literal("stale"),
   Type.Literal("dead"),
   Type.Literal("dismissed"),
 ]);
@@ -87,6 +89,10 @@ export const JobEventSchema = Type.Object({
     Type.Literal("failed"),
     Type.Literal("cancelled"),
     Type.Literal("expired"),
+    Type.Literal("skipped"),
+    Type.Literal("stale"),
+    Type.Literal("heartbeat"),
+    Type.Literal("staleCompletionIgnored"),
     Type.Literal("retried"),
     Type.Literal("dead"),
     Type.Literal("dismissed"),

package/src/server/service.ts

@@ -105,9 +105,11 @@ import {
   ActiveJob as PublicActiveJob,
   type JobIdentity,
   type JobLogEntry,
+  JobNotEnqueuedError,
   type JobProgress,
   JobRef,
   type JobSnapshot,
+  type JobSubmitOutcome,
   JobWorkerHostAdapter,
   type TerminalJob,
 } from "../jobs.js";
@@ -116,6 +118,18 @@ import {
   JobProcessError as InternalJobProcessError,
 } from "./internal_jobs/job-manager.js";
 import { startNatsWorkerHostFromBinding } from "./internal_jobs/runtime-worker.js";
+import {
+  createNatsJobKeyCoordinator,
+  normalizeJobKeyPolicy,
+} from "./internal_jobs/key-coordinator.js";
+import type {
+  JobKeyConcurrencyBinding,
+  JobQueuePolicyBinding,
+} from "./internal_jobs/key-coordinator.js";
+import type {
+  JobsBinding,
+  JobsQueueBinding,
+} from "./internal_jobs/bindings.js";
 import type { ActiveJob as InternalActiveJob } from "./internal_jobs/active-job.js";
 import {
   type Job as InternalJob,
@@ -149,6 +163,8 @@ type ResourceBindingJobsQueue = {
   logs: boolean;
   dlq: boolean;
   concurrency: number;
+  keyConcurrency?: JobKeyConcurrencyBinding;
+  queue?: JobQueuePolicyBinding;
 };
 
 type ResourceBindingJobs = {
@@ -157,6 +173,71 @@ type ResourceBindingJobs = {
   queues: Record<string, ResourceBindingJobsQueue>;
 };
 
+function normalizeResourceJobsBinding(
+  binding: ResourceBindingJobs,
+): JobsBinding {
+  const queues: Record<string, JobsQueueBinding> = {};
+  for (const [name, queue] of Object.entries(binding.queues)) {
+    const baseQueue = baseJobsQueueBinding(queue);
+    if (!queue.keyConcurrency) {
+      queues[name] = {
+        ...baseQueue,
+        ...(queue.queue ? { queue: normalizeQueuePolicy(queue.queue) } : {}),
+      };
+      continue;
+    }
+
+    const policy = normalizeJobKeyPolicy({
+      keyConcurrency: queue.keyConcurrency,
+      queue: queue.queue,
+    });
+    queues[name] = {
+      ...baseQueue,
+      keyConcurrency: {
+        key: policy.key,
+        maxActive: policy.maxActive,
+        heartbeatIntervalMs: policy.heartbeatIntervalMs,
+        heartbeatTtlMs: policy.heartbeatTtlMs,
+        stalePolicy: policy.stalePolicy,
+      },
+      queue: policy.queue,
+    };
+  }
+  return { namespace: binding.namespace, queues };
+}
+
+function baseJobsQueueBinding(
+  queue: ResourceBindingJobsQueue,
+): Omit<JobsQueueBinding, "keyConcurrency" | "queue"> {
+  return {
+    queueType: queue.queueType,
+    publishPrefix: queue.publishPrefix,
+    workSubject: queue.workSubject,
+    consumerName: queue.consumerName,
+    payload: queue.payload,
+    ...(queue.result ? { result: queue.result } : {}),
+    maxDeliver: queue.maxDeliver,
+    backoffMs: queue.backoffMs,
+    ackWaitMs: queue.ackWaitMs,
+    ...(queue.defaultDeadlineMs !== undefined
+      ? { defaultDeadlineMs: queue.defaultDeadlineMs }
+      : {}),
+    progress: queue.progress,
+    logs: queue.logs,
+    dlq: queue.dlq,
+    concurrency: queue.concurrency,
+  };
+}
+
+function normalizeQueuePolicy(
+  queue: JobQueuePolicyBinding,
+): JobsQueueBinding["queue"] {
+  return {
+    maxQueuedPerKey: queue.maxQueuedPerKey ?? 0,
+    whenFull: queue.whenFull ?? "reject",
+  };
+}
+
 type ResourceBindingEventConsumer = {
   stream: string;
   consumerName: string;
@@ -1208,6 +1289,9 @@ export type JobQueue<
   TJobs extends ContractJobsMetadata = {},
 > = {
   create(payload: TPayload): AsyncResult<JobRef<TPayload, TResult>, BaseError>;
+  submit(
+    payload: TPayload,
+  ): AsyncResult<JobSubmitOutcome<TPayload, TResult>, BaseError>;
   handle(
     handler: (args: {
       job: PublicActiveJob<TPayload, TResult>;
@@ -1239,7 +1323,7 @@ type ServiceEventOf<
 type ServiceEventPayloadOf<
   TA extends TrellisAPI,
   E extends ServiceEventName<TA>,
-> = Omit<ServiceEventOf<TA, E>, "header">;
+> = ServiceEventOf<TA, E> & Record<string, unknown>;
 
 type BoundEventHandleFn<
   TEventApi extends TrellisAPI,
@@ -1249,7 +1333,7 @@ type BoundEventHandleFn<
   TJobs extends ContractJobsMetadata,
   TDeps,
 > = (args: {
-  event: ServiceEventOf<TEventApi, E>;
+  event: ServiceEventPayloadOf<TEventApi, E>;
   context: EventListenerContext;
   client: Trellis<TTrellisApi, TKv, TJobs>;
   deps: TDeps;
@@ -1419,6 +1503,9 @@ type BoundJobQueue<
   TDeps,
 > = {
   create(payload: TPayload): AsyncResult<JobRef<TPayload, TResult>, BaseError>;
+  submit(
+    payload: TPayload,
+  ): AsyncResult<JobSubmitOutcome<TPayload, TResult>, BaseError>;
   handle(
     handler: (args: {
       job: PublicActiveJob<TPayload, TResult>;
@@ -1922,6 +2009,7 @@ export async function createConnectedService<
     prepare: (event, data) => outbound.prepare(event, data),
     publish: (event, data) => outbound.publish(event, data),
     publishPrepared: (event) => outbound.publishPrepared(event),
+    stopEventListeners: () => outbound.stopEventListeners(),
     get kv() {
       return getHandlerResources().kv;
     },
@@ -2081,7 +2169,8 @@ function isTerminalJobState(
   state: string,
 ): state is TerminalJob<unknown, unknown>["state"] {
   return state === "completed" || state === "failed" || state === "cancelled" ||
-    state === "expired" || state === "dead" || state === "dismissed";
+    state === "expired" || state === "skipped" || state === "stale" ||
+    state === "dead" || state === "dismissed";
 }
 
 function isTerminalJobSnapshot<TPayload, TResult>(
@@ -2202,6 +2291,9 @@ function snapshotFromLifecycleEvent<TPayload, TResult>(
     case "failed":
     case "cancelled":
     case "expired":
+    case "skipped":
+    case "stale":
+    case "staleCompletionIgnored":
     case "dead":
     case "dismissed":
       return event.error === undefined ? base : {
@@ -2359,8 +2451,8 @@ function createJobLifecycleTracker(nc: NatsConnection): JobLifecycleTracker {
 function createJobRef<TPayload, TResult>(args: {
   nc: NatsConnection;
   queueType: string;
-  jobsBinding: ResourceBindingJobs;
-  queueBinding: ResourceBindingJobsQueue;
+  jobsBinding: JobsBinding;
+  queueBinding: JobsQueueBinding;
   seed: JobSnapshot<TPayload, TResult>;
   lifecycle: JobLifecycleTracker;
 }): JobRef<TPayload, TResult> {
@@ -2467,6 +2559,10 @@ function createJobsFacade<
   const handlers = new Map<string, RegisteredJobHandler<unknown, unknown>>();
   const jobsFacade: Record<string, unknown> = {};
   const lifecycle = createJobLifecycleTracker(args.nc);
+  const keyCoordinator = createNatsJobKeyCoordinator(args.nc);
+  const jobsBinding = args.jobsBinding
+    ? normalizeResourceJobsBinding(args.jobsBinding)
+    : undefined;
   let activeHost: JobWorkerHostAdapter | undefined;
   let startupPromise:
     | Promise<Result<JobWorkerHostAdapter, BaseError>>
@@ -2474,14 +2570,13 @@ function createJobsFacade<
   let stopPromise: Promise<Result<void, BaseError>> | undefined;
 
   for (const queueType of Object.keys(args.contractJobs ?? {})) {
-    const queueBinding = args.jobsBinding?.queues[queueType];
+    const queueBinding = jobsBinding?.queues[queueType];
     if (queueBinding) lifecycle.watch(queueBinding);
 
     jobsFacade[queueType] = {
       create: (payload) =>
         AsyncResult.from((async () => {
           try {
-            const jobsBinding = args.jobsBinding;
             if (!jobsBinding) {
               return Result.err(
                 toUnexpectedError(new Error("Jobs bindings are unavailable")),
@@ -2499,6 +2594,7 @@ function createJobsFacade<
             const manager = new InternalJobManager<unknown, unknown>({
               nc: args.nc,
               jobs: jobsBinding,
+              keyCoordinator,
             });
             await args.nc.flush();
             const created = await manager.create(queueType, payload);
@@ -2510,6 +2606,67 @@ function createJobsFacade<
               seed: created as JobSnapshot<unknown, unknown>,
               lifecycle,
             }));
+          } catch (cause) {
+            if (cause instanceof JobNotEnqueuedError) {
+              return Result.err(cause);
+            }
+            return Result.err(toUnexpectedError(cause));
+          }
+        })()),
+      submit: (payload) =>
+        AsyncResult.from((async () => {
+          try {
+            if (!jobsBinding) {
+              return Result.err(
+                toUnexpectedError(new Error("Jobs bindings are unavailable")),
+              );
+            }
+            const queueBinding = jobsBinding.queues[queueType];
+            if (!queueBinding) {
+              return Result.err(toUnexpectedError(
+                new Error(
+                  `Jobs binding for queue '${queueType}' is unavailable`,
+                ),
+              ));
+            }
+
+            const manager = new InternalJobManager<unknown, unknown>({
+              nc: args.nc,
+              jobs: jobsBinding,
+              keyCoordinator,
+            });
+            await args.nc.flush();
+            const outcome = await manager.submit(queueType, payload);
+            if (outcome.kind === "accepted") {
+              return Result.ok({
+                kind: "accepted",
+                key: outcome.key,
+                ref: createJobRef({
+                  nc: args.nc,
+                  queueType,
+                  jobsBinding,
+                  queueBinding,
+                  seed: outcome.job as JobSnapshot<unknown, unknown>,
+                  lifecycle,
+                }),
+              });
+            }
+            if (outcome.kind === "replaced") {
+              return Result.ok({
+                kind: "replaced",
+                key: outcome.key,
+                replaced: outcome.replaced,
+                ref: createJobRef({
+                  nc: args.nc,
+                  queueType,
+                  jobsBinding,
+                  queueBinding,
+                  seed: outcome.job as JobSnapshot<unknown, unknown>,
+                  lifecycle,
+                }),
+              });
+            }
+            return Result.ok(outcome);
           } catch (cause) {
             return Result.err(toUnexpectedError(cause));
           }
@@ -2554,7 +2711,7 @@ function createJobsFacade<
           return Result.ok(host);
         }
 
-        if (!args.jobsBinding || !args.workStream) {
+        if (!jobsBinding || !args.workStream) {
           return Result.err(toUnexpectedError(
             new Error(
               "Jobs infrastructure bindings are unavailable for this service",
@@ -2562,7 +2719,6 @@ function createJobsFacade<
           ));
         }
 
-        const jobsBinding = args.jobsBinding;
         const workStream = args.workStream;
 
         const hosts = [] as Array<{ stop(): Promise<void> }>;
@@ -2582,6 +2738,7 @@ function createJobsFacade<
             const manager = new InternalJobManager<unknown, unknown>({
               nc: args.nc,
               jobs: jobsBinding,
+              keyCoordinator,
             });
             const host = await startNatsWorkerHostFromBinding<unknown>({
               jobs: jobsBinding,
@@ -2750,6 +2907,7 @@ function createBoundJobsFacade<
     if (!queue) continue;
     boundJobs[queueType] = {
       create: (payload) => queue.create(payload),
+      submit: (payload) => queue.submit(payload),
       handle: (handler) =>
         queue.handle(({ job, client }) =>
           handler({
@@ -3339,6 +3497,7 @@ export class TrellisService<
   async stop(): Promise<void> {
     this.#stopPromise ??= (async () => {
       this.connection.stopObserving();
+      this.#handlerTrellis.stopEventListeners();
 
       try {
         await this.#stopHealthPublishing();

package/src/service/drizzle.ts

@@ -0,0 +1,125 @@
+import "../_dnt.polyfills.js";
+import { type SQL, sql } from "drizzle-orm";
+import type { SqlExecutor, SqlRow } from "./outbox_inbox.js";
+
+/**
+ * Structural Drizzle SQLite database or transaction shape accepted by Trellis
+ * SQL outbox helpers.
+ */
+export type DrizzleSqlDatabase = {
+  /** Runs a SQL statement that returns rows. */
+  all(query: SQL): Promise<readonly SqlRow[]>;
+  /** Runs a SQL statement that does not return rows. */
+  run(query: SQL): Promise<unknown>;
+};
+
+/**
+ * Adapts a caller-owned Drizzle SQLite database or transaction to Trellis'
+ * generic `SqlExecutor` interface.
+ */
+export function createDrizzleSqlExecutor(
+  database: DrizzleSqlDatabase,
+): SqlExecutor {
+  return {
+    query(statement, params) {
+      return database.all(bindDrizzleSqlStatement(statement, params));
+    },
+    async execute(statement, params) {
+      await database.run(bindDrizzleSqlStatement(statement, params));
+    },
+  };
+}
+
+/**
+ * Converts a Trellis SQL statement with positional placeholders into a Drizzle
+ * `SQL` object with parameters bound in order.
+ *
+ * Supports SQLite-style `?` placeholders and PostgreSQL-style `$1`, `$2`, ...
+ * placeholders. A single statement must not mix both styles.
+ */
+export function bindDrizzleSqlStatement(
+  statement: string,
+  params: readonly unknown[],
+): SQL {
+  const matches = [...statement.matchAll(/\?|\$(\d+)/g)];
+  const hasQuestionPlaceholders = matches.some((match) => match[0] === "?");
+  const hasNumberedPlaceholders = matches.some((match) =>
+    match[1] !== undefined
+  );
+
+  if (hasQuestionPlaceholders && hasNumberedPlaceholders) {
+    throw new Error(
+      "SQL statement cannot mix ? and $n placeholders for Drizzle binding",
+    );
+  }
+
+  if (hasQuestionPlaceholders) {
+    validateQuestionPlaceholders(matches.length, params.length);
+  } else if (hasNumberedPlaceholders) {
+    validateNumberedPlaceholders(matches, params.length);
+  } else if (params.length !== 0) {
+    throw new Error(
+      `SQL statement has no placeholders but received ${params.length} parameters`,
+    );
+  }
+
+  const chunks: SQL[] = [];
+  let cursor = 0;
+  let nextQuestionParam = 0;
+  for (const match of matches) {
+    const matchIndex = match.index;
+    chunks.push(sql.raw(statement.slice(cursor, matchIndex)));
+    if (match[0] === "?") {
+      chunks.push(sql`${params[nextQuestionParam]}`);
+      nextQuestionParam += 1;
+    } else {
+      const index = Number(match[1]) - 1;
+      chunks.push(sql`${params[index]}`);
+    }
+    cursor = matchIndex + match[0].length;
+  }
+  chunks.push(sql.raw(statement.slice(cursor)));
+
+  return sql.join(chunks);
+}
+
+function validateQuestionPlaceholders(
+  placeholders: number,
+  paramCount: number,
+): void {
+  if (placeholders !== paramCount) {
+    throw new Error(
+      `SQL statement expected ${placeholders} parameters for ? placeholders, received ${paramCount}`,
+    );
+  }
+}
+
+function validateNumberedPlaceholders(
+  matches: RegExpMatchArray[],
+  paramCount: number,
+): void {
+  const referenced = new Set<number>();
+  for (const match of matches) {
+    if (match[1] === undefined) continue;
+    const index = Number(match[1]);
+    if (index < 1) {
+      throw new Error("PostgreSQL SQL placeholder indexes must start at $1");
+    }
+    referenced.add(index);
+  }
+
+  const maxIndex = Math.max(...referenced);
+  if (maxIndex !== paramCount) {
+    throw new Error(
+      `SQL statement expected ${maxIndex} parameters for $n placeholders, received ${paramCount}`,
+    );
+  }
+
+  for (let index = 1; index <= maxIndex; index += 1) {
+    if (!referenced.has(index)) {
+      throw new Error(
+        `SQL statement is missing PostgreSQL placeholder $${index}`,
+      );
+    }
+  }
+}

package/src/service/mod.ts

@@ -28,7 +28,6 @@ export {
 export { mountStandardHealthRpc } from "../server/health_rpc.js";
 export {
   HealthCheckResultSchema,
-  HealthHeartbeatHeaderSchema,
   HealthHeartbeatSchema,
   HealthHeartbeatServiceSchema,
   HealthInfoSchema,

package/src/service/outbox_inbox.ts

@@ -698,14 +698,13 @@ export async function dispatchOutbox(
 export function outboxMessageToPrepared(
   message: OutboxMessage,
 ): PreparedTrellisEvent {
+  const payload = JSON.parse(message.payload) as Record<string, unknown>;
+  const header = eventHeaderFromMessage(message.headers);
   return Object.freeze({
     event: message.event,
     subject: message.subject,
-    payload: Object.freeze(
-      JSON.parse(message.payload) as Record<string, unknown> & {
-        header: { id: string; time: string };
-      },
-    ),
+    header: Object.freeze(header),
+    payload: Object.freeze(payload),
     encodedPayload: message.payload,
     headers: Object.freeze({ ...message.headers }),
   });
@@ -713,7 +712,18 @@ export function outboxMessageToPrepared(
 
 function messageId(event: PreparedTrellisEvent): string {
   return event.headers["Nats-Msg-Id"] ?? event.headers["nats-msg-id"] ??
-    event.payload.header.id;
+    event.header.id;
+}
+
+function eventHeaderFromMessage(
+  headers: Record<string, string>,
+): { id: string; time: string } {
+  const id = headers["Nats-Msg-Id"] ?? headers["nats-msg-id"];
+  const time = headers["Trellis-Event-Time"] ?? headers["trellis-event-time"];
+  return {
+    id: typeof id === "string" ? id : "",
+    time: typeof time === "string" ? time : new Date(0).toISOString(),
+  };
 }
 
 function rowToOutboxMessage(row: SqlRow): OutboxMessage {