@inneranimalmedia/agentsam-sdk 2.6.10 → 2.6.12

package/packages/agentsam-queue-control/src/contracts.js

@@ -7,6 +7,7 @@ export const JOB_STATUSES = Object.freeze([
   'waiting',
   'batch_waiting',
   'awaiting_input',
+  'retry_scheduled',
   'completed',
   'failed',
   'cancelled',
@@ -36,6 +37,7 @@ export const DEFAULT_RETRY_POLICY = Object.freeze({
   backoff: 'exponential',
   initial_delay_ms: 1_000,
   max_delay_ms: 60_000,
+  jitter: true,
 });
 
 function clean(value) {
@@ -48,6 +50,198 @@ function assertEnum(value, allowed, field) {
   }
 }
 
+function epochSeconds(value, fallback, field = 'timestamp') {
+  if (value == null || value === '') return fallback;
+  const numeric = Number(value);
+  if (!Number.isFinite(numeric) || numeric < 0) throw new TypeError(`${field} must be a non-negative epoch second`);
+  return Math.floor(numeric);
+}
+
+export function computeRetryDelayMs(retry = DEFAULT_RETRY_POLICY, attempt = 1, random = Math.random) {
+  const policy = { ...DEFAULT_RETRY_POLICY, ...(retry || {}) };
+  const n = Math.max(1, Number.isInteger(attempt) ? attempt : 1);
+  const initial = Math.max(0, Number(policy.initial_delay_ms) || 0);
+  const maximum = Math.max(initial, Number(policy.max_delay_ms) || initial);
+  let delay = policy.backoff === 'exponential'
+    ? initial * (2 ** (n - 1))
+    : initial;
+  delay = Math.min(maximum, delay);
+  if (policy.jitter && delay > 0) {
+    const sample = Math.max(0, Math.min(1, Number(random()) || 0));
+    delay = Math.floor((delay / 2) + ((delay / 2) * sample));
+  }
+  return Math.max(0, Math.floor(delay));
+}
+
+export function createRetrySchedule(job, { now_ms = Date.now(), random = Math.random, error = null } = {}) {
+  assertJobEnvelope(job);
+  const nextAttempt = (Number.isInteger(job.attempt) ? job.attempt : 0) + 1;
+  const maxAttempts = Math.max(1, Number(job.retry?.max_attempts) || DEFAULT_RETRY_POLICY.max_attempts);
+  if (nextAttempt >= maxAttempts) {
+    return {
+      retryable: false,
+      exhausted: true,
+      attempt: nextAttempt,
+      delay_ms: 0,
+      available_at: null,
+      job: null,
+    };
+  }
+  const delayMs = computeRetryDelayMs(job.retry, nextAttempt, random);
+  const availableAt = Math.ceil((Number(now_ms) + delayMs) / 1000);
+  return {
+    retryable: true,
+    exhausted: false,
+    attempt: nextAttempt,
+    delay_ms: delayMs,
+    available_at: availableAt,
+    job: {
+      ...job,
+      status: 'retry_scheduled',
+      attempt: nextAttempt,
+      available_at: availableAt,
+      lease_owner: null,
+      lease_expires_at: null,
+      metadata: {
+        ...(job.metadata || {}),
+        ...(error ? { last_error: String(error?.message || error) } : {}),
+      },
+    },
+  };
+}
+
+export function createDeadLetterJob(job, {
+  error = null,
+  attempt = (Number.isInteger(job?.attempt) ? job.attempt : 0) + 1,
+  now = Math.floor(Date.now() / 1000),
+} = {}) {
+  assertJobEnvelope(job);
+  const originalKey = clean(job.idempotency_key) || job.id;
+  return createJobEnvelope({
+    id: job.id + ':dlq:' + attempt,
+    account_id: job.account_id,
+    kind: 'dead_letter',
+    logical_queue: 'dead_letter',
+    executor: 'queue',
+    priority: job.priority === 'urgent' ? 'urgent' : 'high',
+    parent_job_id: job.id,
+    conversation_id: job.conversation_id,
+    agent_id: job.agent_id,
+    source_run_id: job.source_run_id,
+    step_id: job.step_id,
+    idempotency_key: originalKey + ':dlq:' + attempt,
+    payload: {
+      original_job: job,
+      failure: error == null ? null : {
+        message: String(error?.message || error),
+        code: clean(error?.code) || null,
+      },
+    },
+    metadata: {
+      ...(job.metadata || {}),
+      dead_letter: true,
+      original_job_id: job.id,
+      original_kind: job.kind,
+      exhausted_attempt: attempt,
+    },
+    retry: {
+      max_attempts: 1,
+      backoff: 'fixed',
+      initial_delay_ms: 0,
+      max_delay_ms: 0,
+      jitter: false,
+    },
+    attempt: 0,
+    available_at: now,
+    created_at: now,
+  });
+}
+
+export function isJobAvailable(job, now = Math.floor(Date.now() / 1000)) {
+  return (job?.available_at ?? 0) <= now;
+}
+
+export function jobExecutionIdentity(job) {
+  assertJobEnvelope(job);
+  return Object.freeze({
+    job_id: job.id,
+    run_id: job.source_run_id ?? null,
+    step_id: job.step_id ?? null,
+    attempt: job.attempt ?? 0,
+    idempotency_key: clean(job.idempotency_key) || job.id,
+    lease_owner: job.lease_owner ?? null,
+    lease_expires_at: job.lease_expires_at ?? null,
+    lease_generation: job.lease_generation ?? 0,
+  });
+}
+
+export function canClaimJobLease(job, { now = Math.floor(Date.now() / 1000), owner = null } = {}) {
+  assertJobEnvelope(job);
+  if (isTerminalJobStatus(job.status)) return false;
+  if (!isJobAvailable(job, now)) return false;
+  const leaseOwner = clean(job.lease_owner);
+  const leaseExpiresAt = job.lease_expires_at == null
+    ? null
+    : epochSeconds(job.lease_expires_at, null, 'lease_expires_at');
+  if (!leaseOwner || leaseExpiresAt == null || leaseExpiresAt <= now) return true;
+  return Boolean(owner && leaseOwner === clean(owner));
+}
+
+export function claimJobLease(job, { owner, now = Math.floor(Date.now() / 1000), ttl_seconds = 60 } = {}) {
+  assertJobEnvelope(job);
+  const leaseOwner = clean(owner);
+  if (!leaseOwner) throw new TypeError('lease owner is required');
+  const ttl = Math.max(1, Math.floor(Number(ttl_seconds) || 0));
+  if (!canClaimJobLease(job, { now, owner: leaseOwner })) {
+    const error = new Error(isJobAvailable(job, now) ? 'job_lease_held' : 'job_not_available');
+    error.code = isJobAvailable(job, now) ? 'lease_held' : 'job_not_available';
+    throw error;
+  }
+  return {
+    ...job,
+    status: 'claimed',
+    lease_owner: leaseOwner,
+    lease_expires_at: now + ttl,
+    lease_generation: Math.max(0, Number(job.lease_generation) || 0) + 1,
+  };
+}
+
+export function refreshJobLease(job, { owner, now = Math.floor(Date.now() / 1000), ttl_seconds = 60 } = {}) {
+  assertJobEnvelope(job);
+  const leaseOwner = clean(owner);
+  if (!leaseOwner || leaseOwner !== clean(job.lease_owner)) {
+    const error = new Error('job_lease_owner_mismatch');
+    error.code = 'lease_owner_mismatch';
+    throw error;
+  }
+  const currentExpiry = epochSeconds(job.lease_expires_at, 0, 'lease_expires_at');
+  if (currentExpiry <= now) {
+    const error = new Error('job_lease_expired');
+    error.code = 'lease_expired';
+    throw error;
+  }
+  const ttl = Math.max(1, Math.floor(Number(ttl_seconds) || 0));
+  return { ...job, lease_expires_at: now + ttl };
+}
+
+export function releaseJobLease(job, { owner, status = 'queued', available_at = job.available_at } = {}) {
+  assertJobEnvelope(job);
+  const leaseOwner = clean(owner);
+  if (leaseOwner && clean(job.lease_owner) && leaseOwner !== clean(job.lease_owner)) {
+    const error = new Error('job_lease_owner_mismatch');
+    error.code = 'lease_owner_mismatch';
+    throw error;
+  }
+  assertEnum(status, JOB_STATUSES, 'status');
+  return {
+    ...job,
+    status,
+    available_at: epochSeconds(available_at, job.available_at, 'available_at'),
+    lease_owner: null,
+    lease_expires_at: null,
+  };
+}
+
 export function createJobEnvelope(input = {}) {
   const kind = clean(input.kind);
   if (!kind) throw new TypeError('kind is required');
@@ -56,6 +250,8 @@ export function createJobEnvelope(input = {}) {
   if (!accountId) throw new TypeError('account_id is required');
 
   const now = input.created_at ?? Math.floor(Date.now() / 1000);
+  const availableAt = epochSeconds(input.available_at ?? input.availableAt, now, 'available_at');
+  const id = clean(input.id) || `job_${randomUUID().replaceAll('-', '')}`;
   const priority = input.priority ?? 'normal';
   assertEnum(priority, JOB_PRIORITIES, 'priority');
 
@@ -64,7 +260,7 @@ export function createJobEnvelope(input = {}) {
 
   return {
     schema_version: '1',
-    id: clean(input.id) || `job_${randomUUID().replaceAll('-', '')}`,
+    id,
     account_id: accountId,
     kind,
     logical_queue: clean(input.logical_queue) || null,
@@ -74,8 +270,12 @@ export function createJobEnvelope(input = {}) {
     parent_job_id: clean(input.parent_job_id) || null,
     conversation_id: clean(input.conversation_id) || null,
     agent_id: clean(input.agent_id) || null,
-    source_run_id: clean(input.source_run_id) || null,
-    idempotency_key: clean(input.idempotency_key) || null,
+    source_run_id: clean(input.source_run_id ?? input.run_id) || null,
+    step_id: clean(input.step_id) || null,
+    idempotency_key: clean(input.idempotency_key ?? input.idempotencyKey) || id,
+    lease_owner: clean(input.lease_owner) || null,
+    lease_expires_at: input.lease_expires_at == null ? null : epochSeconds(input.lease_expires_at, null, 'lease_expires_at'),
+    lease_generation: Number.isInteger(input.lease_generation) ? Math.max(0, input.lease_generation) : 0,
     payload: input.payload ?? null,
     payload_ref: clean(input.payload_ref) || null,
     metadata: input.metadata && typeof input.metadata === 'object' ? input.metadata : {},
@@ -84,6 +284,7 @@ export function createJobEnvelope(input = {}) {
       ...(input.retry && typeof input.retry === 'object' ? input.retry : {}),
     },
     attempt: Number.isInteger(input.attempt) ? input.attempt : 0,
+    available_at: availableAt,
     created_at: now,
   };
 }
@@ -97,6 +298,9 @@ export function assertJobEnvelope(job) {
   assertEnum(job.priority ?? 'normal', JOB_PRIORITIES, 'job.priority');
   assertEnum(job.executor ?? 'queue', EXECUTOR_KINDS, 'job.executor');
   assertEnum(job.status ?? 'queued', JOB_STATUSES, 'job.status');
+  epochSeconds(job.available_at, job.created_at ?? 0, 'job.available_at');
+  if (job.lease_expires_at != null) epochSeconds(job.lease_expires_at, null, 'job.lease_expires_at');
+  if (job.lease_owner && job.lease_expires_at == null) throw new TypeError('job.lease_expires_at is required when job.lease_owner is set');
   return job;
 }
 

package/packages/agentsam-queue-control/src/control-plane.js

@@ -1,4 +1,4 @@
-import { createJobEnvelope, assertJobEnvelope } from './contracts.js';
+import { createJobEnvelope, assertJobEnvelope, computeRetryDelayMs, createRetrySchedule, createDeadLetterJob } from './contracts.js';
 import { buildQueueTopology, resolvePhysicalQueue } from './topology.js';
 import { routeWork, shouldRequireProvisionApproval } from './policy.js';
 import { JobDispatcher } from './dispatch.js';
@@ -117,6 +117,16 @@ export class QueueControl {
     return this.adapter.ensureTopology(this.topology, options);
   }
 
+  async deadLetter(job, error, { attempt = (job.attempt || 0) + 1 } = {}) {
+    const deadLetter = createDeadLetterJob(job, { error, attempt });
+    const queue = resolvePhysicalQueue(this.topology, 'dead_letter');
+    const receipt = await this.adapter.publish(queue, deadLetter, {
+      logical_queue: 'dead_letter',
+      dead_letter: true,
+    });
+    return { job: deadLetter, queue, receipt: receipt ?? null };
+  }
+
   async consume(messages, context = {}) {
     const results = [];
     for (const message of messages) {
@@ -127,12 +137,62 @@ export class QueueControl {
         if (typeof message?.ack === 'function') message.ack();
         results.push({ job_id: job.id, ok: true, result });
       } catch (error) {
-        if (typeof message?.retry === 'function') message.retry();
+        let retry = null;
+        if (typeof message?.retry === 'function') {
+          const attempt = Number.isInteger(message.attempts) ? message.attempts : (job.attempt + 1);
+          const maxAttempts = Math.max(1, Number(job.retry?.max_attempts) || 3);
+          if (attempt >= maxAttempts) {
+            // Do not ACK an exhausted provider-managed message here. A configured
+            // provider DLQ is the crash-safe fallback; retry/failure delivery lets
+            // the provider move it after max_retries instead of silently deleting it.
+            const delayMs = computeRetryDelayMs(job.retry, attempt);
+            const delaySeconds = Math.max(0, Math.ceil(delayMs / 1000));
+            message.retry(delaySeconds > 0 ? { delaySeconds } : undefined);
+            retry = {
+              scheduled: false,
+              exhausted: true,
+              provider_managed: true,
+              delegated_to_provider_dlq: true,
+              attempt,
+              delay_ms: delayMs,
+              delay_seconds: delaySeconds,
+            };
+          } else {
+            const delayMs = computeRetryDelayMs(job.retry, attempt);
+            const delaySeconds = Math.max(0, Math.ceil(delayMs / 1000));
+            message.retry(delaySeconds > 0 ? { delaySeconds } : undefined);
+            retry = { scheduled: true, provider_managed: true, attempt, delay_ms: delayMs, delay_seconds: delaySeconds };
+          }
+        } else if (job.logical_queue) {
+          const schedule = createRetrySchedule(job, { error });
+          if (schedule.retryable && schedule.job) {
+            const queue = resolvePhysicalQueue(this.topology, job.logical_queue);
+            await this.adapter.publish(queue, schedule.job, { logical_queue: job.logical_queue });
+            retry = {
+              scheduled: true,
+              provider_managed: false,
+              delay_ms: schedule.delay_ms,
+              available_at: schedule.available_at,
+              attempt: schedule.attempt,
+            };
+          } else {
+            const deadLetter = await this.deadLetter(job, error, { attempt: schedule.attempt });
+            retry = {
+              scheduled: false,
+              exhausted: true,
+              attempt: schedule.attempt,
+              dead_lettered: true,
+              dead_letter_job_id: deadLetter.job.id,
+              dead_letter_queue: deadLetter.queue,
+            };
+          }
+        }
         results.push({
           job_id: job.id,
           ok: false,
           error: String(error?.message || error),
           code: error?.code ?? null,
+          retry,
         });
       }
     }

package/packages/agentsam-queue-control/src/index.js

@@ -5,3 +5,4 @@ export * from './dispatch.js';
 export * from './control-plane.js';
 export * from './adapters/memory.js';
 export * from './adapters/cloudflare.js';
+export * from './adapters/sqlite.js';

package/packages/agentsam-queue-control/test/queue-control.test.mjs

@@ -4,6 +4,17 @@ import assert from 'node:assert/strict';
 import {
   QueueControl,
   MemoryQueueAdapter,
+  CloudflareQueueBindingAdapter,
+  CloudflareQueueApiAdapter,
+  canClaimJobLease,
+  claimJobLease,
+  computeRetryDelayMs,
+  createJobEnvelope,
+  createRetrySchedule,
+  delaySecondsForAvailableAt,
+  jobExecutionIdentity,
+  refreshJobLease,
+  releaseJobLease,
   buildQueueTopology,
   resolvePhysicalQueue,
   routeWork,
@@ -98,3 +109,302 @@ test('infrastructure provisioning is approval-gated by default', async () => {
   assert.equal(result.provider, 'memory');
   assert.ok(result.created.includes('demo-test-jobs'));
 });
+
+
+test('job envelopes support delayed availability without making early work runnable', async () => {
+  const adapter = new MemoryQueueAdapter();
+  const control = new QueueControl({
+    adapter,
+    topology: buildQueueTopology({ namespace: 'demo', environment: 'test' }),
+  });
+  const queue = resolvePhysicalQueue(control.topology, 'deployments');
+  const createdAt = 1_000;
+  const availableAt = 1_060;
+  const result = await control.enqueue({
+    account_id: 'acct_real',
+    kind: 'deploy.cloudflare',
+  }, {
+    created_at: createdAt,
+    availableAt,
+  });
+
+  assert.equal(result.job.available_at, availableAt);
+  assert.equal((await adapter.pull(queue, 10, { now: 1_059 })).length, 0);
+  assert.equal(adapter.depth(queue), 1);
+  assert.equal((await adapter.pull(queue, 10, { now: 1_060 })).length, 1);
+});
+
+test('retry scheduling uses bounded exponential backoff with injectable jitter', () => {
+  const job = createJobEnvelope({
+    account_id: 'acct_real',
+    kind: 'repository.index',
+    created_at: 1_000,
+    retry: {
+      max_attempts: 3,
+      initial_delay_ms: 1_000,
+      max_delay_ms: 10_000,
+      backoff: 'exponential',
+      jitter: true,
+    },
+  });
+
+  assert.equal(computeRetryDelayMs(job.retry, 1, () => 0), 500);
+  assert.equal(computeRetryDelayMs(job.retry, 2, () => 1), 2_000);
+
+  const retry = createRetrySchedule(job, { now_ms: 1_000_000, random: () => 1 });
+  assert.equal(retry.retryable, true);
+  assert.equal(retry.job.status, 'retry_scheduled');
+  assert.equal(retry.job.attempt, 1);
+  assert.equal(retry.available_at, 1_001);
+
+  const exhausted = createRetrySchedule({ ...job, attempt: 2 }, { now_ms: 1_000_000 });
+  assert.equal(exhausted.retryable, false);
+  assert.equal(exhausted.exhausted, true);
+});
+
+test('local queue failures are re-enqueued for a future deterministic retry', async () => {
+  const adapter = new MemoryQueueAdapter();
+  const control = new QueueControl({
+    adapter,
+    topology: buildQueueTopology({ namespace: 'demo', environment: 'test' }),
+  });
+  control.register('repository.*', async () => {
+    throw new Error('temporary');
+  });
+
+  await control.enqueue({ account_id: 'acct_real', kind: 'repository.index' }, {
+    created_at: 1_000,
+    retry: { max_attempts: 3, initial_delay_ms: 1_000, max_delay_ms: 1_000, jitter: false },
+  });
+  const queue = resolvePhysicalQueue(control.topology, 'indexing');
+  const [job] = await adapter.pull(queue, 1, { now: 1_000 });
+  const [result] = await control.consume([job]);
+
+  assert.equal(result.ok, false);
+  assert.equal(result.retry.scheduled, true);
+  assert.equal(result.retry.provider_managed, false);
+  assert.equal(adapter.depth(queue), 1);
+  assert.equal((await adapter.pull(queue, 1, { now: result.retry.available_at - 1 })).length, 0);
+  assert.equal((await adapter.pull(queue, 1, { now: result.retry.available_at })).length, 1);
+});
+
+test('Cloudflare queue binding maps available_at to delaySeconds and rejects delays beyond provider limit', async () => {
+  const calls = [];
+  const adapter = new CloudflareQueueBindingAdapter({
+    bindings: {
+      default: {
+        async send(body, options) { calls.push({ body, options }); },
+      },
+    },
+  });
+  const job = createJobEnvelope({
+    account_id: 'acct_real',
+    kind: 'deploy.cloudflare',
+    created_at: 1_000,
+    available_at: 1_600,
+  });
+
+  const receipt = await adapter.publish('demo', job, { now: 1_000 });
+  assert.equal(receipt.delay_seconds, 600);
+  assert.equal(calls[0].options.delaySeconds, 600);
+  assert.equal(delaySecondsForAvailableAt(87_400, 1_000), 86_400);
+  assert.throws(() => delaySecondsForAvailableAt(87_401, 1_000), /exceeds_24h/);
+});
+
+test('provider-managed retries receive a deterministic retry delay instead of immediate polling', async () => {
+  const adapter = new MemoryQueueAdapter();
+  const control = new QueueControl({
+    adapter,
+    topology: buildQueueTopology({ namespace: 'demo', environment: 'test' }),
+  });
+  control.register('deploy.*', async () => {
+    throw new Error('pending');
+  });
+  const job = createJobEnvelope({
+    account_id: 'acct_real',
+    kind: 'deploy.cloudflare',
+    logical_queue: 'deployments',
+    retry: { max_attempts: 3, initial_delay_ms: 2_000, max_delay_ms: 2_000, jitter: false },
+  });
+  let retryOptions = null;
+  const message = {
+    body: job,
+    attempts: 1,
+    retry(options) { retryOptions = options; },
+  };
+
+  const [result] = await control.consume([message]);
+  assert.equal(result.retry.provider_managed, true);
+  assert.equal(result.retry.delay_seconds, 2);
+  assert.deepEqual(retryOptions, { delaySeconds: 2 });
+});
+
+
+test('job execution identity always carries a stable idempotency key and run/step attribution', () => {
+  const job = createJobEnvelope({
+    id: 'job_fixed',
+    account_id: 'acct_real',
+    kind: 'embeddings.generate',
+    run_id: 'arun_parent',
+    step_id: 'step_embed',
+  });
+  assert.equal(job.idempotency_key, 'job_fixed');
+  assert.deepEqual(jobExecutionIdentity(job), {
+    job_id: 'job_fixed',
+    run_id: 'arun_parent',
+    step_id: 'step_embed',
+    attempt: 0,
+    idempotency_key: 'job_fixed',
+    lease_owner: null,
+    lease_expires_at: null,
+    lease_generation: 0,
+  });
+
+  const explicit = createJobEnvelope({
+    id: 'job_other',
+    account_id: 'acct_real',
+    kind: 'deploy.cloudflare',
+    idempotency_key: 'deploy:commit:abc123',
+  });
+  assert.equal(explicit.idempotency_key, 'deploy:commit:abc123');
+});
+
+test('lease contract supports claim, refresh, release, crash expiry, and rejects live double claims', () => {
+  const job = createJobEnvelope({
+    account_id: 'acct_real',
+    kind: 'repository.index',
+    created_at: 1_000,
+  });
+  const claimed = claimJobLease(job, { owner: 'worker-a', now: 1_000, ttl_seconds: 30 });
+  assert.equal(claimed.status, 'claimed');
+  assert.equal(claimed.lease_owner, 'worker-a');
+  assert.equal(claimed.lease_expires_at, 1_030);
+  assert.equal(claimed.lease_generation, 1);
+  assert.equal(canClaimJobLease(claimed, { owner: 'worker-b', now: 1_010 }), false);
+  assert.throws(
+    () => claimJobLease(claimed, { owner: 'worker-b', now: 1_010 }),
+    /job_lease_held/,
+  );
+
+  const refreshed = refreshJobLease(claimed, { owner: 'worker-a', now: 1_010, ttl_seconds: 30 });
+  assert.equal(refreshed.lease_expires_at, 1_040);
+  assert.throws(
+    () => refreshJobLease(refreshed, { owner: 'worker-a', now: 1_041, ttl_seconds: 30 }),
+    /job_lease_expired/,
+  );
+
+  assert.equal(canClaimJobLease(refreshed, { owner: 'worker-b', now: 1_041 }), true);
+  const reclaimed = claimJobLease(refreshed, { owner: 'worker-b', now: 1_041, ttl_seconds: 30 });
+  assert.equal(reclaimed.lease_generation, 2);
+  assert.equal(reclaimed.lease_owner, 'worker-b');
+
+  const released = releaseJobLease(reclaimed, { owner: 'worker-b', status: 'queued' });
+  assert.equal(released.lease_owner, null);
+  assert.equal(released.lease_expires_at, null);
+  assert.equal(released.status, 'queued');
+});
+
+
+test('provider-managed retry exhaustion stops requeueing instead of polling forever', async () => {
+  const adapter = new MemoryQueueAdapter();
+  const control = new QueueControl({
+    adapter,
+    topology: buildQueueTopology({ namespace: 'demo', environment: 'test' }),
+  });
+  control.register('deploy.*', async () => {
+    throw new Error('still pending');
+  });
+  const job = createJobEnvelope({
+    account_id: 'acct_real',
+    kind: 'deploy.cloudflare',
+    logical_queue: 'deployments',
+    retry: { max_attempts: 3, initial_delay_ms: 1_000, max_delay_ms: 1_000, jitter: false },
+  });
+  let retried = false;
+  let acked = false;
+  const [result] = await control.consume([{
+    body: job,
+    attempts: 3,
+    retry() { retried = true; },
+    ack() { acked = true; },
+  }]);
+  assert.equal(retried, true);
+  assert.equal(acked, false);
+  assert.deepEqual(result.retry, {
+    scheduled: false,
+    exhausted: true,
+    provider_managed: true,
+    delegated_to_provider_dlq: true,
+    attempt: 3,
+    delay_ms: 1000,
+    delay_seconds: 1,
+  });
+});
+
+test('exhausted local retries are materialized as a deterministic dead-letter job', async () => {
+  const topology = buildQueueTopology({ namespace: 'acp', environment: 'test' });
+  const adapter = new MemoryQueueAdapter();
+  const control = new QueueControl({ adapter, topology });
+  control.register('always.fail', async () => {
+    throw Object.assign(new Error('boom'), { code: 'provider_unavailable' });
+  });
+
+  const job = createJobEnvelope({
+    id: 'job_exhaust_local',
+    account_id: 'acct_1',
+    kind: 'always.fail',
+    logical_queue: 'jobs',
+    retry: {
+      max_attempts: 1,
+      backoff: 'fixed',
+      initial_delay_ms: 0,
+      max_delay_ms: 0,
+      jitter: false,
+    },
+  });
+
+  const [result] = await control.consume([job]);
+  assert.equal(result.ok, false);
+  assert.equal(result.retry.exhausted, true);
+  assert.equal(result.retry.dead_lettered, true);
+
+  const dlq = resolvePhysicalQueue(topology, 'dead_letter');
+  const [dead] = await adapter.pull(dlq, 1);
+  assert.equal(dead.kind, 'dead_letter');
+  assert.equal(dead.payload.original_job.id, 'job_exhaust_local');
+  assert.equal(dead.payload.failure.code, 'provider_unavailable');
+  assert.equal(dead.metadata.dead_letter, true);
+});
+
+test('Cloudflare consumer provisioning carries the topology dead-letter queue', async () => {
+  const requests = [];
+  const adapter = new CloudflareQueueApiAdapter({
+    cloudflareAccountId: 'acct_cf',
+    apiToken: 'token',
+    fetchImpl: async (url, init = {}) => {
+      requests.push({ url, init });
+      if (init.method === 'POST' && url.endsWith('/consumers')) {
+        return Response.json({ success: true, result: { consumer_id: 'consumer_1' } });
+      }
+      if (init.method === 'POST') {
+        const name = JSON.parse(init.body).queue_name;
+        return Response.json({ success: true, result: { queue_id: name + '_id', queue_name: name } });
+      }
+      return Response.json({ success: true, result: [] });
+    },
+  });
+
+  const topology = buildQueueTopology({ namespace: 'acp', environment: 'prod' });
+  const jobsQueue = resolvePhysicalQueue(topology, 'jobs');
+  await adapter.ensureTopology(topology, {
+    consumers: {
+      [jobsQueue]: { script_name: 'agentsam-acp-consumer', max_retries: 3 },
+    },
+  });
+
+  const consumerRequest = requests.find((row) => row.url.endsWith('/consumers'));
+  assert.ok(consumerRequest);
+  const body = JSON.parse(consumerRequest.init.body);
+  assert.equal(body.dead_letter_queue, resolvePhysicalQueue(topology, 'dead_letter'));
+  assert.equal(body.settings.max_retries, 3);
+});