bunqueue 2.9.0 → 2.9.2

package/dist/infrastructure/persistence/postgres/store.js

@@ -16,6 +16,7 @@ import { cleanPostgresQueue, purgePostgresDlq } from './maintenance';
 import { maintainPostgresDlq } from './dlqLifecycle';
 import { addPostgresJobLog, clearPostgresJobLogs, getPostgresJobLogs } from './logs';
 import { getPostgresCron, listPostgresCrons, processPostgresCrons, removePostgresCron, upsertPostgresCron, } from './crons';
+import { assertGroupPullOptions } from '../../../domain/types/group';
 import { heartbeatPostgresWorker, listPostgresWorkers, removePostgresClientWorkers, removePostgresWorker, savePostgresWorker, } from './workers';
 import { getPostgresQueueMetrics, trimPostgresQueueEvents } from './telemetry';
 import { releasePostgresClientLease } from './leaseRelease';
@@ -28,16 +29,49 @@ import { loadPostgresLifetimeMetrics } from './lifetimeMetrics';
 import { finalizePostgresLifetimeMetrics } from './lifetimeMetricsFinalizer';
 import { loadPostgresCloudReadModel } from './cloudReadModel';
 import { removePostgresUnprocessedChildren } from './removeUnprocessedChildren';
+import { getPostgresGroupActiveCount, getPostgresGroupConcurrency, getPostgresGroupJobsCount, getPostgresGroupRateLimit, getPostgresGroupRateLimitTtl, removePostgresGroupConcurrency, removePostgresGroupRateLimit, setPostgresGroupConcurrency, setPostgresGroupRateLimit, } from './groups';
+import { prunePostgresGroupStates } from './groupStateRetention';
 import { loadPostgresJobProjections, loadPostgresManagerSnapshot, loadPostgresQueueReadModel, } from './readModels';
+function pruneInactiveGroups(ctx) {
+    return runPostgresPostCommitMaintenance(ctx, 'group-state-retention', async () => {
+        await prunePostgresGroupStates(ctx);
+    });
+}
 /** Database-authoritative PostgreSQL queue store safe for concurrent brokers. */
 export class PostgresQueueStore extends PostgresAdmissionStore {
+    getGroupJobsCount = (queue, groupId) => getPostgresGroupJobsCount(this.context, queue, groupId);
+    getGroupActiveCount = (queue, groupId) => getPostgresGroupActiveCount(this.context, queue, groupId);
+    setGroupRateLimit = (queue, groupId, max, duration) => setPostgresGroupRateLimit(this.context, queue, groupId, max, duration);
+    getGroupRateLimit = (queue, groupId) => getPostgresGroupRateLimit(this.context, queue, groupId);
+    removeGroupRateLimit = async (queue, groupId) => {
+        const removed = await removePostgresGroupRateLimit(this.context, queue, groupId);
+        if (removed)
+            await pruneInactiveGroups(this.context);
+        return removed;
+    };
+    getGroupRateLimitTtl = (queue, groupId, maxJobs) => getPostgresGroupRateLimitTtl(this.context, queue, groupId, maxJobs);
+    setGroupConcurrency = (queue, groupId, concurrency) => setPostgresGroupConcurrency(this.context, queue, groupId, concurrency);
+    getGroupConcurrency = (queue, groupId) => getPostgresGroupConcurrency(this.context, queue, groupId);
+    removeGroupConcurrency = async (queue, groupId) => {
+        const removed = await removePostgresGroupConcurrency(this.context, queue, groupId);
+        if (removed)
+            await pruneInactiveGroups(this.context);
+        return removed;
+    };
     async now() {
         await this.initialize();
         return await databaseNow(this.context.sql);
     }
-    async claim(queue, count, owner = this.config.brokerId, leaseDurationMs = this.config.leaseDurationMs) {
+    async claim(queue, count, owner = this.config.brokerId, leaseDurationMs = this.config.leaseDurationMs, groupOptions) {
+        assertGroupPullOptions(groupOptions);
         await this.initialize();
-        return await claimPostgresJobs(this.context, queue, count, owner, leaseDurationMs);
+        return await claimPostgresJobs(this.context, {
+            queue,
+            count,
+            owner,
+            leaseDurationMs,
+            groupOptions,
+        });
     }
     async complete(id, token, result, remove = false) {
         await this.initialize();
@@ -45,6 +79,9 @@ export class PostgresQueueStore extends PostgresAdmissionStore {
         if (transition.applied && (remove || transition.job?.removeOnComplete)) {
             await runPostgresPostCommitMaintenance(this.context, 'completion-retention', () => prunePostgresCompletionTombstones(this.context));
         }
+        if (transition.applied && transition.job?.groupId) {
+            await pruneInactiveGroups(this.context);
+        }
         return transition;
     }
     async completeMany(items) {
@@ -53,11 +90,18 @@ export class PostgresQueueStore extends PostgresAdmissionStore {
         if (transitions.some((transition) => transition.applied && transition.removed)) {
             await runPostgresPostCommitMaintenance(this.context, 'completion-retention', () => prunePostgresCompletionTombstones(this.context));
         }
+        if (transitions.some((transition) => transition.applied && transition.job?.groupId)) {
+            await pruneInactiveGroups(this.context);
+        }
         return transitions;
     }
     async fail(input) {
         await this.initialize();
-        return await failPostgresJob(this.context, input);
+        const transition = await failPostgresJob(this.context, input);
+        if (transition.applied && transition.state === 'failed' && transition.job?.groupId) {
+            await pruneInactiveGroups(this.context);
+        }
+        return transition;
     }
     async renew(id, token, durationMs) {
         await this.initialize();

package/dist/infrastructure/persistence/postgres/telemetry.d.ts

@@ -2,5 +2,7 @@ import type { QueueMetrics, QueueMetricType } from '../../../domain/types/metric
 import type { PostgresContext } from './context';
 export declare function getPostgresQueueMetrics(ctx: PostgresContext, queue: string, type: QueueMetricType, start?: number, end?: number): Promise<QueueMetrics>;
 export declare function trimPostgresQueueEvents(ctx: PostgresContext, queue: string, maxLength: number): Promise<number>;
+/** Try one automatic retention sweep without waiting behind an active writer. */
+export declare function tryTrimPostgresQueueEvents(ctx: PostgresContext, queue: string, maxLength: number): Promise<number | null>;
 /** Converge retained windows after missed notifications or an unclean broker exit. */
 export declare function sweepPostgresEventRetention(ctx: PostgresContext): Promise<number>;

package/dist/infrastructure/persistence/postgres/telemetry.js

@@ -1,5 +1,5 @@
 import { recordPostgresEventPruneWatermarks, summarizePostgresPrunedEvents, } from './eventPruneWatermarks';
-import { lockPostgresEventRetention, prunePostgresQueueEvents } from './eventRetention';
+import { lockPostgresEventRetention, prunePostgresQueueEvents, tryLockPostgresEventRetention, } from './eventRetention';
 import { POSTGRES_EVENT_CHANNEL } from './schema';
 function assertNonNegativeInteger(value, name) {
     if (!Number.isSafeInteger(value) || value < 0) {
@@ -44,30 +44,42 @@ export async function getPostgresQueueMetrics(ctx, queue, type, start = 0, end =
         count,
     };
 }
+async function trimLockedPostgresQueueEvents(tx, ctx, queue, maxLength) {
+    const rows = await prunePostgresQueueEvents(tx, ctx, queue, maxLength);
+    const prune = summarizePostgresPrunedEvents(rows);
+    const { prunedThrough } = prune;
+    if (prunedThrough > 0) {
+        await recordPostgresEventPruneWatermarks(tx, ctx, [
+            {
+                queue,
+                sourceEventId: prunedThrough,
+                ...prune,
+                prunesCurrentTransaction: false,
+            },
+        ]);
+        await tx.notify(POSTGRES_EVENT_CHANNEL, JSON.stringify({
+            namespace: ctx.config.namespace,
+            queue,
+            brokerId: ctx.config.brokerId,
+            prunedThrough,
+        }));
+    }
+    return rows.length;
+}
 export async function trimPostgresQueueEvents(ctx, queue, maxLength) {
     assertNonNegativeInteger(maxLength, 'maxLength');
     return await ctx.sql.begin(async (tx) => {
         await lockPostgresEventRetention(tx, ctx, queue);
-        const rows = await prunePostgresQueueEvents(tx, ctx, queue, maxLength);
-        const prune = summarizePostgresPrunedEvents(rows);
-        const { prunedThrough } = prune;
-        if (prunedThrough > 0) {
-            await recordPostgresEventPruneWatermarks(tx, ctx, [
-                {
-                    queue,
-                    sourceEventId: prunedThrough,
-                    ...prune,
-                    prunesCurrentTransaction: false,
-                },
-            ]);
-            await tx.notify(POSTGRES_EVENT_CHANNEL, JSON.stringify({
-                namespace: ctx.config.namespace,
-                queue,
-                brokerId: ctx.config.brokerId,
-                prunedThrough,
-            }));
-        }
-        return rows.length;
+        return await trimLockedPostgresQueueEvents(tx, ctx, queue, maxLength);
+    });
+}
+/** Try one automatic retention sweep without waiting behind an active writer. */
+export async function tryTrimPostgresQueueEvents(ctx, queue, maxLength) {
+    assertNonNegativeInteger(maxLength, 'maxLength');
+    return await ctx.sql.begin(async (tx) => {
+        if (!(await tryLockPostgresEventRetention(tx, ctx, queue)))
+            return null;
+        return await trimLockedPostgresQueueEvents(tx, ctx, queue, maxLength);
     });
 }
 /** Converge retained windows after missed notifications or an unclean broker exit. */

package/dist/infrastructure/persistence/postgres/types.d.ts

@@ -61,6 +61,7 @@ export interface PostgresJobRow {
     unique_expires_at: number | string | bigint | null;
     custom_id: string | null;
     group_id: string | null;
+    group_order: number | string | bigint | null;
     parent_id: string | null;
     lease_owner: string | null;
     lease_broker_id: string | null;

package/dist/infrastructure/persistence/schema.d.ts

@@ -1,8 +1,8 @@
 /** SQLite PRAGMA settings for optimal performance */
 export declare const PRAGMA_SETTINGS = "\nPRAGMA journal_mode = WAL;\nPRAGMA synchronous = NORMAL;\nPRAGMA cache_size = -64000;\nPRAGMA temp_store = MEMORY;\nPRAGMA mmap_size = 268435456;\nPRAGMA page_size = 4096;\nPRAGMA busy_timeout = 5000;\n";
 /** Main schema creation. */
-export declare const SCHEMA = "\n-- Jobs table (using UUIDv7 for job IDs)\n-- Uses BLOB for data fields (MessagePack serialization for ~2-3x faster than JSON)\nCREATE TABLE IF NOT EXISTS jobs (\n    id TEXT PRIMARY KEY,\n    queue TEXT NOT NULL,\n    name TEXT,\n    data BLOB NOT NULL,\n    priority INTEGER NOT NULL DEFAULT 0,\n    created_at INTEGER NOT NULL,\n    run_at INTEGER NOT NULL,\n    started_at INTEGER,\n    completed_at INTEGER,\n    attempts INTEGER NOT NULL DEFAULT 0,\n    max_attempts INTEGER NOT NULL DEFAULT 3,\n    backoff INTEGER NOT NULL DEFAULT 1000,\n    ttl INTEGER,\n    timeout INTEGER,\n    unique_key TEXT,\n    custom_id TEXT,\n    depends_on BLOB,\n    parent_id TEXT,\n    children_ids BLOB,\n    tags BLOB,\n    state TEXT NOT NULL DEFAULT 'waiting',\n    lifo INTEGER NOT NULL DEFAULT 0,\n    group_id TEXT,\n    progress INTEGER DEFAULT 0,\n    progress_msg TEXT,\n    remove_on_complete INTEGER DEFAULT 0,\n    remove_on_fail INTEGER DEFAULT 0,\n    fail_parent_on_failure INTEGER NOT NULL DEFAULT 0,\n    remove_dependency_on_failure INTEGER NOT NULL DEFAULT 0,\n    continue_parent_on_failure INTEGER NOT NULL DEFAULT 0,\n    ignore_dependency_on_failure INTEGER NOT NULL DEFAULT 0,\n    stall_timeout INTEGER,\n    last_heartbeat INTEGER,\n    stall_count INTEGER NOT NULL DEFAULT 0,\n    timeline BLOB,\n    stacktrace BLOB,\n    dlq_retry_state BLOB,\n    extended_options BLOB\n);\n\n-- Indexes for common queries\nCREATE INDEX IF NOT EXISTS idx_jobs_queue_state\n    ON jobs(queue, state);\n-- Stable createdAt/id pagination for unfiltered and logical-state queue views\nCREATE INDEX IF NOT EXISTS idx_jobs_queue_created\n    ON jobs(queue, created_at, id);\nCREATE INDEX IF NOT EXISTS idx_jobs_queue_state_created\n    ON jobs(queue, state, created_at, id);\nCREATE INDEX IF NOT EXISTS idx_jobs_run_at\n    ON jobs(run_at) WHERE state IN ('waiting', 'prioritized', 'waiting-children', 'delayed');\nCREATE INDEX IF NOT EXISTS idx_jobs_unique\n    ON jobs(queue, unique_key) WHERE unique_key IS NOT NULL;\nCREATE INDEX IF NOT EXISTS idx_jobs_custom_id\n    ON jobs(custom_id) WHERE custom_id IS NOT NULL;\nCREATE INDEX IF NOT EXISTS idx_jobs_parent\n    ON jobs(parent_id) WHERE parent_id IS NOT NULL;\n\n-- Durable outbox/state for terminal child failure propagation.\nCREATE TABLE IF NOT EXISTS flow_failures (\n    parent_id TEXT NOT NULL,\n    child_id TEXT NOT NULL,\n    child_queue TEXT NOT NULL,\n    mode TEXT NOT NULL,\n    error TEXT NOT NULL,\n    created_at INTEGER NOT NULL,\n    PRIMARY KEY (parent_id, child_id)\n);\nCREATE INDEX IF NOT EXISTS idx_flow_failures_parent ON flow_failures(parent_id);\n\n\nCREATE TABLE IF NOT EXISTS dependency_completions (\n    sequence INTEGER PRIMARY KEY AUTOINCREMENT,\n    job_id TEXT NOT NULL UNIQUE,\n    queue TEXT NOT NULL,\n    completed_at INTEGER NOT NULL,\n    pinned INTEGER NOT NULL DEFAULT 0\n);\nCREATE INDEX IF NOT EXISTS idx_dependency_completions_queue\n    ON dependency_completions(queue);\n\n\n-- Job results storage (BLOB for MessagePack)\nCREATE TABLE IF NOT EXISTS job_results (\n    job_id TEXT PRIMARY KEY,\n    result BLOB,\n    completed_at INTEGER NOT NULL\n);\n\n-- Dead letter queue (BLOB for MessagePack - stores full DlqEntry)\nCREATE TABLE IF NOT EXISTS dlq (\n    id INTEGER PRIMARY KEY AUTOINCREMENT,\n    job_id TEXT NOT NULL,\n    queue TEXT NOT NULL,\n    entry BLOB NOT NULL,\n    entered_at INTEGER NOT NULL\n);\n\nCREATE INDEX IF NOT EXISTS idx_dlq_queue ON dlq(queue);\nCREATE INDEX IF NOT EXISTS idx_dlq_job_id ON dlq(job_id);\nCREATE INDEX IF NOT EXISTS idx_dlq_entered_at ON dlq(entered_at);\n\n-- Performance indexes for high-throughput operations\n-- Stall detection: runs every 5s, needs fast lookup of active jobs by started_at\nCREATE INDEX IF NOT EXISTS idx_jobs_state_started\n    ON jobs(state, started_at) WHERE state = 'active';\n\n-- Group operations: fast lookup by group_id\nCREATE INDEX IF NOT EXISTS idx_jobs_group_id\n    ON jobs(group_id) WHERE group_id IS NOT NULL;\n\n-- Pending jobs: compound index for priority-ordered retrieval\nCREATE INDEX IF NOT EXISTS idx_jobs_pending_priority\n    ON jobs(queue, state, priority DESC, run_at ASC) WHERE state IN ('waiting', 'prioritized', 'waiting-children', 'delayed');\n\n-- Completed jobs: index for recovery ordering (issue #84)\nCREATE INDEX IF NOT EXISTS idx_jobs_completed_order\n    ON jobs(completed_at DESC) WHERE state = 'completed';\n\n-- Cron jobs (BLOB for MessagePack)\nCREATE TABLE IF NOT EXISTS cron_jobs (\n    name TEXT PRIMARY KEY,\n    queue TEXT NOT NULL,\n    job_name TEXT,\n    data BLOB NOT NULL,\n    schedule TEXT,\n    repeat_every INTEGER,\n    priority INTEGER NOT NULL DEFAULT 0,\n    next_run INTEGER NOT NULL,\n    executions INTEGER NOT NULL DEFAULT 0,\n    max_limit INTEGER,\n    timezone TEXT,\n    unique_key TEXT,\n    dedup BLOB,\n    skip_missed_on_restart INTEGER NOT NULL DEFAULT 0,\n    skip_if_no_worker INTEGER NOT NULL DEFAULT 0,\n    prevent_overlap INTEGER NOT NULL DEFAULT 1,\n    job_options BLOB\n);\n\n-- Queue state persistence (optional)\nCREATE TABLE IF NOT EXISTS queue_state (\n    name TEXT PRIMARY KEY,\n    paused INTEGER NOT NULL DEFAULT 0,\n    rate_limit INTEGER,\n    concurrency_limit INTEGER,\n    rate_limit_duration INTEGER,\n    rate_limit_expires_at INTEGER,\n    stall_enabled INTEGER,\n    stall_interval INTEGER,\n    max_stalls INTEGER,\n    stall_grace_period INTEGER,\n    dlq_config BLOB\n);\n\n-- Per-queue lifecycle event journal. Entries are kept newest-first by id and\n-- bounded by QueueManagerConfig.maxQueueEvents when they are appended.\nCREATE TABLE IF NOT EXISTS queue_events (\n    id INTEGER PRIMARY KEY AUTOINCREMENT,\n    queue TEXT NOT NULL,\n    event_type TEXT NOT NULL,\n    job_id TEXT NOT NULL,\n    occurred_at INTEGER NOT NULL,\n    payload BLOB\n);\nCREATE INDEX IF NOT EXISTS idx_queue_events_queue_id\n    ON queue_events(queue, id DESC);\n\n-- Cumulative terminal counters are separate from bounded minute buckets so\n-- pruning old data points never resets meta.count.\nCREATE TABLE IF NOT EXISTS queue_metrics_meta (\n    queue TEXT NOT NULL,\n    type TEXT NOT NULL CHECK (type IN ('completed', 'failed')),\n    total_count INTEGER NOT NULL DEFAULT 0,\n    prev_ts INTEGER NOT NULL DEFAULT 0,\n    prev_count INTEGER NOT NULL DEFAULT 0,\n    PRIMARY KEY (queue, type)\n);\nCREATE TABLE IF NOT EXISTS queue_metric_buckets (\n    queue TEXT NOT NULL,\n    type TEXT NOT NULL CHECK (type IN ('completed', 'failed')),\n    minute INTEGER NOT NULL,\n    count INTEGER NOT NULL DEFAULT 0,\n    PRIMARY KEY (queue, type, minute)\n);\nCREATE INDEX IF NOT EXISTS idx_queue_metric_buckets_window\n    ON queue_metric_buckets(queue, type, minute DESC);\n";
+export declare const SCHEMA = "\n-- Jobs table (using UUIDv7 for job IDs)\n-- Uses BLOB for data fields (MessagePack serialization for ~2-3x faster than JSON)\nCREATE TABLE IF NOT EXISTS jobs (\n    id TEXT PRIMARY KEY,\n    queue TEXT NOT NULL,\n    name TEXT,\n    data BLOB NOT NULL,\n    priority INTEGER NOT NULL DEFAULT 0,\n    created_at INTEGER NOT NULL,\n    run_at INTEGER NOT NULL,\n    started_at INTEGER,\n    completed_at INTEGER,\n    attempts INTEGER NOT NULL DEFAULT 0,\n    max_attempts INTEGER NOT NULL DEFAULT 3,\n    backoff INTEGER NOT NULL DEFAULT 1000,\n    ttl INTEGER,\n    timeout INTEGER,\n    unique_key TEXT,\n    custom_id TEXT,\n    depends_on BLOB,\n    parent_id TEXT,\n    children_ids BLOB,\n    tags BLOB,\n    state TEXT NOT NULL DEFAULT 'waiting',\n    lifo INTEGER NOT NULL DEFAULT 0,\n    group_id TEXT,\n    progress INTEGER DEFAULT 0,\n    progress_msg TEXT,\n    remove_on_complete INTEGER DEFAULT 0,\n    remove_on_fail INTEGER DEFAULT 0,\n    fail_parent_on_failure INTEGER NOT NULL DEFAULT 0,\n    remove_dependency_on_failure INTEGER NOT NULL DEFAULT 0,\n    continue_parent_on_failure INTEGER NOT NULL DEFAULT 0,\n    ignore_dependency_on_failure INTEGER NOT NULL DEFAULT 0,\n    stall_timeout INTEGER,\n    last_heartbeat INTEGER,\n    stall_count INTEGER NOT NULL DEFAULT 0,\n    timeline BLOB,\n    stacktrace BLOB,\n    dlq_retry_state BLOB,\n    extended_options BLOB\n);\n\n-- Indexes for common queries\nCREATE INDEX IF NOT EXISTS idx_jobs_queue_state\n    ON jobs(queue, state);\n-- Stable createdAt/id pagination for unfiltered and logical-state queue views\nCREATE INDEX IF NOT EXISTS idx_jobs_queue_created\n    ON jobs(queue, created_at, id);\nCREATE INDEX IF NOT EXISTS idx_jobs_queue_state_created\n    ON jobs(queue, state, created_at, id);\nCREATE INDEX IF NOT EXISTS idx_jobs_run_at\n    ON jobs(run_at) WHERE state IN ('waiting', 'prioritized', 'waiting-children', 'delayed');\nCREATE INDEX IF NOT EXISTS idx_jobs_unique\n    ON jobs(queue, unique_key) WHERE unique_key IS NOT NULL;\nCREATE INDEX IF NOT EXISTS idx_jobs_custom_id\n    ON jobs(custom_id) WHERE custom_id IS NOT NULL;\nCREATE INDEX IF NOT EXISTS idx_jobs_parent\n    ON jobs(parent_id) WHERE parent_id IS NOT NULL;\n\n-- Durable outbox/state for terminal child failure propagation.\nCREATE TABLE IF NOT EXISTS flow_failures (\n    parent_id TEXT NOT NULL,\n    child_id TEXT NOT NULL,\n    child_queue TEXT NOT NULL,\n    mode TEXT NOT NULL,\n    error TEXT NOT NULL,\n    created_at INTEGER NOT NULL,\n    PRIMARY KEY (parent_id, child_id)\n);\nCREATE INDEX IF NOT EXISTS idx_flow_failures_parent ON flow_failures(parent_id);\n\n\nCREATE TABLE IF NOT EXISTS dependency_completions (\n    sequence INTEGER PRIMARY KEY AUTOINCREMENT,\n    job_id TEXT NOT NULL UNIQUE,\n    queue TEXT NOT NULL,\n    completed_at INTEGER NOT NULL,\n    pinned INTEGER NOT NULL DEFAULT 0\n);\nCREATE INDEX IF NOT EXISTS idx_dependency_completions_queue\n    ON dependency_completions(queue);\n\n\n-- Job results storage (BLOB for MessagePack)\nCREATE TABLE IF NOT EXISTS job_results (\n    job_id TEXT PRIMARY KEY,\n    result BLOB,\n    completed_at INTEGER NOT NULL\n);\n\n-- Dead letter queue (BLOB for MessagePack - stores full DlqEntry)\nCREATE TABLE IF NOT EXISTS dlq (\n    id INTEGER PRIMARY KEY AUTOINCREMENT,\n    job_id TEXT NOT NULL,\n    queue TEXT NOT NULL,\n    entry BLOB NOT NULL,\n    entered_at INTEGER NOT NULL\n);\n\nCREATE INDEX IF NOT EXISTS idx_dlq_queue ON dlq(queue);\nCREATE INDEX IF NOT EXISTS idx_dlq_job_id ON dlq(job_id);\nCREATE INDEX IF NOT EXISTS idx_dlq_entered_at ON dlq(entered_at);\n\n-- Performance indexes for high-throughput operations\n-- Stall detection: runs every 5s, needs fast lookup of active jobs by started_at\nCREATE INDEX IF NOT EXISTS idx_jobs_state_started\n    ON jobs(state, started_at) WHERE state = 'active';\n\n-- Group operations: fast lookup by group_id\nCREATE INDEX IF NOT EXISTS idx_jobs_group_id\n    ON jobs(group_id) WHERE group_id IS NOT NULL;\n\n-- Pending jobs: compound index for priority-ordered retrieval\nCREATE INDEX IF NOT EXISTS idx_jobs_pending_priority\n    ON jobs(queue, state, priority DESC, run_at ASC) WHERE state IN ('waiting', 'prioritized', 'waiting-children', 'delayed');\n\n-- Completed jobs: index for recovery ordering (issue #84)\nCREATE INDEX IF NOT EXISTS idx_jobs_completed_order\n    ON jobs(completed_at DESC) WHERE state = 'completed';\n\n-- Cron jobs (BLOB for MessagePack)\nCREATE TABLE IF NOT EXISTS cron_jobs (\n    name TEXT PRIMARY KEY,\n    queue TEXT NOT NULL,\n    job_name TEXT,\n    data BLOB NOT NULL,\n    schedule TEXT,\n    repeat_every INTEGER,\n    priority INTEGER NOT NULL DEFAULT 0,\n    next_run INTEGER NOT NULL,\n    executions INTEGER NOT NULL DEFAULT 0,\n    max_limit INTEGER,\n    timezone TEXT,\n    unique_key TEXT,\n    dedup BLOB,\n    skip_missed_on_restart INTEGER NOT NULL DEFAULT 0,\n    skip_if_no_worker INTEGER NOT NULL DEFAULT 0,\n    prevent_overlap INTEGER NOT NULL DEFAULT 1,\n    job_options BLOB\n);\n\n-- Queue state persistence (optional)\nCREATE TABLE IF NOT EXISTS queue_state (\n    name TEXT PRIMARY KEY,\n    paused INTEGER NOT NULL DEFAULT 0,\n    rate_limit INTEGER,\n    concurrency_limit INTEGER,\n    rate_limit_duration INTEGER,\n    rate_limit_expires_at INTEGER,\n    stall_enabled INTEGER,\n    stall_interval INTEGER,\n    max_stalls INTEGER,\n    stall_grace_period INTEGER,\n    dlq_config BLOB\n);\n\n-- Durable per-group overrides. Runtime rate windows remain in-memory in the\n-- embedded single-broker engine, matching the existing queue-level limiter.\nCREATE TABLE IF NOT EXISTS group_state (\n    queue TEXT NOT NULL,\n    group_id TEXT NOT NULL,\n    rate_limit INTEGER,\n    rate_duration INTEGER,\n    concurrency_limit INTEGER,\n    PRIMARY KEY (queue, group_id)\n);\n\n-- Per-queue lifecycle event journal. Entries are kept newest-first by id and\n-- bounded by QueueManagerConfig.maxQueueEvents when they are appended.\nCREATE TABLE IF NOT EXISTS queue_events (\n    id INTEGER PRIMARY KEY AUTOINCREMENT,\n    queue TEXT NOT NULL,\n    event_type TEXT NOT NULL,\n    job_id TEXT NOT NULL,\n    occurred_at INTEGER NOT NULL,\n    payload BLOB\n);\nCREATE INDEX IF NOT EXISTS idx_queue_events_queue_id\n    ON queue_events(queue, id DESC);\n\n-- Cumulative terminal counters are separate from bounded minute buckets so\n-- pruning old data points never resets meta.count.\nCREATE TABLE IF NOT EXISTS queue_metrics_meta (\n    queue TEXT NOT NULL,\n    type TEXT NOT NULL CHECK (type IN ('completed', 'failed')),\n    total_count INTEGER NOT NULL DEFAULT 0,\n    prev_ts INTEGER NOT NULL DEFAULT 0,\n    prev_count INTEGER NOT NULL DEFAULT 0,\n    PRIMARY KEY (queue, type)\n);\nCREATE TABLE IF NOT EXISTS queue_metric_buckets (\n    queue TEXT NOT NULL,\n    type TEXT NOT NULL CHECK (type IN ('completed', 'failed')),\n    minute INTEGER NOT NULL,\n    count INTEGER NOT NULL DEFAULT 0,\n    PRIMARY KEY (queue, type, minute)\n);\nCREATE INDEX IF NOT EXISTS idx_queue_metric_buckets_window\n    ON queue_metric_buckets(queue, type, minute DESC);\n";
 /** Migration version table */
 export declare const MIGRATION_TABLE = "\nCREATE TABLE IF NOT EXISTS migrations (\n    version INTEGER PRIMARY KEY,\n    applied_at INTEGER NOT NULL\n);\n";
 /** Current schema version */
-export declare const SCHEMA_VERSION = 34;
+export declare const SCHEMA_VERSION = 35;

package/dist/infrastructure/persistence/schema.js

@@ -158,6 +158,17 @@ CREATE TABLE IF NOT EXISTS queue_state (
     dlq_config BLOB
 );
 
+-- Durable per-group overrides. Runtime rate windows remain in-memory in the
+-- embedded single-broker engine, matching the existing queue-level limiter.
+CREATE TABLE IF NOT EXISTS group_state (
+    queue TEXT NOT NULL,
+    group_id TEXT NOT NULL,
+    rate_limit INTEGER,
+    rate_duration INTEGER,
+    concurrency_limit INTEGER,
+    PRIMARY KEY (queue, group_id)
+);
+
 -- Per-queue lifecycle event journal. Entries are kept newest-first by id and
 -- bounded by QueueManagerConfig.maxQueueEvents when they are appended.
 CREATE TABLE IF NOT EXISTS queue_events (
@@ -199,4 +210,4 @@ CREATE TABLE IF NOT EXISTS migrations (
 );
 `;
 /** Current schema version */
-export const SCHEMA_VERSION = 34;
+export const SCHEMA_VERSION = 35;

package/dist/infrastructure/persistence/sqlite/control.d.ts

@@ -12,6 +12,13 @@ export interface PersistedQueueState {
     stallConfig: StallConfig | null;
     dlqConfig: DlqConfig | null;
 }
+export interface PersistedGroupState {
+    queue: string;
+    groupId: string;
+    rateLimit: number | null;
+    rateDuration: number | null;
+    concurrencyLimit: number | null;
+}
 /** Cron schedules and durable queue control-state. */
 export declare abstract class SqliteControl extends SqliteQueries {
     saveCron(cron: CronJob): void;
@@ -28,5 +35,12 @@ export declare abstract class SqliteControl extends SqliteQueries {
         dlqConfig?: DlqConfig | null;
     }): void;
     loadQueueState(): PersistedQueueState[];
+    saveGroupRateLimit(queue: string, groupId: string, max: number, duration: number): void;
+    removeGroupRateLimit(queue: string, groupId: string): void;
+    saveGroupConcurrency(queue: string, groupId: string, concurrency: number): void;
+    removeGroupConcurrency(queue: string, groupId: string): void;
+    loadGroupState(): PersistedGroupState[];
+    deleteGroupState(queue: string): void;
     deleteQueueState(name: string): void;
+    private deleteEmptyGroupState;
 }

package/dist/infrastructure/persistence/sqlite/control.js

@@ -82,9 +82,78 @@ export class SqliteControl extends SqliteQueries {
             dlqConfig: unpack(row.dlq_config, null, `loadQueueDlqConfig:${row.name}`),
         }));
     }
+    saveGroupRateLimit(queue, groupId, max, duration) {
+        this.safeWrite(() => {
+            this.db
+                .prepare(`INSERT INTO group_state (queue, group_id, rate_limit, rate_duration)
+           VALUES (?, ?, ?, ?)
+           ON CONFLICT(queue, group_id) DO UPDATE SET
+             rate_limit = excluded.rate_limit,
+             rate_duration = excluded.rate_duration`)
+                .run(queue, groupId, max, duration);
+        });
+    }
+    removeGroupRateLimit(queue, groupId) {
+        this.safeWrite(() => {
+            const transaction = this.db.transaction(() => {
+                this.db
+                    .prepare(`UPDATE group_state SET rate_limit = NULL, rate_duration = NULL
+             WHERE queue = ? AND group_id = ?`)
+                    .run(queue, groupId);
+                this.deleteEmptyGroupState(queue, groupId);
+            });
+            transaction();
+        });
+    }
+    saveGroupConcurrency(queue, groupId, concurrency) {
+        this.safeWrite(() => {
+            this.db
+                .prepare(`INSERT INTO group_state (queue, group_id, concurrency_limit)
+           VALUES (?, ?, ?)
+           ON CONFLICT(queue, group_id) DO UPDATE SET
+             concurrency_limit = excluded.concurrency_limit`)
+                .run(queue, groupId, concurrency);
+        });
+    }
+    removeGroupConcurrency(queue, groupId) {
+        this.safeWrite(() => {
+            const transaction = this.db.transaction(() => {
+                this.db
+                    .prepare(`UPDATE group_state SET concurrency_limit = NULL
+             WHERE queue = ? AND group_id = ?`)
+                    .run(queue, groupId);
+                this.deleteEmptyGroupState(queue, groupId);
+            });
+            transaction();
+        });
+    }
+    loadGroupState() {
+        return this.db
+            .query('SELECT * FROM group_state')
+            .all()
+            .map((row) => ({
+            queue: row.queue,
+            groupId: row.group_id,
+            rateLimit: row.rate_limit,
+            rateDuration: row.rate_duration,
+            concurrencyLimit: row.concurrency_limit,
+        }));
+    }
+    deleteGroupState(queue) {
+        this.safeWrite(() => {
+            this.db.prepare('DELETE FROM group_state WHERE queue = ?').run(queue);
+        });
+    }
     deleteQueueState(name) {
         this.safeWrite(() => {
             this.statements.get('deleteQueueState').run(name);
         });
     }
+    deleteEmptyGroupState(queue, groupId) {
+        this.db
+            .prepare(`DELETE FROM group_state
+         WHERE queue = ? AND group_id = ?
+           AND rate_limit IS NULL AND concurrency_limit IS NULL`)
+            .run(queue, groupId);
+    }
 }

package/dist/infrastructure/persistence/sqlite/jobs.d.ts

@@ -3,6 +3,16 @@ import type { FlowFailureRecord } from '../../../domain/types/flow';
 import type { Job, JobId, JobTimelineEntry } from '../../../domain/types/job';
 import type { DurableAdmissionMetadata } from '../types/admission';
 import { SqliteAdmission } from './admission';
+export interface ActiveJobWrite {
+    jobId: JobId;
+    startedAt: number;
+    timeline?: JobTimelineEntry[];
+}
+export interface CompletedJobWrite {
+    jobId: JobId;
+    completedAt: number;
+    timeline?: JobTimelineEntry[];
+}
 /** Job insertion, state transitions, and DLQ persistence. */
 export declare abstract class SqliteJobs extends SqliteAdmission {
     insertJob(job: Job, durable?: boolean, admission?: DurableAdmissionMetadata): void;
@@ -10,8 +20,10 @@ export declare abstract class SqliteJobs extends SqliteAdmission {
     protected runInsertJobStmt(job: Job): void;
     protected flushIfBuffered(jobId: JobId): void;
     markActive(jobId: JobId, startedAt: number, timeline?: JobTimelineEntry[]): void;
+    markActiveBatch(updates: readonly ActiveJobWrite[]): void;
     markWaitingChildren(jobId: JobId, timeline?: JobTimelineEntry[]): void;
     markCompleted(jobId: JobId, completedAt: number, timeline?: JobTimelineEntry[]): void;
+    markCompletedBatch(updates: readonly CompletedJobWrite[]): void;
     markFailed(job: Job, error: string | null): void;
     saveDlqEntry(entry: DlqEntry): void;
     commitFailedJob(jobId: JobId, entry: DlqEntry | null, flowFailure: FlowFailureRecord | null): void;

package/dist/infrastructure/persistence/sqlite/jobs.js

@@ -50,6 +50,24 @@ export class SqliteJobs extends SqliteAdmission {
                 .run('active', startedAt, timeline && timeline.length > 0 ? pack(timeline) : null, jobId);
         });
     }
+    markActiveBatch(updates) {
+        if (updates.length === 0)
+            return;
+        for (const update of updates)
+            this.flushIfBuffered(update.jobId);
+        const rows = updates.map((update) => ({
+            ...update,
+            timeline: update.timeline?.length ? pack(update.timeline) : null,
+        }));
+        this.safeWrite(() => {
+            this.db.transaction((batch) => {
+                const statement = this.statements.get('updateJobState');
+                for (const row of batch) {
+                    statement.run('active', row.startedAt, row.timeline, row.jobId);
+                }
+            })(rows);
+        });
+    }
     markWaitingChildren(jobId, timeline) {
         this.flushIfBuffered(jobId);
         this.safeWrite(() => {
@@ -66,6 +84,24 @@ export class SqliteJobs extends SqliteAdmission {
                 .run('completed', completedAt, timeline && timeline.length > 0 ? pack(timeline) : null, jobId);
         });
     }
+    markCompletedBatch(updates) {
+        if (updates.length === 0)
+            return;
+        for (const update of updates)
+            this.flushIfBuffered(update.jobId);
+        const rows = updates.map((update) => ({
+            ...update,
+            timeline: update.timeline?.length ? pack(update.timeline) : null,
+        }));
+        this.safeWrite(() => {
+            this.db.transaction((batch) => {
+                const statement = this.statements.get('completeJob');
+                for (const row of batch) {
+                    statement.run('completed', row.completedAt, row.timeline, row.jobId);
+                }
+            })(rows);
+        });
+    }
     markFailed(job, error) {
         this.flushIfBuffered(job.id);
         this.safeWrite(() => {

package/dist/infrastructure/persistence/sqlite/telemetry.d.ts

@@ -1,5 +1,5 @@
 import type { QueueMetricsMeta, QueueMetricType } from '../../../domain/types/metrics';
-import { type JobEvent } from '../../../domain/types/queue';
+import type { JobEvent } from '../../../domain/types/queue';
 import { SqliteControl } from './control';
 export interface PersistedQueueMetrics {
     meta: QueueMetricsMeta;
@@ -8,6 +8,7 @@ export interface PersistedQueueMetrics {
 /** Durable lifecycle journal and minute-bucket metrics. */
 export declare abstract class SqliteTelemetry extends SqliteControl {
     recordQueueEvent(event: JobEvent, maxEvents: number, maxMetricDataPoints: number): void;
+    recordQueueEventsBatch(events: readonly JobEvent[], maxEvents: number, maxMetricDataPoints: number): void;
     getQueueMetrics(queue: string, type: QueueMetricType, maxMetricDataPoints: number): PersistedQueueMetrics;
     trimQueueEvents(queue: string, maxLength: number): number;
     countQueueEvents(queue: string): number;

package/dist/infrastructure/persistence/sqlite/telemetry.js

@@ -1,71 +1,12 @@
-import { pack } from '../sqliteSerializer';
 import { SqliteControl } from './control';
-function terminalMetricType(event) {
-    if (event.eventType === "completed" /* EventType.Completed */)
-        return 'completed';
-    if (event.eventType === "failed" /* EventType.Failed */ && event.terminal !== false)
-        return 'failed';
-    return null;
-}
-function eventPayload(event) {
-    return pack({
-        data: event.data,
-        error: event.error,
-        progress: event.progress,
-        prev: event.prev,
-        delay: event.delay,
-        terminal: event.terminal,
-    });
-}
+import { writeQueueEvents } from './telemetryWrites';
 /** Durable lifecycle journal and minute-bucket metrics. */
 export class SqliteTelemetry extends SqliteControl {
     recordQueueEvent(event, maxEvents, maxMetricDataPoints) {
-        const eventLimit = Math.max(0, Math.floor(maxEvents));
-        const metricLimit = Math.max(0, Math.floor(maxMetricDataPoints));
-        const metricType = terminalMetricType(event);
-        const minute = Math.floor(event.timestamp / 60_000);
-        const payload = eventPayload(event);
-        this.safeWrite(() => {
-            const transaction = this.db.transaction(() => {
-                this.db
-                    .prepare('INSERT INTO queue_events (queue, event_type, job_id, occurred_at, payload) VALUES (?, ?, ?, ?, ?)')
-                    .run(event.queue, event.eventType, String(event.jobId), event.timestamp, payload);
-                this.db
-                    .prepare('DELETE FROM queue_events WHERE queue = ? AND id IN (SELECT id FROM queue_events WHERE queue = ? ORDER BY id DESC LIMIT -1 OFFSET ?)')
-                    .run(event.queue, event.queue, eventLimit);
-                if (!metricType)
-                    return;
-                this.db
-                    .prepare(`INSERT INTO queue_metric_buckets (queue, type, minute, count)
-             VALUES (?, ?, ?, 1)
-             ON CONFLICT(queue, type, minute) DO UPDATE SET count = count + 1`)
-                    .run(event.queue, metricType, minute);
-                const bucketCount = this.db
-                    .query('SELECT count FROM queue_metric_buckets WHERE queue = ? AND type = ? AND minute = ?')
-                    .get(event.queue, metricType, minute)?.count ?? 1;
-                this.db
-                    .prepare(`INSERT INTO queue_metrics_meta (queue, type, total_count, prev_ts, prev_count)
-             VALUES (?, ?, 1, ?, ?)
-             ON CONFLICT(queue, type) DO UPDATE SET
-               total_count = total_count + 1,
-               prev_ts = excluded.prev_ts,
-               prev_count = excluded.prev_count`)
-                    .run(event.queue, metricType, event.timestamp, bucketCount);
-                if (metricLimit === 0) {
-                    this.db
-                        .prepare('DELETE FROM queue_metric_buckets WHERE queue = ? AND type = ?')
-                        .run(event.queue, metricType);
-                    return;
-                }
-                const newest = this.db
-                    .query('SELECT MAX(minute) AS minute FROM queue_metric_buckets WHERE queue = ? AND type = ?')
-                    .get(event.queue, metricType)?.minute ?? minute;
-                this.db
-                    .prepare('DELETE FROM queue_metric_buckets WHERE queue = ? AND type = ? AND minute < ?')
-                    .run(event.queue, metricType, newest - metricLimit + 1);
-            });
-            transaction();
-        });
+        this.safeWrite(() => writeQueueEvents(this.db, [event], maxEvents, maxMetricDataPoints));
+    }
+    recordQueueEventsBatch(events, maxEvents, maxMetricDataPoints) {
+        this.safeWrite(() => writeQueueEvents(this.db, events, maxEvents, maxMetricDataPoints));
     }
     getQueueMetrics(queue, type, maxMetricDataPoints) {
         const metaRow = this.db

package/dist/infrastructure/persistence/sqlite/telemetryWrites.d.ts

@@ -0,0 +1,4 @@
+import type { Database } from 'bun:sqlite';
+import { type JobEvent } from '../../../domain/types/queue';
+/** Apply events in input order inside one transaction, preserving scalar semantics. */
+export declare function writeQueueEvents(db: Database, events: readonly JobEvent[], maxEvents: number, maxMetricDataPoints: number): void;

package/dist/infrastructure/persistence/sqlite/telemetryWrites.js

@@ -0,0 +1,123 @@
+import { pack } from '../sqliteSerializer';
+function terminalMetricType(event) {
+    if (event.eventType === "completed" /* EventType.Completed */)
+        return 'completed';
+    if (event.eventType === "failed" /* EventType.Failed */ && event.terminal !== false)
+        return 'failed';
+    return null;
+}
+function eventPayload(event) {
+    return pack({
+        data: event.data,
+        error: event.error,
+        progress: event.progress,
+        prev: event.prev,
+        delay: event.delay,
+        terminal: event.terminal,
+    });
+}
+function prepareQueueEvents(events) {
+    return events.map((event) => ({
+        event,
+        metricType: terminalMetricType(event),
+        minute: Math.floor(event.timestamp / 60_000),
+        payload: eventPayload(event),
+    }));
+}
+function groupMetricEvents(events) {
+    const byQueue = new Map();
+    for (const entry of events) {
+        if (!entry.metricType)
+            continue;
+        let byType = byQueue.get(entry.event.queue);
+        if (!byType) {
+            byType = new Map();
+            byQueue.set(entry.event.queue, byType);
+        }
+        let group = byType.get(entry.metricType);
+        if (!group) {
+            group = { queue: entry.event.queue, type: entry.metricType, events: [] };
+            byType.set(entry.metricType, group);
+        }
+        group.events.push({ minute: entry.minute, timestamp: entry.event.timestamp });
+    }
+    return [...byQueue.values()].flatMap((byType) => [...byType.values()]);
+}
+/** Apply events in input order inside one transaction, preserving scalar semantics. */
+export function writeQueueEvents(db, events, maxEvents, maxMetricDataPoints) {
+    if (events.length === 0)
+        return;
+    const preparedEvents = prepareQueueEvents(events);
+    const metricGroups = groupMetricEvents(preparedEvents);
+    const eventLimit = Math.max(0, Math.floor(maxEvents));
+    const metricLimit = Math.max(0, Math.floor(maxMetricDataPoints));
+    const insertEvent = db.prepare('INSERT INTO queue_events (queue, event_type, job_id, occurred_at, payload) VALUES (?, ?, ?, ?, ?)');
+    const trimEvents = db.prepare('DELETE FROM queue_events WHERE queue = ? AND id IN (SELECT id FROM queue_events WHERE queue = ? ORDER BY id DESC LIMIT -1 OFFSET ?)');
+    const incrementBucket = db.prepare(`INSERT INTO queue_metric_buckets (queue, type, minute, count)
+     VALUES (?, ?, ?, ?)
+     ON CONFLICT(queue, type, minute) DO UPDATE SET count = count + excluded.count`);
+    const readBucket = db.query('SELECT count FROM queue_metric_buckets WHERE queue = ? AND type = ? AND minute = ?');
+    const updateMeta = db.prepare(`INSERT INTO queue_metrics_meta (queue, type, total_count, prev_ts, prev_count)
+     VALUES (?, ?, ?, ?, ?)
+     ON CONFLICT(queue, type) DO UPDATE SET
+       total_count = total_count + excluded.total_count,
+       prev_ts = excluded.prev_ts,
+       prev_count = excluded.prev_count`);
+    const clearBuckets = db.prepare('DELETE FROM queue_metric_buckets WHERE queue = ? AND type = ?');
+    const readNewest = db.query('SELECT MAX(minute) AS minute FROM queue_metric_buckets WHERE queue = ? AND type = ?');
+    const trimBuckets = db.prepare('DELETE FROM queue_metric_buckets WHERE queue = ? AND type = ? AND minute < ?');
+    const transaction = db.transaction((batch) => {
+        const affectedQueues = new Set();
+        for (const { event, payload } of batch) {
+            insertEvent.run(event.queue, event.eventType, String(event.jobId), event.timestamp, payload);
+            affectedQueues.add(event.queue);
+        }
+        for (const queue of affectedQueues)
+            trimEvents.run(queue, queue, eventLimit);
+        for (const group of metricGroups) {
+            const eventMinutes = new Set(group.events.map((event) => event.minute));
+            const counts = new Map();
+            for (const minute of eventMinutes) {
+                const count = readBucket.get(group.queue, group.type, minute)?.count;
+                if (count !== undefined)
+                    counts.set(minute, count);
+            }
+            let newest = readNewest.get(group.queue, group.type)?.minute ?? null;
+            const increments = new Map();
+            let lastTimestamp = 0;
+            let lastCount = 0;
+            let cutoff = null;
+            for (const event of group.events) {
+                lastCount = (counts.get(event.minute) ?? 0) + 1;
+                lastTimestamp = event.timestamp;
+                counts.set(event.minute, lastCount);
+                increments.set(event.minute, (increments.get(event.minute) ?? 0) + 1);
+                if (metricLimit === 0) {
+                    counts.clear();
+                    increments.clear();
+                    newest = null;
+                    continue;
+                }
+                newest = newest === null ? event.minute : Math.max(newest, event.minute);
+                cutoff = newest - metricLimit + 1;
+                for (const minute of counts.keys()) {
+                    if (minute >= cutoff)
+                        continue;
+                    counts.delete(minute);
+                    increments.delete(minute);
+                }
+            }
+            updateMeta.run(group.queue, group.type, group.events.length, lastTimestamp, lastCount);
+            if (metricLimit === 0) {
+                clearBuckets.run(group.queue, group.type);
+                continue;
+            }
+            for (const [minute, count] of increments) {
+                incrementBucket.run(group.queue, group.type, minute, count);
+            }
+            if (cutoff !== null)
+                trimBuckets.run(group.queue, group.type, cutoff);
+        }
+    });
+    transaction(preparedEvents);
+}