@nicnocquee/dataqueue 1.26.0-beta.20260223195940 → 1.26.0-beta.20260223202259
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ai/docs-content.json +7 -1
- package/ai/rules/advanced.md +18 -0
- package/ai/skills/dataqueue-advanced/SKILL.md +50 -0
- package/dist/index.cjs +115 -36
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +85 -1
- package/dist/index.d.ts +85 -1
- package/dist/index.js +115 -36
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
- package/src/backends/redis.test.ts +169 -0
- package/src/index.test.ts +352 -11
- package/src/index.ts +66 -13
- package/src/processor.ts +74 -22
- package/src/supervisor.ts +20 -5
- package/src/types.ts +88 -0
package/ai/docs-content.json
CHANGED
|
@@ -39,7 +39,7 @@
|
|
|
39
39
|
"slug": "api/job-queue",
|
|
40
40
|
"title": "JobQueue",
|
|
41
41
|
"description": "",
|
|
42
|
-
"content": "## Initialization\n\n### initJobQueue\n\n```ts\ninitJobQueue(config: JobQueueConfig): JobQueue\n```\n\nInitializes the job queue system with the provided configuration. The `JobQueueConfig` is a discriminated union -- you provide either a PostgreSQL or Redis configuration.\n\n#### PostgresJobQueueConfig\n\nProvide either `databaseConfig` (the library creates a pool) or `pool` (bring your own `pg.Pool`). At least one must be set.\n\n```ts\ninterface PostgresJobQueueConfig {\n backend?: 'postgres'; // Optional, defaults to 'postgres'\n databaseConfig?: {\n connectionString?: string;\n host?: string;\n port?: number;\n database?: string;\n user?: string;\n password?: string;\n ssl?: DatabaseSSLConfig;\n };\n pool?: import('pg').Pool; // Bring your own pool\n verbose?: boolean;\n}\n```\n\n#### RedisJobQueueConfig\n\nProvide either `redisConfig` (the library creates an ioredis client) or `client` (bring your own). At least one must be set.\n\n```ts\ninterface RedisJobQueueConfig {\n backend: 'redis'; // Required\n redisConfig?: {\n url?: string;\n host?: string;\n port?: number;\n password?: string;\n db?: number;\n tls?: RedisTLSConfig;\n keyPrefix?: string; // Default: 'dq:'\n };\n client?: unknown; // Bring your own ioredis client\n keyPrefix?: string; // Key prefix when using external client (default: 'dq:')\n verbose?: boolean;\n}\n```\n\n#### JobQueueConfig\n\n```ts\ntype JobQueueConfig = PostgresJobQueueConfig | RedisJobQueueConfig;\n```\n\n#### DatabaseSSLConfig\n\n```ts\ninterface DatabaseSSLConfig {\n ca?: string;\n cert?: string;\n key?: string;\n rejectUnauthorized?: boolean;\n}\n```\n\n- `ca` - Client certificate authority (CA) as PEM string or file path. If the value starts with 'file://', it will be loaded from file, otherwise treated as PEM string.\n- `cert` - Client certificate as PEM string or file path. If the value starts with 'file://', it will be loaded from file, otherwise treated as PEM string.\n- `key` - Client private key as PEM string or file path. If the value starts with 'file://', it will be loaded from file, otherwise treated as PEM string.\n- `rejectUnauthorized` - Whether to reject unauthorized certificates (default: true)\n\n#### RedisTLSConfig\n\n```ts\ninterface RedisTLSConfig {\n ca?: string;\n cert?: string;\n key?: string;\n rejectUnauthorized?: boolean;\n}\n```\n\n---\n\n## Adding Jobs\n\n### addJob\n\n```ts\naddJob(job: JobOptions, options?: AddJobOptions): Promise<number>\n```\n\nAdds a job to the queue. Returns the job ID.\n\n#### JobOptions\n\n```ts\ninterface JobOptions {\n jobType: string;\n payload: any;\n maxAttempts?: number;\n priority?: number;\n runAt?: Date | null;\n timeoutMs?: number;\n tags?: string[];\n idempotencyKey?: string;\n retryDelay?: number; // Base delay between retries in seconds (default: 60)\n retryBackoff?: boolean; // Use exponential backoff (default: true)\n retryDelayMax?: number; // Max delay cap in seconds (default: none)\n}\n```\n\n- `retryDelay` - Base delay between retries in seconds. When `retryBackoff` is true, this is the base for exponential backoff (`retryDelay * 2^attempts`). When false, retries use this fixed delay. Default: `60`.\n- `retryBackoff` - Whether to use exponential backoff. When true, delay doubles with each attempt and includes jitter. Default: `true`.\n- `retryDelayMax` - Maximum delay cap in seconds. Only meaningful when `retryBackoff` is true. No limit when omitted.\n\n#### AddJobOptions\n\n```ts\ninterface AddJobOptions {\n db?: DatabaseClient;\n}\n```\n\n- `db` — An external database client (e.g., a `pg.PoolClient` inside a transaction). When provided, the INSERT runs on this client instead of the internal pool. **PostgreSQL only.** Throws if used with the Redis backend.\n\n### addJobs\n\n```ts\naddJobs(jobs: JobOptions[], options?: AddJobOptions): Promise<number[]>\n```\n\nAdds multiple jobs to the queue in a single operation. More efficient than calling `addJob` in a loop because it batches the INSERT into a single database round-trip (PostgreSQL) or a single atomic Lua script (Redis).\n\nReturns an array of job IDs in the same order as the input array.\n\nEach job can independently have its own `priority`, `runAt`, `tags`, `idempotencyKey`, and other options. Idempotency keys are handled per-job — duplicates resolve to the existing job's ID without creating a new row.\n\nPassing an empty array returns `[]` immediately without touching the database.\n\n```ts\nconst jobIds = await jobQueue.addJobs([\n {\n jobType: 'email',\n payload: { to: 'a@example.com', subject: 'Hi', body: '...' },\n },\n {\n jobType: 'email',\n payload: { to: 'b@example.com', subject: 'Hi', body: '...' },\n priority: 10,\n },\n {\n jobType: 'report',\n payload: { reportId: '123', userId: '456' },\n tags: ['monthly'],\n },\n]);\n// jobIds = [1, 2, 3]\n```\n\nThe `{ db }` option works the same as `addJob` — pass a transactional client to batch-insert within an existing transaction (PostgreSQL only).\n\n#### DatabaseClient\n\n```ts\ninterface DatabaseClient {\n query(\n text: string,\n values?: any[],\n ): Promise<{ rows: any[]; rowCount: number | null }>;\n}\n```\n\nAny object matching this interface works — `pg.Pool`, `pg.PoolClient`, `pg.Client`, or ORM query runners that expose a raw `query()` method.\n\n---\n\n## Retrieving Jobs\n\n### getJob\n\n```ts\ngetJob(id: number): Promise<JobRecord | null>\n```\n\nRetrieves a job by its ID.\n\n### getJobs\n\n```ts\ngetJobs(\n filters?: {\n jobType?: string;\n priority?: number;\n runAt?: Date | { gt?: Date; gte?: Date; lt?: Date; lte?: Date; eq?: Date };\n tags?: { values: string[]; mode?: 'all' | 'any' | 'none' | 'exact' };\n },\n limit?: number,\n offset?: number\n): Promise<JobRecord[]>\n```\n\nRetrieves jobs matching the provided filters, with optional pagination.\n\n### getJobsByStatus\n\n```ts\ngetJobsByStatus(status: string, limit?: number, offset?: number): Promise<JobRecord[]>\n```\n\nRetrieves jobs by their status, with pagination.\n\n### getAllJobs\n\n```ts\ngetAllJobs(limit?: number, offset?: number): Promise<JobRecord[]>\n```\n\nRetrieves all jobs, with optional pagination.\n\n### getJobsByTags\n\n```ts\ngetJobsByTags(tags: string[], mode?: TagQueryMode, limit?: number, offset?: number): Promise<JobRecord[]>\n```\n\nRetrieves jobs by tag(s).\n\n---\n\n## Managing Jobs\n\n### retryJob\n\n```ts\nretryJob(jobId: number): Promise<void>\n```\n\nRetries a job given its ID.\n\n### cancelJob\n\n```ts\ncancelJob(jobId: number): Promise<void>\n```\n\nCancels a job given its ID.\n\n### editJob\n\n```ts\neditJob(jobId: number, updates: EditJobOptions): Promise<void>\n```\n\nEdits a pending job given its ID. Only works for jobs with status 'pending'. Silently fails for other statuses (processing, completed, failed, cancelled).\n\n#### EditJobOptions\n\n```ts\ninterface EditJobOptions {\n payload?: any;\n maxAttempts?: number;\n priority?: number;\n runAt?: Date | null;\n timeoutMs?: number;\n tags?: string[];\n retryDelay?: number | null;\n retryBackoff?: boolean | null;\n retryDelayMax?: number | null;\n}\n```\n\nAll fields are optional - only provided fields will be updated. Note that `jobType` cannot be changed. Set retry fields to `null` to revert to legacy default behavior.\n\n#### Example\n\n```ts\n// Edit a pending job's payload and priority\nawait jobQueue.editJob(jobId, {\n payload: { to: 'newemail@example.com', subject: 'Updated' },\n priority: 10,\n});\n\n// Edit only the scheduled run time\nawait jobQueue.editJob(jobId, {\n runAt: new Date(Date.now() + 60000), // Run in 1 minute\n});\n\n// Edit multiple fields at once\nawait jobQueue.editJob(jobId, {\n payload: { to: 'updated@example.com' },\n priority: 5,\n maxAttempts: 10,\n timeoutMs: 30000,\n tags: ['urgent', 'priority'],\n});\n```\n\n### editAllPendingJobs\n\n```ts\neditAllPendingJobs(\n filters?: {\n jobType?: string;\n priority?: number;\n runAt?: Date | { gt?: Date; gte?: Date; lt?: Date; lte?: Date; eq?: Date };\n tags?: { values: string[]; mode?: 'all' | 'any' | 'none' | 'exact' };\n },\n updates: EditJobOptions\n): Promise<number>\n```\n\nEdits all pending jobs that match the filters. Only works for jobs with status 'pending'. Non-pending jobs are not affected. Returns the number of jobs that were edited.\n\n#### Parameters\n\n- `filters` (optional): Filters to select which jobs to edit. If not provided, all pending jobs are edited.\n - `jobType`: Filter by job type\n - `priority`: Filter by priority\n - `runAt`: Filter by scheduled run time (supports `gt`, `gte`, `lt`, `lte`, `eq` operators or exact Date match)\n - `tags`: Filter by tags with mode ('all', 'any', 'none', 'exact')\n- `updates`: The fields to update (same as `EditJobOptions`). All fields are optional - only provided fields will be updated.\n\n#### Returns\n\nThe number of jobs that were successfully edited.\n\n#### Examples\n\n```ts\n// Edit all pending jobs\nconst editedCount = await jobQueue.editAllPendingJobs(undefined, {\n priority: 10,\n});\n\n// Edit all pending email jobs\nconst editedCount = await jobQueue.editAllPendingJobs(\n { jobType: 'email' },\n {\n priority: 5,\n },\n);\n\n// Edit all pending jobs with 'urgent' tag\nconst editedCount = await jobQueue.editAllPendingJobs(\n { tags: { values: ['urgent'], mode: 'any' } },\n {\n priority: 10,\n maxAttempts: 5,\n },\n);\n\n// Edit all pending jobs scheduled in the future\nconst editedCount = await jobQueue.editAllPendingJobs(\n { runAt: { gte: new Date() } },\n {\n priority: 10,\n },\n);\n\n// Edit with combined filters\nconst editedCount = await jobQueue.editAllPendingJobs(\n {\n jobType: 'email',\n tags: { values: ['urgent'], mode: 'any' },\n },\n {\n priority: 10,\n maxAttempts: 5,\n },\n);\n```\n\n**Note:** Only pending jobs are edited. Jobs with other statuses (processing, completed, failed, cancelled) are not affected. Edit events are recorded for each affected job, just like single job edits.\n\n### cancelAllUpcomingJobs\n\n```ts\ncancelAllUpcomingJobs(filters?: {\n jobType?: string;\n priority?: number;\n runAt?: Date | { gt?: Date; gte?: Date; lt?: Date; lte?: Date; eq?: Date };\n tags?: { values: string[]; mode?: 'all' | 'any' | 'none' | 'exact' };\n}): Promise<number>\n```\n\nCancels all upcoming jobs that match the filters. Returns the number of jobs cancelled.\n\n### cleanupOldJobs\n\n```ts\ncleanupOldJobs(daysToKeep?: number): Promise<number>\n```\n\nCleans up jobs older than the specified number of days. Returns the number of jobs removed.\n\n### reclaimStuckJobs\n\n```ts\nreclaimStuckJobs(maxProcessingTimeMinutes?: number): Promise<number>\n```\n\nReclaims jobs stuck in 'processing' for too long. Returns the number of jobs reclaimed. If a job has a `timeoutMs` that is longer than the `maxProcessingTimeMinutes` threshold, the job's own timeout is used instead, preventing premature reclamation of long-running jobs.\n\n---\n\n## Job Events\n\n### getJobEvents\n\n```ts\ngetJobEvents(jobId: number): Promise<JobEvent[]>\n```\n\nRetrieves the job events for a job.\n\n#### JobEvent\n\n```ts\ninterface JobEvent {\n id: number;\n jobId: number;\n eventType: JobEventType;\n createdAt: Date;\n metadata: any;\n}\n```\n\n#### JobEventType\n\n```ts\nenum JobEventType {\n Added = 'added',\n Processing = 'processing',\n Completed = 'completed',\n Failed = 'failed',\n Cancelled = 'cancelled',\n Retried = 'retried',\n Edited = 'edited',\n}\n```\n\n---\n\n## Processing Jobs\n\n### createProcessor\n\n```ts\ncreateProcessor(\n handlers: JobHandlers,\n options?: ProcessorOptions\n): Processor\n```\n\nCreates a job processor with the provided handlers and options.\n\n#### ProcessorOptions\n\n```ts\ninterface ProcessorOptions {\n workerId?: string;\n batchSize?: number;\n concurrency?: number;\n pollInterval?: number;\n onError?: (error: Error) => void;\n verbose?: boolean;\n jobType?: string | string[];\n}\n```\n\n---\n\n## Background Supervisor\n\n### createSupervisor\n\n```ts\ncreateSupervisor(options?: SupervisorOptions): Supervisor\n```\n\nCreates a background supervisor that automatically runs maintenance tasks on a configurable interval: reclaiming stuck jobs, cleaning up old completed jobs/events, and expiring timed-out waitpoint tokens.\n\n#### SupervisorOptions\n\n```ts\ninterface SupervisorOptions {\n intervalMs?: number; // default: 60000\n stuckJobsTimeoutMinutes?: number; // default: 10\n cleanupJobsDaysToKeep?: number; // default: 30 (0 to disable)\n cleanupEventsDaysToKeep?: number; // default: 30 (0 to disable)\n cleanupBatchSize?: number; // default: 1000\n reclaimStuckJobs?: boolean; // default: true\n expireTimedOutTokens?: boolean; // default: true\n onError?: (error: Error) => void; // default: console.error\n verbose?: boolean;\n}\n```\n\n#### Supervisor\n\n```ts\ninterface Supervisor {\n start(): Promise<SupervisorRunResult>;\n startInBackground(): void;\n stop(): void;\n stopAndDrain(timeoutMs?: number): Promise<void>;\n isRunning(): boolean;\n}\n```\n\n- `start()` runs all tasks once and returns the results (serverless-friendly).\n- `startInBackground()` starts a background loop that runs every `intervalMs`.\n- `stopAndDrain()` stops the loop and waits for the current run to finish.\n\n#### SupervisorRunResult\n\n```ts\ninterface SupervisorRunResult {\n reclaimedJobs: number;\n cleanedUpJobs: number;\n cleanedUpEvents: number;\n expiredTokens: number;\n}\n```\n\nSee [Long-Running Server](/usage/long-running-server#background-supervisor) for usage examples.\n\n---\n\n## Accessing the Underlying Client\n\n### getPool\n\n```ts\ngetPool(): Pool\n```\n\nReturns the PostgreSQL connection pool instance. Only available when using the PostgreSQL backend.\n\n> **Note:** Throws an error if called when using the Redis backend.\n\n### getRedisClient\n\n```ts\ngetRedisClient(): Redis\n```\n\nReturns the `ioredis` client instance. Only available when using the Redis backend.\n\n> **Note:** Throws an error if called when using the PostgreSQL backend."
|
|
42
|
+
"content": "## Initialization\n\n### initJobQueue\n\n```ts\ninitJobQueue(config: JobQueueConfig): JobQueue\n```\n\nInitializes the job queue system with the provided configuration. The `JobQueueConfig` is a discriminated union -- you provide either a PostgreSQL or Redis configuration.\n\n#### PostgresJobQueueConfig\n\nProvide either `databaseConfig` (the library creates a pool) or `pool` (bring your own `pg.Pool`). At least one must be set.\n\n```ts\ninterface PostgresJobQueueConfig {\n backend?: 'postgres'; // Optional, defaults to 'postgres'\n databaseConfig?: {\n connectionString?: string;\n host?: string;\n port?: number;\n database?: string;\n user?: string;\n password?: string;\n ssl?: DatabaseSSLConfig;\n };\n pool?: import('pg').Pool; // Bring your own pool\n verbose?: boolean;\n}\n```\n\n#### RedisJobQueueConfig\n\nProvide either `redisConfig` (the library creates an ioredis client) or `client` (bring your own). At least one must be set.\n\n```ts\ninterface RedisJobQueueConfig {\n backend: 'redis'; // Required\n redisConfig?: {\n url?: string;\n host?: string;\n port?: number;\n password?: string;\n db?: number;\n tls?: RedisTLSConfig;\n keyPrefix?: string; // Default: 'dq:'\n };\n client?: unknown; // Bring your own ioredis client\n keyPrefix?: string; // Key prefix when using external client (default: 'dq:')\n verbose?: boolean;\n}\n```\n\n#### JobQueueConfig\n\n```ts\ntype JobQueueConfig = PostgresJobQueueConfig | RedisJobQueueConfig;\n```\n\n#### DatabaseSSLConfig\n\n```ts\ninterface DatabaseSSLConfig {\n ca?: string;\n cert?: string;\n key?: string;\n rejectUnauthorized?: boolean;\n}\n```\n\n- `ca` - Client certificate authority (CA) as PEM string or file path. If the value starts with 'file://', it will be loaded from file, otherwise treated as PEM string.\n- `cert` - Client certificate as PEM string or file path. If the value starts with 'file://', it will be loaded from file, otherwise treated as PEM string.\n- `key` - Client private key as PEM string or file path. If the value starts with 'file://', it will be loaded from file, otherwise treated as PEM string.\n- `rejectUnauthorized` - Whether to reject unauthorized certificates (default: true)\n\n#### RedisTLSConfig\n\n```ts\ninterface RedisTLSConfig {\n ca?: string;\n cert?: string;\n key?: string;\n rejectUnauthorized?: boolean;\n}\n```\n\n---\n\n## Adding Jobs\n\n### addJob\n\n```ts\naddJob(job: JobOptions, options?: AddJobOptions): Promise<number>\n```\n\nAdds a job to the queue. Returns the job ID.\n\n#### JobOptions\n\n```ts\ninterface JobOptions {\n jobType: string;\n payload: any;\n maxAttempts?: number;\n priority?: number;\n runAt?: Date | null;\n timeoutMs?: number;\n tags?: string[];\n idempotencyKey?: string;\n retryDelay?: number; // Base delay between retries in seconds (default: 60)\n retryBackoff?: boolean; // Use exponential backoff (default: true)\n retryDelayMax?: number; // Max delay cap in seconds (default: none)\n}\n```\n\n- `retryDelay` - Base delay between retries in seconds. When `retryBackoff` is true, this is the base for exponential backoff (`retryDelay * 2^attempts`). When false, retries use this fixed delay. Default: `60`.\n- `retryBackoff` - Whether to use exponential backoff. When true, delay doubles with each attempt and includes jitter. Default: `true`.\n- `retryDelayMax` - Maximum delay cap in seconds. Only meaningful when `retryBackoff` is true. No limit when omitted.\n\n#### AddJobOptions\n\n```ts\ninterface AddJobOptions {\n db?: DatabaseClient;\n}\n```\n\n- `db` — An external database client (e.g., a `pg.PoolClient` inside a transaction). When provided, the INSERT runs on this client instead of the internal pool. **PostgreSQL only.** Throws if used with the Redis backend.\n\n### addJobs\n\n```ts\naddJobs(jobs: JobOptions[], options?: AddJobOptions): Promise<number[]>\n```\n\nAdds multiple jobs to the queue in a single operation. More efficient than calling `addJob` in a loop because it batches the INSERT into a single database round-trip (PostgreSQL) or a single atomic Lua script (Redis).\n\nReturns an array of job IDs in the same order as the input array.\n\nEach job can independently have its own `priority`, `runAt`, `tags`, `idempotencyKey`, and other options. Idempotency keys are handled per-job — duplicates resolve to the existing job's ID without creating a new row.\n\nPassing an empty array returns `[]` immediately without touching the database.\n\n```ts\nconst jobIds = await jobQueue.addJobs([\n {\n jobType: 'email',\n payload: { to: 'a@example.com', subject: 'Hi', body: '...' },\n },\n {\n jobType: 'email',\n payload: { to: 'b@example.com', subject: 'Hi', body: '...' },\n priority: 10,\n },\n {\n jobType: 'report',\n payload: { reportId: '123', userId: '456' },\n tags: ['monthly'],\n },\n]);\n// jobIds = [1, 2, 3]\n```\n\nThe `{ db }` option works the same as `addJob` — pass a transactional client to batch-insert within an existing transaction (PostgreSQL only).\n\n#### DatabaseClient\n\n```ts\ninterface DatabaseClient {\n query(\n text: string,\n values?: any[],\n ): Promise<{ rows: any[]; rowCount: number | null }>;\n}\n```\n\nAny object matching this interface works — `pg.Pool`, `pg.PoolClient`, `pg.Client`, or ORM query runners that expose a raw `query()` method.\n\n---\n\n## Retrieving Jobs\n\n### getJob\n\n```ts\ngetJob(id: number): Promise<JobRecord | null>\n```\n\nRetrieves a job by its ID.\n\n### getJobs\n\n```ts\ngetJobs(\n filters?: {\n jobType?: string;\n priority?: number;\n runAt?: Date | { gt?: Date; gte?: Date; lt?: Date; lte?: Date; eq?: Date };\n tags?: { values: string[]; mode?: 'all' | 'any' | 'none' | 'exact' };\n },\n limit?: number,\n offset?: number\n): Promise<JobRecord[]>\n```\n\nRetrieves jobs matching the provided filters, with optional pagination.\n\n### getJobsByStatus\n\n```ts\ngetJobsByStatus(status: string, limit?: number, offset?: number): Promise<JobRecord[]>\n```\n\nRetrieves jobs by their status, with pagination.\n\n### getAllJobs\n\n```ts\ngetAllJobs(limit?: number, offset?: number): Promise<JobRecord[]>\n```\n\nRetrieves all jobs, with optional pagination.\n\n### getJobsByTags\n\n```ts\ngetJobsByTags(tags: string[], mode?: TagQueryMode, limit?: number, offset?: number): Promise<JobRecord[]>\n```\n\nRetrieves jobs by tag(s).\n\n---\n\n## Managing Jobs\n\n### retryJob\n\n```ts\nretryJob(jobId: number): Promise<void>\n```\n\nRetries a job given its ID.\n\n### cancelJob\n\n```ts\ncancelJob(jobId: number): Promise<void>\n```\n\nCancels a job given its ID.\n\n### editJob\n\n```ts\neditJob(jobId: number, updates: EditJobOptions): Promise<void>\n```\n\nEdits a pending job given its ID. Only works for jobs with status 'pending'. Silently fails for other statuses (processing, completed, failed, cancelled).\n\n#### EditJobOptions\n\n```ts\ninterface EditJobOptions {\n payload?: any;\n maxAttempts?: number;\n priority?: number;\n runAt?: Date | null;\n timeoutMs?: number;\n tags?: string[];\n retryDelay?: number | null;\n retryBackoff?: boolean | null;\n retryDelayMax?: number | null;\n}\n```\n\nAll fields are optional - only provided fields will be updated. Note that `jobType` cannot be changed. Set retry fields to `null` to revert to legacy default behavior.\n\n#### Example\n\n```ts\n// Edit a pending job's payload and priority\nawait jobQueue.editJob(jobId, {\n payload: { to: 'newemail@example.com', subject: 'Updated' },\n priority: 10,\n});\n\n// Edit only the scheduled run time\nawait jobQueue.editJob(jobId, {\n runAt: new Date(Date.now() + 60000), // Run in 1 minute\n});\n\n// Edit multiple fields at once\nawait jobQueue.editJob(jobId, {\n payload: { to: 'updated@example.com' },\n priority: 5,\n maxAttempts: 10,\n timeoutMs: 30000,\n tags: ['urgent', 'priority'],\n});\n```\n\n### editAllPendingJobs\n\n```ts\neditAllPendingJobs(\n filters?: {\n jobType?: string;\n priority?: number;\n runAt?: Date | { gt?: Date; gte?: Date; lt?: Date; lte?: Date; eq?: Date };\n tags?: { values: string[]; mode?: 'all' | 'any' | 'none' | 'exact' };\n },\n updates: EditJobOptions\n): Promise<number>\n```\n\nEdits all pending jobs that match the filters. Only works for jobs with status 'pending'. Non-pending jobs are not affected. Returns the number of jobs that were edited.\n\n#### Parameters\n\n- `filters` (optional): Filters to select which jobs to edit. If not provided, all pending jobs are edited.\n - `jobType`: Filter by job type\n - `priority`: Filter by priority\n - `runAt`: Filter by scheduled run time (supports `gt`, `gte`, `lt`, `lte`, `eq` operators or exact Date match)\n - `tags`: Filter by tags with mode ('all', 'any', 'none', 'exact')\n- `updates`: The fields to update (same as `EditJobOptions`). All fields are optional - only provided fields will be updated.\n\n#### Returns\n\nThe number of jobs that were successfully edited.\n\n#### Examples\n\n```ts\n// Edit all pending jobs\nconst editedCount = await jobQueue.editAllPendingJobs(undefined, {\n priority: 10,\n});\n\n// Edit all pending email jobs\nconst editedCount = await jobQueue.editAllPendingJobs(\n { jobType: 'email' },\n {\n priority: 5,\n },\n);\n\n// Edit all pending jobs with 'urgent' tag\nconst editedCount = await jobQueue.editAllPendingJobs(\n { tags: { values: ['urgent'], mode: 'any' } },\n {\n priority: 10,\n maxAttempts: 5,\n },\n);\n\n// Edit all pending jobs scheduled in the future\nconst editedCount = await jobQueue.editAllPendingJobs(\n { runAt: { gte: new Date() } },\n {\n priority: 10,\n },\n);\n\n// Edit with combined filters\nconst editedCount = await jobQueue.editAllPendingJobs(\n {\n jobType: 'email',\n tags: { values: ['urgent'], mode: 'any' },\n },\n {\n priority: 10,\n maxAttempts: 5,\n },\n);\n```\n\n**Note:** Only pending jobs are edited. Jobs with other statuses (processing, completed, failed, cancelled) are not affected. Edit events are recorded for each affected job, just like single job edits.\n\n### cancelAllUpcomingJobs\n\n```ts\ncancelAllUpcomingJobs(filters?: {\n jobType?: string;\n priority?: number;\n runAt?: Date | { gt?: Date; gte?: Date; lt?: Date; lte?: Date; eq?: Date };\n tags?: { values: string[]; mode?: 'all' | 'any' | 'none' | 'exact' };\n}): Promise<number>\n```\n\nCancels all upcoming jobs that match the filters. Returns the number of jobs cancelled.\n\n### cleanupOldJobs\n\n```ts\ncleanupOldJobs(daysToKeep?: number): Promise<number>\n```\n\nCleans up jobs older than the specified number of days. Returns the number of jobs removed.\n\n### reclaimStuckJobs\n\n```ts\nreclaimStuckJobs(maxProcessingTimeMinutes?: number): Promise<number>\n```\n\nReclaims jobs stuck in 'processing' for too long. Returns the number of jobs reclaimed. If a job has a `timeoutMs` that is longer than the `maxProcessingTimeMinutes` threshold, the job's own timeout is used instead, preventing premature reclamation of long-running jobs.\n\n---\n\n## Job Events\n\n### getJobEvents\n\n```ts\ngetJobEvents(jobId: number): Promise<JobEvent[]>\n```\n\nRetrieves the job events for a job.\n\n#### JobEvent\n\n```ts\ninterface JobEvent {\n id: number;\n jobId: number;\n eventType: JobEventType;\n createdAt: Date;\n metadata: any;\n}\n```\n\n#### JobEventType\n\n```ts\nenum JobEventType {\n Added = 'added',\n Processing = 'processing',\n Completed = 'completed',\n Failed = 'failed',\n Cancelled = 'cancelled',\n Retried = 'retried',\n Edited = 'edited',\n}\n```\n\n---\n\n## Event Hooks\n\nDataQueue emits real-time events for job lifecycle transitions. Register listeners using `on`, `once`, `off`, and `removeAllListeners`. Works identically with both PostgreSQL and Redis backends.\n\n### QueueEventMap\n\n```ts\ninterface QueueEventMap {\n 'job:added': { jobId: number; jobType: string };\n 'job:processing': { jobId: number; jobType: string };\n 'job:completed': { jobId: number; jobType: string };\n 'job:failed': {\n jobId: number;\n jobType: string;\n error: Error;\n willRetry: boolean;\n };\n 'job:cancelled': { jobId: number };\n 'job:retried': { jobId: number };\n 'job:waiting': { jobId: number; jobType: string };\n 'job:progress': { jobId: number; progress: number };\n error: Error;\n}\n```\n\n### on\n\n```ts\non(event: QueueEventName, listener: (data) => void): void\n```\n\nRegister a listener that fires every time the event is emitted.\n\n### once\n\n```ts\nonce(event: QueueEventName, listener: (data) => void): void\n```\n\nRegister a one-time listener that auto-removes after the first invocation.\n\n### off\n\n```ts\noff(event: QueueEventName, listener: (data) => void): void\n```\n\nRemove a previously registered listener. Pass the exact function reference used with `on` or `once`.\n\n### removeAllListeners\n\n```ts\nremoveAllListeners(event?: QueueEventName): void\n```\n\nRemove all listeners for a specific event, or all listeners for all events when called without arguments.\n\nSee [Event Hooks](/usage/event-hooks) for detailed usage examples.\n\n---\n\n## Processing Jobs\n\n### createProcessor\n\n```ts\ncreateProcessor(\n handlers: JobHandlers,\n options?: ProcessorOptions\n): Processor\n```\n\nCreates a job processor with the provided handlers and options.\n\n#### ProcessorOptions\n\n```ts\ninterface ProcessorOptions {\n workerId?: string;\n batchSize?: number;\n concurrency?: number;\n pollInterval?: number;\n onError?: (error: Error) => void;\n verbose?: boolean;\n jobType?: string | string[];\n}\n```\n\n---\n\n## Background Supervisor\n\n### createSupervisor\n\n```ts\ncreateSupervisor(options?: SupervisorOptions): Supervisor\n```\n\nCreates a background supervisor that automatically runs maintenance tasks on a configurable interval: reclaiming stuck jobs, cleaning up old completed jobs/events, and expiring timed-out waitpoint tokens.\n\n#### SupervisorOptions\n\n```ts\ninterface SupervisorOptions {\n intervalMs?: number; // default: 60000\n stuckJobsTimeoutMinutes?: number; // default: 10\n cleanupJobsDaysToKeep?: number; // default: 30 (0 to disable)\n cleanupEventsDaysToKeep?: number; // default: 30 (0 to disable)\n cleanupBatchSize?: number; // default: 1000\n reclaimStuckJobs?: boolean; // default: true\n expireTimedOutTokens?: boolean; // default: true\n onError?: (error: Error) => void; // default: console.error\n verbose?: boolean;\n}\n```\n\n#### Supervisor\n\n```ts\ninterface Supervisor {\n start(): Promise<SupervisorRunResult>;\n startInBackground(): void;\n stop(): void;\n stopAndDrain(timeoutMs?: number): Promise<void>;\n isRunning(): boolean;\n}\n```\n\n- `start()` runs all tasks once and returns the results (serverless-friendly).\n- `startInBackground()` starts a background loop that runs every `intervalMs`.\n- `stopAndDrain()` stops the loop and waits for the current run to finish.\n\n#### SupervisorRunResult\n\n```ts\ninterface SupervisorRunResult {\n reclaimedJobs: number;\n cleanedUpJobs: number;\n cleanedUpEvents: number;\n expiredTokens: number;\n}\n```\n\nSee [Long-Running Server](/usage/long-running-server#background-supervisor) for usage examples.\n\n---\n\n## Accessing the Underlying Client\n\n### getPool\n\n```ts\ngetPool(): Pool\n```\n\nReturns the PostgreSQL connection pool instance. Only available when using the PostgreSQL backend.\n\n> **Note:** Throws an error if called when using the Redis backend.\n\n### getRedisClient\n\n```ts\ngetRedisClient(): Redis\n```\n\nReturns the `ioredis` client instance. Only available when using the Redis backend.\n\n> **Note:** Throws an error if called when using the PostgreSQL backend."
|
|
43
43
|
},
|
|
44
44
|
{
|
|
45
45
|
"slug": "api/job-record",
|
|
@@ -185,6 +185,12 @@
|
|
|
185
185
|
"description": "",
|
|
186
186
|
"content": "You can edit a pending job by its ID to update its properties before it is processed. Only jobs with status 'pending' can be edited. Attempting to edit a job with any other status (processing, completed, failed, cancelled) will silently fail.\n\n## Basic Usage\n\n```typescript title=\"@/app/api/edit-job/route.ts\"\nimport { NextRequest, NextResponse } from 'next/server';\nimport { getJobQueue } from '@/lib/queue';\n\nexport async function POST(request: NextRequest) {\n try {\n const { jobId, updates } = await request.json();const jobQueue = getJobQueue();\n await jobQueue.editJob(jobId, updates);\n return NextResponse.json({ message: 'Job updated' });\n } catch (error) {\n console.error('Error editing job:', error);\n return NextResponse.json(\n { message: 'Failed to edit job' },\n { status: 500 },\n );\n }\n}\n```\n\n## Editable Fields\n\nAll fields in `EditJobOptions` are optional - only the fields you provide will be updated. The following fields can be edited:\n\n- `payload` - The job payload data\n- `priority` - Job priority (higher runs first)\n- `maxAttempts` - Maximum number of attempts\n- `runAt` - When to run the job (Date or null)\n- `timeoutMs` - Timeout for the job in milliseconds\n- `tags` - Tags for grouping, searching, or batch operations\n\n**Note:** `jobType` cannot be changed. If you need to change the job type, you should cancel the job and create a new one.\n\n## Examples\n\n### Edit Payload\n\n```typescript\n// Update the payload of a pending job\nawait jobQueue.editJob(jobId, {\n payload: { to: 'newemail@example.com', subject: 'Updated Subject' },\n});\n```\n\n### Edit Priority\n\n```typescript\n// Increase the priority of a job\nawait jobQueue.editJob(jobId, {\n priority: 10,\n});\n```\n\n### Edit Scheduled Time\n\n```typescript\n// Reschedule a job to run in 1 hour\nawait jobQueue.editJob(jobId, {\n runAt: new Date(Date.now() + 60 * 60 * 1000),\n});\n\n// Schedule a job to run immediately (or as soon as possible)\nawait jobQueue.editJob(jobId, {\n runAt: null,\n});\n```\n\n### Edit Multiple Fields\n\n```typescript\n// Update multiple fields at once\nawait jobQueue.editJob(jobId, {\n payload: { to: 'updated@example.com', subject: 'New Subject' },\n priority: 5,\n maxAttempts: 10,\n timeoutMs: 30000,\n tags: ['urgent', 'priority'],\n});\n```\n\n### Partial Updates\n\n```typescript\n// Only update what you need - other fields remain unchanged\nawait jobQueue.editJob(jobId, {\n priority: 10,\n // payload, maxAttempts, runAt, timeoutMs, and tags remain unchanged\n});\n```\n\n### Clear Tags or Timeout\n\n```typescript\n// Remove tags by setting to undefined\nawait jobQueue.editJob(jobId, {\n tags: undefined,\n});\n\n// Remove timeout by setting to undefined\nawait jobQueue.editJob(jobId, {\n timeoutMs: undefined,\n});\n```\n\n## Batch Editing\n\nYou can edit multiple pending jobs at once using `editAllPendingJobs`. This is useful when you need to update many jobs that match certain criteria. The function returns the number of jobs that were edited.\n\n### Basic Batch Edit\n\n```typescript\n// Edit all pending jobs\nconst editedCount = await jobQueue.editAllPendingJobs(undefined, {\n priority: 10,\n});\nconsole.log(`Edited ${editedCount} jobs`);\n```\n\n### Filter by Job Type\n\n```typescript\n// Edit all pending email jobs\nconst editedCount = await jobQueue.editAllPendingJobs(\n { jobType: 'email' },\n {\n priority: 5,\n },\n);\n```\n\n### Filter by Priority\n\n```typescript\n// Edit all pending jobs with priority 1\nconst editedCount = await jobQueue.editAllPendingJobs(\n { priority: 1 },\n {\n priority: 5,\n },\n);\n```\n\n### Filter by Tags\n\n```typescript\n// Edit all pending jobs with 'urgent' tag\nconst editedCount = await jobQueue.editAllPendingJobs(\n { tags: { values: ['urgent'], mode: 'any' } },\n {\n priority: 10,\n },\n);\n```\n\n### Filter by Scheduled Time\n\n```typescript\n// Edit all pending jobs scheduled in the future\nconst editedCount = await jobQueue.editAllPendingJobs(\n { runAt: { gte: new Date() } },\n {\n priority: 10,\n },\n);\n\n// Edit all pending jobs scheduled before a specific date\nconst editedCount = await jobQueue.editAllPendingJobs(\n { runAt: { lt: new Date('2024-12-31') } },\n {\n priority: 5,\n },\n);\n```\n\n### Combined Filters\n\n```typescript\n// Edit all pending email jobs with 'urgent' tag\nconst editedCount = await jobQueue.editAllPendingJobs(\n {\n jobType: 'email',\n tags: { values: ['urgent'], mode: 'any' },\n },\n {\n priority: 10,\n maxAttempts: 5,\n },\n);\n```\n\n### Batch Edit Notes\n\n- Only pending jobs are edited. Jobs with other statuses (processing, completed, failed, cancelled) are not affected.\n- The function returns the number of jobs that were successfully edited.\n- Edit events are recorded for each affected job, just like single job edits.\n- If no fields are provided in the updates object, the function returns 0 and no jobs are modified.\n\n## When to Use Edit vs Cancel vs Retry\n\n- **Edit**: Use when you want to modify a pending job's properties before it runs\n- **Cancel**: Use when you want to completely remove a pending job from the queue\n- **Retry**: Use when you want to retry a failed job (sets status back to pending)\n\n## Error Handling\n\nThe `editJob` function silently fails if you try to edit a non-pending job. This means:\n\n- No error is thrown\n- The job remains unchanged\n- The operation completes successfully (but does nothing)\n\nTo check if an edit was successful, you can:\n\n```typescript\nconst job = await jobQueue.getJob(jobId);\nif (job?.status === 'pending') {\n // Job is still pending, edit might have succeeded\n // Check if the fields you wanted to update actually changed\n if (job.priority === newPriority) {\n console.log('Edit successful');\n }\n} else {\n console.log('Job is not pending, edit was ignored');\n}\n```\n\n## Event Tracking\n\nWhen a job is edited, an 'edited' event is recorded in the job's event history. The event metadata contains the fields that were updated:\n\n```typescript\nconst events = await jobQueue.getJobEvents(jobId);\nconst editEvent = events.find((e) => e.eventType === 'edited');\nif (editEvent) {\n console.log('Updated fields:', editEvent.metadata);\n // { payload: {...}, priority: 10, ... }\n}\n```\n\n## Best Practices\n\n1. **Check job status before editing**: If you're unsure whether a job is pending, check its status first:\n\n```typescript\nconst job = await jobQueue.getJob(jobId);\nif (job?.status === 'pending') {\n await jobQueue.editJob(jobId, updates);\n} else {\n console.log('Job is not pending, cannot edit');\n}\n```\n\n2. **Use partial updates**: Only update the fields you need to change. This is more efficient and reduces the chance of accidentally overwriting other fields.\n\n3. **Validate updates**: Ensure the updated values are valid for your job handlers. For example, if your handler expects a specific payload structure, make sure the updated payload matches.\n\n4. **Consider race conditions**: If a job might be picked up for processing while you're editing it, be aware that the edit might not take effect if the job transitions to 'processing' status between your check and the edit operation.\n\n5. **Monitor events**: Use job events to track when and what was edited for audit purposes."
|
|
187
187
|
},
|
|
188
|
+
{
|
|
189
|
+
"slug": "usage/event-hooks",
|
|
190
|
+
"title": "Event Hooks",
|
|
191
|
+
"description": "",
|
|
192
|
+
"content": "DataQueue emits real-time events for job lifecycle transitions, progress updates, and internal errors. Use event hooks to integrate with logging, metrics, alerting, or any custom logic without polling.\n\nEvent hooks work identically with both the PostgreSQL and Redis backends.\n\n## Listening for Events\n\nRegister listeners with `on()`, `once()`, or remove them with `off()` and `removeAllListeners()`.\n\n```typescript\nconst queue = initJobQueue<MyPayloadMap>(config);\n\nqueue.on('job:completed', (event) => {\n console.log(`Job ${event.jobId} (${event.jobType}) completed`);\n});\n\nqueue.on('job:failed', (event) => {\n console.error(`Job ${event.jobId} failed: ${event.error.message}`);\n if (!event.willRetry) {\n alertOps(`Permanent failure for job ${event.jobId}`);\n }\n});\n\nqueue.on('error', (error) => {\n logger.error('Queue internal error:', error);\n});\n```\n\n## Available Events\n\n| Event | Payload | When |\n| ---------------- | -------------------------------------- | --------------------------------------------------------------------------------- |\n| `job:added` | `{ jobId, jobType }` | After `addJob()` or `addJobs()` |\n| `job:processing` | `{ jobId, jobType }` | When a processor claims and starts a job |\n| `job:completed` | `{ jobId, jobType }` | When a handler completes successfully |\n| `job:failed` | `{ jobId, jobType, error, willRetry }` | When a handler throws or times out |\n| `job:cancelled` | `{ jobId }` | After `cancelJob()` |\n| `job:retried` | `{ jobId }` | After `retryJob()` |\n| `job:waiting` | `{ jobId, jobType }` | When a handler enters a wait (`ctx.waitFor`, `ctx.waitUntil`, `ctx.waitForToken`) |\n| `job:progress` | `{ jobId, progress }` | When a handler calls `ctx.setProgress()` |\n| `error` | `Error` | Internal errors from the processor or supervisor |\n\n## One-Time Listeners\n\nUse `once()` when you only need to react to the first occurrence of an event.\n\n```typescript\nqueue.once('job:added', (event) => {\n console.log('First job added:', event.jobId);\n});\n```\n\n## Removing Listeners\n\n```typescript\nconst listener = (event) => console.log(event);\n\nqueue.on('job:completed', listener);\n\n// Remove a specific listener\nqueue.off('job:completed', listener);\n\n// Remove all listeners for one event\nqueue.removeAllListeners('job:completed');\n\n// Remove all listeners for all events\nqueue.removeAllListeners();\n```\n\n## Error Monitoring\n\nThe `error` event fires for internal errors in the processor and supervisor. It works alongside the existing `onError` callback in `ProcessorOptions` and `SupervisorOptions` -- both fire independently.\n\n```typescript\nqueue.on('error', (error) => {\n Sentry.captureException(error);\n});\n\n// onError still works as before\nconst processor = queue.createProcessor(handlers, {\n onError: (error) => console.error('Processor error:', error),\n});\n```\n\n## Failure Retry Detection\n\nThe `job:failed` event includes a `willRetry` boolean that tells you whether the job will be retried automatically.\n\n```typescript\nqueue.on('job:failed', (event) => {\n if (event.willRetry) {\n metrics.increment('job.retry', { jobType: event.jobType });\n } else {\n metrics.increment('job.permanent_failure', { jobType: event.jobType });\n pagerDuty.alert(`Job ${event.jobId} permanently failed`);\n }\n});\n```\n\n## Progress Tracking\n\nThe `job:progress` event fires whenever a handler calls `ctx.setProgress()`, giving you real-time progress updates.\n\n```typescript\nqueue.on('job:progress', (event) => {\n websocket.broadcast(`job:${event.jobId}`, { progress: event.progress });\n});\n```\n\n> **Note:** Events are emitted synchronously after the corresponding database operation\n completes. Slow event listeners will delay the return of methods like\n `addJob()` or the processing of the next job. Use async patterns in listeners\n if they perform I/O."
|
|
193
|
+
},
|
|
188
194
|
{
|
|
189
195
|
"slug": "usage/failed-jobs",
|
|
190
196
|
"title": "Failed Jobs",
|
package/ai/rules/advanced.md
CHANGED
|
@@ -116,6 +116,24 @@ await queue.addJob({
|
|
|
116
116
|
- No config — legacy `2^attempts * 60s` formula (backward compatible).
|
|
117
117
|
- Cron schedules propagate retry config to enqueued jobs.
|
|
118
118
|
|
|
119
|
+
## Event Hooks
|
|
120
|
+
|
|
121
|
+
Subscribe to real-time lifecycle events via `on`, `once`, `off`, `removeAllListeners`. Works with both Postgres and Redis.
|
|
122
|
+
|
|
123
|
+
```typescript
|
|
124
|
+
queue.on('job:completed', ({ jobId, jobType }) => {
|
|
125
|
+
metrics.increment('job.completed', { jobType });
|
|
126
|
+
});
|
|
127
|
+
queue.on('job:failed', ({ jobId, jobType, error, willRetry }) => {
|
|
128
|
+
if (!willRetry) alertOps(`Permanent failure: ${jobId}`);
|
|
129
|
+
});
|
|
130
|
+
queue.on('error', (error) => Sentry.captureException(error));
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
Events: `job:added`, `job:processing`, `job:completed`, `job:failed` (with `willRetry`), `job:cancelled`, `job:retried`, `job:waiting`, `job:progress`, `error`.
|
|
134
|
+
|
|
135
|
+
`error` events fire alongside `onError` callbacks in `ProcessorOptions` / `SupervisorOptions` — both mechanisms work independently.
|
|
136
|
+
|
|
119
137
|
## Scaling
|
|
120
138
|
|
|
121
139
|
- Increase `batchSize` and `concurrency` for higher throughput.
|
|
@@ -170,6 +170,56 @@ await queue.addJob({
|
|
|
170
170
|
- Handler must be serializable (no closures over external variables).
|
|
171
171
|
- `prolong`, `onTimeout`, `ctx.run`, waits are NOT available.
|
|
172
172
|
|
|
173
|
+
## Event Hooks
|
|
174
|
+
|
|
175
|
+
Subscribe to real-time job lifecycle events. Works identically with PostgreSQL and Redis.
|
|
176
|
+
|
|
177
|
+
```typescript
|
|
178
|
+
const queue = initJobQueue<MyPayloadMap>(config);
|
|
179
|
+
|
|
180
|
+
queue.on('job:completed', ({ jobId, jobType }) => {
|
|
181
|
+
console.log(`Job ${jobId} (${jobType}) completed`);
|
|
182
|
+
});
|
|
183
|
+
|
|
184
|
+
queue.on('job:failed', ({ jobId, jobType, error, willRetry }) => {
|
|
185
|
+
console.error(`Job ${jobId} failed: ${error.message}`);
|
|
186
|
+
if (!willRetry) {
|
|
187
|
+
alertOps(`Permanent failure for job ${jobId}`);
|
|
188
|
+
}
|
|
189
|
+
});
|
|
190
|
+
|
|
191
|
+
queue.on('error', (error) => {
|
|
192
|
+
Sentry.captureException(error);
|
|
193
|
+
});
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
### Available events
|
|
197
|
+
|
|
198
|
+
| Event | Payload |
|
|
199
|
+
| ---------------- | -------------------------------------- |
|
|
200
|
+
| `job:added` | `{ jobId, jobType }` |
|
|
201
|
+
| `job:processing` | `{ jobId, jobType }` |
|
|
202
|
+
| `job:completed` | `{ jobId, jobType }` |
|
|
203
|
+
| `job:failed` | `{ jobId, jobType, error, willRetry }` |
|
|
204
|
+
| `job:cancelled` | `{ jobId }` |
|
|
205
|
+
| `job:retried` | `{ jobId }` |
|
|
206
|
+
| `job:waiting` | `{ jobId, jobType }` |
|
|
207
|
+
| `job:progress` | `{ jobId, progress }` |
|
|
208
|
+
| `error` | `Error` |
|
|
209
|
+
|
|
210
|
+
### Listener management
|
|
211
|
+
|
|
212
|
+
```typescript
|
|
213
|
+
const listener = ({ jobId }) => console.log(jobId);
|
|
214
|
+
queue.on('job:completed', listener);
|
|
215
|
+
queue.off('job:completed', listener);
|
|
216
|
+
queue.once('job:added', ({ jobId }) => console.log('First job:', jobId));
|
|
217
|
+
queue.removeAllListeners('job:completed');
|
|
218
|
+
queue.removeAllListeners(); // all events
|
|
219
|
+
```
|
|
220
|
+
|
|
221
|
+
The `error` event fires alongside `onError` callbacks in `ProcessorOptions` and `SupervisorOptions` -- both mechanisms work independently.
|
|
222
|
+
|
|
173
223
|
## Tags
|
|
174
224
|
|
|
175
225
|
```typescript
|
package/dist/index.cjs
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
+
var events = require('events');
|
|
3
4
|
var worker_threads = require('worker_threads');
|
|
4
5
|
var async_hooks = require('async_hooks');
|
|
5
6
|
var pg = require('pg');
|
|
@@ -14,7 +15,7 @@ function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
|
|
|
14
15
|
|
|
15
16
|
var fs__default = /*#__PURE__*/_interopDefault(fs);
|
|
16
17
|
|
|
17
|
-
// src/
|
|
18
|
+
// src/index.ts
|
|
18
19
|
|
|
19
20
|
// src/types.ts
|
|
20
21
|
var JobEventType = /* @__PURE__ */ ((JobEventType2) => {
|
|
@@ -365,18 +366,23 @@ function buildWaitContext(backend, jobId, stepData, baseCtx) {
|
|
|
365
366
|
};
|
|
366
367
|
return ctx;
|
|
367
368
|
}
|
|
368
|
-
async function processJobWithHandlers(backend, job, jobHandlers) {
|
|
369
|
+
async function processJobWithHandlers(backend, job, jobHandlers, emit) {
|
|
369
370
|
const handler = jobHandlers[job.jobType];
|
|
370
371
|
if (!handler) {
|
|
371
372
|
await backend.setPendingReasonForUnpickedJobs(
|
|
372
373
|
`No handler registered for job type: ${job.jobType}`,
|
|
373
374
|
job.jobType
|
|
374
375
|
);
|
|
375
|
-
|
|
376
|
-
job.
|
|
377
|
-
new Error(`No handler registered for job type: ${job.jobType}`),
|
|
378
|
-
"no_handler" /* NoHandler */
|
|
376
|
+
const noHandlerError = new Error(
|
|
377
|
+
`No handler registered for job type: ${job.jobType}`
|
|
379
378
|
);
|
|
379
|
+
await backend.failJob(job.id, noHandlerError, "no_handler" /* NoHandler */);
|
|
380
|
+
emit?.("job:failed", {
|
|
381
|
+
jobId: job.id,
|
|
382
|
+
jobType: job.jobType,
|
|
383
|
+
error: noHandlerError,
|
|
384
|
+
willRetry: false
|
|
385
|
+
});
|
|
380
386
|
return;
|
|
381
387
|
}
|
|
382
388
|
const stepData = { ...job.stepData || {} };
|
|
@@ -445,6 +451,16 @@ async function processJobWithHandlers(backend, job, jobHandlers) {
|
|
|
445
451
|
}
|
|
446
452
|
};
|
|
447
453
|
const ctx = buildWaitContext(backend, job.id, stepData, baseCtx);
|
|
454
|
+
if (emit) {
|
|
455
|
+
const originalSetProgress = ctx.setProgress;
|
|
456
|
+
ctx.setProgress = async (percent) => {
|
|
457
|
+
await originalSetProgress(percent);
|
|
458
|
+
emit("job:progress", {
|
|
459
|
+
jobId: job.id,
|
|
460
|
+
progress: Math.round(percent)
|
|
461
|
+
});
|
|
462
|
+
};
|
|
463
|
+
}
|
|
448
464
|
if (forceKillOnTimeout && !hasTimeout) {
|
|
449
465
|
log(
|
|
450
466
|
`forceKillOnTimeout is set but no timeoutMs for job ${job.id}, running without force kill`
|
|
@@ -465,6 +481,7 @@ async function processJobWithHandlers(backend, job, jobHandlers) {
|
|
|
465
481
|
}
|
|
466
482
|
if (timeoutId) clearTimeout(timeoutId);
|
|
467
483
|
await backend.completeJob(job.id);
|
|
484
|
+
emit?.("job:completed", { jobId: job.id, jobType: job.jobType });
|
|
468
485
|
} catch (error) {
|
|
469
486
|
if (timeoutId) clearTimeout(timeoutId);
|
|
470
487
|
if (error instanceof WaitSignal) {
|
|
@@ -476,6 +493,7 @@ async function processJobWithHandlers(backend, job, jobHandlers) {
|
|
|
476
493
|
waitTokenId: error.tokenId,
|
|
477
494
|
stepData: error.stepData
|
|
478
495
|
});
|
|
496
|
+
emit?.("job:waiting", { jobId: job.id, jobType: job.jobType });
|
|
479
497
|
return;
|
|
480
498
|
}
|
|
481
499
|
console.error(`Error processing job ${job.id}:`, error);
|
|
@@ -483,22 +501,32 @@ async function processJobWithHandlers(backend, job, jobHandlers) {
|
|
|
483
501
|
if (error && typeof error === "object" && "failureReason" in error && error.failureReason === "timeout" /* Timeout */) {
|
|
484
502
|
failureReason = "timeout" /* Timeout */;
|
|
485
503
|
}
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
504
|
+
const failError = error instanceof Error ? error : new Error(String(error));
|
|
505
|
+
await backend.failJob(job.id, failError, failureReason);
|
|
506
|
+
emit?.("job:failed", {
|
|
507
|
+
jobId: job.id,
|
|
508
|
+
jobType: job.jobType,
|
|
509
|
+
error: failError,
|
|
510
|
+
willRetry: job.attempts + 1 < job.maxAttempts
|
|
511
|
+
});
|
|
491
512
|
}
|
|
492
513
|
}
|
|
493
|
-
async function processBatchWithHandlers(backend, workerId, batchSize, jobType, jobHandlers, concurrency, onError) {
|
|
514
|
+
async function processBatchWithHandlers(backend, workerId, batchSize, jobType, jobHandlers, concurrency, onError, emit) {
|
|
494
515
|
const jobs = await backend.getNextBatch(
|
|
495
516
|
workerId,
|
|
496
517
|
batchSize,
|
|
497
518
|
jobType
|
|
498
519
|
);
|
|
520
|
+
if (emit) {
|
|
521
|
+
for (const job of jobs) {
|
|
522
|
+
emit("job:processing", { jobId: job.id, jobType: job.jobType });
|
|
523
|
+
}
|
|
524
|
+
}
|
|
499
525
|
if (!concurrency || concurrency >= jobs.length) {
|
|
500
526
|
await Promise.all(
|
|
501
|
-
jobs.map(
|
|
527
|
+
jobs.map(
|
|
528
|
+
(job) => processJobWithHandlers(backend, job, jobHandlers, emit)
|
|
529
|
+
)
|
|
502
530
|
);
|
|
503
531
|
return jobs.length;
|
|
504
532
|
}
|
|
@@ -511,7 +539,7 @@ async function processBatchWithHandlers(backend, workerId, batchSize, jobType, j
|
|
|
511
539
|
while (running < concurrency && idx < jobs.length) {
|
|
512
540
|
const job = jobs[idx++];
|
|
513
541
|
running++;
|
|
514
|
-
processJobWithHandlers(backend, job, jobHandlers).then(() => {
|
|
542
|
+
processJobWithHandlers(backend, job, jobHandlers, emit).then(() => {
|
|
515
543
|
running--;
|
|
516
544
|
finished++;
|
|
517
545
|
next();
|
|
@@ -528,7 +556,7 @@ async function processBatchWithHandlers(backend, workerId, batchSize, jobType, j
|
|
|
528
556
|
next();
|
|
529
557
|
});
|
|
530
558
|
}
|
|
531
|
-
var createProcessor = (backend, handlers, options = {}, onBeforeBatch) => {
|
|
559
|
+
var createProcessor = (backend, handlers, options = {}, onBeforeBatch, emit) => {
|
|
532
560
|
const {
|
|
533
561
|
workerId = `worker-${Math.random().toString(36).substring(2, 9)}`,
|
|
534
562
|
batchSize = 10,
|
|
@@ -548,11 +576,11 @@ var createProcessor = (backend, handlers, options = {}, onBeforeBatch) => {
|
|
|
548
576
|
await onBeforeBatch();
|
|
549
577
|
} catch (hookError) {
|
|
550
578
|
log(`onBeforeBatch hook error: ${hookError}`);
|
|
579
|
+
const err = hookError instanceof Error ? hookError : new Error(String(hookError));
|
|
551
580
|
if (onError) {
|
|
552
|
-
onError(
|
|
553
|
-
hookError instanceof Error ? hookError : new Error(String(hookError))
|
|
554
|
-
);
|
|
581
|
+
onError(err);
|
|
555
582
|
}
|
|
583
|
+
emit?.("error", err);
|
|
556
584
|
}
|
|
557
585
|
}
|
|
558
586
|
log(
|
|
@@ -566,11 +594,14 @@ var createProcessor = (backend, handlers, options = {}, onBeforeBatch) => {
|
|
|
566
594
|
jobType,
|
|
567
595
|
handlers,
|
|
568
596
|
concurrency,
|
|
569
|
-
onError
|
|
597
|
+
onError,
|
|
598
|
+
emit
|
|
570
599
|
);
|
|
571
600
|
return processed;
|
|
572
601
|
} catch (error) {
|
|
573
|
-
|
|
602
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
603
|
+
onError(err);
|
|
604
|
+
emit?.("error", err);
|
|
574
605
|
}
|
|
575
606
|
return 0;
|
|
576
607
|
};
|
|
@@ -651,7 +682,7 @@ var createProcessor = (backend, handlers, options = {}, onBeforeBatch) => {
|
|
|
651
682
|
};
|
|
652
683
|
|
|
653
684
|
// src/supervisor.ts
|
|
654
|
-
var createSupervisor = (backend, options = {}) => {
|
|
685
|
+
var createSupervisor = (backend, options = {}, emit) => {
|
|
655
686
|
const {
|
|
656
687
|
intervalMs = 6e4,
|
|
657
688
|
stuckJobsTimeoutMinutes = 10,
|
|
@@ -684,7 +715,9 @@ var createSupervisor = (backend, options = {}) => {
|
|
|
684
715
|
log(`Supervisor: reclaimed ${result.reclaimedJobs} stuck jobs`);
|
|
685
716
|
}
|
|
686
717
|
} catch (e) {
|
|
687
|
-
|
|
718
|
+
const err = e instanceof Error ? e : new Error(String(e));
|
|
719
|
+
onError(err);
|
|
720
|
+
emit?.("error", err);
|
|
688
721
|
}
|
|
689
722
|
}
|
|
690
723
|
if (cleanupJobsDaysToKeep > 0) {
|
|
@@ -697,7 +730,9 @@ var createSupervisor = (backend, options = {}) => {
|
|
|
697
730
|
log(`Supervisor: cleaned up ${result.cleanedUpJobs} old jobs`);
|
|
698
731
|
}
|
|
699
732
|
} catch (e) {
|
|
700
|
-
|
|
733
|
+
const err = e instanceof Error ? e : new Error(String(e));
|
|
734
|
+
onError(err);
|
|
735
|
+
emit?.("error", err);
|
|
701
736
|
}
|
|
702
737
|
}
|
|
703
738
|
if (cleanupEventsDaysToKeep > 0) {
|
|
@@ -712,7 +747,9 @@ var createSupervisor = (backend, options = {}) => {
|
|
|
712
747
|
);
|
|
713
748
|
}
|
|
714
749
|
} catch (e) {
|
|
715
|
-
|
|
750
|
+
const err = e instanceof Error ? e : new Error(String(e));
|
|
751
|
+
onError(err);
|
|
752
|
+
emit?.("error", err);
|
|
716
753
|
}
|
|
717
754
|
}
|
|
718
755
|
if (expireTimedOutTokens) {
|
|
@@ -722,7 +759,9 @@ var createSupervisor = (backend, options = {}) => {
|
|
|
722
759
|
log(`Supervisor: expired ${result.expiredTokens} timed-out tokens`);
|
|
723
760
|
}
|
|
724
761
|
} catch (e) {
|
|
725
|
-
|
|
762
|
+
const err = e instanceof Error ? e : new Error(String(e));
|
|
763
|
+
onError(err);
|
|
764
|
+
emit?.("error", err);
|
|
726
765
|
}
|
|
727
766
|
}
|
|
728
767
|
return result;
|
|
@@ -4876,6 +4915,10 @@ var initJobQueue = (config) => {
|
|
|
4876
4915
|
} else {
|
|
4877
4916
|
throw new Error(`Unknown backend: ${backendType}`);
|
|
4878
4917
|
}
|
|
4918
|
+
const emitter = new events.EventEmitter();
|
|
4919
|
+
const emit = (event, data) => {
|
|
4920
|
+
emitter.emit(event, data);
|
|
4921
|
+
};
|
|
4879
4922
|
const enqueueDueCronJobsImpl = async () => {
|
|
4880
4923
|
const dueSchedules = await backend.getDueCronSchedules();
|
|
4881
4924
|
let count = 0;
|
|
@@ -4925,11 +4968,21 @@ var initJobQueue = (config) => {
|
|
|
4925
4968
|
return {
|
|
4926
4969
|
// Job queue operations
|
|
4927
4970
|
addJob: withLogContext(
|
|
4928
|
-
(job, options) =>
|
|
4971
|
+
async (job, options) => {
|
|
4972
|
+
const jobId = await backend.addJob(job, options);
|
|
4973
|
+
emit("job:added", { jobId, jobType: job.jobType });
|
|
4974
|
+
return jobId;
|
|
4975
|
+
},
|
|
4929
4976
|
config.verbose ?? false
|
|
4930
4977
|
),
|
|
4931
4978
|
addJobs: withLogContext(
|
|
4932
|
-
(jobs, options) =>
|
|
4979
|
+
async (jobs, options) => {
|
|
4980
|
+
const jobIds = await backend.addJobs(jobs, options);
|
|
4981
|
+
for (let i = 0; i < jobIds.length; i++) {
|
|
4982
|
+
emit("job:added", { jobId: jobIds[i], jobType: jobs[i].jobType });
|
|
4983
|
+
}
|
|
4984
|
+
return jobIds;
|
|
4985
|
+
},
|
|
4933
4986
|
config.verbose ?? false
|
|
4934
4987
|
),
|
|
4935
4988
|
getJob: withLogContext(
|
|
@@ -4948,13 +5001,16 @@ var initJobQueue = (config) => {
|
|
|
4948
5001
|
(filters, limit, offset) => backend.getJobs(filters, limit, offset),
|
|
4949
5002
|
config.verbose ?? false
|
|
4950
5003
|
),
|
|
4951
|
-
retryJob: (jobId) =>
|
|
5004
|
+
retryJob: async (jobId) => {
|
|
5005
|
+
await backend.retryJob(jobId);
|
|
5006
|
+
emit("job:retried", { jobId });
|
|
5007
|
+
},
|
|
4952
5008
|
cleanupOldJobs: (daysToKeep, batchSize) => backend.cleanupOldJobs(daysToKeep, batchSize),
|
|
4953
5009
|
cleanupOldJobEvents: (daysToKeep, batchSize) => backend.cleanupOldJobEvents(daysToKeep, batchSize),
|
|
4954
|
-
cancelJob: withLogContext(
|
|
4955
|
-
|
|
4956
|
-
|
|
4957
|
-
),
|
|
5010
|
+
cancelJob: withLogContext(async (jobId) => {
|
|
5011
|
+
await backend.cancelJob(jobId);
|
|
5012
|
+
emit("job:cancelled", { jobId });
|
|
5013
|
+
}, config.verbose ?? false),
|
|
4958
5014
|
editJob: withLogContext(
|
|
4959
5015
|
(jobId, updates) => backend.editJob(jobId, updates),
|
|
4960
5016
|
config.verbose ?? false
|
|
@@ -4979,11 +5035,17 @@ var initJobQueue = (config) => {
|
|
|
4979
5035
|
config.verbose ?? false
|
|
4980
5036
|
),
|
|
4981
5037
|
// Job processing — automatically enqueues due cron jobs before each batch
|
|
4982
|
-
createProcessor: (handlers, options) => createProcessor(
|
|
4983
|
-
|
|
4984
|
-
|
|
5038
|
+
createProcessor: (handlers, options) => createProcessor(
|
|
5039
|
+
backend,
|
|
5040
|
+
handlers,
|
|
5041
|
+
options,
|
|
5042
|
+
async () => {
|
|
5043
|
+
await enqueueDueCronJobsImpl();
|
|
5044
|
+
},
|
|
5045
|
+
emit
|
|
5046
|
+
),
|
|
4985
5047
|
// Background supervisor — automated maintenance
|
|
4986
|
-
createSupervisor: (options) => createSupervisor(backend, options),
|
|
5048
|
+
createSupervisor: (options) => createSupervisor(backend, options, emit),
|
|
4987
5049
|
// Job events
|
|
4988
5050
|
getJobEvents: withLogContext(
|
|
4989
5051
|
(jobId) => backend.getJobEvents(jobId),
|
|
@@ -5085,6 +5147,23 @@ var initJobQueue = (config) => {
|
|
|
5085
5147
|
() => enqueueDueCronJobsImpl(),
|
|
5086
5148
|
config.verbose ?? false
|
|
5087
5149
|
),
|
|
5150
|
+
// Event hooks
|
|
5151
|
+
on: (event, listener) => {
|
|
5152
|
+
emitter.on(event, listener);
|
|
5153
|
+
},
|
|
5154
|
+
once: (event, listener) => {
|
|
5155
|
+
emitter.once(event, listener);
|
|
5156
|
+
},
|
|
5157
|
+
off: (event, listener) => {
|
|
5158
|
+
emitter.off(event, listener);
|
|
5159
|
+
},
|
|
5160
|
+
removeAllListeners: (event) => {
|
|
5161
|
+
if (event) {
|
|
5162
|
+
emitter.removeAllListeners(event);
|
|
5163
|
+
} else {
|
|
5164
|
+
emitter.removeAllListeners();
|
|
5165
|
+
}
|
|
5166
|
+
},
|
|
5088
5167
|
// Advanced access
|
|
5089
5168
|
getPool: () => {
|
|
5090
5169
|
if (!(backend instanceof PostgresBackend)) {
|