@pattern-stack/codegen 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +70 -0
- package/dist/runtime/subsystems/auth/controllers/auth.controller.d.ts +1 -0
- package/dist/runtime/subsystems/auth/index.d.ts +2 -0
- package/dist/runtime/subsystems/auth/index.js +55 -0
- package/dist/runtime/subsystems/auth/index.js.map +1 -1
- package/dist/runtime/subsystems/auth/middleware/requester-context.d.ts +81 -0
- package/dist/runtime/subsystems/auth/middleware/requester-context.js +60 -0
- package/dist/runtime/subsystems/auth/middleware/requester-context.js.map +1 -0
- package/dist/runtime/subsystems/auth/protocols/user-context.d.ts +18 -0
- package/dist/runtime/subsystems/bridge/bridge-delivery-handler.js.map +1 -1
- package/dist/runtime/subsystems/bridge/bridge-outbox-drain-hook.js.map +1 -1
- package/dist/runtime/subsystems/bridge/bridge.module.d.ts +3 -0
- package/dist/runtime/subsystems/bridge/bridge.module.js +930 -275
- package/dist/runtime/subsystems/bridge/bridge.module.js.map +1 -1
- package/dist/runtime/subsystems/bridge/event-flow.service.js.map +1 -1
- package/dist/runtime/subsystems/bridge/index.d.ts +3 -0
- package/dist/runtime/subsystems/bridge/index.js +837 -182
- package/dist/runtime/subsystems/bridge/index.js.map +1 -1
- package/dist/runtime/subsystems/events/event-bus.drizzle-backend.d.ts +3 -1
- package/dist/runtime/subsystems/events/event-bus.drizzle-backend.js +92 -1
- package/dist/runtime/subsystems/events/event-bus.drizzle-backend.js.map +1 -1
- package/dist/runtime/subsystems/events/event-bus.memory-backend.d.ts +3 -1
- package/dist/runtime/subsystems/events/event-bus.memory-backend.js +99 -0
- package/dist/runtime/subsystems/events/event-bus.memory-backend.js.map +1 -1
- package/dist/runtime/subsystems/events/event-bus.redis-backend.js.map +1 -1
- package/dist/runtime/subsystems/events/event-keyset-cursor.d.ts +32 -0
- package/dist/runtime/subsystems/events/event-keyset-cursor.js +38 -0
- package/dist/runtime/subsystems/events/event-keyset-cursor.js.map +1 -0
- package/dist/runtime/subsystems/events/event-read.protocol.d.ts +94 -0
- package/dist/runtime/subsystems/events/event-read.protocol.js +9 -0
- package/dist/runtime/subsystems/events/event-read.protocol.js.map +1 -0
- package/dist/runtime/subsystems/events/events.module.js +177 -3
- package/dist/runtime/subsystems/events/events.module.js.map +1 -1
- package/dist/runtime/subsystems/events/events.tokens.d.ts +16 -1
- package/dist/runtime/subsystems/events/events.tokens.js +2 -0
- package/dist/runtime/subsystems/events/events.tokens.js.map +1 -1
- package/dist/runtime/subsystems/events/generated/bus.js.map +1 -1
- package/dist/runtime/subsystems/events/generated/index.js.map +1 -1
- package/dist/runtime/subsystems/events/index.d.ts +2 -1
- package/dist/runtime/subsystems/events/index.js +178 -3
- package/dist/runtime/subsystems/events/index.js.map +1 -1
- package/dist/runtime/subsystems/index.d.ts +2 -0
- package/dist/runtime/subsystems/index.js +1198 -264
- package/dist/runtime/subsystems/index.js.map +1 -1
- package/dist/runtime/subsystems/jobs/bullmq.config.d.ts +98 -0
- package/dist/runtime/subsystems/jobs/bullmq.config.js +143 -0
- package/dist/runtime/subsystems/jobs/bullmq.config.js.map +1 -0
- package/dist/runtime/subsystems/jobs/index.d.ts +6 -2
- package/dist/runtime/subsystems/jobs/index.js +861 -201
- package/dist/runtime/subsystems/jobs/index.js.map +1 -1
- package/dist/runtime/subsystems/jobs/job-orchestrator.bullmq-backend.d.ts +107 -0
- package/dist/runtime/subsystems/jobs/job-orchestrator.bullmq-backend.js +922 -0
- package/dist/runtime/subsystems/jobs/job-orchestrator.bullmq-backend.js.map +1 -0
- package/dist/runtime/subsystems/jobs/job-run-keyset-cursor.d.ts +52 -0
- package/dist/runtime/subsystems/jobs/job-run-keyset-cursor.js +57 -0
- package/dist/runtime/subsystems/jobs/job-run-keyset-cursor.js.map +1 -0
- package/dist/runtime/subsystems/jobs/job-run-service.drizzle-backend.d.ts +2 -1
- package/dist/runtime/subsystems/jobs/job-run-service.drizzle-backend.js +81 -1
- package/dist/runtime/subsystems/jobs/job-run-service.drizzle-backend.js.map +1 -1
- package/dist/runtime/subsystems/jobs/job-run-service.memory-backend.d.ts +2 -1
- package/dist/runtime/subsystems/jobs/job-run-service.memory-backend.js +81 -0
- package/dist/runtime/subsystems/jobs/job-run-service.memory-backend.js.map +1 -1
- package/dist/runtime/subsystems/jobs/job-run-service.protocol.d.ts +74 -1
- package/dist/runtime/subsystems/jobs/job-worker.bullmq-backend.d.ts +48 -0
- package/dist/runtime/subsystems/jobs/job-worker.bullmq-backend.js +374 -0
- package/dist/runtime/subsystems/jobs/job-worker.bullmq-backend.js.map +1 -0
- package/dist/runtime/subsystems/jobs/job-worker.module.d.ts +42 -4
- package/dist/runtime/subsystems/jobs/job-worker.module.js +832 -178
- package/dist/runtime/subsystems/jobs/job-worker.module.js.map +1 -1
- package/dist/runtime/subsystems/jobs/jobs-domain.module.d.ts +10 -1
- package/dist/runtime/subsystems/jobs/jobs-domain.module.js +519 -20
- package/dist/runtime/subsystems/jobs/jobs-domain.module.js.map +1 -1
- package/dist/runtime/subsystems/jobs/pool-config.loader.d.ts +9 -1
- package/dist/runtime/subsystems/jobs/pool-config.loader.js +4 -0
- package/dist/runtime/subsystems/jobs/pool-config.loader.js.map +1 -1
- package/dist/runtime/subsystems/observability/index.d.ts +4 -3
- package/dist/runtime/subsystems/observability/index.js +109 -2
- package/dist/runtime/subsystems/observability/index.js.map +1 -1
- package/dist/runtime/subsystems/observability/observability.module.js +109 -2
- package/dist/runtime/subsystems/observability/observability.module.js.map +1 -1
- package/dist/runtime/subsystems/observability/observability.protocol.d.ts +63 -2
- package/dist/runtime/subsystems/observability/observability.service.d.ts +21 -3
- package/dist/runtime/subsystems/observability/observability.service.js +109 -2
- package/dist/runtime/subsystems/observability/observability.service.js.map +1 -1
- package/dist/runtime/subsystems/observability/reporters/bridge-metrics.reporter.d.ts +1 -0
- package/dist/runtime/subsystems/observability/reporters/index.d.ts +1 -0
- package/dist/src/cli/index.js +43 -7
- package/dist/src/cli/index.js.map +1 -1
- package/package.json +1 -1
- package/runtime/subsystems/auth/index.ts +8 -0
- package/runtime/subsystems/auth/middleware/requester-context.ts +141 -0
- package/runtime/subsystems/auth/protocols/user-context.ts +17 -0
- package/runtime/subsystems/bridge/bridge.module.ts +5 -0
- package/runtime/subsystems/events/event-bus.drizzle-backend.ts +109 -3
- package/runtime/subsystems/events/event-bus.memory-backend.ts +103 -1
- package/runtime/subsystems/events/event-keyset-cursor.ts +59 -0
- package/runtime/subsystems/events/event-read.protocol.ts +97 -0
- package/runtime/subsystems/events/events.module.ts +18 -2
- package/runtime/subsystems/events/events.tokens.ts +16 -0
- package/runtime/subsystems/events/index.ts +7 -0
- package/runtime/subsystems/jobs/bullmq.config.ts +125 -0
- package/runtime/subsystems/jobs/index.ts +22 -0
- package/runtime/subsystems/jobs/job-orchestrator.bullmq-backend.ts +381 -0
- package/runtime/subsystems/jobs/job-run-keyset-cursor.ts +88 -0
- package/runtime/subsystems/jobs/job-run-service.drizzle-backend.ts +59 -1
- package/runtime/subsystems/jobs/job-run-service.memory-backend.ts +53 -0
- package/runtime/subsystems/jobs/job-run-service.protocol.ts +77 -0
- package/runtime/subsystems/jobs/job-worker.bullmq-backend.ts +311 -0
- package/runtime/subsystems/jobs/job-worker.module.ts +124 -10
- package/runtime/subsystems/jobs/jobs-domain.module.ts +40 -21
- package/runtime/subsystems/jobs/pool-config.loader.ts +11 -0
- package/runtime/subsystems/observability/index.ts +8 -0
- package/runtime/subsystems/observability/observability.protocol.ts +76 -0
- package/runtime/subsystems/observability/observability.service.ts +148 -1
- package/templates/entity/new/clean-lite-ps/prompt-extension.js +14 -12
- package/templates/relationship/new/prompt.js +8 -5
- package/templates/subsystem/jobs/worker.ejs.t +30 -7
- package/templates/subsystem/sync/sync-audit.schema.ejs.t +12 -16
|
@@ -4,8 +4,15 @@
|
|
|
4
4
|
* Import the module in AppModule, inject the bus via EVENT_BUS token.
|
|
5
5
|
*/
|
|
6
6
|
export type { DomainEvent, IEventBus, DrizzleTransaction } from './event-bus.protocol';
|
|
7
|
+
export type {
|
|
8
|
+
IEventReadPort,
|
|
9
|
+
ListEventsQuery,
|
|
10
|
+
EventSummary,
|
|
11
|
+
EventPage,
|
|
12
|
+
} from './event-read.protocol';
|
|
7
13
|
export {
|
|
8
14
|
EVENT_BUS,
|
|
15
|
+
EVENT_READ_PORT,
|
|
9
16
|
EVENTS_MODULE_OPTIONS,
|
|
10
17
|
EVENTS_MULTI_TENANT,
|
|
11
18
|
TYPED_EVENT_BUS,
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BullMQ backend configuration surface (BULLMQ-1, ADR-022 extension slot).
|
|
3
|
+
*
|
|
4
|
+
* The core `IJobOrchestrator` contract is backend-agnostic; everything in
|
|
5
|
+
* this file is BullMQ-specific and lives behind the
|
|
6
|
+
* `jobs.extensions.bullmq.*` config namespace (CLAUDE.md core/extension
|
|
7
|
+
* protocol). The Drizzle backend never reads any of it.
|
|
8
|
+
*/
|
|
9
|
+
import type { ConnectionOptions } from 'bullmq';
|
|
10
|
+
import { loadPoolConfig, type PoolConfig } from './pool-config.loader';
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Typed shape of `codegen.config.yaml: jobs.extensions.bullmq`. Snake_case
|
|
14
|
+
* because it mirrors the YAML the consumer authors.
|
|
15
|
+
*
|
|
16
|
+
* ```yaml
|
|
17
|
+
* jobs:
|
|
18
|
+
* backend: bullmq
|
|
19
|
+
* extensions:
|
|
20
|
+
* bullmq:
|
|
21
|
+
* redis_url: redis://localhost:6379 # or env REDIS_URL
|
|
22
|
+
* queue_prefix: myapp # optional namespace (ADR-022 OQ)
|
|
23
|
+
* bull_board:
|
|
24
|
+
* enabled: true
|
|
25
|
+
* mount_path: /api/admin/queues
|
|
26
|
+
* ```
|
|
27
|
+
*/
|
|
28
|
+
export interface BullMqExtensionsConfig {
|
|
29
|
+
/**
|
|
30
|
+
* Redis/Valkey connection URL. When omitted, the runtime resolves
|
|
31
|
+
* `process.env.REDIS_URL`, then falls back to `redis://localhost:6379`.
|
|
32
|
+
*/
|
|
33
|
+
redis_url?: string;
|
|
34
|
+
/**
|
|
35
|
+
* Optional queue-name prefix to avoid collisions when several codegen apps
|
|
36
|
+
* share one Redis (ADR-022 §"BullMQ queue naming collisions"). Applied to
|
|
37
|
+
* every pool queue alias.
|
|
38
|
+
*/
|
|
39
|
+
queue_prefix?: string;
|
|
40
|
+
/**
|
|
41
|
+
* Bull Board dashboard — opt-in extension (not core). Mounting is the
|
|
42
|
+
* consumer's responsibility (it needs the consumer's Express/Nest adapter +
|
|
43
|
+
* admin auth); we only carry the config. See README + spec §Extensions.
|
|
44
|
+
*/
|
|
45
|
+
bull_board?: {
|
|
46
|
+
enabled: boolean;
|
|
47
|
+
mount_path?: string;
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* The runtime form after `redis_url`/env resolution. This is what the
|
|
53
|
+
* orchestrator + worker actually consume.
|
|
54
|
+
*/
|
|
55
|
+
export interface BullMqResolvedConfig {
|
|
56
|
+
connection: ConnectionOptions;
|
|
57
|
+
queuePrefix?: string;
|
|
58
|
+
bullBoard?: { enabled: boolean; mountPath: string };
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** DI token for the resolved BullMQ `ConnectionOptions` (ioredis-compatible). */
|
|
62
|
+
export const BULLMQ_CONNECTION = Symbol('BULLMQ_CONNECTION');
|
|
63
|
+
|
|
64
|
+
/** DI token for the full resolved BullMQ config (prefix + bull board). */
|
|
65
|
+
export const BULLMQ_RESOLVED_CONFIG = Symbol('BULLMQ_RESOLVED_CONFIG');
|
|
66
|
+
|
|
67
|
+
const DEFAULT_REDIS_URL = 'redis://localhost:6379';
|
|
68
|
+
const DEFAULT_BULL_BOARD_MOUNT = '/admin/queues';
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Resolve the BullMQ runtime config from the extension block.
|
|
72
|
+
*
|
|
73
|
+
* Precedence for the connection URL:
|
|
74
|
+
* 1. explicit `extensions.bullmq.redis_url`
|
|
75
|
+
* 2. `process.env.REDIS_URL`
|
|
76
|
+
* 3. `redis://localhost:6379`
|
|
77
|
+
*
|
|
78
|
+
* Returns a `{ url }` connection shape — BullMQ/ioredis accept a URL string
|
|
79
|
+
* via the `{ url }` ConnectionOptions form.
|
|
80
|
+
*/
|
|
81
|
+
export function resolveBullMqConfig(
|
|
82
|
+
ext: BullMqExtensionsConfig | undefined,
|
|
83
|
+
): BullMqResolvedConfig {
|
|
84
|
+
const url =
|
|
85
|
+
ext?.redis_url ?? process.env.REDIS_URL ?? DEFAULT_REDIS_URL;
|
|
86
|
+
|
|
87
|
+
const resolved: BullMqResolvedConfig = {
|
|
88
|
+
connection: { url } as ConnectionOptions,
|
|
89
|
+
queuePrefix: ext?.queue_prefix,
|
|
90
|
+
};
|
|
91
|
+
if (ext?.bull_board?.enabled) {
|
|
92
|
+
resolved.bullBoard = {
|
|
93
|
+
enabled: true,
|
|
94
|
+
mountPath: ext.bull_board.mount_path ?? DEFAULT_BULL_BOARD_MOUNT,
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
return resolved;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Resolve the BullMQ queue name for a *logical pool name*. The orchestrator
|
|
102
|
+
* and worker MUST agree on this mapping or jobs are enqueued onto a queue
|
|
103
|
+
* nobody consumes. Both derive it identically:
|
|
104
|
+
*
|
|
105
|
+
* 1. Look up the pool's `queue` alias (e.g. `jobs-batch`) in the resolved
|
|
106
|
+
* pool config — the same alias `JobWorkerModule.onModuleInit` logs and
|
|
107
|
+
* that the BullMQ `Worker` binds to.
|
|
108
|
+
* 2. Fall back to the logical pool name when the pool is unknown (defensive;
|
|
109
|
+
* still a stable, colon-free identifier).
|
|
110
|
+
* 3. Apply the optional `queue_prefix` namespace for multi-app Redis
|
|
111
|
+
* sharing — `:` is fine in the *queue name* (it is only forbidden in the
|
|
112
|
+
* `jobId`, hence the sha1 there).
|
|
113
|
+
*
|
|
114
|
+
* `poolConfig` defaults to the cached `loadPoolConfig()` so callers that only
|
|
115
|
+
* hold the logical pool name (the orchestrator) don't need to thread the map.
|
|
116
|
+
*/
|
|
117
|
+
export function resolvePoolQueueName(
|
|
118
|
+
pool: string,
|
|
119
|
+
config: BullMqResolvedConfig | null | undefined,
|
|
120
|
+
poolConfig: PoolConfig = loadPoolConfig(),
|
|
121
|
+
): string {
|
|
122
|
+
const alias = poolConfig.get(pool)?.queue ?? pool;
|
|
123
|
+
const prefix = config?.queuePrefix;
|
|
124
|
+
return prefix ? `${prefix}:${alias}` : alias;
|
|
125
|
+
}
|
|
@@ -42,6 +42,9 @@ export type {
|
|
|
42
42
|
RescheduleForScopeOptions,
|
|
43
43
|
PoolStatusCount,
|
|
44
44
|
JobRunFailure,
|
|
45
|
+
ListJobRunsQuery,
|
|
46
|
+
JobRunSummary,
|
|
47
|
+
JobRunPage,
|
|
45
48
|
} from './job-run-service.protocol';
|
|
46
49
|
|
|
47
50
|
// ─── JOB-2: step-service protocol ──────────────────────────────────────────
|
|
@@ -76,6 +79,24 @@ export type {
|
|
|
76
79
|
export { DrizzleJobOrchestrator } from './job-orchestrator.drizzle-backend';
|
|
77
80
|
export { DrizzleJobRunService } from './job-run-service.drizzle-backend';
|
|
78
81
|
export { DrizzleJobStepService } from './job-step-service.drizzle-backend';
|
|
82
|
+
|
|
83
|
+
// ─── BULLMQ-1: BullMQ backend (additive; opt-in via jobs.backend: bullmq) ──
|
|
84
|
+
export {
|
|
85
|
+
BullMQJobOrchestrator,
|
|
86
|
+
sha1JobId,
|
|
87
|
+
} from './job-orchestrator.bullmq-backend';
|
|
88
|
+
export {
|
|
89
|
+
BullMQJobWorker,
|
|
90
|
+
type BullMQJobWorkerOptions,
|
|
91
|
+
} from './job-worker.bullmq-backend';
|
|
92
|
+
export {
|
|
93
|
+
BULLMQ_CONNECTION,
|
|
94
|
+
BULLMQ_RESOLVED_CONFIG,
|
|
95
|
+
resolveBullMqConfig,
|
|
96
|
+
resolvePoolQueueName,
|
|
97
|
+
type BullMqExtensionsConfig,
|
|
98
|
+
type BullMqResolvedConfig,
|
|
99
|
+
} from './bullmq.config';
|
|
79
100
|
export {
|
|
80
101
|
JobWorker,
|
|
81
102
|
JOB_WORKER_OPTIONS,
|
|
@@ -115,6 +136,7 @@ export {
|
|
|
115
136
|
export {
|
|
116
137
|
loadPoolConfig,
|
|
117
138
|
allNonReservedPoolNames,
|
|
139
|
+
allPoolNames,
|
|
118
140
|
FRAMEWORK_POOLS,
|
|
119
141
|
RESERVED_POOL_NAMES,
|
|
120
142
|
type PoolConfig,
|
|
@@ -0,0 +1,381 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BullMQJobOrchestrator — BullMQ-backed implementation of `IJobOrchestrator`
|
|
3
|
+
* (BULLMQ-1, ADR-022 §58 — the reserved "Phase 6+" backend, now built).
|
|
4
|
+
*
|
|
5
|
+
* Split-of-responsibility (spec §"Postgres + BullMQ coordination"):
|
|
6
|
+
* - Postgres `job_run` stays the **domain source of truth** — scoping,
|
|
7
|
+
* hierarchy (`parent_run_id`/`root_run_id`), dedupe/concurrency state,
|
|
8
|
+
* `listForScope`. All of that is the Drizzle backend's job and is reused
|
|
9
|
+
* verbatim by extending `DrizzleJobOrchestrator`.
|
|
10
|
+
* - BullMQ owns the **claim/dispatch** half. `start` adds a job to the
|
|
11
|
+
* pool's queue (or to a FlowProducer flow when parented); the BullMQ
|
|
12
|
+
* `Worker` (see `job-worker.bullmq-backend.ts`) consumes it and runs the
|
|
13
|
+
* handler through the existing `JobHandlerBase` path. `cancel` removes
|
|
14
|
+
* the queued job; `replay` re-adds it after the shared DB reset.
|
|
15
|
+
*
|
|
16
|
+
* This is **additive**: the Drizzle backend, the core protocol, and app code
|
|
17
|
+
* are untouched. Consumers flip `jobs.backend: bullmq` with no code change —
|
|
18
|
+
* the same `IJobOrchestrator` surface is satisfied.
|
|
19
|
+
*
|
|
20
|
+
* `jobId` (spec §Gotcha 1): BullMQ treats `:` as a Redis key separator and
|
|
21
|
+
* consumers use `vendor:externalId`-shaped idempotency keys, so we derive the
|
|
22
|
+
* `jobId` as `sha1(idempotencyKey)` — colon-safe and stable (same logical key
|
|
23
|
+
* → same id → BullMQ-native dedup). When no dedupe key is configured we fall
|
|
24
|
+
* back to the `job_run.id` (a fresh UUID), which is already colon-safe.
|
|
25
|
+
*/
|
|
26
|
+
import { createHash } from 'node:crypto';
|
|
27
|
+
import { Inject, Injectable, Logger, Optional } from '@nestjs/common';
|
|
28
|
+
import { eq } from 'drizzle-orm';
|
|
29
|
+
// `bullmq` is an OPTIONAL peer dependency. Only TYPE imports here — types are
|
|
30
|
+
// erased at compile time and never resolve `'bullmq'` at runtime, so a
|
|
31
|
+
// `drizzle`-only consumer who didn't install bullmq can still load this file
|
|
32
|
+
// (it is statically imported by `jobs-domain.module.ts`). The VALUE
|
|
33
|
+
// constructors (`Queue`, `FlowProducer`) are loaded lazily via `await
|
|
34
|
+
// import('bullmq')` in `loadBullMq()` — mirrors
|
|
35
|
+
// `event-bus.redis-backend.ts:createRedisClient`. See BULLMQ-1 §Lazy import.
|
|
36
|
+
import type { ConnectionOptions, FlowProducer, Queue } from 'bullmq';
|
|
37
|
+
import type { DrizzleClient } from '../../types/drizzle';
|
|
38
|
+
import type { DrizzleTransaction } from '../events/event-bus.protocol';
|
|
39
|
+
import { DRIZZLE } from '../../constants/tokens';
|
|
40
|
+
import { jobRuns, jobs, type JobDefinitionRow } from './job-orchestration.schema';
|
|
41
|
+
import { DrizzleJobOrchestrator } from './job-orchestrator.drizzle-backend';
|
|
42
|
+
import type {
|
|
43
|
+
CancelOptions,
|
|
44
|
+
JobRun,
|
|
45
|
+
StartOptions,
|
|
46
|
+
} from './job-orchestrator.protocol';
|
|
47
|
+
import { JOBS_MULTI_TENANT } from './jobs-domain.tokens';
|
|
48
|
+
import {
|
|
49
|
+
BULLMQ_CONNECTION,
|
|
50
|
+
resolvePoolQueueName,
|
|
51
|
+
type BullMqResolvedConfig,
|
|
52
|
+
BULLMQ_RESOLVED_CONFIG,
|
|
53
|
+
} from './bullmq.config';
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Derive a colon-safe, stable BullMQ `jobId` from a logical idempotency key.
|
|
57
|
+
*
|
|
58
|
+
* SHA-1 over the raw key. Collision analysis (spec §Gotcha 1, resolved during
|
|
59
|
+
* implementation): SHA-1's 160-bit space makes an accidental collision between
|
|
60
|
+
* two *distinct* logical keys astronomically unlikely at any realistic job
|
|
61
|
+
* volume (the birthday bound is ~2^80 keys before a 50% collision chance —
|
|
62
|
+
* orders of magnitude beyond any job throughput). SHA-1's cryptographic
|
|
63
|
+
* weakness is irrelevant here: there is no adversary forging idempotency keys,
|
|
64
|
+
* and even a forged collision only deduplicates two jobs that the caller chose
|
|
65
|
+
* to key identically. We therefore accept SHA-1 with no mitigation. The *same*
|
|
66
|
+
* logical key intentionally maps to the *same* jobId — that is the dedup
|
|
67
|
+
* mechanism, not a collision.
|
|
68
|
+
*/
|
|
69
|
+
export function sha1JobId(idempotencyKey: string): string {
|
|
70
|
+
return createHash('sha1').update(idempotencyKey).digest('hex');
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
// Constructor types for the lazily-loaded `bullmq` value exports. Typed via
|
|
74
|
+
// `typeof` the type-only imports so the cached ctors stay strongly typed
|
|
75
|
+
// without a runtime `import`.
|
|
76
|
+
type QueueCtor = typeof import('bullmq').Queue;
|
|
77
|
+
type FlowProducerCtor = typeof import('bullmq').FlowProducer;
|
|
78
|
+
|
|
79
|
+
@Injectable()
|
|
80
|
+
export class BullMQJobOrchestrator extends DrizzleJobOrchestrator {
|
|
81
|
+
// TODO(logging-subsystem): swap to ILogger once ADR-028 lands
|
|
82
|
+
private readonly bullLogger = new Logger(BullMQJobOrchestrator.name);
|
|
83
|
+
|
|
84
|
+
/** Lazily-opened `Queue` handles, one per pool. */
|
|
85
|
+
private readonly queues = new Map<string, Queue>();
|
|
86
|
+
/** Single FlowProducer for parent/child hierarchies. Lazily opened. */
|
|
87
|
+
private _flow: FlowProducer | null = null;
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* Cached `bullmq` value constructors, populated by `loadBullMq()` on first
|
|
91
|
+
* use (the `start`/`cancel`/`replay` entrypoints `await` it before touching
|
|
92
|
+
* a queue). Kept off the import graph so a `drizzle`-only consumer never
|
|
93
|
+
* resolves the optional `'bullmq'` package.
|
|
94
|
+
*/
|
|
95
|
+
private QueueCtor: QueueCtor | null = null;
|
|
96
|
+
private FlowProducerCtor: FlowProducerCtor | null = null;
|
|
97
|
+
private bullMqLoad: Promise<void> | null = null;
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Own reference to the Drizzle client. `DrizzleJobOrchestrator.db` is
|
|
101
|
+
* `private` (can't be redeclared even privately in a subclass), and the
|
|
102
|
+
* spec forbids touching that file — so the subclass keeps its own handle
|
|
103
|
+
* under a distinct name (same instance, passed through to `super`) for the
|
|
104
|
+
* cancel-cascade snapshot + definition/run loads below.
|
|
105
|
+
*/
|
|
106
|
+
private readonly bullDb: DrizzleClient;
|
|
107
|
+
|
|
108
|
+
constructor(
|
|
109
|
+
@Inject(DRIZZLE) db: DrizzleClient,
|
|
110
|
+
@Inject(JOBS_MULTI_TENANT) multiTenant: boolean,
|
|
111
|
+
@Inject(BULLMQ_CONNECTION) private readonly connection: ConnectionOptions,
|
|
112
|
+
@Optional()
|
|
113
|
+
@Inject(BULLMQ_RESOLVED_CONFIG)
|
|
114
|
+
private readonly bullConfig: BullMqResolvedConfig | null = null,
|
|
115
|
+
) {
|
|
116
|
+
super(db, multiTenant);
|
|
117
|
+
this.bullDb = db;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Lazily load the optional `bullmq` package and cache its value
|
|
122
|
+
* constructors. Idempotent (single in-flight promise). Throws a friendly,
|
|
123
|
+
* actionable error when the consumer selected `backend: 'bullmq'` but did
|
|
124
|
+
* not install the package — mirrors `createRedisClient` in the redis event
|
|
125
|
+
* backend. Must be `await`ed before any `queueFor`/`flow` access.
|
|
126
|
+
*/
|
|
127
|
+
private async loadBullMq(): Promise<void> {
|
|
128
|
+
if (this.QueueCtor && this.FlowProducerCtor) return;
|
|
129
|
+
if (!this.bullMqLoad) {
|
|
130
|
+
this.bullMqLoad = (async () => {
|
|
131
|
+
try {
|
|
132
|
+
const mod = await import('bullmq');
|
|
133
|
+
this.QueueCtor = mod.Queue;
|
|
134
|
+
this.FlowProducerCtor = mod.FlowProducer;
|
|
135
|
+
} catch {
|
|
136
|
+
throw new Error(
|
|
137
|
+
'BullMQ backend requires the "bullmq" package. Install it with: npm install bullmq',
|
|
138
|
+
);
|
|
139
|
+
}
|
|
140
|
+
})();
|
|
141
|
+
}
|
|
142
|
+
await this.bullMqLoad;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* Open (or reuse) the `Queue` for a pool. Synchronous — callers `await
|
|
147
|
+
* loadBullMq()` first so `QueueCtor` is populated.
|
|
148
|
+
*/
|
|
149
|
+
private queueFor(pool: string): Queue {
|
|
150
|
+
if (!this.QueueCtor) {
|
|
151
|
+
throw new Error('BullMQJobOrchestrator: queueFor called before loadBullMq()');
|
|
152
|
+
}
|
|
153
|
+
const name = resolvePoolQueueName(pool, this.bullConfig);
|
|
154
|
+
let q = this.queues.get(name);
|
|
155
|
+
if (!q) {
|
|
156
|
+
q = new this.QueueCtor(name, { connection: this.connection });
|
|
157
|
+
this.queues.set(name, q);
|
|
158
|
+
}
|
|
159
|
+
return q;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
private flow(): FlowProducer {
|
|
163
|
+
if (!this.FlowProducerCtor) {
|
|
164
|
+
throw new Error('BullMQJobOrchestrator: flow called before loadBullMq()');
|
|
165
|
+
}
|
|
166
|
+
if (!this._flow) {
|
|
167
|
+
this._flow = new this.FlowProducerCtor({ connection: this.connection });
|
|
168
|
+
}
|
|
169
|
+
return this._flow;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
// ==========================================================================
|
|
173
|
+
// start — Postgres insert (super) + BullMQ dispatch
|
|
174
|
+
// ==========================================================================
|
|
175
|
+
|
|
176
|
+
override async start(
|
|
177
|
+
type: string,
|
|
178
|
+
input: unknown,
|
|
179
|
+
opts: StartOptions = {},
|
|
180
|
+
tx?: DrizzleTransaction,
|
|
181
|
+
): Promise<JobRun> {
|
|
182
|
+
// (1) Postgres remains source of truth — the Drizzle backend handles the
|
|
183
|
+
// job-definition lookup, dedupe short-circuit, concurrency collision,
|
|
184
|
+
// parent/root resolution, and the `job_run` INSERT. If dedupe
|
|
185
|
+
// short-circuited it returns the incumbent row whose dispatch already
|
|
186
|
+
// happened on the original start; we must not enqueue again.
|
|
187
|
+
const run = await super.start(type, input, opts, tx);
|
|
188
|
+
|
|
189
|
+
// Dedupe returned an existing run (its createdAt predates this call) —
|
|
190
|
+
// BullMQ-native dedup already covered the dispatch. Skip re-enqueue.
|
|
191
|
+
// We detect this by checking the run was freshly created in THIS call:
|
|
192
|
+
// a brand-new run has status 'pending' and zero attempts AND its id is
|
|
193
|
+
// not yet known to BullMQ. The cheapest reliable signal is the dedupe
|
|
194
|
+
// path's contract: super.start returns the incumbent unchanged. Since we
|
|
195
|
+
// cannot distinguish purely from the row, we rely on `jobId` idempotency
|
|
196
|
+
// — re-adding with the same jobId is a no-op in BullMQ, so the enqueue is
|
|
197
|
+
// safe to attempt unconditionally.
|
|
198
|
+
|
|
199
|
+
await this.dispatch(run, type);
|
|
200
|
+
return run;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/**
|
|
204
|
+
* Map a `job_run` row onto a BullMQ job via `queue.add`. When the run has a
|
|
205
|
+
* `parentRunId` we attach it to the parent's existing BullMQ job through the
|
|
206
|
+
* `parent: { id, queue }` opt — BullMQ then tracks the parent/child link in
|
|
207
|
+
* its own graph. (The FlowProducer is reserved for whole-tree atomic
|
|
208
|
+
* submits, exposed as an opt-in extension via `flowProducer()`; runtime
|
|
209
|
+
* `ctx.spawnChild` is incremental, so `queue.add` with a parent ref is the
|
|
210
|
+
* correct primitive here.)
|
|
211
|
+
*
|
|
212
|
+
* The `jobId` is colon-safe + stable: `sha1(dedupeKey)` when a dedupe key is
|
|
213
|
+
* present (so the same logical key dedups), else the `job_run.id` UUID
|
|
214
|
+
* (already colon-free).
|
|
215
|
+
*
|
|
216
|
+
* The domain `parentClosePolicy` cascade is still enforced in Postgres by
|
|
217
|
+
* the shared `cancel` path — BullMQ's parent link is dispatch bookkeeping,
|
|
218
|
+
* not the authority.
|
|
219
|
+
*/
|
|
220
|
+
private async dispatch(run: JobRun, type: string): Promise<void> {
|
|
221
|
+
await this.loadBullMq();
|
|
222
|
+
const def = await this.loadDefinition(type);
|
|
223
|
+
const jobId = run.dedupeKey ? sha1JobId(run.dedupeKey) : run.id;
|
|
224
|
+
|
|
225
|
+
const jobOpts: Record<string, unknown> = {
|
|
226
|
+
jobId,
|
|
227
|
+
...this.retryOpts(def),
|
|
228
|
+
...this.dedupeOpts(run, def),
|
|
229
|
+
};
|
|
230
|
+
|
|
231
|
+
if (run.parentRunId) {
|
|
232
|
+
const parentRow = await this.loadRun(run.parentRunId);
|
|
233
|
+
if (parentRow) {
|
|
234
|
+
const parentJobId = parentRow.dedupeKey
|
|
235
|
+
? sha1JobId(parentRow.dedupeKey)
|
|
236
|
+
: parentRow.id;
|
|
237
|
+
jobOpts.parent = {
|
|
238
|
+
id: parentJobId,
|
|
239
|
+
queue: resolvePoolQueueName(parentRow.pool, this.bullConfig),
|
|
240
|
+
};
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
// The processor reads the authoritative input from `job_run`; the payload
|
|
245
|
+
// carries the runId so it can load the row, plus type/input for logging.
|
|
246
|
+
const payload = { runId: run.id, type, input: run.input };
|
|
247
|
+
await this.queueFor(run.pool).add(type, payload, jobOpts);
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
/**
|
|
251
|
+
* Opt-in extension (spec §Extensions): expose the FlowProducer for
|
|
252
|
+
* consumers that want to submit a whole parent/child DAG atomically up
|
|
253
|
+
* front, rather than incrementally via `ctx.spawnChild`. Backend-specific —
|
|
254
|
+
* code using it is not portable to the Drizzle backend. Async because it
|
|
255
|
+
* lazily loads the optional `bullmq` package on first use.
|
|
256
|
+
*/
|
|
257
|
+
async flowProducer(): Promise<FlowProducer> {
|
|
258
|
+
await this.loadBullMq();
|
|
259
|
+
return this.flow();
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
private retryOpts(def: JobDefinitionRow): {
|
|
263
|
+
attempts?: number;
|
|
264
|
+
backoff?: { type: 'fixed' | 'exponential'; delay: number };
|
|
265
|
+
} {
|
|
266
|
+
const policy = def.retryPolicy;
|
|
267
|
+
if (!policy) return {};
|
|
268
|
+
return {
|
|
269
|
+
attempts: policy.attempts,
|
|
270
|
+
backoff: {
|
|
271
|
+
type: policy.backoff === 'exponential' ? 'exponential' : 'fixed',
|
|
272
|
+
delay: policy.baseMs,
|
|
273
|
+
},
|
|
274
|
+
};
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
private dedupeOpts(
|
|
278
|
+
run: JobRun,
|
|
279
|
+
def: JobDefinitionRow,
|
|
280
|
+
): { deduplication?: { id: string; ttl?: number } } {
|
|
281
|
+
if (!run.dedupeKey || !def.dedupeWindowMs) return {};
|
|
282
|
+
return {
|
|
283
|
+
deduplication: {
|
|
284
|
+
id: sha1JobId(run.dedupeKey),
|
|
285
|
+
ttl: def.dedupeWindowMs,
|
|
286
|
+
},
|
|
287
|
+
};
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
// ==========================================================================
|
|
291
|
+
// cancel — Postgres cascade (super) + remove from queue
|
|
292
|
+
// ==========================================================================
|
|
293
|
+
|
|
294
|
+
override async cancel(runId: string, opts: CancelOptions = {}): Promise<void> {
|
|
295
|
+
// Snapshot the subtree BEFORE the DB cascade flips rows to canceled, so we
|
|
296
|
+
// can remove every affected BullMQ job. We read the target's rootRunId and
|
|
297
|
+
// the non-terminal descendants the same way the Drizzle cascade does.
|
|
298
|
+
const target = await this.loadRun(runId);
|
|
299
|
+
|
|
300
|
+
await super.cancel(runId, opts);
|
|
301
|
+
|
|
302
|
+
if (!target) return;
|
|
303
|
+
await this.loadBullMq();
|
|
304
|
+
// Remove the target's own queued job.
|
|
305
|
+
await this.removeFromQueue(target);
|
|
306
|
+
|
|
307
|
+
if (opts.cascade === false) return;
|
|
308
|
+
|
|
309
|
+
// Remove descendants' queued jobs (the DB rows were just canceled by
|
|
310
|
+
// super.cancel; we mirror that into BullMQ so workers don't pick them up).
|
|
311
|
+
const descendants = await this.bullDb
|
|
312
|
+
.select()
|
|
313
|
+
.from(jobRuns)
|
|
314
|
+
.where(eq(jobRuns.rootRunId, target.rootRunId));
|
|
315
|
+
for (const child of descendants) {
|
|
316
|
+
if (child.id === runId) continue;
|
|
317
|
+
await this.removeFromQueue(child as JobRun);
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
private async removeFromQueue(run: JobRun): Promise<void> {
|
|
322
|
+
const jobId = run.dedupeKey ? sha1JobId(run.dedupeKey) : run.id;
|
|
323
|
+
try {
|
|
324
|
+
const job = await this.queueFor(run.pool).getJob(jobId);
|
|
325
|
+
if (job) await job.remove();
|
|
326
|
+
} catch (err) {
|
|
327
|
+
// A job already moved to active/completed cannot always be removed;
|
|
328
|
+
// the Postgres cancel is authoritative either way.
|
|
329
|
+
this.bullLogger.warn(
|
|
330
|
+
`cancel: could not remove BullMQ job ${jobId} (pool=${run.pool}): ${(err as Error).message}`,
|
|
331
|
+
);
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
// ==========================================================================
|
|
336
|
+
// replay — Postgres reset (super) + re-enqueue
|
|
337
|
+
// ==========================================================================
|
|
338
|
+
|
|
339
|
+
override async replay(runId: string): Promise<JobRun> {
|
|
340
|
+
const run = await super.replay(runId);
|
|
341
|
+
await this.dispatch(run, run.jobType);
|
|
342
|
+
return run;
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
// ==========================================================================
|
|
346
|
+
// Internals
|
|
347
|
+
// ==========================================================================
|
|
348
|
+
|
|
349
|
+
private async loadDefinition(type: string): Promise<JobDefinitionRow> {
|
|
350
|
+
const [def] = await this.bullDb
|
|
351
|
+
.select()
|
|
352
|
+
.from(jobs)
|
|
353
|
+
.where(eq(jobs.type, type))
|
|
354
|
+
.limit(1);
|
|
355
|
+
if (!def) {
|
|
356
|
+
throw new Error(`BullMQJobOrchestrator: no job definition for '${type}'`);
|
|
357
|
+
}
|
|
358
|
+
return def as JobDefinitionRow;
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
private async loadRun(id: string): Promise<JobRun | null> {
|
|
362
|
+
const [row] = await this.bullDb
|
|
363
|
+
.select()
|
|
364
|
+
.from(jobRuns)
|
|
365
|
+
.where(eq(jobRuns.id, id))
|
|
366
|
+
.limit(1);
|
|
367
|
+
return (row as JobRun) ?? null;
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
/** Close all open queue + flow connections. Called on module destroy. */
|
|
371
|
+
async closeConnections(): Promise<void> {
|
|
372
|
+
for (const q of this.queues.values()) {
|
|
373
|
+
await q.close().catch(() => undefined);
|
|
374
|
+
}
|
|
375
|
+
this.queues.clear();
|
|
376
|
+
if (this._flow) {
|
|
377
|
+
await this._flow.close().catch(() => undefined);
|
|
378
|
+
this._flow = null;
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Keyset (seek) cursor codec for `IJobRunService.listJobRuns` (OBS-LIST-1).
|
|
3
|
+
*
|
|
4
|
+
* The list is ordered `created_at DESC, id DESC`. The cursor encodes the
|
|
5
|
+
* `(createdAt, id)` of the last row on the previous page so the next page
|
|
6
|
+
* can seek with `WHERE (created_at, id) < (cursorCreatedAt, cursorId)`
|
|
7
|
+
* rather than an `OFFSET`. Keyset pagination stays O(log n) on deep pages
|
|
8
|
+
* and is stable as new rows arrive at the head.
|
|
9
|
+
*
|
|
10
|
+
* The cursor is opaque to consumers: a base64url-encoded JSON tuple. Shape
|
|
11
|
+
* is an implementation detail — never parse it outside this module.
|
|
12
|
+
*
|
|
13
|
+
* Also hosts `toJobRunSummary`, the single `JobRunRow → JobRunSummary`
|
|
14
|
+
* projection shared by both backends so the narrow shape stays in sync.
|
|
15
|
+
*/
|
|
16
|
+
import type { JobRunRow } from './job-orchestration.schema';
|
|
17
|
+
import type { JobRunSummary } from './job-run-service.protocol';
|
|
18
|
+
|
|
19
|
+
export interface JobRunKeyset {
|
|
20
|
+
/** `created_at` of the last row on the previous page. */
|
|
21
|
+
createdAt: Date;
|
|
22
|
+
/** `id` (UUID) tie-break of the last row on the previous page. */
|
|
23
|
+
id: string;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** Default page size when `limit` is omitted. */
|
|
27
|
+
export const DEFAULT_LIST_LIMIT = 50;
|
|
28
|
+
/** Hard upper bound on page size to keep a single read bounded. */
|
|
29
|
+
export const MAX_LIST_LIMIT = 200;
|
|
30
|
+
|
|
31
|
+
/** Clamp a caller-supplied `limit` into `[1, MAX_LIST_LIMIT]`. */
|
|
32
|
+
export function clampLimit(limit: number | undefined): number {
|
|
33
|
+
if (typeof limit !== 'number' || !Number.isFinite(limit)) {
|
|
34
|
+
return DEFAULT_LIST_LIMIT;
|
|
35
|
+
}
|
|
36
|
+
const floored = Math.floor(limit);
|
|
37
|
+
if (floored < 1) return 1;
|
|
38
|
+
if (floored > MAX_LIST_LIMIT) return MAX_LIST_LIMIT;
|
|
39
|
+
return floored;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export function encodeKeysetCursor(keyset: JobRunKeyset): string {
|
|
43
|
+
const tuple = [keyset.createdAt.toISOString(), keyset.id];
|
|
44
|
+
return Buffer.from(JSON.stringify(tuple), 'utf8').toString('base64url');
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Decode an opaque cursor back into its `(createdAt, id)` keyset. Returns
|
|
49
|
+
* `null` for a malformed cursor so callers can treat garbage input as
|
|
50
|
+
* "start from the beginning" rather than throwing on user-supplied data.
|
|
51
|
+
*/
|
|
52
|
+
export function decodeKeysetCursor(cursor: string): JobRunKeyset | null {
|
|
53
|
+
try {
|
|
54
|
+
const json = Buffer.from(cursor, 'base64url').toString('utf8');
|
|
55
|
+
const parsed = JSON.parse(json) as unknown;
|
|
56
|
+
if (!Array.isArray(parsed) || parsed.length !== 2) return null;
|
|
57
|
+
const [iso, id] = parsed;
|
|
58
|
+
if (typeof iso !== 'string' || typeof id !== 'string') return null;
|
|
59
|
+
const createdAt = new Date(iso);
|
|
60
|
+
if (Number.isNaN(createdAt.getTime())) return null;
|
|
61
|
+
return { createdAt, id };
|
|
62
|
+
} catch {
|
|
63
|
+
return null;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Project a raw `job_run` row into the narrow `JobRunSummary` shape exposed
|
|
69
|
+
* by `listJobRuns`. `errorMessage` is pulled from the jsonb `error.message`.
|
|
70
|
+
*/
|
|
71
|
+
export function toJobRunSummary(r: JobRunRow): JobRunSummary {
|
|
72
|
+
return {
|
|
73
|
+
runId: r.id,
|
|
74
|
+
rootRunId: r.rootRunId,
|
|
75
|
+
jobType: r.jobType,
|
|
76
|
+
pool: r.pool,
|
|
77
|
+
status: r.status,
|
|
78
|
+
scopeEntityType: r.scopeEntityType,
|
|
79
|
+
scopeEntityId: r.scopeEntityId,
|
|
80
|
+
tenantId: r.tenantId,
|
|
81
|
+
attempts: r.attempts,
|
|
82
|
+
errorMessage: r.error?.message ?? null,
|
|
83
|
+
runAt: r.runAt,
|
|
84
|
+
startedAt: r.startedAt,
|
|
85
|
+
finishedAt: r.finishedAt,
|
|
86
|
+
createdAt: r.createdAt,
|
|
87
|
+
};
|
|
88
|
+
}
|