@pattern-stack/codegen 0.9.2 → 0.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +73 -0
- package/README.md +5 -0
- package/consumer-skills/bridge/SKILL.md +265 -0
- package/consumer-skills/codegen/SKILL.md +115 -0
- package/consumer-skills/entities/SKILL.md +111 -0
- package/consumer-skills/entities/families-and-queries.md +82 -0
- package/consumer-skills/entities/yaml-reference.md +118 -0
- package/consumer-skills/events/SKILL.md +71 -0
- package/consumer-skills/events/authoring-events.md +164 -0
- package/consumer-skills/events/typed-bus-and-outbox.md +163 -0
- package/consumer-skills/jobs/SKILL.md +66 -0
- package/consumer-skills/jobs/handler-authoring.md +236 -0
- package/consumer-skills/jobs/pools-and-ordering.md +161 -0
- package/consumer-skills/subsystems/SKILL.md +161 -0
- package/consumer-skills/subsystems/wiring-and-order.md +120 -0
- package/consumer-skills/sync/SKILL.md +134 -0
- package/consumer-skills/sync/audit-and-detection.md +302 -0
- package/consumer-skills/sync/change-sources-and-sinks.md +442 -0
- package/dist/runtime/subsystems/bridge/bridge.module.d.ts +0 -1
- package/dist/runtime/subsystems/bridge/bridge.module.js +294 -710
- package/dist/runtime/subsystems/bridge/bridge.module.js.map +1 -1
- package/dist/runtime/subsystems/bridge/index.d.ts +0 -1
- package/dist/runtime/subsystems/bridge/index.js +248 -664
- package/dist/runtime/subsystems/bridge/index.js.map +1 -1
- package/dist/runtime/subsystems/events/event-bus.drizzle-backend.js +18 -10
- package/dist/runtime/subsystems/events/event-bus.drizzle-backend.js.map +1 -1
- package/dist/runtime/subsystems/events/events.module.js +43 -244
- package/dist/runtime/subsystems/events/events.module.js.map +1 -1
- package/dist/runtime/subsystems/events/index.d.ts +0 -1
- package/dist/runtime/subsystems/events/index.js +39 -241
- package/dist/runtime/subsystems/events/index.js.map +1 -1
- package/dist/runtime/subsystems/index.js +174 -791
- package/dist/runtime/subsystems/index.js.map +1 -1
- package/dist/runtime/subsystems/jobs/bullmq.config.d.ts +22 -3
- package/dist/runtime/subsystems/jobs/bullmq.config.js.map +1 -1
- package/dist/runtime/subsystems/jobs/index.d.ts +1 -4
- package/dist/runtime/subsystems/jobs/index.js +87 -506
- package/dist/runtime/subsystems/jobs/index.js.map +1 -1
- package/dist/runtime/subsystems/jobs/job-orchestrator.bullmq-backend.js.map +1 -1
- package/dist/runtime/subsystems/jobs/job-worker.bullmq-backend.js +3 -0
- package/dist/runtime/subsystems/jobs/job-worker.bullmq-backend.js.map +1 -1
- package/dist/runtime/subsystems/jobs/job-worker.module.d.ts +11 -4
- package/dist/runtime/subsystems/jobs/job-worker.module.js +248 -664
- package/dist/runtime/subsystems/jobs/job-worker.module.js.map +1 -1
- package/dist/runtime/subsystems/jobs/jobs-domain.module.d.ts +0 -1
- package/dist/runtime/subsystems/jobs/jobs-domain.module.js +89 -391
- package/dist/runtime/subsystems/jobs/jobs-domain.module.js.map +1 -1
- package/dist/src/cli/index.js +1065 -440
- package/dist/src/cli/index.js.map +1 -1
- package/dist/src/index.js +26 -4
- package/dist/src/index.js.map +1 -1
- package/package.json +2 -1
- package/runtime/subsystems/events/event-bus.drizzle-backend.ts +32 -10
- package/runtime/subsystems/events/events.module.ts +38 -6
- package/runtime/subsystems/events/index.ts +7 -1
- package/runtime/subsystems/jobs/bullmq.config.ts +23 -3
- package/runtime/subsystems/jobs/index.ts +13 -8
- package/runtime/subsystems/jobs/job-worker.bullmq-backend.ts +5 -2
- package/runtime/subsystems/jobs/job-worker.module.ts +27 -7
- package/runtime/subsystems/jobs/jobs-domain.module.ts +27 -2
- package/templates/subsystem/events/domain-events.schema.ejs.t +43 -2
|
@@ -933,18 +933,18 @@ EventFlowService = __decorateClass([
|
|
|
933
933
|
|
|
934
934
|
// runtime/subsystems/bridge/bridge.module.ts
|
|
935
935
|
import {
|
|
936
|
-
Inject as
|
|
936
|
+
Inject as Inject12,
|
|
937
937
|
Module as Module3,
|
|
938
|
-
Optional as
|
|
938
|
+
Optional as Optional7
|
|
939
939
|
} from "@nestjs/common";
|
|
940
940
|
|
|
941
941
|
// runtime/subsystems/jobs/job-worker.module.ts
|
|
942
942
|
import {
|
|
943
|
-
Inject as
|
|
944
|
-
Injectable as
|
|
945
|
-
Logger as
|
|
943
|
+
Inject as Inject11,
|
|
944
|
+
Injectable as Injectable12,
|
|
945
|
+
Logger as Logger6,
|
|
946
946
|
Module as Module2,
|
|
947
|
-
Optional as
|
|
947
|
+
Optional as Optional6
|
|
948
948
|
} from "@nestjs/common";
|
|
949
949
|
|
|
950
950
|
// runtime/subsystems/jobs/jobs-domain.module.ts
|
|
@@ -1629,394 +1629,9 @@ DrizzleJobStepService = __decorateClass([
|
|
|
1629
1629
|
__decorateParam(0, Inject7(DRIZZLE))
|
|
1630
1630
|
], DrizzleJobStepService);
|
|
1631
1631
|
|
|
1632
|
-
// runtime/subsystems/jobs/job-orchestrator.bullmq-backend.ts
|
|
1633
|
-
import { createHash } from "crypto";
|
|
1634
|
-
import { Inject as Inject8, Injectable as Injectable8, Logger as Logger4, Optional as Optional5 } from "@nestjs/common";
|
|
1635
|
-
import { eq as eq5 } from "drizzle-orm";
|
|
1636
|
-
|
|
1637
|
-
// runtime/subsystems/jobs/pool-config.loader.ts
|
|
1638
|
-
import { existsSync, readFileSync } from "fs";
|
|
1639
|
-
import { resolve } from "path";
|
|
1640
|
-
import { parse as parseYaml } from "yaml";
|
|
1641
|
-
var FRAMEWORK_POOLS = Object.freeze({
|
|
1642
|
-
events_inbound: Object.freeze({
|
|
1643
|
-
queue: "jobs-events-inbound",
|
|
1644
|
-
concurrency: 20,
|
|
1645
|
-
reserved: true,
|
|
1646
|
-
description: "Inbound events drain (events subsystem outbox)."
|
|
1647
|
-
}),
|
|
1648
|
-
events_change: Object.freeze({
|
|
1649
|
-
queue: "jobs-events-change",
|
|
1650
|
-
concurrency: 30,
|
|
1651
|
-
reserved: true,
|
|
1652
|
-
description: "Change events drain (events subsystem outbox)."
|
|
1653
|
-
}),
|
|
1654
|
-
events_outbound: Object.freeze({
|
|
1655
|
-
queue: "jobs-events-outbound",
|
|
1656
|
-
concurrency: 10,
|
|
1657
|
-
reserved: true,
|
|
1658
|
-
description: "Outbound events drain (events subsystem outbox)."
|
|
1659
|
-
}),
|
|
1660
|
-
interactive: Object.freeze({
|
|
1661
|
-
queue: "jobs-interactive",
|
|
1662
|
-
concurrency: 20,
|
|
1663
|
-
reserved: false,
|
|
1664
|
-
description: "User-facing latency-sensitive jobs."
|
|
1665
|
-
}),
|
|
1666
|
-
batch: Object.freeze({
|
|
1667
|
-
queue: "jobs-batch",
|
|
1668
|
-
concurrency: 5,
|
|
1669
|
-
reserved: false,
|
|
1670
|
-
description: "Default pool for background jobs."
|
|
1671
|
-
})
|
|
1672
|
-
});
|
|
1673
|
-
var RESERVED_POOL_NAMES = new Set(
|
|
1674
|
-
Object.entries(FRAMEWORK_POOLS).filter(([, def]) => def.reserved).map(([name]) => name)
|
|
1675
|
-
);
|
|
1676
|
-
var cache = /* @__PURE__ */ new Map();
|
|
1677
|
-
function loadPoolConfig(configPath) {
|
|
1678
|
-
const resolved = resolve(configPath ?? `${process.cwd()}/codegen.config.yaml`);
|
|
1679
|
-
const cached = cache.get(resolved);
|
|
1680
|
-
if (cached) return cached;
|
|
1681
|
-
const merged = /* @__PURE__ */ new Map();
|
|
1682
|
-
for (const [name, def] of Object.entries(FRAMEWORK_POOLS)) {
|
|
1683
|
-
merged.set(name, { ...def });
|
|
1684
|
-
}
|
|
1685
|
-
if (!existsSync(resolved)) {
|
|
1686
|
-
cache.set(resolved, merged);
|
|
1687
|
-
return merged;
|
|
1688
|
-
}
|
|
1689
|
-
let raw;
|
|
1690
|
-
try {
|
|
1691
|
-
raw = parseYaml(readFileSync(resolved, "utf8"));
|
|
1692
|
-
} catch (err) {
|
|
1693
|
-
throw new Error(
|
|
1694
|
-
`pool-config.loader: failed to parse YAML at ${resolved}: ${err.message}`
|
|
1695
|
-
);
|
|
1696
|
-
}
|
|
1697
|
-
const userPools = extractUserPools(raw);
|
|
1698
|
-
for (const [name, userDef] of Object.entries(userPools)) {
|
|
1699
|
-
const existing = merged.get(name);
|
|
1700
|
-
if (existing) {
|
|
1701
|
-
const next = {
|
|
1702
|
-
queue: existing.queue,
|
|
1703
|
-
concurrency: typeof userDef.concurrency === "number" ? userDef.concurrency : existing.concurrency,
|
|
1704
|
-
reserved: existing.reserved,
|
|
1705
|
-
description: userDef.description ?? existing.description
|
|
1706
|
-
};
|
|
1707
|
-
merged.set(name, next);
|
|
1708
|
-
continue;
|
|
1709
|
-
}
|
|
1710
|
-
if (typeof userDef.queue !== "string" || userDef.queue.length === 0) {
|
|
1711
|
-
throw new Error(
|
|
1712
|
-
`pool-config.loader: pool '${name}' must declare a non-empty 'queue'.`
|
|
1713
|
-
);
|
|
1714
|
-
}
|
|
1715
|
-
if (typeof userDef.concurrency !== "number" || userDef.concurrency <= 0) {
|
|
1716
|
-
throw new Error(
|
|
1717
|
-
`pool-config.loader: pool '${name}' must declare a positive 'concurrency'.`
|
|
1718
|
-
);
|
|
1719
|
-
}
|
|
1720
|
-
if (userDef.reserved === true) {
|
|
1721
|
-
throw new Error(
|
|
1722
|
-
`pool-config.loader: user-defined pool '${name}' cannot set 'reserved: true' \u2014 reserved is framework-only.`
|
|
1723
|
-
);
|
|
1724
|
-
}
|
|
1725
|
-
merged.set(name, {
|
|
1726
|
-
queue: userDef.queue,
|
|
1727
|
-
concurrency: userDef.concurrency,
|
|
1728
|
-
reserved: false,
|
|
1729
|
-
description: userDef.description
|
|
1730
|
-
});
|
|
1731
|
-
}
|
|
1732
|
-
cache.set(resolved, merged);
|
|
1733
|
-
return merged;
|
|
1734
|
-
}
|
|
1735
|
-
function allNonReservedPoolNames(config) {
|
|
1736
|
-
const out = [];
|
|
1737
|
-
for (const [name, def] of config) {
|
|
1738
|
-
if (!def.reserved) out.push(name);
|
|
1739
|
-
}
|
|
1740
|
-
return out;
|
|
1741
|
-
}
|
|
1742
|
-
function allPoolNames(config) {
|
|
1743
|
-
return [...config.keys()];
|
|
1744
|
-
}
|
|
1745
|
-
function extractUserPools(raw) {
|
|
1746
|
-
if (!raw || typeof raw !== "object") return {};
|
|
1747
|
-
const jobs2 = raw.jobs;
|
|
1748
|
-
if (!jobs2 || typeof jobs2 !== "object") return {};
|
|
1749
|
-
const pools = jobs2.pools;
|
|
1750
|
-
if (!pools || typeof pools !== "object") return {};
|
|
1751
|
-
const out = {};
|
|
1752
|
-
for (const [name, def] of Object.entries(pools)) {
|
|
1753
|
-
if (!def || typeof def !== "object") continue;
|
|
1754
|
-
out[name] = def;
|
|
1755
|
-
}
|
|
1756
|
-
return out;
|
|
1757
|
-
}
|
|
1758
|
-
|
|
1759
|
-
// runtime/subsystems/jobs/bullmq.config.ts
|
|
1760
|
-
var BULLMQ_CONNECTION = /* @__PURE__ */ Symbol("BULLMQ_CONNECTION");
|
|
1761
|
-
var BULLMQ_RESOLVED_CONFIG = /* @__PURE__ */ Symbol("BULLMQ_RESOLVED_CONFIG");
|
|
1762
|
-
var DEFAULT_REDIS_URL = "redis://localhost:6379";
|
|
1763
|
-
var DEFAULT_BULL_BOARD_MOUNT = "/admin/queues";
|
|
1764
|
-
function resolveBullMqConfig(ext) {
|
|
1765
|
-
const url = ext?.redis_url ?? process.env.REDIS_URL ?? DEFAULT_REDIS_URL;
|
|
1766
|
-
const resolved = {
|
|
1767
|
-
connection: { url },
|
|
1768
|
-
queuePrefix: ext?.queue_prefix
|
|
1769
|
-
};
|
|
1770
|
-
if (ext?.bull_board?.enabled) {
|
|
1771
|
-
resolved.bullBoard = {
|
|
1772
|
-
enabled: true,
|
|
1773
|
-
mountPath: ext.bull_board.mount_path ?? DEFAULT_BULL_BOARD_MOUNT
|
|
1774
|
-
};
|
|
1775
|
-
}
|
|
1776
|
-
return resolved;
|
|
1777
|
-
}
|
|
1778
|
-
function resolvePoolQueueName(pool, config, poolConfig = loadPoolConfig()) {
|
|
1779
|
-
const alias = poolConfig.get(pool)?.queue ?? pool;
|
|
1780
|
-
const prefix = config?.queuePrefix;
|
|
1781
|
-
return prefix ? `${prefix}:${alias}` : alias;
|
|
1782
|
-
}
|
|
1783
|
-
|
|
1784
|
-
// runtime/subsystems/jobs/job-orchestrator.bullmq-backend.ts
|
|
1785
|
-
function sha1JobId(idempotencyKey) {
|
|
1786
|
-
return createHash("sha1").update(idempotencyKey).digest("hex");
|
|
1787
|
-
}
|
|
1788
|
-
var BullMQJobOrchestrator = class extends DrizzleJobOrchestrator {
|
|
1789
|
-
constructor(db, multiTenant, connection, bullConfig = null) {
|
|
1790
|
-
super(db, multiTenant);
|
|
1791
|
-
this.connection = connection;
|
|
1792
|
-
this.bullConfig = bullConfig;
|
|
1793
|
-
this.bullDb = db;
|
|
1794
|
-
}
|
|
1795
|
-
connection;
|
|
1796
|
-
bullConfig;
|
|
1797
|
-
// TODO(logging-subsystem): swap to ILogger once ADR-028 lands
|
|
1798
|
-
bullLogger = new Logger4(BullMQJobOrchestrator.name);
|
|
1799
|
-
/** Lazily-opened `Queue` handles, one per pool. */
|
|
1800
|
-
queues = /* @__PURE__ */ new Map();
|
|
1801
|
-
/** Single FlowProducer for parent/child hierarchies. Lazily opened. */
|
|
1802
|
-
_flow = null;
|
|
1803
|
-
/**
|
|
1804
|
-
* Cached `bullmq` value constructors, populated by `loadBullMq()` on first
|
|
1805
|
-
* use (the `start`/`cancel`/`replay` entrypoints `await` it before touching
|
|
1806
|
-
* a queue). Kept off the import graph so a `drizzle`-only consumer never
|
|
1807
|
-
* resolves the optional `'bullmq'` package.
|
|
1808
|
-
*/
|
|
1809
|
-
QueueCtor = null;
|
|
1810
|
-
FlowProducerCtor = null;
|
|
1811
|
-
bullMqLoad = null;
|
|
1812
|
-
/**
|
|
1813
|
-
* Own reference to the Drizzle client. `DrizzleJobOrchestrator.db` is
|
|
1814
|
-
* `private` (can't be redeclared even privately in a subclass), and the
|
|
1815
|
-
* spec forbids touching that file — so the subclass keeps its own handle
|
|
1816
|
-
* under a distinct name (same instance, passed through to `super`) for the
|
|
1817
|
-
* cancel-cascade snapshot + definition/run loads below.
|
|
1818
|
-
*/
|
|
1819
|
-
bullDb;
|
|
1820
|
-
/**
|
|
1821
|
-
* Lazily load the optional `bullmq` package and cache its value
|
|
1822
|
-
* constructors. Idempotent (single in-flight promise). Throws a friendly,
|
|
1823
|
-
* actionable error when the consumer selected `backend: 'bullmq'` but did
|
|
1824
|
-
* not install the package — mirrors `createRedisClient` in the redis event
|
|
1825
|
-
* backend. Must be `await`ed before any `queueFor`/`flow` access.
|
|
1826
|
-
*/
|
|
1827
|
-
async loadBullMq() {
|
|
1828
|
-
if (this.QueueCtor && this.FlowProducerCtor) return;
|
|
1829
|
-
if (!this.bullMqLoad) {
|
|
1830
|
-
this.bullMqLoad = (async () => {
|
|
1831
|
-
try {
|
|
1832
|
-
const mod = await import("bullmq");
|
|
1833
|
-
this.QueueCtor = mod.Queue;
|
|
1834
|
-
this.FlowProducerCtor = mod.FlowProducer;
|
|
1835
|
-
} catch {
|
|
1836
|
-
throw new Error(
|
|
1837
|
-
'BullMQ backend requires the "bullmq" package. Install it with: npm install bullmq'
|
|
1838
|
-
);
|
|
1839
|
-
}
|
|
1840
|
-
})();
|
|
1841
|
-
}
|
|
1842
|
-
await this.bullMqLoad;
|
|
1843
|
-
}
|
|
1844
|
-
/**
|
|
1845
|
-
* Open (or reuse) the `Queue` for a pool. Synchronous — callers `await
|
|
1846
|
-
* loadBullMq()` first so `QueueCtor` is populated.
|
|
1847
|
-
*/
|
|
1848
|
-
queueFor(pool) {
|
|
1849
|
-
if (!this.QueueCtor) {
|
|
1850
|
-
throw new Error("BullMQJobOrchestrator: queueFor called before loadBullMq()");
|
|
1851
|
-
}
|
|
1852
|
-
const name = resolvePoolQueueName(pool, this.bullConfig);
|
|
1853
|
-
let q = this.queues.get(name);
|
|
1854
|
-
if (!q) {
|
|
1855
|
-
q = new this.QueueCtor(name, { connection: this.connection });
|
|
1856
|
-
this.queues.set(name, q);
|
|
1857
|
-
}
|
|
1858
|
-
return q;
|
|
1859
|
-
}
|
|
1860
|
-
flow() {
|
|
1861
|
-
if (!this.FlowProducerCtor) {
|
|
1862
|
-
throw new Error("BullMQJobOrchestrator: flow called before loadBullMq()");
|
|
1863
|
-
}
|
|
1864
|
-
if (!this._flow) {
|
|
1865
|
-
this._flow = new this.FlowProducerCtor({ connection: this.connection });
|
|
1866
|
-
}
|
|
1867
|
-
return this._flow;
|
|
1868
|
-
}
|
|
1869
|
-
// ==========================================================================
|
|
1870
|
-
// start — Postgres insert (super) + BullMQ dispatch
|
|
1871
|
-
// ==========================================================================
|
|
1872
|
-
async start(type, input, opts = {}, tx) {
|
|
1873
|
-
const run = await super.start(type, input, opts, tx);
|
|
1874
|
-
await this.dispatch(run, type);
|
|
1875
|
-
return run;
|
|
1876
|
-
}
|
|
1877
|
-
/**
|
|
1878
|
-
* Map a `job_run` row onto a BullMQ job via `queue.add`. When the run has a
|
|
1879
|
-
* `parentRunId` we attach it to the parent's existing BullMQ job through the
|
|
1880
|
-
* `parent: { id, queue }` opt — BullMQ then tracks the parent/child link in
|
|
1881
|
-
* its own graph. (The FlowProducer is reserved for whole-tree atomic
|
|
1882
|
-
* submits, exposed as an opt-in extension via `flowProducer()`; runtime
|
|
1883
|
-
* `ctx.spawnChild` is incremental, so `queue.add` with a parent ref is the
|
|
1884
|
-
* correct primitive here.)
|
|
1885
|
-
*
|
|
1886
|
-
* The `jobId` is colon-safe + stable: `sha1(dedupeKey)` when a dedupe key is
|
|
1887
|
-
* present (so the same logical key dedups), else the `job_run.id` UUID
|
|
1888
|
-
* (already colon-free).
|
|
1889
|
-
*
|
|
1890
|
-
* The domain `parentClosePolicy` cascade is still enforced in Postgres by
|
|
1891
|
-
* the shared `cancel` path — BullMQ's parent link is dispatch bookkeeping,
|
|
1892
|
-
* not the authority.
|
|
1893
|
-
*/
|
|
1894
|
-
async dispatch(run, type) {
|
|
1895
|
-
await this.loadBullMq();
|
|
1896
|
-
const def = await this.loadDefinition(type);
|
|
1897
|
-
const jobId = run.dedupeKey ? sha1JobId(run.dedupeKey) : run.id;
|
|
1898
|
-
const jobOpts = {
|
|
1899
|
-
jobId,
|
|
1900
|
-
...this.retryOpts(def),
|
|
1901
|
-
...this.dedupeOpts(run, def)
|
|
1902
|
-
};
|
|
1903
|
-
if (run.parentRunId) {
|
|
1904
|
-
const parentRow = await this.loadRun(run.parentRunId);
|
|
1905
|
-
if (parentRow) {
|
|
1906
|
-
const parentJobId = parentRow.dedupeKey ? sha1JobId(parentRow.dedupeKey) : parentRow.id;
|
|
1907
|
-
jobOpts.parent = {
|
|
1908
|
-
id: parentJobId,
|
|
1909
|
-
queue: resolvePoolQueueName(parentRow.pool, this.bullConfig)
|
|
1910
|
-
};
|
|
1911
|
-
}
|
|
1912
|
-
}
|
|
1913
|
-
const payload = { runId: run.id, type, input: run.input };
|
|
1914
|
-
await this.queueFor(run.pool).add(type, payload, jobOpts);
|
|
1915
|
-
}
|
|
1916
|
-
/**
|
|
1917
|
-
* Opt-in extension (spec §Extensions): expose the FlowProducer for
|
|
1918
|
-
* consumers that want to submit a whole parent/child DAG atomically up
|
|
1919
|
-
* front, rather than incrementally via `ctx.spawnChild`. Backend-specific —
|
|
1920
|
-
* code using it is not portable to the Drizzle backend. Async because it
|
|
1921
|
-
* lazily loads the optional `bullmq` package on first use.
|
|
1922
|
-
*/
|
|
1923
|
-
async flowProducer() {
|
|
1924
|
-
await this.loadBullMq();
|
|
1925
|
-
return this.flow();
|
|
1926
|
-
}
|
|
1927
|
-
retryOpts(def) {
|
|
1928
|
-
const policy = def.retryPolicy;
|
|
1929
|
-
if (!policy) return {};
|
|
1930
|
-
return {
|
|
1931
|
-
attempts: policy.attempts,
|
|
1932
|
-
backoff: {
|
|
1933
|
-
type: policy.backoff === "exponential" ? "exponential" : "fixed",
|
|
1934
|
-
delay: policy.baseMs
|
|
1935
|
-
}
|
|
1936
|
-
};
|
|
1937
|
-
}
|
|
1938
|
-
dedupeOpts(run, def) {
|
|
1939
|
-
if (!run.dedupeKey || !def.dedupeWindowMs) return {};
|
|
1940
|
-
return {
|
|
1941
|
-
deduplication: {
|
|
1942
|
-
id: sha1JobId(run.dedupeKey),
|
|
1943
|
-
ttl: def.dedupeWindowMs
|
|
1944
|
-
}
|
|
1945
|
-
};
|
|
1946
|
-
}
|
|
1947
|
-
// ==========================================================================
|
|
1948
|
-
// cancel — Postgres cascade (super) + remove from queue
|
|
1949
|
-
// ==========================================================================
|
|
1950
|
-
async cancel(runId, opts = {}) {
|
|
1951
|
-
const target = await this.loadRun(runId);
|
|
1952
|
-
await super.cancel(runId, opts);
|
|
1953
|
-
if (!target) return;
|
|
1954
|
-
await this.loadBullMq();
|
|
1955
|
-
await this.removeFromQueue(target);
|
|
1956
|
-
if (opts.cascade === false) return;
|
|
1957
|
-
const descendants = await this.bullDb.select().from(jobRuns).where(eq5(jobRuns.rootRunId, target.rootRunId));
|
|
1958
|
-
for (const child of descendants) {
|
|
1959
|
-
if (child.id === runId) continue;
|
|
1960
|
-
await this.removeFromQueue(child);
|
|
1961
|
-
}
|
|
1962
|
-
}
|
|
1963
|
-
async removeFromQueue(run) {
|
|
1964
|
-
const jobId = run.dedupeKey ? sha1JobId(run.dedupeKey) : run.id;
|
|
1965
|
-
try {
|
|
1966
|
-
const job = await this.queueFor(run.pool).getJob(jobId);
|
|
1967
|
-
if (job) await job.remove();
|
|
1968
|
-
} catch (err) {
|
|
1969
|
-
this.bullLogger.warn(
|
|
1970
|
-
`cancel: could not remove BullMQ job ${jobId} (pool=${run.pool}): ${err.message}`
|
|
1971
|
-
);
|
|
1972
|
-
}
|
|
1973
|
-
}
|
|
1974
|
-
// ==========================================================================
|
|
1975
|
-
// replay — Postgres reset (super) + re-enqueue
|
|
1976
|
-
// ==========================================================================
|
|
1977
|
-
async replay(runId) {
|
|
1978
|
-
const run = await super.replay(runId);
|
|
1979
|
-
await this.dispatch(run, run.jobType);
|
|
1980
|
-
return run;
|
|
1981
|
-
}
|
|
1982
|
-
// ==========================================================================
|
|
1983
|
-
// Internals
|
|
1984
|
-
// ==========================================================================
|
|
1985
|
-
async loadDefinition(type) {
|
|
1986
|
-
const [def] = await this.bullDb.select().from(jobs).where(eq5(jobs.type, type)).limit(1);
|
|
1987
|
-
if (!def) {
|
|
1988
|
-
throw new Error(`BullMQJobOrchestrator: no job definition for '${type}'`);
|
|
1989
|
-
}
|
|
1990
|
-
return def;
|
|
1991
|
-
}
|
|
1992
|
-
async loadRun(id) {
|
|
1993
|
-
const [row] = await this.bullDb.select().from(jobRuns).where(eq5(jobRuns.id, id)).limit(1);
|
|
1994
|
-
return row ?? null;
|
|
1995
|
-
}
|
|
1996
|
-
/** Close all open queue + flow connections. Called on module destroy. */
|
|
1997
|
-
async closeConnections() {
|
|
1998
|
-
for (const q of this.queues.values()) {
|
|
1999
|
-
await q.close().catch(() => void 0);
|
|
2000
|
-
}
|
|
2001
|
-
this.queues.clear();
|
|
2002
|
-
if (this._flow) {
|
|
2003
|
-
await this._flow.close().catch(() => void 0);
|
|
2004
|
-
this._flow = null;
|
|
2005
|
-
}
|
|
2006
|
-
}
|
|
2007
|
-
};
|
|
2008
|
-
BullMQJobOrchestrator = __decorateClass([
|
|
2009
|
-
Injectable8(),
|
|
2010
|
-
__decorateParam(0, Inject8(DRIZZLE)),
|
|
2011
|
-
__decorateParam(1, Inject8(JOBS_MULTI_TENANT)),
|
|
2012
|
-
__decorateParam(2, Inject8(BULLMQ_CONNECTION)),
|
|
2013
|
-
__decorateParam(3, Optional5()),
|
|
2014
|
-
__decorateParam(3, Inject8(BULLMQ_RESOLVED_CONFIG))
|
|
2015
|
-
], BullMQJobOrchestrator);
|
|
2016
|
-
|
|
2017
1632
|
// runtime/subsystems/jobs/job-orchestrator.memory-backend.ts
|
|
2018
1633
|
import { randomUUID as randomUUID4 } from "crypto";
|
|
2019
|
-
import { Inject as
|
|
1634
|
+
import { Inject as Inject8, Injectable as Injectable8, Logger as Logger4, Optional as Optional5 } from "@nestjs/common";
|
|
2020
1635
|
var QUEUED_RUN_AT = /* @__PURE__ */ new Date(864e13);
|
|
2021
1636
|
var TERMINAL_STATUSES2 = [
|
|
2022
1637
|
"completed",
|
|
@@ -2063,7 +1678,7 @@ var MemoryJobOrchestrator = class {
|
|
|
2063
1678
|
stepService;
|
|
2064
1679
|
multiTenant;
|
|
2065
1680
|
moduleRef;
|
|
2066
|
-
logger = new
|
|
1681
|
+
logger = new Logger4(MemoryJobOrchestrator.name);
|
|
2067
1682
|
mutex = new PromiseMutex();
|
|
2068
1683
|
handlerRegistry = /* @__PURE__ */ new Map();
|
|
2069
1684
|
/**
|
|
@@ -2412,7 +2027,7 @@ var MemoryJobOrchestrator = class {
|
|
|
2412
2027
|
run,
|
|
2413
2028
|
step: this.makeStepFn(run),
|
|
2414
2029
|
spawnChild: this.makeSpawnFn(run),
|
|
2415
|
-
logger: new
|
|
2030
|
+
logger: new Logger4(`JobRun:${run.id}`)
|
|
2416
2031
|
};
|
|
2417
2032
|
const attemptsBefore = run.attempts ?? 0;
|
|
2418
2033
|
try {
|
|
@@ -2585,9 +2200,9 @@ var MemoryJobOrchestrator = class {
|
|
|
2585
2200
|
}
|
|
2586
2201
|
};
|
|
2587
2202
|
MemoryJobOrchestrator = __decorateClass([
|
|
2588
|
-
|
|
2589
|
-
__decorateParam(2,
|
|
2590
|
-
__decorateParam(3,
|
|
2203
|
+
Injectable8(),
|
|
2204
|
+
__decorateParam(2, Inject8(JOBS_MULTI_TENANT)),
|
|
2205
|
+
__decorateParam(3, Optional5())
|
|
2591
2206
|
], MemoryJobOrchestrator);
|
|
2592
2207
|
function classifyError(err, policy, currentAttempts) {
|
|
2593
2208
|
if (!policy) return "fail";
|
|
@@ -2621,7 +2236,7 @@ function serialiseError(err, attempt, retryable) {
|
|
|
2621
2236
|
}
|
|
2622
2237
|
|
|
2623
2238
|
// runtime/subsystems/jobs/job-run-service.memory-backend.ts
|
|
2624
|
-
import { Inject as
|
|
2239
|
+
import { Inject as Inject9, Injectable as Injectable9 } from "@nestjs/common";
|
|
2625
2240
|
var NON_TERMINAL_STATUSES2 = [
|
|
2626
2241
|
"pending",
|
|
2627
2242
|
"running",
|
|
@@ -2788,9 +2403,9 @@ var MemoryJobRunService = class {
|
|
|
2788
2403
|
}
|
|
2789
2404
|
};
|
|
2790
2405
|
MemoryJobRunService = __decorateClass([
|
|
2791
|
-
|
|
2792
|
-
__decorateParam(1,
|
|
2793
|
-
__decorateParam(2,
|
|
2406
|
+
Injectable9(),
|
|
2407
|
+
__decorateParam(1, Inject9(JOB_ORCHESTRATOR)),
|
|
2408
|
+
__decorateParam(2, Inject9(JOBS_MULTI_TENANT))
|
|
2794
2409
|
], MemoryJobRunService);
|
|
2795
2410
|
function compareBy(a, b, order) {
|
|
2796
2411
|
switch (order) {
|
|
@@ -2808,7 +2423,7 @@ function compareBy(a, b, order) {
|
|
|
2808
2423
|
|
|
2809
2424
|
// runtime/subsystems/jobs/job-step-service.memory-backend.ts
|
|
2810
2425
|
import { randomUUID as randomUUID5 } from "crypto";
|
|
2811
|
-
import { Injectable as
|
|
2426
|
+
import { Injectable as Injectable10 } from "@nestjs/common";
|
|
2812
2427
|
var MemoryJobStepService = class {
|
|
2813
2428
|
constructor(store) {
|
|
2814
2429
|
this.store = store;
|
|
@@ -2901,7 +2516,7 @@ var MemoryJobStepService = class {
|
|
|
2901
2516
|
}
|
|
2902
2517
|
};
|
|
2903
2518
|
MemoryJobStepService = __decorateClass([
|
|
2904
|
-
|
|
2519
|
+
Injectable10()
|
|
2905
2520
|
], MemoryJobStepService);
|
|
2906
2521
|
|
|
2907
2522
|
// runtime/subsystems/jobs/memory-job-store.ts
|
|
@@ -2920,32 +2535,192 @@ var MemoryJobStore = class {
|
|
|
2920
2535
|
}
|
|
2921
2536
|
};
|
|
2922
2537
|
|
|
2923
|
-
// runtime/subsystems/jobs/
|
|
2924
|
-
|
|
2925
|
-
|
|
2926
|
-
|
|
2927
|
-
|
|
2928
|
-
|
|
2929
|
-
|
|
2930
|
-
|
|
2931
|
-
|
|
2932
|
-
|
|
2933
|
-
|
|
2934
|
-
|
|
2935
|
-
|
|
2936
|
-
|
|
2937
|
-
|
|
2938
|
-
|
|
2939
|
-
|
|
2940
|
-
|
|
2941
|
-
|
|
2942
|
-
|
|
2943
|
-
|
|
2538
|
+
// runtime/subsystems/jobs/pool-config.loader.ts
|
|
2539
|
+
import { existsSync, readFileSync } from "fs";
|
|
2540
|
+
import { resolve } from "path";
|
|
2541
|
+
import { parse as parseYaml } from "yaml";
|
|
2542
|
+
var FRAMEWORK_POOLS = Object.freeze({
|
|
2543
|
+
events_inbound: Object.freeze({
|
|
2544
|
+
queue: "jobs-events-inbound",
|
|
2545
|
+
concurrency: 20,
|
|
2546
|
+
reserved: true,
|
|
2547
|
+
description: "Inbound events drain (events subsystem outbox)."
|
|
2548
|
+
}),
|
|
2549
|
+
events_change: Object.freeze({
|
|
2550
|
+
queue: "jobs-events-change",
|
|
2551
|
+
concurrency: 30,
|
|
2552
|
+
reserved: true,
|
|
2553
|
+
description: "Change events drain (events subsystem outbox)."
|
|
2554
|
+
}),
|
|
2555
|
+
events_outbound: Object.freeze({
|
|
2556
|
+
queue: "jobs-events-outbound",
|
|
2557
|
+
concurrency: 10,
|
|
2558
|
+
reserved: true,
|
|
2559
|
+
description: "Outbound events drain (events subsystem outbox)."
|
|
2560
|
+
}),
|
|
2561
|
+
interactive: Object.freeze({
|
|
2562
|
+
queue: "jobs-interactive",
|
|
2563
|
+
concurrency: 20,
|
|
2564
|
+
reserved: false,
|
|
2565
|
+
description: "User-facing latency-sensitive jobs."
|
|
2566
|
+
}),
|
|
2567
|
+
batch: Object.freeze({
|
|
2568
|
+
queue: "jobs-batch",
|
|
2569
|
+
concurrency: 5,
|
|
2570
|
+
reserved: false,
|
|
2571
|
+
description: "Default pool for background jobs."
|
|
2572
|
+
})
|
|
2573
|
+
});
|
|
2574
|
+
var RESERVED_POOL_NAMES = new Set(
|
|
2575
|
+
Object.entries(FRAMEWORK_POOLS).filter(([, def]) => def.reserved).map(([name]) => name)
|
|
2576
|
+
);
|
|
2577
|
+
var cache = /* @__PURE__ */ new Map();
|
|
2578
|
+
function loadPoolConfig(configPath) {
|
|
2579
|
+
const resolved = resolve(configPath ?? `${process.cwd()}/codegen.config.yaml`);
|
|
2580
|
+
const cached = cache.get(resolved);
|
|
2581
|
+
if (cached) return cached;
|
|
2582
|
+
const merged = /* @__PURE__ */ new Map();
|
|
2583
|
+
for (const [name, def] of Object.entries(FRAMEWORK_POOLS)) {
|
|
2584
|
+
merged.set(name, { ...def });
|
|
2585
|
+
}
|
|
2586
|
+
if (!existsSync(resolved)) {
|
|
2587
|
+
cache.set(resolved, merged);
|
|
2588
|
+
return merged;
|
|
2589
|
+
}
|
|
2590
|
+
let raw;
|
|
2591
|
+
try {
|
|
2592
|
+
raw = parseYaml(readFileSync(resolved, "utf8"));
|
|
2593
|
+
} catch (err) {
|
|
2594
|
+
throw new Error(
|
|
2595
|
+
`pool-config.loader: failed to parse YAML at ${resolved}: ${err.message}`
|
|
2596
|
+
);
|
|
2597
|
+
}
|
|
2598
|
+
const userPools = extractUserPools(raw);
|
|
2599
|
+
for (const [name, userDef] of Object.entries(userPools)) {
|
|
2600
|
+
const existing = merged.get(name);
|
|
2601
|
+
if (existing) {
|
|
2602
|
+
const next = {
|
|
2603
|
+
queue: existing.queue,
|
|
2604
|
+
concurrency: typeof userDef.concurrency === "number" ? userDef.concurrency : existing.concurrency,
|
|
2605
|
+
reserved: existing.reserved,
|
|
2606
|
+
description: userDef.description ?? existing.description
|
|
2607
|
+
};
|
|
2608
|
+
merged.set(name, next);
|
|
2609
|
+
continue;
|
|
2610
|
+
}
|
|
2611
|
+
if (typeof userDef.queue !== "string" || userDef.queue.length === 0) {
|
|
2612
|
+
throw new Error(
|
|
2613
|
+
`pool-config.loader: pool '${name}' must declare a non-empty 'queue'.`
|
|
2614
|
+
);
|
|
2615
|
+
}
|
|
2616
|
+
if (typeof userDef.concurrency !== "number" || userDef.concurrency <= 0) {
|
|
2617
|
+
throw new Error(
|
|
2618
|
+
`pool-config.loader: pool '${name}' must declare a positive 'concurrency'.`
|
|
2619
|
+
);
|
|
2620
|
+
}
|
|
2621
|
+
if (userDef.reserved === true) {
|
|
2622
|
+
throw new Error(
|
|
2623
|
+
`pool-config.loader: user-defined pool '${name}' cannot set 'reserved: true' \u2014 reserved is framework-only.`
|
|
2624
|
+
);
|
|
2625
|
+
}
|
|
2626
|
+
merged.set(name, {
|
|
2627
|
+
queue: userDef.queue,
|
|
2628
|
+
concurrency: userDef.concurrency,
|
|
2629
|
+
reserved: false,
|
|
2630
|
+
description: userDef.description
|
|
2631
|
+
});
|
|
2632
|
+
}
|
|
2633
|
+
cache.set(resolved, merged);
|
|
2634
|
+
return merged;
|
|
2635
|
+
}
|
|
2636
|
+
function allNonReservedPoolNames(config) {
|
|
2637
|
+
const out = [];
|
|
2638
|
+
for (const [name, def] of config) {
|
|
2639
|
+
if (!def.reserved) out.push(name);
|
|
2640
|
+
}
|
|
2641
|
+
return out;
|
|
2642
|
+
}
|
|
2643
|
+
function allPoolNames(config) {
|
|
2644
|
+
return [...config.keys()];
|
|
2645
|
+
}
|
|
2646
|
+
function extractUserPools(raw) {
|
|
2647
|
+
if (!raw || typeof raw !== "object") return {};
|
|
2648
|
+
const jobs2 = raw.jobs;
|
|
2649
|
+
if (!jobs2 || typeof jobs2 !== "object") return {};
|
|
2650
|
+
const pools = jobs2.pools;
|
|
2651
|
+
if (!pools || typeof pools !== "object") return {};
|
|
2652
|
+
const out = {};
|
|
2653
|
+
for (const [name, def] of Object.entries(pools)) {
|
|
2654
|
+
if (!def || typeof def !== "object") continue;
|
|
2655
|
+
out[name] = def;
|
|
2656
|
+
}
|
|
2657
|
+
return out;
|
|
2658
|
+
}
|
|
2659
|
+
|
|
2660
|
+
// runtime/subsystems/jobs/bullmq.config.ts
|
|
2661
|
+
var BULLMQ_CONNECTION = /* @__PURE__ */ Symbol("BULLMQ_CONNECTION");
|
|
2662
|
+
var BULLMQ_RESOLVED_CONFIG = /* @__PURE__ */ Symbol("BULLMQ_RESOLVED_CONFIG");
|
|
2663
|
+
var DEFAULT_REDIS_URL = "redis://localhost:6379";
|
|
2664
|
+
var DEFAULT_BULL_BOARD_MOUNT = "/admin/queues";
|
|
2665
|
+
function resolveBullMqConfig(ext) {
|
|
2666
|
+
const url = ext?.redis_url ?? process.env.REDIS_URL ?? DEFAULT_REDIS_URL;
|
|
2667
|
+
const resolved = {
|
|
2668
|
+
connection: { url },
|
|
2669
|
+
queuePrefix: ext?.queue_prefix
|
|
2670
|
+
};
|
|
2671
|
+
if (ext?.bull_board?.enabled) {
|
|
2672
|
+
resolved.bullBoard = {
|
|
2673
|
+
enabled: true,
|
|
2674
|
+
mountPath: ext.bull_board.mount_path ?? DEFAULT_BULL_BOARD_MOUNT
|
|
2675
|
+
};
|
|
2676
|
+
}
|
|
2677
|
+
return resolved;
|
|
2678
|
+
}
|
|
2679
|
+
function resolvePoolQueueName(pool, config, poolConfig = loadPoolConfig()) {
|
|
2680
|
+
const alias = poolConfig.get(pool)?.queue ?? pool;
|
|
2681
|
+
const prefix = config?.queuePrefix;
|
|
2682
|
+
return prefix ? `${prefix}:${alias}` : alias;
|
|
2683
|
+
}
|
|
2684
|
+
|
|
2685
|
+
// runtime/subsystems/jobs/jobs-domain.module.ts
|
|
2686
|
+
var JobsDomainModule = class {
|
|
2687
|
+
static forRoot(opts) {
|
|
2688
|
+
const multiTenant = opts.multiTenant ?? false;
|
|
2689
|
+
const providers = [
|
|
2690
|
+
// JOB-8 — boolean provider consumed by the four service-layer backends.
|
|
2691
|
+
// Always provided (even when `multiTenant === false`) so `@Inject`
|
|
2692
|
+
// always resolves; backends short-circuit the enforcement path when
|
|
2693
|
+
// the value is `false`. See `jobs-domain.tokens.ts` for the claim-loop
|
|
2694
|
+
// cross-tenant-by-design decision.
|
|
2695
|
+
{ provide: JOBS_MULTI_TENANT, useValue: multiTenant }
|
|
2696
|
+
];
|
|
2697
|
+
if (opts.backend === "memory") {
|
|
2698
|
+
const store = new MemoryJobStore();
|
|
2699
|
+
providers.push({ provide: MemoryJobStore, useValue: store });
|
|
2700
|
+
providers.push(MemoryJobStepService);
|
|
2701
|
+
providers.push({ provide: JOB_STEP_SERVICE, useExisting: MemoryJobStepService });
|
|
2702
|
+
providers.push(MemoryJobOrchestrator);
|
|
2703
|
+
providers.push({ provide: JOB_ORCHESTRATOR, useExisting: MemoryJobOrchestrator });
|
|
2704
|
+
providers.push(MemoryJobRunService);
|
|
2705
|
+
providers.push({ provide: JOB_RUN_SERVICE, useExisting: MemoryJobRunService });
|
|
2944
2706
|
} else if (opts.backend === "bullmq") {
|
|
2945
2707
|
const resolved = resolveBullMqConfig(opts.extensions?.bullmq);
|
|
2946
2708
|
providers.push({ provide: BULLMQ_CONNECTION, useValue: resolved.connection });
|
|
2947
2709
|
providers.push({ provide: BULLMQ_RESOLVED_CONFIG, useValue: resolved });
|
|
2948
|
-
providers.push({
|
|
2710
|
+
providers.push({
|
|
2711
|
+
provide: JOB_ORCHESTRATOR,
|
|
2712
|
+
useFactory: async (...args) => {
|
|
2713
|
+
const specifier = "./job-orchestrator.bullmq-backend";
|
|
2714
|
+
const mod = await import(specifier);
|
|
2715
|
+
return new mod.BullMQJobOrchestrator(...args);
|
|
2716
|
+
},
|
|
2717
|
+
// The bullmq orchestrator constructor mirrors DrizzleJobOrchestrator's
|
|
2718
|
+
// injection list: DRIZZLE + JOBS_MULTI_TENANT + the resolved BullMQ
|
|
2719
|
+
// tokens. Importing token references would force a static dep on the
|
|
2720
|
+
// tokens file in this module's import graph; using the existing
|
|
2721
|
+
// symbols already in scope is sufficient.
|
|
2722
|
+
inject: [DRIZZLE, JOBS_MULTI_TENANT, BULLMQ_CONNECTION, BULLMQ_RESOLVED_CONFIG]
|
|
2723
|
+
});
|
|
2949
2724
|
providers.push({ provide: JOB_RUN_SERVICE, useClass: DrizzleJobRunService });
|
|
2950
2725
|
providers.push({ provide: JOB_STEP_SERVICE, useClass: DrizzleJobStepService });
|
|
2951
2726
|
} else {
|
|
@@ -2975,8 +2750,8 @@ JobsDomainModule = __decorateClass([
|
|
|
2975
2750
|
], JobsDomainModule);
|
|
2976
2751
|
|
|
2977
2752
|
// runtime/subsystems/jobs/job-worker.ts
|
|
2978
|
-
import { Inject as
|
|
2979
|
-
import { and as and5, asc as asc2, desc as desc3, eq as
|
|
2753
|
+
import { Inject as Inject10, Injectable as Injectable11, Logger as Logger5 } from "@nestjs/common";
|
|
2754
|
+
import { and as and5, asc as asc2, desc as desc3, eq as eq5, inArray as inArray3, lt as lt2, lte, sql as sql7 } from "drizzle-orm";
|
|
2980
2755
|
var JOB_WORKER_OPTIONS = /* @__PURE__ */ Symbol("JOB_WORKER_OPTIONS");
|
|
2981
2756
|
var DEFAULT_POLL_INTERVAL_MS = 1e3;
|
|
2982
2757
|
var DEFAULT_STALE_SWEEPER_INTERVAL_MS = 6e4;
|
|
@@ -3039,7 +2814,7 @@ var JobWorker = class {
|
|
|
3039
2814
|
stepService;
|
|
3040
2815
|
options;
|
|
3041
2816
|
moduleRef;
|
|
3042
|
-
logger = new
|
|
2817
|
+
logger = new Logger5(JobWorker.name);
|
|
3043
2818
|
shuttingDown = false;
|
|
3044
2819
|
inFlight = /* @__PURE__ */ new Set();
|
|
3045
2820
|
pollTimer = null;
|
|
@@ -3080,7 +2855,7 @@ var JobWorker = class {
|
|
|
3080
2855
|
await this.drainInFlight();
|
|
3081
2856
|
try {
|
|
3082
2857
|
await this.db.update(jobRuns).set({ status: "pending", claimedAt: null, startedAt: null }).where(
|
|
3083
|
-
and5(
|
|
2858
|
+
and5(eq5(jobRuns.status, "running"), eq5(jobRuns.pool, this.options.pool))
|
|
3084
2859
|
);
|
|
3085
2860
|
} catch (err) {
|
|
3086
2861
|
this.logger.error(`shutdown reset failed: ${err.message}`);
|
|
@@ -3130,8 +2905,8 @@ var JobWorker = class {
|
|
|
3130
2905
|
return this.db.transaction(async (tx) => {
|
|
3131
2906
|
const candidates = await tx.select({ id: jobRuns.id }).from(jobRuns).where(
|
|
3132
2907
|
and5(
|
|
3133
|
-
|
|
3134
|
-
|
|
2908
|
+
eq5(jobRuns.status, "pending"),
|
|
2909
|
+
eq5(jobRuns.pool, pool),
|
|
3135
2910
|
lte(jobRuns.runAt, /* @__PURE__ */ new Date())
|
|
3136
2911
|
)
|
|
3137
2912
|
).orderBy(desc3(jobRuns.priority), asc2(jobRuns.runAt)).limit(1).for("update", { skipLocked: true });
|
|
@@ -3142,7 +2917,7 @@ var JobWorker = class {
|
|
|
3142
2917
|
claimedAt: /* @__PURE__ */ new Date(),
|
|
3143
2918
|
startedAt: /* @__PURE__ */ new Date(),
|
|
3144
2919
|
updatedAt: /* @__PURE__ */ new Date()
|
|
3145
|
-
}).where(
|
|
2920
|
+
}).where(eq5(jobRuns.id, candidate.id)).returning();
|
|
3146
2921
|
return claimed ?? null;
|
|
3147
2922
|
});
|
|
3148
2923
|
}
|
|
@@ -3160,7 +2935,7 @@ var JobWorker = class {
|
|
|
3160
2935
|
await this.db.transaction(async (tx) => {
|
|
3161
2936
|
const threshold = new Date(Date.now() - this.staleThresholdMs);
|
|
3162
2937
|
const stale = await tx.select({ id: jobRuns.id }).from(jobRuns).where(
|
|
3163
|
-
and5(
|
|
2938
|
+
and5(eq5(jobRuns.status, "running"), lt2(jobRuns.claimedAt, threshold))
|
|
3164
2939
|
).for("update", { skipLocked: true });
|
|
3165
2940
|
if (stale.length === 0) return;
|
|
3166
2941
|
const ids = stale.map((r) => r.id);
|
|
@@ -3193,8 +2968,8 @@ var JobWorker = class {
|
|
|
3193
2968
|
if (claimed.concurrencyKey) {
|
|
3194
2969
|
const inflight = await this.db.select({ id: jobRuns.id }).from(jobRuns).where(
|
|
3195
2970
|
and5(
|
|
3196
|
-
|
|
3197
|
-
|
|
2971
|
+
eq5(jobRuns.concurrencyKey, claimed.concurrencyKey),
|
|
2972
|
+
eq5(jobRuns.status, "running")
|
|
3198
2973
|
)
|
|
3199
2974
|
);
|
|
3200
2975
|
const other = inflight.find((r) => r.id !== claimed.id);
|
|
@@ -3204,7 +2979,7 @@ var JobWorker = class {
|
|
|
3204
2979
|
claimedAt: null,
|
|
3205
2980
|
startedAt: null,
|
|
3206
2981
|
updatedAt: /* @__PURE__ */ new Date()
|
|
3207
|
-
}).where(
|
|
2982
|
+
}).where(eq5(jobRuns.id, claimed.id));
|
|
3208
2983
|
return;
|
|
3209
2984
|
}
|
|
3210
2985
|
}
|
|
@@ -3219,7 +2994,7 @@ var JobWorker = class {
|
|
|
3219
2994
|
run: claimed,
|
|
3220
2995
|
step: this.makeStepFn(claimed),
|
|
3221
2996
|
spawnChild: this.makeSpawnFn(claimed),
|
|
3222
|
-
logger: new
|
|
2997
|
+
logger: new Logger5(`JobRun:${claimed.id}`)
|
|
3223
2998
|
};
|
|
3224
2999
|
const attemptsBefore = claimed.attempts ?? 0;
|
|
3225
3000
|
try {
|
|
@@ -3230,7 +3005,7 @@ var JobWorker = class {
|
|
|
3230
3005
|
finishedAt: /* @__PURE__ */ new Date(),
|
|
3231
3006
|
updatedAt: /* @__PURE__ */ new Date(),
|
|
3232
3007
|
attempts: attemptsBefore + 1
|
|
3233
|
-
}).where(
|
|
3008
|
+
}).where(eq5(jobRuns.id, claimed.id));
|
|
3234
3009
|
} catch (err) {
|
|
3235
3010
|
const policy = meta.retry;
|
|
3236
3011
|
const decision = classifyError2(err, policy, attemptsBefore);
|
|
@@ -3245,7 +3020,7 @@ var JobWorker = class {
|
|
|
3245
3020
|
claimedAt: null,
|
|
3246
3021
|
error: serialiseError2(err, nextAttempts, true),
|
|
3247
3022
|
updatedAt: /* @__PURE__ */ new Date()
|
|
3248
|
-
}).where(
|
|
3023
|
+
}).where(eq5(jobRuns.id, claimed.id));
|
|
3249
3024
|
} else {
|
|
3250
3025
|
await this.markFailed(claimed, err, nextAttempts);
|
|
3251
3026
|
}
|
|
@@ -3258,7 +3033,7 @@ var JobWorker = class {
|
|
|
3258
3033
|
finishedAt: /* @__PURE__ */ new Date(),
|
|
3259
3034
|
error: serialiseError2(err, finalAttempts, false),
|
|
3260
3035
|
updatedAt: /* @__PURE__ */ new Date()
|
|
3261
|
-
}).where(
|
|
3036
|
+
}).where(eq5(jobRuns.id, claimed.id));
|
|
3262
3037
|
if (claimed.parentClosePolicy === "terminate") {
|
|
3263
3038
|
try {
|
|
3264
3039
|
await this.orchestrator.cancel(claimed.id, {
|
|
@@ -3361,215 +3136,14 @@ var JobWorker = class {
|
|
|
3361
3136
|
// ============================================================================
|
|
3362
3137
|
};
|
|
3363
3138
|
JobWorker = __decorateClass([
|
|
3364
|
-
|
|
3365
|
-
__decorateParam(0,
|
|
3366
|
-
__decorateParam(1,
|
|
3367
|
-
__decorateParam(2,
|
|
3368
|
-
__decorateParam(3,
|
|
3369
|
-
__decorateParam(4,
|
|
3139
|
+
Injectable11(),
|
|
3140
|
+
__decorateParam(0, Inject10(DRIZZLE)),
|
|
3141
|
+
__decorateParam(1, Inject10(JOB_ORCHESTRATOR)),
|
|
3142
|
+
__decorateParam(2, Inject10(JOB_RUN_SERVICE)),
|
|
3143
|
+
__decorateParam(3, Inject10(JOB_STEP_SERVICE)),
|
|
3144
|
+
__decorateParam(4, Inject10(JOB_WORKER_OPTIONS))
|
|
3370
3145
|
], JobWorker);
|
|
3371
3146
|
|
|
3372
|
-
// runtime/subsystems/jobs/job-worker.bullmq-backend.ts
|
|
3373
|
-
import { Logger as Logger7 } from "@nestjs/common";
|
|
3374
|
-
import { eq as eq7 } from "drizzle-orm";
|
|
3375
|
-
function serialiseError3(err, attempt, retryable) {
|
|
3376
|
-
const e = err;
|
|
3377
|
-
return {
|
|
3378
|
-
message: e?.message ?? String(err),
|
|
3379
|
-
stack: e?.stack,
|
|
3380
|
-
retryable,
|
|
3381
|
-
attempt
|
|
3382
|
-
};
|
|
3383
|
-
}
|
|
3384
|
-
var BullMQJobWorker = class _BullMQJobWorker {
|
|
3385
|
-
constructor(db, orchestrator, stepService, options, moduleRef) {
|
|
3386
|
-
this.db = db;
|
|
3387
|
-
this.orchestrator = orchestrator;
|
|
3388
|
-
this.stepService = stepService;
|
|
3389
|
-
this.options = options;
|
|
3390
|
-
this.moduleRef = moduleRef;
|
|
3391
|
-
}
|
|
3392
|
-
db;
|
|
3393
|
-
orchestrator;
|
|
3394
|
-
stepService;
|
|
3395
|
-
options;
|
|
3396
|
-
moduleRef;
|
|
3397
|
-
logger = new Logger7(_BullMQJobWorker.name);
|
|
3398
|
-
worker = null;
|
|
3399
|
-
async onModuleInit() {
|
|
3400
|
-
let WorkerCtor;
|
|
3401
|
-
try {
|
|
3402
|
-
const mod = await import("bullmq");
|
|
3403
|
-
WorkerCtor = mod.Worker;
|
|
3404
|
-
} catch {
|
|
3405
|
-
throw new Error(
|
|
3406
|
-
'BullMQ backend requires the "bullmq" package. Install it with: npm install bullmq'
|
|
3407
|
-
);
|
|
3408
|
-
}
|
|
3409
|
-
this.worker = new WorkerCtor(
|
|
3410
|
-
this.options.queueName,
|
|
3411
|
-
(job) => this.process(job),
|
|
3412
|
-
{
|
|
3413
|
-
connection: this.options.connection,
|
|
3414
|
-
concurrency: this.options.concurrency
|
|
3415
|
-
}
|
|
3416
|
-
);
|
|
3417
|
-
this.worker.on("failed", (job, err) => {
|
|
3418
|
-
if (!job) return;
|
|
3419
|
-
const attemptsMade = job.attemptsMade;
|
|
3420
|
-
const maxAttempts = job.opts.attempts ?? 1;
|
|
3421
|
-
if (attemptsMade >= maxAttempts) {
|
|
3422
|
-
void this.markFailed(job.data.runId, err, attemptsMade);
|
|
3423
|
-
}
|
|
3424
|
-
});
|
|
3425
|
-
this.logger.log(
|
|
3426
|
-
`BullMQ worker started: pool='${this.options.pool}' queue='${this.options.queueName}' concurrency=${this.options.concurrency}`
|
|
3427
|
-
);
|
|
3428
|
-
}
|
|
3429
|
-
async onModuleDestroy() {
|
|
3430
|
-
if (this.worker) {
|
|
3431
|
-
await this.worker.close();
|
|
3432
|
-
this.worker = null;
|
|
3433
|
-
}
|
|
3434
|
-
}
|
|
3435
|
-
/**
|
|
3436
|
-
* Process one BullMQ job. Returns the handler output (stored by BullMQ as
|
|
3437
|
-
* the job return value AND written to `job_run.output`). Throws on handler
|
|
3438
|
-
* failure so BullMQ applies the retry policy.
|
|
3439
|
-
*/
|
|
3440
|
-
async process(job) {
|
|
3441
|
-
const { runId } = job.data;
|
|
3442
|
-
const [row] = await this.db.select().from(jobRuns).where(eq7(jobRuns.id, runId)).limit(1);
|
|
3443
|
-
if (!row) {
|
|
3444
|
-
this.logger.warn(`process: job_run ${runId} not found; skipping`);
|
|
3445
|
-
return {};
|
|
3446
|
-
}
|
|
3447
|
-
const run = row;
|
|
3448
|
-
if (run.status === "canceled") {
|
|
3449
|
-
return {};
|
|
3450
|
-
}
|
|
3451
|
-
const registryEntry = JOB_HANDLER_REGISTRY.get(run.jobType);
|
|
3452
|
-
if (!registryEntry) {
|
|
3453
|
-
throw new Error(
|
|
3454
|
-
`No handler registered for jobType='${run.jobType}' (run ${run.id})`
|
|
3455
|
-
);
|
|
3456
|
-
}
|
|
3457
|
-
await this.db.update(jobRuns).set({
|
|
3458
|
-
status: "running",
|
|
3459
|
-
claimedAt: /* @__PURE__ */ new Date(),
|
|
3460
|
-
startedAt: /* @__PURE__ */ new Date(),
|
|
3461
|
-
attempts: job.attemptsMade + 1,
|
|
3462
|
-
updatedAt: /* @__PURE__ */ new Date()
|
|
3463
|
-
}).where(eq7(jobRuns.id, run.id));
|
|
3464
|
-
const HandlerClass = registryEntry.handlerClass;
|
|
3465
|
-
const handler = this.moduleRef.get(
|
|
3466
|
-
HandlerClass,
|
|
3467
|
-
{ strict: false }
|
|
3468
|
-
);
|
|
3469
|
-
const ctx = {
|
|
3470
|
-
input: run.input,
|
|
3471
|
-
run,
|
|
3472
|
-
step: this.makeStepFn(run),
|
|
3473
|
-
spawnChild: this.makeSpawnFn(run),
|
|
3474
|
-
logger: new Logger7(`JobRun:${run.id}`)
|
|
3475
|
-
};
|
|
3476
|
-
const output = await handler.run(ctx);
|
|
3477
|
-
await this.db.update(jobRuns).set({
|
|
3478
|
-
status: "completed",
|
|
3479
|
-
output: output ?? {},
|
|
3480
|
-
finishedAt: /* @__PURE__ */ new Date(),
|
|
3481
|
-
updatedAt: /* @__PURE__ */ new Date()
|
|
3482
|
-
}).where(eq7(jobRuns.id, run.id));
|
|
3483
|
-
return output ?? {};
|
|
3484
|
-
}
|
|
3485
|
-
async markFailed(runId, err, finalAttempts) {
|
|
3486
|
-
const [row] = await this.db.select().from(jobRuns).where(eq7(jobRuns.id, runId)).limit(1);
|
|
3487
|
-
if (!row) return;
|
|
3488
|
-
const run = row;
|
|
3489
|
-
await this.db.update(jobRuns).set({
|
|
3490
|
-
status: "failed",
|
|
3491
|
-
attempts: finalAttempts,
|
|
3492
|
-
finishedAt: /* @__PURE__ */ new Date(),
|
|
3493
|
-
error: serialiseError3(err, finalAttempts, false),
|
|
3494
|
-
updatedAt: /* @__PURE__ */ new Date()
|
|
3495
|
-
}).where(eq7(jobRuns.id, runId));
|
|
3496
|
-
if (run.parentClosePolicy === "terminate") {
|
|
3497
|
-
try {
|
|
3498
|
-
await this.orchestrator.cancel(run.id, {
|
|
3499
|
-
cascade: true,
|
|
3500
|
-
reason: "parent-failed",
|
|
3501
|
-
tenantId: run.tenantId
|
|
3502
|
-
});
|
|
3503
|
-
} catch (cascadeErr) {
|
|
3504
|
-
this.logger.warn(
|
|
3505
|
-
`cascade on failed run ${run.id}: ${cascadeErr.message}`
|
|
3506
|
-
);
|
|
3507
|
-
}
|
|
3508
|
-
}
|
|
3509
|
-
}
|
|
3510
|
-
// ── ctx.step / ctx.spawnChild (mirror JobWorker) ──────────────────────────
|
|
3511
|
-
makeStepFn(run) {
|
|
3512
|
-
return async (stepId, fn, _opts) => {
|
|
3513
|
-
void _opts;
|
|
3514
|
-
const existing = await this.stepService.findStep(run.id, stepId);
|
|
3515
|
-
if (existing?.status === "completed") {
|
|
3516
|
-
return existing.output;
|
|
3517
|
-
}
|
|
3518
|
-
const nextAttempts = (existing?.attempts ?? 0) + 1;
|
|
3519
|
-
const seq = nextAttempts;
|
|
3520
|
-
await this.stepService.recordStep({
|
|
3521
|
-
jobRunId: run.id,
|
|
3522
|
-
stepId,
|
|
3523
|
-
kind: "task",
|
|
3524
|
-
seq,
|
|
3525
|
-
status: "running",
|
|
3526
|
-
startedAt: /* @__PURE__ */ new Date(),
|
|
3527
|
-
attempts: nextAttempts
|
|
3528
|
-
});
|
|
3529
|
-
try {
|
|
3530
|
-
const output = await fn();
|
|
3531
|
-
await this.stepService.recordStep({
|
|
3532
|
-
jobRunId: run.id,
|
|
3533
|
-
stepId,
|
|
3534
|
-
kind: "task",
|
|
3535
|
-
seq,
|
|
3536
|
-
status: "completed",
|
|
3537
|
-
output,
|
|
3538
|
-
finishedAt: /* @__PURE__ */ new Date(),
|
|
3539
|
-
attempts: nextAttempts
|
|
3540
|
-
});
|
|
3541
|
-
return output;
|
|
3542
|
-
} catch (err) {
|
|
3543
|
-
await this.stepService.recordStep({
|
|
3544
|
-
jobRunId: run.id,
|
|
3545
|
-
stepId,
|
|
3546
|
-
kind: "task",
|
|
3547
|
-
seq,
|
|
3548
|
-
status: "failed",
|
|
3549
|
-
error: serialiseError3(err, nextAttempts, false),
|
|
3550
|
-
finishedAt: /* @__PURE__ */ new Date(),
|
|
3551
|
-
attempts: nextAttempts
|
|
3552
|
-
});
|
|
3553
|
-
throw err;
|
|
3554
|
-
}
|
|
3555
|
-
};
|
|
3556
|
-
}
|
|
3557
|
-
makeSpawnFn(run) {
|
|
3558
|
-
return async (type, input, opts) => {
|
|
3559
|
-
return this.orchestrator.start(type, input, {
|
|
3560
|
-
parentRunId: run.id,
|
|
3561
|
-
parentClosePolicy: opts?.closePolicy,
|
|
3562
|
-
runAt: opts?.runAt,
|
|
3563
|
-
priority: opts?.priority,
|
|
3564
|
-
tags: opts?.tags,
|
|
3565
|
-
triggerSource: "parent",
|
|
3566
|
-
triggerRef: run.id,
|
|
3567
|
-
tenantId: run.tenantId
|
|
3568
|
-
});
|
|
3569
|
-
};
|
|
3570
|
-
}
|
|
3571
|
-
};
|
|
3572
|
-
|
|
3573
3147
|
// runtime/subsystems/jobs/job-worker.module.ts
|
|
3574
3148
|
var DEFAULT_SHUTDOWN_TIMEOUT_MS2 = 3e4;
|
|
3575
3149
|
var JOB_WORKER_MODULE_OPTIONS = /* @__PURE__ */ Symbol("JOB_WORKER_MODULE_OPTIONS");
|
|
@@ -3592,7 +3166,7 @@ var JobWorkerOrchestrator = class {
|
|
|
3592
3166
|
moduleRef;
|
|
3593
3167
|
bullConnection;
|
|
3594
3168
|
bullConfig;
|
|
3595
|
-
logger = new
|
|
3169
|
+
logger = new Logger6(JobWorkerOrchestrator.name);
|
|
3596
3170
|
workers = [];
|
|
3597
3171
|
// ============================================================================
|
|
3598
3172
|
// Lifecycle
|
|
@@ -3622,7 +3196,7 @@ var JobWorkerOrchestrator = class {
|
|
|
3622
3196
|
concurrency: def.concurrency,
|
|
3623
3197
|
shutdownTimeoutMs: this.options.shutdownTimeoutMs ?? DEFAULT_SHUTDOWN_TIMEOUT_MS2
|
|
3624
3198
|
};
|
|
3625
|
-
const worker = this.options.workerFactory ? this.options.workerFactory(workerOptions) : backend === "bullmq" ? this.spawnBullMQWorker(poolName, def.queue, def.concurrency, poolConfig) : this.spawnWorker(workerOptions);
|
|
3199
|
+
const worker = this.options.workerFactory ? this.options.workerFactory(workerOptions) : backend === "bullmq" ? await this.spawnBullMQWorker(poolName, def.queue, def.concurrency, poolConfig) : this.spawnWorker(workerOptions);
|
|
3626
3200
|
await worker.onModuleInit();
|
|
3627
3201
|
this.workers.push(worker);
|
|
3628
3202
|
this.logger.log(
|
|
@@ -3722,7 +3296,15 @@ var JobWorkerOrchestrator = class {
|
|
|
3722
3296
|
* orchestrator's `dispatch` via `resolvePoolQueueName(pool, …)` so producer
|
|
3723
3297
|
* and consumer agree.
|
|
3724
3298
|
*/
|
|
3725
|
-
|
|
3299
|
+
/**
|
|
3300
|
+
* #6 — async + dynamic-import. The `job-worker.bullmq-backend.ts` file is
|
|
3301
|
+
* filtered out of the vendor set for drizzle/memory installs (no `bullmq`
|
|
3302
|
+
* peer dep needed). The non-literal import specifier makes TS treat the
|
|
3303
|
+
* module as `any` so the consumer's tsc never tries to resolve an absent
|
|
3304
|
+
* file. This method is only entered when `backend === 'bullmq'` — at which
|
|
3305
|
+
* point the file IS vendored.
|
|
3306
|
+
*/
|
|
3307
|
+
async spawnBullMQWorker(pool, _queueAlias, concurrency, poolConfig) {
|
|
3726
3308
|
if (!this.db) {
|
|
3727
3309
|
throw new Error(
|
|
3728
3310
|
`JobWorkerModule: BullMQ worker spawning requires the Drizzle client (no DRIZZLE provider available) \u2014 job_run remains the source of truth.`
|
|
@@ -3739,7 +3321,9 @@ var JobWorkerOrchestrator = class {
|
|
|
3739
3321
|
);
|
|
3740
3322
|
}
|
|
3741
3323
|
const queueName = resolvePoolQueueName(pool, this.bullConfig, poolConfig);
|
|
3742
|
-
|
|
3324
|
+
const specifier = "./job-worker.bullmq-backend";
|
|
3325
|
+
const mod = await import(specifier);
|
|
3326
|
+
return new mod.BullMQJobWorker(
|
|
3743
3327
|
this.db,
|
|
3744
3328
|
this.orchestrator,
|
|
3745
3329
|
this.stepService,
|
|
@@ -3754,17 +3338,17 @@ var JobWorkerOrchestrator = class {
|
|
|
3754
3338
|
}
|
|
3755
3339
|
};
|
|
3756
3340
|
JobWorkerOrchestrator = __decorateClass([
|
|
3757
|
-
|
|
3758
|
-
__decorateParam(0,
|
|
3759
|
-
__decorateParam(1,
|
|
3760
|
-
__decorateParam(2,
|
|
3761
|
-
__decorateParam(3,
|
|
3762
|
-
__decorateParam(4,
|
|
3763
|
-
__decorateParam(4,
|
|
3764
|
-
__decorateParam(6,
|
|
3765
|
-
__decorateParam(6,
|
|
3766
|
-
__decorateParam(7,
|
|
3767
|
-
__decorateParam(7,
|
|
3341
|
+
Injectable12(),
|
|
3342
|
+
__decorateParam(0, Inject11(JOB_ORCHESTRATOR)),
|
|
3343
|
+
__decorateParam(1, Inject11(JOB_RUN_SERVICE)),
|
|
3344
|
+
__decorateParam(2, Inject11(JOB_STEP_SERVICE)),
|
|
3345
|
+
__decorateParam(3, Inject11(JOB_WORKER_MODULE_OPTIONS)),
|
|
3346
|
+
__decorateParam(4, Optional6()),
|
|
3347
|
+
__decorateParam(4, Inject11(DRIZZLE)),
|
|
3348
|
+
__decorateParam(6, Optional6()),
|
|
3349
|
+
__decorateParam(6, Inject11(BULLMQ_CONNECTION)),
|
|
3350
|
+
__decorateParam(7, Optional6()),
|
|
3351
|
+
__decorateParam(7, Inject11(BULLMQ_RESOLVED_CONFIG))
|
|
3768
3352
|
], JobWorkerOrchestrator);
|
|
3769
3353
|
var JobWorkerModule = class {
|
|
3770
3354
|
static forRoot(opts) {
|
|
@@ -3864,8 +3448,8 @@ var BridgeModule = class {
|
|
|
3864
3448
|
};
|
|
3865
3449
|
BridgeModule = __decorateClass([
|
|
3866
3450
|
Module3({}),
|
|
3867
|
-
__decorateParam(0,
|
|
3868
|
-
__decorateParam(0,
|
|
3451
|
+
__decorateParam(0, Optional7()),
|
|
3452
|
+
__decorateParam(0, Inject12(JOB_WORKER_MODULE_OPTIONS))
|
|
3869
3453
|
], BridgeModule);
|
|
3870
3454
|
export {
|
|
3871
3455
|
BRIDGE_DELIVERY_JOB_TYPE,
|