@cosmicdrift/kumiko-framework 0.383.1 → 0.385.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agents/examples/custom-fields-basic.md +13 -3
- package/agents/examples/designer-demo.md +6 -2
- package/agents/examples/i18n.md +36 -1
- package/agents/features.md +4 -0
- package/dist/api/auth-rate-limiters.d.ts +8 -0
- package/dist/api/auth-rate-limiters.js +105 -0
- package/dist/api/auth-routes-context.d.ts +34 -0
- package/dist/api/auth-routes-context.js +192 -0
- package/dist/api/auth-routes-invite.d.ts +2 -0
- package/dist/api/auth-routes-invite.js +171 -0
- package/dist/api/auth-routes-login.d.ts +2 -0
- package/dist/api/auth-routes-login.js +103 -0
- package/dist/api/auth-routes-mfa.d.ts +4 -0
- package/dist/api/auth-routes-mfa.js +211 -0
- package/dist/api/auth-routes-session.d.ts +2 -0
- package/dist/api/auth-routes-session.js +144 -0
- package/dist/api/auth-routes-shared.d.ts +33 -0
- package/dist/api/auth-routes-shared.js +81 -0
- package/dist/api/auth-routes-signup.d.ts +2 -0
- package/dist/api/auth-routes-signup.js +75 -0
- package/dist/api/auth-routes-token-flows.d.ts +4 -0
- package/dist/api/auth-routes-token-flows.js +91 -0
- package/dist/api/auth-routes-types.d.ts +76 -0
- package/dist/api/auth-routes-types.js +1 -0
- package/dist/api/auth-routes.d.ts +5 -85
- package/dist/api/auth-routes.js +20 -1095
- package/dist/config/index.d.ts +1 -1
- package/dist/crypto/pii-ciphertext-format.js +1 -1
- package/dist/db/blind-index-cleanup.js +1 -1
- package/dist/db/dialect.js +1 -1
- package/dist/db/distinct-on.d.ts +9 -0
- package/dist/db/distinct-on.js +28 -0
- package/dist/db/event-store-executor-write.js +2 -2
- package/dist/db/migrate-runner.js +4 -4
- package/dist/db/pg-error.js +0 -12
- package/dist/db/queries/event-consumer.js +3 -3
- package/dist/db/queries/postgres-array-literal.js +2 -2
- package/dist/db/query.js +48 -20
- package/dist/db/tenant-db.js +1 -1
- package/dist/engine/boot-validator/i18n-keys.d.ts +2 -0
- package/dist/engine/boot-validator/i18n-keys.js +15 -1
- package/dist/engine/boot-validator/index.d.ts +4 -0
- package/dist/engine/boot-validator/index.js +3 -1
- package/dist/engine/boot-validator/pii-retention.js +6 -7
- package/dist/engine/boot-validator/screens-actions.d.ts +67 -0
- package/dist/engine/boot-validator/screens-actions.js +488 -0
- package/dist/engine/boot-validator/screens-dashboard.d.ts +15 -0
- package/dist/engine/boot-validator/screens-dashboard.js +467 -0
- package/dist/engine/boot-validator/screens-forms.d.ts +24 -0
- package/dist/engine/boot-validator/screens-forms.js +516 -0
- package/dist/engine/boot-validator/screens-lists.d.ts +47 -0
- package/dist/engine/boot-validator/screens-lists.js +319 -0
- package/dist/engine/boot-validator/screens.d.ts +4 -21
- package/dist/engine/boot-validator/screens.js +26 -1799
- package/dist/engine/build-app-schema.js +1 -1
- package/dist/engine/factories.d.ts +3 -3
- package/dist/engine/steps/read-find-one.js +1 -1
- package/dist/engine/types/index.d.ts +1 -1
- package/dist/event-store/provenance-append.js +1 -1
- package/dist/files/file-ref-entity.d.ts +6 -126
- package/dist/files/file-ref-table.d.ts +6 -126
- package/dist/files/file-routes.js +39 -16
- package/dist/i18n/required-surface-keys.d.ts +22 -2
- package/dist/i18n/required-surface-keys.js +91 -3
- package/dist/jobs/job-runner-helpers.d.ts +45 -0
- package/dist/jobs/job-runner-helpers.js +281 -0
- package/dist/jobs/job-runner-schedulers.d.ts +23 -0
- package/dist/jobs/job-runner-schedulers.js +106 -0
- package/dist/jobs/job-runner-types.d.ts +65 -0
- package/dist/jobs/job-runner-types.js +2 -0
- package/dist/jobs/job-runner.d.ts +4 -75
- package/dist/jobs/job-runner.js +7 -384
- package/dist/pipeline/dispatch-shared.js +2 -2
- package/dist/pipeline/event-dispatcher-admin.d.ts +15 -8
- package/dist/pipeline/event-dispatcher-admin.js +28 -7
- package/dist/pipeline/event-dispatcher.d.ts +2 -2
- package/dist/pipeline/event-dispatcher.js +3 -4
- package/dist/pipeline/index.d.ts +2 -2
- package/dist/pipeline/index.js +1 -1
- package/dist/schema-cli.js +5 -0
- package/package.json +3 -3
- package/src/changes.json +39 -0
|
@@ -0,0 +1,281 @@
|
|
|
1
|
+
/// <reference types="temporal-polyfill/global" preserve="true" />
|
|
2
|
+
import { isPlainObject } from "@cosmicdrift/kumiko-types/type-guards";
|
|
3
|
+
import { requestContext } from "../api/request-context.js";
|
|
4
|
+
import { InternalError } from "../errors/index.js";
|
|
5
|
+
import { isKumikoError } from "../errors/kumiko-error.js";
|
|
6
|
+
import { createFallbackLogger, scrubLogText } from "../logging/index.js";
|
|
7
|
+
import { emitJobQueueDepth } from "../observability/index.js";
|
|
8
|
+
import { TENANT_TIMEZONE_CONFIG_KEY } from "../pipeline/tenant-timezone-config-key.js";
|
|
9
|
+
import { isPersonalDataGated } from "../pipeline/write-origin.js";
|
|
10
|
+
import { createTzContext, isValidIanaTimeZone } from "../time/index.js";
|
|
11
|
+
// Shape-checked after the BullMQ/Redis JSON round-trip. Unlike `_writeOrigin`
|
|
12
|
+
// (a security-relevant escape hatch that must fail the run closed), a
|
|
13
|
+
// malformed `_triggerEvent` only costs an idempotency-key convenience — the
|
|
14
|
+
// run proceeds without it rather than failing over what would be a framework
|
|
15
|
+
// bug, not untrusted input.
|
|
16
|
+
export function isJobTriggerEvent(value) {
|
|
17
|
+
if (!isPlainObject(value))
|
|
18
|
+
return false;
|
|
19
|
+
if (typeof value["id"] !== "string")
|
|
20
|
+
return false;
|
|
21
|
+
const headers = value["headers"];
|
|
22
|
+
if (typeof headers !== "object" || headers === null)
|
|
23
|
+
return false;
|
|
24
|
+
return Object.values(headers).every((v) => typeof v === "string" || typeof v === "number" || typeof v === "boolean");
|
|
25
|
+
}
|
|
26
|
+
// A payload's own `_writeOrigin` is never trusted; only the ambient gated origin is stamped.
|
|
27
|
+
export function stampGatedWriteOrigin(data) {
|
|
28
|
+
delete data["_writeOrigin"];
|
|
29
|
+
const origin = requestContext.get()?.writeOrigin;
|
|
30
|
+
if (origin && isPersonalDataGated(origin)) {
|
|
31
|
+
data["_writeOrigin"] = origin;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
export function createJobLogger(logs, baseContext = {}) {
|
|
35
|
+
function push(level, msg, data) {
|
|
36
|
+
const merged = { ...baseContext, ...data };
|
|
37
|
+
const message = Object.keys(merged).length > 0 ? `${msg} ${JSON.stringify(merged)}` : msg;
|
|
38
|
+
logs.push({ level, message, timestamp: Temporal.Now.instant() });
|
|
39
|
+
}
|
|
40
|
+
const logger = {
|
|
41
|
+
info(msg, data) {
|
|
42
|
+
push("info", msg, data);
|
|
43
|
+
},
|
|
44
|
+
warn(msg, data) {
|
|
45
|
+
push("warn", msg, data);
|
|
46
|
+
},
|
|
47
|
+
error(msg, data) {
|
|
48
|
+
push("error", msg, data);
|
|
49
|
+
},
|
|
50
|
+
// No-op by design, not missing coverage: JobLogEntry.level (and the
|
|
51
|
+
// persisted run-completed/-failed event schema in bundled-features/
|
|
52
|
+
// jobs/events.ts) only has info|warn|error — a job's debug() calls have
|
|
53
|
+
// nowhere durable to land.
|
|
54
|
+
debug() { },
|
|
55
|
+
child(context) {
|
|
56
|
+
return createJobLogger(logs, { ...baseContext, ...context });
|
|
57
|
+
},
|
|
58
|
+
};
|
|
59
|
+
return logger;
|
|
60
|
+
}
|
|
61
|
+
// Single source of the stored subject string: the writer (jobSubjectKey) and
|
|
62
|
+
// the tenant failures query both go through it. Matching is exact — field names
|
|
63
|
+
// and JSON types must equal what the writer stored (`"1"` is not `1`).
|
|
64
|
+
export function serializeJobSubject(subject) {
|
|
65
|
+
return JSON.stringify(Object.keys(subject)
|
|
66
|
+
.sort()
|
|
67
|
+
.map((field) => [field, subject[field] ?? null]));
|
|
68
|
+
}
|
|
69
|
+
// Stable identity for the declared payload fields: sorted keys, so two runs
|
|
70
|
+
// with the same subject values produce the same string and the later one
|
|
71
|
+
// replaces the earlier record. Non-primitives would serialize into something
|
|
72
|
+
// no caller can reconstruct for a lookup, so they fail the run loudly.
|
|
73
|
+
export function jobSubjectKey(jobName, payload, fields, declaration) {
|
|
74
|
+
if (fields === undefined || fields.length === 0)
|
|
75
|
+
return null;
|
|
76
|
+
const subject = {};
|
|
77
|
+
for (const field of fields) {
|
|
78
|
+
const value = payload[field] ?? null;
|
|
79
|
+
if (value !== null && typeof value === "object") {
|
|
80
|
+
throw new Error(`Job "${jobName}": ${declaration}.subjectFields["${field}"] must be a primitive, got ${typeof value}`);
|
|
81
|
+
}
|
|
82
|
+
subject[field] = value;
|
|
83
|
+
}
|
|
84
|
+
return serializeJobSubject(subject);
|
|
85
|
+
}
|
|
86
|
+
export async function resolveJobTz(config, context) {
|
|
87
|
+
const tenantTz = config ? await config(TENANT_TIMEZONE_CONFIG_KEY) : undefined;
|
|
88
|
+
return createTzContext({
|
|
89
|
+
...(context.geoTzProvider !== undefined && { geoTz: context.geoTzProvider }),
|
|
90
|
+
...(context.clock !== undefined && { clock: context.clock }),
|
|
91
|
+
tenant: typeof tenantTz === "string" && isValidIanaTimeZone(tenantTz) ? tenantTz : "UTC",
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
// Jobs that never touch ctx.db run fine on a runner built without one; a job
|
|
95
|
+
// that does gets a named error instead of a bare TypeError on undefined.
|
|
96
|
+
export function missingDbProxy(jobName) {
|
|
97
|
+
// @cast-boundary db-operator — every access throws, so no TenantDb member is ever returned
|
|
98
|
+
return new Proxy({}, {
|
|
99
|
+
get() {
|
|
100
|
+
throw new InternalError({
|
|
101
|
+
message: `[jobs] job runner constructed without a db connection — ctx.db is unavailable for "${jobName}"`,
|
|
102
|
+
});
|
|
103
|
+
},
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
export function createCrossTenantReads(systemModeDb, reason, report) {
|
|
107
|
+
return {
|
|
108
|
+
selectMany(table, where, options) {
|
|
109
|
+
report("cross-tenant-read", reason);
|
|
110
|
+
return systemModeDb.selectMany(table, where, options);
|
|
111
|
+
},
|
|
112
|
+
fetchOne(table, where) {
|
|
113
|
+
report("cross-tenant-read", reason);
|
|
114
|
+
return systemModeDb.fetchOne(table, where);
|
|
115
|
+
},
|
|
116
|
+
count(table, where) {
|
|
117
|
+
report("cross-tenant-read", reason);
|
|
118
|
+
return systemModeDb.count(table, where);
|
|
119
|
+
},
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
export function priorAttemptsOf(data) {
|
|
123
|
+
const value = data["_priorAttempts"];
|
|
124
|
+
return typeof value === "number" && Number.isInteger(value) && value > 0 ? value : 0;
|
|
125
|
+
}
|
|
126
|
+
// Only a translation key ever travels to the tenant: the thrown error's own
|
|
127
|
+
// i18nKey when it carries one, otherwise the key declared at the job. The
|
|
128
|
+
// error message itself can echo provider or payload content and stays on the
|
|
129
|
+
// run row and in the run log.
|
|
130
|
+
export function tenantFailureMessageKey(err, declaration) {
|
|
131
|
+
// skip: job did not opt in — nothing tenant-visible to record
|
|
132
|
+
if (!declaration)
|
|
133
|
+
return null;
|
|
134
|
+
if (isKumikoError(err) && err.i18nKey.length > 0)
|
|
135
|
+
return err.i18nKey;
|
|
136
|
+
return declaration.messageKey;
|
|
137
|
+
}
|
|
138
|
+
// Serialized trace context lives under this key in the BullMQ job data.
|
|
139
|
+
// Leading underscore matches the existing internal-meta convention
|
|
140
|
+
// (_triggeredById, _tenantId, _payload).
|
|
141
|
+
export const TRACE_CONTEXT_KEY = "_traceContext";
|
|
142
|
+
export const DEFER_COUNT_KEY = "_deferCount";
|
|
143
|
+
// Separate from the maxPerTenant counter so tenant-limit deferrals do not eat into maxDefers.
|
|
144
|
+
export const HANDLER_DEFER_COUNT_KEY = "_handlerDeferCount";
|
|
145
|
+
export const DEFAULT_MAX_DEFERS = 100;
|
|
146
|
+
export const DEFER_DELAY_CAP_MS = 30_000;
|
|
147
|
+
export const DEFER_BACKOFF_MAX_DOUBLINGS = 5;
|
|
148
|
+
export function readTraceContext(data) {
|
|
149
|
+
const raw = data[TRACE_CONTEXT_KEY];
|
|
150
|
+
if (!raw || typeof raw !== "object")
|
|
151
|
+
return undefined;
|
|
152
|
+
const ctx = raw; // @cast-boundary engine-payload
|
|
153
|
+
if (!ctx.traceId || !ctx.spanId)
|
|
154
|
+
return undefined;
|
|
155
|
+
return { traceId: ctx.traceId, spanId: ctx.spanId };
|
|
156
|
+
}
|
|
157
|
+
export function captureTraceContext(tracer) {
|
|
158
|
+
const span = tracer.getActiveSpan();
|
|
159
|
+
if (!span?.traceId)
|
|
160
|
+
return undefined;
|
|
161
|
+
return { traceId: span.traceId, spanId: span.spanId };
|
|
162
|
+
}
|
|
163
|
+
const QUEUE_DEPTH_POLL_INTERVAL_MS = 15_000;
|
|
164
|
+
const QUEUE_DEPTH_WARN_INTERVAL_MS = 5 * 60_000;
|
|
165
|
+
// kumiko_job_queue_depth: only the lane's own consumer polls — the count is
|
|
166
|
+
// a global Redis-backed value (BullMQ, not per-process), so one reporter per
|
|
167
|
+
// lane is enough. Fires once immediately (metrics exist before the first
|
|
168
|
+
// poll tick) then on an interval; the caller stores + clears the handle.
|
|
169
|
+
export async function startQueueDepthPolling(queue, lane, meter) {
|
|
170
|
+
let lastWarnAt;
|
|
171
|
+
const pollQueueDepth = async () => {
|
|
172
|
+
try {
|
|
173
|
+
const counts = await queue.getJobCounts("waiting", "active", "delayed", "failed", "paused");
|
|
174
|
+
emitJobQueueDepth(meter, lane, counts);
|
|
175
|
+
}
|
|
176
|
+
catch (err) {
|
|
177
|
+
// Next poll retries; the warn is throttled so a Redis outage logs once per interval, not once per poll.
|
|
178
|
+
const nowMs = Date.now();
|
|
179
|
+
if (lastWarnAt === undefined || nowMs - lastWarnAt >= QUEUE_DEPTH_WARN_INTERVAL_MS) {
|
|
180
|
+
lastWarnAt = nowMs;
|
|
181
|
+
createFallbackLogger("job-runner").warn("queue depth poll failed", {
|
|
182
|
+
lane,
|
|
183
|
+
error: scrubLogText(err instanceof Error ? err.message : String(err)),
|
|
184
|
+
});
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
};
|
|
188
|
+
await pollQueueDepth();
|
|
189
|
+
return setInterval(() => void pollQueueDepth(), QUEUE_DEPTH_POLL_INTERVAL_MS);
|
|
190
|
+
}
|
|
191
|
+
export function parseRedisOpts(url) {
|
|
192
|
+
const parsed = new URL(url);
|
|
193
|
+
const result = {
|
|
194
|
+
host: parsed.hostname,
|
|
195
|
+
port: Number(parsed.port) || 6379,
|
|
196
|
+
};
|
|
197
|
+
if (parsed.pathname.length > 1) {
|
|
198
|
+
result.db = Number(parsed.pathname.slice(1));
|
|
199
|
+
}
|
|
200
|
+
return result;
|
|
201
|
+
}
|
|
202
|
+
// redisOpts carries no connectTimeout/retry cap, so an unreachable Redis
|
|
203
|
+
// would otherwise hang start() forever with no health endpoint to notice.
|
|
204
|
+
export const BOOT_REDIS_TIMEOUT_MS = 10_000;
|
|
205
|
+
// Shutdown must fit a k8s grace period even when Redis is down, so the
|
|
206
|
+
// readiness wait in stop() is capped well below the boot timeout.
|
|
207
|
+
export const STOP_QUEUE_READY_TIMEOUT_MS = 1_000;
|
|
208
|
+
// A gate without an explicit `timeout` may legitimately run long (migration-style
|
|
209
|
+
// checks), so it is never rejected; a hang just must not stay silent.
|
|
210
|
+
export const BOOT_GATE_SLOW_WARN_MS = 60_000;
|
|
211
|
+
export function timeoutReject(ms, message) {
|
|
212
|
+
let timer;
|
|
213
|
+
const promise = new Promise((_, reject) => {
|
|
214
|
+
timer = setTimeout(() => reject(new Error(message)), ms);
|
|
215
|
+
});
|
|
216
|
+
return {
|
|
217
|
+
promise,
|
|
218
|
+
cancel: () => {
|
|
219
|
+
if (timer !== undefined)
|
|
220
|
+
clearTimeout(timer);
|
|
221
|
+
},
|
|
222
|
+
};
|
|
223
|
+
}
|
|
224
|
+
// Default base delay when a job opts into backoff without an explicit
|
|
225
|
+
// delayMs. Without a `delay`, BullMQ's fixed/exponential strategies compute
|
|
226
|
+
// NaN/undefined, which is falsy — the job retries immediately instead of
|
|
227
|
+
// waiting.
|
|
228
|
+
const DEFAULT_JOB_BACKOFF_DELAY_MS = 1_000;
|
|
229
|
+
// Shared by every enqueue path (dispatch(), handleEvent(), cron, runOnBoot,
|
|
230
|
+
// perTenant wrapper/children, sequential re-enqueue) — a job with `retries`
|
|
231
|
+
// set must retry the same way regardless of how it got enqueued, or it
|
|
232
|
+
// fails for good on the very first error on whichever path skips this.
|
|
233
|
+
export function buildRetryBullOpts(jobDef, priorAttempts = 0) {
|
|
234
|
+
const opts = {};
|
|
235
|
+
if (jobDef.retries !== undefined)
|
|
236
|
+
opts.attempts = Math.max(1, jobDef.retries + 1 - priorAttempts);
|
|
237
|
+
if (jobDef.backoff) {
|
|
238
|
+
opts.backoff =
|
|
239
|
+
typeof jobDef.backoff === "string"
|
|
240
|
+
? { type: jobDef.backoff, delay: DEFAULT_JOB_BACKOFF_DELAY_MS }
|
|
241
|
+
: {
|
|
242
|
+
type: jobDef.backoff.type,
|
|
243
|
+
delay: jobDef.backoff.delayMs ?? DEFAULT_JOB_BACKOFF_DELAY_MS,
|
|
244
|
+
};
|
|
245
|
+
}
|
|
246
|
+
return opts;
|
|
247
|
+
}
|
|
248
|
+
// Equality filter on top-level payload fields for a job's trigger.where —
|
|
249
|
+
// lets N jobs share one broad event QN, partitioned by a payload
|
|
250
|
+
// discriminant instead of needing N distinct event types.
|
|
251
|
+
export function payloadMatchesTriggerFilter(where, payload) {
|
|
252
|
+
if (!where)
|
|
253
|
+
return true;
|
|
254
|
+
return Object.entries(where).every(([key, value]) => payload[key] === value);
|
|
255
|
+
}
|
|
256
|
+
// BullMQ sweeps queue-wide: any job finishing with keepJobs set can evict
|
|
257
|
+
// OTHER jobs in the same completed/failed zset (moveToFinished lua,
|
|
258
|
+
// removeJobsByMaxAge/removeJobsByMaxCount), not just the finishing job, and
|
|
259
|
+
// the sweep only runs lazily on a later finish into that set. Per-job
|
|
260
|
+
// retention is therefore meaningless on a shared queue — only queue-wide,
|
|
261
|
+
// age-only retention (no count) is safe: a count would evict perTenant
|
|
262
|
+
// children still inside their wrapper's retry window, and boot jobs. The
|
|
263
|
+
// completed age must stay above every perTenant wrapper's own worst-case
|
|
264
|
+
// retry window (see maxRetryWindowMs and the invariant check in
|
|
265
|
+
// createJobRunner below), because child dedup (perTenantChildJobId) relies
|
|
266
|
+
// on the children still existing in Redis when a wrapper retry re-derives
|
|
267
|
+
// their ids. The margin here is generous — queue wait time isn't bounded by
|
|
268
|
+
// backoff — to cover realistic windows. Audit trail lives in read_job_runs,
|
|
269
|
+
// not BullMQ (fw#3199).
|
|
270
|
+
export const COMPLETED_JOB_RETENTION_AGE_SEC = 86_400; // 24h
|
|
271
|
+
export const FAILED_JOB_RETENTION_AGE_SEC = 604_800; // 7d
|
|
272
|
+
// Total wall-clock time BullMQ can hold a perTenant wrapper across all its
|
|
273
|
+
// retries, consistent with buildRetryBullOpts's own backoff computation.
|
|
274
|
+
export function maxRetryWindowMs(jobDef) {
|
|
275
|
+
if (!jobDef.backoff || jobDef.retries === undefined)
|
|
276
|
+
return 0;
|
|
277
|
+
const delayMs = (typeof jobDef.backoff === "string" ? undefined : jobDef.backoff.delayMs) ??
|
|
278
|
+
DEFAULT_JOB_BACKOFF_DELAY_MS;
|
|
279
|
+
const type = typeof jobDef.backoff === "string" ? jobDef.backoff : jobDef.backoff.type;
|
|
280
|
+
return type === "exponential" ? delayMs * (2 ** jobDef.retries - 1) : jobDef.retries * delayMs;
|
|
281
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { Queue } from "bullmq";
|
|
2
|
+
import type { JobRunIn } from "../engine/types/index.ts";
|
|
3
|
+
export declare const DEFAULT_QUEUE_NAME_PREFIX = "kumiko-jobs";
|
|
4
|
+
export declare function queueNameFor(prefix: string, lane: JobRunIn): string;
|
|
5
|
+
export declare const ACTIVE_TENANT_IDS_QUERY_NAME = "tenant:query:active-tenant-ids";
|
|
6
|
+
/**
|
|
7
|
+
* BullMQ job ids are `repeat:<schedulerId>:<millis>`. Colons inside the
|
|
8
|
+
* scheduler id push the segment count to ≥5, which BullMQ's legacy heuristic
|
|
9
|
+
* treated as old repeatables — spawning a new scheduler entry every tick and
|
|
10
|
+
* leaking permanent `repeat:*` hashes (taskforcesh/bullmq#3828, fw#1603 /
|
|
11
|
+
* publicstatus Redis OOM). Strip `.` and `:` from the job QN.
|
|
12
|
+
*/
|
|
13
|
+
export declare function schedulerIdForJobName(jobName: string): string;
|
|
14
|
+
export declare function bootJobIdForJobName(jobName: string): string;
|
|
15
|
+
export declare const ORPHAN_SCHEDULER_GRACE_MS: number;
|
|
16
|
+
export declare function markSchedulersSeen(queue: Queue, schedulerIds: ReadonlySet<string>): Promise<void>;
|
|
17
|
+
type OrphanSchedulerLog = {
|
|
18
|
+
info(msg: string, data?: Record<string, unknown>): void;
|
|
19
|
+
warn(msg: string, data?: Record<string, unknown>): void;
|
|
20
|
+
};
|
|
21
|
+
export declare function removeOrphanJobSchedulers(queue: Queue, expectedSchedulerIds: ReadonlySet<string>, lane: JobRunIn, log: OrphanSchedulerLog): Promise<void>;
|
|
22
|
+
export declare function perTenantChildJobId(wrapperJobId: string, tenantId: string): string;
|
|
23
|
+
export {};
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
// Queue-name convention: <prefix>-<lane>. The prefix is fixed in prod
|
|
2
|
+
// ("kumiko-jobs") — it must match between enqueuers and consumers, and an
|
|
3
|
+
// accidental drift would silently drop jobs. Tests override via
|
|
4
|
+
// `queueNamePrefix` for per-run isolation (stale jobs from a prior run
|
|
5
|
+
// don't leak into a new test because the queue name includes a timestamp).
|
|
6
|
+
export const DEFAULT_QUEUE_NAME_PREFIX = "kumiko-jobs";
|
|
7
|
+
export function queueNameFor(prefix, lane) {
|
|
8
|
+
return `${prefix}-${lane}`;
|
|
9
|
+
}
|
|
10
|
+
// perTenant jobs need a source of truth for "which tenants are active" —
|
|
11
|
+
// see createJobRunner's defaultGetActiveTenantIds below. The tenant feature
|
|
12
|
+
// (bundled-features package) registers a query under this exact name when
|
|
13
|
+
// mounted. framework must not depend on bundled-features (wrong direction —
|
|
14
|
+
// bundled-features already depends on framework), so this is a name-only
|
|
15
|
+
// coupling: createJobRunner looks the handler up in the registry by string
|
|
16
|
+
// instead of importing it.
|
|
17
|
+
export const ACTIVE_TENANT_IDS_QUERY_NAME = "tenant:query:active-tenant-ids";
|
|
18
|
+
/**
|
|
19
|
+
* BullMQ job ids are `repeat:<schedulerId>:<millis>`. Colons inside the
|
|
20
|
+
* scheduler id push the segment count to ≥5, which BullMQ's legacy heuristic
|
|
21
|
+
* treated as old repeatables — spawning a new scheduler entry every tick and
|
|
22
|
+
* leaking permanent `repeat:*` hashes (taskforcesh/bullmq#3828, fw#1603 /
|
|
23
|
+
* publicstatus Redis OOM). Strip `.` and `:` from the job QN.
|
|
24
|
+
*/
|
|
25
|
+
export function schedulerIdForJobName(jobName) {
|
|
26
|
+
return `scheduler-${jobName.replace(/[.:]/g, "-")}`;
|
|
27
|
+
}
|
|
28
|
+
// Same colon-in-BullMQ-id hazard as schedulerIdForJobName (fw#1603/#1604) —
|
|
29
|
+
// a QN like "publicstatus:job:uptime-probe" must not leave ":" in the id.
|
|
30
|
+
export function bootJobIdForJobName(jobName) {
|
|
31
|
+
return `boot-${jobName.replace(/[.:]/g, "-")}`;
|
|
32
|
+
}
|
|
33
|
+
const SCHEDULER_PAGE_SIZE = 100;
|
|
34
|
+
// Rolling deploys run old and new pods against the same Redis for a while; an
|
|
35
|
+
// old runner restarting in that window must not delete the scheduler of a
|
|
36
|
+
// cron job the new version just added.
|
|
37
|
+
export const ORPHAN_SCHEDULER_GRACE_MS = 60 * 60 * 1000;
|
|
38
|
+
function schedulerSeenKey(queue) {
|
|
39
|
+
return queue.toKey("kumiko-scheduler-seen");
|
|
40
|
+
}
|
|
41
|
+
// "Seen" has to mean a live runner holds the scheduler, so runners re-stamp
|
|
42
|
+
// their own ids periodically; a single stamp at start() would let a runner
|
|
43
|
+
// restarting during a mixed-version window longer than the grace period delete
|
|
44
|
+
// a scheduler a healthy newer pod still owns.
|
|
45
|
+
export async function markSchedulersSeen(queue, schedulerIds) {
|
|
46
|
+
// skip: no cron scheduler on this lane, nothing to stamp
|
|
47
|
+
if (schedulerIds.size === 0)
|
|
48
|
+
return;
|
|
49
|
+
const stamp = String(Date.now());
|
|
50
|
+
const entries = Object.fromEntries([...schedulerIds].map((id) => [id, stamp]));
|
|
51
|
+
await (await queue.client).hset(schedulerSeenKey(queue), entries);
|
|
52
|
+
}
|
|
53
|
+
// A cron job dropped from the app (or an older scheduler-id format) leaves its
|
|
54
|
+
// scheduler in Redis, where it keeps firing jobs the worker no longer knows
|
|
55
|
+
// ("Unknown job"). Best-effort: a Redis hiccup here must not stop the start.
|
|
56
|
+
export async function removeOrphanJobSchedulers(queue, expectedSchedulerIds, lane, log) {
|
|
57
|
+
try {
|
|
58
|
+
// Collect first: removing while paging would shift the indices.
|
|
59
|
+
const existingKeys = [];
|
|
60
|
+
for (let start = 0;; start += SCHEDULER_PAGE_SIZE) {
|
|
61
|
+
const page = await queue.getJobSchedulers(start, start + SCHEDULER_PAGE_SIZE - 1);
|
|
62
|
+
for (const scheduler of page)
|
|
63
|
+
existingKeys.push(scheduler.key);
|
|
64
|
+
if (page.length < SCHEDULER_PAGE_SIZE)
|
|
65
|
+
break;
|
|
66
|
+
}
|
|
67
|
+
const client = await queue.client;
|
|
68
|
+
const seenAt = await client.hgetall(schedulerSeenKey(queue));
|
|
69
|
+
const now = Date.now();
|
|
70
|
+
for (const key of existingKeys) {
|
|
71
|
+
if (expectedSchedulerIds.has(key))
|
|
72
|
+
continue;
|
|
73
|
+
const lastSeen = Number(seenAt[key]);
|
|
74
|
+
if (key in seenAt && now - lastSeen < ORPHAN_SCHEDULER_GRACE_MS)
|
|
75
|
+
continue;
|
|
76
|
+
await queue.removeJobScheduler(key);
|
|
77
|
+
await client.hdel(schedulerSeenKey(queue), key);
|
|
78
|
+
log.info("removed orphan job scheduler", { schedulerId: key, lane });
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
catch (err) {
|
|
82
|
+
log.warn("orphan job scheduler cleanup failed", {
|
|
83
|
+
lane,
|
|
84
|
+
error: err instanceof Error ? err.message : String(err),
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
// Deterministic per-tenant child job id, derived from the perTenant
|
|
89
|
+
// wrapper's own BullMQ job id. A cron tick's wrapper id is
|
|
90
|
+
// `repeat:<schedulerId>:<millis>` (stable for that tick, including across
|
|
91
|
+
// BullMQ's own retry/stalled-job redelivery of the *same* job); a manual
|
|
92
|
+
// dispatch's wrapper id is BullMQ's auto-generated id, equally stable
|
|
93
|
+
// across retries of that job. Reusing it means a wrapper that runs twice
|
|
94
|
+
// for the same trigger (retry after failure, a Redis drop, an instance
|
|
95
|
+
// restarting mid-run) re-derives the *same* child ids, and BullMQ's
|
|
96
|
+
// existing-jobId add() no-ops the second batch instead of creating
|
|
97
|
+
// duplicates — safe only as long as the child job hash outlives the
|
|
98
|
+
// wrapper's retry window, enforced at construction against
|
|
99
|
+
// COMPLETED_JOB_RETENTION_AGE_SEC (see the invariant check in
|
|
100
|
+
// createJobRunner). Same colon-in-BullMQ-id hazard as schedulerIdForJobName
|
|
101
|
+
// (fw#1603/#1604) — the wrapper id already contains ":", so strip
|
|
102
|
+
// separators from both halves before joining instead of interpolating raw.
|
|
103
|
+
export function perTenantChildJobId(wrapperJobId, tenantId) {
|
|
104
|
+
const sanitize = (value) => value.replace(/[.:]/g, "-");
|
|
105
|
+
return `child-${sanitize(wrapperJobId)}-${sanitize(tenantId)}`;
|
|
106
|
+
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
/// <reference types="temporal-polyfill/global" preserve="true" />
|
|
2
|
+
import type { AppContext, DispatchWriteRef, JobRunIn, JobTriggerEvent, Registry, SessionUser, TenantId } from "../engine/types/index.ts";
|
|
3
|
+
export type JobLogEntry = {
|
|
4
|
+
level: "info" | "warn" | "error";
|
|
5
|
+
message: string;
|
|
6
|
+
timestamp: Temporal.Instant;
|
|
7
|
+
};
|
|
8
|
+
export type JobMeta = {
|
|
9
|
+
triggeredById?: string | undefined;
|
|
10
|
+
payload?: string | undefined;
|
|
11
|
+
attempt?: number | undefined;
|
|
12
|
+
priority?: number | undefined;
|
|
13
|
+
jobId?: string | undefined;
|
|
14
|
+
tenantVisibleRun?: {
|
|
15
|
+
readonly tenantId: string;
|
|
16
|
+
readonly subject: string | null;
|
|
17
|
+
} | undefined;
|
|
18
|
+
};
|
|
19
|
+
export type JobQueuedMeta = {
|
|
20
|
+
readonly tenantId: string;
|
|
21
|
+
readonly subject: string | null;
|
|
22
|
+
};
|
|
23
|
+
export type JobDroppedMeta = {
|
|
24
|
+
readonly tenantId: string;
|
|
25
|
+
};
|
|
26
|
+
export type JobOutcomeMeta = {
|
|
27
|
+
readonly tenantId: string;
|
|
28
|
+
readonly finalAttempt: boolean;
|
|
29
|
+
readonly tenantVisible?: {
|
|
30
|
+
readonly subject: string | null;
|
|
31
|
+
readonly messageKey: string | null;
|
|
32
|
+
} | undefined;
|
|
33
|
+
readonly tenantVisibleRun?: {
|
|
34
|
+
readonly subject: string | null;
|
|
35
|
+
} | undefined;
|
|
36
|
+
};
|
|
37
|
+
export type JobSubjectValue = string | number | boolean | null;
|
|
38
|
+
export type JobRunner = {
|
|
39
|
+
start(): Promise<void>;
|
|
40
|
+
stop(): Promise<void>;
|
|
41
|
+
dispatch(jobName: string, payload?: Record<string, unknown>, meta?: JobMeta): Promise<string>;
|
|
42
|
+
handleEvent(eventName: string, payload: Record<string, unknown>, user?: SessionUser, triggerEvent?: JobTriggerEvent): Promise<void>;
|
|
43
|
+
attachDispatcher(ref: DispatchWriteRef): void;
|
|
44
|
+
countPendingJobs(): Promise<number>;
|
|
45
|
+
};
|
|
46
|
+
export type JobRunnerOptions = {
|
|
47
|
+
registry: Registry;
|
|
48
|
+
context: AppContext;
|
|
49
|
+
redisUrl: string;
|
|
50
|
+
consumerLane?: JobRunIn | undefined;
|
|
51
|
+
queueNamePrefix?: string | undefined;
|
|
52
|
+
bootRedisTimeoutMs?: number | undefined;
|
|
53
|
+
schedulerSeenRefreshMs?: number | undefined;
|
|
54
|
+
jobRetention?: {
|
|
55
|
+
completedAgeSec?: number | undefined;
|
|
56
|
+
failedAgeSec?: number | undefined;
|
|
57
|
+
} | undefined;
|
|
58
|
+
getActiveTenantIds?: () => Promise<TenantId[]>;
|
|
59
|
+
onJobStart?: (jobName: string, jobId: string, meta: JobMeta) => void;
|
|
60
|
+
onJobComplete?: (jobName: string, jobId: string, duration: number, logs: JobLogEntry[], outcome?: JobOutcomeMeta) => void;
|
|
61
|
+
onJobFailed?: (jobName: string, jobId: string, error: string, logs: JobLogEntry[], outcome?: JobOutcomeMeta) => void;
|
|
62
|
+
onJobDeferred?: (jobName: string, jobId: string, meta: JobMeta) => void;
|
|
63
|
+
onJobQueued?: (jobName: string, jobId: string, meta: JobQueuedMeta) => void;
|
|
64
|
+
onJobDropped?: (jobName: string, jobId: string, meta: JobDroppedMeta) => void;
|
|
65
|
+
};
|
|
@@ -1,77 +1,6 @@
|
|
|
1
1
|
/// <reference types="temporal-polyfill/global" preserve="true" />
|
|
2
|
-
import
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
* treated as old repeatables — spawning a new scheduler entry every tick and
|
|
7
|
-
* leaking permanent `repeat:*` hashes (taskforcesh/bullmq#3828, fw#1603 /
|
|
8
|
-
* publicstatus Redis OOM). Strip `.` and `:` from the job QN.
|
|
9
|
-
*/
|
|
10
|
-
export declare function schedulerIdForJobName(jobName: string): string;
|
|
11
|
-
export declare function bootJobIdForJobName(jobName: string): string;
|
|
12
|
-
export declare const ORPHAN_SCHEDULER_GRACE_MS: number;
|
|
13
|
-
export type JobLogEntry = {
|
|
14
|
-
level: "info" | "warn" | "error";
|
|
15
|
-
message: string;
|
|
16
|
-
timestamp: Temporal.Instant;
|
|
17
|
-
};
|
|
18
|
-
export type JobMeta = {
|
|
19
|
-
triggeredById?: string | undefined;
|
|
20
|
-
payload?: string | undefined;
|
|
21
|
-
attempt?: number | undefined;
|
|
22
|
-
priority?: number | undefined;
|
|
23
|
-
jobId?: string | undefined;
|
|
24
|
-
tenantVisibleRun?: {
|
|
25
|
-
readonly tenantId: string;
|
|
26
|
-
readonly subject: string | null;
|
|
27
|
-
} | undefined;
|
|
28
|
-
};
|
|
29
|
-
export type JobQueuedMeta = {
|
|
30
|
-
readonly tenantId: string;
|
|
31
|
-
readonly subject: string | null;
|
|
32
|
-
};
|
|
33
|
-
export type JobDroppedMeta = {
|
|
34
|
-
readonly tenantId: string;
|
|
35
|
-
};
|
|
36
|
-
export type JobOutcomeMeta = {
|
|
37
|
-
readonly tenantId: string;
|
|
38
|
-
readonly finalAttempt: boolean;
|
|
39
|
-
readonly tenantVisible?: {
|
|
40
|
-
readonly subject: string | null;
|
|
41
|
-
readonly messageKey: string | null;
|
|
42
|
-
} | undefined;
|
|
43
|
-
readonly tenantVisibleRun?: {
|
|
44
|
-
readonly subject: string | null;
|
|
45
|
-
} | undefined;
|
|
46
|
-
};
|
|
47
|
-
export type JobSubjectValue = string | number | boolean | null;
|
|
48
|
-
export declare function serializeJobSubject(subject: Readonly<Record<string, JobSubjectValue>>): string;
|
|
49
|
-
export type JobRunner = {
|
|
50
|
-
start(): Promise<void>;
|
|
51
|
-
stop(): Promise<void>;
|
|
52
|
-
dispatch(jobName: string, payload?: Record<string, unknown>, meta?: JobMeta): Promise<string>;
|
|
53
|
-
handleEvent(eventName: string, payload: Record<string, unknown>, user?: SessionUser, triggerEvent?: JobTriggerEvent): Promise<void>;
|
|
54
|
-
attachDispatcher(ref: DispatchWriteRef): void;
|
|
55
|
-
countPendingJobs(): Promise<number>;
|
|
56
|
-
};
|
|
57
|
-
export type JobRunnerOptions = {
|
|
58
|
-
registry: Registry;
|
|
59
|
-
context: AppContext;
|
|
60
|
-
redisUrl: string;
|
|
61
|
-
consumerLane?: JobRunIn | undefined;
|
|
62
|
-
queueNamePrefix?: string | undefined;
|
|
63
|
-
bootRedisTimeoutMs?: number | undefined;
|
|
64
|
-
schedulerSeenRefreshMs?: number | undefined;
|
|
65
|
-
jobRetention?: {
|
|
66
|
-
completedAgeSec?: number | undefined;
|
|
67
|
-
failedAgeSec?: number | undefined;
|
|
68
|
-
} | undefined;
|
|
69
|
-
getActiveTenantIds?: () => Promise<TenantId[]>;
|
|
70
|
-
onJobStart?: (jobName: string, jobId: string, meta: JobMeta) => void;
|
|
71
|
-
onJobComplete?: (jobName: string, jobId: string, duration: number, logs: JobLogEntry[], outcome?: JobOutcomeMeta) => void;
|
|
72
|
-
onJobFailed?: (jobName: string, jobId: string, error: string, logs: JobLogEntry[], outcome?: JobOutcomeMeta) => void;
|
|
73
|
-
onJobDeferred?: (jobName: string, jobId: string, meta: JobMeta) => void;
|
|
74
|
-
onJobQueued?: (jobName: string, jobId: string, meta: JobQueuedMeta) => void;
|
|
75
|
-
onJobDropped?: (jobName: string, jobId: string, meta: JobDroppedMeta) => void;
|
|
76
|
-
};
|
|
2
|
+
import type { JobRunner, JobRunnerOptions } from "./job-runner-types.ts";
|
|
3
|
+
export { serializeJobSubject } from "./job-runner-helpers.ts";
|
|
4
|
+
export { bootJobIdForJobName, ORPHAN_SCHEDULER_GRACE_MS, schedulerIdForJobName, } from "./job-runner-schedulers.ts";
|
|
5
|
+
export type { JobDroppedMeta, JobLogEntry, JobMeta, JobOutcomeMeta, JobQueuedMeta, JobRunner, JobRunnerOptions, JobSubjectValue, } from "./job-runner-types.ts";
|
|
77
6
|
export declare function createJobRunner(options: JobRunnerOptions): JobRunner;
|