@cosmicdrift/kumiko-framework 0.383.1 → 0.385.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/agents/examples/custom-fields-basic.md +13 -3
  2. package/agents/examples/designer-demo.md +6 -2
  3. package/agents/examples/i18n.md +36 -1
  4. package/agents/features.md +4 -0
  5. package/dist/api/auth-rate-limiters.d.ts +8 -0
  6. package/dist/api/auth-rate-limiters.js +105 -0
  7. package/dist/api/auth-routes-context.d.ts +34 -0
  8. package/dist/api/auth-routes-context.js +192 -0
  9. package/dist/api/auth-routes-invite.d.ts +2 -0
  10. package/dist/api/auth-routes-invite.js +171 -0
  11. package/dist/api/auth-routes-login.d.ts +2 -0
  12. package/dist/api/auth-routes-login.js +103 -0
  13. package/dist/api/auth-routes-mfa.d.ts +4 -0
  14. package/dist/api/auth-routes-mfa.js +211 -0
  15. package/dist/api/auth-routes-session.d.ts +2 -0
  16. package/dist/api/auth-routes-session.js +144 -0
  17. package/dist/api/auth-routes-shared.d.ts +33 -0
  18. package/dist/api/auth-routes-shared.js +81 -0
  19. package/dist/api/auth-routes-signup.d.ts +2 -0
  20. package/dist/api/auth-routes-signup.js +75 -0
  21. package/dist/api/auth-routes-token-flows.d.ts +4 -0
  22. package/dist/api/auth-routes-token-flows.js +91 -0
  23. package/dist/api/auth-routes-types.d.ts +76 -0
  24. package/dist/api/auth-routes-types.js +1 -0
  25. package/dist/api/auth-routes.d.ts +5 -85
  26. package/dist/api/auth-routes.js +20 -1095
  27. package/dist/config/index.d.ts +1 -1
  28. package/dist/crypto/pii-ciphertext-format.js +1 -1
  29. package/dist/db/blind-index-cleanup.js +1 -1
  30. package/dist/db/dialect.js +1 -1
  31. package/dist/db/distinct-on.d.ts +9 -0
  32. package/dist/db/distinct-on.js +28 -0
  33. package/dist/db/event-store-executor-write.js +2 -2
  34. package/dist/db/migrate-runner.js +4 -4
  35. package/dist/db/pg-error.js +0 -12
  36. package/dist/db/queries/event-consumer.js +3 -3
  37. package/dist/db/queries/postgres-array-literal.js +2 -2
  38. package/dist/db/query.js +48 -20
  39. package/dist/db/tenant-db.js +1 -1
  40. package/dist/engine/boot-validator/i18n-keys.d.ts +2 -0
  41. package/dist/engine/boot-validator/i18n-keys.js +15 -1
  42. package/dist/engine/boot-validator/index.d.ts +4 -0
  43. package/dist/engine/boot-validator/index.js +3 -1
  44. package/dist/engine/boot-validator/pii-retention.js +6 -7
  45. package/dist/engine/boot-validator/screens-actions.d.ts +67 -0
  46. package/dist/engine/boot-validator/screens-actions.js +488 -0
  47. package/dist/engine/boot-validator/screens-dashboard.d.ts +15 -0
  48. package/dist/engine/boot-validator/screens-dashboard.js +467 -0
  49. package/dist/engine/boot-validator/screens-forms.d.ts +24 -0
  50. package/dist/engine/boot-validator/screens-forms.js +516 -0
  51. package/dist/engine/boot-validator/screens-lists.d.ts +47 -0
  52. package/dist/engine/boot-validator/screens-lists.js +319 -0
  53. package/dist/engine/boot-validator/screens.d.ts +4 -21
  54. package/dist/engine/boot-validator/screens.js +26 -1799
  55. package/dist/engine/build-app-schema.js +1 -1
  56. package/dist/engine/factories.d.ts +3 -3
  57. package/dist/engine/steps/read-find-one.js +1 -1
  58. package/dist/engine/types/index.d.ts +1 -1
  59. package/dist/event-store/provenance-append.js +1 -1
  60. package/dist/files/file-ref-entity.d.ts +6 -126
  61. package/dist/files/file-ref-table.d.ts +6 -126
  62. package/dist/files/file-routes.js +39 -16
  63. package/dist/i18n/required-surface-keys.d.ts +22 -2
  64. package/dist/i18n/required-surface-keys.js +91 -3
  65. package/dist/jobs/job-runner-helpers.d.ts +45 -0
  66. package/dist/jobs/job-runner-helpers.js +281 -0
  67. package/dist/jobs/job-runner-schedulers.d.ts +23 -0
  68. package/dist/jobs/job-runner-schedulers.js +106 -0
  69. package/dist/jobs/job-runner-types.d.ts +65 -0
  70. package/dist/jobs/job-runner-types.js +2 -0
  71. package/dist/jobs/job-runner.d.ts +4 -75
  72. package/dist/jobs/job-runner.js +7 -384
  73. package/dist/pipeline/dispatch-shared.js +2 -2
  74. package/dist/pipeline/event-dispatcher-admin.d.ts +15 -8
  75. package/dist/pipeline/event-dispatcher-admin.js +28 -7
  76. package/dist/pipeline/event-dispatcher.d.ts +2 -2
  77. package/dist/pipeline/event-dispatcher.js +3 -4
  78. package/dist/pipeline/index.d.ts +2 -2
  79. package/dist/pipeline/index.js +1 -1
  80. package/dist/schema-cli.js +5 -0
  81. package/package.json +3 -3
  82. package/src/changes.json +39 -0
@@ -0,0 +1,281 @@
1
+ /// <reference types="temporal-polyfill/global" preserve="true" />
2
+ import { isPlainObject } from "@cosmicdrift/kumiko-types/type-guards";
3
+ import { requestContext } from "../api/request-context.js";
4
+ import { InternalError } from "../errors/index.js";
5
+ import { isKumikoError } from "../errors/kumiko-error.js";
6
+ import { createFallbackLogger, scrubLogText } from "../logging/index.js";
7
+ import { emitJobQueueDepth } from "../observability/index.js";
8
+ import { TENANT_TIMEZONE_CONFIG_KEY } from "../pipeline/tenant-timezone-config-key.js";
9
+ import { isPersonalDataGated } from "../pipeline/write-origin.js";
10
+ import { createTzContext, isValidIanaTimeZone } from "../time/index.js";
11
+ // Shape-checked after the BullMQ/Redis JSON round-trip. Unlike `_writeOrigin`
12
+ // (a security-relevant escape hatch that must fail the run closed), a
13
+ // malformed `_triggerEvent` only costs an idempotency-key convenience — the
14
+ // run proceeds without it rather than failing over what would be a framework
15
+ // bug, not untrusted input.
16
+ export function isJobTriggerEvent(value) {
17
+ if (!isPlainObject(value))
18
+ return false;
19
+ if (typeof value["id"] !== "string")
20
+ return false;
21
+ const headers = value["headers"];
22
+ if (typeof headers !== "object" || headers === null)
23
+ return false;
24
+ return Object.values(headers).every((v) => typeof v === "string" || typeof v === "number" || typeof v === "boolean");
25
+ }
26
+ // A payload's own `_writeOrigin` is never trusted; only the ambient gated origin is stamped.
27
+ export function stampGatedWriteOrigin(data) {
28
+ delete data["_writeOrigin"];
29
+ const origin = requestContext.get()?.writeOrigin;
30
+ if (origin && isPersonalDataGated(origin)) {
31
+ data["_writeOrigin"] = origin;
32
+ }
33
+ }
34
+ export function createJobLogger(logs, baseContext = {}) {
35
+ function push(level, msg, data) {
36
+ const merged = { ...baseContext, ...data };
37
+ const message = Object.keys(merged).length > 0 ? `${msg} ${JSON.stringify(merged)}` : msg;
38
+ logs.push({ level, message, timestamp: Temporal.Now.instant() });
39
+ }
40
+ const logger = {
41
+ info(msg, data) {
42
+ push("info", msg, data);
43
+ },
44
+ warn(msg, data) {
45
+ push("warn", msg, data);
46
+ },
47
+ error(msg, data) {
48
+ push("error", msg, data);
49
+ },
50
+ // No-op by design, not missing coverage: JobLogEntry.level (and the
51
+ // persisted run-completed/-failed event schema in bundled-features/
52
+ // jobs/events.ts) only has info|warn|error — a job's debug() calls have
53
+ // nowhere durable to land.
54
+ debug() { },
55
+ child(context) {
56
+ return createJobLogger(logs, { ...baseContext, ...context });
57
+ },
58
+ };
59
+ return logger;
60
+ }
61
+ // Single source of the stored subject string: the writer (jobSubjectKey) and
62
+ // the tenant failures query both go through it. Matching is exact — field names
63
+ // and JSON types must equal what the writer stored (`"1"` is not `1`).
64
+ export function serializeJobSubject(subject) {
65
+ return JSON.stringify(Object.keys(subject)
66
+ .sort()
67
+ .map((field) => [field, subject[field] ?? null]));
68
+ }
69
+ // Stable identity for the declared payload fields: sorted keys, so two runs
70
+ // with the same subject values produce the same string and the later one
71
+ // replaces the earlier record. Non-primitives would serialize into something
72
+ // no caller can reconstruct for a lookup, so they fail the run loudly.
73
+ export function jobSubjectKey(jobName, payload, fields, declaration) {
74
+ if (fields === undefined || fields.length === 0)
75
+ return null;
76
+ const subject = {};
77
+ for (const field of fields) {
78
+ const value = payload[field] ?? null;
79
+ if (value !== null && typeof value === "object") {
80
+ throw new Error(`Job "${jobName}": ${declaration}.subjectFields["${field}"] must be a primitive, got ${typeof value}`);
81
+ }
82
+ subject[field] = value;
83
+ }
84
+ return serializeJobSubject(subject);
85
+ }
86
+ export async function resolveJobTz(config, context) {
87
+ const tenantTz = config ? await config(TENANT_TIMEZONE_CONFIG_KEY) : undefined;
88
+ return createTzContext({
89
+ ...(context.geoTzProvider !== undefined && { geoTz: context.geoTzProvider }),
90
+ ...(context.clock !== undefined && { clock: context.clock }),
91
+ tenant: typeof tenantTz === "string" && isValidIanaTimeZone(tenantTz) ? tenantTz : "UTC",
92
+ });
93
+ }
94
+ // Jobs that never touch ctx.db run fine on a runner built without one; a job
95
+ // that does gets a named error instead of a bare TypeError on undefined.
96
+ export function missingDbProxy(jobName) {
97
+ // @cast-boundary db-operator — every access throws, so no TenantDb member is ever returned
98
+ return new Proxy({}, {
99
+ get() {
100
+ throw new InternalError({
101
+ message: `[jobs] job runner constructed without a db connection — ctx.db is unavailable for "${jobName}"`,
102
+ });
103
+ },
104
+ });
105
+ }
106
+ export function createCrossTenantReads(systemModeDb, reason, report) {
107
+ return {
108
+ selectMany(table, where, options) {
109
+ report("cross-tenant-read", reason);
110
+ return systemModeDb.selectMany(table, where, options);
111
+ },
112
+ fetchOne(table, where) {
113
+ report("cross-tenant-read", reason);
114
+ return systemModeDb.fetchOne(table, where);
115
+ },
116
+ count(table, where) {
117
+ report("cross-tenant-read", reason);
118
+ return systemModeDb.count(table, where);
119
+ },
120
+ };
121
+ }
122
+ export function priorAttemptsOf(data) {
123
+ const value = data["_priorAttempts"];
124
+ return typeof value === "number" && Number.isInteger(value) && value > 0 ? value : 0;
125
+ }
126
+ // Only a translation key ever travels to the tenant: the thrown error's own
127
+ // i18nKey when it carries one, otherwise the key declared at the job. The
128
+ // error message itself can echo provider or payload content and stays on the
129
+ // run row and in the run log.
130
+ export function tenantFailureMessageKey(err, declaration) {
131
+ // skip: job did not opt in — nothing tenant-visible to record
132
+ if (!declaration)
133
+ return null;
134
+ if (isKumikoError(err) && err.i18nKey.length > 0)
135
+ return err.i18nKey;
136
+ return declaration.messageKey;
137
+ }
138
+ // Serialized trace context lives under this key in the BullMQ job data.
139
+ // Leading underscore matches the existing internal-meta convention
140
+ // (_triggeredById, _tenantId, _payload).
141
+ export const TRACE_CONTEXT_KEY = "_traceContext";
142
+ export const DEFER_COUNT_KEY = "_deferCount";
143
+ // Separate from the maxPerTenant counter so tenant-limit deferrals do not eat into maxDefers.
144
+ export const HANDLER_DEFER_COUNT_KEY = "_handlerDeferCount";
145
+ export const DEFAULT_MAX_DEFERS = 100;
146
+ export const DEFER_DELAY_CAP_MS = 30_000;
147
+ export const DEFER_BACKOFF_MAX_DOUBLINGS = 5;
148
+ export function readTraceContext(data) {
149
+ const raw = data[TRACE_CONTEXT_KEY];
150
+ if (!raw || typeof raw !== "object")
151
+ return undefined;
152
+ const ctx = raw; // @cast-boundary engine-payload
153
+ if (!ctx.traceId || !ctx.spanId)
154
+ return undefined;
155
+ return { traceId: ctx.traceId, spanId: ctx.spanId };
156
+ }
157
+ export function captureTraceContext(tracer) {
158
+ const span = tracer.getActiveSpan();
159
+ if (!span?.traceId)
160
+ return undefined;
161
+ return { traceId: span.traceId, spanId: span.spanId };
162
+ }
163
+ const QUEUE_DEPTH_POLL_INTERVAL_MS = 15_000;
164
+ const QUEUE_DEPTH_WARN_INTERVAL_MS = 5 * 60_000;
165
+ // kumiko_job_queue_depth: only the lane's own consumer polls — the count is
166
+ // a global Redis-backed value (BullMQ, not per-process), so one reporter per
167
+ // lane is enough. Fires once immediately (metrics exist before the first
168
+ // poll tick) then on an interval; the caller stores + clears the handle.
169
+ export async function startQueueDepthPolling(queue, lane, meter) {
170
+ let lastWarnAt;
171
+ const pollQueueDepth = async () => {
172
+ try {
173
+ const counts = await queue.getJobCounts("waiting", "active", "delayed", "failed", "paused");
174
+ emitJobQueueDepth(meter, lane, counts);
175
+ }
176
+ catch (err) {
177
+ // Next poll retries; the warn is throttled so a Redis outage logs once per interval, not once per poll.
178
+ const nowMs = Date.now();
179
+ if (lastWarnAt === undefined || nowMs - lastWarnAt >= QUEUE_DEPTH_WARN_INTERVAL_MS) {
180
+ lastWarnAt = nowMs;
181
+ createFallbackLogger("job-runner").warn("queue depth poll failed", {
182
+ lane,
183
+ error: scrubLogText(err instanceof Error ? err.message : String(err)),
184
+ });
185
+ }
186
+ }
187
+ };
188
+ await pollQueueDepth();
189
+ return setInterval(() => void pollQueueDepth(), QUEUE_DEPTH_POLL_INTERVAL_MS);
190
+ }
191
+ export function parseRedisOpts(url) {
192
+ const parsed = new URL(url);
193
+ const result = {
194
+ host: parsed.hostname,
195
+ port: Number(parsed.port) || 6379,
196
+ };
197
+ if (parsed.pathname.length > 1) {
198
+ result.db = Number(parsed.pathname.slice(1));
199
+ }
200
+ return result;
201
+ }
202
+ // redisOpts carries no connectTimeout/retry cap, so an unreachable Redis
203
+ // would otherwise hang start() forever with no health endpoint to notice.
204
+ export const BOOT_REDIS_TIMEOUT_MS = 10_000;
205
+ // Shutdown must fit a k8s grace period even when Redis is down, so the
206
+ // readiness wait in stop() is capped well below the boot timeout.
207
+ export const STOP_QUEUE_READY_TIMEOUT_MS = 1_000;
208
+ // A gate without an explicit `timeout` may legitimately run long (migration-style
209
+ // checks), so it is never rejected; a hang just must not stay silent.
210
+ export const BOOT_GATE_SLOW_WARN_MS = 60_000;
211
+ export function timeoutReject(ms, message) {
212
+ let timer;
213
+ const promise = new Promise((_, reject) => {
214
+ timer = setTimeout(() => reject(new Error(message)), ms);
215
+ });
216
+ return {
217
+ promise,
218
+ cancel: () => {
219
+ if (timer !== undefined)
220
+ clearTimeout(timer);
221
+ },
222
+ };
223
+ }
224
+ // Default base delay when a job opts into backoff without an explicit
225
+ // delayMs. Without a `delay`, BullMQ's fixed/exponential strategies compute
226
+ // NaN/undefined, which is falsy — the job retries immediately instead of
227
+ // waiting.
228
+ const DEFAULT_JOB_BACKOFF_DELAY_MS = 1_000;
229
+ // Shared by every enqueue path (dispatch(), handleEvent(), cron, runOnBoot,
230
+ // perTenant wrapper/children, sequential re-enqueue) — a job with `retries`
231
+ // set must retry the same way regardless of how it got enqueued, or it
232
+ // fails for good on the very first error on whichever path skips this.
233
+ export function buildRetryBullOpts(jobDef, priorAttempts = 0) {
234
+ const opts = {};
235
+ if (jobDef.retries !== undefined)
236
+ opts.attempts = Math.max(1, jobDef.retries + 1 - priorAttempts);
237
+ if (jobDef.backoff) {
238
+ opts.backoff =
239
+ typeof jobDef.backoff === "string"
240
+ ? { type: jobDef.backoff, delay: DEFAULT_JOB_BACKOFF_DELAY_MS }
241
+ : {
242
+ type: jobDef.backoff.type,
243
+ delay: jobDef.backoff.delayMs ?? DEFAULT_JOB_BACKOFF_DELAY_MS,
244
+ };
245
+ }
246
+ return opts;
247
+ }
248
+ // Equality filter on top-level payload fields for a job's trigger.where —
249
+ // lets N jobs share one broad event QN, partitioned by a payload
250
+ // discriminant instead of needing N distinct event types.
251
+ export function payloadMatchesTriggerFilter(where, payload) {
252
+ if (!where)
253
+ return true;
254
+ return Object.entries(where).every(([key, value]) => payload[key] === value);
255
+ }
256
+ // BullMQ sweeps queue-wide: any job finishing with keepJobs set can evict
257
+ // OTHER jobs in the same completed/failed zset (moveToFinished lua,
258
+ // removeJobsByMaxAge/removeJobsByMaxCount), not just the finishing job, and
259
+ // the sweep only runs lazily on a later finish into that set. Per-job
260
+ // retention is therefore meaningless on a shared queue — only queue-wide,
261
+ // age-only retention (no count) is safe: a count would evict perTenant
262
+ // children still inside their wrapper's retry window, and boot jobs. The
263
+ // completed age must stay above every perTenant wrapper's own worst-case
264
+ // retry window (see maxRetryWindowMs and the invariant check in
265
+ // createJobRunner below), because child dedup (perTenantChildJobId) relies
266
+ // on the children still existing in Redis when a wrapper retry re-derives
267
+ // their ids. The margin here is generous — queue wait time isn't bounded by
268
+ // backoff — to cover realistic windows. Audit trail lives in read_job_runs,
269
+ // not BullMQ (fw#3199).
270
+ export const COMPLETED_JOB_RETENTION_AGE_SEC = 86_400; // 24h
271
+ export const FAILED_JOB_RETENTION_AGE_SEC = 604_800; // 7d
272
+ // Total wall-clock time BullMQ can hold a perTenant wrapper across all its
273
+ // retries, consistent with buildRetryBullOpts's own backoff computation.
274
+ export function maxRetryWindowMs(jobDef) {
275
+ if (!jobDef.backoff || jobDef.retries === undefined)
276
+ return 0;
277
+ const delayMs = (typeof jobDef.backoff === "string" ? undefined : jobDef.backoff.delayMs) ??
278
+ DEFAULT_JOB_BACKOFF_DELAY_MS;
279
+ const type = typeof jobDef.backoff === "string" ? jobDef.backoff : jobDef.backoff.type;
280
+ return type === "exponential" ? delayMs * (2 ** jobDef.retries - 1) : jobDef.retries * delayMs;
281
+ }
@@ -0,0 +1,23 @@
1
+ import type { Queue } from "bullmq";
2
+ import type { JobRunIn } from "../engine/types/index.ts";
3
+ export declare const DEFAULT_QUEUE_NAME_PREFIX = "kumiko-jobs";
4
+ export declare function queueNameFor(prefix: string, lane: JobRunIn): string;
5
+ export declare const ACTIVE_TENANT_IDS_QUERY_NAME = "tenant:query:active-tenant-ids";
6
+ /**
7
+ * BullMQ job ids are `repeat:<schedulerId>:<millis>`. Colons inside the
8
+ * scheduler id push the segment count to ≥5, which BullMQ's legacy heuristic
9
+ * treated as old repeatables — spawning a new scheduler entry every tick and
10
+ * leaking permanent `repeat:*` hashes (taskforcesh/bullmq#3828, fw#1603 /
11
+ * publicstatus Redis OOM). Strip `.` and `:` from the job QN.
12
+ */
13
+ export declare function schedulerIdForJobName(jobName: string): string;
14
+ export declare function bootJobIdForJobName(jobName: string): string;
15
+ export declare const ORPHAN_SCHEDULER_GRACE_MS: number;
16
+ export declare function markSchedulersSeen(queue: Queue, schedulerIds: ReadonlySet<string>): Promise<void>;
17
+ type OrphanSchedulerLog = {
18
+ info(msg: string, data?: Record<string, unknown>): void;
19
+ warn(msg: string, data?: Record<string, unknown>): void;
20
+ };
21
+ export declare function removeOrphanJobSchedulers(queue: Queue, expectedSchedulerIds: ReadonlySet<string>, lane: JobRunIn, log: OrphanSchedulerLog): Promise<void>;
22
+ export declare function perTenantChildJobId(wrapperJobId: string, tenantId: string): string;
23
+ export {};
@@ -0,0 +1,106 @@
1
+ // Queue-name convention: <prefix>-<lane>. The prefix is fixed in prod
2
+ // ("kumiko-jobs") — it must match between enqueuers and consumers, and an
3
+ // accidental drift would silently drop jobs. Tests override via
4
+ // `queueNamePrefix` for per-run isolation (stale jobs from a prior run
5
+ // don't leak into a new test because the queue name includes a timestamp).
6
+ export const DEFAULT_QUEUE_NAME_PREFIX = "kumiko-jobs";
7
+ export function queueNameFor(prefix, lane) {
8
+ return `${prefix}-${lane}`;
9
+ }
10
+ // perTenant jobs need a source of truth for "which tenants are active" —
11
+ // see createJobRunner's defaultGetActiveTenantIds below. The tenant feature
12
+ // (bundled-features package) registers a query under this exact name when
13
+ // mounted. framework must not depend on bundled-features (wrong direction —
14
+ // bundled-features already depends on framework), so this is a name-only
15
+ // coupling: createJobRunner looks the handler up in the registry by string
16
+ // instead of importing it.
17
+ export const ACTIVE_TENANT_IDS_QUERY_NAME = "tenant:query:active-tenant-ids";
18
+ /**
19
+ * BullMQ job ids are `repeat:<schedulerId>:<millis>`. Colons inside the
20
+ * scheduler id push the segment count to ≥5, which BullMQ's legacy heuristic
21
+ * treated as old repeatables — spawning a new scheduler entry every tick and
22
+ * leaking permanent `repeat:*` hashes (taskforcesh/bullmq#3828, fw#1603 /
23
+ * publicstatus Redis OOM). Strip `.` and `:` from the job QN.
24
+ */
25
+ export function schedulerIdForJobName(jobName) {
26
+ return `scheduler-${jobName.replace(/[.:]/g, "-")}`;
27
+ }
28
+ // Same colon-in-BullMQ-id hazard as schedulerIdForJobName (fw#1603/#1604) —
29
+ // a QN like "publicstatus:job:uptime-probe" must not leave ":" in the id.
30
+ export function bootJobIdForJobName(jobName) {
31
+ return `boot-${jobName.replace(/[.:]/g, "-")}`;
32
+ }
33
+ const SCHEDULER_PAGE_SIZE = 100;
34
+ // Rolling deploys run old and new pods against the same Redis for a while; an
35
+ // old runner restarting in that window must not delete the scheduler of a
36
+ // cron job the new version just added.
37
+ export const ORPHAN_SCHEDULER_GRACE_MS = 60 * 60 * 1000;
38
+ function schedulerSeenKey(queue) {
39
+ return queue.toKey("kumiko-scheduler-seen");
40
+ }
41
+ // "Seen" has to mean a live runner holds the scheduler, so runners re-stamp
42
+ // their own ids periodically; a single stamp at start() would let a runner
43
+ // restarting during a mixed-version window longer than the grace period delete
44
+ // a scheduler a healthy newer pod still owns.
45
+ export async function markSchedulersSeen(queue, schedulerIds) {
46
+ // skip: no cron scheduler on this lane, nothing to stamp
47
+ if (schedulerIds.size === 0)
48
+ return;
49
+ const stamp = String(Date.now());
50
+ const entries = Object.fromEntries([...schedulerIds].map((id) => [id, stamp]));
51
+ await (await queue.client).hset(schedulerSeenKey(queue), entries);
52
+ }
53
+ // A cron job dropped from the app (or an older scheduler-id format) leaves its
54
+ // scheduler in Redis, where it keeps firing jobs the worker no longer knows
55
+ // ("Unknown job"). Best-effort: a Redis hiccup here must not stop the start.
56
+ export async function removeOrphanJobSchedulers(queue, expectedSchedulerIds, lane, log) {
57
+ try {
58
+ // Collect first: removing while paging would shift the indices.
59
+ const existingKeys = [];
60
+ for (let start = 0;; start += SCHEDULER_PAGE_SIZE) {
61
+ const page = await queue.getJobSchedulers(start, start + SCHEDULER_PAGE_SIZE - 1);
62
+ for (const scheduler of page)
63
+ existingKeys.push(scheduler.key);
64
+ if (page.length < SCHEDULER_PAGE_SIZE)
65
+ break;
66
+ }
67
+ const client = await queue.client;
68
+ const seenAt = await client.hgetall(schedulerSeenKey(queue));
69
+ const now = Date.now();
70
+ for (const key of existingKeys) {
71
+ if (expectedSchedulerIds.has(key))
72
+ continue;
73
+ const lastSeen = Number(seenAt[key]);
74
+ if (key in seenAt && now - lastSeen < ORPHAN_SCHEDULER_GRACE_MS)
75
+ continue;
76
+ await queue.removeJobScheduler(key);
77
+ await client.hdel(schedulerSeenKey(queue), key);
78
+ log.info("removed orphan job scheduler", { schedulerId: key, lane });
79
+ }
80
+ }
81
+ catch (err) {
82
+ log.warn("orphan job scheduler cleanup failed", {
83
+ lane,
84
+ error: err instanceof Error ? err.message : String(err),
85
+ });
86
+ }
87
+ }
88
+ // Deterministic per-tenant child job id, derived from the perTenant
89
+ // wrapper's own BullMQ job id. A cron tick's wrapper id is
90
+ // `repeat:<schedulerId>:<millis>` (stable for that tick, including across
91
+ // BullMQ's own retry/stalled-job redelivery of the *same* job); a manual
92
+ // dispatch's wrapper id is BullMQ's auto-generated id, equally stable
93
+ // across retries of that job. Reusing it means a wrapper that runs twice
94
+ // for the same trigger (retry after failure, a Redis drop, an instance
95
+ // restarting mid-run) re-derives the *same* child ids, and BullMQ's
96
+ // existing-jobId add() no-ops the second batch instead of creating
97
+ // duplicates — safe only as long as the child job hash outlives the
98
+ // wrapper's retry window, enforced at construction against
99
+ // COMPLETED_JOB_RETENTION_AGE_SEC (see the invariant check in
100
+ // createJobRunner). Same colon-in-BullMQ-id hazard as schedulerIdForJobName
101
+ // (fw#1603/#1604) — the wrapper id already contains ":", so strip
102
+ // separators from both halves before joining instead of interpolating raw.
103
+ export function perTenantChildJobId(wrapperJobId, tenantId) {
104
+ const sanitize = (value) => value.replace(/[.:]/g, "-");
105
+ return `child-${sanitize(wrapperJobId)}-${sanitize(tenantId)}`;
106
+ }
@@ -0,0 +1,65 @@
1
+ /// <reference types="temporal-polyfill/global" preserve="true" />
2
+ import type { AppContext, DispatchWriteRef, JobRunIn, JobTriggerEvent, Registry, SessionUser, TenantId } from "../engine/types/index.ts";
3
+ export type JobLogEntry = {
4
+ level: "info" | "warn" | "error";
5
+ message: string;
6
+ timestamp: Temporal.Instant;
7
+ };
8
+ export type JobMeta = {
9
+ triggeredById?: string | undefined;
10
+ payload?: string | undefined;
11
+ attempt?: number | undefined;
12
+ priority?: number | undefined;
13
+ jobId?: string | undefined;
14
+ tenantVisibleRun?: {
15
+ readonly tenantId: string;
16
+ readonly subject: string | null;
17
+ } | undefined;
18
+ };
19
+ export type JobQueuedMeta = {
20
+ readonly tenantId: string;
21
+ readonly subject: string | null;
22
+ };
23
+ export type JobDroppedMeta = {
24
+ readonly tenantId: string;
25
+ };
26
+ export type JobOutcomeMeta = {
27
+ readonly tenantId: string;
28
+ readonly finalAttempt: boolean;
29
+ readonly tenantVisible?: {
30
+ readonly subject: string | null;
31
+ readonly messageKey: string | null;
32
+ } | undefined;
33
+ readonly tenantVisibleRun?: {
34
+ readonly subject: string | null;
35
+ } | undefined;
36
+ };
37
+ export type JobSubjectValue = string | number | boolean | null;
38
+ export type JobRunner = {
39
+ start(): Promise<void>;
40
+ stop(): Promise<void>;
41
+ dispatch(jobName: string, payload?: Record<string, unknown>, meta?: JobMeta): Promise<string>;
42
+ handleEvent(eventName: string, payload: Record<string, unknown>, user?: SessionUser, triggerEvent?: JobTriggerEvent): Promise<void>;
43
+ attachDispatcher(ref: DispatchWriteRef): void;
44
+ countPendingJobs(): Promise<number>;
45
+ };
46
+ export type JobRunnerOptions = {
47
+ registry: Registry;
48
+ context: AppContext;
49
+ redisUrl: string;
50
+ consumerLane?: JobRunIn | undefined;
51
+ queueNamePrefix?: string | undefined;
52
+ bootRedisTimeoutMs?: number | undefined;
53
+ schedulerSeenRefreshMs?: number | undefined;
54
+ jobRetention?: {
55
+ completedAgeSec?: number | undefined;
56
+ failedAgeSec?: number | undefined;
57
+ } | undefined;
58
+ getActiveTenantIds?: () => Promise<TenantId[]>;
59
+ onJobStart?: (jobName: string, jobId: string, meta: JobMeta) => void;
60
+ onJobComplete?: (jobName: string, jobId: string, duration: number, logs: JobLogEntry[], outcome?: JobOutcomeMeta) => void;
61
+ onJobFailed?: (jobName: string, jobId: string, error: string, logs: JobLogEntry[], outcome?: JobOutcomeMeta) => void;
62
+ onJobDeferred?: (jobName: string, jobId: string, meta: JobMeta) => void;
63
+ onJobQueued?: (jobName: string, jobId: string, meta: JobQueuedMeta) => void;
64
+ onJobDropped?: (jobName: string, jobId: string, meta: JobDroppedMeta) => void;
65
+ };
@@ -0,0 +1,2 @@
1
+ /// <reference types="temporal-polyfill/global" preserve="true" />
2
+ export {};
@@ -1,77 +1,6 @@
1
1
  /// <reference types="temporal-polyfill/global" preserve="true" />
2
- import { type AppContext, type DispatchWriteRef, type JobRunIn, type JobTriggerEvent, type Registry, type SessionUser, type TenantId } from "../engine/types/index.ts";
3
- /**
4
- * BullMQ job ids are `repeat:<schedulerId>:<millis>`. Colons inside the
5
- * scheduler id push the segment count to ≥5, which BullMQ's legacy heuristic
6
- * treated as old repeatables — spawning a new scheduler entry every tick and
7
- * leaking permanent `repeat:*` hashes (taskforcesh/bullmq#3828, fw#1603 /
8
- * publicstatus Redis OOM). Strip `.` and `:` from the job QN.
9
- */
10
- export declare function schedulerIdForJobName(jobName: string): string;
11
- export declare function bootJobIdForJobName(jobName: string): string;
12
- export declare const ORPHAN_SCHEDULER_GRACE_MS: number;
13
- export type JobLogEntry = {
14
- level: "info" | "warn" | "error";
15
- message: string;
16
- timestamp: Temporal.Instant;
17
- };
18
- export type JobMeta = {
19
- triggeredById?: string | undefined;
20
- payload?: string | undefined;
21
- attempt?: number | undefined;
22
- priority?: number | undefined;
23
- jobId?: string | undefined;
24
- tenantVisibleRun?: {
25
- readonly tenantId: string;
26
- readonly subject: string | null;
27
- } | undefined;
28
- };
29
- export type JobQueuedMeta = {
30
- readonly tenantId: string;
31
- readonly subject: string | null;
32
- };
33
- export type JobDroppedMeta = {
34
- readonly tenantId: string;
35
- };
36
- export type JobOutcomeMeta = {
37
- readonly tenantId: string;
38
- readonly finalAttempt: boolean;
39
- readonly tenantVisible?: {
40
- readonly subject: string | null;
41
- readonly messageKey: string | null;
42
- } | undefined;
43
- readonly tenantVisibleRun?: {
44
- readonly subject: string | null;
45
- } | undefined;
46
- };
47
- export type JobSubjectValue = string | number | boolean | null;
48
- export declare function serializeJobSubject(subject: Readonly<Record<string, JobSubjectValue>>): string;
49
- export type JobRunner = {
50
- start(): Promise<void>;
51
- stop(): Promise<void>;
52
- dispatch(jobName: string, payload?: Record<string, unknown>, meta?: JobMeta): Promise<string>;
53
- handleEvent(eventName: string, payload: Record<string, unknown>, user?: SessionUser, triggerEvent?: JobTriggerEvent): Promise<void>;
54
- attachDispatcher(ref: DispatchWriteRef): void;
55
- countPendingJobs(): Promise<number>;
56
- };
57
- export type JobRunnerOptions = {
58
- registry: Registry;
59
- context: AppContext;
60
- redisUrl: string;
61
- consumerLane?: JobRunIn | undefined;
62
- queueNamePrefix?: string | undefined;
63
- bootRedisTimeoutMs?: number | undefined;
64
- schedulerSeenRefreshMs?: number | undefined;
65
- jobRetention?: {
66
- completedAgeSec?: number | undefined;
67
- failedAgeSec?: number | undefined;
68
- } | undefined;
69
- getActiveTenantIds?: () => Promise<TenantId[]>;
70
- onJobStart?: (jobName: string, jobId: string, meta: JobMeta) => void;
71
- onJobComplete?: (jobName: string, jobId: string, duration: number, logs: JobLogEntry[], outcome?: JobOutcomeMeta) => void;
72
- onJobFailed?: (jobName: string, jobId: string, error: string, logs: JobLogEntry[], outcome?: JobOutcomeMeta) => void;
73
- onJobDeferred?: (jobName: string, jobId: string, meta: JobMeta) => void;
74
- onJobQueued?: (jobName: string, jobId: string, meta: JobQueuedMeta) => void;
75
- onJobDropped?: (jobName: string, jobId: string, meta: JobDroppedMeta) => void;
76
- };
2
+ import type { JobRunner, JobRunnerOptions } from "./job-runner-types.ts";
3
+ export { serializeJobSubject } from "./job-runner-helpers.ts";
4
+ export { bootJobIdForJobName, ORPHAN_SCHEDULER_GRACE_MS, schedulerIdForJobName, } from "./job-runner-schedulers.ts";
5
+ export type { JobDroppedMeta, JobLogEntry, JobMeta, JobOutcomeMeta, JobQueuedMeta, JobRunner, JobRunnerOptions, JobSubjectValue, } from "./job-runner-types.ts";
77
6
  export declare function createJobRunner(options: JobRunnerOptions): JobRunner;