@cosmicdrift/kumiko-bundled-features 0.363.0 → 0.364.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/audit/handlers/list.query.js +9 -5
- package/dist/auth-email-password/bootstrap.d.ts +8 -5
- package/dist/auth-email-password/bootstrap.js +12 -4
- package/dist/auth-email-password/email-rate-limits.d.ts +4 -0
- package/dist/auth-email-password/email-rate-limits.js +13 -0
- package/dist/auth-email-password/handlers/change-password.write.d.ts +1 -1
- package/dist/auth-email-password/handlers/change-password.write.js +6 -2
- package/dist/auth-email-password/handlers/confirm-account-unlock.write.d.ts +1 -1
- package/dist/auth-email-password/handlers/invite-accept-with-login.write.d.ts +1 -1
- package/dist/auth-email-password/handlers/invite-accept.write.d.ts +1 -1
- package/dist/auth-email-password/handlers/invite-create.write.d.ts +2 -2
- package/dist/auth-email-password/handlers/invite-signup-complete.write.d.ts +1 -1
- package/dist/auth-email-password/handlers/login.write.d.ts +1 -1
- package/dist/auth-email-password/handlers/logout.write.d.ts +1 -1
- package/dist/auth-email-password/handlers/logout.write.js +6 -2
- package/dist/auth-email-password/handlers/request-account-unlock.write.d.ts +1 -1
- package/dist/auth-email-password/handlers/request-email-verification.write.d.ts +1 -1
- package/dist/auth-email-password/handlers/request-password-reset.write.d.ts +1 -1
- package/dist/auth-email-password/handlers/reset-password.write.d.ts +1 -1
- package/dist/auth-email-password/handlers/signup-confirm.write.d.ts +1 -1
- package/dist/auth-email-password/handlers/signup-request.write.d.ts +1 -1
- package/dist/auth-email-password/handlers/signup-request.write.js +2 -1
- package/dist/auth-email-password/handlers/switch-tenant-mfa-gate.write.d.ts +1 -1
- package/dist/auth-email-password/handlers/token-request-handler.d.ts +1 -1
- package/dist/auth-email-password/handlers/token-request-handler.js +2 -1
- package/dist/auth-email-password/handlers/verify-email.write.d.ts +1 -1
- package/dist/auth-email-password/web/auth-gate.js +16 -4
- package/dist/auth-mfa/handlers/disable.write.d.ts +1 -1
- package/dist/auth-mfa/handlers/enable-confirm-preauth.write.d.ts +1 -1
- package/dist/auth-mfa/handlers/enable-confirm.write.d.ts +1 -1
- package/dist/auth-mfa/handlers/enable-start-preauth.write.d.ts +1 -1
- package/dist/auth-mfa/handlers/enable-start.write.d.ts +1 -1
- package/dist/auth-mfa/handlers/regenerate-recovery.write.d.ts +1 -1
- package/dist/auth-mfa/handlers/verify.write.d.ts +1 -1
- package/dist/billing-foundation/consumer-protection/confirmation-mail.js +52 -5
- package/dist/billing-foundation/consumer-protection/consent-gate.d.ts +3 -0
- package/dist/billing-foundation/consumer-protection/consent-gate.js +21 -2
- package/dist/billing-foundation/consumer-protection/mail-notifications.js +1 -1
- package/dist/billing-foundation/consumer-protection/register-termination.js +11 -1
- package/dist/billing-foundation/consumer-protection/termination-mail.d.ts +5 -3
- package/dist/billing-foundation/consumer-protection/termination-mail.js +1 -2
- package/dist/billing-foundation/consumer-protection/termination-receipt-limit.d.ts +6 -0
- package/dist/billing-foundation/consumer-protection/termination-receipt-limit.js +34 -0
- package/dist/billing-foundation/consumer-protection/termination-record.d.ts +2 -0
- package/dist/billing-foundation/consumer-protection/termination-record.js +7 -1
- package/dist/billing-foundation/consumer-protection/termination-request.js +30 -20
- package/dist/billing-foundation/consumer-protection/termination-texts.d.ts +1 -1
- package/dist/billing-foundation/consumer-protection/termination-texts.js +6 -0
- package/dist/billing-foundation/consumer-protection/withdrawal-period.d.ts +10 -0
- package/dist/billing-foundation/consumer-protection/withdrawal-period.js +26 -0
- package/dist/billing-foundation/handlers/create-checkout-session.write.js +11 -5
- package/dist/billing-foundation/handlers/process-event.write.js +10 -6
- package/dist/billing-foundation/i18n.js +3 -0
- package/dist/billing-foundation/subscription-tier-sync.d.ts +7 -1
- package/dist/billing-foundation/subscription-tier-sync.js +13 -5
- package/dist/billing-foundation/types.d.ts +7 -2
- package/dist/cap-counter/calendar-cap-usage-reservation.js +2 -0
- package/dist/cap-counter/enforce-cap.d.ts +18 -1
- package/dist/cap-counter/enforce-cap.js +49 -6
- package/dist/cap-counter/index.d.ts +1 -1
- package/dist/cap-counter/index.js +1 -1
- package/dist/cap-counter/with-cap-enforcement.d.ts +2 -0
- package/dist/channel-email/email-channel.js +16 -4
- package/dist/channel-in-app/handlers/mark-all-read.write.d.ts +1 -1
- package/dist/channel-in-app/handlers/mark-read.write.d.ts +1 -1
- package/dist/compliance-profiles/handlers/set-profile.write.d.ts +1 -1
- package/dist/config/handlers/reset.write.d.ts +1 -1
- package/dist/config/handlers/set.write.d.ts +1 -1
- package/dist/config/handlers/set.write.js +2 -2
- package/dist/config/resolver.d.ts +1 -0
- package/dist/config/resolver.js +11 -14
- package/dist/config/write-helpers.d.ts +2 -0
- package/dist/config/write-helpers.js +19 -0
- package/dist/crypto-shredding/handlers/forget-subject.write.d.ts +1 -1
- package/dist/crypto-shredding/handlers/forget-subject.write.js +29 -19
- package/dist/custom-fields/handlers/update-tenant-field.write.js +0 -1
- package/dist/delivery/attachment-ref.d.ts +2 -0
- package/dist/delivery/attachment-ref.js +14 -0
- package/dist/delivery/channel-context.js +19 -1
- package/dist/delivery/handlers/resubscribe-address.write.d.ts +1 -1
- package/dist/delivery/handlers/resubscribe-user.write.d.ts +1 -1
- package/dist/delivery/handlers/set-preference.write.d.ts +1 -1
- package/dist/delivery/handlers/unsubscribe-address.write.d.ts +1 -1
- package/dist/delivery/handlers/unsubscribe-user.write.d.ts +1 -1
- package/dist/delivery/index.d.ts +1 -1
- package/dist/delivery/index.js +1 -1
- package/dist/delivery/jobs.js +43 -11
- package/dist/delivery/types.d.ts +7 -1
- package/dist/feature-toggles/handlers/set.write.d.ts +1 -1
- package/dist/file-derivatives/handlers/public-variant.query.js +1 -1
- package/dist/file-foundation/read-file-ref-content.d.ts +1 -1
- package/dist/file-foundation/read-file-ref-content.js +6 -4
- package/dist/files-tenant-data/hooks.js +4 -0
- package/dist/form-draft/handlers/discard.write.d.ts +1 -1
- package/dist/form-draft/handlers/save.write.d.ts +1 -1
- package/dist/inbound-mail-foundation/oauth-access-token.js +8 -3
- package/dist/inbound-mail-foundation/watch-supervisor.js +3 -1
- package/dist/jobs/db/queries/stale-run-sweep.d.ts +1 -1
- package/dist/jobs/db/queries/stale-run-sweep.js +12 -5
- package/dist/jobs/feature.js +5 -3
- package/dist/jobs/handlers/list.query.js +3 -1
- package/dist/jobs/handlers/retry.write.d.ts +1 -1
- package/dist/jobs/handlers/stale-run-sweep.job.d.ts +1 -1
- package/dist/jobs/handlers/stale-run-sweep.job.js +2 -2
- package/dist/jobs/handlers/trigger.write.d.ts +1 -1
- package/dist/jobs/job-run-logger.js +32 -10
- package/dist/jobs/tenant-job-run-table.js +3 -2
- package/dist/ledger/entity.js +3 -1
- package/dist/managed-pages/handlers/set.write.d.ts +1 -1
- package/dist/personal-access-tokens/handlers/create.write.d.ts +1 -1
- package/dist/personal-access-tokens/handlers/revoke.write.d.ts +1 -1
- package/dist/renderer-foundation/types.d.ts +2 -0
- package/dist/renderer-pdf/render-table-pdf.d.ts +27 -33
- package/dist/renderer-pdf/render-table-pdf.js +95 -34
- package/dist/secrets/feature.js +4 -3
- package/dist/secrets/handlers/delete.write.d.ts +1 -1
- package/dist/secrets/handlers/set.write.d.ts +1 -1
- package/dist/secrets/secrets-context.js +15 -3
- package/dist/sessions/handlers/revoke-all-for-user.write.d.ts +1 -1
- package/dist/sessions/handlers/revoke-all-others.write.d.ts +1 -1
- package/dist/sessions/handlers/revoke.write.d.ts +1 -1
- package/dist/shared/chunked-entity-migration.d.ts +1 -0
- package/dist/shared/chunked-entity-migration.js +1 -1
- package/dist/shared/mail-template-body.d.ts +3 -0
- package/dist/shared/mail-template-body.js +10 -0
- package/dist/subscription-stripe/plugin-methods.js +18 -8
- package/dist/subscription-stripe/verify-webhook.d.ts +1 -1
- package/dist/template-resolver/api.js +3 -2
- package/dist/template-resolver/compile-document-body.d.ts +4 -0
- package/dist/template-resolver/compile-document-body.js +58 -16
- package/dist/template-resolver/constants.d.ts +2 -2
- package/dist/template-resolver/constants.js +3 -1
- package/dist/template-resolver/document-schema.d.ts +1 -1
- package/dist/template-resolver/document-schema.js +23 -3
- package/dist/template-resolver/feature.d.ts +2 -0
- package/dist/template-resolver/feature.js +4 -1
- package/dist/template-resolver/handlers/collection-set.write.d.ts +1 -1
- package/dist/template-resolver/handlers/copy-locale.write.d.ts +1 -1
- package/dist/template-resolver/handlers/copy-locale.write.js +29 -11
- package/dist/template-resolver/handlers/diff-to-system.query.js +4 -38
- package/dist/template-resolver/handlers/discard-draft.write.d.ts +13 -0
- package/dist/template-resolver/handlers/discard-draft.write.js +37 -0
- package/dist/template-resolver/handlers/export-bundle.query.js +51 -1
- package/dist/template-resolver/handlers/find-by-id.query.d.ts +1 -0
- package/dist/template-resolver/handlers/find-by-id.query.js +6 -3
- package/dist/template-resolver/handlers/import-bundle-system.write.d.ts +1 -1
- package/dist/template-resolver/handlers/import-bundle.write.d.ts +1 -1
- package/dist/template-resolver/handlers/layout-source.query.d.ts +5 -0
- package/dist/template-resolver/handlers/layout-source.query.js +46 -23
- package/dist/template-resolver/handlers/list.query.js +4 -1
- package/dist/template-resolver/handlers/override-diff.d.ts +20 -0
- package/dist/template-resolver/handlers/override-diff.js +41 -0
- package/dist/template-resolver/handlers/overview.query.d.ts +6 -0
- package/dist/template-resolver/handlers/overview.query.js +51 -11
- package/dist/template-resolver/handlers/reset-to-system.write.d.ts +1 -1
- package/dist/template-resolver/handlers/resource-link-pdf.write.d.ts +1 -1
- package/dist/template-resolver/handlers/resource-remove.write.d.ts +1 -1
- package/dist/template-resolver/handlers/resource-upload.write.d.ts +1 -1
- package/dist/template-resolver/handlers/resource-upload.write.js +1 -1
- package/dist/template-resolver/handlers/send-test.write.d.ts +3 -1
- package/dist/template-resolver/handlers/send-test.write.js +37 -10
- package/dist/template-resolver/handlers/set.write.d.ts +1 -1
- package/dist/template-resolver/handlers/template-gates.js +13 -2
- package/dist/template-resolver/handlers/template-write.js +36 -4
- package/dist/template-resolver/handlers/tenant-overrides.query.d.ts +32 -0
- package/dist/template-resolver/handlers/tenant-overrides.query.js +80 -16
- package/dist/template-resolver/handlers/toggle-status.write.d.ts +17 -4
- package/dist/template-resolver/handlers/toggle-status.write.js +35 -19
- package/dist/template-resolver/handlers/upsert-system.write.d.ts +1 -1
- package/dist/template-resolver/handlers/upsert-tenant.write.d.ts +1 -1
- package/dist/template-resolver/handlers/upsert-tenant.write.js +1 -1
- package/dist/template-resolver/handlers/usage.query.js +11 -2
- package/dist/template-resolver/index.d.ts +4 -3
- package/dist/template-resolver/index.js +4 -3
- package/dist/template-resolver/layout-appearance.d.ts +10 -0
- package/dist/template-resolver/layout-appearance.js +26 -0
- package/dist/template-resolver/pdf-constants.d.ts +7 -0
- package/dist/template-resolver/pdf-constants.js +8 -0
- package/dist/template-resolver/qualified-names.d.ts +4 -0
- package/dist/template-resolver/qualified-names.js +4 -0
- package/dist/template-resolver/seeding.js +21 -4
- package/dist/template-resolver/store-template-resource.js +2 -2
- package/dist/template-resolver/table.d.ts +103 -0
- package/dist/template-resolver/table.js +9 -2
- package/dist/template-resolver/template-draft.d.ts +51 -0
- package/dist/template-resolver/template-draft.js +72 -0
- package/dist/template-resolver/template-meta.d.ts +30 -5
- package/dist/template-resolver/template-meta.js +60 -16
- package/dist/template-resolver/template-preview.d.ts +19 -2
- package/dist/template-resolver/template-preview.js +98 -54
- package/dist/template-resolver/template-resources.d.ts +5 -0
- package/dist/template-resolver/template-resources.js +35 -4
- package/dist/template-resolver/template-usage.d.ts +5 -1
- package/dist/template-resolver/template-usage.js +14 -9
- package/dist/template-resolver/template-validation.js +1 -0
- package/dist/template-resolver/web/client-plugin.js +4 -2
- package/dist/template-resolver/web/document-editor-extensions.d.ts +4 -1
- package/dist/template-resolver/web/document-editor-extensions.js +50 -12
- package/dist/template-resolver/web/document-editor.d.ts +2 -1
- package/dist/template-resolver/web/document-editor.js +8 -2
- package/dist/template-resolver/web/document-tiptap-editor.d.ts +1 -1
- package/dist/template-resolver/web/document-tiptap-editor.js +144 -28
- package/dist/template-resolver/web/document-toolbar-blocks.d.ts +8 -0
- package/dist/template-resolver/web/document-toolbar-blocks.js +4 -0
- package/dist/template-resolver/web/i18n.js +19 -4
- package/dist/template-resolver/web/index.d.ts +1 -0
- package/dist/template-resolver/web/index.js +1 -0
- package/dist/template-resolver/web/placeholder-line-editor.d.ts +3 -1
- package/dist/template-resolver/web/placeholder-line-editor.js +6 -6
- package/dist/tenant/handlers/add-member.write.d.ts +1 -1
- package/dist/tenant/handlers/cancel-invitation.write.d.ts +1 -1
- package/dist/tenant/handlers/create.write.d.ts +1 -1
- package/dist/tenant/handlers/remove-member.write.d.ts +1 -1
- package/dist/tenant/handlers/toggle-enabled.write.d.ts +2 -2
- package/dist/tenant/handlers/update-member-roles.write.d.ts +1 -1
- package/dist/tenant/handlers/update.write.d.ts +1 -1
- package/dist/tenant-handover/handlers/claim.write.d.ts +1 -1
- package/dist/tenant-lifecycle/handlers/cancel-destruction.write.d.ts +1 -1
- package/dist/tenant-lifecycle/handlers/request-destruction.write.d.ts +1 -1
- package/dist/tier-engine/feature.js +9 -3
- package/dist/tier-engine/handlers/set-tenant-tier.write.d.ts +1 -1
- package/dist/user/handlers/create.write.d.ts +1 -1
- package/dist/user/handlers/update.write.d.ts +1 -1
- package/dist/user/resolve-display-names.js +5 -9
- package/dist/user/seeding.js +20 -1
- package/dist/user-data-rights/handlers/cancel-deletion.write.d.ts +1 -1
- package/dist/user-data-rights/handlers/confirm-deletion-by-token.write.d.ts +1 -1
- package/dist/user-data-rights/handlers/lift-restriction.write.d.ts +1 -1
- package/dist/user-data-rights/handlers/request-deletion-by-email.write.d.ts +1 -1
- package/dist/user-data-rights/handlers/request-deletion.write.d.ts +1 -1
- package/dist/user-data-rights/handlers/request-export.write.d.ts +1 -1
- package/dist/user-data-rights/handlers/restrict-account.write.d.ts +1 -1
- package/dist/user-data-rights/handlers/run-forget-cleanup.write.d.ts +1 -1
- package/dist/user-profile/handlers/change-email.write.d.ts +1 -1
- package/dist/waitlist/handlers/invite.write.d.ts +1 -1
- package/dist/waitlist/handlers/reject.write.d.ts +1 -1
- package/dist/waitlist/handlers/submit.write.d.ts +1 -1
- package/dist/waitlist/options.js +1 -1
- package/dist/workflow-runner/event-trigger.js +2 -2
- package/dist/workflow-runner/handlers/resume-run.write.js +7 -2
- package/package.json +19 -10
- package/src/audit/changes.json +6 -0
- package/src/auth-email-password/changes.json +37 -0
- package/src/billing-foundation/changes.json +36 -0
- package/src/cap-counter/changes.json +6 -0
- package/src/config/changes.json +6 -0
- package/src/crypto-shredding/changes.json +6 -0
- package/src/custom-fields/changes.json +6 -0
- package/src/delivery/changes.json +18 -0
- package/src/file-foundation/changes.json +6 -0
- package/src/files-tenant-data/changes.json +6 -0
- package/src/inbound-mail-foundation/changes.json +6 -0
- package/src/jobs/changes.json +19 -0
- package/src/ledger/changes.json +7 -0
- package/src/renderer-pdf/changes.json +12 -0
- package/src/secrets/changes.json +6 -0
- package/src/shared/changes.json +6 -0
- package/src/subscription-stripe/changes.json +6 -0
- package/src/template-resolver/changes.json +68 -0
- package/src/tier-engine/changes.json +6 -0
- package/src/user/changes.json +6 -0
- package/src/user-data-rights/changes.json +6 -0
- package/src/user-profile/changes.json +1 -1
- package/src/workflow-runner/changes.json +6 -0
package/dist/delivery/jobs.js
CHANGED
|
@@ -36,6 +36,8 @@ const sendJobPayloadSchema = renderJobPayloadSchema.extend({
|
|
|
36
36
|
html: z.string(),
|
|
37
37
|
subject: z.string(),
|
|
38
38
|
text: z.string().optional(),
|
|
39
|
+
fromName: z.string().optional(),
|
|
40
|
+
replyTo: z.string().optional(),
|
|
39
41
|
attachments: z.array(emailAttachmentRefSchema).max(MAX_EMAIL_ATTACHMENTS).optional(),
|
|
40
42
|
})
|
|
41
43
|
.optional(),
|
|
@@ -63,6 +65,8 @@ function toSendRendered(rendered) {
|
|
|
63
65
|
html: rendered.html,
|
|
64
66
|
subject: rendered.subject,
|
|
65
67
|
...(rendered.text !== undefined && { text: rendered.text }),
|
|
68
|
+
...(rendered.fromName !== undefined && { fromName: rendered.fromName }),
|
|
69
|
+
...(rendered.replyTo !== undefined && { replyTo: rendered.replyTo }),
|
|
66
70
|
...(rendered.attachments !== undefined && {
|
|
67
71
|
attachments: rendered.attachments.map(toAttachmentRef),
|
|
68
72
|
}),
|
|
@@ -96,12 +100,11 @@ function entryFor(p, status, error, address, confirmed) {
|
|
|
96
100
|
...(confirmed === false && { confirmed }),
|
|
97
101
|
};
|
|
98
102
|
}
|
|
99
|
-
//
|
|
100
|
-
//
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
if (!ctx.finalAttempt || !p.idempotencyKey)
|
|
103
|
+
// Assumes ctx.redis is the handle the service claimed on; runProdApp wires the same
|
|
104
|
+
// instance into both.
|
|
105
|
+
async function releaseIdempotencyKey(ctx, p) {
|
|
106
|
+
// skip: no idempotency key was claimed for this notification
|
|
107
|
+
if (!p.idempotencyKey)
|
|
105
108
|
return;
|
|
106
109
|
if (!ctx.redis) {
|
|
107
110
|
ctx.log.warn("delivery: idempotency key not released, job context has no redis", {
|
|
@@ -118,6 +121,26 @@ async function releaseIdempotencyOnTerminalFailure(ctx, p) {
|
|
|
118
121
|
});
|
|
119
122
|
}
|
|
120
123
|
}
|
|
124
|
+
// Only the final retry frees the key: earlier failures are retried by the runner and
|
|
125
|
+
// must still dedup.
|
|
126
|
+
async function releaseIdempotencyOnTerminalFailure(ctx, p) {
|
|
127
|
+
// skip: not the last attempt, the runner retries and the key must keep deduplicating
|
|
128
|
+
if (!ctx.finalAttempt)
|
|
129
|
+
return;
|
|
130
|
+
await releaseIdempotencyKey(ctx, p);
|
|
131
|
+
}
|
|
132
|
+
// Covers every throw after the payload parsed, not just the attempt-logged ones: a
|
|
133
|
+
// missing registry or unknown channel on the last attempt would otherwise hold the key
|
|
134
|
+
// until its TTL.
|
|
135
|
+
async function releasingKeyOnTerminalFailure(ctx, p, run) {
|
|
136
|
+
try {
|
|
137
|
+
return await run();
|
|
138
|
+
}
|
|
139
|
+
catch (err) {
|
|
140
|
+
await releaseIdempotencyOnTerminalFailure(ctx, p);
|
|
141
|
+
throw err;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
121
144
|
// The job runner stores err.message in the run row, and BullMQ keeps failedReason and
|
|
122
145
|
// the stack trace in Redis in plain text, so the rethrown error carries only the code.
|
|
123
146
|
// The redacted message goes to the log.
|
|
@@ -128,7 +151,6 @@ async function failAttempt(ctx, db, registry, p, code, err) {
|
|
|
128
151
|
channel: p.channelName,
|
|
129
152
|
});
|
|
130
153
|
await appendAttemptEvent(db, registry, p.deliveryAttemptId, entryFor(p, "failed", code, p.address));
|
|
131
|
-
await releaseIdempotencyOnTerminalFailure(ctx, p);
|
|
132
154
|
throw new Error(code);
|
|
133
155
|
}
|
|
134
156
|
// Render the message and hand off to delivery.send. On failure: record the
|
|
@@ -136,10 +158,13 @@ async function failAttempt(ctx, db, registry, p, code, err) {
|
|
|
136
158
|
// the send step is never reached, so a stuck render can't half-send.
|
|
137
159
|
export const deliveryRenderJob = async (payload, ctx) => {
|
|
138
160
|
const p = renderJobPayloadSchema.parse(payload);
|
|
161
|
+
return releasingKeyOnTerminalFailure(ctx, p, () => renderStep(p, ctx));
|
|
162
|
+
};
|
|
163
|
+
async function renderStep(p, ctx) {
|
|
139
164
|
const tenantId = p.tenantId; // @cast-boundary engine-payload — stream tenant
|
|
140
165
|
const { db, registry } = requireTenantScopedDeps(ctx, tenantId);
|
|
141
166
|
const channel = resolveChannel(registry, p.channelName);
|
|
142
|
-
const channelCtx = buildChannelContext(db, registry, undefined, tenantId, ctx.secrets, ctx.config, (fileTenantId) => fileContextForTenant(ctx, fileTenantId));
|
|
167
|
+
const channelCtx = buildChannelContext(db, registry, undefined, tenantId, ctx.secrets, ctx.config, (fileTenantId) => fileContextForTenant(ctx, ctx.systemUser.tenantId, fileTenantId));
|
|
143
168
|
let rendered;
|
|
144
169
|
try {
|
|
145
170
|
if (!channel.render) {
|
|
@@ -157,21 +182,28 @@ export const deliveryRenderJob = async (payload, ctx) => {
|
|
|
157
182
|
catch (err) {
|
|
158
183
|
return failAttempt(ctx, db, registry, p, "channel_error", err);
|
|
159
184
|
}
|
|
160
|
-
}
|
|
185
|
+
}
|
|
161
186
|
// Deliver via the channel using the rendered payload (if any). On failure:
|
|
162
187
|
// record the terminal failed event and rethrow so BullMQ retries the send with
|
|
163
188
|
// the same already-rendered HTML — no re-render needed.
|
|
164
189
|
export const deliverySendJob = async (payload, ctx) => {
|
|
165
190
|
const p = sendJobPayloadSchema.parse(payload);
|
|
191
|
+
return releasingKeyOnTerminalFailure(ctx, p, () => sendStep(p, ctx));
|
|
192
|
+
};
|
|
193
|
+
async function sendStep(p, ctx) {
|
|
166
194
|
const tenantId = p.tenantId; // @cast-boundary engine-payload — stream tenant
|
|
167
195
|
const { db, registry } = requireTenantScopedDeps(ctx, tenantId);
|
|
168
196
|
const channel = resolveChannel(registry, p.channelName);
|
|
169
|
-
const channelCtx = buildChannelContext(db, registry, undefined, tenantId, ctx.secrets, ctx.config, (fileTenantId) => fileContextForTenant(ctx, fileTenantId));
|
|
197
|
+
const channelCtx = buildChannelContext(db, registry, undefined, tenantId, ctx.secrets, ctx.config, (fileTenantId) => fileContextForTenant(ctx, ctx.systemUser.tenantId, fileTenantId));
|
|
170
198
|
try {
|
|
171
199
|
const result = await channel.send(p.address, toMessage(p), channelCtx, p.rendered);
|
|
172
200
|
await appendAttemptEvent(db, registry, p.deliveryAttemptId, entryFor(p, result.status, result.error ?? null, result.address ?? p.address, result.confirmed));
|
|
201
|
+
// A soft failure ends the job successfully, so no retry follows and the caller's
|
|
202
|
+
// retry must not be skipped as a duplicate.
|
|
203
|
+
if (result.status === "failed")
|
|
204
|
+
await releaseIdempotencyKey(ctx, p);
|
|
173
205
|
}
|
|
174
206
|
catch (err) {
|
|
175
207
|
return failAttempt(ctx, db, registry, p, "send_failed", err);
|
|
176
208
|
}
|
|
177
|
-
}
|
|
209
|
+
}
|
package/dist/delivery/types.d.ts
CHANGED
|
@@ -3,7 +3,7 @@ import type { DbConnection, TenantDb } from "@cosmicdrift/kumiko-framework/db";
|
|
|
3
3
|
import type { DeliveryErrorCode, NotifyDeliveryStatus, NotifyDispatched, NotifyJobDispatcher, NotifyOptions, NotifyPriority, Registry, SessionUser, TenantId } from "@cosmicdrift/kumiko-framework/engine";
|
|
4
4
|
import type { FileProviderResolver } from "@cosmicdrift/kumiko-framework/files";
|
|
5
5
|
import type { SecretsContext } from "@cosmicdrift/kumiko-framework/secrets";
|
|
6
|
-
import type { MailPayload } from "../renderer-foundation/index.js";
|
|
6
|
+
import type { MailPayload, RenderedTemplateRef } from "../renderer-foundation/index.js";
|
|
7
7
|
import type { EmailAttachmentRef } from "./attachment-ref.js";
|
|
8
8
|
import { DELIVERY_CHANNEL_EXTENSION } from "./constants.js";
|
|
9
9
|
export type RenderedMail = {
|
|
@@ -11,7 +11,11 @@ export type RenderedMail = {
|
|
|
11
11
|
readonly html: string;
|
|
12
12
|
readonly text: string;
|
|
13
13
|
readonly previewText?: string;
|
|
14
|
+
readonly fromName?: string;
|
|
15
|
+
readonly replyTo?: string;
|
|
14
16
|
readonly attachments?: readonly EmailAttachmentRef[];
|
|
17
|
+
/** The layout version the html was rendered through; the editor preview names it. */
|
|
18
|
+
readonly layout?: RenderedTemplateRef;
|
|
15
19
|
};
|
|
16
20
|
export type ChannelContext = {
|
|
17
21
|
readonly db: TenantDb;
|
|
@@ -43,6 +47,8 @@ export type RenderedMessage = {
|
|
|
43
47
|
readonly html: string;
|
|
44
48
|
readonly subject: string;
|
|
45
49
|
readonly text?: string;
|
|
50
|
+
readonly fromName?: string;
|
|
51
|
+
readonly replyTo?: string;
|
|
46
52
|
readonly attachments?: readonly EmailAttachmentRef[];
|
|
47
53
|
};
|
|
48
54
|
export declare const DELIVERY_CHANNEL_MODES: readonly ["inline", "queued"];
|
|
@@ -7,4 +7,4 @@ export declare function createSetWriteHandler(getRuntime: (() => GlobalFeatureTo
|
|
|
7
7
|
featureName: string;
|
|
8
8
|
enabled: boolean;
|
|
9
9
|
previousEnabled: boolean | null;
|
|
10
|
-
}, import("@cosmicdrift/kumiko-types/event-type-map").KumikoEventTypeMap>;
|
|
10
|
+
}, import("@cosmicdrift/kumiko-types/event-type-map").KumikoEventTypeMap, undefined>;
|
|
@@ -99,7 +99,7 @@ export const publicVariantQuery = defineQueryHandler({
|
|
|
99
99
|
if (!spec)
|
|
100
100
|
return null;
|
|
101
101
|
const result = await derivatives.variant(query.payload.fileRefId, spec, query.payload.variant);
|
|
102
|
-
const bytes = await files.
|
|
102
|
+
const bytes = await files.refFromFileRef(result).read();
|
|
103
103
|
// ponytail: fully buffered (no streaming), +33% over the wire from
|
|
104
104
|
// base64 — HttpRouteHandlerDeps only exposes {app, systemQuery} (no
|
|
105
105
|
// direct ctx.files access from the Hono handler itself), and the local
|
|
@@ -9,7 +9,7 @@ export type FileRefContent = {
|
|
|
9
9
|
export declare function fileContextForTenant(ctx: {
|
|
10
10
|
readonly _fileProviderResolver?: FileProviderResolver | undefined;
|
|
11
11
|
readonly files?: FileContext | undefined;
|
|
12
|
-
}, tenantId: string): FileContext | undefined;
|
|
12
|
+
}, contextTenantId: string, tenantId: string): FileContext | undefined;
|
|
13
13
|
export declare function fileContextFromResolver(resolver: FileProviderResolver, tenantId: string): FileContext;
|
|
14
14
|
export declare class FileRefTooLargeError extends Error {
|
|
15
15
|
readonly size: number;
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import { selectMany } from "@cosmicdrift/kumiko-framework/bun-db";
|
|
2
2
|
import { isUuid } from "@cosmicdrift/kumiko-framework/engine";
|
|
3
3
|
import { createFileContext, fileRefsTable, } from "@cosmicdrift/kumiko-framework/files";
|
|
4
|
-
// Tenant-bound on purpose:
|
|
5
|
-
|
|
4
|
+
// Tenant-bound on purpose: without a resolver `ctx.files` only reaches the storage of the
|
|
5
|
+
// context's own tenant (`contextTenantId`), so a request for another tenant's files is
|
|
6
|
+
// undefined instead of silently reading through the wrong tenant's storage.
|
|
7
|
+
export function fileContextForTenant(ctx, contextTenantId, tenantId) {
|
|
6
8
|
const resolver = ctx._fileProviderResolver;
|
|
7
9
|
if (resolver === undefined)
|
|
8
|
-
return ctx.files;
|
|
10
|
+
return tenantId === contextTenantId ? ctx.files : undefined;
|
|
9
11
|
return fileContextFromResolver(resolver, tenantId);
|
|
10
12
|
}
|
|
11
13
|
export function fileContextFromResolver(resolver, tenantId) {
|
|
@@ -44,6 +46,6 @@ export async function readFileRefContent(args) {
|
|
|
44
46
|
if (files === undefined) {
|
|
45
47
|
throw new Error("readFileRefContent: no file storage is available for the file's tenant");
|
|
46
48
|
}
|
|
47
|
-
const content = await files.
|
|
49
|
+
const content = await files.refFromFileRef(row).read();
|
|
48
50
|
return { content, contentType: row.mimeType, tenantId: row.tenantId, size };
|
|
49
51
|
}
|
|
@@ -97,6 +97,7 @@ export const fileRefTenantDestroyHook = async (ctx) => {
|
|
|
97
97
|
},
|
|
98
98
|
maxFailures: 1,
|
|
99
99
|
deadlineAt: ctx.deadlineAt ?? Number.POSITIVE_INFINITY,
|
|
100
|
+
minBatches: 1,
|
|
100
101
|
onRowError: (_row, err) => {
|
|
101
102
|
firstFailure = err instanceof Error ? err.message : String(err);
|
|
102
103
|
},
|
|
@@ -176,6 +177,9 @@ export const fileRefStorageDestroyHook = async (tenantId, ctx) => {
|
|
|
176
177
|
},
|
|
177
178
|
maxFailures: 1,
|
|
178
179
|
deadlineAt: ctx.deadlineAt ?? Number.POSITIVE_INFINITY,
|
|
180
|
+
// list() and the survivor lookup above can use up the whole budget; one chunk per tick
|
|
181
|
+
// still deletes, so the stage never reports "no progress".
|
|
182
|
+
minBatches: 1,
|
|
179
183
|
onRowError: (_key, err) => {
|
|
180
184
|
firstFailure = err instanceof Error ? err.message : String(err);
|
|
181
185
|
},
|
|
@@ -3,4 +3,4 @@ export declare const discardDraftWrite: import("@cosmicdrift/kumiko-types/define
|
|
|
3
3
|
releaseFiles: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
4
4
|
}, import("zod/v4/core").$strip>, {
|
|
5
5
|
discarded: boolean;
|
|
6
|
-
}, import("@cosmicdrift/kumiko-types/event-type-map").KumikoEventTypeMap>;
|
|
6
|
+
}, import("@cosmicdrift/kumiko-types/event-type-map").KumikoEventTypeMap, undefined>;
|
|
@@ -2,4 +2,4 @@ export declare const saveDraftWrite: import("@cosmicdrift/kumiko-types/define-ha
|
|
|
2
2
|
draftKey: import("zod").ZodString;
|
|
3
3
|
values: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodUnknown>;
|
|
4
4
|
stepIndex: import("zod").ZodNumber;
|
|
5
|
-
}, import("zod/v4/core").$strip>, import("@cosmicdrift/kumiko-types/hooks").SaveContext, import("@cosmicdrift/kumiko-types/event-type-map").KumikoEventTypeMap>;
|
|
5
|
+
}, import("zod/v4/core").$strip>, import("@cosmicdrift/kumiko-types/hooks").SaveContext, import("@cosmicdrift/kumiko-types/event-type-map").KumikoEventTypeMap, undefined>;
|
|
@@ -82,14 +82,17 @@ export function createOAuthAccessTokenManager(deps) {
|
|
|
82
82
|
const secretKey = inboundCredentialSecretKey(account.id);
|
|
83
83
|
return transaction(deps.db, async (tx) => {
|
|
84
84
|
await acquireNamespacedAdvisoryLock(tx, OAUTH_REFRESH_LOCK_NAMESPACE, lockKey(account));
|
|
85
|
-
|
|
85
|
+
// Secret read/write join the lock transaction: a second pool connection per refresh
|
|
86
|
+
// would exhaust a small pool when many accounts refresh at once.
|
|
87
|
+
const secrets = deps.secrets.withRunner(tx);
|
|
88
|
+
const stored = await secrets.get(account.tenantId, secretKey);
|
|
86
89
|
if (!stored) {
|
|
87
90
|
throw new InboundAuthError("no refresh token stored for account");
|
|
88
91
|
}
|
|
89
92
|
const currentRefreshToken = stored.reveal();
|
|
90
93
|
const tokens = await refreshWithTimeout(plugin, account, currentRefreshToken);
|
|
91
94
|
if (tokens.refreshToken && tokens.refreshToken !== currentRefreshToken) {
|
|
92
|
-
await
|
|
95
|
+
await secrets.set(account.tenantId, secretKey, tokens.refreshToken, inboundRefreshTokenSecretOptions(account.id));
|
|
93
96
|
}
|
|
94
97
|
remember(account.id, tokens);
|
|
95
98
|
return tokens.accessToken;
|
|
@@ -117,7 +120,9 @@ export function createOAuthAccessTokenManager(deps) {
|
|
|
117
120
|
async storeRefreshToken(account, refreshToken) {
|
|
118
121
|
await transaction(deps.db, async (tx) => {
|
|
119
122
|
await acquireNamespacedAdvisoryLock(tx, OAUTH_REFRESH_LOCK_NAMESPACE, lockKey(account));
|
|
120
|
-
await deps.secrets
|
|
123
|
+
await deps.secrets
|
|
124
|
+
.withRunner(tx)
|
|
125
|
+
.set(account.tenantId, inboundCredentialSecretKey(account.id), refreshToken, inboundRefreshTokenSecretOptions(account.id));
|
|
121
126
|
});
|
|
122
127
|
cache.delete(account.id);
|
|
123
128
|
},
|
|
@@ -120,8 +120,10 @@ export function createInboundMailSupervisor(deps) {
|
|
|
120
120
|
: row;
|
|
121
121
|
const decryptedAddress = decrypted["address"];
|
|
122
122
|
// Falling back to row.address would hand the ciphertext to a provider.
|
|
123
|
+
// One undecryptable account must not stop polling of every other tenant's accounts.
|
|
123
124
|
if (typeof decryptedAddress !== "string") {
|
|
124
|
-
|
|
125
|
+
log(`inbound-mail: account ${row.id} skipped — address did not decrypt to text`);
|
|
126
|
+
continue;
|
|
125
127
|
}
|
|
126
128
|
records.push({
|
|
127
129
|
id: row.id,
|
|
@@ -5,4 +5,4 @@ export type StaleRunSweepResult = {
|
|
|
5
5
|
readonly tenantRunsMarkedFailed: number;
|
|
6
6
|
readonly tenantQueuedDropped: number;
|
|
7
7
|
};
|
|
8
|
-
export declare function markStaleJobRunsFailed(db: DbConnection, timeoutHours: number): Promise<StaleRunSweepResult>;
|
|
8
|
+
export declare function markStaleJobRunsFailed(db: DbConnection, timeoutHours: number, queuedRetentionDays?: number): Promise<StaleRunSweepResult>;
|
|
@@ -13,20 +13,26 @@
|
|
|
13
13
|
// (retry.write.ts only allows retry from "failed").
|
|
14
14
|
import { deleteManyBatched, selectMany, updateMany } from "@cosmicdrift/kumiko-framework/bun-db";
|
|
15
15
|
import { mapWithConcurrency } from "../../../shared/index.js";
|
|
16
|
+
import { DEFAULT_JOB_RUN_RETENTION_DAYS } from "../../handlers/retention-cleanup.job.js";
|
|
16
17
|
import { encryptFailureError } from "../../job-run-logger.js";
|
|
17
18
|
import { jobRunsTable } from "../../job-run-table.js";
|
|
18
19
|
import { tenantJobRunsTable } from "../../tenant-job-run-table.js";
|
|
19
20
|
const KMS_POOL_CONCURRENCY = 4;
|
|
20
21
|
const STALE_DELETE_BATCH_SIZE = 500;
|
|
21
22
|
export const STALE_JOB_RUN_ERROR = "job run exceeded the stale-run timeout without a completion signal (likely a crashed worker process)";
|
|
22
|
-
|
|
23
|
+
// A queued tenant row can legitimately wait much longer than a run takes (backlog, delay/backoff,
|
|
24
|
+
// sequential requeue), so it is only treated as lost once it is older than the retention window,
|
|
25
|
+
// not after the running-timeout.
|
|
26
|
+
export async function markStaleJobRunsFailed(db, timeoutHours, queuedRetentionDays = DEFAULT_JOB_RUN_RETENTION_DAYS) {
|
|
23
27
|
const cutoff = Temporal.Now.instant().subtract({ hours: timeoutHours });
|
|
28
|
+
const queuedCutoff = Temporal.Now.instant().subtract({ hours: queuedRetentionDays * 24 });
|
|
24
29
|
const now = Temporal.Now.instant();
|
|
25
30
|
// Fetch the matched rows first: `error` carries the same per-triggering-
|
|
26
31
|
// user subject annotation as job-run-logger.ts's onJobFailed path
|
|
27
32
|
// (personal: { of: "triggeredById" } on job-run-table.ts), so it is encrypted
|
|
28
|
-
// under
|
|
29
|
-
//
|
|
33
|
+
// under the triggering user's DEK, or the system tenant key when
|
|
34
|
+
// triggeredById is null (whenAbsent: "tenant", fw#3654). Runs are grouped by
|
|
35
|
+
// that subject: one encrypt and one updateMany per subject instead of per run.
|
|
30
36
|
const stale = await selectMany(db, jobRunsTable, {
|
|
31
37
|
status: "running",
|
|
32
38
|
startedAt: { lt: cutoff },
|
|
@@ -61,9 +67,10 @@ export async function markStaleJobRunsFailed(db, timeoutHours) {
|
|
|
61
67
|
const runsMarkedFailed = updatedPerSubject.reduce((sum, count) => sum + count, 0);
|
|
62
68
|
// Tenant-visible run state follows the same clock: a running row whose
|
|
63
69
|
// worker died is a failure, a queued row that never started (job lost from
|
|
64
|
-
// Redis, enqueue hook without a run) would otherwise show "waiting" forever
|
|
70
|
+
// Redis, enqueue hook without a run) would otherwise show "waiting" forever;
|
|
71
|
+
// it is dropped once it has outlived the retention window.
|
|
65
72
|
const tenantRunsFailed = await updateMany(db, tenantJobRunsTable, { status: "failed", finishedAt: now, updatedAt: now }, { status: "running", startedAt: { lt: cutoff } });
|
|
66
|
-
const tenantQueuedStale = await deleteManyBatched(db, tenantJobRunsTable, { status: "queued", updatedAt: { lt:
|
|
73
|
+
const tenantQueuedStale = await deleteManyBatched(db, tenantJobRunsTable, { status: "queued", updatedAt: { lt: queuedCutoff } }, { limit: STALE_DELETE_BATCH_SIZE });
|
|
67
74
|
return {
|
|
68
75
|
runsMarkedFailed,
|
|
69
76
|
tenantRunsMarkedFailed: tenantRunsFailed.length,
|
package/dist/jobs/feature.js
CHANGED
|
@@ -19,7 +19,7 @@ export function createJobsFeature(options = {}) {
|
|
|
19
19
|
const retentionDays = options.retentionDays ?? DEFAULT_JOB_RUN_RETENTION_DAYS;
|
|
20
20
|
const staleRunTimeoutHours = options.staleRunTimeoutHours ?? DEFAULT_JOB_RUN_STALE_TIMEOUT_HOURS;
|
|
21
21
|
return defineFeature("jobs", (r) => {
|
|
22
|
-
r.describe("Persistence and operator tooling for background jobs registered via `r.job(...)`. Every job execution writes directly into `store_job_runs` (current status + duration) and `store_job_run_logs` (per-line log rows) from the BullMQ callbacks — no event stream in between (#2243). A daily `retention-cleanup` job deletes runs (and their logs) older than `retentionDays`; an hourly `stale-run-sweep` job marks runs stuck at status `running` past `staleRunTimeoutHours` as `failed` (#2246 — a crashed worker never fires the completion callback, so nothing else ever revisits the row). Exposes `jobs:write:trigger` (manual run) and `jobs:write:retry` (operator retry of a failed run), plus `jobs:query:list`, `jobs:query:details`, and `jobs:query:catalog` (manual jobs) for the operator UI. A job that declares `tenantVisibleFailure` also records its last failed attempt per tenant and subject in `store_tenant_job_failures`, which the tenant itself reads through `jobs:query:failures` — a translation key only, never the provider's message (fw#3079). A job that declares `tenantVisibleRun` also keeps its run state per tenant and subject in `store_tenant_job_runs` (queued, running, plus the latest completed or failed run, no payload, error text or logs), which the tenant reads through `jobs:query:tenant-runs` (fw#3616); the stale-run sweep fails its stuck `running` rows and drops
|
|
22
|
+
r.describe("Persistence and operator tooling for background jobs registered via `r.job(...)`. Every job execution writes directly into `store_job_runs` (current status + duration) and `store_job_run_logs` (per-line log rows) from the BullMQ callbacks — no event stream in between (#2243). A daily `retention-cleanup` job deletes runs (and their logs) older than `retentionDays`; an hourly `stale-run-sweep` job marks runs stuck at status `running` past `staleRunTimeoutHours` as `failed` (#2246 — a crashed worker never fires the completion callback, so nothing else ever revisits the row). Exposes `jobs:write:trigger` (manual run) and `jobs:write:retry` (operator retry of a failed run), plus `jobs:query:list`, `jobs:query:details`, and `jobs:query:catalog` (manual jobs) for the operator UI. A job that declares `tenantVisibleFailure` also records its last failed attempt per tenant and subject in `store_tenant_job_failures`, which the tenant itself reads through `jobs:query:failures` — a translation key only, never the provider's message (fw#3079). A job that declares `tenantVisibleRun` also keeps its run state per tenant and subject in `store_tenant_job_runs` (queued, running, plus the latest completed or failed run, no payload, error text or logs), which the tenant reads through `jobs:query:tenant-runs` (fw#3616); the stale-run sweep fails its stuck `running` rows and drops `queued` rows older than `retentionDays`.");
|
|
23
23
|
r.uiHints({
|
|
24
24
|
displayLabel: "Jobs · Audit & Operator UI",
|
|
25
25
|
category: "operations",
|
|
@@ -28,7 +28,9 @@ export function createJobsFeature(options = {}) {
|
|
|
28
28
|
r.systemScope();
|
|
29
29
|
r.storeTable(jobRunsTableMeta, {
|
|
30
30
|
reason: "direct_write.job_runs",
|
|
31
|
-
// payload/error carry personal:{of:"triggeredById"} (job-run-table.ts)
|
|
31
|
+
// payload/error carry personal:{of:"triggeredById"} (job-run-table.ts): encrypted under the
|
|
32
|
+
// triggering user's DEK, or the system tenant key when triggeredById is null
|
|
33
|
+
// (whenAbsent: "tenant", fw#3654).
|
|
32
34
|
// job-run-logger.ts manually encrypts both before every insert/update
|
|
33
35
|
// (encryptStartedPayload/encryptFailureError) since this store bypasses
|
|
34
36
|
// the executor's automatic PII pipeline (#2243).
|
|
@@ -63,7 +65,7 @@ export function createJobsFeature(options = {}) {
|
|
|
63
65
|
// frequent enough to catch a stuck run soon after it crosses the
|
|
64
66
|
// threshold, cheap enough that an (almost always empty) result set
|
|
65
67
|
// doesn't matter.
|
|
66
|
-
r.job("stale-run-sweep", { trigger: { cron: "0 * * * *" }, concurrency: "skip" }, createStaleRunSweepJob(staleRunTimeoutHours));
|
|
68
|
+
r.job("stale-run-sweep", { trigger: { cron: "0 * * * *" }, concurrency: "skip" }, createStaleRunSweepJob(staleRunTimeoutHours, retentionDays));
|
|
67
69
|
const handlers = {
|
|
68
70
|
trigger: r.writeHandler(triggerWrite),
|
|
69
71
|
retry: r.writeHandler(retryWrite),
|
|
@@ -7,6 +7,8 @@ import { decryptStoredPii, mapWithConcurrency } from "../../shared/index.js";
|
|
|
7
7
|
import { jobRunsTable } from "../job-run-table.js";
|
|
8
8
|
const KMS_POOL_CONCURRENCY = 4;
|
|
9
9
|
const DEFAULT_PAGE_SIZE = 50;
|
|
10
|
+
// The offset cursor re-reads everything up to the page end, so the page size bounds that read.
|
|
11
|
+
const MAX_LIST_LIMIT = 200;
|
|
10
12
|
const jobRunStatusSchema = z.enum(["queued", "running", "completed", "failed"]);
|
|
11
13
|
async function decryptRunRow(row) {
|
|
12
14
|
let result = row;
|
|
@@ -56,7 +58,7 @@ export const listQuery = defineQueryHandler({
|
|
|
56
58
|
.optional(),
|
|
57
59
|
sort: z.enum(["jobName", "status", "startedAt", "duration"]).optional(),
|
|
58
60
|
sortDirection: z.enum(["asc", "desc"]).optional(),
|
|
59
|
-
limit: z.number().int().min(1).optional(),
|
|
61
|
+
limit: z.number().int().min(1).max(MAX_LIST_LIMIT).optional(),
|
|
60
62
|
cursor: z.string().optional(),
|
|
61
63
|
totalCount: z.boolean().optional(),
|
|
62
64
|
}),
|
|
@@ -5,4 +5,4 @@ export declare const retryWrite: import("@cosmicdrift/kumiko-types/define-handle
|
|
|
5
5
|
jobName: string;
|
|
6
6
|
bullJobId: string;
|
|
7
7
|
retriedFromRunId: string;
|
|
8
|
-
}, import("@cosmicdrift/kumiko-types/event-type-map").KumikoEventTypeMap>;
|
|
8
|
+
}, import("@cosmicdrift/kumiko-types/event-type-map").KumikoEventTypeMap, undefined>;
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
import type { JobHandlerFn } from "@cosmicdrift/kumiko-framework/engine";
|
|
2
2
|
export declare const DEFAULT_JOB_RUN_STALE_TIMEOUT_HOURS = 24;
|
|
3
|
-
export declare function createStaleRunSweepJob(timeoutHours: number): JobHandlerFn;
|
|
3
|
+
export declare function createStaleRunSweepJob(timeoutHours: number, queuedRetentionDays: number): JobHandlerFn;
|
|
@@ -9,7 +9,7 @@ import { markStaleJobRunsFailed } from "../db/queries/stale-run-sweep.js";
|
|
|
9
9
|
// default has to clear any plausible real job, not just be "long".
|
|
10
10
|
// Configurable per-app via JobsFeatureOptions.staleRunTimeoutHours.
|
|
11
11
|
export const DEFAULT_JOB_RUN_STALE_TIMEOUT_HOURS = 24;
|
|
12
|
-
export function createStaleRunSweepJob(timeoutHours) {
|
|
12
|
+
export function createStaleRunSweepJob(timeoutHours, queuedRetentionDays) {
|
|
13
13
|
return async (_payload, ctx) => {
|
|
14
14
|
if (!ctx.systemDb) {
|
|
15
15
|
throw new InternalError({
|
|
@@ -17,7 +17,7 @@ export function createStaleRunSweepJob(timeoutHours) {
|
|
|
17
17
|
});
|
|
18
18
|
}
|
|
19
19
|
const db = ctx.systemDb.unsafeRaw("stale-run sweep marks stuck runs of every tenant as failed"); // @cast-boundary db-operator — DbRunner narrows to DbConnection, jobs never run inside a DbTx
|
|
20
|
-
const result = await markStaleJobRunsFailed(db, timeoutHours);
|
|
20
|
+
const result = await markStaleJobRunsFailed(db, timeoutHours, queuedRetentionDays);
|
|
21
21
|
ctx.log?.info?.(`[jobs:stale-run-sweep] complete: ${JSON.stringify(result)}`);
|
|
22
22
|
};
|
|
23
23
|
}
|
|
@@ -5,4 +5,4 @@ export declare const triggerWrite: import("@cosmicdrift/kumiko-types/define-hand
|
|
|
5
5
|
}, z.core.$strip>, {
|
|
6
6
|
jobName: string;
|
|
7
7
|
bullJobId: string;
|
|
8
|
-
}, import("@cosmicdrift/kumiko-types/event-type-map").KumikoEventTypeMap>;
|
|
8
|
+
}, import("@cosmicdrift/kumiko-types/event-type-map").KumikoEventTypeMap, undefined>;
|
|
@@ -103,6 +103,7 @@ async function encryptStartedPayload(payload, triggeredById) {
|
|
|
103
103
|
const TENANT_JOB_FAILURE_LOCK_NAMESPACE = 0x746a666c;
|
|
104
104
|
// 'tjrl': serialises the finished-row write + prune per tenant run key.
|
|
105
105
|
const TENANT_JOB_RUN_LOCK_NAMESPACE = 0x746a726c;
|
|
106
|
+
const TENANT_RUN_CONFLICT_KEYS = ["tenantId", "jobName", "bullJobId"];
|
|
106
107
|
// fw#3079 — which row the tenant-visible failure record lives in: one per
|
|
107
108
|
// (tenant, job, subject), so the next outcome of the same work replaces or
|
|
108
109
|
// clears it. `subject` is null for a job that declares no subjectFields.
|
|
@@ -156,9 +157,22 @@ async function recordTenantRunQueued(db, jobName, bullJobId, meta) {
|
|
|
156
157
|
if (!key)
|
|
157
158
|
return;
|
|
158
159
|
const now = Temporal.Now.instant();
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
160
|
+
await transaction(db, async (tx) => {
|
|
161
|
+
// A finished row under the same id is left over from before a Redis reset
|
|
162
|
+
// (ids restart at 1), not this job. Only rows that went through this hook
|
|
163
|
+
// (queuedAt set) qualify: a fast worker can finish before the hook runs,
|
|
164
|
+
// and that row is this job's own and must stay.
|
|
165
|
+
await deleteMany(tx, tenantJobRunsTable, {
|
|
166
|
+
tenantId: key.tenantId,
|
|
167
|
+
jobName,
|
|
168
|
+
bullJobId,
|
|
169
|
+
status: { in: ["completed", "failed"] },
|
|
170
|
+
queuedAt: { gt: Temporal.Instant.fromEpochMilliseconds(0) },
|
|
171
|
+
});
|
|
172
|
+
// DO NOTHING: a fast worker can report the start before this enqueue hook
|
|
173
|
+
// returns, and a repeated jobId (debounce, fan-out dedup) is the same job.
|
|
174
|
+
await insertOnConflictDoNothing(tx, tenantJobRunsTable, { ...key, bullJobId, status: "queued", queuedAt: now, updatedAt: now }, { conflictKeys: [...TENANT_RUN_CONFLICT_KEYS] });
|
|
175
|
+
});
|
|
162
176
|
}
|
|
163
177
|
async function recordTenantRunStarted(db, jobName, bullJobId, meta) {
|
|
164
178
|
// skip: job did not opt in to tenantVisibleRun
|
|
@@ -170,7 +184,7 @@ async function recordTenantRunStarted(db, jobName, bullJobId, meta) {
|
|
|
170
184
|
return;
|
|
171
185
|
const now = Temporal.Now.instant();
|
|
172
186
|
await upsertOnConflict(db, tenantJobRunsTable, { ...key, bullJobId, status: "running", startedAt: now, updatedAt: now }, {
|
|
173
|
-
conflictKeys: [
|
|
187
|
+
conflictKeys: [...TENANT_RUN_CONFLICT_KEYS],
|
|
174
188
|
update: { status: "running", startedAt: now, finishedAt: null, updatedAt: now },
|
|
175
189
|
});
|
|
176
190
|
}
|
|
@@ -186,7 +200,7 @@ async function recordTenantRunOutcome(db, jobName, bullJobId, outcome, result) {
|
|
|
186
200
|
// A failed attempt BullMQ will retry goes back to waiting: the tenant keeps
|
|
187
201
|
// seeing pending work, not a failure the next attempt may still resolve.
|
|
188
202
|
if (result === "failed" && !outcome.finalAttempt) {
|
|
189
|
-
await updateMany(db, tenantJobRunsTable, { status: "queued", startedAt: null, finishedAt: null, updatedAt: now }, { bullJobId });
|
|
203
|
+
await updateMany(db, tenantJobRunsTable, { status: "queued", startedAt: null, finishedAt: null, updatedAt: now }, { tenantId: key.tenantId, jobName, bullJobId });
|
|
190
204
|
// skip: retry pending, the row stays active until the final attempt
|
|
191
205
|
return;
|
|
192
206
|
}
|
|
@@ -196,7 +210,10 @@ async function recordTenantRunOutcome(db, jobName, bullJobId, outcome, result) {
|
|
|
196
210
|
// both behind.
|
|
197
211
|
await transaction(db, async (tx) => {
|
|
198
212
|
await acquireNamespacedAdvisoryLock(tx, TENANT_JOB_RUN_LOCK_NAMESPACE, JSON.stringify([key.tenantId, key.jobName, key.subject]));
|
|
199
|
-
await upsertOnConflict(tx, tenantJobRunsTable, { ...key, bullJobId, status: result, finishedAt: now, updatedAt: now }, {
|
|
213
|
+
await upsertOnConflict(tx, tenantJobRunsTable, { ...key, bullJobId, status: result, finishedAt: now, updatedAt: now }, {
|
|
214
|
+
conflictKeys: [...TENANT_RUN_CONFLICT_KEYS],
|
|
215
|
+
update: { status: result, finishedAt: now, updatedAt: now },
|
|
216
|
+
});
|
|
200
217
|
await deleteMany(tx, tenantJobRunsTable, {
|
|
201
218
|
...key,
|
|
202
219
|
status: { in: ["completed", "failed"] },
|
|
@@ -204,8 +221,13 @@ async function recordTenantRunOutcome(db, jobName, bullJobId, outcome, result) {
|
|
|
204
221
|
});
|
|
205
222
|
});
|
|
206
223
|
}
|
|
207
|
-
async function dropTenantRunQueued(db, bullJobId) {
|
|
208
|
-
await deleteMany(db, tenantJobRunsTable, {
|
|
224
|
+
async function dropTenantRunQueued(db, jobName, bullJobId, meta) {
|
|
225
|
+
await deleteMany(db, tenantJobRunsTable, {
|
|
226
|
+
tenantId: meta.tenantId,
|
|
227
|
+
jobName,
|
|
228
|
+
bullJobId,
|
|
229
|
+
status: "queued",
|
|
230
|
+
});
|
|
209
231
|
}
|
|
210
232
|
const log = createFallbackLogger("job-run-logger");
|
|
211
233
|
// The tenant run state is an optional side write: a throw here must not
|
|
@@ -287,8 +309,8 @@ export function createJobRunLogger(opts) {
|
|
|
287
309
|
onJobQueued: async (jobName, bullJobId, meta) => {
|
|
288
310
|
await isolateTenantRunWrite(() => recordTenantRunQueued(db, jobName, bullJobId, meta), jobName, bullJobId);
|
|
289
311
|
},
|
|
290
|
-
onJobDropped: async (jobName, bullJobId) => {
|
|
291
|
-
await isolateTenantRunWrite(() => dropTenantRunQueued(db, bullJobId), jobName, bullJobId);
|
|
312
|
+
onJobDropped: async (jobName, bullJobId, meta) => {
|
|
313
|
+
await isolateTenantRunWrite(() => dropTenantRunQueued(db, jobName, bullJobId, meta), jobName, bullJobId);
|
|
292
314
|
},
|
|
293
315
|
onJobStart: async (jobName, bullJobId, meta) => {
|
|
294
316
|
await isolateTenantRunWrite(() => recordTenantRunStarted(db, jobName, bullJobId, meta), jobName, bullJobId);
|
|
@@ -8,7 +8,7 @@ import { index, instant, table as pgTable, serial, sql, text, uniqueIndex, uuid,
|
|
|
8
8
|
// job is invisible there. This table holds no payload, error text or logs by
|
|
9
9
|
// design — the tenant learns only that work is waiting, running or finished.
|
|
10
10
|
//
|
|
11
|
-
// One row per BullMQ job. Active rows (queued/running) are kept as they are;
|
|
11
|
+
// One row per BullMQ job and (tenant, job name). Active rows (queued/running) are kept as they are;
|
|
12
12
|
// a finished row replaces earlier finished rows of the same (tenant, job,
|
|
13
13
|
// subject), so only the latest outcome stays. `subject` is the canonical JSON
|
|
14
14
|
// of the job's declared subjectFields, or NULL for none — stored in clear, so
|
|
@@ -25,7 +25,8 @@ export const tenantJobRunsTable = pgTable("store_tenant_job_runs", {
|
|
|
25
25
|
finishedAt: instant("finished_at"),
|
|
26
26
|
updatedAt: instant("updated_at").default(sql `now()`).notNull(),
|
|
27
27
|
}, (t) => [
|
|
28
|
-
|
|
28
|
+
// BullMQ ids are unique per queue only; the id alone would let two tenants collide.
|
|
29
|
+
uniqueIndex("store_tenant_job_runs_tenant_job_bull_job_id_unique").on(t.tenantId, t.jobName, t.bullJobId),
|
|
29
30
|
// Finishing a run prunes older outcomes by this key; the tenant query reads by it.
|
|
30
31
|
index("store_tenant_job_runs_tenant_job_subject_idx").on(t.tenantId, t.jobName, t.subject),
|
|
31
32
|
// Stale sweep and retention filter on age alone.
|
package/dist/ledger/entity.js
CHANGED
|
@@ -160,7 +160,9 @@ export const scheduleEntity = createEntity({
|
|
|
160
160
|
},
|
|
161
161
|
// Find-then-create by sourceRef races across tabs/devices; only the DB closes
|
|
162
162
|
// that gap. Partial because sourceRef is optional. A new name (not the old
|
|
163
|
-
// non-unique default) keeps the migration an in-place CREATE UNIQUE INDEX
|
|
163
|
+
// non-unique default) keeps the migration an in-place CREATE UNIQUE INDEX;
|
|
164
|
+
// that statement, like a projection rebuild, fails on existing duplicate
|
|
165
|
+
// (tenant_id, source_ref) pairs, so duplicates must be cleaned up beforehand.
|
|
164
166
|
indexes: [
|
|
165
167
|
{ columns: ["tenantId", "subjectType", "subjectId"] },
|
|
166
168
|
{
|
|
@@ -12,4 +12,4 @@ export declare const setWrite: import("@cosmicdrift/kumiko-types/define-handler"
|
|
|
12
12
|
slug: string;
|
|
13
13
|
lang: string;
|
|
14
14
|
isNew: boolean;
|
|
15
|
-
}, import("@cosmicdrift/kumiko-types/event-type-map").KumikoEventTypeMap>;
|
|
15
|
+
}, import("@cosmicdrift/kumiko-types/event-type-map").KumikoEventTypeMap, undefined>;
|
|
@@ -17,4 +17,4 @@ export declare function createPatCreateHandler(opts?: CreatePatOptions): import(
|
|
|
17
17
|
id: string;
|
|
18
18
|
token: string;
|
|
19
19
|
prefix: string;
|
|
20
|
-
}, import("@cosmicdrift/kumiko-types/event-type-map").KumikoEventTypeMap>;
|
|
20
|
+
}, import("@cosmicdrift/kumiko-types/event-type-map").KumikoEventTypeMap, undefined>;
|
|
@@ -3,4 +3,4 @@ export declare const revokePatWrite: import("@cosmicdrift/kumiko-types/define-ha
|
|
|
3
3
|
id: z.ZodUUID;
|
|
4
4
|
}, z.core.$strip>, {
|
|
5
5
|
id: string;
|
|
6
|
-
}, import("@cosmicdrift/kumiko-types/event-type-map").KumikoEventTypeMap>;
|
|
6
|
+
}, import("@cosmicdrift/kumiko-types/event-type-map").KumikoEventTypeMap, undefined>;
|