@cosmicdrift/kumiko-framework 0.365.0 → 0.365.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -68,7 +68,7 @@ bun test
68
68
  ```ts
69
69
  // webhook-step integration test — drives r.step.webhook.send end-to-end:
70
70
  // handler appends step.dispatch-requested in TX, dispatcher subscription
71
- // drains after COMMIT and calls the (stubbed) fetch, follow-up
71
+ // enqueues a delivery job after COMMIT, the job calls the (stubbed) fetch, follow-up
72
72
  // step.dispatched / step.dispatch-failed events land on the same stream.
73
73
 
74
74
  import { afterAll, beforeAll, beforeEach, describe, expect, mock, test } from "bun:test";
@@ -140,6 +140,7 @@ beforeAll(async () => {
140
140
  stack = await setupTestStack({
141
141
  features: [createStepDispatcherFeature(), createSecretsFeature(), webhookDemoFeature],
142
142
  systemHooks: [],
143
+ jobs: { consumerLane: "worker" },
143
144
  extraContext: ({ db, registry }) => ({
144
145
  secrets: createSecretsContext({ db, masterKeyProvider, registry }),
145
146
  }),
@@ -184,6 +185,7 @@ describe("webhook-step Sample", () => {
184
185
 
185
186
  // Drain dispatcher (MSP runs async via runOnce in test mode)
186
187
  await stack.eventDispatcher?.runOnce();
188
+ await stack.drainJobs();
187
189
 
188
190
  expect(fetchMock).toHaveBeenCalledTimes(1);
189
191
  const [calledUrl, init] = fetchMock.mock.calls[0]!;
@@ -202,6 +204,7 @@ describe("webhook-step Sample", () => {
202
204
  expect(id).toMatch(/^[0-9a-f-]{36}$/);
203
205
 
204
206
  await stack.eventDispatcher?.runOnce();
207
+ await stack.drainJobs();
205
208
 
206
209
  expect(mailMock).toHaveBeenCalledTimes(1);
207
210
  expect(mailMock).toHaveBeenCalledWith(
@@ -231,6 +234,7 @@ describe("webhook-step Sample", () => {
231
234
 
232
235
  // And the inner handler's webhook fired
233
236
  await stack.eventDispatcher?.runOnce();
237
+ await stack.drainJobs();
234
238
  expect(fetchMock).toHaveBeenCalledTimes(1);
235
239
  });
236
240
 
@@ -243,6 +247,7 @@ describe("webhook-step Sample", () => {
243
247
  expect(res.status).toBe(500);
244
248
 
245
249
  await stack.eventDispatcher?.runOnce();
250
+ await stack.drainJobs();
246
251
 
247
252
  expect(fetchMock).not.toHaveBeenCalled();
248
253
  const rows = await selectMany(stack.db, incidentTable);
@@ -261,6 +266,7 @@ describe("access-denied — role-restricted handlers", () => {
261
266
  );
262
267
  expect(error.code).toBe("access_denied");
263
268
  await stack.eventDispatcher?.runOnce();
269
+ await stack.drainJobs();
264
270
  expect(mailMock).not.toHaveBeenCalled();
265
271
  });
266
272
 
@@ -287,6 +293,7 @@ describe("access-denied — role-restricted handlers", () => {
287
293
  );
288
294
  expect(error.code).toBe("access_denied");
289
295
  await stack.eventDispatcher?.runOnce();
296
+ await stack.drainJobs();
290
297
  expect(fetchMock).not.toHaveBeenCalled();
291
298
  });
292
299
  });
@@ -317,6 +324,7 @@ describe("incident:open-authenticated — tenant-owned webhook auth secret", ()
317
324
  adminB,
318
325
  );
319
326
  await stack.eventDispatcher?.runOnce();
327
+ await stack.drainJobs();
320
328
 
321
329
  expect(fetchMock).toHaveBeenCalledTimes(1);
322
330
  const [, init] = fetchMock.mock.calls[0]!;
@@ -348,6 +356,7 @@ describe("incident:open-authenticated — tenant-owned webhook auth secret", ()
348
356
  adminA,
349
357
  );
350
358
  await stack.eventDispatcher?.runOnce();
359
+ await stack.drainJobs();
351
360
 
352
361
  expect(fetchMock).not.toHaveBeenCalled();
353
362
 
@@ -378,6 +387,7 @@ describe("incident:open-authenticated — tenant-owned webhook auth secret", ()
378
387
  adminC,
379
388
  );
380
389
  await stack.eventDispatcher?.runOnce();
390
+ await stack.drainJobs();
381
391
 
382
392
  expect(fetchMock).not.toHaveBeenCalled();
383
393
  const failedEvents = await selectMany(stack.db, eventsTable, {
@@ -594,7 +594,7 @@ Exposes APIs: `sessions.revokeAllForUser`
594
594
 
595
595
  ## step-dispatcher
596
596
 
597
- Internal system feature that drains deferred Tier-2 side-effects (currently `webhook.send` and `mail.send`) after their originating transaction commits. Listens via `r.multiStreamProjection` on the `kumiko:system:step.dispatch-requested` system event, performs the actual HTTP or mail delivery, then appends `kumiko:system:step.dispatched` or `kumiko:system:step.dispatch-failed` back onto the same stream so the outcome is recorded in the event log without a separate status table. Mount this feature explicitly via `createStepDispatcherFeature()` in your app's feature list alongside any features that use `r.step.webhook.send` or `r.step.mail.send`. Requires the `secrets` feature (`createSecretsFeature()`) to be mounted — `webhook.send` auth resolves per-tenant through it, under `step-dispatcher:webhook-auth.<name>`. Every `webhook.send` request carries `Idempotency-Key: <dispatch stream id>`, stable across redeliveries of the same dispatch request; an explicit `Idempotency-Key` in `headers` takes precedence.
597
+ Internal system feature that drains deferred Tier-2 side-effects (currently `webhook.send` and `mail.send`) after their originating transaction commits. An `r.multiStreamProjection` on the `kumiko:system:step.dispatch-requested` system event enqueues one `step-dispatcher:job:deliver` BullMQ job per dispatch (Redis carries only the stream id and tenant, never the request payload). The job performs the actual HTTP or mail delivery and appends `kumiko:system:step.dispatched` or `kumiko:system:step.dispatch-failed` back onto the same stream, so the outcome is recorded in the event log without a separate status table. Transient failures (network error, timeout, 5xx, 408, 429, any mail adapter failure) are retried 3 times with exponential backoff; the outcome is recorded after the final attempt and the payload key is erased only then. Permanent failures (invalid url, blocked host, other 4xx, unavailable auth secret) are recorded immediately. At most 2 deliveries per tenant run at once (`maxPerTenant` with `onLimit: "defer"`), so a slow receiver of one tenant does not delay other tenants. Requires a job runner (Redis) and a worker consuming the job's default `worker` lane; without a job runner the projection fails loudly. Mount this feature explicitly via `createStepDispatcherFeature()` in your app's feature list alongside any features that use `r.step.webhook.send` or `r.step.mail.send`. Requires the `secrets` feature (`createSecretsFeature()`) to be mounted — `webhook.send` auth resolves per-tenant through it, under `step-dispatcher:webhook-auth.<name>`. Every `webhook.send` request carries `Idempotency-Key: <dispatch stream id>`, stable across retries and redeliveries of the same dispatch request; an explicit `Idempotency-Key` in `headers` takes precedence.
598
598
 
599
599
  Category: infrastructure
600
600
 
@@ -310,6 +310,13 @@ export function buildServer(options) {
310
310
  const searchConsumerEnabled = options.eventDispatcher?.systemConsumers?.search ?? true;
311
311
  const jobTriggerConsumerEnabled = options.eventDispatcher?.systemConsumers?.jobTrigger ?? true;
312
312
  const jobRunnerForTriggers = options.dispatcherOptions?.jobRunner;
313
+ const mspJobs = jobRunnerForTriggers?.dispatch
314
+ ? {
315
+ dispatch: async (jobName, payload, dispatchOptions) => {
316
+ await jobRunnerForTriggers.dispatch?.(jobName, payload, dispatchOptions);
317
+ },
318
+ }
319
+ : undefined;
313
320
  const systemConsumers = [];
314
321
  if (sseConsumerEnabled) {
315
322
  systemConsumers.push(createSseBroadcastEventConsumer(sseBroker));
@@ -391,6 +398,7 @@ export function buildServer(options) {
391
398
  ...(mspOwner && { callerFeature: mspOwner }),
392
399
  ...(mspFiles && { files: mspFiles }),
393
400
  ...(contextWithObservability.secrets && { secrets: contextWithObservability.secrets }),
401
+ ...(mspJobs && { jobs: mspJobs }),
394
402
  ...(mspFiles && {
395
403
  derivatives: createDerivativesContext({
396
404
  files: mspFiles,
@@ -1,7 +1,8 @@
1
1
  import { randomBytes } from "node:crypto";
2
2
  import { KeyAlreadyExistsError, KeyErasedError, KeyNotFoundError, subjectIdToKey, } from "./kms-adapter.js";
3
3
  // Non-persistent adapter for tests and dev mode. Erased entries stay as
4
- // tombstones so the create-after-erase contract holds within a process.
4
+ // tombstones (also for never-keyed subjects) so the create-after-erase
5
+ // contract holds within a process.
5
6
  export class InMemoryKmsAdapter {
6
7
  capabilities = { mode: "local-key" };
7
8
  keys = new Map();
@@ -20,12 +21,8 @@ export class InMemoryKmsAdapter {
20
21
  return entry.key;
21
22
  }
22
23
  async eraseKey(subject) {
23
- const entry = this.keys.get(subjectIdToKey(subject));
24
- // skip: eraseKey is contractually idempotent — unknown subject is a no-op
25
- if (!entry)
26
- return;
27
- entry.key = null;
28
- entry.erased = true;
24
+ // A never-keyed subject gets a tombstone too, so createKey cannot mint one later.
25
+ this.keys.set(subjectIdToKey(subject), { key: null, erased: true });
29
26
  }
30
27
  async health() {
31
28
  return { ok: true, latencyMs: 0 };
@@ -107,15 +107,19 @@ export class PgKmsAdapter {
107
107
  }
108
108
  async eraseKey(subject, ctx) {
109
109
  await this.ensureSchema();
110
- // Guard on erased_at IS NULL keeps repeat erases from overwriting the
111
- // original tombstone audit fields; unknown subjects update zero rows.
110
+ // Upsert: a never-keyed subject (pre-KMS plaintext) still gets a tombstone,
111
+ // so a later createKey cannot mint a fresh key and re-encrypt the plaintext.
112
+ // The erased_at IS NULL guard keeps repeat erases from overwriting the
113
+ // original tombstone audit fields.
112
114
  await this.sql `
113
- UPDATE kumiko_subject_keys
115
+ INSERT INTO kumiko_subject_keys (subject_id, cipher_key, kek_version, erased_at, erased_by, erase_reason)
116
+ VALUES (${subjectIdToKey(subject)}, NULL, ${this.kekVersion}, now(), ${ctx.userId ?? null}, ${ctx.eraseReason ?? null})
117
+ ON CONFLICT (subject_id) DO UPDATE
114
118
  SET cipher_key = NULL,
115
119
  erased_at = now(),
116
- erased_by = ${ctx.userId ?? null},
117
- erase_reason = ${ctx.eraseReason ?? null}
118
- WHERE subject_id = ${subjectIdToKey(subject)} AND erased_at IS NULL`;
120
+ erased_by = EXCLUDED.erased_by,
121
+ erase_reason = EXCLUDED.erase_reason
122
+ WHERE kumiko_subject_keys.erased_at IS NULL`;
119
123
  }
120
124
  async health() {
121
125
  const start = Date.now();
@@ -169,7 +169,7 @@ export function createWriteVerbs(ctx) {
169
169
  failure: writeFailure(new FrameworkVersionConflict({
170
170
  entityId: id,
171
171
  expectedVersion: currentVersion,
172
- currentVersion: -1,
172
+ currentVersion: await versionAfterConflict(runner, String(id), streamTenantFor(user, streamTenantId)),
173
173
  })),
174
174
  };
175
175
  }
@@ -181,6 +181,16 @@ export function createWriteVerbs(ctx) {
181
181
  throw e;
182
182
  }
183
183
  }
184
+ // After a lost append race the stream version read before the append is stale;
185
+ // report the actual one. -1 when the lookup itself fails.
186
+ async function versionAfterConflict(runner, aggregateId, tenantId) {
187
+ try {
188
+ return await getStreamVersion(runner, aggregateId, tenantId);
189
+ }
190
+ catch {
191
+ return -1;
192
+ }
193
+ }
184
194
  async function expectedStreamVersion(db, id, streamTenant, expect) {
185
195
  const snapshot = await loadExpectSnapshot(db, id, streamTenant, Object.keys(expect));
186
196
  const row = snapshot.row;
@@ -298,13 +308,7 @@ export function createWriteVerbs(ctx) {
298
308
  }
299
309
  catch (e) {
300
310
  if (e instanceof EventStoreVersionConflict) {
301
- let currentVersion = -1;
302
- try {
303
- currentVersion = await getStreamVersion(runner, aggregateId, streamTenantFor(user, streamTenantId));
304
- }
305
- catch {
306
- // Lookup failure — keep the sentinel.
307
- }
311
+ const currentVersion = await versionAfterConflict(runner, aggregateId, streamTenantFor(user, streamTenantId));
308
312
  return writeFailure(new FrameworkVersionConflict({
309
313
  entityId: aggregateId,
310
314
  expectedVersion: 0,
@@ -603,7 +607,7 @@ export function createWriteVerbs(ctx) {
603
607
  return writeFailure(new FrameworkVersionConflict({
604
608
  entityId: payload.id,
605
609
  expectedVersion: payload.version ?? 0,
606
- currentVersion,
610
+ currentVersion: await versionAfterConflict(runner, String(payload.id), streamTenant),
607
611
  }));
608
612
  }
609
613
  if (e instanceof EventStoreIdempotentAppendConflict) {
@@ -675,7 +679,7 @@ export function createWriteVerbs(ctx) {
675
679
  return writeFailure(new FrameworkVersionConflict({
676
680
  entityId: payload.id,
677
681
  expectedVersion: currentVersion,
678
- currentVersion: -1,
682
+ currentVersion: await versionAfterConflict(runner, String(payload.id), streamTenantFor(user, streamTenantId)),
679
683
  }));
680
684
  }
681
685
  if (e instanceof EventStoreIdempotentAppendConflict) {
@@ -760,7 +764,7 @@ export function createWriteVerbs(ctx) {
760
764
  return writeFailure(new FrameworkVersionConflict({
761
765
  entityId: payload.id,
762
766
  expectedVersion: currentVersion,
763
- currentVersion: -1,
767
+ currentVersion: await versionAfterConflict(runner, String(payload.id), streamTenantFor(user, streamTenantId)),
764
768
  }));
765
769
  }
766
770
  if (e instanceof EventStoreIdempotentAppendConflict) {
@@ -825,7 +829,7 @@ export function createWriteVerbs(ctx) {
825
829
  // table. Rows predating the subject-KMS rollout may still hold plaintext
826
830
  // pii, and envelope encryption is not idempotent: decrypt, then
827
831
  // re-encrypt as delete does.
828
- const previousPlain = await decryptForRead(data);
832
+ const previousPlain = await decryptForRead(rehydrateCompoundTypes(data, entity));
829
833
  const appended = await appendRestoredEvent(runner, user, payload.id, previousPlain, streamTenantId, currentVersion);
830
834
  if ("failure" in appended)
831
835
  return appended.failure;
@@ -81,7 +81,7 @@ export { assertNotificationVariables, templateVariable, templateVariablesToZod,
81
81
  export { type EffectiveFeaturesResolver, findTierResolverUsage, isTierResolverPlugin, requireTierResolverPlugin, TENANT_TIER_RESOLVER_EXT, type TierResolverPlugin, type TrialGate, } from "./tier-resolver-extension.js";
82
82
  export { findOverDeepTransferChain, resolveTransferAdjacency as resolveEntityTransferAdjacency, type TransferAdjacency, type TransferEdge, type TransferEdgeLink, } from "./transfer-adjacency.js";
83
83
  export { isSystemTenant, isUuid, parseTenantId, SYSTEM_TENANT_ID } from "./types/identifiers.js";
84
- export type { AccessRule, ActionFormScreenDefinition, ActiveMembership, ActiveMembershipRejection, ActiveMembershipResult, AgentExposure, AgentHandlerHints, AgentRisk, AppContext, AppendEventArgs, AppendEventFn, AuthClaimsContext, AuthClaimsFn, AuthClaimsHookDef, BeforeTransactionPhase, BeforeTransactionResult, BelongsToRelation, BigIntFieldDef, BooleanFieldDef, CamelToKebab, ChatSendFailureCode, ClaimKeyDefinition, ClaimKeyHandle, ClaimKeyJsType, ClaimKeyType, ConcurrencyMode, ConfigAccessor, ConfigAccessorFactory, ConfigBacking, ConfigCascade, ConfigCascadeLevel, ConfigDefinition, ConfigEditScreenDefinition, ConfigKeyAccess, ConfigKeyDefinition, ConfigKeyHandle, ConfigKeyType, ConfigMask, ConfigResolver, ConfigScope, ConfigSecretsReader, ConfigSeedDef, ConfigStoredRow, ConfigStoredRowWithSource, ConfigValidateContext, ConfigValidateFn, ConfigValue, ConfigValueSource, ConfigValueWithSource, ContentCollectionDefinition, CreateSeedOptions, CreateTenantSeedOptions, CreateUserSeedOptions, CustomScreenDefinition, CustomScreenRoute, DashboardChartPanel, DashboardCustomPanel, DashboardDateParam, DashboardFeedPanel, DashboardFilterDefinition, DashboardI18nText, DashboardListPanel, DashboardMoneyParam, DashboardPanelDefinition, DashboardPanelVisibility, DashboardProgressListPanel, DashboardScreenDefinition, DashboardScreenPanel, DashboardStatGroupPanel, DashboardStatPanel, DashboardText, DashboardTextParam, DashboardValueFormat, DateFieldDef, DecimalFieldDef, DeleteContext, DeliveryErrorCode, DeliveryFailureCode, DeliverySkipReason, DeriveContext, DerivedFieldDef, DerivedFieldsMap, DerivedValueType, EditExtensionSection, EditFieldPrefill, EditFieldSpec, EditFieldsSection, EditLayout, EditRelatedListSection, EditSectionSpec, EmbeddedListEditSpec, EntityDefinition, EntityEditAction, EntityEditActionPlacement, EntityEditScreenDefinition, EntityId, EntityListExpandableRow, EntityListScreenDefinition, EntityRef, EntityRelations, EscapeHatchAuditSink, EscapeHatchDeclaration, EscapeHatchKind, EscapeHatchReporter, EscapeHatchTarget, EscapeHatchUseEvent, EventDef, ExtensionOptionsArgs, ExtensionOptionsFor, FeatureDefinition, FeatureRegistrar, FieldAccess, FieldCondition, FieldDefinition, FieldFormatRegistry, FieldIconKey, FieldRenderer, FileFieldDef, FilesFieldDef, Findability, FormatSpec, FormWidth, HandlerContext, HasManyRelation, HookMap, ImageFieldDef, ImagesFieldDef, JobBackoff, JobBackoffStrategy, JobContext, JobDefinition, JobHandlerFn, JobTrigger, JobTriggerEvent, JsonbFieldDef, KumikoEntityTypeMap, KumikoEventTypeMap, KumikoExtensionOptionsMap, KumikoHandlerPayloadMap, KumikoHandlerResultMap, LifecycleHookType, ListColumnSpec, ListFacetSpec, ListPaginationMode, ListSortDir, ListSortSpec, LongTextFindability, ManyToManyRelation, MemberReader, MetricNavigate, MetricSpec, MspErrorMode, MspErrorPolicy, MultiSelectFieldDef, MultiStreamApplyFn, MultiStreamProjectionDefinition, NameOrRef, NavDefinition, NavIconKey, NotificationDataFn, NotificationDefinition, NotificationLocalizedTemplate, NotificationRecipientFn, NotificationRecordMapping, NotificationTemplateBody, NotificationTemplateFn, NotificationTemplateMeta, NotifyDeferred, NotifyDelivery, NotifyDeliveryStatus, NotifyDispatched, NotifyFactory, NotifyFn, NotifyJobDispatcher, NotifyOptions, NotifyPriority, NotifyResult, NumberFieldDef, OnDeleteStrategy, OpenToAllAccessRule, OpenToAllDeclaration, OpenToAllPersonalData, ParentRefDef, PersonalAnnotations, PersonalAnnotationsLongText, PersonalAnnotationsNoFind, PersonalSubject, PlatformComponent, PluralForms, PostDeleteHookFn, PostSaveHookFn, PreDeleteHookFn, PreparedWriteEvent, PreQueryHookFn, PreSaveHookFn, ProjectionDefinition, ProjectionDetailScreenDefinition, ProjectionListScreenDefinition, ProjectionTable, QualifiedEventName, QueryEvent, QueryHandlerDef, QueryHandlerFn, RateLimitPer, RecordHeaderSpec, RecordHeaderSubtitlePart, ReferenceDataDef, Registry, RelationDefinition, ResolvedPiiFlags, RetentionDef, RoleAccessPersonalData, RoleAccessRule, RowAction, RowActionNavigate, RowActionNavigateBase, RowActionWriteHandler, RowFieldExtractor, SaveContext, ScreenDefinition, ScreenFilter, ScreenFilterOp, ScreenNavSugar, ScreenSlots, SecretKeyDefinition, SecretKeyHandle, SecretMintConfirmStep, SecretMintScreenDefinition, SecretNamespaceDefinition, SecretNamespaceHandle, SecretNamespaceOptions, SecretReveal, SecretRevealField, SecretsEditScreenDefinition, SecretsEditSection, SelectFieldDef, SessionUser, SessionUserOrigin, StreamHandlerDef, StreamHandlerFn, Subscribe, TargetRef, TemplateListVariableDef, TemplateScalarVariableDef, TemplateVariableDef, TemplateVariables, TenantId, TenantWriteTarget, TextFieldDef, ToolbarAction, TranslationEntry, TranslationKeys, TranslationsDef, TranslationValue, TreeAction, TreeActionDef, TreeActionsHandle, TreeChildrenSubscribe, TreeNode, TreeNodeState, UnitKey, UnsafeAppendEventFn, ValidationError, ValidationHookFn, WorkspaceDefinition, WriteEvent, WriteHandlerDef, WriteHandlerFn, WriteResult, } from "./types/index.js";
84
+ export type { AccessRule, ActionFormScreenDefinition, ActiveMembership, ActiveMembershipRejection, ActiveMembershipResult, AgentExposure, AgentHandlerHints, AgentRisk, AppContext, AppendEventArgs, AppendEventFn, AuthClaimsContext, AuthClaimsFn, AuthClaimsHookDef, BeforeTransactionPhase, BeforeTransactionResult, BelongsToRelation, BigIntFieldDef, BooleanFieldDef, CamelToKebab, ChatSendFailureCode, ClaimKeyDefinition, ClaimKeyHandle, ClaimKeyJsType, ClaimKeyType, ConcurrencyMode, ConfigAccessor, ConfigAccessorFactory, ConfigBacking, ConfigCascade, ConfigCascadeLevel, ConfigDefinition, ConfigEditScreenDefinition, ConfigKeyAccess, ConfigKeyDefinition, ConfigKeyHandle, ConfigKeyType, ConfigMask, ConfigResolver, ConfigScope, ConfigSecretsReader, ConfigSeedDef, ConfigStoredRow, ConfigStoredRowWithSource, ConfigValidateContext, ConfigValidateFn, ConfigValue, ConfigValueSource, ConfigValueWithSource, ContentCollectionDefinition, CreateSeedOptions, CreateTenantSeedOptions, CreateUserSeedOptions, CustomScreenDefinition, CustomScreenRoute, DashboardChartPanel, DashboardCustomPanel, DashboardDateParam, DashboardFeedPanel, DashboardFilterDefinition, DashboardI18nText, DashboardListPanel, DashboardMoneyParam, DashboardPanelDefinition, DashboardPanelVisibility, DashboardProgressListPanel, DashboardScreenDefinition, DashboardScreenPanel, DashboardStatGroupPanel, DashboardStatPanel, DashboardText, DashboardTextParam, DashboardValueFormat, DateFieldDef, DecimalFieldDef, DeleteContext, DeliveryErrorCode, DeliveryFailureCode, DeliverySkipReason, DeriveContext, DerivedFieldDef, DerivedFieldsMap, DerivedValueType, EditExtensionSection, EditFieldPrefill, EditFieldSpec, EditFieldsSection, EditLayout, EditRelatedListSection, EditSectionSpec, EmbeddedListEditSpec, EntityDefinition, EntityEditAction, EntityEditActionPlacement, EntityEditScreenDefinition, EntityId, EntityListExpandableRow, EntityListScreenDefinition, EntityRef, EntityRelations, EscapeHatchAuditSink, EscapeHatchDeclaration, EscapeHatchKind, EscapeHatchReporter, EscapeHatchTarget, EscapeHatchUseEvent, EventDef, ExtensionOptionsArgs, ExtensionOptionsFor, FeatureDefinition, FeatureRegistrar, FieldAccess, FieldCondition, FieldDefinition, FieldFormatRegistry, FieldIconKey, FieldRenderer, FileFieldDef, FilesFieldDef, Findability, FormatSpec, FormWidth, HandlerContext, HasManyRelation, HookMap, ImageFieldDef, ImagesFieldDef, JobBackoff, JobBackoffStrategy, JobContext, JobDefinition, JobHandlerFn, JobTrigger, JobTriggerEvent, JsonbFieldDef, KumikoEntityTypeMap, KumikoEventTypeMap, KumikoExtensionOptionsMap, KumikoHandlerPayloadMap, KumikoHandlerResultMap, LifecycleHookType, ListColumnSpec, ListFacetSpec, ListPaginationMode, ListSortDir, ListSortSpec, LongTextFindability, ManyToManyRelation, MaxPerTenantOptions, MemberReader, MetricNavigate, MetricSpec, MspErrorMode, MspErrorPolicy, MultiSelectFieldDef, MultiStreamApplyFn, MultiStreamProjectionDefinition, NameOrRef, NavDefinition, NavIconKey, NotificationDataFn, NotificationDefinition, NotificationLocalizedTemplate, NotificationRecipientFn, NotificationRecordMapping, NotificationTemplateBody, NotificationTemplateFn, NotificationTemplateMeta, NotifyDeferred, NotifyDelivery, NotifyDeliveryStatus, NotifyDispatched, NotifyFactory, NotifyFn, NotifyJobDispatcher, NotifyOptions, NotifyPriority, NotifyResult, NumberFieldDef, OnDeleteStrategy, OpenToAllAccessRule, OpenToAllDeclaration, OpenToAllPersonalData, ParentRefDef, PersonalAnnotations, PersonalAnnotationsLongText, PersonalAnnotationsNoFind, PersonalSubject, PlatformComponent, PluralForms, PostDeleteHookFn, PostSaveHookFn, PreDeleteHookFn, PreparedWriteEvent, PreQueryHookFn, PreSaveHookFn, ProjectionDefinition, ProjectionDetailScreenDefinition, ProjectionListScreenDefinition, ProjectionTable, QualifiedEventName, QueryEvent, QueryHandlerDef, QueryHandlerFn, RateLimitPer, RecordHeaderSpec, RecordHeaderSubtitlePart, ReferenceDataDef, Registry, RelationDefinition, ResolvedPiiFlags, RetentionDef, RoleAccessPersonalData, RoleAccessRule, RowAction, RowActionNavigate, RowActionNavigateBase, RowActionWriteHandler, RowFieldExtractor, SaveContext, ScreenDefinition, ScreenFilter, ScreenFilterOp, ScreenNavSugar, ScreenSlots, SecretKeyDefinition, SecretKeyHandle, SecretMintConfirmStep, SecretMintScreenDefinition, SecretNamespaceDefinition, SecretNamespaceHandle, SecretNamespaceOptions, SecretReveal, SecretRevealField, SecretsEditScreenDefinition, SecretsEditSection, SelectFieldDef, SessionUser, SessionUserOrigin, StreamHandlerDef, StreamHandlerFn, Subscribe, TargetRef, TemplateListVariableDef, TemplateScalarVariableDef, TemplateVariableDef, TemplateVariables, TenantId, TenantWriteTarget, TextFieldDef, ToolbarAction, TranslationEntry, TranslationKeys, TranslationsDef, TranslationValue, TreeAction, TreeActionDef, TreeActionsHandle, TreeChildrenSubscribe, TreeNode, TreeNodeState, UnitKey, UnsafeAppendEventFn, ValidationError, ValidationHookFn, WorkspaceDefinition, WriteEvent, WriteHandlerDef, WriteHandlerFn, WriteResult, } from "./types/index.js";
85
85
  export { DEFAULT_CURRENCIES, DEFAULT_LOCALES, DELIVERY_FAILURE_CODES, DELIVERY_SKIP_REASONS, HookPhases, HTTP_ERROR_CODE_PATTERN, isAgentVisibleScreen, isDeliveryErrorCode, isOpenToAllGranted, resolveAgentExposure, } from "./types/index.js";
86
86
  export type { AwaitedEventType, PipelineBuildCtx, PipelineCtx, PipelineDef, StepBuilder, StepDef, StepFailureStrategy, StepInstance, StepKind, StepNamespace, StepResolver, } from "./types/step.js";
87
87
  export { runValidation } from "./validation.js";
@@ -26,6 +26,7 @@ export declare function resolveJobTriggers(state: RegistryState): void;
26
26
  export declare function validateLiveEntities(state: RegistryState): void;
27
27
  export declare function validateBootGates(state: RegistryState): void;
28
28
  export declare function validateJobBackoff(state: RegistryState): void;
29
+ export declare function validateMaxPerTenant(state: RegistryState): void;
29
30
  export declare function validateExtensionUsageTargets(state: RegistryState): void;
30
31
  export declare function computeHasRateLimitedHandler(state: RegistryState): void;
31
32
  export declare function publishEventPiiCatalog(state: RegistryState): void;
@@ -1,6 +1,7 @@
1
1
  import { configureEventPiiCatalog } from "../crypto/event-pii.js";
2
2
  import { hasSearchablePlaintext, isSensitiveLabelField } from "../db/entity-field-encryption.js";
3
3
  import { entityEventName } from "../db/event-store-executor-context.js";
4
+ import { resolveMaxPerTenant } from "../jobs/max-per-tenant.js";
4
5
  import { bindHookEscapeHatchGrant } from "../pipeline/system-identity-switch.js";
5
6
  import { resolveName } from "./handler-helpers.js";
6
7
  import { parseRefTargetEntityName } from "./parse-ref-target.js";
@@ -744,6 +745,26 @@ export function validateJobBackoff(state) {
744
745
  }
745
746
  }
746
747
  }
748
+ export function validateMaxPerTenant(state) {
749
+ for (const [jobName, jobDef] of state.jobMap) {
750
+ const resolved = resolveMaxPerTenant(jobDef);
751
+ if (!resolved)
752
+ continue;
753
+ if (!Number.isInteger(resolved.limit) || resolved.limit <= 0) {
754
+ throw new Error(`Job "${jobName}" maxPerTenant limit must be a positive integer (got ${resolved.limit})`);
755
+ }
756
+ if (typeof jobDef.maxPerTenant !== "object")
757
+ continue;
758
+ const { deferMs } = jobDef.maxPerTenant;
759
+ if (deferMs !== undefined && (!Number.isInteger(deferMs) || deferMs <= 0)) {
760
+ throw new Error(`Job "${jobName}" maxPerTenant.deferMs must be a positive integer (got ${deferMs})`);
761
+ }
762
+ // The inline boot-gate path has no BullMQ token to move the job to delayed with.
763
+ if (resolved.onLimit === "defer" && jobDef.bootGate) {
764
+ throw new Error(`Job "${jobName}" cannot combine bootGate with maxPerTenant onLimit "defer"`);
765
+ }
766
+ }
767
+ }
747
768
  export function validateExtensionUsageTargets(state) {
748
769
  // Validate: extension usages must reference existing extensions
749
770
  for (const usage of state.extensionUsages) {
@@ -3,7 +3,7 @@ import { dedupeFeatures } from "./dedupe-features.js";
3
3
  import { buildRegistryFacade } from "./registry-facade.js";
4
4
  import { populateClaimsAndAuth, populateConfigKeys, populateEvents, populateExtensionsAndSeeds, populateFeatureCore, populateHandlers, populateHooks, populateJobsAndNotifications, populateMetricsAndSecrets, populateProjectionsAndTables, populateScreensNavWorkspaces, populateSystemEvents, populateTranslations, } from "./registry-ingest.js";
5
5
  import { createInitialState } from "./registry-state.js";
6
- import { applyExtensionUsages, autoWireSoftDeleteJobs, buildEntityEventUpcasterChains, buildEventUpcasterChains, buildImplicitProjections, buildIncomingRelations, buildSearchableSortableCaches, computeHasRateLimitedHandler, finalizeWorkspaceNavMembership, populateHandlerEntityMappings, publishEventPiiCatalog, resolveJobTriggers, resolveNotificationTriggersAndRegisterHooks, validateBootGates, validateEntityHookTargets, validateEventMigrationVersions, validateExtensionSelectors, validateExtensionUsageTargets, validateFieldAccessHandlersAreEntityMapped, validateJobBackoff, validateLifecycleHookTargets, validateLiveEntities, validateNotificationRecordMappings, validateProjectionApplyKeys, validateRelationTargetsExist, validateRequiredFeatures, } from "./registry-validate.js";
6
+ import { applyExtensionUsages, autoWireSoftDeleteJobs, buildEntityEventUpcasterChains, buildEventUpcasterChains, buildImplicitProjections, buildIncomingRelations, buildSearchableSortableCaches, computeHasRateLimitedHandler, finalizeWorkspaceNavMembership, populateHandlerEntityMappings, publishEventPiiCatalog, resolveJobTriggers, resolveNotificationTriggersAndRegisterHooks, validateBootGates, validateEntityHookTargets, validateEventMigrationVersions, validateExtensionSelectors, validateExtensionUsageTargets, validateFieldAccessHandlersAreEntityMapped, validateJobBackoff, validateLifecycleHookTargets, validateLiveEntities, validateMaxPerTenant, validateNotificationRecordMappings, validateProjectionApplyKeys, validateRelationTargetsExist, validateRequiredFeatures, } from "./registry-validate.js";
7
7
  // This is where the magic happens. By "magic" I mean: precomputed maps.
8
8
  // I build everything once at boot (hooks, relations, searchable fields, ...)
9
9
  // so nothing has to iterate over objects at runtime. O(1) instead of O(n*m).
@@ -48,6 +48,7 @@ export function createRegistry(rawFeatures) {
48
48
  validateBootGates(state);
49
49
  validateLiveEntities(state);
50
50
  validateJobBackoff(state);
51
+ validateMaxPerTenant(state);
51
52
  validateExtensionUsageTargets(state);
52
53
  computeHasRateLimitedHandler(state);
53
54
  publishEventPiiCatalog(state);
@@ -1,5 +1,5 @@
1
1
  export { isAgentVisibleScreen, resolveAgentExposure, } from "@cosmicdrift/kumiko-types/agent-exposure";
2
- export type { ConfigAccessor, ConfigAccessorFactory, ConfigBacking, ConfigBounds, ConfigCascade, ConfigCascadeLevel, ConfigComputedContext, ConfigComputedFn, ConfigDefinition, ConfigKeyAccess, ConfigKeyDefinition, ConfigKeyHandle, ConfigKeyType, ConfigMask, ConfigResolver, ConfigSecretsReader, ConfigSeedDef, ConfigStoredRow, ConfigStoredRowWithSource, ConfigValidateContext, ConfigValidateFn, ConfigValue, ConfigValueSource, ConfigValueWithSource, CreateSeedOptions, CreateTenantSeedOptions, CreateUserSeedOptions, ExtensionSelectorDef, ExtensionSelectorPanel, JobBackoff, JobBackoffStrategy, JobDefinition, JobHandlerFn, JobRunIn, JobTrigger, NotificationDataFn, NotificationDefinition, NotificationLocalizedTemplate, NotificationRecipientFn, NotificationRecordMapping, NotificationTemplateBody, NotificationTemplateFn, NotificationTemplateMeta, PluralForms, ReferenceDataDef, RegistrarExtensionDef, RegistrarExtensionHooks, RegistrarExtensionRegistration, RunIn, TemplateListVariableDef, TemplateScalarVariableDef, TemplateVariableDef, TemplateVariables, TranslationEntry, TranslationKeys, TranslationsDef, TranslationValue, UiExtensionDef, } from "@cosmicdrift/kumiko-types/config";
2
+ export type { ConfigAccessor, ConfigAccessorFactory, ConfigBacking, ConfigBounds, ConfigCascade, ConfigCascadeLevel, ConfigComputedContext, ConfigComputedFn, ConfigDefinition, ConfigKeyAccess, ConfigKeyDefinition, ConfigKeyHandle, ConfigKeyType, ConfigMask, ConfigResolver, ConfigSecretsReader, ConfigSeedDef, ConfigStoredRow, ConfigStoredRowWithSource, ConfigValidateContext, ConfigValidateFn, ConfigValue, ConfigValueSource, ConfigValueWithSource, CreateSeedOptions, CreateTenantSeedOptions, CreateUserSeedOptions, ExtensionSelectorDef, ExtensionSelectorPanel, JobBackoff, JobBackoffStrategy, JobDefinition, JobHandlerFn, JobRunIn, JobTrigger, MaxPerTenantOptions, NotificationDataFn, NotificationDefinition, NotificationLocalizedTemplate, NotificationRecipientFn, NotificationRecordMapping, NotificationTemplateBody, NotificationTemplateFn, NotificationTemplateMeta, PluralForms, ReferenceDataDef, RegistrarExtensionDef, RegistrarExtensionHooks, RegistrarExtensionRegistration, RunIn, TemplateListVariableDef, TemplateScalarVariableDef, TemplateVariableDef, TemplateVariables, TranslationEntry, TranslationKeys, TranslationsDef, TranslationValue, UiExtensionDef, } from "@cosmicdrift/kumiko-types/config";
3
3
  export type { QueryHandlerDefinition, StreamHandlerDefinition, WriteHandlerDefinition, WriteHandlerInput, } from "@cosmicdrift/kumiko-types/define-handler";
4
4
  export type { EntityCrudRegistrar, EntityCrudVerb, EntityHandlerOptions, EntityQueryHandlerOptions, EntityWriteHandlerOptions, RegisterEntityCrudOptions, } from "@cosmicdrift/kumiko-types/entity-handlers";
5
5
  export type { KumikoEntityTypeMap, KumikoEventTypeMap, KumikoHandlerPayloadMap, KumikoHandlerResultMap, } from "@cosmicdrift/kumiko-types/event-type-map";
@@ -20,6 +20,7 @@ export type JobMeta = {
20
20
  payload?: string | undefined;
21
21
  attempt?: number | undefined;
22
22
  priority?: number | undefined;
23
+ jobId?: string | undefined;
23
24
  tenantVisibleRun?: {
24
25
  readonly tenantId: string;
25
26
  readonly subject: string | null;
@@ -1,6 +1,6 @@
1
1
  import { escapeHatchFor } from "@cosmicdrift/kumiko-types/handlers";
2
2
  import { isPlainObject } from "@cosmicdrift/kumiko-types/type-guards";
3
- import { Queue, UnrecoverableError, Worker } from "bullmq";
3
+ import { DelayedError, Queue, UnrecoverableError, Worker } from "bullmq";
4
4
  import { Redis } from "ioredis";
5
5
  import { requestContext } from "../api/request-context.js";
6
6
  import { createTenantDb, createUncheckedSystemDb } from "../db/tenant-db.js";
@@ -19,6 +19,7 @@ import { createGatedIdentitySwitch } from "../pipeline/system-identity-switch.js
19
19
  import { buildPersonalDataGate, isPersonalDataGated, parseWriteOrigin, } from "../pipeline/write-origin.js";
20
20
  import { bridgeStub } from "../testing/handler-context.js";
21
21
  import { cronRepeatOptions } from "./cron-repeat-options.js";
22
+ import { resolveMaxPerTenant } from "./max-per-tenant.js";
22
23
  // Shape-checked after the BullMQ/Redis JSON round-trip. Unlike `_writeOrigin`
23
24
  // (a security-relevant escape hatch that must fail the run closed), a
24
25
  // malformed `_triggerEvent` only costs an idempotency-key convenience — the
@@ -248,6 +249,9 @@ function tenantFailureMessageKey(err, declaration) {
248
249
  // Leading underscore matches the existing internal-meta convention
249
250
  // (_triggeredById, _tenantId, _payload).
250
251
  const TRACE_CONTEXT_KEY = "_traceContext";
252
+ const DEFER_COUNT_KEY = "_deferCount";
253
+ const DEFER_DELAY_CAP_MS = 30_000;
254
+ const DEFER_BACKOFF_MAX_DOUBLINGS = 5;
251
255
  function readTraceContext(data) {
252
256
  const raw = data[TRACE_CONTEXT_KEY];
253
257
  if (!raw || typeof raw !== "object")
@@ -555,18 +559,23 @@ export function createJobRunner(options) {
555
559
  // lane (jobDef.runIn is static), but walking both is cheap and avoids
556
560
  // a subtle bug if someone ever reassigns a job to a different lane
557
561
  // between deploys while old queue contents are still draining.
558
- async function isOverPerTenantLimit(jobName, tenantId, max) {
562
+ // `activeOnly` + `self` serve the defer check, which runs inside the job being
563
+ // counted: it must not see waiting jobs or itself, and only counts jobs that
564
+ // became active before it. Without that order, workers that fetch a burst of
565
+ // jobs together all see each other as active and all defer, again and again.
566
+ async function isOverPerTenantLimit(jobName, tenantId, max, scope = {}) {
559
567
  const results = await Promise.all([
560
568
  queues.api.getActive(),
561
- queues.api.getWaiting(),
562
569
  queues.worker.getActive(),
563
- queues.worker.getWaiting(),
570
+ ...(scope.activeOnly ? [] : [queues.api.getWaiting(), queues.worker.getWaiting()]),
564
571
  ]);
565
572
  let count = 0;
566
573
  for (const list of results) {
567
574
  for (const j of list) {
568
575
  if (j.name !== jobName)
569
576
  continue;
577
+ if (scope.self && !isActiveBefore(j, scope.self))
578
+ continue;
570
579
  const t = j.data?._tenantId; // @cast-boundary dynamic-key
571
580
  if (t === tenantId) {
572
581
  count += 1;
@@ -577,6 +586,22 @@ export function createJobRunner(options) {
577
586
  }
578
587
  return false;
579
588
  }
589
+ function isActiveBefore(other, self) {
590
+ if (other.id === self.id)
591
+ return false;
592
+ if (self.processedOn === undefined)
593
+ return true;
594
+ const otherProcessedOn = other.processedOn ?? 0;
595
+ if (otherProcessedOn !== self.processedOn)
596
+ return otherProcessedOn < self.processedOn;
597
+ return (other.id ?? "") < (self.id ?? "");
598
+ }
599
+ // Growing delay bounds Redis and worker load while one tenant has a large
600
+ // backlog: every deferred job would otherwise be re-checked every deferMs.
601
+ function deferDelayMs(deferMs, deferCount) {
602
+ const grown = Math.min(deferMs * 2 ** Math.min(deferCount, DEFER_BACKOFF_MAX_DOUBLINGS), DEFER_DELAY_CAP_MS);
603
+ return grown + Math.floor(Math.random() * deferMs);
604
+ }
580
605
  // Only err.message, never payload or the logs array: job payloads can carry PII.
581
606
  function logFailedAttempt(entry) {
582
607
  errorLogger.error("job failed", {
@@ -647,7 +672,7 @@ export function createJobRunner(options) {
647
672
  });
648
673
  }
649
674
  }
650
- async function handleJob(bullJob) {
675
+ async function handleJob(bullJob, token) {
651
676
  const rawName = bullJob.name;
652
677
  // Handle perTenant dispatch jobs — fan out to one job per tenant. The
653
678
  // fan-out re-enqueues into the lane the actual job is assigned to;
@@ -729,6 +754,25 @@ export function createJobRunner(options) {
729
754
  await reportPreRunFailure(jobName, bullJob, err);
730
755
  throw err;
731
756
  }
757
+ // maxPerTenant defer: checked before anything is recorded for this run, so a
758
+ // deferred pickup leaves no span, run-log entry or failure trace and does not
759
+ // consume an attempt (moveToDelayed + DelayedError, not a retry).
760
+ const maxPerTenant = resolveMaxPerTenant(jobDef);
761
+ const deferTenantId = bullJob.data["_tenantId"];
762
+ if (maxPerTenant?.onLimit === "defer" &&
763
+ typeof deferTenantId === "string" &&
764
+ token !== undefined &&
765
+ bullJob.moveToDelayed &&
766
+ (await isOverPerTenantLimit(jobName, deferTenantId, maxPerTenant.limit, {
767
+ activeOnly: true,
768
+ self: { id: bullJob.id, processedOn: bullJob.processedOn },
769
+ }))) {
770
+ const priorDeferCount = bullJob.data[DEFER_COUNT_KEY];
771
+ const deferCount = typeof priorDeferCount === "number" ? priorDeferCount : 0;
772
+ await bullJob.updateData?.({ ...bullJob.data, [DEFER_COUNT_KEY]: deferCount + 1 });
773
+ await bullJob.moveToDelayed(Date.now() + deferDelayMs(maxPerTenant.deferMs, deferCount), token);
774
+ throw new DelayedError();
775
+ }
732
776
  // Sequential gate: try to claim the per-name lock. If another worker
733
777
  // (or this worker on a different bullJob) holds it, re-enqueue with a
734
778
  // small delay and exit *successfully* — using throw would burn the
@@ -1202,7 +1246,7 @@ export function createJobRunner(options) {
1202
1246
  return;
1203
1247
  }
1204
1248
  const consumerQueue = queues[consumerLane];
1205
- worker = new Worker(queueNameFor(queueNamePrefix, consumerLane), handleJob, {
1249
+ worker = new Worker(queueNameFor(queueNamePrefix, consumerLane), (job, token) => handleJob(job, token), {
1206
1250
  connection: redisOpts,
1207
1251
  concurrency: 5,
1208
1252
  // Not consuming until the boot gates passed: a job already queued from a
@@ -1361,10 +1405,11 @@ export function createJobRunner(options) {
1361
1405
  // against; without it the guard is inactive (system jobs, ambient
1362
1406
  // dispatch). Fan-out children of perTenant jobs land here on their
1363
1407
  // recursive queue.add and DO carry _tenantId.
1364
- if (jobDef.maxPerTenant !== undefined) {
1408
+ const maxPerTenant = resolveMaxPerTenant(jobDef);
1409
+ if (maxPerTenant?.onLimit === "skip") {
1365
1410
  const tenantId = payload?._tenantId; // @cast-boundary dynamic-key
1366
1411
  if (tenantId !== undefined &&
1367
- (await isOverPerTenantLimit(jobName, tenantId, jobDef.maxPerTenant))) {
1412
+ (await isOverPerTenantLimit(jobName, tenantId, maxPerTenant.limit))) {
1368
1413
  return "skipped:max-per-tenant";
1369
1414
  }
1370
1415
  }
@@ -1406,6 +1451,8 @@ export function createJobRunner(options) {
1406
1451
  bullOpts["timeout"] = jobDef.timeout;
1407
1452
  if (meta?.priority !== undefined)
1408
1453
  bullOpts["priority"] = meta.priority;
1454
+ if (meta?.jobId !== undefined)
1455
+ bullOpts["jobId"] = meta.jobId;
1409
1456
  // Pack meta into job data with _ prefix
1410
1457
  const data = { ...payload };
1411
1458
  if (meta?.triggeredById !== undefined)
@@ -1467,8 +1514,9 @@ export function createJobRunner(options) {
1467
1514
  data["_correlationId"] = reqCtx.correlationId;
1468
1515
  // Same maxPerTenant guard as dispatch — events that fan into many
1469
1516
  // jobs must respect the per-tenant cap or the limit is one-sided.
1470
- if (jobDef.maxPerTenant !== undefined && user?.tenantId !== undefined) {
1471
- if (await isOverPerTenantLimit(name, String(user.tenantId), jobDef.maxPerTenant)) {
1517
+ const maxPerTenant = resolveMaxPerTenant(jobDef);
1518
+ if (maxPerTenant?.onLimit === "skip" && user?.tenantId !== undefined) {
1519
+ if (await isOverPerTenantLimit(name, String(user.tenantId), maxPerTenant.limit)) {
1472
1520
  continue;
1473
1521
  }
1474
1522
  }
@@ -1485,17 +1533,21 @@ export function createJobRunner(options) {
1485
1533
  async countPendingJobs() {
1486
1534
  let total = 0;
1487
1535
  for (const queue of Object.values(queues)) {
1488
- const counts = await queue.getJobCounts("waiting", "active", "prioritized", "waiting-children");
1536
+ // One getJobCounts call is a single atomic snapshot. Reading delayed in a
1537
+ // second call misses a deferred job that cycles delayed -> active between
1538
+ // the reads, so delayed is counted here and only repeat schedulers (which
1539
+ // stay delayed between cron fires) are subtracted afterwards.
1540
+ const counts = await queue.getJobCounts("waiting", "active", "prioritized", "waiting-children", "delayed");
1489
1541
  total +=
1490
1542
  (counts["waiting"] ?? 0) +
1491
1543
  (counts["active"] ?? 0) +
1492
1544
  (counts["prioritized"] ?? 0) +
1493
- (counts["waiting-children"] ?? 0);
1494
- // getJobCounts has no id-prefix filter for "delayed" — getJobs(["delayed"])
1495
- // loads the full Job objects (payload included) just to read .id; fine for
1545
+ (counts["waiting-children"] ?? 0) +
1546
+ (counts["delayed"] ?? 0);
1547
+ // getJobs(["delayed"]) loads full Job objects just to read .id; fine for
1496
1548
  // test-draining's low volume, not a pattern for a prod-path hot loop.
1497
1549
  const delayed = await queue.getJobs(["delayed"]);
1498
- total += delayed.filter((job) => !(job.id ?? "").startsWith("repeat:")).length;
1550
+ total -= delayed.filter((job) => (job.id ?? "").startsWith("repeat:")).length;
1499
1551
  }
1500
1552
  return total;
1501
1553
  },
@@ -0,0 +1,8 @@
1
+ import type { JobDefinition } from "../engine/types/index.js";
2
+ export declare const DEFAULT_MAX_PER_TENANT_DEFER_MS = 1000;
3
+ export type ResolvedMaxPerTenant = {
4
+ readonly limit: number;
5
+ readonly onLimit: "skip" | "defer";
6
+ readonly deferMs: number;
7
+ };
8
+ export declare function resolveMaxPerTenant(jobDef: Pick<JobDefinition, "maxPerTenant">): ResolvedMaxPerTenant | undefined;
@@ -0,0 +1,14 @@
1
+ export const DEFAULT_MAX_PER_TENANT_DEFER_MS = 1_000;
2
+ export function resolveMaxPerTenant(jobDef) {
3
+ const raw = jobDef.maxPerTenant;
4
+ if (raw === undefined)
5
+ return undefined;
6
+ if (typeof raw === "number") {
7
+ return { limit: raw, onLimit: "skip", deferMs: DEFAULT_MAX_PER_TENANT_DEFER_MS };
8
+ }
9
+ return {
10
+ limit: raw.limit,
11
+ onLimit: raw.onLimit,
12
+ deferMs: raw.deferMs ?? DEFAULT_MAX_PER_TENANT_DEFER_MS,
13
+ };
14
+ }
@@ -31,6 +31,8 @@ export type { LifecycleHooks, SystemHookDef, SystemHooks } from "./lifecycle-pip
31
31
  export { createLifecycleHooks } from "./lifecycle-pipeline.js";
32
32
  export type { MspRebuildDeps } from "./msp-rebuild.js";
33
33
  export { rebuildMultiStreamProjection } from "./msp-rebuild.js";
34
+ export type { MultiStreamApplyContext, MultiStreamApplyContextDeps, } from "./multi-stream-apply-context.js";
35
+ export { createMultiStreamApplyContext } from "./multi-stream-apply-context.js";
34
36
  export { assertNotifyTargetsOwnTenant, createDeferringNotify } from "./notify-scope.js";
35
37
  export type { ProjectionProgress, RebuildResult } from "./projection-rebuild.js";
36
38
  export { getAllProjectionProgress, getProjectionState, listProjectionsWithState, rebuildProjection, } from "./projection-rebuild.js";
@@ -16,6 +16,7 @@ export { createIdempotencyGuard } from "./idempotency.js";
16
16
  export { assertInstructionFieldWriteAllowed, assertIrreversibleOperationAllowed, isIrreversibleEntityVerb, } from "./irreversible-operation-gate.js";
17
17
  export { createLifecycleHooks } from "./lifecycle-pipeline.js";
18
18
  export { rebuildMultiStreamProjection } from "./msp-rebuild.js";
19
+ export { createMultiStreamApplyContext } from "./multi-stream-apply-context.js";
19
20
  export { assertNotifyTargetsOwnTenant, createDeferringNotify } from "./notify-scope.js";
20
21
  export { getAllProjectionProgress, getProjectionState, listProjectionsWithState, rebuildProjection, } from "./projection-rebuild.js";
21
22
  export { createProjectionStateTable, ProjectionStatuses, projectionStateTable, } from "./projection-state.js";
@@ -14,5 +14,6 @@ export type MultiStreamApplyContextDeps = {
14
14
  readonly files?: FileContext;
15
15
  readonly derivatives?: DerivativesContext;
16
16
  readonly secrets?: SecretsContext;
17
+ readonly jobs?: MultiStreamApplyContext["jobs"];
17
18
  };
18
19
  export declare function createMultiStreamApplyContext(deps: MultiStreamApplyContextDeps): MultiStreamApplyContext;
@@ -7,6 +7,7 @@ export function createMultiStreamApplyContext(deps) {
7
7
  ...(deps.files ? { files: deps.files } : {}),
8
8
  ...(deps.derivatives ? { derivatives: deps.derivatives } : {}),
9
9
  ...(deps.secrets ? { secrets: deps.secrets } : {}),
10
+ ...(deps.jobs ? { jobs: deps.jobs } : {}),
10
11
  appendEvent: (async (args) => {
11
12
  await appendDomainEventCore({
12
13
  registry: deps.registry,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@cosmicdrift/kumiko-framework",
3
- "version": "0.365.0",
3
+ "version": "0.365.1",
4
4
  "description": "Framework core — engine, pipeline, API, DB, and every other bit that makes Kumiko go.",
5
5
  "license": "BUSL-1.1",
6
6
  "author": "Marc Frost <marc@cosmicdriftgamestudio.com>",
@@ -210,8 +210,8 @@
210
210
  "./package.json": "./package.json"
211
211
  },
212
212
  "dependencies": {
213
- "@cosmicdrift/kumiko-http": "0.365.0",
214
- "@cosmicdrift/kumiko-types": "0.365.0",
213
+ "@cosmicdrift/kumiko-http": "0.365.1",
214
+ "@cosmicdrift/kumiko-types": "0.365.1",
215
215
  "bullmq": "^5.76.7",
216
216
  "bun-types": "^1.3.13",
217
217
  "hono": "^4.13.1",
@@ -227,7 +227,7 @@
227
227
  "zod": "^4.4.3"
228
228
  },
229
229
  "devDependencies": {
230
- "@cosmicdrift/kumiko-dispatcher-live": "0.365.0",
230
+ "@cosmicdrift/kumiko-dispatcher-live": "0.365.1",
231
231
  "bun-types": "^1.3.13",
232
232
  "pino-pretty": "^13.1.3"
233
233
  },
package/src/changes.json CHANGED
@@ -1,4 +1,22 @@
1
1
  [
2
+ {
3
+ "version": "0.365.1",
4
+ "type": "fix",
5
+ "title": "App and worker boot wait for Postgres instead of crashing",
6
+ "detail": "runProdApp and worker processes now probe the database before the schema-drift check and retry with exponential backoff (1s to 15s, up to 90s) on transient connection errors such as too_many_connections or ECONNREFUSED. The KMS health check at boot retries the same way. Non-transient errors like auth failures still abort immediately."
7
+ },
8
+ {
9
+ "version": "0.365.1",
10
+ "type": "fix",
11
+ "title": "Restoring a soft-deleted row with a money field no longer fails with a 500",
12
+ "detail": "restore() wrote the raw stored row (money as bigint minor units) into payload.previous, and the jsonb bind threw. It now builds previous in the same domain form as delete(), so the restored event carries { amount, currency, ... } for money fields."
13
+ },
14
+ {
15
+ "version": "0.365.1",
16
+ "type": "fix",
17
+ "title": "version_conflict from delete, restore, forget and update reports the actual stream version",
18
+ "detail": "After a lost append race, delete, restore and forget returned currentVersion -1 and update returned the version read before the race, so expectedVersion and currentVersion looked equal. They now look up the stream version after the conflict; -1 remains only when that lookup fails."
19
+ },
2
20
  {
3
21
  "version": "0.365.0",
4
22
  "type": "improvement",