@cosmicdrift/kumiko-framework 0.365.0 → 0.365.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agents/examples/webhook-step.md +11 -1
- package/agents/features.md +1 -1
- package/dist/api/server.js +8 -0
- package/dist/crypto/in-memory-kms-adapter.js +4 -7
- package/dist/crypto/pg-kms-adapter.js +10 -6
- package/dist/db/event-store-executor-write.js +16 -12
- package/dist/engine/index.d.ts +1 -1
- package/dist/engine/registry-validate.d.ts +1 -0
- package/dist/engine/registry-validate.js +21 -0
- package/dist/engine/registry.js +2 -1
- package/dist/engine/types/index.d.ts +1 -1
- package/dist/jobs/job-runner.d.ts +1 -0
- package/dist/jobs/job-runner.js +67 -15
- package/dist/jobs/max-per-tenant.d.ts +8 -0
- package/dist/jobs/max-per-tenant.js +14 -0
- package/dist/pipeline/index.d.ts +2 -0
- package/dist/pipeline/index.js +1 -0
- package/dist/pipeline/multi-stream-apply-context.d.ts +1 -0
- package/dist/pipeline/multi-stream-apply-context.js +1 -0
- package/package.json +4 -4
- package/src/changes.json +18 -0
|
@@ -68,7 +68,7 @@ bun test
|
|
|
68
68
|
```ts
|
|
69
69
|
// webhook-step integration test — drives r.step.webhook.send end-to-end:
|
|
70
70
|
// handler appends step.dispatch-requested in TX, dispatcher subscription
|
|
71
|
-
//
|
|
71
|
+
// enqueues a delivery job after COMMIT, the job calls the (stubbed) fetch, follow-up
|
|
72
72
|
// step.dispatched / step.dispatch-failed events land on the same stream.
|
|
73
73
|
|
|
74
74
|
import { afterAll, beforeAll, beforeEach, describe, expect, mock, test } from "bun:test";
|
|
@@ -140,6 +140,7 @@ beforeAll(async () => {
|
|
|
140
140
|
stack = await setupTestStack({
|
|
141
141
|
features: [createStepDispatcherFeature(), createSecretsFeature(), webhookDemoFeature],
|
|
142
142
|
systemHooks: [],
|
|
143
|
+
jobs: { consumerLane: "worker" },
|
|
143
144
|
extraContext: ({ db, registry }) => ({
|
|
144
145
|
secrets: createSecretsContext({ db, masterKeyProvider, registry }),
|
|
145
146
|
}),
|
|
@@ -184,6 +185,7 @@ describe("webhook-step Sample", () => {
|
|
|
184
185
|
|
|
185
186
|
// Drain dispatcher (MSP runs async via runOnce in test mode)
|
|
186
187
|
await stack.eventDispatcher?.runOnce();
|
|
188
|
+
await stack.drainJobs();
|
|
187
189
|
|
|
188
190
|
expect(fetchMock).toHaveBeenCalledTimes(1);
|
|
189
191
|
const [calledUrl, init] = fetchMock.mock.calls[0]!;
|
|
@@ -202,6 +204,7 @@ describe("webhook-step Sample", () => {
|
|
|
202
204
|
expect(id).toMatch(/^[0-9a-f-]{36}$/);
|
|
203
205
|
|
|
204
206
|
await stack.eventDispatcher?.runOnce();
|
|
207
|
+
await stack.drainJobs();
|
|
205
208
|
|
|
206
209
|
expect(mailMock).toHaveBeenCalledTimes(1);
|
|
207
210
|
expect(mailMock).toHaveBeenCalledWith(
|
|
@@ -231,6 +234,7 @@ describe("webhook-step Sample", () => {
|
|
|
231
234
|
|
|
232
235
|
// And the inner handler's webhook fired
|
|
233
236
|
await stack.eventDispatcher?.runOnce();
|
|
237
|
+
await stack.drainJobs();
|
|
234
238
|
expect(fetchMock).toHaveBeenCalledTimes(1);
|
|
235
239
|
});
|
|
236
240
|
|
|
@@ -243,6 +247,7 @@ describe("webhook-step Sample", () => {
|
|
|
243
247
|
expect(res.status).toBe(500);
|
|
244
248
|
|
|
245
249
|
await stack.eventDispatcher?.runOnce();
|
|
250
|
+
await stack.drainJobs();
|
|
246
251
|
|
|
247
252
|
expect(fetchMock).not.toHaveBeenCalled();
|
|
248
253
|
const rows = await selectMany(stack.db, incidentTable);
|
|
@@ -261,6 +266,7 @@ describe("access-denied — role-restricted handlers", () => {
|
|
|
261
266
|
);
|
|
262
267
|
expect(error.code).toBe("access_denied");
|
|
263
268
|
await stack.eventDispatcher?.runOnce();
|
|
269
|
+
await stack.drainJobs();
|
|
264
270
|
expect(mailMock).not.toHaveBeenCalled();
|
|
265
271
|
});
|
|
266
272
|
|
|
@@ -287,6 +293,7 @@ describe("access-denied — role-restricted handlers", () => {
|
|
|
287
293
|
);
|
|
288
294
|
expect(error.code).toBe("access_denied");
|
|
289
295
|
await stack.eventDispatcher?.runOnce();
|
|
296
|
+
await stack.drainJobs();
|
|
290
297
|
expect(fetchMock).not.toHaveBeenCalled();
|
|
291
298
|
});
|
|
292
299
|
});
|
|
@@ -317,6 +324,7 @@ describe("incident:open-authenticated — tenant-owned webhook auth secret", ()
|
|
|
317
324
|
adminB,
|
|
318
325
|
);
|
|
319
326
|
await stack.eventDispatcher?.runOnce();
|
|
327
|
+
await stack.drainJobs();
|
|
320
328
|
|
|
321
329
|
expect(fetchMock).toHaveBeenCalledTimes(1);
|
|
322
330
|
const [, init] = fetchMock.mock.calls[0]!;
|
|
@@ -348,6 +356,7 @@ describe("incident:open-authenticated — tenant-owned webhook auth secret", ()
|
|
|
348
356
|
adminA,
|
|
349
357
|
);
|
|
350
358
|
await stack.eventDispatcher?.runOnce();
|
|
359
|
+
await stack.drainJobs();
|
|
351
360
|
|
|
352
361
|
expect(fetchMock).not.toHaveBeenCalled();
|
|
353
362
|
|
|
@@ -378,6 +387,7 @@ describe("incident:open-authenticated — tenant-owned webhook auth secret", ()
|
|
|
378
387
|
adminC,
|
|
379
388
|
);
|
|
380
389
|
await stack.eventDispatcher?.runOnce();
|
|
390
|
+
await stack.drainJobs();
|
|
381
391
|
|
|
382
392
|
expect(fetchMock).not.toHaveBeenCalled();
|
|
383
393
|
const failedEvents = await selectMany(stack.db, eventsTable, {
|
package/agents/features.md
CHANGED
|
@@ -594,7 +594,7 @@ Exposes APIs: `sessions.revokeAllForUser`
|
|
|
594
594
|
|
|
595
595
|
## step-dispatcher
|
|
596
596
|
|
|
597
|
-
Internal system feature that drains deferred Tier-2 side-effects (currently `webhook.send` and `mail.send`) after their originating transaction commits.
|
|
597
|
+
Internal system feature that drains deferred Tier-2 side-effects (currently `webhook.send` and `mail.send`) after their originating transaction commits. An `r.multiStreamProjection` on the `kumiko:system:step.dispatch-requested` system event enqueues one `step-dispatcher:job:deliver` BullMQ job per dispatch (Redis carries only the stream id and tenant, never the request payload). The job performs the actual HTTP or mail delivery and appends `kumiko:system:step.dispatched` or `kumiko:system:step.dispatch-failed` back onto the same stream, so the outcome is recorded in the event log without a separate status table. Transient failures (network error, timeout, 5xx, 408, 429, any mail adapter failure) are retried 3 times with exponential backoff; the outcome is recorded after the final attempt and the payload key is erased only then. Permanent failures (invalid url, blocked host, other 4xx, unavailable auth secret) are recorded immediately. At most 2 deliveries per tenant run at once (`maxPerTenant` with `onLimit: "defer"`), so a slow receiver of one tenant does not delay other tenants. Requires a job runner (Redis) and a worker consuming the job's default `worker` lane; without a job runner the projection fails loudly. Mount this feature explicitly via `createStepDispatcherFeature()` in your app's feature list alongside any features that use `r.step.webhook.send` or `r.step.mail.send`. Requires the `secrets` feature (`createSecretsFeature()`) to be mounted — `webhook.send` auth resolves per-tenant through it, under `step-dispatcher:webhook-auth.<name>`. Every `webhook.send` request carries `Idempotency-Key: <dispatch stream id>`, stable across retries and redeliveries of the same dispatch request; an explicit `Idempotency-Key` in `headers` takes precedence.
|
|
598
598
|
|
|
599
599
|
Category: infrastructure
|
|
600
600
|
|
package/dist/api/server.js
CHANGED
|
@@ -310,6 +310,13 @@ export function buildServer(options) {
|
|
|
310
310
|
const searchConsumerEnabled = options.eventDispatcher?.systemConsumers?.search ?? true;
|
|
311
311
|
const jobTriggerConsumerEnabled = options.eventDispatcher?.systemConsumers?.jobTrigger ?? true;
|
|
312
312
|
const jobRunnerForTriggers = options.dispatcherOptions?.jobRunner;
|
|
313
|
+
const mspJobs = jobRunnerForTriggers?.dispatch
|
|
314
|
+
? {
|
|
315
|
+
dispatch: async (jobName, payload, dispatchOptions) => {
|
|
316
|
+
await jobRunnerForTriggers.dispatch?.(jobName, payload, dispatchOptions);
|
|
317
|
+
},
|
|
318
|
+
}
|
|
319
|
+
: undefined;
|
|
313
320
|
const systemConsumers = [];
|
|
314
321
|
if (sseConsumerEnabled) {
|
|
315
322
|
systemConsumers.push(createSseBroadcastEventConsumer(sseBroker));
|
|
@@ -391,6 +398,7 @@ export function buildServer(options) {
|
|
|
391
398
|
...(mspOwner && { callerFeature: mspOwner }),
|
|
392
399
|
...(mspFiles && { files: mspFiles }),
|
|
393
400
|
...(contextWithObservability.secrets && { secrets: contextWithObservability.secrets }),
|
|
401
|
+
...(mspJobs && { jobs: mspJobs }),
|
|
394
402
|
...(mspFiles && {
|
|
395
403
|
derivatives: createDerivativesContext({
|
|
396
404
|
files: mspFiles,
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { randomBytes } from "node:crypto";
|
|
2
2
|
import { KeyAlreadyExistsError, KeyErasedError, KeyNotFoundError, subjectIdToKey, } from "./kms-adapter.js";
|
|
3
3
|
// Non-persistent adapter for tests and dev mode. Erased entries stay as
|
|
4
|
-
// tombstones so the create-after-erase
|
|
4
|
+
// tombstones (also for never-keyed subjects) so the create-after-erase
|
|
5
|
+
// contract holds within a process.
|
|
5
6
|
export class InMemoryKmsAdapter {
|
|
6
7
|
capabilities = { mode: "local-key" };
|
|
7
8
|
keys = new Map();
|
|
@@ -20,12 +21,8 @@ export class InMemoryKmsAdapter {
|
|
|
20
21
|
return entry.key;
|
|
21
22
|
}
|
|
22
23
|
async eraseKey(subject) {
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
if (!entry)
|
|
26
|
-
return;
|
|
27
|
-
entry.key = null;
|
|
28
|
-
entry.erased = true;
|
|
24
|
+
// A never-keyed subject gets a tombstone too, so createKey cannot mint one later.
|
|
25
|
+
this.keys.set(subjectIdToKey(subject), { key: null, erased: true });
|
|
29
26
|
}
|
|
30
27
|
async health() {
|
|
31
28
|
return { ok: true, latencyMs: 0 };
|
|
@@ -107,15 +107,19 @@ export class PgKmsAdapter {
|
|
|
107
107
|
}
|
|
108
108
|
async eraseKey(subject, ctx) {
|
|
109
109
|
await this.ensureSchema();
|
|
110
|
-
//
|
|
111
|
-
//
|
|
110
|
+
// Upsert: a never-keyed subject (pre-KMS plaintext) still gets a tombstone,
|
|
111
|
+
// so a later createKey cannot mint a fresh key and re-encrypt the plaintext.
|
|
112
|
+
// The erased_at IS NULL guard keeps repeat erases from overwriting the
|
|
113
|
+
// original tombstone audit fields.
|
|
112
114
|
await this.sql `
|
|
113
|
-
|
|
115
|
+
INSERT INTO kumiko_subject_keys (subject_id, cipher_key, kek_version, erased_at, erased_by, erase_reason)
|
|
116
|
+
VALUES (${subjectIdToKey(subject)}, NULL, ${this.kekVersion}, now(), ${ctx.userId ?? null}, ${ctx.eraseReason ?? null})
|
|
117
|
+
ON CONFLICT (subject_id) DO UPDATE
|
|
114
118
|
SET cipher_key = NULL,
|
|
115
119
|
erased_at = now(),
|
|
116
|
-
erased_by =
|
|
117
|
-
erase_reason =
|
|
118
|
-
WHERE
|
|
120
|
+
erased_by = EXCLUDED.erased_by,
|
|
121
|
+
erase_reason = EXCLUDED.erase_reason
|
|
122
|
+
WHERE kumiko_subject_keys.erased_at IS NULL`;
|
|
119
123
|
}
|
|
120
124
|
async health() {
|
|
121
125
|
const start = Date.now();
|
|
@@ -169,7 +169,7 @@ export function createWriteVerbs(ctx) {
|
|
|
169
169
|
failure: writeFailure(new FrameworkVersionConflict({
|
|
170
170
|
entityId: id,
|
|
171
171
|
expectedVersion: currentVersion,
|
|
172
|
-
currentVersion:
|
|
172
|
+
currentVersion: await versionAfterConflict(runner, String(id), streamTenantFor(user, streamTenantId)),
|
|
173
173
|
})),
|
|
174
174
|
};
|
|
175
175
|
}
|
|
@@ -181,6 +181,16 @@ export function createWriteVerbs(ctx) {
|
|
|
181
181
|
throw e;
|
|
182
182
|
}
|
|
183
183
|
}
|
|
184
|
+
// After a lost append race the stream version read before the append is stale;
|
|
185
|
+
// report the actual one. -1 when the lookup itself fails.
|
|
186
|
+
async function versionAfterConflict(runner, aggregateId, tenantId) {
|
|
187
|
+
try {
|
|
188
|
+
return await getStreamVersion(runner, aggregateId, tenantId);
|
|
189
|
+
}
|
|
190
|
+
catch {
|
|
191
|
+
return -1;
|
|
192
|
+
}
|
|
193
|
+
}
|
|
184
194
|
async function expectedStreamVersion(db, id, streamTenant, expect) {
|
|
185
195
|
const snapshot = await loadExpectSnapshot(db, id, streamTenant, Object.keys(expect));
|
|
186
196
|
const row = snapshot.row;
|
|
@@ -298,13 +308,7 @@ export function createWriteVerbs(ctx) {
|
|
|
298
308
|
}
|
|
299
309
|
catch (e) {
|
|
300
310
|
if (e instanceof EventStoreVersionConflict) {
|
|
301
|
-
|
|
302
|
-
try {
|
|
303
|
-
currentVersion = await getStreamVersion(runner, aggregateId, streamTenantFor(user, streamTenantId));
|
|
304
|
-
}
|
|
305
|
-
catch {
|
|
306
|
-
// Lookup failure — keep the sentinel.
|
|
307
|
-
}
|
|
311
|
+
const currentVersion = await versionAfterConflict(runner, aggregateId, streamTenantFor(user, streamTenantId));
|
|
308
312
|
return writeFailure(new FrameworkVersionConflict({
|
|
309
313
|
entityId: aggregateId,
|
|
310
314
|
expectedVersion: 0,
|
|
@@ -603,7 +607,7 @@ export function createWriteVerbs(ctx) {
|
|
|
603
607
|
return writeFailure(new FrameworkVersionConflict({
|
|
604
608
|
entityId: payload.id,
|
|
605
609
|
expectedVersion: payload.version ?? 0,
|
|
606
|
-
currentVersion,
|
|
610
|
+
currentVersion: await versionAfterConflict(runner, String(payload.id), streamTenant),
|
|
607
611
|
}));
|
|
608
612
|
}
|
|
609
613
|
if (e instanceof EventStoreIdempotentAppendConflict) {
|
|
@@ -675,7 +679,7 @@ export function createWriteVerbs(ctx) {
|
|
|
675
679
|
return writeFailure(new FrameworkVersionConflict({
|
|
676
680
|
entityId: payload.id,
|
|
677
681
|
expectedVersion: currentVersion,
|
|
678
|
-
currentVersion:
|
|
682
|
+
currentVersion: await versionAfterConflict(runner, String(payload.id), streamTenantFor(user, streamTenantId)),
|
|
679
683
|
}));
|
|
680
684
|
}
|
|
681
685
|
if (e instanceof EventStoreIdempotentAppendConflict) {
|
|
@@ -760,7 +764,7 @@ export function createWriteVerbs(ctx) {
|
|
|
760
764
|
return writeFailure(new FrameworkVersionConflict({
|
|
761
765
|
entityId: payload.id,
|
|
762
766
|
expectedVersion: currentVersion,
|
|
763
|
-
currentVersion:
|
|
767
|
+
currentVersion: await versionAfterConflict(runner, String(payload.id), streamTenantFor(user, streamTenantId)),
|
|
764
768
|
}));
|
|
765
769
|
}
|
|
766
770
|
if (e instanceof EventStoreIdempotentAppendConflict) {
|
|
@@ -825,7 +829,7 @@ export function createWriteVerbs(ctx) {
|
|
|
825
829
|
// table. Rows predating the subject-KMS rollout may still hold plaintext
|
|
826
830
|
// pii, and envelope encryption is not idempotent: decrypt, then
|
|
827
831
|
// re-encrypt as delete does.
|
|
828
|
-
const previousPlain = await decryptForRead(data);
|
|
832
|
+
const previousPlain = await decryptForRead(rehydrateCompoundTypes(data, entity));
|
|
829
833
|
const appended = await appendRestoredEvent(runner, user, payload.id, previousPlain, streamTenantId, currentVersion);
|
|
830
834
|
if ("failure" in appended)
|
|
831
835
|
return appended.failure;
|
package/dist/engine/index.d.ts
CHANGED
|
@@ -81,7 +81,7 @@ export { assertNotificationVariables, templateVariable, templateVariablesToZod,
|
|
|
81
81
|
export { type EffectiveFeaturesResolver, findTierResolverUsage, isTierResolverPlugin, requireTierResolverPlugin, TENANT_TIER_RESOLVER_EXT, type TierResolverPlugin, type TrialGate, } from "./tier-resolver-extension.js";
|
|
82
82
|
export { findOverDeepTransferChain, resolveTransferAdjacency as resolveEntityTransferAdjacency, type TransferAdjacency, type TransferEdge, type TransferEdgeLink, } from "./transfer-adjacency.js";
|
|
83
83
|
export { isSystemTenant, isUuid, parseTenantId, SYSTEM_TENANT_ID } from "./types/identifiers.js";
|
|
84
|
-
export type { AccessRule, ActionFormScreenDefinition, ActiveMembership, ActiveMembershipRejection, ActiveMembershipResult, AgentExposure, AgentHandlerHints, AgentRisk, AppContext, AppendEventArgs, AppendEventFn, AuthClaimsContext, AuthClaimsFn, AuthClaimsHookDef, BeforeTransactionPhase, BeforeTransactionResult, BelongsToRelation, BigIntFieldDef, BooleanFieldDef, CamelToKebab, ChatSendFailureCode, ClaimKeyDefinition, ClaimKeyHandle, ClaimKeyJsType, ClaimKeyType, ConcurrencyMode, ConfigAccessor, ConfigAccessorFactory, ConfigBacking, ConfigCascade, ConfigCascadeLevel, ConfigDefinition, ConfigEditScreenDefinition, ConfigKeyAccess, ConfigKeyDefinition, ConfigKeyHandle, ConfigKeyType, ConfigMask, ConfigResolver, ConfigScope, ConfigSecretsReader, ConfigSeedDef, ConfigStoredRow, ConfigStoredRowWithSource, ConfigValidateContext, ConfigValidateFn, ConfigValue, ConfigValueSource, ConfigValueWithSource, ContentCollectionDefinition, CreateSeedOptions, CreateTenantSeedOptions, CreateUserSeedOptions, CustomScreenDefinition, CustomScreenRoute, DashboardChartPanel, DashboardCustomPanel, DashboardDateParam, DashboardFeedPanel, DashboardFilterDefinition, DashboardI18nText, DashboardListPanel, DashboardMoneyParam, DashboardPanelDefinition, DashboardPanelVisibility, DashboardProgressListPanel, DashboardScreenDefinition, DashboardScreenPanel, DashboardStatGroupPanel, DashboardStatPanel, DashboardText, DashboardTextParam, DashboardValueFormat, DateFieldDef, DecimalFieldDef, DeleteContext, DeliveryErrorCode, DeliveryFailureCode, DeliverySkipReason, DeriveContext, DerivedFieldDef, DerivedFieldsMap, DerivedValueType, EditExtensionSection, EditFieldPrefill, EditFieldSpec, EditFieldsSection, EditLayout, EditRelatedListSection, EditSectionSpec, EmbeddedListEditSpec, EntityDefinition, EntityEditAction, EntityEditActionPlacement, EntityEditScreenDefinition, EntityId, EntityListExpandableRow, EntityListScreenDefinition, EntityRef, EntityRelations, EscapeHatchAuditSink, EscapeHatchDeclaration, EscapeHatchKind, EscapeHatchReporter, EscapeHatchTarget, EscapeHatchUseEvent, EventDef, ExtensionOptionsArgs, ExtensionOptionsFor, FeatureDefinition, FeatureRegistrar, FieldAccess, FieldCondition, FieldDefinition, FieldFormatRegistry, FieldIconKey, FieldRenderer, FileFieldDef, FilesFieldDef, Findability, FormatSpec, FormWidth, HandlerContext, HasManyRelation, HookMap, ImageFieldDef, ImagesFieldDef, JobBackoff, JobBackoffStrategy, JobContext, JobDefinition, JobHandlerFn, JobTrigger, JobTriggerEvent, JsonbFieldDef, KumikoEntityTypeMap, KumikoEventTypeMap, KumikoExtensionOptionsMap, KumikoHandlerPayloadMap, KumikoHandlerResultMap, LifecycleHookType, ListColumnSpec, ListFacetSpec, ListPaginationMode, ListSortDir, ListSortSpec, LongTextFindability, ManyToManyRelation, MemberReader, MetricNavigate, MetricSpec, MspErrorMode, MspErrorPolicy, MultiSelectFieldDef, MultiStreamApplyFn, MultiStreamProjectionDefinition, NameOrRef, NavDefinition, NavIconKey, NotificationDataFn, NotificationDefinition, NotificationLocalizedTemplate, NotificationRecipientFn, NotificationRecordMapping, NotificationTemplateBody, NotificationTemplateFn, NotificationTemplateMeta, NotifyDeferred, NotifyDelivery, NotifyDeliveryStatus, NotifyDispatched, NotifyFactory, NotifyFn, NotifyJobDispatcher, NotifyOptions, NotifyPriority, NotifyResult, NumberFieldDef, OnDeleteStrategy, OpenToAllAccessRule, OpenToAllDeclaration, OpenToAllPersonalData, ParentRefDef, PersonalAnnotations, PersonalAnnotationsLongText, PersonalAnnotationsNoFind, PersonalSubject, PlatformComponent, PluralForms, PostDeleteHookFn, PostSaveHookFn, PreDeleteHookFn, PreparedWriteEvent, PreQueryHookFn, PreSaveHookFn, ProjectionDefinition, ProjectionDetailScreenDefinition, ProjectionListScreenDefinition, ProjectionTable, QualifiedEventName, QueryEvent, QueryHandlerDef, QueryHandlerFn, RateLimitPer, RecordHeaderSpec, RecordHeaderSubtitlePart, ReferenceDataDef, Registry, RelationDefinition, ResolvedPiiFlags, RetentionDef, RoleAccessPersonalData, RoleAccessRule, RowAction, RowActionNavigate, RowActionNavigateBase, RowActionWriteHandler, RowFieldExtractor, SaveContext, ScreenDefinition, ScreenFilter, ScreenFilterOp, ScreenNavSugar, ScreenSlots, SecretKeyDefinition, SecretKeyHandle, SecretMintConfirmStep, SecretMintScreenDefinition, SecretNamespaceDefinition, SecretNamespaceHandle, SecretNamespaceOptions, SecretReveal, SecretRevealField, SecretsEditScreenDefinition, SecretsEditSection, SelectFieldDef, SessionUser, SessionUserOrigin, StreamHandlerDef, StreamHandlerFn, Subscribe, TargetRef, TemplateListVariableDef, TemplateScalarVariableDef, TemplateVariableDef, TemplateVariables, TenantId, TenantWriteTarget, TextFieldDef, ToolbarAction, TranslationEntry, TranslationKeys, TranslationsDef, TranslationValue, TreeAction, TreeActionDef, TreeActionsHandle, TreeChildrenSubscribe, TreeNode, TreeNodeState, UnitKey, UnsafeAppendEventFn, ValidationError, ValidationHookFn, WorkspaceDefinition, WriteEvent, WriteHandlerDef, WriteHandlerFn, WriteResult, } from "./types/index.js";
|
|
84
|
+
export type { AccessRule, ActionFormScreenDefinition, ActiveMembership, ActiveMembershipRejection, ActiveMembershipResult, AgentExposure, AgentHandlerHints, AgentRisk, AppContext, AppendEventArgs, AppendEventFn, AuthClaimsContext, AuthClaimsFn, AuthClaimsHookDef, BeforeTransactionPhase, BeforeTransactionResult, BelongsToRelation, BigIntFieldDef, BooleanFieldDef, CamelToKebab, ChatSendFailureCode, ClaimKeyDefinition, ClaimKeyHandle, ClaimKeyJsType, ClaimKeyType, ConcurrencyMode, ConfigAccessor, ConfigAccessorFactory, ConfigBacking, ConfigCascade, ConfigCascadeLevel, ConfigDefinition, ConfigEditScreenDefinition, ConfigKeyAccess, ConfigKeyDefinition, ConfigKeyHandle, ConfigKeyType, ConfigMask, ConfigResolver, ConfigScope, ConfigSecretsReader, ConfigSeedDef, ConfigStoredRow, ConfigStoredRowWithSource, ConfigValidateContext, ConfigValidateFn, ConfigValue, ConfigValueSource, ConfigValueWithSource, ContentCollectionDefinition, CreateSeedOptions, CreateTenantSeedOptions, CreateUserSeedOptions, CustomScreenDefinition, CustomScreenRoute, DashboardChartPanel, DashboardCustomPanel, DashboardDateParam, DashboardFeedPanel, DashboardFilterDefinition, DashboardI18nText, DashboardListPanel, DashboardMoneyParam, DashboardPanelDefinition, DashboardPanelVisibility, DashboardProgressListPanel, DashboardScreenDefinition, DashboardScreenPanel, DashboardStatGroupPanel, DashboardStatPanel, DashboardText, DashboardTextParam, DashboardValueFormat, DateFieldDef, DecimalFieldDef, DeleteContext, DeliveryErrorCode, DeliveryFailureCode, DeliverySkipReason, DeriveContext, DerivedFieldDef, DerivedFieldsMap, DerivedValueType, EditExtensionSection, EditFieldPrefill, EditFieldSpec, EditFieldsSection, EditLayout, EditRelatedListSection, EditSectionSpec, EmbeddedListEditSpec, EntityDefinition, EntityEditAction, EntityEditActionPlacement, EntityEditScreenDefinition, EntityId, EntityListExpandableRow, EntityListScreenDefinition, EntityRef, EntityRelations, EscapeHatchAuditSink, EscapeHatchDeclaration, EscapeHatchKind, EscapeHatchReporter, EscapeHatchTarget, EscapeHatchUseEvent, EventDef, ExtensionOptionsArgs, ExtensionOptionsFor, FeatureDefinition, FeatureRegistrar, FieldAccess, FieldCondition, FieldDefinition, FieldFormatRegistry, FieldIconKey, FieldRenderer, FileFieldDef, FilesFieldDef, Findability, FormatSpec, FormWidth, HandlerContext, HasManyRelation, HookMap, ImageFieldDef, ImagesFieldDef, JobBackoff, JobBackoffStrategy, JobContext, JobDefinition, JobHandlerFn, JobTrigger, JobTriggerEvent, JsonbFieldDef, KumikoEntityTypeMap, KumikoEventTypeMap, KumikoExtensionOptionsMap, KumikoHandlerPayloadMap, KumikoHandlerResultMap, LifecycleHookType, ListColumnSpec, ListFacetSpec, ListPaginationMode, ListSortDir, ListSortSpec, LongTextFindability, ManyToManyRelation, MaxPerTenantOptions, MemberReader, MetricNavigate, MetricSpec, MspErrorMode, MspErrorPolicy, MultiSelectFieldDef, MultiStreamApplyFn, MultiStreamProjectionDefinition, NameOrRef, NavDefinition, NavIconKey, NotificationDataFn, NotificationDefinition, NotificationLocalizedTemplate, NotificationRecipientFn, NotificationRecordMapping, NotificationTemplateBody, NotificationTemplateFn, NotificationTemplateMeta, NotifyDeferred, NotifyDelivery, NotifyDeliveryStatus, NotifyDispatched, NotifyFactory, NotifyFn, NotifyJobDispatcher, NotifyOptions, NotifyPriority, NotifyResult, NumberFieldDef, OnDeleteStrategy, OpenToAllAccessRule, OpenToAllDeclaration, OpenToAllPersonalData, ParentRefDef, PersonalAnnotations, PersonalAnnotationsLongText, PersonalAnnotationsNoFind, PersonalSubject, PlatformComponent, PluralForms, PostDeleteHookFn, PostSaveHookFn, PreDeleteHookFn, PreparedWriteEvent, PreQueryHookFn, PreSaveHookFn, ProjectionDefinition, ProjectionDetailScreenDefinition, ProjectionListScreenDefinition, ProjectionTable, QualifiedEventName, QueryEvent, QueryHandlerDef, QueryHandlerFn, RateLimitPer, RecordHeaderSpec, RecordHeaderSubtitlePart, ReferenceDataDef, Registry, RelationDefinition, ResolvedPiiFlags, RetentionDef, RoleAccessPersonalData, RoleAccessRule, RowAction, RowActionNavigate, RowActionNavigateBase, RowActionWriteHandler, RowFieldExtractor, SaveContext, ScreenDefinition, ScreenFilter, ScreenFilterOp, ScreenNavSugar, ScreenSlots, SecretKeyDefinition, SecretKeyHandle, SecretMintConfirmStep, SecretMintScreenDefinition, SecretNamespaceDefinition, SecretNamespaceHandle, SecretNamespaceOptions, SecretReveal, SecretRevealField, SecretsEditScreenDefinition, SecretsEditSection, SelectFieldDef, SessionUser, SessionUserOrigin, StreamHandlerDef, StreamHandlerFn, Subscribe, TargetRef, TemplateListVariableDef, TemplateScalarVariableDef, TemplateVariableDef, TemplateVariables, TenantId, TenantWriteTarget, TextFieldDef, ToolbarAction, TranslationEntry, TranslationKeys, TranslationsDef, TranslationValue, TreeAction, TreeActionDef, TreeActionsHandle, TreeChildrenSubscribe, TreeNode, TreeNodeState, UnitKey, UnsafeAppendEventFn, ValidationError, ValidationHookFn, WorkspaceDefinition, WriteEvent, WriteHandlerDef, WriteHandlerFn, WriteResult, } from "./types/index.js";
|
|
85
85
|
export { DEFAULT_CURRENCIES, DEFAULT_LOCALES, DELIVERY_FAILURE_CODES, DELIVERY_SKIP_REASONS, HookPhases, HTTP_ERROR_CODE_PATTERN, isAgentVisibleScreen, isDeliveryErrorCode, isOpenToAllGranted, resolveAgentExposure, } from "./types/index.js";
|
|
86
86
|
export type { AwaitedEventType, PipelineBuildCtx, PipelineCtx, PipelineDef, StepBuilder, StepDef, StepFailureStrategy, StepInstance, StepKind, StepNamespace, StepResolver, } from "./types/step.js";
|
|
87
87
|
export { runValidation } from "./validation.js";
|
|
@@ -26,6 +26,7 @@ export declare function resolveJobTriggers(state: RegistryState): void;
|
|
|
26
26
|
export declare function validateLiveEntities(state: RegistryState): void;
|
|
27
27
|
export declare function validateBootGates(state: RegistryState): void;
|
|
28
28
|
export declare function validateJobBackoff(state: RegistryState): void;
|
|
29
|
+
export declare function validateMaxPerTenant(state: RegistryState): void;
|
|
29
30
|
export declare function validateExtensionUsageTargets(state: RegistryState): void;
|
|
30
31
|
export declare function computeHasRateLimitedHandler(state: RegistryState): void;
|
|
31
32
|
export declare function publishEventPiiCatalog(state: RegistryState): void;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { configureEventPiiCatalog } from "../crypto/event-pii.js";
|
|
2
2
|
import { hasSearchablePlaintext, isSensitiveLabelField } from "../db/entity-field-encryption.js";
|
|
3
3
|
import { entityEventName } from "../db/event-store-executor-context.js";
|
|
4
|
+
import { resolveMaxPerTenant } from "../jobs/max-per-tenant.js";
|
|
4
5
|
import { bindHookEscapeHatchGrant } from "../pipeline/system-identity-switch.js";
|
|
5
6
|
import { resolveName } from "./handler-helpers.js";
|
|
6
7
|
import { parseRefTargetEntityName } from "./parse-ref-target.js";
|
|
@@ -744,6 +745,26 @@ export function validateJobBackoff(state) {
|
|
|
744
745
|
}
|
|
745
746
|
}
|
|
746
747
|
}
|
|
748
|
+
export function validateMaxPerTenant(state) {
|
|
749
|
+
for (const [jobName, jobDef] of state.jobMap) {
|
|
750
|
+
const resolved = resolveMaxPerTenant(jobDef);
|
|
751
|
+
if (!resolved)
|
|
752
|
+
continue;
|
|
753
|
+
if (!Number.isInteger(resolved.limit) || resolved.limit <= 0) {
|
|
754
|
+
throw new Error(`Job "${jobName}" maxPerTenant limit must be a positive integer (got ${resolved.limit})`);
|
|
755
|
+
}
|
|
756
|
+
if (typeof jobDef.maxPerTenant !== "object")
|
|
757
|
+
continue;
|
|
758
|
+
const { deferMs } = jobDef.maxPerTenant;
|
|
759
|
+
if (deferMs !== undefined && (!Number.isInteger(deferMs) || deferMs <= 0)) {
|
|
760
|
+
throw new Error(`Job "${jobName}" maxPerTenant.deferMs must be a positive integer (got ${deferMs})`);
|
|
761
|
+
}
|
|
762
|
+
// The inline boot-gate path has no BullMQ token to move the job to delayed with.
|
|
763
|
+
if (resolved.onLimit === "defer" && jobDef.bootGate) {
|
|
764
|
+
throw new Error(`Job "${jobName}" cannot combine bootGate with maxPerTenant onLimit "defer"`);
|
|
765
|
+
}
|
|
766
|
+
}
|
|
767
|
+
}
|
|
747
768
|
export function validateExtensionUsageTargets(state) {
|
|
748
769
|
// Validate: extension usages must reference existing extensions
|
|
749
770
|
for (const usage of state.extensionUsages) {
|
package/dist/engine/registry.js
CHANGED
|
@@ -3,7 +3,7 @@ import { dedupeFeatures } from "./dedupe-features.js";
|
|
|
3
3
|
import { buildRegistryFacade } from "./registry-facade.js";
|
|
4
4
|
import { populateClaimsAndAuth, populateConfigKeys, populateEvents, populateExtensionsAndSeeds, populateFeatureCore, populateHandlers, populateHooks, populateJobsAndNotifications, populateMetricsAndSecrets, populateProjectionsAndTables, populateScreensNavWorkspaces, populateSystemEvents, populateTranslations, } from "./registry-ingest.js";
|
|
5
5
|
import { createInitialState } from "./registry-state.js";
|
|
6
|
-
import { applyExtensionUsages, autoWireSoftDeleteJobs, buildEntityEventUpcasterChains, buildEventUpcasterChains, buildImplicitProjections, buildIncomingRelations, buildSearchableSortableCaches, computeHasRateLimitedHandler, finalizeWorkspaceNavMembership, populateHandlerEntityMappings, publishEventPiiCatalog, resolveJobTriggers, resolveNotificationTriggersAndRegisterHooks, validateBootGates, validateEntityHookTargets, validateEventMigrationVersions, validateExtensionSelectors, validateExtensionUsageTargets, validateFieldAccessHandlersAreEntityMapped, validateJobBackoff, validateLifecycleHookTargets, validateLiveEntities, validateNotificationRecordMappings, validateProjectionApplyKeys, validateRelationTargetsExist, validateRequiredFeatures, } from "./registry-validate.js";
|
|
6
|
+
import { applyExtensionUsages, autoWireSoftDeleteJobs, buildEntityEventUpcasterChains, buildEventUpcasterChains, buildImplicitProjections, buildIncomingRelations, buildSearchableSortableCaches, computeHasRateLimitedHandler, finalizeWorkspaceNavMembership, populateHandlerEntityMappings, publishEventPiiCatalog, resolveJobTriggers, resolveNotificationTriggersAndRegisterHooks, validateBootGates, validateEntityHookTargets, validateEventMigrationVersions, validateExtensionSelectors, validateExtensionUsageTargets, validateFieldAccessHandlersAreEntityMapped, validateJobBackoff, validateLifecycleHookTargets, validateLiveEntities, validateMaxPerTenant, validateNotificationRecordMappings, validateProjectionApplyKeys, validateRelationTargetsExist, validateRequiredFeatures, } from "./registry-validate.js";
|
|
7
7
|
// This is where the magic happens. By "magic" I mean: precomputed maps.
|
|
8
8
|
// I build everything once at boot (hooks, relations, searchable fields, ...)
|
|
9
9
|
// so nothing has to iterate over objects at runtime. O(1) instead of O(n*m).
|
|
@@ -48,6 +48,7 @@ export function createRegistry(rawFeatures) {
|
|
|
48
48
|
validateBootGates(state);
|
|
49
49
|
validateLiveEntities(state);
|
|
50
50
|
validateJobBackoff(state);
|
|
51
|
+
validateMaxPerTenant(state);
|
|
51
52
|
validateExtensionUsageTargets(state);
|
|
52
53
|
computeHasRateLimitedHandler(state);
|
|
53
54
|
publishEventPiiCatalog(state);
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export { isAgentVisibleScreen, resolveAgentExposure, } from "@cosmicdrift/kumiko-types/agent-exposure";
|
|
2
|
-
export type { ConfigAccessor, ConfigAccessorFactory, ConfigBacking, ConfigBounds, ConfigCascade, ConfigCascadeLevel, ConfigComputedContext, ConfigComputedFn, ConfigDefinition, ConfigKeyAccess, ConfigKeyDefinition, ConfigKeyHandle, ConfigKeyType, ConfigMask, ConfigResolver, ConfigSecretsReader, ConfigSeedDef, ConfigStoredRow, ConfigStoredRowWithSource, ConfigValidateContext, ConfigValidateFn, ConfigValue, ConfigValueSource, ConfigValueWithSource, CreateSeedOptions, CreateTenantSeedOptions, CreateUserSeedOptions, ExtensionSelectorDef, ExtensionSelectorPanel, JobBackoff, JobBackoffStrategy, JobDefinition, JobHandlerFn, JobRunIn, JobTrigger, NotificationDataFn, NotificationDefinition, NotificationLocalizedTemplate, NotificationRecipientFn, NotificationRecordMapping, NotificationTemplateBody, NotificationTemplateFn, NotificationTemplateMeta, PluralForms, ReferenceDataDef, RegistrarExtensionDef, RegistrarExtensionHooks, RegistrarExtensionRegistration, RunIn, TemplateListVariableDef, TemplateScalarVariableDef, TemplateVariableDef, TemplateVariables, TranslationEntry, TranslationKeys, TranslationsDef, TranslationValue, UiExtensionDef, } from "@cosmicdrift/kumiko-types/config";
|
|
2
|
+
export type { ConfigAccessor, ConfigAccessorFactory, ConfigBacking, ConfigBounds, ConfigCascade, ConfigCascadeLevel, ConfigComputedContext, ConfigComputedFn, ConfigDefinition, ConfigKeyAccess, ConfigKeyDefinition, ConfigKeyHandle, ConfigKeyType, ConfigMask, ConfigResolver, ConfigSecretsReader, ConfigSeedDef, ConfigStoredRow, ConfigStoredRowWithSource, ConfigValidateContext, ConfigValidateFn, ConfigValue, ConfigValueSource, ConfigValueWithSource, CreateSeedOptions, CreateTenantSeedOptions, CreateUserSeedOptions, ExtensionSelectorDef, ExtensionSelectorPanel, JobBackoff, JobBackoffStrategy, JobDefinition, JobHandlerFn, JobRunIn, JobTrigger, MaxPerTenantOptions, NotificationDataFn, NotificationDefinition, NotificationLocalizedTemplate, NotificationRecipientFn, NotificationRecordMapping, NotificationTemplateBody, NotificationTemplateFn, NotificationTemplateMeta, PluralForms, ReferenceDataDef, RegistrarExtensionDef, RegistrarExtensionHooks, RegistrarExtensionRegistration, RunIn, TemplateListVariableDef, TemplateScalarVariableDef, TemplateVariableDef, TemplateVariables, TranslationEntry, TranslationKeys, TranslationsDef, TranslationValue, UiExtensionDef, } from "@cosmicdrift/kumiko-types/config";
|
|
3
3
|
export type { QueryHandlerDefinition, StreamHandlerDefinition, WriteHandlerDefinition, WriteHandlerInput, } from "@cosmicdrift/kumiko-types/define-handler";
|
|
4
4
|
export type { EntityCrudRegistrar, EntityCrudVerb, EntityHandlerOptions, EntityQueryHandlerOptions, EntityWriteHandlerOptions, RegisterEntityCrudOptions, } from "@cosmicdrift/kumiko-types/entity-handlers";
|
|
5
5
|
export type { KumikoEntityTypeMap, KumikoEventTypeMap, KumikoHandlerPayloadMap, KumikoHandlerResultMap, } from "@cosmicdrift/kumiko-types/event-type-map";
|
package/dist/jobs/job-runner.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { escapeHatchFor } from "@cosmicdrift/kumiko-types/handlers";
|
|
2
2
|
import { isPlainObject } from "@cosmicdrift/kumiko-types/type-guards";
|
|
3
|
-
import { Queue, UnrecoverableError, Worker } from "bullmq";
|
|
3
|
+
import { DelayedError, Queue, UnrecoverableError, Worker } from "bullmq";
|
|
4
4
|
import { Redis } from "ioredis";
|
|
5
5
|
import { requestContext } from "../api/request-context.js";
|
|
6
6
|
import { createTenantDb, createUncheckedSystemDb } from "../db/tenant-db.js";
|
|
@@ -19,6 +19,7 @@ import { createGatedIdentitySwitch } from "../pipeline/system-identity-switch.js
|
|
|
19
19
|
import { buildPersonalDataGate, isPersonalDataGated, parseWriteOrigin, } from "../pipeline/write-origin.js";
|
|
20
20
|
import { bridgeStub } from "../testing/handler-context.js";
|
|
21
21
|
import { cronRepeatOptions } from "./cron-repeat-options.js";
|
|
22
|
+
import { resolveMaxPerTenant } from "./max-per-tenant.js";
|
|
22
23
|
// Shape-checked after the BullMQ/Redis JSON round-trip. Unlike `_writeOrigin`
|
|
23
24
|
// (a security-relevant escape hatch that must fail the run closed), a
|
|
24
25
|
// malformed `_triggerEvent` only costs an idempotency-key convenience — the
|
|
@@ -248,6 +249,9 @@ function tenantFailureMessageKey(err, declaration) {
|
|
|
248
249
|
// Leading underscore matches the existing internal-meta convention
|
|
249
250
|
// (_triggeredById, _tenantId, _payload).
|
|
250
251
|
const TRACE_CONTEXT_KEY = "_traceContext";
|
|
252
|
+
const DEFER_COUNT_KEY = "_deferCount";
|
|
253
|
+
const DEFER_DELAY_CAP_MS = 30_000;
|
|
254
|
+
const DEFER_BACKOFF_MAX_DOUBLINGS = 5;
|
|
251
255
|
function readTraceContext(data) {
|
|
252
256
|
const raw = data[TRACE_CONTEXT_KEY];
|
|
253
257
|
if (!raw || typeof raw !== "object")
|
|
@@ -555,18 +559,23 @@ export function createJobRunner(options) {
|
|
|
555
559
|
// lane (jobDef.runIn is static), but walking both is cheap and avoids
|
|
556
560
|
// a subtle bug if someone ever reassigns a job to a different lane
|
|
557
561
|
// between deploys while old queue contents are still draining.
|
|
558
|
-
|
|
562
|
+
// `activeOnly` + `self` serve the defer check, which runs inside the job being
|
|
563
|
+
// counted: it must not see waiting jobs or itself, and only counts jobs that
|
|
564
|
+
// became active before it. Without that order, workers that fetch a burst of
|
|
565
|
+
// jobs together all see each other as active and all defer, again and again.
|
|
566
|
+
async function isOverPerTenantLimit(jobName, tenantId, max, scope = {}) {
|
|
559
567
|
const results = await Promise.all([
|
|
560
568
|
queues.api.getActive(),
|
|
561
|
-
queues.api.getWaiting(),
|
|
562
569
|
queues.worker.getActive(),
|
|
563
|
-
queues.worker.getWaiting(),
|
|
570
|
+
...(scope.activeOnly ? [] : [queues.api.getWaiting(), queues.worker.getWaiting()]),
|
|
564
571
|
]);
|
|
565
572
|
let count = 0;
|
|
566
573
|
for (const list of results) {
|
|
567
574
|
for (const j of list) {
|
|
568
575
|
if (j.name !== jobName)
|
|
569
576
|
continue;
|
|
577
|
+
if (scope.self && !isActiveBefore(j, scope.self))
|
|
578
|
+
continue;
|
|
570
579
|
const t = j.data?._tenantId; // @cast-boundary dynamic-key
|
|
571
580
|
if (t === tenantId) {
|
|
572
581
|
count += 1;
|
|
@@ -577,6 +586,22 @@ export function createJobRunner(options) {
|
|
|
577
586
|
}
|
|
578
587
|
return false;
|
|
579
588
|
}
|
|
589
|
+
function isActiveBefore(other, self) {
|
|
590
|
+
if (other.id === self.id)
|
|
591
|
+
return false;
|
|
592
|
+
if (self.processedOn === undefined)
|
|
593
|
+
return true;
|
|
594
|
+
const otherProcessedOn = other.processedOn ?? 0;
|
|
595
|
+
if (otherProcessedOn !== self.processedOn)
|
|
596
|
+
return otherProcessedOn < self.processedOn;
|
|
597
|
+
return (other.id ?? "") < (self.id ?? "");
|
|
598
|
+
}
|
|
599
|
+
// Growing delay bounds Redis and worker load while one tenant has a large
|
|
600
|
+
// backlog: every deferred job would otherwise be re-checked every deferMs.
|
|
601
|
+
function deferDelayMs(deferMs, deferCount) {
|
|
602
|
+
const grown = Math.min(deferMs * 2 ** Math.min(deferCount, DEFER_BACKOFF_MAX_DOUBLINGS), DEFER_DELAY_CAP_MS);
|
|
603
|
+
return grown + Math.floor(Math.random() * deferMs);
|
|
604
|
+
}
|
|
580
605
|
// Only err.message, never payload or the logs array: job payloads can carry PII.
|
|
581
606
|
function logFailedAttempt(entry) {
|
|
582
607
|
errorLogger.error("job failed", {
|
|
@@ -647,7 +672,7 @@ export function createJobRunner(options) {
|
|
|
647
672
|
});
|
|
648
673
|
}
|
|
649
674
|
}
|
|
650
|
-
async function handleJob(bullJob) {
|
|
675
|
+
async function handleJob(bullJob, token) {
|
|
651
676
|
const rawName = bullJob.name;
|
|
652
677
|
// Handle perTenant dispatch jobs — fan out to one job per tenant. The
|
|
653
678
|
// fan-out re-enqueues into the lane the actual job is assigned to;
|
|
@@ -729,6 +754,25 @@ export function createJobRunner(options) {
|
|
|
729
754
|
await reportPreRunFailure(jobName, bullJob, err);
|
|
730
755
|
throw err;
|
|
731
756
|
}
|
|
757
|
+
// maxPerTenant defer: checked before anything is recorded for this run, so a
|
|
758
|
+
// deferred pickup leaves no span, run-log entry or failure trace and does not
|
|
759
|
+
// consume an attempt (moveToDelayed + DelayedError, not a retry).
|
|
760
|
+
const maxPerTenant = resolveMaxPerTenant(jobDef);
|
|
761
|
+
const deferTenantId = bullJob.data["_tenantId"];
|
|
762
|
+
if (maxPerTenant?.onLimit === "defer" &&
|
|
763
|
+
typeof deferTenantId === "string" &&
|
|
764
|
+
token !== undefined &&
|
|
765
|
+
bullJob.moveToDelayed &&
|
|
766
|
+
(await isOverPerTenantLimit(jobName, deferTenantId, maxPerTenant.limit, {
|
|
767
|
+
activeOnly: true,
|
|
768
|
+
self: { id: bullJob.id, processedOn: bullJob.processedOn },
|
|
769
|
+
}))) {
|
|
770
|
+
const priorDeferCount = bullJob.data[DEFER_COUNT_KEY];
|
|
771
|
+
const deferCount = typeof priorDeferCount === "number" ? priorDeferCount : 0;
|
|
772
|
+
await bullJob.updateData?.({ ...bullJob.data, [DEFER_COUNT_KEY]: deferCount + 1 });
|
|
773
|
+
await bullJob.moveToDelayed(Date.now() + deferDelayMs(maxPerTenant.deferMs, deferCount), token);
|
|
774
|
+
throw new DelayedError();
|
|
775
|
+
}
|
|
732
776
|
// Sequential gate: try to claim the per-name lock. If another worker
|
|
733
777
|
// (or this worker on a different bullJob) holds it, re-enqueue with a
|
|
734
778
|
// small delay and exit *successfully* — using throw would burn the
|
|
@@ -1202,7 +1246,7 @@ export function createJobRunner(options) {
|
|
|
1202
1246
|
return;
|
|
1203
1247
|
}
|
|
1204
1248
|
const consumerQueue = queues[consumerLane];
|
|
1205
|
-
worker = new Worker(queueNameFor(queueNamePrefix, consumerLane), handleJob, {
|
|
1249
|
+
worker = new Worker(queueNameFor(queueNamePrefix, consumerLane), (job, token) => handleJob(job, token), {
|
|
1206
1250
|
connection: redisOpts,
|
|
1207
1251
|
concurrency: 5,
|
|
1208
1252
|
// Not consuming until the boot gates passed: a job already queued from a
|
|
@@ -1361,10 +1405,11 @@ export function createJobRunner(options) {
|
|
|
1361
1405
|
// against; without it the guard is inactive (system jobs, ambient
|
|
1362
1406
|
// dispatch). Fan-out children of perTenant jobs land here on their
|
|
1363
1407
|
// recursive queue.add and DO carry _tenantId.
|
|
1364
|
-
|
|
1408
|
+
const maxPerTenant = resolveMaxPerTenant(jobDef);
|
|
1409
|
+
if (maxPerTenant?.onLimit === "skip") {
|
|
1365
1410
|
const tenantId = payload?._tenantId; // @cast-boundary dynamic-key
|
|
1366
1411
|
if (tenantId !== undefined &&
|
|
1367
|
-
(await isOverPerTenantLimit(jobName, tenantId,
|
|
1412
|
+
(await isOverPerTenantLimit(jobName, tenantId, maxPerTenant.limit))) {
|
|
1368
1413
|
return "skipped:max-per-tenant";
|
|
1369
1414
|
}
|
|
1370
1415
|
}
|
|
@@ -1406,6 +1451,8 @@ export function createJobRunner(options) {
|
|
|
1406
1451
|
bullOpts["timeout"] = jobDef.timeout;
|
|
1407
1452
|
if (meta?.priority !== undefined)
|
|
1408
1453
|
bullOpts["priority"] = meta.priority;
|
|
1454
|
+
if (meta?.jobId !== undefined)
|
|
1455
|
+
bullOpts["jobId"] = meta.jobId;
|
|
1409
1456
|
// Pack meta into job data with _ prefix
|
|
1410
1457
|
const data = { ...payload };
|
|
1411
1458
|
if (meta?.triggeredById !== undefined)
|
|
@@ -1467,8 +1514,9 @@ export function createJobRunner(options) {
|
|
|
1467
1514
|
data["_correlationId"] = reqCtx.correlationId;
|
|
1468
1515
|
// Same maxPerTenant guard as dispatch — events that fan into many
|
|
1469
1516
|
// jobs must respect the per-tenant cap or the limit is one-sided.
|
|
1470
|
-
|
|
1471
|
-
|
|
1517
|
+
const maxPerTenant = resolveMaxPerTenant(jobDef);
|
|
1518
|
+
if (maxPerTenant?.onLimit === "skip" && user?.tenantId !== undefined) {
|
|
1519
|
+
if (await isOverPerTenantLimit(name, String(user.tenantId), maxPerTenant.limit)) {
|
|
1472
1520
|
continue;
|
|
1473
1521
|
}
|
|
1474
1522
|
}
|
|
@@ -1485,17 +1533,21 @@ export function createJobRunner(options) {
|
|
|
1485
1533
|
async countPendingJobs() {
|
|
1486
1534
|
let total = 0;
|
|
1487
1535
|
for (const queue of Object.values(queues)) {
|
|
1488
|
-
|
|
1536
|
+
// One getJobCounts call is a single atomic snapshot. Reading delayed in a
|
|
1537
|
+
// second call misses a deferred job that cycles delayed -> active between
|
|
1538
|
+
// the reads, so delayed is counted here and only repeat schedulers (which
|
|
1539
|
+
// stay delayed between cron fires) are subtracted afterwards.
|
|
1540
|
+
const counts = await queue.getJobCounts("waiting", "active", "prioritized", "waiting-children", "delayed");
|
|
1489
1541
|
total +=
|
|
1490
1542
|
(counts["waiting"] ?? 0) +
|
|
1491
1543
|
(counts["active"] ?? 0) +
|
|
1492
1544
|
(counts["prioritized"] ?? 0) +
|
|
1493
|
-
(counts["waiting-children"] ?? 0)
|
|
1494
|
-
|
|
1495
|
-
// loads
|
|
1545
|
+
(counts["waiting-children"] ?? 0) +
|
|
1546
|
+
(counts["delayed"] ?? 0);
|
|
1547
|
+
// getJobs(["delayed"]) loads full Job objects just to read .id; fine for
|
|
1496
1548
|
// test-draining's low volume, not a pattern for a prod-path hot loop.
|
|
1497
1549
|
const delayed = await queue.getJobs(["delayed"]);
|
|
1498
|
-
total
|
|
1550
|
+
total -= delayed.filter((job) => (job.id ?? "").startsWith("repeat:")).length;
|
|
1499
1551
|
}
|
|
1500
1552
|
return total;
|
|
1501
1553
|
},
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { JobDefinition } from "../engine/types/index.js";
|
|
2
|
+
export declare const DEFAULT_MAX_PER_TENANT_DEFER_MS = 1000;
|
|
3
|
+
export type ResolvedMaxPerTenant = {
|
|
4
|
+
readonly limit: number;
|
|
5
|
+
readonly onLimit: "skip" | "defer";
|
|
6
|
+
readonly deferMs: number;
|
|
7
|
+
};
|
|
8
|
+
export declare function resolveMaxPerTenant(jobDef: Pick<JobDefinition, "maxPerTenant">): ResolvedMaxPerTenant | undefined;
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export const DEFAULT_MAX_PER_TENANT_DEFER_MS = 1_000;
|
|
2
|
+
export function resolveMaxPerTenant(jobDef) {
|
|
3
|
+
const raw = jobDef.maxPerTenant;
|
|
4
|
+
if (raw === undefined)
|
|
5
|
+
return undefined;
|
|
6
|
+
if (typeof raw === "number") {
|
|
7
|
+
return { limit: raw, onLimit: "skip", deferMs: DEFAULT_MAX_PER_TENANT_DEFER_MS };
|
|
8
|
+
}
|
|
9
|
+
return {
|
|
10
|
+
limit: raw.limit,
|
|
11
|
+
onLimit: raw.onLimit,
|
|
12
|
+
deferMs: raw.deferMs ?? DEFAULT_MAX_PER_TENANT_DEFER_MS,
|
|
13
|
+
};
|
|
14
|
+
}
|
package/dist/pipeline/index.d.ts
CHANGED
|
@@ -31,6 +31,8 @@ export type { LifecycleHooks, SystemHookDef, SystemHooks } from "./lifecycle-pip
|
|
|
31
31
|
export { createLifecycleHooks } from "./lifecycle-pipeline.js";
|
|
32
32
|
export type { MspRebuildDeps } from "./msp-rebuild.js";
|
|
33
33
|
export { rebuildMultiStreamProjection } from "./msp-rebuild.js";
|
|
34
|
+
export type { MultiStreamApplyContext, MultiStreamApplyContextDeps, } from "./multi-stream-apply-context.js";
|
|
35
|
+
export { createMultiStreamApplyContext } from "./multi-stream-apply-context.js";
|
|
34
36
|
export { assertNotifyTargetsOwnTenant, createDeferringNotify } from "./notify-scope.js";
|
|
35
37
|
export type { ProjectionProgress, RebuildResult } from "./projection-rebuild.js";
|
|
36
38
|
export { getAllProjectionProgress, getProjectionState, listProjectionsWithState, rebuildProjection, } from "./projection-rebuild.js";
|
package/dist/pipeline/index.js
CHANGED
|
@@ -16,6 +16,7 @@ export { createIdempotencyGuard } from "./idempotency.js";
|
|
|
16
16
|
export { assertInstructionFieldWriteAllowed, assertIrreversibleOperationAllowed, isIrreversibleEntityVerb, } from "./irreversible-operation-gate.js";
|
|
17
17
|
export { createLifecycleHooks } from "./lifecycle-pipeline.js";
|
|
18
18
|
export { rebuildMultiStreamProjection } from "./msp-rebuild.js";
|
|
19
|
+
export { createMultiStreamApplyContext } from "./multi-stream-apply-context.js";
|
|
19
20
|
export { assertNotifyTargetsOwnTenant, createDeferringNotify } from "./notify-scope.js";
|
|
20
21
|
export { getAllProjectionProgress, getProjectionState, listProjectionsWithState, rebuildProjection, } from "./projection-rebuild.js";
|
|
21
22
|
export { createProjectionStateTable, ProjectionStatuses, projectionStateTable, } from "./projection-state.js";
|
|
@@ -14,5 +14,6 @@ export type MultiStreamApplyContextDeps = {
|
|
|
14
14
|
readonly files?: FileContext;
|
|
15
15
|
readonly derivatives?: DerivativesContext;
|
|
16
16
|
readonly secrets?: SecretsContext;
|
|
17
|
+
readonly jobs?: MultiStreamApplyContext["jobs"];
|
|
17
18
|
};
|
|
18
19
|
export declare function createMultiStreamApplyContext(deps: MultiStreamApplyContextDeps): MultiStreamApplyContext;
|
|
@@ -7,6 +7,7 @@ export function createMultiStreamApplyContext(deps) {
|
|
|
7
7
|
...(deps.files ? { files: deps.files } : {}),
|
|
8
8
|
...(deps.derivatives ? { derivatives: deps.derivatives } : {}),
|
|
9
9
|
...(deps.secrets ? { secrets: deps.secrets } : {}),
|
|
10
|
+
...(deps.jobs ? { jobs: deps.jobs } : {}),
|
|
10
11
|
appendEvent: (async (args) => {
|
|
11
12
|
await appendDomainEventCore({
|
|
12
13
|
registry: deps.registry,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cosmicdrift/kumiko-framework",
|
|
3
|
-
"version": "0.365.
|
|
3
|
+
"version": "0.365.1",
|
|
4
4
|
"description": "Framework core — engine, pipeline, API, DB, and every other bit that makes Kumiko go.",
|
|
5
5
|
"license": "BUSL-1.1",
|
|
6
6
|
"author": "Marc Frost <marc@cosmicdriftgamestudio.com>",
|
|
@@ -210,8 +210,8 @@
|
|
|
210
210
|
"./package.json": "./package.json"
|
|
211
211
|
},
|
|
212
212
|
"dependencies": {
|
|
213
|
-
"@cosmicdrift/kumiko-http": "0.365.
|
|
214
|
-
"@cosmicdrift/kumiko-types": "0.365.
|
|
213
|
+
"@cosmicdrift/kumiko-http": "0.365.1",
|
|
214
|
+
"@cosmicdrift/kumiko-types": "0.365.1",
|
|
215
215
|
"bullmq": "^5.76.7",
|
|
216
216
|
"bun-types": "^1.3.13",
|
|
217
217
|
"hono": "^4.13.1",
|
|
@@ -227,7 +227,7 @@
|
|
|
227
227
|
"zod": "^4.4.3"
|
|
228
228
|
},
|
|
229
229
|
"devDependencies": {
|
|
230
|
-
"@cosmicdrift/kumiko-dispatcher-live": "0.365.
|
|
230
|
+
"@cosmicdrift/kumiko-dispatcher-live": "0.365.1",
|
|
231
231
|
"bun-types": "^1.3.13",
|
|
232
232
|
"pino-pretty": "^13.1.3"
|
|
233
233
|
},
|
package/src/changes.json
CHANGED
|
@@ -1,4 +1,22 @@
|
|
|
1
1
|
[
|
|
2
|
+
{
|
|
3
|
+
"version": "0.365.1",
|
|
4
|
+
"type": "fix",
|
|
5
|
+
"title": "App and worker boot wait for Postgres instead of crashing",
|
|
6
|
+
"detail": "runProdApp and worker processes now probe the database before the schema-drift check and retry with exponential backoff (1s to 15s, up to 90s) on transient connection errors such as too_many_connections or ECONNREFUSED. The KMS health check at boot retries the same way. Non-transient errors like auth failures still abort immediately."
|
|
7
|
+
},
|
|
8
|
+
{
|
|
9
|
+
"version": "0.365.1",
|
|
10
|
+
"type": "fix",
|
|
11
|
+
"title": "Restoring a soft-deleted row with a money field no longer fails with a 500",
|
|
12
|
+
"detail": "restore() wrote the raw stored row (money as bigint minor units) into payload.previous, and the jsonb bind threw. It now builds previous in the same domain form as delete(), so the restored event carries { amount, currency, ... } for money fields."
|
|
13
|
+
},
|
|
14
|
+
{
|
|
15
|
+
"version": "0.365.1",
|
|
16
|
+
"type": "fix",
|
|
17
|
+
"title": "version_conflict from delete, restore, forget and update reports the actual stream version",
|
|
18
|
+
"detail": "After a lost append race, delete, restore and forget returned currentVersion -1 and update returned the version read before the race, so expectedVersion and currentVersion looked equal. They now look up the stream version after the conflict; -1 remains only when that lookup fails."
|
|
19
|
+
},
|
|
2
20
|
{
|
|
3
21
|
"version": "0.365.0",
|
|
4
22
|
"type": "improvement",
|