@cosmicdrift/kumiko-bundled-features 0.211.0 → 0.213.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +8 -8
- package/src/admin-shell/feature.ts +2 -0
- package/src/admin-shell/web/platform-overview-screen.tsx +1 -0
- package/src/admin-shell/web/tenant-overview-screen.tsx +1 -0
- package/src/audit/__tests__/audit-log-screen.test.tsx +228 -0
- package/src/audit/constants.ts +3 -0
- package/src/audit/feature.ts +2 -0
- package/src/audit/i18n.ts +1 -0
- package/src/audit/web/audit-log-detail-screen.tsx +28 -6
- package/src/audit/web/audit-log-screen.tsx +51 -23
- package/src/auth-email-password/__tests__/password-reset.integration.test.ts +31 -2
- package/src/auth-email-password/__tests__/signup-flow.integration.test.ts +175 -2
- package/src/auth-email-password/feature.ts +9 -4
- package/src/auth-email-password/handlers/invite-create.write.ts +14 -4
- package/src/auth-email-password/handlers/signup-request.write.ts +48 -34
- package/src/auth-email-password/handlers/token-request-handler.ts +20 -4
- package/src/auth-email-password/i18n.ts +2 -0
- package/src/auth-email-password/magic-link-mail.ts +11 -2
- package/src/auth-email-password/web/__tests__/signup-complete-screen.test.tsx +15 -6
- package/src/auth-email-password/web/auth-form-primitives.tsx +2 -0
- package/src/auth-email-password/web/auth-gate.tsx +1 -0
- package/src/auth-email-password/web/confirm-account-unlock-screen.tsx +1 -0
- package/src/auth-email-password/web/invite-accept-screen.tsx +3 -0
- package/src/auth-email-password/web/session.tsx +1 -0
- package/src/auth-email-password/web/signup-complete-screen.tsx +29 -14
- package/src/auth-email-password/web/verify-email-screen.tsx +1 -0
- package/src/auth-mfa/feature.ts +1 -0
- package/src/auth-mfa/web/mfa-disable-dialog.tsx +1 -0
- package/src/auth-mfa/web/mfa-regenerate-recovery-dialog.tsx +1 -0
- package/src/compliance-profiles/feature.ts +1 -5
- package/src/config/db/queries/__tests__/resolver.test.ts +64 -0
- package/src/config/db/queries/resolver.ts +5 -3
- package/src/crypto-shredding/changes.json +2 -1
- package/src/custom-fields/__tests__/feature.test.ts +42 -0
- package/src/custom-fields/__tests__/field-definition-role-config.integration.test.ts +139 -0
- package/src/custom-fields/constants.ts +6 -0
- package/src/custom-fields/feature.ts +56 -10
- package/src/custom-fields/handlers/define-tenant-field.write.ts +7 -1
- package/src/custom-fields/handlers/delete-tenant-field.write.ts +54 -33
- package/src/custom-fields/handlers/update-tenant-field.write.ts +58 -35
- package/src/delivery/feature.ts +1 -3
- package/src/feature-toggles/feature.ts +1 -0
- package/src/feature-toggles/web/toggle-admin-screen.tsx +1 -0
- package/src/folders/web/folder-manager.tsx +1 -0
- package/src/inbound-mail-foundation/__tests__/watch-supervisor.integration.test.ts +141 -0
- package/src/inbound-mail-foundation/watch-supervisor.ts +152 -11
- package/src/jobs/__tests__/jobs-pii-kms.integration.test.ts +174 -0
- package/src/jobs/__tests__/jobs-stale-run-sweep.integration.test.ts +148 -0
- package/src/jobs/db/queries/stale-run-sweep.ts +51 -0
- package/src/jobs/feature.ts +25 -1
- package/src/jobs/handlers/detail.query.ts +13 -1
- package/src/jobs/handlers/stale-run-sweep.job.ts +28 -0
- package/src/jobs/job-run-logger.ts +68 -18
- package/src/jobs/web/job-run-detail-screen.tsx +1 -0
- package/src/page-render/web.ts +1 -0
- package/src/personal-access-tokens/feature.ts +1 -0
- package/src/secrets/__tests__/access-options.integration.test.ts +114 -0
- package/src/secrets/constants.ts +12 -0
- package/src/secrets/feature.ts +35 -9
- package/src/secrets/handlers/delete.write.ts +23 -16
- package/src/secrets/handlers/list.query.ts +37 -30
- package/src/secrets/handlers/set.write.ts +43 -36
- package/src/secrets/index.ts +5 -0
- package/src/seo/feature.ts +1 -0
- package/src/tags/feature.ts +1 -0
- package/src/tags/web/tag-chip.tsx +2 -0
- package/src/tags/web/tag-manager.tsx +2 -1
- package/src/tags/web/tag-picker.tsx +1 -0
- package/src/template-resolver/web/client-plugin.tsx +2 -0
- package/src/tier-engine/feature.ts +2 -0
- package/src/tier-engine/web/tier-admin-screen.tsx +5 -5
- package/src/user-data-rights/feature.ts +2 -0
- package/src/user-data-rights/web/privacy-center-screen.tsx +2 -1
- package/src/user-profile/i18n.ts +1 -0
- package/src/user-profile/web/profile-screen.tsx +1 -1
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import { isSystemTenant, type WriteHandlerDef } from "@cosmicdrift/kumiko-framework/engine";
|
|
2
2
|
import { fieldDefinitionAggregateId } from "../aggregate-id";
|
|
3
|
-
import {
|
|
3
|
+
import {
|
|
4
|
+
DEFAULT_FIELD_DEFINITION_WRITE_ROLES,
|
|
5
|
+
FIELD_DEFINITION_AGGREGATE_TYPE,
|
|
6
|
+
} from "../constants";
|
|
4
7
|
import { fieldDefinitionExecutor } from "../executor";
|
|
5
8
|
import { customFieldsFeature } from "../feature";
|
|
6
9
|
import { type DeleteFieldPayload, deleteFieldPayloadSchema } from "../schemas";
|
|
@@ -15,36 +18,54 @@ import { type DeleteFieldPayload, deleteFieldPayloadSchema } from "../schemas";
|
|
|
15
18
|
// **Idempotency:** Delete auf nicht-existente Definition → version_conflict
|
|
16
19
|
// (executor.delete returns failure, dispatcher 404/422). Caller sieht "not
|
|
17
20
|
// found" — kein Crash.
|
|
18
|
-
export
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
21
|
+
export interface DeleteTenantFieldOptions {
|
|
22
|
+
/** Access roles for this handler. Default ["TenantAdmin"] (#2296) — see
|
|
23
|
+
* DefineTenantFieldOptions.roles, the definition-CRUD triad shares one gate. */
|
|
24
|
+
readonly roles?: readonly string[];
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function createDeleteTenantFieldHandler(
|
|
28
|
+
opts: DeleteTenantFieldOptions = {},
|
|
29
|
+
): WriteHandlerDef {
|
|
30
|
+
return {
|
|
31
|
+
name: "delete-tenant-field",
|
|
32
|
+
schema: deleteFieldPayloadSchema,
|
|
33
|
+
access: { roles: opts.roles ?? DEFAULT_FIELD_DEFINITION_WRITE_ROLES },
|
|
34
|
+
handler: async (event, ctx) => {
|
|
35
|
+
const payload = event.payload as DeleteFieldPayload; // @cast-boundary engine-payload
|
|
36
|
+
const tenantId = event.user.tenantId;
|
|
37
|
+
|
|
38
|
+
if (isSystemTenant(tenantId)) {
|
|
39
|
+
throw new Error(
|
|
40
|
+
"delete-tenant-field: tenantId is SYSTEM_TENANT_ID — use delete-system-field for system-scope deletions",
|
|
41
|
+
);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
const aggregateId = fieldDefinitionAggregateId(
|
|
45
|
+
tenantId,
|
|
46
|
+
payload.entityName,
|
|
47
|
+
payload.fieldKey,
|
|
29
48
|
);
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
49
|
+
|
|
50
|
+
const result = await fieldDefinitionExecutor.delete({ id: aggregateId }, event.user, ctx.db);
|
|
51
|
+
|
|
52
|
+
// Emit cascade-cleanup-Event NACH erfolgreichem Delete. host-entity-MSPs
|
|
53
|
+
// (registriert via wireCustomFieldsFor) konsumieren das + entfernen orphan
|
|
54
|
+
// values aus ihrer customFields jsonb. Im selben TX = atomic cleanup.
|
|
55
|
+
if (result.isSuccess) {
|
|
56
|
+
await ctx.unsafeAppendEvent({
|
|
57
|
+
aggregateId,
|
|
58
|
+
aggregateType: FIELD_DEFINITION_AGGREGATE_TYPE,
|
|
59
|
+
type: customFieldsFeature.exports.fieldDefinitionDeletedEvent.name,
|
|
60
|
+
payload: { entityName: payload.entityName, fieldKey: payload.fieldKey, tenantId },
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
return result;
|
|
65
|
+
},
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// Backwards-compat: existing imports of `deleteTenantFieldHandler` keep
|
|
70
|
+
// working — identical to createDeleteTenantFieldHandler() with default roles.
|
|
71
|
+
export const deleteTenantFieldHandler: WriteHandlerDef = createDeleteTenantFieldHandler();
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { isSystemTenant, type WriteHandlerDef } from "@cosmicdrift/kumiko-framework/engine";
|
|
2
2
|
import { failNotFound, failUnprocessable } from "@cosmicdrift/kumiko-framework/errors";
|
|
3
3
|
import { fieldDefinitionAggregateId } from "../aggregate-id";
|
|
4
|
+
import { DEFAULT_FIELD_DEFINITION_WRITE_ROLES } from "../constants";
|
|
4
5
|
import { fieldDefinitionExecutor } from "../executor";
|
|
5
6
|
import { buildFieldDefinitionColumns } from "../lib/field-definition-row";
|
|
6
7
|
import { type UpdateFieldPayload, updateFieldPayloadSchema } from "../schemas";
|
|
@@ -29,45 +30,67 @@ import { type UpdateFieldPayload, updateFieldPayloadSchema } from "../schemas";
|
|
|
29
30
|
// **skipOptimisticLock:** Definition-Edits sind admin-only + low-frequency
|
|
30
31
|
// (gleiche Abwägung wie der Quota-soft-cap in define). Last-write-wins
|
|
31
32
|
// statt version-Roundtrip durch den Edit-Screen.
|
|
32
|
-
export
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
const payload = event.payload as UpdateFieldPayload; // @cast-boundary engine-payload
|
|
38
|
-
const tenantId = event.user.tenantId;
|
|
33
|
+
export interface UpdateTenantFieldOptions {
|
|
34
|
+
/** Access roles for this handler. Default ["TenantAdmin"] (#2296) — see
|
|
35
|
+
* DefineTenantFieldOptions.roles, the definition-CRUD triad shares one gate. */
|
|
36
|
+
readonly roles?: readonly string[];
|
|
37
|
+
}
|
|
39
38
|
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
39
|
+
export function createUpdateTenantFieldHandler(
|
|
40
|
+
opts: UpdateTenantFieldOptions = {},
|
|
41
|
+
): WriteHandlerDef {
|
|
42
|
+
return {
|
|
43
|
+
name: "update-tenant-field",
|
|
44
|
+
schema: updateFieldPayloadSchema,
|
|
45
|
+
access: { roles: opts.roles ?? DEFAULT_FIELD_DEFINITION_WRITE_ROLES },
|
|
46
|
+
handler: async (event, ctx) => {
|
|
47
|
+
const payload = event.payload as UpdateFieldPayload; // @cast-boundary engine-payload
|
|
48
|
+
const tenantId = event.user.tenantId;
|
|
49
|
+
|
|
50
|
+
if (isSystemTenant(tenantId)) {
|
|
51
|
+
throw new Error(
|
|
52
|
+
"update-tenant-field: tenantId is SYSTEM_TENANT_ID — system-scope definitions have no update handler (delete + re-define via the system-field handlers)",
|
|
53
|
+
);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
const aggregateId = fieldDefinitionAggregateId(
|
|
57
|
+
tenantId,
|
|
58
|
+
payload.entityName,
|
|
59
|
+
payload.fieldKey,
|
|
43
60
|
);
|
|
44
|
-
}
|
|
45
61
|
|
|
46
|
-
|
|
62
|
+
const existing = await fieldDefinitionExecutor.detail(
|
|
63
|
+
{ id: aggregateId },
|
|
64
|
+
event.user,
|
|
65
|
+
ctx.db,
|
|
66
|
+
);
|
|
67
|
+
if (!existing) {
|
|
68
|
+
return failNotFound("field-definition", aggregateId);
|
|
69
|
+
}
|
|
47
70
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
71
|
+
if (existing["type"] !== payload.serializedField.type) {
|
|
72
|
+
return failUnprocessable("field_type_immutable", {
|
|
73
|
+
entityName: payload.entityName,
|
|
74
|
+
fieldKey: payload.fieldKey,
|
|
75
|
+
currentType: existing["type"],
|
|
76
|
+
requestedType: payload.serializedField.type,
|
|
77
|
+
});
|
|
78
|
+
}
|
|
52
79
|
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
entityName:
|
|
56
|
-
fieldKey:
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
});
|
|
60
|
-
}
|
|
80
|
+
// entityName/fieldKey sind die Identität — nicht Teil der changes.
|
|
81
|
+
const {
|
|
82
|
+
entityName: _entityName,
|
|
83
|
+
fieldKey: _fieldKey,
|
|
84
|
+
...changes
|
|
85
|
+
} = buildFieldDefinitionColumns(payload);
|
|
61
86
|
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
87
|
+
return fieldDefinitionExecutor.update({ id: aggregateId, changes }, event.user, ctx.db, {
|
|
88
|
+
skipOptimisticLock: true,
|
|
89
|
+
});
|
|
90
|
+
},
|
|
91
|
+
};
|
|
92
|
+
}
|
|
68
93
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
},
|
|
73
|
-
};
|
|
94
|
+
// Backwards-compat: existing imports of `updateTenantFieldHandler` keep
|
|
95
|
+
// working — identical to createUpdateTenantFieldHandler() with default roles.
|
|
96
|
+
export const updateTenantFieldHandler: WriteHandlerDef = createUpdateTenantFieldHandler();
|
package/src/delivery/feature.ts
CHANGED
|
@@ -163,9 +163,7 @@ export function createDeliveryFeature(options?: DeliveryFeatureOptions): Feature
|
|
|
163
163
|
r.nav({
|
|
164
164
|
id: "delivery-log",
|
|
165
165
|
label: "delivery:nav.deliveryLog",
|
|
166
|
-
|
|
167
|
-
// nav-icon.ts) — "bell" matches the feature's own notifications framing.
|
|
168
|
-
icon: "bell",
|
|
166
|
+
icon: "send",
|
|
169
167
|
screen: "delivery:screen:delivery-log",
|
|
170
168
|
order: 40,
|
|
171
169
|
});
|
|
@@ -108,6 +108,7 @@ export function FolderManager({
|
|
|
108
108
|
const [pendingDelete, setPendingDelete] = useState<FolderNode | null>(null);
|
|
109
109
|
|
|
110
110
|
// Persist expand/collapse so a reload (F5) doesn't re-expand everything.
|
|
111
|
+
// kumiko-lint-ignore no-raw-hooks DOM integration: writes to window.localStorage, no framework query/mutation equivalent
|
|
111
112
|
useEffect(() => {
|
|
112
113
|
if (typeof window === "undefined") return;
|
|
113
114
|
try {
|
|
@@ -10,6 +10,7 @@ import { configurePiiSubjectKms, InMemoryKmsAdapter } from "@cosmicdrift/kumiko-
|
|
|
10
10
|
import type { DbConnection } from "@cosmicdrift/kumiko-framework/db";
|
|
11
11
|
import { createSystemUser, type TenantId } from "@cosmicdrift/kumiko-framework/engine";
|
|
12
12
|
import { createEventsTable } from "@cosmicdrift/kumiko-framework/event-store";
|
|
13
|
+
import { createDistributedLock } from "@cosmicdrift/kumiko-framework/pipeline";
|
|
13
14
|
import {
|
|
14
15
|
createTestUser,
|
|
15
16
|
setupTestStack,
|
|
@@ -429,3 +430,143 @@ describe("watch-supervisor — watch restart + ingest resilience", () => {
|
|
|
429
430
|
}
|
|
430
431
|
});
|
|
431
432
|
});
|
|
433
|
+
|
|
434
|
+
describe("watch-supervisor — multi-worker watch coordination (#1719)", () => {
|
|
435
|
+
// Counts only watch-path ingests (payload.providerCursor === "watch") —
|
|
436
|
+
// the reconciliation poll is deliberately N-fold (every worker polls
|
|
437
|
+
// every account) and would otherwise inflate this count regardless of
|
|
438
|
+
// which worker actually holds the watch lease.
|
|
439
|
+
function makeWatchIngestCounter() {
|
|
440
|
+
let count = 0;
|
|
441
|
+
const dispatchWrite = ({
|
|
442
|
+
handlerQn,
|
|
443
|
+
payload,
|
|
444
|
+
tenantId,
|
|
445
|
+
}: {
|
|
446
|
+
handlerQn: string;
|
|
447
|
+
payload: unknown;
|
|
448
|
+
tenantId: string;
|
|
449
|
+
}) => {
|
|
450
|
+
if (handlerQn === InboundMailFoundationHandlers.ingestMessage) {
|
|
451
|
+
const cursor = (payload as { providerCursor?: string }).providerCursor;
|
|
452
|
+
if (cursor === "watch") count += 1;
|
|
453
|
+
}
|
|
454
|
+
return stack.dispatcher.write(
|
|
455
|
+
handlerQn,
|
|
456
|
+
payload,
|
|
457
|
+
createSystemUser(tenantId as TenantId, [ROLES.SystemAdmin]),
|
|
458
|
+
);
|
|
459
|
+
};
|
|
460
|
+
return { dispatchWrite, watchIngestCount: () => count };
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
test("only one supervisor holds the watch lease; the peer takes over after stop()", async () => {
|
|
464
|
+
const admin = adminFor(4200);
|
|
465
|
+
const accountId = await connectSharedAccount(admin);
|
|
466
|
+
const lock = createDistributedLock(stack.redis.redis, "test-watch-lease:");
|
|
467
|
+
const a = makeWatchIngestCounter();
|
|
468
|
+
const b = makeWatchIngestCounter();
|
|
469
|
+
const supervisorA = createInboundMailSupervisor({
|
|
470
|
+
providerCtx: { registry: stack.registry },
|
|
471
|
+
db,
|
|
472
|
+
dispatchWrite: a.dispatchWrite,
|
|
473
|
+
lock,
|
|
474
|
+
watchLeaseTtlSeconds: 5,
|
|
475
|
+
pollIntervalMs: 60_000,
|
|
476
|
+
});
|
|
477
|
+
const supervisorB = createInboundMailSupervisor({
|
|
478
|
+
providerCtx: { registry: stack.registry },
|
|
479
|
+
db,
|
|
480
|
+
dispatchWrite: b.dispatchWrite,
|
|
481
|
+
lock,
|
|
482
|
+
watchLeaseTtlSeconds: 5,
|
|
483
|
+
pollIntervalMs: 60_000,
|
|
484
|
+
});
|
|
485
|
+
|
|
486
|
+
try {
|
|
487
|
+
await supervisorA.start();
|
|
488
|
+
await waitFor(() => {
|
|
489
|
+
expect(isWatching(accountId)).toBe(true);
|
|
490
|
+
});
|
|
491
|
+
|
|
492
|
+
// B's own start() polls too (poll stays N-fold by design) but must
|
|
493
|
+
// not claim the watch lease — A already holds it.
|
|
494
|
+
await supervisorB.start();
|
|
495
|
+
|
|
496
|
+
await seedInboundMessage(accountId, rawMsg({ providerMessageId: "mw-1" }));
|
|
497
|
+
await waitFor(() => {
|
|
498
|
+
expect(a.watchIngestCount()).toBe(1);
|
|
499
|
+
});
|
|
500
|
+
expect(b.watchIngestCount()).toBe(0);
|
|
501
|
+
|
|
502
|
+
await supervisorA.stop();
|
|
503
|
+
expect(isWatching(accountId)).toBe(false);
|
|
504
|
+
|
|
505
|
+
await supervisorB.pollOnce();
|
|
506
|
+
await waitFor(() => {
|
|
507
|
+
expect(isWatching(accountId)).toBe(true);
|
|
508
|
+
});
|
|
509
|
+
|
|
510
|
+
await seedInboundMessage(accountId, rawMsg({ providerMessageId: "mw-2" }));
|
|
511
|
+
await waitFor(() => {
|
|
512
|
+
expect(b.watchIngestCount()).toBe(1);
|
|
513
|
+
});
|
|
514
|
+
expect(a.watchIngestCount()).toBe(1);
|
|
515
|
+
} finally {
|
|
516
|
+
await supervisorA.stop();
|
|
517
|
+
await supervisorB.stop();
|
|
518
|
+
}
|
|
519
|
+
});
|
|
520
|
+
|
|
521
|
+
test("the holder renews its lease past the TTL — the peer still can't claim it", async () => {
|
|
522
|
+
const admin = adminFor(4201);
|
|
523
|
+
const accountId = await connectSharedAccount(admin);
|
|
524
|
+
const lock = createDistributedLock(stack.redis.redis, "test-watch-lease-renew:");
|
|
525
|
+
const a = makeWatchIngestCounter();
|
|
526
|
+
const b = makeWatchIngestCounter();
|
|
527
|
+
const supervisorA = createInboundMailSupervisor({
|
|
528
|
+
providerCtx: { registry: stack.registry },
|
|
529
|
+
db,
|
|
530
|
+
dispatchWrite: a.dispatchWrite,
|
|
531
|
+
lock,
|
|
532
|
+
watchLeaseTtlSeconds: 1,
|
|
533
|
+
pollIntervalMs: 60_000,
|
|
534
|
+
});
|
|
535
|
+
const supervisorB = createInboundMailSupervisor({
|
|
536
|
+
providerCtx: { registry: stack.registry },
|
|
537
|
+
db,
|
|
538
|
+
dispatchWrite: b.dispatchWrite,
|
|
539
|
+
lock,
|
|
540
|
+
watchLeaseTtlSeconds: 1,
|
|
541
|
+
pollIntervalMs: 60_000,
|
|
542
|
+
});
|
|
543
|
+
|
|
544
|
+
try {
|
|
545
|
+
await supervisorA.start();
|
|
546
|
+
await waitFor(() => {
|
|
547
|
+
expect(isWatching(accountId)).toBe(true);
|
|
548
|
+
});
|
|
549
|
+
|
|
550
|
+
// 2.5x the 1s TTL: long enough that a dead heartbeat guarantees the
|
|
551
|
+
// key expired (1.5x left this indistinguishable from Redis's
|
|
552
|
+
// second-granularity EX not having rolled over yet).
|
|
553
|
+
await new Promise((r) => setTimeout(r, 2500));
|
|
554
|
+
|
|
555
|
+
// Direct probe of the invariant: the lease itself must still be held,
|
|
556
|
+
// independent of B's behavior below (which only infers it indirectly).
|
|
557
|
+
expect(await lock.acquire(accountId, { ttlSeconds: 1 })).toBeNull();
|
|
558
|
+
|
|
559
|
+
await supervisorB.start();
|
|
560
|
+
expect(isWatching(accountId)).toBe(true);
|
|
561
|
+
|
|
562
|
+
await seedInboundMessage(accountId, rawMsg({ providerMessageId: "mw-renew-1" }));
|
|
563
|
+
await waitFor(() => {
|
|
564
|
+
expect(a.watchIngestCount()).toBe(1);
|
|
565
|
+
});
|
|
566
|
+
expect(b.watchIngestCount()).toBe(0);
|
|
567
|
+
} finally {
|
|
568
|
+
await supervisorA.stop();
|
|
569
|
+
await supervisorB.stop();
|
|
570
|
+
}
|
|
571
|
+
});
|
|
572
|
+
});
|
|
@@ -5,27 +5,44 @@
|
|
|
5
5
|
// Dedup im ingest-Handler macht Watch/Poll-Überschneidung idempotent —
|
|
6
6
|
// der Poll ist Korrektheits-Anker, watch nur Latenz-Optimierung.
|
|
7
7
|
//
|
|
8
|
-
// **Plan
|
|
9
|
-
//
|
|
10
|
-
// run-export-jobs.ts
|
|
11
|
-
//
|
|
12
|
-
//
|
|
13
|
-
//
|
|
14
|
-
//
|
|
15
|
-
// bin/server.ts:
|
|
8
|
+
// **Plan deviation (documented):** the plan had the poll as an `r.job`
|
|
9
|
+
// cron trigger. At the time this was written, JobContext had no
|
|
10
|
+
// dispatcher (verified against run-export-jobs.ts). It now does
|
|
11
|
+
// (`ctx.write`/`ctx.queryAs`, job-runner.ts) — but converting the poll to
|
|
12
|
+
// a cron job is a separate migration (job concurrency instead of this
|
|
13
|
+
// dispatcher contract), not part of #1719, and stays untouched here. The
|
|
14
|
+
// app owner still wires the supervisor in bin/server.ts:
|
|
16
15
|
//
|
|
17
16
|
// const supervisor = createInboundMailSupervisor({
|
|
18
17
|
// providerCtx: { registry: deps.registry, secrets },
|
|
19
18
|
// db,
|
|
20
19
|
// dispatchWrite: ({ handlerQn, payload, tenantId }) =>
|
|
21
20
|
// deps.dispatchSystemWrite({ handlerQn, payload, tenantId: tenantId as TenantId }),
|
|
21
|
+
// // Multi-worker deployments: share one DistributedLock (same Redis,
|
|
22
|
+
// // same key prefix) across every worker process so only one of them
|
|
23
|
+
// // holds the IMAP IDLE connection per account (#1719). Omit `lock`
|
|
24
|
+
// // for a single-process deployment — every active account gets
|
|
25
|
+
// // watched locally, same as before.
|
|
26
|
+
// lock: createDistributedLock(redis, `${RedisKeys.lock}inbound-mail:watch:`),
|
|
22
27
|
// });
|
|
23
28
|
// await supervisor.start();
|
|
24
29
|
// // shutdown-hook: await supervisor.stop();
|
|
25
30
|
//
|
|
26
|
-
// **
|
|
27
|
-
//
|
|
28
|
-
//
|
|
31
|
+
// **IDLE operational risk (plan §7.3):** long-lived sockets in-process.
|
|
32
|
+
// Mitigated here via backoff-restart on onError, a clean stop() of every
|
|
33
|
+
// watcher on shutdown, and the poll covering every gap.
|
|
34
|
+
//
|
|
35
|
+
// **Multi-worker coordination (#1719):** the reconciliation poll stays
|
|
36
|
+
// deliberately N-fold — every worker polls every active account, which
|
|
37
|
+
// is idempotent and cheap. Only `plugin.watch()` holds a long-lived
|
|
38
|
+
// connection, and only one worker may hold it per account. With
|
|
39
|
+
// `deps.lock`, `ensureWatcher` claims a TTL lease (`lock.acquire`) for
|
|
40
|
+
// the account before connecting; a worker that doesn't get it leaves the
|
|
41
|
+
// account to its current holder — the poll still covers it. The holder
|
|
42
|
+
// renews the claim via heartbeat (`lock.renew`, every ttl/3) for as long
|
|
43
|
+
// as the watcher state lives, including across reconnect backoff, not
|
|
44
|
+
// only while the connection is open. Losing the claim (Redis outage, TTL
|
|
45
|
+
// exceeded) tears down the local watcher instead of racing a new holder.
|
|
29
46
|
|
|
30
47
|
import { fetchOne, insertOne, selectMany, updateMany } from "@cosmicdrift/kumiko-framework/bun-db";
|
|
31
48
|
import {
|
|
@@ -33,6 +50,7 @@ import {
|
|
|
33
50
|
decryptPiiFieldValues,
|
|
34
51
|
} from "@cosmicdrift/kumiko-framework/crypto";
|
|
35
52
|
import type { DbConnection, EntityTableMeta } from "@cosmicdrift/kumiko-framework/db";
|
|
53
|
+
import type { DistributedLock } from "@cosmicdrift/kumiko-framework/pipeline";
|
|
36
54
|
import { Temporal } from "temporal-polyfill";
|
|
37
55
|
import { InboundMailAccountStatuses, InboundMailFoundationHandlers } from "./constants";
|
|
38
56
|
import { MAIL_ACCOUNT_PII_FIELDS, syncCursorTable } from "./entities";
|
|
@@ -54,6 +72,7 @@ const DEFAULT_BACKFILL_WINDOW_DAYS = 30;
|
|
|
54
72
|
const DEFAULT_MAX_MESSAGES_PER_POLL = 200;
|
|
55
73
|
const DEFAULT_WATCH_BACKOFF_INITIAL_MS = 5_000;
|
|
56
74
|
const DEFAULT_WATCH_BACKOFF_MAX_MS = 5 * 60 * 1000;
|
|
75
|
+
const DEFAULT_WATCH_LEASE_TTL_SECONDS = 90;
|
|
57
76
|
/** V1: ein Cursor pro Account (eine Mailbox-Inbox). Multi-Folder später
|
|
58
77
|
* über weitere scopes ohne Schema-Änderung. */
|
|
59
78
|
const CURSOR_SCOPE = "default";
|
|
@@ -85,6 +104,15 @@ export type InboundMailSupervisorDeps = {
|
|
|
85
104
|
readonly maxMessagesPerPoll?: number;
|
|
86
105
|
readonly watchBackoffInitialMs?: number;
|
|
87
106
|
readonly watchBackoffMaxMs?: number;
|
|
107
|
+
/** Coordinates the IMAP watch (not the poll) across worker processes:
|
|
108
|
+
* before `plugin.watch()`, ensureWatcher claims a TTL lease for the
|
|
109
|
+
* account via `lock.acquire` and renews it via `lock.renew` while the
|
|
110
|
+
* watcher lives. Omit for single-process deployments — every active
|
|
111
|
+
* account is watched locally, same as before #1719. */
|
|
112
|
+
readonly lock?: DistributedLock;
|
|
113
|
+
/** TTL (seconds) for the per-account watch lease. Default 90s, renewed
|
|
114
|
+
* every ttl/3. Only used when `lock` is set. */
|
|
115
|
+
readonly watchLeaseTtlSeconds?: number;
|
|
88
116
|
readonly log?: (line: string) => void;
|
|
89
117
|
};
|
|
90
118
|
|
|
@@ -95,8 +123,14 @@ type WatcherState = {
|
|
|
95
123
|
/** Bump beim stop() — verhindert dass ein nachzügelnder Restart einen
|
|
96
124
|
* bereits gestoppten Watcher wiederbelebt. */
|
|
97
125
|
generation: number;
|
|
126
|
+
/** Token from `deps.lock.acquire` while this worker owns the account's
|
|
127
|
+
* watch lease; null when unclaimed (no `deps.lock`, or claim lost). */
|
|
128
|
+
lockToken: string | null;
|
|
129
|
+
renewTimer: ReturnType<typeof setTimeout> | null;
|
|
98
130
|
};
|
|
99
131
|
|
|
132
|
+
type WatchLeaseResult = "acquired" | "no-token" | "stale";
|
|
133
|
+
|
|
100
134
|
export type InboundMailSupervisor = {
|
|
101
135
|
readonly start: () => Promise<void>;
|
|
102
136
|
/** Ein Reconciliation-Durchlauf über alle aktiven Accounts — auch
|
|
@@ -113,6 +147,11 @@ export function createInboundMailSupervisor(
|
|
|
113
147
|
const maxMessagesPerPoll = deps.maxMessagesPerPoll ?? DEFAULT_MAX_MESSAGES_PER_POLL;
|
|
114
148
|
const backoffInitialMs = deps.watchBackoffInitialMs ?? DEFAULT_WATCH_BACKOFF_INITIAL_MS;
|
|
115
149
|
const backoffMaxMs = deps.watchBackoffMaxMs ?? DEFAULT_WATCH_BACKOFF_MAX_MS;
|
|
150
|
+
const leaseTtlSeconds = deps.watchLeaseTtlSeconds ?? DEFAULT_WATCH_LEASE_TTL_SECONDS;
|
|
151
|
+
// ttl/3 keeps at least two renewal attempts inside the TTL window before
|
|
152
|
+
// it lapses. The 50ms floor only guards against a pathologically small
|
|
153
|
+
// configured TTL — real deployments (default 90s) never hit it.
|
|
154
|
+
const renewIntervalMs = Math.max(50, Math.floor((leaseTtlSeconds * 1000) / 3));
|
|
116
155
|
const log = deps.log ?? (() => {});
|
|
117
156
|
|
|
118
157
|
let running = false;
|
|
@@ -343,6 +382,72 @@ export function createInboundMailSupervisor(
|
|
|
343
382
|
// ---------------------------------------------------------------
|
|
344
383
|
// Watch-Lifecycle mit Backoff-Restart.
|
|
345
384
|
// ---------------------------------------------------------------
|
|
385
|
+
// Heartbeat for a held watch lease. Independent of the connection's own
|
|
386
|
+
// lifecycle — it keeps renewing across reconnect backoff, not just while
|
|
387
|
+
// `plugin.watch()` is actually connected, so a flaky IMAP link doesn't
|
|
388
|
+
// make this worker lose the account to a peer mid-backoff.
|
|
389
|
+
function scheduleRenew(
|
|
390
|
+
account: MailAccountRecord,
|
|
391
|
+
state: WatcherState,
|
|
392
|
+
generation: number,
|
|
393
|
+
): void {
|
|
394
|
+
state.renewTimer = setTimeout(() => {
|
|
395
|
+
void (async () => {
|
|
396
|
+
state.renewTimer = null;
|
|
397
|
+
// skip: supervisor stopped, generation superseded, or lease already lost — stale timer fire, nothing to renew.
|
|
398
|
+
if (!running || state.generation !== generation || !state.lockToken || !deps.lock) return;
|
|
399
|
+
let renewed: boolean;
|
|
400
|
+
try {
|
|
401
|
+
renewed = await deps.lock.renew(account.id, state.lockToken, leaseTtlSeconds);
|
|
402
|
+
} catch (err) {
|
|
403
|
+
log(
|
|
404
|
+
`inbound-mail: watch lease renew for account ${account.id} failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
405
|
+
);
|
|
406
|
+
// Treat a renew error as transient (Redis blip) — keep the local
|
|
407
|
+
// watcher running and retry next tick within the TTL grace window.
|
|
408
|
+
if (running && state.generation === generation && state.lockToken) {
|
|
409
|
+
scheduleRenew(account, state, generation);
|
|
410
|
+
}
|
|
411
|
+
// skip: renew already rescheduled above (or conditions no longer hold) — nothing left to do this tick.
|
|
412
|
+
return;
|
|
413
|
+
}
|
|
414
|
+
if (!renewed) {
|
|
415
|
+
log(
|
|
416
|
+
`inbound-mail: watch lease for account ${account.id} lost — tearing down local watcher`,
|
|
417
|
+
);
|
|
418
|
+
state.lockToken = null;
|
|
419
|
+
await stopWatcher(account.id);
|
|
420
|
+
// skip: watcher already torn down by stopWatcher() above — nothing left to do.
|
|
421
|
+
return;
|
|
422
|
+
}
|
|
423
|
+
if (running && state.generation === generation && state.lockToken) {
|
|
424
|
+
scheduleRenew(account, state, generation);
|
|
425
|
+
}
|
|
426
|
+
})();
|
|
427
|
+
}, renewIntervalMs);
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
async function acquireWatchLease(
|
|
431
|
+
account: MailAccountRecord,
|
|
432
|
+
state: WatcherState,
|
|
433
|
+
): Promise<WatchLeaseResult> {
|
|
434
|
+
if (!deps.lock || state.lockToken) return "acquired";
|
|
435
|
+
const token = await deps.lock.acquire(account.id, { ttlSeconds: leaseTtlSeconds });
|
|
436
|
+
if (!token) return "no-token";
|
|
437
|
+
// The acquire() await is a yield point: a concurrent stopWatcher()
|
|
438
|
+
// (e.g. from a lost-lease renewal on a different generation) may have
|
|
439
|
+
// already retired this exact state and removed it from the map. If so,
|
|
440
|
+
// committing the fresh token onto it would leak the lease — nothing
|
|
441
|
+
// would ever release it, blocking failover for the full TTL.
|
|
442
|
+
if (watchers.get(account.id) !== state) {
|
|
443
|
+
await deps.lock.release(account.id, token);
|
|
444
|
+
return "stale";
|
|
445
|
+
}
|
|
446
|
+
state.lockToken = token;
|
|
447
|
+
scheduleRenew(account, state, state.generation);
|
|
448
|
+
return "acquired";
|
|
449
|
+
}
|
|
450
|
+
|
|
346
451
|
async function ensureWatcher(
|
|
347
452
|
account: MailAccountRecord,
|
|
348
453
|
plugin: InboundMailProviderPlugin,
|
|
@@ -358,8 +463,26 @@ export function createInboundMailSupervisor(
|
|
|
358
463
|
backoffMs: backoffInitialMs,
|
|
359
464
|
restartTimer: null,
|
|
360
465
|
generation: 0,
|
|
466
|
+
lockToken: null,
|
|
467
|
+
renewTimer: null,
|
|
361
468
|
};
|
|
362
469
|
watchers.set(account.id, state);
|
|
470
|
+
|
|
471
|
+
const leaseResult = await acquireWatchLease(account, state);
|
|
472
|
+
if (leaseResult === "no-token") {
|
|
473
|
+
// Another worker already holds this account's watch lease — the poll
|
|
474
|
+
// still covers it, this worker just doesn't open a second IDLE
|
|
475
|
+
// connection. Retried on the next tick.
|
|
476
|
+
if (!existing) watchers.delete(account.id);
|
|
477
|
+
// skip: no lease token acquired above — nothing more to set up here.
|
|
478
|
+
return;
|
|
479
|
+
}
|
|
480
|
+
if (leaseResult === "stale") {
|
|
481
|
+
// skip: state was retired by a concurrent stopWatcher while acquire()
|
|
482
|
+
// awaited — committing here would leak the lease.
|
|
483
|
+
return;
|
|
484
|
+
}
|
|
485
|
+
|
|
363
486
|
const generation = state.generation;
|
|
364
487
|
|
|
365
488
|
const scheduleRestart = (err: unknown) => {
|
|
@@ -434,8 +557,17 @@ export function createInboundMailSupervisor(
|
|
|
434
557
|
clearTimeout(state.restartTimer);
|
|
435
558
|
state.restartTimer = null;
|
|
436
559
|
}
|
|
560
|
+
if (state.renewTimer) {
|
|
561
|
+
clearTimeout(state.renewTimer);
|
|
562
|
+
state.renewTimer = null;
|
|
563
|
+
}
|
|
437
564
|
const stop = state.stop;
|
|
438
565
|
state.stop = null;
|
|
566
|
+
// Read the token before clearing it: only OUR claim gets released — a
|
|
567
|
+
// caller that already cleared lockToken (lease-lost path in
|
|
568
|
+
// scheduleRenew) means someone else may own it by now.
|
|
569
|
+
const lockToken = state.lockToken;
|
|
570
|
+
state.lockToken = null;
|
|
439
571
|
watchers.delete(accountId);
|
|
440
572
|
if (stop) {
|
|
441
573
|
try {
|
|
@@ -446,6 +578,15 @@ export function createInboundMailSupervisor(
|
|
|
446
578
|
);
|
|
447
579
|
}
|
|
448
580
|
}
|
|
581
|
+
if (deps.lock && lockToken) {
|
|
582
|
+
try {
|
|
583
|
+
await deps.lock.release(accountId, lockToken);
|
|
584
|
+
} catch (err) {
|
|
585
|
+
log(
|
|
586
|
+
`inbound-mail: watch lease release for account ${accountId} failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
587
|
+
);
|
|
588
|
+
}
|
|
589
|
+
}
|
|
449
590
|
}
|
|
450
591
|
|
|
451
592
|
// ---------------------------------------------------------------
|