@cosmicdrift/kumiko-samples 0.353.0 → 0.354.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/packages/bundled-features/package.json +18 -2
- package/packages/bundled-features/src/agent-tools/agent-doc-lint.ts +66 -0
- package/packages/bundled-features/src/agent-tools/agent-manifest.ts +11 -5
- package/packages/bundled-features/src/agent-tools/feature.ts +13 -2
- package/packages/bundled-features/src/agent-tools/index.ts +6 -1
- package/packages/bundled-features/src/agent-tools/tool-dispatch.ts +1 -1
- package/packages/bundled-features/src/auth-email-password/changes.json +24 -0
- package/packages/bundled-features/src/auth-email-password/handlers/change-password.write.ts +1 -0
- package/packages/bundled-features/src/auth-email-password/handlers/invite-accept-with-login.write.ts +3 -2
- package/packages/bundled-features/src/auth-email-password/handlers/invite-accept.write.ts +3 -2
- package/packages/bundled-features/src/auth-email-password/handlers/invite-create.write.ts +21 -0
- package/packages/bundled-features/src/auth-email-password/handlers/invite-signup-complete.write.ts +4 -1
- package/packages/bundled-features/src/auth-email-password/handlers/login.write.ts +1 -0
- package/packages/bundled-features/src/auth-email-password/handlers/reset-password.write.ts +1 -0
- package/packages/bundled-features/src/auth-email-password/handlers/signup-confirm.write.ts +5 -0
- package/packages/bundled-features/src/auth-email-password/handlers/signup-request.write.ts +53 -96
- package/packages/bundled-features/src/auth-email-password/handlers/token-request-handler.ts +1 -0
- package/packages/bundled-features/src/auth-email-password/handlers/verify-email.write.ts +1 -0
- package/packages/bundled-features/src/auth-email-password/index.ts +12 -0
- package/packages/bundled-features/src/auth-email-password/seeding.ts +38 -7
- package/packages/bundled-features/src/auth-email-password/signup-activation.ts +83 -0
- package/packages/bundled-features/src/auth-email-password/signup-token-store.ts +9 -3
- package/packages/bundled-features/src/auth-email-password/web/auth-gate.tsx +7 -7
- package/packages/bundled-features/src/auth-email-password/web/auth-redirect.ts +15 -0
- package/packages/bundled-features/src/auth-email-password/web/login-screen.tsx +2 -2
- package/packages/bundled-features/src/auth-email-password/web/user-menu.tsx +1 -1
- package/packages/bundled-features/src/auth-mfa/changes.json +7 -0
- package/packages/bundled-features/src/auth-mfa/feature.ts +5 -0
- package/packages/bundled-features/src/auth-mfa/handlers/enable-confirm-preauth.write.ts +1 -0
- package/packages/bundled-features/src/auth-mfa/handlers/verify.write.ts +1 -0
- package/packages/bundled-features/src/auth-mfa/web/mfa-recovery-codes-reveal.tsx +6 -8
- package/packages/bundled-features/src/auth-mfa/web/mfa-setup-preauth-screen.tsx +18 -21
- package/packages/bundled-features/src/auth-mfa/web/mfa-verify-screen.tsx +2 -2
- package/packages/bundled-features/src/billing-foundation/billing-info-query.ts +4 -4
- package/packages/bundled-features/src/billing-foundation/changes.json +6 -0
- package/packages/bundled-features/src/channel-in-app/changes.json +8 -1
- package/packages/bundled-features/src/channel-in-app/in-app-channel.ts +1 -1
- package/packages/bundled-features/src/channel-in-app/tables.ts +18 -12
- package/packages/bundled-features/src/crypto-shredding/changes.json +12 -0
- package/packages/bundled-features/src/crypto-shredding/handlers/forget-subject.write.ts +154 -69
- package/packages/bundled-features/src/delivery/changes.json +6 -0
- package/packages/bundled-features/src/delivery/db/queries/address-opt-outs.ts +18 -12
- package/packages/bundled-features/src/delivery/delivery-service.ts +103 -16
- package/packages/bundled-features/src/delivery/unsubscribe.ts +81 -13
- package/packages/bundled-features/src/feature-toggles/feature.ts +2 -3
- package/packages/bundled-features/src/feature-toggles/toggle-runtime.ts +34 -5
- package/packages/bundled-features/src/feature-toggles/toggle-sync-signal.ts +31 -8
- package/packages/bundled-features/src/file-derivatives/handlers/public-variant.query.ts +10 -1
- package/packages/bundled-features/src/files/changes.json +24 -0
- package/packages/bundled-features/src/files-tenant-data/changes.json +5 -0
- package/packages/bundled-features/src/files-tenant-data/handlers/sweep-orphaned-derivatives.job.ts +4 -0
- package/packages/bundled-features/src/files-tenant-data/hooks.ts +72 -23
- package/packages/bundled-features/src/folders/changes.json +6 -0
- package/packages/bundled-features/src/folders/handlers/delete-folder.write.ts +41 -9
- package/packages/bundled-features/src/foundation-shared/index.ts +1 -0
- package/packages/bundled-features/src/foundation-shared/mail-host-policy.ts +16 -4
- package/packages/bundled-features/src/inbound-provider-imap/changes.json +6 -0
- package/packages/bundled-features/src/inbound-provider-imap/feature.ts +2 -0
- package/packages/bundled-features/src/inbound-provider-imap/imap-client.ts +3 -4
- package/packages/bundled-features/src/jobs/changes.json +30 -0
- package/packages/bundled-features/src/jobs/constants.ts +1 -0
- package/packages/bundled-features/src/jobs/db/queries/retention.ts +12 -0
- package/packages/bundled-features/src/jobs/db/queries/stale-run-sweep.ts +26 -2
- package/packages/bundled-features/src/jobs/feature.ts +7 -1
- package/packages/bundled-features/src/jobs/handlers/list.query.ts +1 -1
- package/packages/bundled-features/src/jobs/handlers/tenant-failures.query.ts +1 -1
- package/packages/bundled-features/src/jobs/handlers/tenant-runs.query.ts +94 -0
- package/packages/bundled-features/src/jobs/index.ts +2 -0
- package/packages/bundled-features/src/jobs/job-run-logger.ts +177 -10
- package/packages/bundled-features/src/jobs/tenant-job-failure-table.ts +27 -8
- package/packages/bundled-features/src/jobs/tenant-job-run-table.ts +55 -0
- package/packages/bundled-features/src/ledger/changes.json +23 -0
- package/packages/bundled-features/src/ledger/entity.ts +13 -5
- package/packages/bundled-features/src/ledger/feature.ts +3 -0
- package/packages/bundled-features/src/ledger/handlers/confirm-schedule-period.write.ts +15 -3
- package/packages/bundled-features/src/ledger/handlers/create-transaction.write.ts +1 -0
- package/packages/bundled-features/src/ledger/handlers/reverse-transaction.write.ts +1 -0
- package/packages/bundled-features/src/ledger/schedule-subject-pair.ts +27 -0
- package/packages/bundled-features/src/mail-transport-smtp/feature.ts +4 -15
- package/packages/bundled-features/src/notes-history/changes.json +13 -0
- package/packages/bundled-features/src/notes-history/entity.ts +4 -0
- package/packages/bundled-features/src/personal-access-tokens/handlers/create.write.ts +1 -0
- package/packages/bundled-features/src/sessions/feature.ts +21 -8
- package/packages/bundled-features/src/sessions/handlers/detail.query.ts +34 -33
- package/packages/bundled-features/src/sessions/handlers/list.query.ts +68 -65
- package/packages/bundled-features/src/sessions/screens.ts +10 -8
- package/packages/bundled-features/src/shared/encrypt-for-direct-write.ts +15 -8
- package/packages/bundled-features/src/shared/index.ts +2 -1
- package/packages/bundled-features/src/shared/parent-visibility.ts +31 -1
- package/packages/bundled-features/src/shared/row-bound-grant.ts +18 -2
- package/packages/bundled-features/src/shared/session-locale-field.ts +7 -0
- package/packages/bundled-features/src/template-resolver/api.ts +3 -3
- package/packages/bundled-features/src/template-resolver/changes.json +5 -0
- package/packages/bundled-features/src/template-resolver/handlers/list.query.ts +2 -2
- package/packages/bundled-features/src/tenant/changes.json +12 -0
- package/packages/bundled-features/src/tenant/feature.ts +8 -1
- package/packages/bundled-features/src/tenant/handlers/toggle-enabled.write.ts +2 -0
- package/packages/bundled-features/src/tenant/handlers/update-member-roles.write.ts +1 -0
- package/packages/bundled-features/src/tenant/invitation-table.ts +6 -0
- package/packages/bundled-features/src/tenant/invited-membership.ts +45 -12
- package/packages/bundled-features/src/tenant-handover/changes.json +6 -0
- package/packages/bundled-features/src/tenant-handover/transfer-graph.ts +14 -96
- package/packages/bundled-features/src/tenant-lifecycle/boot-checks.ts +16 -0
- package/packages/bundled-features/src/tenant-lifecycle/changes.json +22 -0
- package/packages/bundled-features/src/tenant-lifecycle/constants.ts +8 -0
- package/packages/bundled-features/src/tenant-lifecycle/events.ts +6 -0
- package/packages/bundled-features/src/tenant-lifecycle/feature.ts +13 -1
- package/packages/bundled-features/src/tenant-lifecycle/run-tenant-destroy.ts +100 -29
- package/packages/bundled-features/src/tenant-lifecycle/stages.ts +41 -9
- package/packages/bundled-features/src/tenant-lifecycle/testing.ts +58 -0
- package/packages/bundled-features/src/tier-engine/changes.json +5 -0
- package/packages/bundled-features/src/tier-engine/handlers/set-tenant-tier.write.ts +1 -0
- package/packages/bundled-features/src/user/changes.json +15 -0
- package/packages/bundled-features/src/user/handlers/detail.query.ts +1 -0
- package/packages/bundled-features/src/user/handlers/list.query.ts +3 -0
- package/packages/bundled-features/src/user/handlers/update.write.ts +1 -0
- package/packages/bundled-features/src/user/seeding.ts +34 -0
- package/packages/bundled-features/src/user-data-rights/changes.json +6 -0
- package/packages/bundled-features/src/user-data-rights/handlers/request-deletion.write.ts +1 -0
- package/packages/bundled-features/src/user-data-rights/index.ts +1 -0
- package/packages/bundled-features/src/user-data-rights/schema/download-attempt.ts +17 -9
- package/packages/bundled-features/src/user-profile/handlers/change-email.write.ts +8 -2
- package/packages/bundled-features/src/workflow-runner/event-trigger.ts +72 -14
- package/packages/bundled-features/src/workflow-runner/handlers/resume-run.write.ts +6 -1
- package/packages/bundled-features/src/workflow-runner/runner.ts +8 -2
- package/samples/apps/showcase/src/features/items/feature.ts +2 -1
- package/samples/apps/styleguide/src/features/widgets/feature.ts +83 -0
- package/samples/apps/styleguide/src/features/widgets/i18n.ts +12 -0
- package/samples/apps/styleguide/src/features/widgets/web/Widgets.tsx +4 -0
- package/samples/apps/ui-walkthrough/package.json +3 -0
- package/samples/apps/use-all-bundled/README.md +1 -2
- package/samples/apps/use-all-bundled/feature-manifest.json +1 -1
- package/samples/apps/user-data-rights-demo/README.md +1 -3
- package/samples/recipes/legal-pages/README.md +3 -4
- package/samples/recipes/managed-pages/README.md +2 -2
- package/samples/recipes/seed-migration/README.md +0 -2
|
@@ -3,6 +3,7 @@ import {
|
|
|
3
3
|
createEventStoreExecutor,
|
|
4
4
|
createTenantDb,
|
|
5
5
|
type DbRunner,
|
|
6
|
+
type TenantDb,
|
|
6
7
|
} from "@cosmicdrift/kumiko-framework/db";
|
|
7
8
|
import { createSystemUser, type TenantId } from "@cosmicdrift/kumiko-framework/engine";
|
|
8
9
|
import { ConflictError, writeFailure } from "@cosmicdrift/kumiko-framework/errors";
|
|
@@ -23,7 +24,13 @@ export type InvitedMembershipOptions = {
|
|
|
23
24
|
readonly userId: string;
|
|
24
25
|
readonly tenantId: TenantId;
|
|
25
26
|
readonly role: string;
|
|
26
|
-
readonly
|
|
27
|
+
readonly invitation: InvitationIssuance;
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
export type InvitationIssuance = {
|
|
31
|
+
readonly membershipVersion: number | null;
|
|
32
|
+
readonly insertedAt: Temporal.Instant;
|
|
33
|
+
readonly modifiedAt: Temporal.Instant | null;
|
|
27
34
|
};
|
|
28
35
|
|
|
29
36
|
type MembershipRow = {
|
|
@@ -34,6 +41,21 @@ type MembershipRow = {
|
|
|
34
41
|
readonly modifiedAt: Temporal.Instant | null;
|
|
35
42
|
};
|
|
36
43
|
|
|
44
|
+
const MEMBERSHIP_CREATED_VERSION = 1;
|
|
45
|
+
|
|
46
|
+
// 0 = the user has no membership in the tenant yet.
|
|
47
|
+
export async function membershipVersionOf(
|
|
48
|
+
db: Pick<TenantDb, "fetchOne">,
|
|
49
|
+
tenantId: string,
|
|
50
|
+
userId: string,
|
|
51
|
+
): Promise<number> {
|
|
52
|
+
const membership = await db.fetchOne<{ readonly version: number }>(tenantMembershipsTable, {
|
|
53
|
+
tenantId,
|
|
54
|
+
userId,
|
|
55
|
+
});
|
|
56
|
+
return membership?.version ?? 0;
|
|
57
|
+
}
|
|
58
|
+
|
|
37
59
|
// Add-only, so unlike updateMemberRoles no session revoke or last-TenantAdmin check.
|
|
38
60
|
// Read unfiltered: tenant:query:memberships hides disabled tenants.
|
|
39
61
|
export async function grantInvitedMembershipRole(db: DbRunner, options: InvitedMembershipOptions) {
|
|
@@ -54,7 +76,7 @@ export async function grantInvitedMembershipRole(db: DbRunner, options: InvitedM
|
|
|
54
76
|
}
|
|
55
77
|
|
|
56
78
|
// A membership decision made after the invitation was issued wins over it.
|
|
57
|
-
if (
|
|
79
|
+
if (isSupersededByMembership(existing, options.invitation)) {
|
|
58
80
|
return writeFailure(invitationSupersededError());
|
|
59
81
|
}
|
|
60
82
|
|
|
@@ -71,17 +93,28 @@ export async function grantInvitedMembershipRole(db: DbRunner, options: InvitedM
|
|
|
71
93
|
return grantedMembership(true, roles);
|
|
72
94
|
}
|
|
73
95
|
|
|
74
|
-
//
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
function
|
|
96
|
+
// The pinned version is the membership's version when the invitation was issued
|
|
97
|
+
// (0 = none yet). A different version now means the membership was decided after
|
|
98
|
+
// the invite. Pin 0: a membership that appeared afterwards is an additive grant
|
|
99
|
+
// only while it is untouched since its creation (version 1). A role change bumps
|
|
100
|
+
// the version above 1 and counts as a decision. A removal in between hard-deletes
|
|
101
|
+
// the row and cancels the invitation (remove-member), so it never reaches this check.
|
|
102
|
+
// Rows issued before the pin existed (null) fall back to wall-clock timestamps,
|
|
103
|
+
// which can misjudge changes within the same millisecond but is all they carry.
|
|
104
|
+
function isSupersededByMembership(
|
|
105
|
+
membership: MembershipRow,
|
|
106
|
+
invitation: InvitationIssuance,
|
|
107
|
+
): boolean {
|
|
108
|
+
if (invitation.membershipVersion !== null) {
|
|
109
|
+
if (invitation.membershipVersion === 0) {
|
|
110
|
+
return membership.version > MEMBERSHIP_CREATED_VERSION;
|
|
111
|
+
}
|
|
112
|
+
return membership.version !== invitation.membershipVersion;
|
|
113
|
+
}
|
|
114
|
+
// Resend reuses the invitation row, so its last modification is the latest issuance.
|
|
115
|
+
const issuedAt = invitation.modifiedAt ?? invitation.insertedAt;
|
|
83
116
|
const lastChangedAt = membership.modifiedAt ?? membership.insertedAt;
|
|
84
|
-
return Temporal.Instant.compare(lastChangedAt,
|
|
117
|
+
return Temporal.Instant.compare(lastChangedAt, issuedAt) > 0;
|
|
85
118
|
}
|
|
86
119
|
|
|
87
120
|
function invitationSupersededError(): ConflictError {
|
|
@@ -1,4 +1,10 @@
|
|
|
1
1
|
[
|
|
2
|
+
{
|
|
3
|
+
"version": "0.354.0",
|
|
4
|
+
"type": "improvement",
|
|
5
|
+
"title": "Reference fields can opt out of the handover with handover: \"stay\"; boot depth limit and mover share one graph definition",
|
|
6
|
+
"detail": "ReferenceFieldDef gains handover: \"stay\". An entity that should NOT travel with its host but points at it through a plain reference no longer blocks the claim with entity_not_transferable when that reference field carries handover: \"stay\": the edge is not walked, its rows stay in the source tenant, and the reference crosses the tenant boundary after the handover. Without the opt-out the behaviour is unchanged (entity_not_transferable at claim time). Migration: an entity that should NOT travel with its host: mark the reference field handover: \"stay\" (removing transferable alone does not help once rows exist, the claim still fails). The framework now exports one adjacency definition (resolveEntityTransferAdjacency, reference and parentRef edges, non-transferable intermediate nodes included, handover: \"stay\" respected) that the boot validator's depth limit and the tenant-handover mover both use. Consequence: the boot-time MAX_TRANSFER_DEPTH check now also counts parentRef edges and chains running through non-transferable entities, as the mover always did; a graph that was deeper than 5 hops only through such nodes now fails the boot instead of the claim."
|
|
7
|
+
},
|
|
2
8
|
{
|
|
3
9
|
"version": "0.353.0",
|
|
4
10
|
"type": "fix",
|
|
@@ -4,18 +4,12 @@
|
|
|
4
4
|
// the graph depth is read off the existing declarations, never a hand-
|
|
5
5
|
// maintained per-app list).
|
|
6
6
|
//
|
|
7
|
-
//
|
|
8
|
-
//
|
|
9
|
-
//
|
|
10
|
-
//
|
|
11
|
-
//
|
|
12
|
-
//
|
|
13
|
-
// These nest freely.
|
|
14
|
-
//
|
|
15
|
-
// What comes out is static adjacency, not a walk: which edges lead away from
|
|
16
|
-
// each type, read straight off the declarations. The mover drives the actual
|
|
17
|
-
// traversal from the row ids it moves (#3131), because how deep a type sits
|
|
18
|
-
// depends on the rows found, not on the schema alone.
|
|
7
|
+
// The edge definition (parentRef + single-valued reference fields, minus
|
|
8
|
+
// `handover: "stay"`) lives in the framework so the boot validator's depth
|
|
9
|
+
// limit and the mover's traversal measure the same graph. What comes out is
|
|
10
|
+
// static adjacency, not a walk: the mover drives the actual traversal from the
|
|
11
|
+
// row ids it moves (#3131), because how deep a type sits depends on the rows
|
|
12
|
+
// found, not on the schema alone.
|
|
19
13
|
//
|
|
20
14
|
// `transferable: true` stays the only gate on what actually moves, and it is
|
|
21
15
|
// enforced by the caller against edges that turn out to have rows rather than
|
|
@@ -25,22 +19,16 @@
|
|
|
25
19
|
|
|
26
20
|
import {
|
|
27
21
|
type EntityDefinition,
|
|
28
|
-
parseRefTargetEntityName,
|
|
29
22
|
type Registry,
|
|
23
|
+
resolveEntityTransferAdjacency,
|
|
24
|
+
type TransferAdjacency,
|
|
30
25
|
} from "@cosmicdrift/kumiko-framework/engine";
|
|
31
26
|
|
|
32
|
-
export type
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
readonly entityName: string;
|
|
38
|
-
readonly entity: EntityDefinition;
|
|
39
|
-
/** The already-resolved type this edge hangs off — its moved row ids are
|
|
40
|
-
* the parent ids the mover matches this edge's rows against. */
|
|
41
|
-
readonly parentEntityName: string;
|
|
42
|
-
readonly link: TransferEdgeLink;
|
|
43
|
-
};
|
|
27
|
+
export type {
|
|
28
|
+
TransferAdjacency,
|
|
29
|
+
TransferEdge,
|
|
30
|
+
TransferEdgeLink,
|
|
31
|
+
} from "@cosmicdrift/kumiko-framework/engine";
|
|
44
32
|
|
|
45
33
|
export function resolveTransferableRoot(
|
|
46
34
|
registry: Registry,
|
|
@@ -50,79 +38,9 @@ export function resolveTransferableRoot(
|
|
|
50
38
|
return entity?.transferable === true ? entity : undefined;
|
|
51
39
|
}
|
|
52
40
|
|
|
53
|
-
// A `multiple` reference stores a jsonb array rather than a single id column,
|
|
54
|
-
// which the mover's `= ANY($ids)` match cannot address. The boot validator
|
|
55
|
-
// (engine/boot-validator/transfer-graph.ts) rejects that combination up front,
|
|
56
|
-
// so skipping it here can only ever hit an entity that is not transferable —
|
|
57
|
-
// never a silent partial move.
|
|
58
|
-
function referenceEdgesFrom(entityName: string, entity: EntityDefinition): readonly TransferEdge[] {
|
|
59
|
-
const edges: TransferEdge[] = [];
|
|
60
|
-
for (const [fieldName, field] of Object.entries(entity.fields)) {
|
|
61
|
-
if (field.type !== "reference") continue;
|
|
62
|
-
if (field.multiple === true) continue;
|
|
63
|
-
edges.push({
|
|
64
|
-
entityName,
|
|
65
|
-
entity,
|
|
66
|
-
parentEntityName: parseRefTargetEntityName(field.entity),
|
|
67
|
-
link: { kind: "reference", field: fieldName },
|
|
68
|
-
});
|
|
69
|
-
}
|
|
70
|
-
return edges;
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
// A parentRef names the types it accepts; without `allowedTypes` it accepts
|
|
74
|
-
// any of them, so the edge is recorded against every registered type.
|
|
75
|
-
function parentRefEdgesFrom(
|
|
76
|
-
entityName: string,
|
|
77
|
-
entity: EntityDefinition,
|
|
78
|
-
allEntityNames: readonly string[],
|
|
79
|
-
): readonly TransferEdge[] {
|
|
80
|
-
const parentRef = entity.parentRef;
|
|
81
|
-
if (!parentRef) return [];
|
|
82
|
-
const link: TransferEdgeLink = {
|
|
83
|
-
kind: "parentRef",
|
|
84
|
-
typeField: parentRef.entityTypeField,
|
|
85
|
-
idField: parentRef.entityIdField,
|
|
86
|
-
};
|
|
87
|
-
return (parentRef.allowedTypes ?? allEntityNames).map((parentEntityName) => ({
|
|
88
|
-
entityName,
|
|
89
|
-
entity,
|
|
90
|
-
parentEntityName,
|
|
91
|
-
link,
|
|
92
|
-
}));
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
/** Outgoing edges per type: keyed by the type an edge hangs off, so the mover
|
|
96
|
-
* can ask "what hangs off the rows I just moved" for any type it reaches, at
|
|
97
|
-
* any depth, as often as it reaches it. */
|
|
98
|
-
export type TransferAdjacency = ReadonlyMap<string, readonly TransferEdge[]>;
|
|
99
|
-
|
|
100
|
-
// A type can legitimately be reached by more than one edge — `note.vehicleId`
|
|
101
|
-
// and `note.campaignId` are distinct, and collapsing them by entity name would
|
|
102
|
-
// leave one edge's rows behind, the silent partial move #3088 exists to end.
|
|
103
|
-
// Both are kept here; the mover runs each against whatever parent ids it has.
|
|
104
41
|
export function resolveTransferAdjacency(
|
|
105
42
|
registry: Registry,
|
|
106
43
|
rootEntityType: string,
|
|
107
44
|
): TransferAdjacency {
|
|
108
|
-
|
|
109
|
-
const allEntityNames = [...entities.keys()];
|
|
110
|
-
const byParent = new Map<string, TransferEdge[]>();
|
|
111
|
-
|
|
112
|
-
for (const [entityName, entity] of entities) {
|
|
113
|
-
// skip: the root is never collected as someone else's descendant. A self
|
|
114
|
-
// reference would otherwise drag unrelated rows of the root's own type
|
|
115
|
-
// across the tenant boundary along with the one that was claimed.
|
|
116
|
-
if (entityName === rootEntityType) continue;
|
|
117
|
-
for (const edge of [
|
|
118
|
-
...parentRefEdgesFrom(entityName, entity, allEntityNames),
|
|
119
|
-
...referenceEdgesFrom(entityName, entity),
|
|
120
|
-
]) {
|
|
121
|
-
const edges = byParent.get(edge.parentEntityName) ?? [];
|
|
122
|
-
edges.push(edge);
|
|
123
|
-
byParent.set(edge.parentEntityName, edges);
|
|
124
|
-
}
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
return byParent;
|
|
45
|
+
return resolveEntityTransferAdjacency(registry.getAllEntities(), rootEntityType);
|
|
128
46
|
}
|
|
@@ -9,8 +9,11 @@ import {
|
|
|
9
9
|
isTenantResourceExtensionHooks,
|
|
10
10
|
type ResolvedPiiFlags,
|
|
11
11
|
} from "@cosmicdrift/kumiko-framework/engine";
|
|
12
|
+
import { createFallbackLogger } from "@cosmicdrift/kumiko-framework/logging";
|
|
12
13
|
import { entitiesOf } from "../shared/index.js";
|
|
13
14
|
|
|
15
|
+
const log = createFallbackLogger("kumiko:boot");
|
|
16
|
+
|
|
14
17
|
// V4: tenantOwned-entity-without-hook gate. Mirrors user-data-rights' V3
|
|
15
18
|
// (validateGdprPiiHookCoverage) but for EXT_TENANT_DATA. Registered as this
|
|
16
19
|
// feature's own `r.bootCheck()` (#1314, moved off the framework-internal
|
|
@@ -77,3 +80,16 @@ export function validateTenantDestroyHookShapes(features: readonly FeatureDefini
|
|
|
77
80
|
}
|
|
78
81
|
}
|
|
79
82
|
}
|
|
83
|
+
|
|
84
|
+
// `files-tenant-data` is opt-in (files stays usable without tenant-lifecycle),
|
|
85
|
+
// so a tenant-lifecycle app that mounts `files` but forgets it destroys the
|
|
86
|
+
// tenant's fileRef rows and storage binaries never — only a warning, since
|
|
87
|
+
// some apps legitimately handle file erasure themselves.
|
|
88
|
+
export function warnIfFilesMountedWithoutTenantData(features: readonly FeatureDefinition[]): void {
|
|
89
|
+
const mounted = new Set(features.map((f) => f.name));
|
|
90
|
+
// skip: nothing to cover (no files) or already covered.
|
|
91
|
+
if (!mounted.has("files") || mounted.has("files-tenant-data")) return;
|
|
92
|
+
log.warn(
|
|
93
|
+
'feature "files" is mounted with "tenant-lifecycle" but "files-tenant-data" is not — tenant destroy will leave fileRef rows and stored file binaries behind. Mount createFilesTenantDataFeature() to cover them.',
|
|
94
|
+
);
|
|
95
|
+
}
|
|
@@ -1,4 +1,26 @@
|
|
|
1
1
|
[
|
|
2
|
+
{
|
|
3
|
+
"version": "0.354.0",
|
|
4
|
+
"type": "improvement",
|
|
5
|
+
"title": "tenant-lifecycle/testing exports runTenantDestructionSweep, seedDestroyingTenant and driveDestructionToCompletion"
|
|
6
|
+
},
|
|
7
|
+
{
|
|
8
|
+
"version": "0.354.0",
|
|
9
|
+
"type": "improvement",
|
|
10
|
+
"title": "Tenant destroy stages resume partial progress next tick; hooks may return { done: false } and get deadlineAt; file hooks work in chunks",
|
|
11
|
+
"detail": "Tenant destroy stages resume partial progress in the next sweep tick instead of burning failed attempts\nA tenant with tens of thousands of files used to exhaust the stage's three attempts and leave the DSGVO destroy stuck. Destroy hooks (`destroyTenant`, EXT_TENANT_DATA `destroy`) may now return `{ done: false, processed }` and receive `deadlineAt` in their ctx; the runner records a `tenant-destruction-stage-progressed` event in the tenant stream and continues the stage next tick without counting a failed attempt. Hooks returning nothing stay valid. The `fileRef` row purge and the storage prefix wipe work in chunks against that deadline."
|
|
12
|
+
},
|
|
13
|
+
{
|
|
14
|
+
"version": "0.354.0",
|
|
15
|
+
"type": "improvement",
|
|
16
|
+
"title": "Boot warns when files is mounted without files-tenant-data",
|
|
17
|
+
"detail": "files-tenant-data is opt-in, so an app with tenant-lifecycle and files that forgot it silently left fileRef rows and stored binaries behind on tenant destroy. tenant-lifecycle now logs a boot warning in that case. Action: mount createFilesTenantDataFeature(), or ignore the warning if the app erases files itself."
|
|
18
|
+
},
|
|
19
|
+
{
|
|
20
|
+
"version": "0.354.0",
|
|
21
|
+
"type": "improvement",
|
|
22
|
+
"title": "Tenant destroy: a stage tick without progress counts as a failed attempt; done:false requires processed"
|
|
23
|
+
},
|
|
2
24
|
{
|
|
3
25
|
"version": "0.337.0",
|
|
4
26
|
"type": "fix",
|
|
@@ -11,6 +11,8 @@ export const TENANT_DESTRUCTION_STAGE_STARTED_EVENT_SHORT =
|
|
|
11
11
|
"tenant-destruction-stage-started" as const;
|
|
12
12
|
export const TENANT_DESTRUCTION_STAGE_SUCCEEDED_EVENT_SHORT =
|
|
13
13
|
"tenant-destruction-stage-succeeded" as const;
|
|
14
|
+
export const TENANT_DESTRUCTION_STAGE_PROGRESSED_EVENT_SHORT =
|
|
15
|
+
"tenant-destruction-stage-progressed" as const;
|
|
14
16
|
export const TENANT_DESTRUCTION_STAGE_FAILED_EVENT_SHORT =
|
|
15
17
|
"tenant-destruction-stage-failed" as const;
|
|
16
18
|
export const TENANT_DESTRUCTION_STAGE_ABANDONED_EVENT_SHORT =
|
|
@@ -26,6 +28,8 @@ export const TENANT_DESTRUCTION_STARTED_EVENT_QN =
|
|
|
26
28
|
`${EVENT_PREFIX}${TENANT_DESTRUCTION_STARTED_EVENT_SHORT}` as const;
|
|
27
29
|
export const TENANT_DESTRUCTION_STAGE_SUCCEEDED_EVENT_QN =
|
|
28
30
|
`${EVENT_PREFIX}${TENANT_DESTRUCTION_STAGE_SUCCEEDED_EVENT_SHORT}` as const;
|
|
31
|
+
export const TENANT_DESTRUCTION_STAGE_PROGRESSED_EVENT_QN =
|
|
32
|
+
`${EVENT_PREFIX}${TENANT_DESTRUCTION_STAGE_PROGRESSED_EVENT_SHORT}` as const;
|
|
29
33
|
export const TENANT_DESTRUCTION_STAGE_FAILED_EVENT_QN =
|
|
30
34
|
`${EVENT_PREFIX}${TENANT_DESTRUCTION_STAGE_FAILED_EVENT_SHORT}` as const;
|
|
31
35
|
export const TENANT_DESTRUCTION_STAGE_ABANDONED_EVENT_QN =
|
|
@@ -54,3 +58,7 @@ export const TENANT_DESTRUCTION_STAGES = [
|
|
|
54
58
|
] as const;
|
|
55
59
|
|
|
56
60
|
export type TenantDestructionStageName = (typeof TENANT_DESTRUCTION_STAGES)[number];
|
|
61
|
+
|
|
62
|
+
// The sweep cron fires every minute with concurrency "skip", so a tick must
|
|
63
|
+
// finish well inside that window or the next one is skipped.
|
|
64
|
+
export const DESTRUCTION_STAGE_TICK_BUDGET_MS = 30_000;
|
|
@@ -23,6 +23,12 @@ export const tenantDestructionStageSucceededSchema = z.object({
|
|
|
23
23
|
attempts: z.number().int().positive(),
|
|
24
24
|
});
|
|
25
25
|
|
|
26
|
+
export const tenantDestructionStageProgressedSchema = z.object({
|
|
27
|
+
stage: z.string().min(1),
|
|
28
|
+
attempts: z.number().int().positive(),
|
|
29
|
+
processed: z.number().int().nonnegative(),
|
|
30
|
+
});
|
|
31
|
+
|
|
26
32
|
export const tenantDestructionStageFailedSchema = z.object({
|
|
27
33
|
stage: z.string().min(1),
|
|
28
34
|
attempts: z.number().int().positive(),
|
|
@@ -9,7 +9,11 @@ import {
|
|
|
9
9
|
type FeatureDefinition,
|
|
10
10
|
type TenantLifecycleStatusPlugin,
|
|
11
11
|
} from "@cosmicdrift/kumiko-framework/engine";
|
|
12
|
-
import {
|
|
12
|
+
import {
|
|
13
|
+
validateTenantDataHookCoverage,
|
|
14
|
+
validateTenantDestroyHookShapes,
|
|
15
|
+
warnIfFilesMountedWithoutTenantData,
|
|
16
|
+
} from "./boot-checks.js";
|
|
13
17
|
import {
|
|
14
18
|
DESTRUCTION_CANCELLED_EVENT_SHORT,
|
|
15
19
|
DESTRUCTION_REQUESTED_EVENT_SHORT,
|
|
@@ -17,6 +21,7 @@ import {
|
|
|
17
21
|
TENANT_DESTRUCTION_FAILED_EVENT_SHORT,
|
|
18
22
|
TENANT_DESTRUCTION_STAGE_ABANDONED_EVENT_SHORT,
|
|
19
23
|
TENANT_DESTRUCTION_STAGE_FAILED_EVENT_SHORT,
|
|
24
|
+
TENANT_DESTRUCTION_STAGE_PROGRESSED_EVENT_SHORT,
|
|
20
25
|
TENANT_DESTRUCTION_STAGE_STARTED_EVENT_SHORT,
|
|
21
26
|
TENANT_DESTRUCTION_STAGE_SUCCEEDED_EVENT_SHORT,
|
|
22
27
|
TENANT_DESTRUCTION_STARTED_EVENT_SHORT,
|
|
@@ -28,6 +33,7 @@ import {
|
|
|
28
33
|
tenantDestructionFailedSchema,
|
|
29
34
|
tenantDestructionStageAbandonedSchema,
|
|
30
35
|
tenantDestructionStageFailedSchema,
|
|
36
|
+
tenantDestructionStageProgressedSchema,
|
|
31
37
|
tenantDestructionStageStartedSchema,
|
|
32
38
|
tenantDestructionStageSucceededSchema,
|
|
33
39
|
tenantDestructionStartedSchema,
|
|
@@ -78,6 +84,7 @@ export function createTenantLifecycleFeature(): FeatureDefinition {
|
|
|
78
84
|
// guard's "tenant-lifecycle mounted" gate exactly.
|
|
79
85
|
r.bootCheck(({ features }) => validateTenantDataHookCoverage(features));
|
|
80
86
|
r.bootCheck(({ features }) => validateTenantDestroyHookShapes(features));
|
|
87
|
+
r.bootCheck(({ features }) => warnIfFilesMountedWithoutTenantData(features));
|
|
81
88
|
|
|
82
89
|
r.defineEvent(DESTRUCTION_REQUESTED_EVENT_SHORT, destructionRequestedSchema, {
|
|
83
90
|
piiFields: "none",
|
|
@@ -100,6 +107,11 @@ export function createTenantLifecycleFeature(): FeatureDefinition {
|
|
|
100
107
|
tenantDestructionStageSucceededSchema,
|
|
101
108
|
{ piiFields: "none" },
|
|
102
109
|
);
|
|
110
|
+
r.defineEvent(
|
|
111
|
+
TENANT_DESTRUCTION_STAGE_PROGRESSED_EVENT_SHORT,
|
|
112
|
+
tenantDestructionStageProgressedSchema,
|
|
113
|
+
{ piiFields: "none" },
|
|
114
|
+
);
|
|
103
115
|
r.defineEvent(TENANT_DESTRUCTION_STAGE_FAILED_EVENT_SHORT, tenantDestructionStageFailedSchema, {
|
|
104
116
|
piiFields: "none",
|
|
105
117
|
});
|
|
@@ -18,17 +18,20 @@ import type { SearchAdapter } from "@cosmicdrift/kumiko-framework/search";
|
|
|
18
18
|
import { getTemporal } from "@cosmicdrift/kumiko-framework/time";
|
|
19
19
|
import { tenantEntity, tenantTable } from "../tenant/index.js";
|
|
20
20
|
import {
|
|
21
|
+
DESTRUCTION_STAGE_TICK_BUDGET_MS,
|
|
21
22
|
TENANT_AGGREGATE_TYPE,
|
|
22
23
|
TENANT_DESTRUCTION_COMPLETED_EVENT_QN,
|
|
23
24
|
TENANT_DESTRUCTION_FAILED_EVENT_QN,
|
|
24
25
|
TENANT_DESTRUCTION_STAGE_ABANDONED_EVENT_SHORT,
|
|
25
26
|
TENANT_DESTRUCTION_STAGE_FAILED_EVENT_SHORT,
|
|
27
|
+
TENANT_DESTRUCTION_STAGE_PROGRESSED_EVENT_SHORT,
|
|
26
28
|
TENANT_DESTRUCTION_STAGE_STARTED_EVENT_SHORT,
|
|
27
29
|
TENANT_DESTRUCTION_STAGE_SUCCEEDED_EVENT_SHORT,
|
|
28
30
|
TENANT_DESTRUCTION_STARTED_EVENT_SHORT,
|
|
29
31
|
} from "./constants.js";
|
|
30
32
|
import { invalidateTenantLifecycleGate, TEARDOWN_GATE_SETTLE_MS } from "./lifecycle-gate.js";
|
|
31
33
|
import {
|
|
34
|
+
type DestructionStage,
|
|
32
35
|
type DestructionStageCtx,
|
|
33
36
|
isDestructionPipelineComplete,
|
|
34
37
|
pickNextStage,
|
|
@@ -43,6 +46,7 @@ type StageEventName =
|
|
|
43
46
|
| typeof TENANT_DESTRUCTION_STAGE_STARTED_EVENT_SHORT
|
|
44
47
|
| typeof TENANT_DESTRUCTION_STAGE_SUCCEEDED_EVENT_SHORT
|
|
45
48
|
| typeof TENANT_DESTRUCTION_STAGE_FAILED_EVENT_SHORT
|
|
49
|
+
| typeof TENANT_DESTRUCTION_STAGE_PROGRESSED_EVENT_SHORT
|
|
46
50
|
| typeof TENANT_DESTRUCTION_STAGE_ABANDONED_EVENT_SHORT;
|
|
47
51
|
|
|
48
52
|
function qualified(short: StageEventName): string {
|
|
@@ -78,6 +82,23 @@ function replayStageState(
|
|
|
78
82
|
return { completed, abandoned, attemptsByStage };
|
|
79
83
|
}
|
|
80
84
|
|
|
85
|
+
// A progressed event as the stage's latest lifecycle event means the previous
|
|
86
|
+
// tick handed over mid-stage; its stage-started is still the open one.
|
|
87
|
+
function isStageContinuation(
|
|
88
|
+
events: ReadonlyArray<{ type: string; payload: Record<string, unknown> }>,
|
|
89
|
+
stageName: string,
|
|
90
|
+
): boolean {
|
|
91
|
+
const stageEventTypes = new Set([
|
|
92
|
+
qualified(TENANT_DESTRUCTION_STAGE_STARTED_EVENT_SHORT),
|
|
93
|
+
qualified(TENANT_DESTRUCTION_STAGE_PROGRESSED_EVENT_SHORT),
|
|
94
|
+
qualified(TENANT_DESTRUCTION_STAGE_FAILED_EVENT_SHORT),
|
|
95
|
+
]);
|
|
96
|
+
const lastStageEvent = events.findLast(
|
|
97
|
+
(event) => stageEventTypes.has(event.type) && event.payload["stage"] === stageName,
|
|
98
|
+
);
|
|
99
|
+
return lastStageEvent?.type === qualified(TENANT_DESTRUCTION_STAGE_PROGRESSED_EVENT_SHORT);
|
|
100
|
+
}
|
|
101
|
+
|
|
81
102
|
function lastEventVersion(events: ReadonlyArray<{ version: number }>): number {
|
|
82
103
|
const last = events.at(-1);
|
|
83
104
|
return last?.version ?? 0;
|
|
@@ -217,6 +238,43 @@ async function maybeAppendDestructionCompleted(
|
|
|
217
238
|
);
|
|
218
239
|
}
|
|
219
240
|
|
|
241
|
+
type StageTickResult = {
|
|
242
|
+
readonly done: boolean;
|
|
243
|
+
readonly error?: string;
|
|
244
|
+
readonly halted?: boolean;
|
|
245
|
+
readonly progressed?: boolean;
|
|
246
|
+
};
|
|
247
|
+
|
|
248
|
+
// `version` differs per caller: the catch path appends against the pre-run
|
|
249
|
+
// version, the no-progress path against the version reloaded after run().
|
|
250
|
+
async function recordFailedStageAttempt(args: {
|
|
251
|
+
readonly db: DbRunner;
|
|
252
|
+
readonly tenantId: TenantId;
|
|
253
|
+
readonly stage: DestructionStage;
|
|
254
|
+
readonly attempt: number;
|
|
255
|
+
readonly error: string;
|
|
256
|
+
readonly version: number;
|
|
257
|
+
}): Promise<StageTickResult> {
|
|
258
|
+
if (args.attempt >= args.stage.maxAttempts) {
|
|
259
|
+
await haltPipelineOnAbandon({
|
|
260
|
+
db: args.db,
|
|
261
|
+
tenantId: args.tenantId,
|
|
262
|
+
stage: args.stage.name,
|
|
263
|
+
attempts: args.attempt,
|
|
264
|
+
error: args.error,
|
|
265
|
+
});
|
|
266
|
+
return { done: false, error: args.error, halted: true };
|
|
267
|
+
}
|
|
268
|
+
await appendTenantStageEvent(
|
|
269
|
+
args.db,
|
|
270
|
+
args.tenantId,
|
|
271
|
+
qualified(TENANT_DESTRUCTION_STAGE_FAILED_EVENT_SHORT),
|
|
272
|
+
{ stage: args.stage.name, attempts: args.attempt, error: args.error },
|
|
273
|
+
args.version,
|
|
274
|
+
);
|
|
275
|
+
return { done: false, error: args.error };
|
|
276
|
+
}
|
|
277
|
+
|
|
220
278
|
export async function runNextDestructionStage(args: {
|
|
221
279
|
readonly db: DbRunner;
|
|
222
280
|
readonly registry: Registry;
|
|
@@ -226,7 +284,7 @@ export async function runNextDestructionStage(args: {
|
|
|
226
284
|
readonly searchAdapter?: SearchAdapter;
|
|
227
285
|
readonly escapeHatchAuditSink?: EscapeHatchAuditSink;
|
|
228
286
|
readonly actor?: string;
|
|
229
|
-
}): Promise<
|
|
287
|
+
}): Promise<StageTickResult> {
|
|
230
288
|
const events = await loadAggregate(args.db, args.tenantId, args.tenantId);
|
|
231
289
|
const { completed, abandoned, attemptsByStage } = replayStageState(events);
|
|
232
290
|
|
|
@@ -251,25 +309,50 @@ export async function runNextDestructionStage(args: {
|
|
|
251
309
|
searchAdapter: args.searchAdapter,
|
|
252
310
|
escapeHatchAuditSink: args.escapeHatchAuditSink,
|
|
253
311
|
actor: args.actor,
|
|
312
|
+
deadlineAt: Date.now() + DESTRUCTION_STAGE_TICK_BUDGET_MS,
|
|
254
313
|
};
|
|
255
314
|
|
|
256
315
|
let version = lastEventVersion(events);
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
316
|
+
if (!isStageContinuation(events, next.name)) {
|
|
317
|
+
version = await appendTenantStageEventIdempotent(
|
|
318
|
+
args.db,
|
|
319
|
+
args.tenantId,
|
|
320
|
+
qualified(TENANT_DESTRUCTION_STAGE_STARTED_EVENT_SHORT),
|
|
321
|
+
{ stage: next.name, attempts: attempt },
|
|
322
|
+
version,
|
|
323
|
+
stageOutcomeRecorded(events, next.name, TENANT_DESTRUCTION_STAGE_SUCCEEDED_EVENT_SHORT),
|
|
324
|
+
);
|
|
325
|
+
}
|
|
265
326
|
|
|
266
327
|
try {
|
|
267
|
-
await next.run(ctx);
|
|
328
|
+
const outcome = await next.run(ctx);
|
|
268
329
|
// next.run() may itself have appended events to this aggregate (e.g.
|
|
269
330
|
// tombstoneTenantRow updates the tenant row on the same stream) — reload
|
|
270
331
|
// the version fresh instead of trusting the pre-run `version`, or this
|
|
271
332
|
// append throws VersionConflictError on every stage whose run() writes.
|
|
272
333
|
const eventsAfterRun = await loadAggregate(args.db, args.tenantId, args.tenantId);
|
|
334
|
+
// A tick that neither finishes nor processes anything would otherwise loop
|
|
335
|
+
// forever, because progressed events do not count as attempts.
|
|
336
|
+
if (!outcome.done && outcome.processed === 0) {
|
|
337
|
+
return await recordFailedStageAttempt({
|
|
338
|
+
db: args.db,
|
|
339
|
+
tenantId: args.tenantId,
|
|
340
|
+
stage: next,
|
|
341
|
+
attempt,
|
|
342
|
+
error: `stage "${next.name}" made no progress`,
|
|
343
|
+
version: lastEventVersion(eventsAfterRun),
|
|
344
|
+
});
|
|
345
|
+
}
|
|
346
|
+
if (!outcome.done) {
|
|
347
|
+
await appendTenantStageEvent(
|
|
348
|
+
args.db,
|
|
349
|
+
args.tenantId,
|
|
350
|
+
qualified(TENANT_DESTRUCTION_STAGE_PROGRESSED_EVENT_SHORT),
|
|
351
|
+
{ stage: next.name, attempts: attempt, processed: outcome.processed },
|
|
352
|
+
lastEventVersion(eventsAfterRun),
|
|
353
|
+
);
|
|
354
|
+
return { done: false, progressed: true };
|
|
355
|
+
}
|
|
273
356
|
version = await appendTenantStageEventIdempotent(
|
|
274
357
|
args.db,
|
|
275
358
|
args.tenantId,
|
|
@@ -287,26 +370,14 @@ export async function runNextDestructionStage(args: {
|
|
|
287
370
|
await maybeAppendDestructionCompleted(args.db, args.tenantId, completedAfter);
|
|
288
371
|
return { done: isDestructionPipelineComplete(completedAfter) };
|
|
289
372
|
} catch (err) {
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
stage: next.name,
|
|
297
|
-
attempts: attempt,
|
|
298
|
-
error: message,
|
|
299
|
-
});
|
|
300
|
-
return { done: false, error: message, halted: true };
|
|
301
|
-
}
|
|
302
|
-
await appendTenantStageEvent(
|
|
303
|
-
args.db,
|
|
304
|
-
args.tenantId,
|
|
305
|
-
qualified(TENANT_DESTRUCTION_STAGE_FAILED_EVENT_SHORT),
|
|
306
|
-
{ stage: next.name, attempts: attempt, error: message },
|
|
373
|
+
return await recordFailedStageAttempt({
|
|
374
|
+
db: args.db,
|
|
375
|
+
tenantId: args.tenantId,
|
|
376
|
+
stage: next,
|
|
377
|
+
attempt,
|
|
378
|
+
error: err instanceof Error ? err.message : String(err),
|
|
307
379
|
version,
|
|
308
|
-
);
|
|
309
|
-
return { done: false, error: message };
|
|
380
|
+
});
|
|
310
381
|
}
|
|
311
382
|
}
|
|
312
383
|
|