@sekiban/dcb-runtime 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +86 -0
- package/README.md +13 -0
- package/dist/allocator/AllocatorDurableObject.d.ts +36 -0
- package/dist/allocator/AllocatorDurableObject.js +658 -0
- package/dist/allocator/IssuanceLedger.d.ts +188 -0
- package/dist/allocator/IssuanceLedger.js +608 -0
- package/dist/allocator/IssuanceReconciler.d.ts +41 -0
- package/dist/allocator/IssuanceReconciler.js +191 -0
- package/dist/allocator/OrderClock.d.ts +41 -0
- package/dist/allocator/OrderClock.js +92 -0
- package/dist/allocator/SortableUniqueId.d.ts +46 -0
- package/dist/allocator/SortableUniqueId.js +156 -0
- package/dist/allocator/types.d.ts +52 -0
- package/dist/allocator/types.js +1 -0
- package/dist/bootstrap/BootstrapCoordinatorDurableObject.d.ts +28 -0
- package/dist/bootstrap/BootstrapCoordinatorDurableObject.js +357 -0
- package/dist/bootstrap/BootstrapStoreAdapter.d.ts +51 -0
- package/dist/bootstrap/BootstrapStoreAdapter.js +162 -0
- package/dist/bootstrap/OperatorBootstrap.d.ts +20 -0
- package/dist/bootstrap/OperatorBootstrap.js +104 -0
- package/dist/bootstrap/manifest.d.ts +9 -0
- package/dist/bootstrap/manifest.js +115 -0
- package/dist/bootstrap/types.d.ts +75 -0
- package/dist/bootstrap/types.js +1 -0
- package/dist/cli/OperatorRepairCli.d.ts +11 -0
- package/dist/cli/OperatorRepairCli.js +108 -0
- package/dist/cloudflare.d.ts +164 -0
- package/dist/cloudflare.js +19487 -0
- package/dist/commit/CommitWorker.d.ts +101 -0
- package/dist/commit/CommitWorker.js +921 -0
- package/dist/commit/types.d.ts +25 -0
- package/dist/commit/types.js +13 -0
- package/dist/completeness/GlobalCompletenessReconciler.d.ts +77 -0
- package/dist/completeness/GlobalCompletenessReconciler.js +572 -0
- package/dist/completeness/types.d.ts +85 -0
- package/dist/completeness/types.js +6 -0
- package/dist/composition.d.ts +123 -0
- package/dist/composition.js +237 -0
- package/dist/cosmos.d.ts +11 -0
- package/dist/cosmos.js +1411 -0
- package/dist/d1-mv.d.ts +8 -0
- package/dist/d1-mv.js +2001 -0
- package/dist/d1.d.ts +16 -0
- package/dist/d1.js +3418 -0
- package/dist/diagnostics/G60DurableHop.d.ts +99 -0
- package/dist/diagnostics/G60DurableHop.js +248 -0
- package/dist/diagnostics/G65Admission.d.ts +23 -0
- package/dist/diagnostics/G65Admission.js +43 -0
- package/dist/diagnostics/G65DirectRing.d.ts +33 -0
- package/dist/diagnostics/G65DirectRing.js +85 -0
- package/dist/diagnostics/G69AdmissionAttempt.d.ts +43 -0
- package/dist/diagnostics/G69AdmissionAttempt.js +22 -0
- package/dist/downstream/DeliveryCore.d.ts +127 -0
- package/dist/downstream/DeliveryCore.js +304 -0
- package/dist/downstream/Doorbell.d.ts +43 -0
- package/dist/downstream/Doorbell.js +137 -0
- package/dist/downstream/DownstreamAdapter.d.ts +52 -0
- package/dist/downstream/DownstreamAdapter.js +299 -0
- package/dist/downstream/ExclusionLookup.d.ts +36 -0
- package/dist/downstream/ExclusionLookup.js +50 -0
- package/dist/downstream/InconsistencyDetector.d.ts +27 -0
- package/dist/downstream/InconsistencyDetector.js +82 -0
- package/dist/downstream/OutboxDrain.d.ts +35 -0
- package/dist/downstream/OutboxDrain.js +126 -0
- package/dist/downstream/types.d.ts +77 -0
- package/dist/downstream/types.js +64 -0
- package/dist/eventIdentity.d.ts +30 -0
- package/dist/eventIdentity.js +49 -0
- package/dist/eventRecord.d.ts +26 -0
- package/dist/eventRecord.js +53 -0
- package/dist/g42-probe.d.ts +9 -0
- package/dist/g42-probe.js +9 -0
- package/dist/http/SerializedQueryWorker.d.ts +58 -0
- package/dist/http/SerializedQueryWorker.js +497 -0
- package/dist/http/commitResponse.d.ts +46 -0
- package/dist/http/commitResponse.js +80 -0
- package/dist/http/testServiceId.d.ts +17 -0
- package/dist/http/testServiceId.js +14 -0
- package/dist/index.d.ts +112 -0
- package/dist/index.js +19926 -0
- package/dist/journal/JournalDurableObject.d.ts +30 -0
- package/dist/journal/JournalDurableObject.js +481 -0
- package/dist/journal/JournalFirstTouchProbe.d.ts +190 -0
- package/dist/journal/JournalFirstTouchProbe.js +375 -0
- package/dist/journal/types.d.ts +116 -0
- package/dist/journal/types.js +52 -0
- package/dist/mv/MaterializedViewCatchUp.d.ts +60 -0
- package/dist/mv/MaterializedViewCatchUp.js +353 -0
- package/dist/mv/MaterializedViewRuntime.d.ts +80 -0
- package/dist/mv/MaterializedViewRuntime.js +328 -0
- package/dist/mv/MaterializedViewStore.d.ts +257 -0
- package/dist/mv/MaterializedViewStore.js +875 -0
- package/dist/mv/UnsafeWindowMaterializedView.d.ts +112 -0
- package/dist/mv/UnsafeWindowMaterializedView.js +495 -0
- package/dist/mv/index.d.ts +8 -0
- package/dist/mv/index.js +2357 -0
- package/dist/projection/LiveProjectionWorker.d.ts +86 -0
- package/dist/projection/LiveProjectionWorker.js +240 -0
- package/dist/projection/ProjectionRuntime.d.ts +113 -0
- package/dist/projection/ProjectionRuntime.js +346 -0
- package/dist/projection/ProjectorRegistry.d.ts +51 -0
- package/dist/projection/ProjectorRegistry.js +116 -0
- package/dist/query/ProjectionQueryStore.d.ts +102 -0
- package/dist/query/ProjectionQueryStore.js +185 -0
- package/dist/query/QueryRegistry.d.ts +22 -0
- package/dist/query/QueryRegistry.js +36 -0
- package/dist/read/SerializedReadWorker.d.ts +36 -0
- package/dist/read/SerializedReadWorker.js +225 -0
- package/dist/repair/RepairWorker.d.ts +72 -0
- package/dist/repair/RepairWorker.js +461 -0
- package/dist/safeWindow.d.ts +20 -0
- package/dist/safeWindow.js +36 -0
- package/dist/scope/ControlRouteScope.d.ts +22 -0
- package/dist/scope/ControlRouteScope.js +36 -0
- package/dist/scope/ScopeName.d.ts +34 -0
- package/dist/scope/ScopeName.js +67 -0
- package/dist/service/ServiceIdentityProvider.d.ts +35 -0
- package/dist/service/ServiceIdentityProvider.js +56 -0
- package/dist/store/CosmosEventStore.d.ts +126 -0
- package/dist/store/CosmosEventStore.js +950 -0
- package/dist/store/D1EventStore.d.ts +111 -0
- package/dist/store/D1EventStore.js +1111 -0
- package/dist/store/PostgresEventStore.d.ts +35 -0
- package/dist/store/PostgresEventStore.js +811 -0
- package/dist/store/provider.d.ts +25 -0
- package/dist/store/provider.js +32 -0
- package/dist/store/types.d.ts +205 -0
- package/dist/store/types.js +18 -0
- package/dist/tag/EventDigest.d.ts +40 -0
- package/dist/tag/EventDigest.js +158 -0
- package/dist/tag/TagDurableObject.d.ts +286 -0
- package/dist/tag/TagDurableObject.js +4071 -0
- package/dist/tag/TagSqlMeasurement.d.ts +26 -0
- package/dist/tag/TagSqlMeasurement.js +44 -0
- package/dist/tag/TagSqlSchema.d.ts +29 -0
- package/dist/tag/TagSqlSchema.js +211 -0
- package/dist/tag/types.d.ts +173 -0
- package/dist/tag/types.js +3 -0
- package/dist/tagstate/TagStateDurableObject.d.ts +57 -0
- package/dist/tagstate/TagStateDurableObject.js +499 -0
- package/dist/trace/AttributionRatio.d.ts +12 -0
- package/dist/trace/AttributionRatio.js +68 -0
- package/dist/trace/CloudflareTracing.d.ts +7 -0
- package/dist/trace/CloudflareTracing.js +9 -0
- package/dist/trace/CommitTrace.d.ts +405 -0
- package/dist/trace/CommitTrace.js +1008 -0
- package/dist/trace/CommitTraceConsoleSink.d.ts +68 -0
- package/dist/trace/CommitTraceConsoleSink.js +114 -0
- package/dist/trace/CommitTraceVerifier.d.ts +22 -0
- package/dist/trace/CommitTraceVerifier.js +295 -0
- package/dist/trace/ObservationStream.d.ts +118 -0
- package/dist/trace/ObservationStream.js +148 -0
- package/package.json +71 -0
|
@@ -0,0 +1,921 @@
|
|
|
1
|
+
import { durationSince, } from "../http/commitResponse";
|
|
2
|
+
import { COMMIT_TEST_FAULTS, } from "./types";
|
|
3
|
+
import { canonicalEventType } from "../eventIdentity";
|
|
4
|
+
import { createUuidV7, serializedEventMetadata, writeTimestampUtc } from "../eventRecord";
|
|
5
|
+
import { assertSortableUniqueId } from "../allocator/SortableUniqueId";
|
|
6
|
+
import { PARTIAL_WRITE_FENCE_REASON } from "../tag/types";
|
|
7
|
+
import { beginWorkerInvocationObservation, CommitTrace, createTraceCorrelationId, } from "../trace/CommitTrace";
|
|
8
|
+
import { observeWorkerInvocation } from "../trace/ObservationStream";
|
|
9
|
+
import { verifyCommitTrace } from "../trace/CommitTraceVerifier";
|
|
10
|
+
import { scopeIdFor } from "../scope/ScopeName";
|
|
11
|
+
import { envServiceIdentity, requestServiceIdentity, } from "../service/ServiceIdentityProvider";
|
|
12
|
+
const INITIAL_OWNER_EPOCH = 0;
|
|
13
|
+
const MAX_WRITE_ATTEMPTS = 2;
|
|
14
|
+
const OUTCOME_UNDETERMINED_ERROR = "Commit outcome is undetermined; reread tag heads and event/query state before retrying because blind retry may create duplicate events.";
|
|
15
|
+
const GLOBAL_ADMISSION_HEADER = "x-sdt-global-admission";
|
|
16
|
+
/**
|
|
17
|
+
* The static authority checker proves the manifest bytes; this is the
|
|
18
|
+
* separate runtime structural/attribution verifier. Its outcome is retained
|
|
19
|
+
* only in the in-process trace snapshot and is fail-open by design: G30 must
|
|
20
|
+
* not change the protocol when an observation adapter is unavailable.
|
|
21
|
+
*/
|
|
22
|
+
const runtimeCommitTraceVerifier = {
|
|
23
|
+
verify(snapshot) {
|
|
24
|
+
const accepted = snapshot.spans.some((span) => span.face === "accepted" && typeof span.attributes["attempt.id"] === "string");
|
|
25
|
+
verifyCommitTrace(snapshot, { accepted });
|
|
26
|
+
},
|
|
27
|
+
};
|
|
28
|
+
function json(body, status = 200, headers) {
|
|
29
|
+
const responseHeaders = new Headers(headers);
|
|
30
|
+
responseHeaders.set("content-type", "application/json; charset=utf-8");
|
|
31
|
+
return new Response(JSON.stringify(body), { status, headers: responseHeaders });
|
|
32
|
+
}
|
|
33
|
+
function error(status, code, message, retryable = false) {
|
|
34
|
+
return json(retryable ? { error: message, code, retryable: true } : { error: message, code }, status);
|
|
35
|
+
}
|
|
36
|
+
function readGlobalAdmission(response) {
|
|
37
|
+
const value = response.headers.get(GLOBAL_ADMISSION_HEADER);
|
|
38
|
+
return value === "admitted" || value === "not-admitted" || value === "unknown" ? value : "unknown";
|
|
39
|
+
}
|
|
40
|
+
function mergeGlobalAdmission(left, right) {
|
|
41
|
+
if (left === "unknown" || right === "unknown")
|
|
42
|
+
return "unknown";
|
|
43
|
+
if (left === "not-admitted" || right === "not-admitted")
|
|
44
|
+
return "not-admitted";
|
|
45
|
+
return "admitted";
|
|
46
|
+
}
|
|
47
|
+
function withGlobalAdmission(response, status) {
|
|
48
|
+
const headers = new Headers(response.headers);
|
|
49
|
+
headers.set(GLOBAL_ADMISSION_HEADER, status);
|
|
50
|
+
return new Response(response.body, { status: response.status, headers });
|
|
51
|
+
}
|
|
52
|
+
function isObject(value) {
|
|
53
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
54
|
+
}
|
|
55
|
+
function isNonEmptyString(value) {
|
|
56
|
+
return typeof value === "string" && value.length > 0;
|
|
57
|
+
}
|
|
58
|
+
function unique(values) {
|
|
59
|
+
return [...new Set(values)];
|
|
60
|
+
}
|
|
61
|
+
function isStandardBase64(value) {
|
|
62
|
+
if (value.length === 0) {
|
|
63
|
+
return true;
|
|
64
|
+
}
|
|
65
|
+
return (value.length % 4 === 0 &&
|
|
66
|
+
/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(value));
|
|
67
|
+
}
|
|
68
|
+
class CommitPayloadAdmissionError extends Error {
|
|
69
|
+
code;
|
|
70
|
+
constructor(code, message) {
|
|
71
|
+
super(message);
|
|
72
|
+
this.code = code;
|
|
73
|
+
this.name = "CommitPayloadAdmissionError";
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
function decodeBase64Utf8Json(value) {
|
|
77
|
+
let bytes;
|
|
78
|
+
try {
|
|
79
|
+
const binary = atob(value);
|
|
80
|
+
bytes = Uint8Array.from(binary, (character) => character.charCodeAt(0));
|
|
81
|
+
}
|
|
82
|
+
catch {
|
|
83
|
+
throw new CommitPayloadAdmissionError("invalid_payload_utf8", "Payload base64 could not be decoded");
|
|
84
|
+
}
|
|
85
|
+
if (bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf) {
|
|
86
|
+
throw new CommitPayloadAdmissionError("invalid_payload_utf8", "Payload must not start with a UTF-8 BOM");
|
|
87
|
+
}
|
|
88
|
+
let text;
|
|
89
|
+
try {
|
|
90
|
+
// The fatal decoder is deliberate: replacement characters would destroy
|
|
91
|
+
// the payload-byte equality contract before storage is even attempted.
|
|
92
|
+
text = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
|
|
93
|
+
}
|
|
94
|
+
catch {
|
|
95
|
+
throw new CommitPayloadAdmissionError("invalid_payload_utf8", "Payload must be valid UTF-8 JSON text");
|
|
96
|
+
}
|
|
97
|
+
try {
|
|
98
|
+
return Object.freeze({ text, parsed: JSON.parse(text) });
|
|
99
|
+
}
|
|
100
|
+
catch {
|
|
101
|
+
throw new CommitPayloadAdmissionError("invalid_payload_json", "Payload must be syntactically valid JSON");
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
function assertNoPayloadDiscriminator(value, path = "$") {
|
|
105
|
+
if (Array.isArray(value)) {
|
|
106
|
+
value.forEach((entry, index) => assertNoPayloadDiscriminator(entry, `${path}[${index}]`));
|
|
107
|
+
return;
|
|
108
|
+
}
|
|
109
|
+
if (typeof value !== "object" || value === null)
|
|
110
|
+
return;
|
|
111
|
+
for (const [key, child] of Object.entries(value)) {
|
|
112
|
+
if (key === "eventType" || key === "eventName" || key === "eventPayloadName") {
|
|
113
|
+
throw new CommitPayloadAdmissionError("payload_type_discriminator", `Payload discriminator ${path}.${key} is forbidden`);
|
|
114
|
+
}
|
|
115
|
+
assertNoPayloadDiscriminator(child, `${path}.${key}`);
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* A parser may coerce values, but it may not silently accept a member missing
|
|
120
|
+
* from its returned schema value. This catches additional members and
|
|
121
|
+
* case-only spelling drift without reserializing the admitted bytes.
|
|
122
|
+
*/
|
|
123
|
+
function assertExactMemberPaths(actual, registered, path = "$") {
|
|
124
|
+
if (Array.isArray(actual)) {
|
|
125
|
+
if (!Array.isArray(registered)) {
|
|
126
|
+
throw new CommitPayloadAdmissionError("payload_case_mismatch", `Registered schema shape differs at ${path}`);
|
|
127
|
+
}
|
|
128
|
+
actual.forEach((entry, index) => assertExactMemberPaths(entry, registered[index], `${path}[${index}]`));
|
|
129
|
+
return;
|
|
130
|
+
}
|
|
131
|
+
if (typeof actual !== "object" || actual === null || Array.isArray(actual))
|
|
132
|
+
return;
|
|
133
|
+
if (typeof registered !== "object" || registered === null || Array.isArray(registered)) {
|
|
134
|
+
throw new CommitPayloadAdmissionError("payload_case_mismatch", `Registered schema shape differs at ${path}`);
|
|
135
|
+
}
|
|
136
|
+
const registeredRecord = registered;
|
|
137
|
+
for (const [key, value] of Object.entries(actual)) {
|
|
138
|
+
if (!Object.prototype.hasOwnProperty.call(registeredRecord, key)) {
|
|
139
|
+
const caseOnly = Object.keys(registeredRecord).find((registeredKey) => registeredKey.toLocaleLowerCase("en-US") === key.toLocaleLowerCase("en-US"));
|
|
140
|
+
const suffix = caseOnly === undefined ? "is not registered" : `must use exact case ${caseOnly}`;
|
|
141
|
+
throw new CommitPayloadAdmissionError("payload_case_mismatch", `Payload member ${path}.${key} ${suffix}`);
|
|
142
|
+
}
|
|
143
|
+
assertExactMemberPaths(value, registeredRecord[key], `${path}.${key}`);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
function encodeBase64Utf8(value) {
|
|
147
|
+
const bytes = new TextEncoder().encode(value);
|
|
148
|
+
let binary = "";
|
|
149
|
+
for (const byte of bytes)
|
|
150
|
+
binary += String.fromCharCode(byte);
|
|
151
|
+
return btoa(binary);
|
|
152
|
+
}
|
|
153
|
+
function faultFromRequest(request) {
|
|
154
|
+
// The test pool calls the Worker through this synthetic host. Production
|
|
155
|
+
// V1 traffic cannot enable fault injection through a wire/header extension.
|
|
156
|
+
if (new URL(request.url).hostname !== "commit.test") {
|
|
157
|
+
return undefined;
|
|
158
|
+
}
|
|
159
|
+
const candidate = request.headers.get("x-sdt-g4-test-fault");
|
|
160
|
+
return candidate !== null && COMMIT_TEST_FAULTS.includes(candidate)
|
|
161
|
+
? candidate
|
|
162
|
+
: undefined;
|
|
163
|
+
}
|
|
164
|
+
function testAttemptIdFromRequest(request) {
|
|
165
|
+
if (new URL(request.url).hostname !== "commit.test") {
|
|
166
|
+
return undefined;
|
|
167
|
+
}
|
|
168
|
+
const candidate = request.headers.get("x-sdt-g4-test-attempt-id");
|
|
169
|
+
return candidate !== null && isNonEmptyString(candidate) ? candidate : undefined;
|
|
170
|
+
}
|
|
171
|
+
function responseWithAttempt(response, attemptId, expose) {
|
|
172
|
+
if (!expose) {
|
|
173
|
+
return response;
|
|
174
|
+
}
|
|
175
|
+
const headers = new Headers(response.headers);
|
|
176
|
+
headers.set("x-sdt-g4-attempt-id", attemptId);
|
|
177
|
+
return new Response(response.body, { status: response.status, headers });
|
|
178
|
+
}
|
|
179
|
+
/** Parse the complete V1 envelope before any Durable Object is contacted. */
|
|
180
|
+
export function validateCommitEnvelope(value, registeredEventParsers = {}) {
|
|
181
|
+
if (!isObject(value)) {
|
|
182
|
+
return { error: error(400, "malformed_commit_envelope", "Commit envelope must contain numeric version 1") };
|
|
183
|
+
}
|
|
184
|
+
const clientModelAliasMembers = ["candidates", "consistency"].filter((member) => Object.prototype.hasOwnProperty.call(value, member));
|
|
185
|
+
if (typeof value.version !== "number") {
|
|
186
|
+
if (clientModelAliasMembers.length > 0) {
|
|
187
|
+
return {
|
|
188
|
+
error: error(400, "malformed_commit_envelope", `Client-model member(s) ${clientModelAliasMembers.join(", ")} require the transport adapter; use version 1 with eventCandidates and consistencyTags on the V1 wire.`),
|
|
189
|
+
};
|
|
190
|
+
}
|
|
191
|
+
return { error: error(400, "malformed_commit_envelope", "Commit envelope must contain numeric version 1") };
|
|
192
|
+
}
|
|
193
|
+
if (value.version !== 1) {
|
|
194
|
+
return {
|
|
195
|
+
error: error(400, "unsupported_commit_envelope_version", "Only serialized commit envelope version 1 is supported"),
|
|
196
|
+
};
|
|
197
|
+
}
|
|
198
|
+
const missingV1ArrayMembers = ["eventCandidates", "consistencyTags"].filter((member) => !Object.prototype.hasOwnProperty.call(value, member));
|
|
199
|
+
if (missingV1ArrayMembers.length > 0 || clientModelAliasMembers.length > 0) {
|
|
200
|
+
const missing = missingV1ArrayMembers.length === 0
|
|
201
|
+
? ""
|
|
202
|
+
: `Missing required V1 member(s): ${missingV1ArrayMembers.join(", ")}. `;
|
|
203
|
+
const aliases = clientModelAliasMembers.length === 0
|
|
204
|
+
? ""
|
|
205
|
+
: `Client-model member(s) ${clientModelAliasMembers.join(", ")} require the transport adapter; use eventCandidates and consistencyTags on the V1 wire. `;
|
|
206
|
+
return { error: error(400, "malformed_commit_envelope", `${missing}${aliases}`.trim()) };
|
|
207
|
+
}
|
|
208
|
+
if (value.eventCandidates === undefined) {
|
|
209
|
+
return { error: error(400, "malformed_commit_envelope", "eventCandidates must be an array") };
|
|
210
|
+
}
|
|
211
|
+
if (value.consistencyTags === undefined) {
|
|
212
|
+
return { error: error(400, "malformed_commit_envelope", "consistencyTags must be an array") };
|
|
213
|
+
}
|
|
214
|
+
// The explicit required-member check above intentionally stays separate
|
|
215
|
+
// from these historical defaults. Its omission mutant restores the former
|
|
216
|
+
// fail-open behavior, while the shipped path still permits explicit []
|
|
217
|
+
// members exactly as the V1 contract does.
|
|
218
|
+
const rawCandidates = value.eventCandidates ?? [];
|
|
219
|
+
const rawConsistencyTags = value.consistencyTags ?? [];
|
|
220
|
+
if (!Array.isArray(rawCandidates)) {
|
|
221
|
+
return { error: error(400, "malformed_commit_envelope", "eventCandidates must be an array") };
|
|
222
|
+
}
|
|
223
|
+
if (!Array.isArray(rawConsistencyTags)) {
|
|
224
|
+
return { error: error(400, "malformed_commit_envelope", "consistencyTags must be an array") };
|
|
225
|
+
}
|
|
226
|
+
const eventCandidates = [];
|
|
227
|
+
for (const rawCandidate of rawCandidates) {
|
|
228
|
+
if (!isObject(rawCandidate)) {
|
|
229
|
+
return { error: error(400, "malformed_commit_envelope", "Each event candidate must be an object") };
|
|
230
|
+
}
|
|
231
|
+
if (typeof rawCandidate.payload !== "string" ||
|
|
232
|
+
!isStandardBase64(rawCandidate.payload) ||
|
|
233
|
+
!isNonEmptyString(rawCandidate.eventPayloadName) ||
|
|
234
|
+
!Array.isArray(rawCandidate.tags)) {
|
|
235
|
+
return {
|
|
236
|
+
error: error(400, "malformed_commit_envelope", "Each event candidate needs base64 payload, eventPayloadName, and tags"),
|
|
237
|
+
};
|
|
238
|
+
}
|
|
239
|
+
if (rawCandidate.tags.length === 0 ||
|
|
240
|
+
!rawCandidate.tags.every(isNonEmptyString) ||
|
|
241
|
+
new Set(rawCandidate.tags).size !== rawCandidate.tags.length) {
|
|
242
|
+
return { error: error(400, "validation_error", "Candidate tags must be unique non-empty strings") };
|
|
243
|
+
}
|
|
244
|
+
let eventType;
|
|
245
|
+
let payload;
|
|
246
|
+
try {
|
|
247
|
+
if (Object.prototype.hasOwnProperty.call(rawCandidate, "eventPayloadVersion")) {
|
|
248
|
+
throw new Error("V1 commit candidates must not contain eventPayloadVersion; the registered event definition is authoritative");
|
|
249
|
+
}
|
|
250
|
+
eventType = canonicalEventType(rawCandidate.eventPayloadName);
|
|
251
|
+
const decoded = decodeBase64Utf8Json(rawCandidate.payload);
|
|
252
|
+
assertNoPayloadDiscriminator(decoded.parsed);
|
|
253
|
+
const parser = registeredEventParsers[rawCandidate.eventPayloadName];
|
|
254
|
+
if (parser !== undefined) {
|
|
255
|
+
let registered;
|
|
256
|
+
try {
|
|
257
|
+
registered = parser(decoded.parsed);
|
|
258
|
+
}
|
|
259
|
+
catch {
|
|
260
|
+
throw new CommitPayloadAdmissionError("payload_case_mismatch", `Payload was rejected by registered ${rawCandidate.eventPayloadName} schema`);
|
|
261
|
+
}
|
|
262
|
+
assertExactMemberPaths(decoded.parsed, registered);
|
|
263
|
+
}
|
|
264
|
+
payload = decoded.text;
|
|
265
|
+
}
|
|
266
|
+
catch (identityError) {
|
|
267
|
+
if (identityError instanceof CommitPayloadAdmissionError) {
|
|
268
|
+
return { error: error(400, identityError.code, identityError.message) };
|
|
269
|
+
}
|
|
270
|
+
return { error: error(400, "invalid_event_identity", identityError instanceof Error ? identityError.message : "Event identity is invalid") };
|
|
271
|
+
}
|
|
272
|
+
eventCandidates.push({
|
|
273
|
+
payload,
|
|
274
|
+
eventPayloadName: rawCandidate.eventPayloadName,
|
|
275
|
+
eventType,
|
|
276
|
+
tags: [...rawCandidate.tags],
|
|
277
|
+
});
|
|
278
|
+
}
|
|
279
|
+
const allTags = unique(eventCandidates.flatMap((candidate) => candidate.tags));
|
|
280
|
+
const consistencyTags = [];
|
|
281
|
+
for (const rawTag of rawConsistencyTags) {
|
|
282
|
+
if (!isObject(rawTag) || !isNonEmptyString(rawTag.tag) || !("lastSortableUniqueId" in rawTag)) {
|
|
283
|
+
return { error: error(400, "malformed_commit_envelope", "Each consistency tag has a required V1 shape") };
|
|
284
|
+
}
|
|
285
|
+
if (rawTag.lastSortableUniqueId === null || typeof rawTag.lastSortableUniqueId !== "string") {
|
|
286
|
+
return {
|
|
287
|
+
error: error(400, "malformed_commit_envelope", "lastSortableUniqueId must be a non-null string; omit the entry for an unobserved tag"),
|
|
288
|
+
};
|
|
289
|
+
}
|
|
290
|
+
if (rawTag.lastSortableUniqueId !== "") {
|
|
291
|
+
try {
|
|
292
|
+
assertSortableUniqueId(rawTag.lastSortableUniqueId);
|
|
293
|
+
}
|
|
294
|
+
catch {
|
|
295
|
+
return { error: error(400, "invalid_sortable_unique_id", "lastSortableUniqueId must be a 30-digit SortableUniqueId") };
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
if (!allTags.includes(rawTag.tag)) {
|
|
299
|
+
return { error: error(400, "validation_error", "Each consistency tag must occur in an event candidate") };
|
|
300
|
+
}
|
|
301
|
+
consistencyTags.push({ tag: rawTag.tag, lastSortableUniqueId: rawTag.lastSortableUniqueId });
|
|
302
|
+
}
|
|
303
|
+
if (new Set(consistencyTags.map((entry) => entry.tag)).size !== consistencyTags.length) {
|
|
304
|
+
return { error: error(400, "validation_error", "Consistency tags must be unique") };
|
|
305
|
+
}
|
|
306
|
+
return { value: { eventCandidates, consistencyTags, allTags } };
|
|
307
|
+
}
|
|
308
|
+
/**
|
|
309
|
+
* Stateless request orchestrator. Every mutation is delegated to the durable
|
|
310
|
+
* actor that owns it; this class retains only one request's generated IDs.
|
|
311
|
+
*/
|
|
312
|
+
export class CommitWorker {
|
|
313
|
+
env;
|
|
314
|
+
serviceId;
|
|
315
|
+
hooks;
|
|
316
|
+
constructor(env, serviceId, hooks = {}) {
|
|
317
|
+
this.env = env;
|
|
318
|
+
this.serviceId = serviceId;
|
|
319
|
+
this.hooks = hooks;
|
|
320
|
+
}
|
|
321
|
+
async handle(request) {
|
|
322
|
+
const workerObservation = beginWorkerInvocationObservation();
|
|
323
|
+
// CF-Ray is provider-owned ingress identity. It is retained exclusively
|
|
324
|
+
// in sdt.observe/v1 after admission; neither V1 JSON nor durable/internal
|
|
325
|
+
// request shapes gain a diagnostic field.
|
|
326
|
+
const requestId = request.headers.get("cf-ray")?.trim();
|
|
327
|
+
const trace = new CommitTrace({
|
|
328
|
+
schema: "sdt.commit/v1",
|
|
329
|
+
correlationId: createTraceCorrelationId(),
|
|
330
|
+
serviceId: this.serviceId,
|
|
331
|
+
nativeTracing: this.hooks.nativeTracing,
|
|
332
|
+
sink: this.hooks.commitTraceSink,
|
|
333
|
+
clock: this.hooks.commitTraceClock,
|
|
334
|
+
provider: this.hooks.commitTraceProvider,
|
|
335
|
+
runtimeVerifier: runtimeCommitTraceVerifier,
|
|
336
|
+
diagnostics: {
|
|
337
|
+
"worker.isolate.id": workerObservation.isolateInstanceId,
|
|
338
|
+
"worker.isolate.first": workerObservation.firstInvocation,
|
|
339
|
+
},
|
|
340
|
+
});
|
|
341
|
+
return trace.root("S00", {
|
|
342
|
+
actorKey: `root:${this.serviceId}`,
|
|
343
|
+
// This is a trace-only observation generated at the handler boundary.
|
|
344
|
+
// It is intentionally not persisted, used for control, or returned on
|
|
345
|
+
// the V1 response.
|
|
346
|
+
attributes: { "activation.first": workerObservation.firstInvocation },
|
|
347
|
+
}, async (root) => {
|
|
348
|
+
let workerObservationEmitted = false;
|
|
349
|
+
let workerObservationEligible = false;
|
|
350
|
+
const state = {
|
|
351
|
+
trace,
|
|
352
|
+
scope: root,
|
|
353
|
+
enableWorkerObservation: () => {
|
|
354
|
+
workerObservationEligible = true;
|
|
355
|
+
},
|
|
356
|
+
emitWorkerObservation: () => {
|
|
357
|
+
if (!workerObservationEligible || workerObservationEmitted || requestId === undefined || requestId.length === 0)
|
|
358
|
+
return;
|
|
359
|
+
workerObservationEmitted = true;
|
|
360
|
+
// This is deliberately a separate structured Workers Logs event.
|
|
361
|
+
// It uses existing accepted identity without adding a span row,
|
|
362
|
+
// storage write, control input, or V1 response field.
|
|
363
|
+
observeWorkerInvocation({
|
|
364
|
+
...workerObservation,
|
|
365
|
+
requestId,
|
|
366
|
+
correlationId: trace.observationCorrelationId(),
|
|
367
|
+
emittedWorkerRowIds: trace.emittedWorkerRowIds(),
|
|
368
|
+
scriptVersion: this.hooks.commitTraceProvider?.scriptVersion,
|
|
369
|
+
colo: this.hooks.commitTraceProvider?.colo,
|
|
370
|
+
}, this.hooks.workerObservationSink);
|
|
371
|
+
},
|
|
372
|
+
};
|
|
373
|
+
try {
|
|
374
|
+
const response = await this.handleUntraced(request, state);
|
|
375
|
+
// The one Worker observation is emitted only after S15 has settled,
|
|
376
|
+
// so its inventory covers every native Worker span entered by this
|
|
377
|
+
// request. Its console transport remains observation-only and is not
|
|
378
|
+
// awaited as part of the V1 response.
|
|
379
|
+
return await state.scope.span("S15", { httpStatus: response.status }, async () => response);
|
|
380
|
+
}
|
|
381
|
+
finally {
|
|
382
|
+
state.emitWorkerObservation?.();
|
|
383
|
+
}
|
|
384
|
+
});
|
|
385
|
+
}
|
|
386
|
+
async handleUntraced(request, traceState) {
|
|
387
|
+
if (request.method !== "POST") {
|
|
388
|
+
return error(404, "commit_route_not_found", "Commit route requires POST");
|
|
389
|
+
}
|
|
390
|
+
const commandReceivedAt = Date.now();
|
|
391
|
+
const decodeAndValidate = async () => {
|
|
392
|
+
let body;
|
|
393
|
+
try {
|
|
394
|
+
body = await request.json();
|
|
395
|
+
}
|
|
396
|
+
catch {
|
|
397
|
+
return { error: error(400, "malformed_commit_envelope", "Commit envelope must be JSON") };
|
|
398
|
+
}
|
|
399
|
+
return validateCommitEnvelope(body, this.hooks.registeredEventParsers);
|
|
400
|
+
};
|
|
401
|
+
const validated = traceState === undefined
|
|
402
|
+
? await decodeAndValidate()
|
|
403
|
+
: await traceState.scope.span("S01", {}, async () => decodeAndValidate());
|
|
404
|
+
if ("error" in validated) {
|
|
405
|
+
return validated.error;
|
|
406
|
+
}
|
|
407
|
+
const startedAt = Date.now();
|
|
408
|
+
const input = validated.value;
|
|
409
|
+
if (input.eventCandidates.length === 0) {
|
|
410
|
+
return json({ writtenEvents: [], tagWriteResults: [], duration: durationSince(startedAt) });
|
|
411
|
+
}
|
|
412
|
+
const fault = faultFromRequest(request);
|
|
413
|
+
const attemptId = fault === undefined
|
|
414
|
+
? crypto.randomUUID()
|
|
415
|
+
: testAttemptIdFromRequest(request) ?? crypto.randomUUID();
|
|
416
|
+
if (traceState !== undefined) {
|
|
417
|
+
traceState.trace.markAccepted(attemptId);
|
|
418
|
+
traceState.scope = traceState.scope.accepted(attemptId);
|
|
419
|
+
traceState.enableWorkerObservation?.();
|
|
420
|
+
}
|
|
421
|
+
const writeTimestamp = writeTimestampUtc(startedAt);
|
|
422
|
+
const candidates = input.eventCandidates.map((candidate) => ({
|
|
423
|
+
...candidate,
|
|
424
|
+
eventId: createUuidV7(startedAt),
|
|
425
|
+
timestamp: writeTimestamp,
|
|
426
|
+
}));
|
|
427
|
+
const bootstrapEpoch = await this.bootstrapCommand("admit", attemptId, undefined, traceState?.scope, "S02");
|
|
428
|
+
if (bootstrapEpoch === undefined) {
|
|
429
|
+
return responseWithAttempt(error(409, "bootstrap_command_rejected", "Bootstrap is importing; commit is unavailable"), attemptId, fault !== undefined);
|
|
430
|
+
}
|
|
431
|
+
// Admission only linearizes the entry check. Do not retain it through
|
|
432
|
+
// remote work: PLANNED may start after in-flight work reaches zero, and
|
|
433
|
+
// the epoch check at the authoritative write is what fences that race.
|
|
434
|
+
if ((await this.bootstrapCommand("release", attemptId, bootstrapEpoch, traceState?.scope, "S03")) === undefined) {
|
|
435
|
+
return responseWithAttempt(error(409, "bootstrap_command_rejected", "Bootstrap admission could not be released"), attemptId, fault !== undefined);
|
|
436
|
+
}
|
|
437
|
+
// G41: there is no attempt-level Journal admission or state machine on
|
|
438
|
+
// this path. A tag reservation is the durable prepare fact and every
|
|
439
|
+
// tag's local append transaction is the commit fact. The retained S04 /
|
|
440
|
+
// S05 trace identities are compatibility observations only; they cannot
|
|
441
|
+
// resolve JOURNAL and are removed from the actual dependency path.
|
|
442
|
+
await this.retiredJournalMilestone("S04", traceState?.scope);
|
|
443
|
+
const reservations = await this.acquireReservations(input, attemptId, fault, traceState?.scope);
|
|
444
|
+
await this.retiredJournalMilestone("S05a", traceState?.scope, 0);
|
|
445
|
+
if (reservations.failure !== undefined) {
|
|
446
|
+
return this.finishReservationFailure([...reservations.successes.keys()], attemptId, reservations.failure, fault, traceState?.scope);
|
|
447
|
+
}
|
|
448
|
+
if (fault === "after-reservations-before-allocation") {
|
|
449
|
+
// Boundary 1: every durable prepare fact is force-tombstoned before
|
|
450
|
+
// returning the intentionally undetermined outcome. There is no
|
|
451
|
+
// attempt-level Journal recovery record to make this safe later.
|
|
452
|
+
await this.cancelReservations([...reservations.successes.keys()], attemptId, fault, traceState?.scope);
|
|
453
|
+
return this.noApplicationOutcome(attemptId, true);
|
|
454
|
+
}
|
|
455
|
+
await this.hooks.beforeBootstrapAllocation?.();
|
|
456
|
+
const allocation = await this.allocate(candidates, attemptId, fault, bootstrapEpoch, traceState?.scope);
|
|
457
|
+
if (allocation === undefined) {
|
|
458
|
+
return this.finishReservationFailure([...reservations.successes.keys()], attemptId, {
|
|
459
|
+
outcome: "FAILED",
|
|
460
|
+
failureCause: "allocator-failure",
|
|
461
|
+
reason: "allocator could not durably allocate the complete vector",
|
|
462
|
+
}, fault, traceState?.scope);
|
|
463
|
+
}
|
|
464
|
+
const allocatedCandidates = this.withAllocatedSuids(candidates, allocation);
|
|
465
|
+
if (allocatedCandidates === undefined) {
|
|
466
|
+
await this.cancelReservations([...reservations.successes.keys()], attemptId, fault, traceState?.scope);
|
|
467
|
+
return this.noApplicationOutcome(attemptId, fault !== undefined);
|
|
468
|
+
}
|
|
469
|
+
// Allocation supplies the exact EventId/SUID pair needed by the durable
|
|
470
|
+
// operational ledger. The timestamp is captured at the request boundary;
|
|
471
|
+
// the observer writes after the response path through waitUntil and never
|
|
472
|
+
// becomes an admission, ordering, or response dependency.
|
|
473
|
+
for (const candidate of allocatedCandidates) {
|
|
474
|
+
this.hooks.durableHopObserver?.observe({
|
|
475
|
+
stage: "command-receipt",
|
|
476
|
+
serviceId: this.serviceId,
|
|
477
|
+
eventId: candidate.eventId,
|
|
478
|
+
suid: candidate.suid,
|
|
479
|
+
attemptId,
|
|
480
|
+
observedAt: commandReceivedAt,
|
|
481
|
+
});
|
|
482
|
+
}
|
|
483
|
+
await this.retiredJournalMilestone("S05b", traceState?.scope, 1);
|
|
484
|
+
if (fault === "journal-cas-after-allocator") {
|
|
485
|
+
// This retained fault marks the allocation-to-first-append crash
|
|
486
|
+
// boundary. With no Journal alarm, tags receive the same best-effort
|
|
487
|
+
// tombstone barrier immediately and retain their own expiry alarm.
|
|
488
|
+
await this.cancelReservations(input.allTags, attemptId, fault, traceState?.scope);
|
|
489
|
+
return this.noApplicationOutcome(attemptId, true);
|
|
490
|
+
}
|
|
491
|
+
await this.retiredJournalMilestone("S05c", traceState?.scope, 2);
|
|
492
|
+
await this.hooks.beforeBootstrapFinalization?.();
|
|
493
|
+
// This is immediately before the first final authoritative tag mutation.
|
|
494
|
+
// The same service epoch obtained at admission must still be current.
|
|
495
|
+
if ((await this.bootstrapCommand("finalize", attemptId, bootstrapEpoch, traceState?.scope, "S10")) === undefined) {
|
|
496
|
+
await this.cancelReservations(input.allTags, attemptId, fault, traceState?.scope);
|
|
497
|
+
return responseWithAttempt(error(409, "bootstrap_command_rejected", "Bootstrap fencing epoch changed before commit write"), attemptId, fault !== undefined);
|
|
498
|
+
}
|
|
499
|
+
const writes = await this.appendAllTags(input, allocatedCandidates, attemptId, allocation.allocatorLineageId, reservations.successes, fault, traceState?.scope);
|
|
500
|
+
if (writes.pendingTags.length > 0) {
|
|
501
|
+
// Cancel is deliberately best effort: a cancellation error cannot
|
|
502
|
+
// replace the primary partial-write outcome or delete a committed row.
|
|
503
|
+
await this.cancelReservations(input.allTags, attemptId, fault, traceState?.scope);
|
|
504
|
+
// A missing participant must retain a tag-local, durable partial fact.
|
|
505
|
+
// This is the direct replacement for Journal reconciliation's
|
|
506
|
+
// /fence/install loop: G44 can discover the incomplete source universe
|
|
507
|
+
// without a delivery, and G45/G46 retain a readable fenced frontier.
|
|
508
|
+
if (!await this.installPartialWriteFences(writes.pendingTags, attemptId, traceState?.scope)) {
|
|
509
|
+
return this.noApplicationOutcome(attemptId, true);
|
|
510
|
+
}
|
|
511
|
+
if (writes.committedTags.size === 0 &&
|
|
512
|
+
writes.registrationUnavailableTags.length === writes.pendingTags.length) {
|
|
513
|
+
return responseWithAttempt(error(503, "partition_registration_unavailable", "Source partition registration is unavailable; retry the commit.", true), attemptId, fault !== undefined);
|
|
514
|
+
}
|
|
515
|
+
return this.partialWriteOutcome(input, allocatedCandidates, writes, attemptId, fault !== undefined);
|
|
516
|
+
}
|
|
517
|
+
await this.retiredJournalMilestone("S05d", traceState?.scope, 3);
|
|
518
|
+
if (fault === "sealing-after-cas") {
|
|
519
|
+
// The all-tags-written/response-lost boundary has no single terminal
|
|
520
|
+
// Journal response. Durable tag receipts and G44 reconciliation are
|
|
521
|
+
// now the detection authority.
|
|
522
|
+
return this.noApplicationOutcome(attemptId, true);
|
|
523
|
+
}
|
|
524
|
+
try {
|
|
525
|
+
const success = await this.successResponse(input, allocatedCandidates, attemptId, startedAt, fault, traceState?.scope);
|
|
526
|
+
return responseWithAttempt(withGlobalAdmission(json(success, 200), writes.globalAdmission), attemptId, fault !== undefined);
|
|
527
|
+
}
|
|
528
|
+
catch {
|
|
529
|
+
return responseWithAttempt(error(500, "internal_error", "Committed records could not be read while preparing the response"), attemptId, fault !== undefined);
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
tagFor(tag) {
|
|
533
|
+
return this.env.TAG.get(scopeIdFor(this.env.TAG, {
|
|
534
|
+
serviceId: this.serviceId,
|
|
535
|
+
doClass: "tag",
|
|
536
|
+
identity: tag,
|
|
537
|
+
}));
|
|
538
|
+
}
|
|
539
|
+
/**
|
|
540
|
+
* `sdt.commit/v1` still has the historical S04/S05 row identities in its
|
|
541
|
+
* separately-owned G30 manifest. G41 keeps those observations zero-work
|
|
542
|
+
* until that trace schema is revised by its owner, while proving that they
|
|
543
|
+
* cannot perform a JOURNAL namespace operation. These spans are not a
|
|
544
|
+
* substitute for a Journal record or terminal-outcome authority.
|
|
545
|
+
*/
|
|
546
|
+
async retiredJournalMilestone(rowId, traceScope, phaseOrdinal) {
|
|
547
|
+
if (traceScope === undefined)
|
|
548
|
+
return;
|
|
549
|
+
await traceScope.span(rowId, phaseOrdinal === undefined ? {} : { phaseOrdinal }, async () => undefined);
|
|
550
|
+
}
|
|
551
|
+
async bootstrapCommand(action, commandId, leaseEpoch, traceScope, rowId) {
|
|
552
|
+
if (this.env.BOOTSTRAP === undefined)
|
|
553
|
+
return 0;
|
|
554
|
+
const invoke = async () => {
|
|
555
|
+
const url = new URL(`https://commit-worker.internal/command/${action}`);
|
|
556
|
+
url.searchParams.set("__serviceId", this.serviceId);
|
|
557
|
+
const response = await this.env.BOOTSTRAP.get(scopeIdFor(this.env.BOOTSTRAP, {
|
|
558
|
+
serviceId: this.serviceId,
|
|
559
|
+
doClass: "bootstrap",
|
|
560
|
+
identity: "coordinator",
|
|
561
|
+
})).fetch(new Request(url, {
|
|
562
|
+
method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ commandId, ...(leaseEpoch === undefined ? {} : { leaseEpoch }) }),
|
|
563
|
+
}));
|
|
564
|
+
if (!response.ok)
|
|
565
|
+
return { response, leaseEpoch: undefined };
|
|
566
|
+
const body = await response.clone().json().catch(() => undefined);
|
|
567
|
+
return { response, leaseEpoch: typeof body?.leaseEpoch === "number" ? body.leaseEpoch : undefined };
|
|
568
|
+
};
|
|
569
|
+
const result = traceScope === undefined || rowId === undefined
|
|
570
|
+
? await invoke()
|
|
571
|
+
: await traceScope.span(rowId, {}, invoke);
|
|
572
|
+
return result.leaseEpoch;
|
|
573
|
+
}
|
|
574
|
+
async postJson(stub, path, body, traceScope, rowId, traceOptions = {}) {
|
|
575
|
+
const invoke = async () => {
|
|
576
|
+
const response = await stub.fetch(new Request(`https://commit-worker.internal${path}`, {
|
|
577
|
+
method: "POST",
|
|
578
|
+
headers: { "content-type": "application/json" },
|
|
579
|
+
body: JSON.stringify(body),
|
|
580
|
+
}));
|
|
581
|
+
let parsed;
|
|
582
|
+
try {
|
|
583
|
+
parsed = (await response.clone().json());
|
|
584
|
+
}
|
|
585
|
+
catch {
|
|
586
|
+
// Callers use the status first; malformed internal bodies are failures.
|
|
587
|
+
}
|
|
588
|
+
return { response, body: parsed };
|
|
589
|
+
};
|
|
590
|
+
return traceScope === undefined || rowId === undefined
|
|
591
|
+
? invoke()
|
|
592
|
+
: traceScope.span(rowId, traceOptions, invoke);
|
|
593
|
+
}
|
|
594
|
+
async tagRequest(tag, path, body, traceScope, rowId, traceOptions = {}) {
|
|
595
|
+
const invoke = async () => {
|
|
596
|
+
const url = new URL(`https://commit-worker.internal${path}`);
|
|
597
|
+
url.searchParams.set("__tag", tag);
|
|
598
|
+
url.searchParams.set("__serviceId", this.serviceId);
|
|
599
|
+
if (path === "/append" && this.hooks.domainDeliveryClass !== undefined) {
|
|
600
|
+
url.searchParams.set("__domainDeliveryClass", this.hooks.domainDeliveryClass);
|
|
601
|
+
}
|
|
602
|
+
return this.tagFor(tag).fetch(new Request(url.toString(), {
|
|
603
|
+
...(body === undefined ? {} : { method: "POST", body: JSON.stringify(body) }),
|
|
604
|
+
...(body === undefined ? {} : { headers: { "content-type": "application/json" } }),
|
|
605
|
+
}));
|
|
606
|
+
};
|
|
607
|
+
return traceScope === undefined || rowId === undefined
|
|
608
|
+
? invoke()
|
|
609
|
+
: traceScope.span(rowId, { tag, ...traceOptions }, invoke);
|
|
610
|
+
}
|
|
611
|
+
async acquireReservations(input, attemptId, fault, traceScope) {
|
|
612
|
+
const acquire = (stageScope) => Promise.allSettled(input.consistencyTags.map(async (entry) => ({
|
|
613
|
+
tag: entry.tag,
|
|
614
|
+
response: await this.tagRequest(entry.tag, "/acquire", {
|
|
615
|
+
attemptId,
|
|
616
|
+
epoch: INITIAL_OWNER_EPOCH,
|
|
617
|
+
eventTags: input.allTags,
|
|
618
|
+
consistencyTags: input.consistencyTags,
|
|
619
|
+
}, stageScope?.fork(), stageScope === undefined ? undefined : "S07", {
|
|
620
|
+
memberIndex: input.consistencyTags.findIndex((candidate) => candidate.tag === entry.tag),
|
|
621
|
+
attemptId,
|
|
622
|
+
}),
|
|
623
|
+
})));
|
|
624
|
+
const responses = traceScope === undefined
|
|
625
|
+
? await acquire()
|
|
626
|
+
: await traceScope.span("S06", {}, async (stage) => acquire(stage));
|
|
627
|
+
const successes = new Map();
|
|
628
|
+
let failure;
|
|
629
|
+
for (const result of responses) {
|
|
630
|
+
if (result.status === "rejected") {
|
|
631
|
+
failure ??= {
|
|
632
|
+
outcome: "FAILED",
|
|
633
|
+
failureCause: "reservation-timeout",
|
|
634
|
+
reason: "a consistency reservation provider did not settle successfully",
|
|
635
|
+
};
|
|
636
|
+
continue;
|
|
637
|
+
}
|
|
638
|
+
const { tag, response } = result.value;
|
|
639
|
+
const body = await response.clone().json().catch(() => undefined);
|
|
640
|
+
const reservation = isObject(body) && isObject(body.reservation) ? body.reservation : undefined;
|
|
641
|
+
if (response.status >= 200 &&
|
|
642
|
+
response.status < 300 &&
|
|
643
|
+
reservation !== undefined &&
|
|
644
|
+
isNonEmptyString(reservation.token)) {
|
|
645
|
+
successes.set(tag, { tag, reservationToken: reservation.token });
|
|
646
|
+
continue;
|
|
647
|
+
}
|
|
648
|
+
const code = isObject(body) && typeof body.code === "string" ? body.code : undefined;
|
|
649
|
+
if (response.status === 500 && code === "internal_error") {
|
|
650
|
+
failure ??= {
|
|
651
|
+
outcome: "FAILED",
|
|
652
|
+
failureCause: "guard-rejection",
|
|
653
|
+
reason: "a consistency tag is fenced while its durable state is reconciled",
|
|
654
|
+
};
|
|
655
|
+
continue;
|
|
656
|
+
}
|
|
657
|
+
const reason = isObject(body) && typeof body.reason === "string" ? body.reason : "reservation provider rejected the attempt";
|
|
658
|
+
const logical = reason === "consistency_head_mismatch" ||
|
|
659
|
+
reason === "consistency_head_mismatch_assert_empty" ||
|
|
660
|
+
reason === "active_reservation_conflict";
|
|
661
|
+
failure ??= logical
|
|
662
|
+
? { outcome: "REFUSED", failureCause: "reservation-conflict", reason }
|
|
663
|
+
: { outcome: "FAILED", failureCause: "reservation-timeout", reason };
|
|
664
|
+
}
|
|
665
|
+
if (fault === "reservation-delayed-success" && successes.size > 0) {
|
|
666
|
+
failure = {
|
|
667
|
+
outcome: "FAILED",
|
|
668
|
+
failureCause: "reservation-timeout",
|
|
669
|
+
reason: "simulated delayed reservation success after an unavailable response",
|
|
670
|
+
};
|
|
671
|
+
}
|
|
672
|
+
return { successes, failure };
|
|
673
|
+
}
|
|
674
|
+
async allocate(candidates, attemptId, fault, bootstrapEpoch, traceScope) {
|
|
675
|
+
const allocator = this.env.ALLOCATOR.get(scopeIdFor(this.env.ALLOCATOR, {
|
|
676
|
+
serviceId: this.serviceId,
|
|
677
|
+
doClass: "allocator",
|
|
678
|
+
identity: this.hooks.allocatorScopeIdentity ?? "allocator",
|
|
679
|
+
}));
|
|
680
|
+
try {
|
|
681
|
+
const result = await this.postJson(allocator, "/allocate", {
|
|
682
|
+
attemptId,
|
|
683
|
+
serviceId: this.serviceId,
|
|
684
|
+
bootstrapCommandId: attemptId,
|
|
685
|
+
bootstrapEpoch,
|
|
686
|
+
candidates: candidates.map((candidate, candidateIndex) => ({
|
|
687
|
+
candidateIndex,
|
|
688
|
+
eventId: candidate.eventId,
|
|
689
|
+
targetTags: candidate.tags,
|
|
690
|
+
pinnedWriterEpoch: INITIAL_OWNER_EPOCH,
|
|
691
|
+
})),
|
|
692
|
+
faultInjection: fault === "allocator-commit" ? "between-vector-and-watermark" : undefined,
|
|
693
|
+
}, traceScope, "S08");
|
|
694
|
+
if (result.response.status >= 200 && result.response.status < 300 && result.body !== undefined) {
|
|
695
|
+
return result.body;
|
|
696
|
+
}
|
|
697
|
+
}
|
|
698
|
+
catch {
|
|
699
|
+
// A lost response is not evidence that allocation did not commit.
|
|
700
|
+
}
|
|
701
|
+
return this.readAllocation(allocator, attemptId);
|
|
702
|
+
}
|
|
703
|
+
async readAllocation(allocator, attemptId) {
|
|
704
|
+
try {
|
|
705
|
+
const response = await allocator.fetch(new Request(`https://commit-worker.internal/attempts/${encodeURIComponent(attemptId)}`));
|
|
706
|
+
if (response.status !== 200) {
|
|
707
|
+
return undefined;
|
|
708
|
+
}
|
|
709
|
+
const vector = (await response.json());
|
|
710
|
+
return Array.isArray(vector.candidates) ? vector : undefined;
|
|
711
|
+
}
|
|
712
|
+
catch {
|
|
713
|
+
return undefined;
|
|
714
|
+
}
|
|
715
|
+
}
|
|
716
|
+
withAllocatedSuids(candidates, vector) {
|
|
717
|
+
if (vector.candidates.length !== candidates.length) {
|
|
718
|
+
return undefined;
|
|
719
|
+
}
|
|
720
|
+
const suids = new Map(vector.candidates.map((candidate) => [candidate.eventId, candidate.suid]));
|
|
721
|
+
const allocated = candidates.map((candidate) => {
|
|
722
|
+
const suid = suids.get(candidate.eventId);
|
|
723
|
+
return suid === undefined ? undefined : { ...candidate, suid };
|
|
724
|
+
});
|
|
725
|
+
return allocated.every((candidate) => candidate !== undefined)
|
|
726
|
+
? allocated
|
|
727
|
+
: undefined;
|
|
728
|
+
}
|
|
729
|
+
async appendAllTags(input, candidates, attemptId, allocatorLineageId, reservations, fault, traceScope) {
|
|
730
|
+
let pending = [...input.allTags];
|
|
731
|
+
const committedTags = new Set();
|
|
732
|
+
const lastFailureCodeByTag = new Map();
|
|
733
|
+
let globalAdmission = "admitted";
|
|
734
|
+
for (let attempt = 0; attempt < MAX_WRITE_ATTEMPTS && pending.length > 0; attempt += 1) {
|
|
735
|
+
const append = (stageScope) => Promise.allSettled(pending.map(async (tag) => {
|
|
736
|
+
const injectFault = fault === "tag-append-always" ||
|
|
737
|
+
((fault === "tag-append-last" ||
|
|
738
|
+
fault === "fence-not-durable" ||
|
|
739
|
+
fault === "fence-install-partial") &&
|
|
740
|
+
tag === input.allTags[input.allTags.length - 1]);
|
|
741
|
+
const response = await this.tagRequest(tag, "/append", {
|
|
742
|
+
attemptId,
|
|
743
|
+
epoch: INITIAL_OWNER_EPOCH,
|
|
744
|
+
allocatorLineageId,
|
|
745
|
+
reservationToken: reservations.get(tag)?.reservationToken,
|
|
746
|
+
candidates: candidates
|
|
747
|
+
.filter((candidate) => candidate.tags.includes(tag))
|
|
748
|
+
.map(({ eventId, suid, payload, eventType, tags, timestamp }) => ({
|
|
749
|
+
eventId,
|
|
750
|
+
suid,
|
|
751
|
+
payload,
|
|
752
|
+
eventType,
|
|
753
|
+
provenance: "g32",
|
|
754
|
+
eventTags: tags,
|
|
755
|
+
allocatorLineageId,
|
|
756
|
+
timestamp,
|
|
757
|
+
})),
|
|
758
|
+
faultInjection: injectFault ? "after-append-before-confirm" : undefined,
|
|
759
|
+
}, stageScope?.fork(), stageScope === undefined ? undefined : "S12", {
|
|
760
|
+
memberIndex: input.allTags.findIndex((candidate) => candidate === tag),
|
|
761
|
+
retryIndex: attempt,
|
|
762
|
+
attemptId,
|
|
763
|
+
});
|
|
764
|
+
const body = response.status === 503
|
|
765
|
+
? await response.clone().json().catch(() => undefined)
|
|
766
|
+
: undefined;
|
|
767
|
+
return {
|
|
768
|
+
tag,
|
|
769
|
+
success: response.status >= 200 && response.status < 300,
|
|
770
|
+
globalAdmission: readGlobalAdmission(response),
|
|
771
|
+
code: isObject(body) && typeof body.code === "string" ? body.code : undefined,
|
|
772
|
+
};
|
|
773
|
+
}));
|
|
774
|
+
const results = traceScope === undefined
|
|
775
|
+
? await append()
|
|
776
|
+
: await traceScope.span("S11", { retryIndex: attempt }, async (stage) => append(stage));
|
|
777
|
+
const nextPending = [];
|
|
778
|
+
for (const [index, result] of results.entries()) {
|
|
779
|
+
const tag = pending[index];
|
|
780
|
+
if (result.status === "fulfilled" && result.value.success) {
|
|
781
|
+
committedTags.add(tag);
|
|
782
|
+
lastFailureCodeByTag.delete(tag);
|
|
783
|
+
globalAdmission = mergeGlobalAdmission(globalAdmission, result.value.globalAdmission);
|
|
784
|
+
}
|
|
785
|
+
else {
|
|
786
|
+
lastFailureCodeByTag.set(tag, result.status === "fulfilled" ? result.value.code : undefined);
|
|
787
|
+
nextPending.push(tag);
|
|
788
|
+
}
|
|
789
|
+
}
|
|
790
|
+
pending = nextPending;
|
|
791
|
+
}
|
|
792
|
+
return {
|
|
793
|
+
committedTags,
|
|
794
|
+
pendingTags: pending,
|
|
795
|
+
registrationUnavailableTags: pending.filter((tag) => lastFailureCodeByTag.get(tag) === "partition_registration_unavailable"),
|
|
796
|
+
globalAdmission,
|
|
797
|
+
};
|
|
798
|
+
}
|
|
799
|
+
/**
|
|
800
|
+
* Settle the same force-tombstone barrier used by the durable reservation
|
|
801
|
+
* failure path. A concurrently-started admission can reject after one or
|
|
802
|
+
* more acquire calls have succeeded; in that branch there is no terminal
|
|
803
|
+
* Journal record to perform this cleanup on the caller's behalf.
|
|
804
|
+
*/
|
|
805
|
+
async cancelReservations(tags, attemptId, fault, traceScope) {
|
|
806
|
+
const cancel = (stageScope) => Promise.allSettled(tags.map(async (tag, memberIndex) => {
|
|
807
|
+
const response = await this.tagRequest(tag, "/cancel", {
|
|
808
|
+
attemptId,
|
|
809
|
+
epoch: INITIAL_OWNER_EPOCH,
|
|
810
|
+
forceTombstone: true,
|
|
811
|
+
}, stageScope?.fork(), stageScope === undefined ? undefined : "S19", { memberIndex, attemptId });
|
|
812
|
+
return response.status >= 200 && response.status < 300;
|
|
813
|
+
}));
|
|
814
|
+
const cancelled = traceScope === undefined
|
|
815
|
+
? await cancel()
|
|
816
|
+
: await traceScope.span("S18", {}, async (stage) => cancel(stage));
|
|
817
|
+
const failedTags = cancelled.flatMap((result, index) => result.status === "fulfilled" && result.value ? [] : [tags[index]]);
|
|
818
|
+
// A test-only fault models an acknowledgement loss after the target
|
|
819
|
+
// performed its durable tombstone write. Either way, cancellation is
|
|
820
|
+
// never allowed to replace the primary prepare/commit result.
|
|
821
|
+
if (fault === "tombstone-after-durable" && tags.length > 0 && !failedTags.includes(tags[0])) {
|
|
822
|
+
failedTags.push(tags[0]);
|
|
823
|
+
}
|
|
824
|
+
return Object.freeze({ attemptedTags: [...tags], failedTags });
|
|
825
|
+
}
|
|
826
|
+
async finishReservationFailure(tags, attemptId, failure, fault, traceScope) {
|
|
827
|
+
if (traceScope !== undefined) {
|
|
828
|
+
await traceScope.span("S17", {}, async () => undefined);
|
|
829
|
+
}
|
|
830
|
+
await this.cancelReservations(tags, attemptId, fault, traceScope);
|
|
831
|
+
await this.retiredJournalMilestone("S05e", traceScope, 4);
|
|
832
|
+
const response = failure.outcome === "REFUSED"
|
|
833
|
+
? error(400, "consistency_conflict", "serialized commit was refused by a consistency reservation")
|
|
834
|
+
: failure.failureCause === "reservation-timeout"
|
|
835
|
+
? error(504, "timeout", "serialized commit timed out")
|
|
836
|
+
: error(500, "internal_error", "serialized commit failed before writing requested records");
|
|
837
|
+
return responseWithAttempt(response, attemptId, fault !== undefined);
|
|
838
|
+
}
|
|
839
|
+
/**
|
|
840
|
+
* Persist the local partial-write fact for each tag whose append did not
|
|
841
|
+
* commit. This is not a response decoration: the Tag DO's fence is the
|
|
842
|
+
* source-side authority used by read and repair/reconciliation seams.
|
|
843
|
+
*/
|
|
844
|
+
async installPartialWriteFences(tags, attemptId, traceScope) {
|
|
845
|
+
const install = (stageScope) => Promise.allSettled(tags.map(async (tag) => {
|
|
846
|
+
const response = await this.tagRequest(tag, "/fence/install", {
|
|
847
|
+
attemptId,
|
|
848
|
+
epoch: INITIAL_OWNER_EPOCH,
|
|
849
|
+
reason: PARTIAL_WRITE_FENCE_REASON,
|
|
850
|
+
}, stageScope?.fork());
|
|
851
|
+
return response.status >= 200 && response.status < 300;
|
|
852
|
+
}));
|
|
853
|
+
// S20 remains the frozen G30 observation boundary, but now covers the
|
|
854
|
+
// actual tag-owned partial-fact transition rather than a Journal handoff.
|
|
855
|
+
const installed = traceScope === undefined
|
|
856
|
+
? await install()
|
|
857
|
+
: await traceScope.span("S20", {}, async (stage) => install(stage));
|
|
858
|
+
return installed.every((result) => result.status === "fulfilled" && result.value);
|
|
859
|
+
}
|
|
860
|
+
async partialWriteOutcome(input, candidates, writes, attemptId, exposeAttempt) {
|
|
861
|
+
const writtenEventIds = candidates
|
|
862
|
+
.filter((candidate) => candidate.tags.some((tag) => writes.committedTags.has(tag)))
|
|
863
|
+
.map((candidate) => candidate.eventId);
|
|
864
|
+
const failedEventIds = candidates
|
|
865
|
+
.filter((candidate) => !writtenEventIds.includes(candidate.eventId))
|
|
866
|
+
.map((candidate) => candidate.eventId);
|
|
867
|
+
return responseWithAttempt(json({
|
|
868
|
+
error: "serialized commit partially failed",
|
|
869
|
+
code: "partial_write",
|
|
870
|
+
partial: {
|
|
871
|
+
writtenEventIds,
|
|
872
|
+
failedEventIds,
|
|
873
|
+
writtenTags: input.allTags.filter((tag) => writes.committedTags.has(tag)),
|
|
874
|
+
missingTags: [...writes.pendingTags],
|
|
875
|
+
eventsDeleted: false,
|
|
876
|
+
retryable: false,
|
|
877
|
+
},
|
|
878
|
+
}, 500), attemptId, exposeAttempt);
|
|
879
|
+
}
|
|
880
|
+
async successResponse(input, candidates, attemptId, startedAt, fault, traceScope) {
|
|
881
|
+
const readStates = (stageScope) => Promise.all(input.allTags.map(async (tag, memberIndex) => {
|
|
882
|
+
if (fault === "tag-state-unavailable") {
|
|
883
|
+
throw new Error("Test fault made the committed tag state unavailable");
|
|
884
|
+
}
|
|
885
|
+
const response = await this.tagRequest(tag, "/head-facts", undefined, stageScope?.fork(), stageScope === undefined ? undefined : "S14", { memberIndex, attemptId });
|
|
886
|
+
if (response.status !== 200) {
|
|
887
|
+
throw new Error("Committed tag state was unavailable while preparing the response");
|
|
888
|
+
}
|
|
889
|
+
const body = (await response.json());
|
|
890
|
+
if (typeof body.version !== "number" || typeof body.updatedAt !== "string" || typeof body.head !== "string") {
|
|
891
|
+
throw new Error("Committed tag state was malformed while preparing the response");
|
|
892
|
+
}
|
|
893
|
+
return { tag, version: body.version, writtenAt: body.updatedAt };
|
|
894
|
+
}));
|
|
895
|
+
const tagWriteResults = traceScope === undefined
|
|
896
|
+
? await readStates()
|
|
897
|
+
: await traceScope.span("S13", {}, async (stage) => readStates(stage));
|
|
898
|
+
return {
|
|
899
|
+
writtenEvents: candidates.map((candidate) => ({
|
|
900
|
+
payload: encodeBase64Utf8(candidate.payload),
|
|
901
|
+
sortableUniqueIdValue: candidate.suid,
|
|
902
|
+
id: candidate.eventId,
|
|
903
|
+
eventMetadata: serializedEventMetadata(candidate.eventId),
|
|
904
|
+
tags: candidate.tags,
|
|
905
|
+
eventPayloadName: candidate.eventPayloadName,
|
|
906
|
+
})),
|
|
907
|
+
tagWriteResults,
|
|
908
|
+
duration: durationSince(startedAt),
|
|
909
|
+
};
|
|
910
|
+
}
|
|
911
|
+
/** The response was lost after tag-local facts may already be authoritative. */
|
|
912
|
+
noApplicationOutcome(attemptId, exposeAttempt) {
|
|
913
|
+
return responseWithAttempt(error(504, "timeout", OUTCOME_UNDETERMINED_ERROR), attemptId, exposeAttempt);
|
|
914
|
+
}
|
|
915
|
+
}
|
|
916
|
+
export async function handleSerializedCommit(request, env, hooks = {}) {
|
|
917
|
+
const serviceIdentity = hooks.serviceIdentityProvider ?? envServiceIdentity(env);
|
|
918
|
+
return new CommitWorker(env, requestServiceIdentity(request, serviceIdentity, {
|
|
919
|
+
allowG11Verification: env.G11_VERIFICATION_ENABLED === "true",
|
|
920
|
+
}), hooks).handle(request);
|
|
921
|
+
}
|