@sekiban/dcb-runtime 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +86 -0
- package/README.md +13 -0
- package/dist/allocator/AllocatorDurableObject.d.ts +36 -0
- package/dist/allocator/AllocatorDurableObject.js +658 -0
- package/dist/allocator/IssuanceLedger.d.ts +188 -0
- package/dist/allocator/IssuanceLedger.js +608 -0
- package/dist/allocator/IssuanceReconciler.d.ts +41 -0
- package/dist/allocator/IssuanceReconciler.js +191 -0
- package/dist/allocator/OrderClock.d.ts +41 -0
- package/dist/allocator/OrderClock.js +92 -0
- package/dist/allocator/SortableUniqueId.d.ts +46 -0
- package/dist/allocator/SortableUniqueId.js +156 -0
- package/dist/allocator/types.d.ts +52 -0
- package/dist/allocator/types.js +1 -0
- package/dist/bootstrap/BootstrapCoordinatorDurableObject.d.ts +28 -0
- package/dist/bootstrap/BootstrapCoordinatorDurableObject.js +357 -0
- package/dist/bootstrap/BootstrapStoreAdapter.d.ts +51 -0
- package/dist/bootstrap/BootstrapStoreAdapter.js +162 -0
- package/dist/bootstrap/OperatorBootstrap.d.ts +20 -0
- package/dist/bootstrap/OperatorBootstrap.js +104 -0
- package/dist/bootstrap/manifest.d.ts +9 -0
- package/dist/bootstrap/manifest.js +115 -0
- package/dist/bootstrap/types.d.ts +75 -0
- package/dist/bootstrap/types.js +1 -0
- package/dist/cli/OperatorRepairCli.d.ts +11 -0
- package/dist/cli/OperatorRepairCli.js +108 -0
- package/dist/cloudflare.d.ts +164 -0
- package/dist/cloudflare.js +19487 -0
- package/dist/commit/CommitWorker.d.ts +101 -0
- package/dist/commit/CommitWorker.js +921 -0
- package/dist/commit/types.d.ts +25 -0
- package/dist/commit/types.js +13 -0
- package/dist/completeness/GlobalCompletenessReconciler.d.ts +77 -0
- package/dist/completeness/GlobalCompletenessReconciler.js +572 -0
- package/dist/completeness/types.d.ts +85 -0
- package/dist/completeness/types.js +6 -0
- package/dist/composition.d.ts +123 -0
- package/dist/composition.js +237 -0
- package/dist/cosmos.d.ts +11 -0
- package/dist/cosmos.js +1411 -0
- package/dist/d1-mv.d.ts +8 -0
- package/dist/d1-mv.js +2001 -0
- package/dist/d1.d.ts +16 -0
- package/dist/d1.js +3418 -0
- package/dist/diagnostics/G60DurableHop.d.ts +99 -0
- package/dist/diagnostics/G60DurableHop.js +248 -0
- package/dist/diagnostics/G65Admission.d.ts +23 -0
- package/dist/diagnostics/G65Admission.js +43 -0
- package/dist/diagnostics/G65DirectRing.d.ts +33 -0
- package/dist/diagnostics/G65DirectRing.js +85 -0
- package/dist/diagnostics/G69AdmissionAttempt.d.ts +43 -0
- package/dist/diagnostics/G69AdmissionAttempt.js +22 -0
- package/dist/downstream/DeliveryCore.d.ts +127 -0
- package/dist/downstream/DeliveryCore.js +304 -0
- package/dist/downstream/Doorbell.d.ts +43 -0
- package/dist/downstream/Doorbell.js +137 -0
- package/dist/downstream/DownstreamAdapter.d.ts +52 -0
- package/dist/downstream/DownstreamAdapter.js +299 -0
- package/dist/downstream/ExclusionLookup.d.ts +36 -0
- package/dist/downstream/ExclusionLookup.js +50 -0
- package/dist/downstream/InconsistencyDetector.d.ts +27 -0
- package/dist/downstream/InconsistencyDetector.js +82 -0
- package/dist/downstream/OutboxDrain.d.ts +35 -0
- package/dist/downstream/OutboxDrain.js +126 -0
- package/dist/downstream/types.d.ts +77 -0
- package/dist/downstream/types.js +64 -0
- package/dist/eventIdentity.d.ts +30 -0
- package/dist/eventIdentity.js +49 -0
- package/dist/eventRecord.d.ts +26 -0
- package/dist/eventRecord.js +53 -0
- package/dist/g42-probe.d.ts +9 -0
- package/dist/g42-probe.js +9 -0
- package/dist/http/SerializedQueryWorker.d.ts +58 -0
- package/dist/http/SerializedQueryWorker.js +497 -0
- package/dist/http/commitResponse.d.ts +46 -0
- package/dist/http/commitResponse.js +80 -0
- package/dist/http/testServiceId.d.ts +17 -0
- package/dist/http/testServiceId.js +14 -0
- package/dist/index.d.ts +112 -0
- package/dist/index.js +19926 -0
- package/dist/journal/JournalDurableObject.d.ts +30 -0
- package/dist/journal/JournalDurableObject.js +481 -0
- package/dist/journal/JournalFirstTouchProbe.d.ts +190 -0
- package/dist/journal/JournalFirstTouchProbe.js +375 -0
- package/dist/journal/types.d.ts +116 -0
- package/dist/journal/types.js +52 -0
- package/dist/mv/MaterializedViewCatchUp.d.ts +60 -0
- package/dist/mv/MaterializedViewCatchUp.js +353 -0
- package/dist/mv/MaterializedViewRuntime.d.ts +80 -0
- package/dist/mv/MaterializedViewRuntime.js +328 -0
- package/dist/mv/MaterializedViewStore.d.ts +257 -0
- package/dist/mv/MaterializedViewStore.js +875 -0
- package/dist/mv/UnsafeWindowMaterializedView.d.ts +112 -0
- package/dist/mv/UnsafeWindowMaterializedView.js +495 -0
- package/dist/mv/index.d.ts +8 -0
- package/dist/mv/index.js +2357 -0
- package/dist/projection/LiveProjectionWorker.d.ts +86 -0
- package/dist/projection/LiveProjectionWorker.js +240 -0
- package/dist/projection/ProjectionRuntime.d.ts +113 -0
- package/dist/projection/ProjectionRuntime.js +346 -0
- package/dist/projection/ProjectorRegistry.d.ts +51 -0
- package/dist/projection/ProjectorRegistry.js +116 -0
- package/dist/query/ProjectionQueryStore.d.ts +102 -0
- package/dist/query/ProjectionQueryStore.js +185 -0
- package/dist/query/QueryRegistry.d.ts +22 -0
- package/dist/query/QueryRegistry.js +36 -0
- package/dist/read/SerializedReadWorker.d.ts +36 -0
- package/dist/read/SerializedReadWorker.js +225 -0
- package/dist/repair/RepairWorker.d.ts +72 -0
- package/dist/repair/RepairWorker.js +461 -0
- package/dist/safeWindow.d.ts +20 -0
- package/dist/safeWindow.js +36 -0
- package/dist/scope/ControlRouteScope.d.ts +22 -0
- package/dist/scope/ControlRouteScope.js +36 -0
- package/dist/scope/ScopeName.d.ts +34 -0
- package/dist/scope/ScopeName.js +67 -0
- package/dist/service/ServiceIdentityProvider.d.ts +35 -0
- package/dist/service/ServiceIdentityProvider.js +56 -0
- package/dist/store/CosmosEventStore.d.ts +126 -0
- package/dist/store/CosmosEventStore.js +950 -0
- package/dist/store/D1EventStore.d.ts +111 -0
- package/dist/store/D1EventStore.js +1111 -0
- package/dist/store/PostgresEventStore.d.ts +35 -0
- package/dist/store/PostgresEventStore.js +811 -0
- package/dist/store/provider.d.ts +25 -0
- package/dist/store/provider.js +32 -0
- package/dist/store/types.d.ts +205 -0
- package/dist/store/types.js +18 -0
- package/dist/tag/EventDigest.d.ts +40 -0
- package/dist/tag/EventDigest.js +158 -0
- package/dist/tag/TagDurableObject.d.ts +286 -0
- package/dist/tag/TagDurableObject.js +4071 -0
- package/dist/tag/TagSqlMeasurement.d.ts +26 -0
- package/dist/tag/TagSqlMeasurement.js +44 -0
- package/dist/tag/TagSqlSchema.d.ts +29 -0
- package/dist/tag/TagSqlSchema.js +211 -0
- package/dist/tag/types.d.ts +173 -0
- package/dist/tag/types.js +3 -0
- package/dist/tagstate/TagStateDurableObject.d.ts +57 -0
- package/dist/tagstate/TagStateDurableObject.js +499 -0
- package/dist/trace/AttributionRatio.d.ts +12 -0
- package/dist/trace/AttributionRatio.js +68 -0
- package/dist/trace/CloudflareTracing.d.ts +7 -0
- package/dist/trace/CloudflareTracing.js +9 -0
- package/dist/trace/CommitTrace.d.ts +405 -0
- package/dist/trace/CommitTrace.js +1008 -0
- package/dist/trace/CommitTraceConsoleSink.d.ts +68 -0
- package/dist/trace/CommitTraceConsoleSink.js +114 -0
- package/dist/trace/CommitTraceVerifier.d.ts +22 -0
- package/dist/trace/CommitTraceVerifier.js +295 -0
- package/dist/trace/ObservationStream.d.ts +118 -0
- package/dist/trace/ObservationStream.js +148 -0
- package/package.json +71 -0
|
@@ -0,0 +1,4071 @@
|
|
|
1
|
+
import { PARTIAL_WRITE_FENCE_REASON, RESERVATION_WINDOW_MS, SEGMENT_ROTATION_FENCE_REASON, } from "./types";
|
|
2
|
+
import { downstreamEnvelopeBytes, classifyDirectDoorbellFailure, readDomainDeliveryClass, preflightDirectDoorbell, readDirectDoorbellConfig, } from "../downstream/Doorbell";
|
|
3
|
+
import { deliveryCorrelationId } from "../downstream/DeliveryCore";
|
|
4
|
+
import { assertCanonicalEventType } from "../eventIdentity";
|
|
5
|
+
import { assertSortableUniqueId, compareSortableUniqueId } from "../allocator/SortableUniqueId";
|
|
6
|
+
import { CANONICAL_UTC_TIMESTAMP_PATTERN, isRfc4122Uuid, isUuidV7, serializedEventMetadata } from "../eventRecord";
|
|
7
|
+
import { DurableObjectActivation, enterNativeActorHandleSpan, noOpNativeTracing, } from "../trace/CommitTrace";
|
|
8
|
+
import { beginDurableObjectHandlerObservation, } from "../trace/ObservationStream";
|
|
9
|
+
import { canonicalDeclaredTagSet, eventDigestBytes, eventDigestHex } from "./EventDigest";
|
|
10
|
+
import { hasTagSqlStorage, initializeTagSqlSchema, TAG_READ_AFTER_SQL, TAG_READ_AFTER_THROUGH_SQL, } from "./TagSqlSchema";
|
|
11
|
+
import { G43SqlMeasurement } from "./TagSqlMeasurement";
|
|
12
|
+
import { recordDurableHop } from "../diagnostics/G60DurableHop";
|
|
13
|
+
import { recordG65AdmissionAttempt } from "../diagnostics/G65Admission";
|
|
14
|
+
import { D1EventStore } from "../store/D1EventStore";
|
|
15
|
+
const REPAIR_FACTS_KEY = "repair-facts";
|
|
16
|
+
/**
|
|
17
|
+
* Only the deliberately SQL-less direct-test seam uses this legacy-shaped
|
|
18
|
+
* delivery clock fixture. Deployed Tag DOs always use tag_outbox_obligation;
|
|
19
|
+
* this is never a second persisted event-record authority.
|
|
20
|
+
*/
|
|
21
|
+
const OUTBOX_DELIVERIES_KEY = "outbox-deliveries";
|
|
22
|
+
const MAX_EPOCH = Number.MAX_SAFE_INTEGER;
|
|
23
|
+
const DEFAULT_REPAIR_LEASE_MS = 30_000;
|
|
24
|
+
const MAX_REPAIR_LEASE_MS = 5 * 60_000;
|
|
25
|
+
const OBLIGATION_RETRY_MS = 1_000;
|
|
26
|
+
const OBLIGATION_MAX_ATTEMPTS = 3;
|
|
27
|
+
const OBLIGATION_ALARM_BATCH_LIMIT = 32;
|
|
28
|
+
/**
|
|
29
|
+
* G65 bounds derived writes, not the durable Tag commit. The value is short
|
|
30
|
+
* enough to keep the commit root near the G52 baseline while still allowing a
|
|
31
|
+
* healthy same-colo D1/doorbell attempt to complete before the response.
|
|
32
|
+
*/
|
|
33
|
+
export const G65_DERIVED_WRITE_BUDGET_MS = 300;
|
|
34
|
+
/**
|
|
35
|
+
* SDT-G98: first-append source-partition registration is on the commit path
|
|
36
|
+
* when G44 D1 is configured, but cold remote D1 registration routinely needs
|
|
37
|
+
* more than the doorbell/admission 300 ms budget. Keep registration on its own
|
|
38
|
+
* measured ceiling so multi-tag first-touch does not invent partial_write.
|
|
39
|
+
* Chosen from 2026-09-17 live cold-reserve observation (300 ms shared budget
|
|
40
|
+
* timed out registration) plus margin; refined against cold-loop evidence.
|
|
41
|
+
*/
|
|
42
|
+
export const G65_SOURCE_REGISTRATION_BUDGET_MS = 1_500;
|
|
43
|
+
export const G65_SOURCE_REGISTRATION_MAX_ATTEMPTS = 3;
|
|
44
|
+
export const G65_SOURCE_REGISTRATION_RETRY_DELAY_MS = 25;
|
|
45
|
+
export const G65_GLOBAL_ADMISSION_HEADER = "x-sdt-global-admission";
|
|
46
|
+
/** Schema identity is immutable for the lifetime of a Worker isolate. */
|
|
47
|
+
const g44GlobalArrayAuthorityByD1 = new WeakMap();
|
|
48
|
+
const g44GlobalArrayAuthorityResultByD1 = new WeakMap();
|
|
49
|
+
class AppendTransactionFault extends Error {
|
|
50
|
+
}
|
|
51
|
+
class PartitionRegistrationUnavailableError extends Error {
|
|
52
|
+
constructor(cause) {
|
|
53
|
+
super("partition_registration_unavailable");
|
|
54
|
+
this.name = "PartitionRegistrationUnavailableError";
|
|
55
|
+
this.cause = cause;
|
|
56
|
+
}
|
|
57
|
+
cause;
|
|
58
|
+
}
|
|
59
|
+
/** A mismatched immutable tag identity is a typed 409, not an internal error. */
|
|
60
|
+
export class TagIdentityConflict extends Error {
|
|
61
|
+
constructor() {
|
|
62
|
+
super("Tag Durable Object identity changed");
|
|
63
|
+
this.name = "TagIdentityConflict";
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
function json(body, status = 200) {
|
|
67
|
+
return new Response(JSON.stringify(body), {
|
|
68
|
+
status,
|
|
69
|
+
headers: { "content-type": "application/json; charset=utf-8" },
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
function error(status, code, message, retryable = false) {
|
|
73
|
+
return json(retryable ? { error: message, code, retryable: true } : { error: message, code }, status);
|
|
74
|
+
}
|
|
75
|
+
function withGlobalAdmission(response, status) {
|
|
76
|
+
// `json()` creates this Response locally, so its Headers are mutable. Keep
|
|
77
|
+
// the original object so the existing G60 response-order guard remains a
|
|
78
|
+
// direct assertion over the same response returned by the Tag DO.
|
|
79
|
+
response.headers.set(G65_GLOBAL_ADMISSION_HEADER, status);
|
|
80
|
+
return response;
|
|
81
|
+
}
|
|
82
|
+
function rejected(reason, status = 409) {
|
|
83
|
+
return {
|
|
84
|
+
status,
|
|
85
|
+
body: { code: "tag_operation_rejected", error: reason, reason },
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
const ASSERT_EMPTY_CONFLICT_REASON = "consistency_head_mismatch_assert_empty";
|
|
89
|
+
function repairRejected(reason, status = 409) {
|
|
90
|
+
return {
|
|
91
|
+
status,
|
|
92
|
+
body: { code: "repair_operation_rejected", error: reason, reason },
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
function isObject(value) {
|
|
96
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
97
|
+
}
|
|
98
|
+
function isNonEmptyString(value) {
|
|
99
|
+
return typeof value === "string" && value.length > 0;
|
|
100
|
+
}
|
|
101
|
+
function isEpoch(value) {
|
|
102
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
|
|
103
|
+
}
|
|
104
|
+
function isSafeInteger(value) {
|
|
105
|
+
return typeof value === "number" && Number.isSafeInteger(value);
|
|
106
|
+
}
|
|
107
|
+
function isNonNegativeInteger(value) {
|
|
108
|
+
return isSafeInteger(value) && value >= 0;
|
|
109
|
+
}
|
|
110
|
+
function sqlString(value, column) {
|
|
111
|
+
if (typeof value !== "string")
|
|
112
|
+
throw new Error(`Tag SQL column ${column} is not a string`);
|
|
113
|
+
return value;
|
|
114
|
+
}
|
|
115
|
+
function sqlNumber(value, column) {
|
|
116
|
+
if (typeof value !== "number" || !Number.isSafeInteger(value)) {
|
|
117
|
+
throw new Error(`Tag SQL column ${column} is not a safe integer`);
|
|
118
|
+
}
|
|
119
|
+
return value;
|
|
120
|
+
}
|
|
121
|
+
function sqlNullableNumber(value, column) {
|
|
122
|
+
return value === null || value === undefined ? null : sqlNumber(value, column);
|
|
123
|
+
}
|
|
124
|
+
function sqlNullableString(value, column) {
|
|
125
|
+
return value === null || value === undefined ? null : sqlString(value, column);
|
|
126
|
+
}
|
|
127
|
+
function sqlArrayBuffer(value, column) {
|
|
128
|
+
if (value instanceof ArrayBuffer)
|
|
129
|
+
return value;
|
|
130
|
+
if (ArrayBuffer.isView(value)) {
|
|
131
|
+
return value.buffer.slice(value.byteOffset, value.byteOffset + value.byteLength);
|
|
132
|
+
}
|
|
133
|
+
throw new Error(`Tag SQL column ${column} is not binary`);
|
|
134
|
+
}
|
|
135
|
+
function sqlJson(value, column) {
|
|
136
|
+
try {
|
|
137
|
+
return JSON.parse(sqlString(value, column));
|
|
138
|
+
}
|
|
139
|
+
catch {
|
|
140
|
+
throw new Error(`Tag SQL column ${column} is not valid JSON`);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
function copyArrayBuffer(bytes) {
|
|
144
|
+
return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
|
|
145
|
+
}
|
|
146
|
+
/** JSON envelope form of the exact G43 digest preimage stored as a BLOB. */
|
|
147
|
+
function arrayBufferBase64(value) {
|
|
148
|
+
let binary = "";
|
|
149
|
+
for (const byte of new Uint8Array(value))
|
|
150
|
+
binary += String.fromCharCode(byte);
|
|
151
|
+
return btoa(binary);
|
|
152
|
+
}
|
|
153
|
+
function nowIso() {
|
|
154
|
+
return new Date().toISOString();
|
|
155
|
+
}
|
|
156
|
+
function epochFrom(value) {
|
|
157
|
+
if (!isObject(value) || !isNonEmptyString(value.attemptId) || !isEpoch(value.epoch)) {
|
|
158
|
+
return { error: "attemptId and a non-negative safe-integer epoch are required" };
|
|
159
|
+
}
|
|
160
|
+
return { value: { attemptId: value.attemptId, epoch: value.epoch } };
|
|
161
|
+
}
|
|
162
|
+
function stringArrayFrom(value, name) {
|
|
163
|
+
if (!Array.isArray(value) || !value.every(isNonEmptyString)) {
|
|
164
|
+
return { error: `${name} must be an array of non-empty strings` };
|
|
165
|
+
}
|
|
166
|
+
if (new Set(value).size !== value.length) {
|
|
167
|
+
return { error: `${name} values must be unique` };
|
|
168
|
+
}
|
|
169
|
+
return { value: [...value] };
|
|
170
|
+
}
|
|
171
|
+
function consistencyTagsFrom(value) {
|
|
172
|
+
if (!Array.isArray(value)) {
|
|
173
|
+
return { error: "consistencyTags must be an array" };
|
|
174
|
+
}
|
|
175
|
+
const entries = [];
|
|
176
|
+
for (const rawEntry of value) {
|
|
177
|
+
if (!isObject(rawEntry) || !isNonEmptyString(rawEntry.tag)) {
|
|
178
|
+
return { error: "each consistency tag needs a non-empty tag" };
|
|
179
|
+
}
|
|
180
|
+
if (!("lastSortableUniqueId" in rawEntry)) {
|
|
181
|
+
return { error: "each consistency tag needs lastSortableUniqueId" };
|
|
182
|
+
}
|
|
183
|
+
if (rawEntry.lastSortableUniqueId === null) {
|
|
184
|
+
return { error: "lastSortableUniqueId must not be null" };
|
|
185
|
+
}
|
|
186
|
+
if (typeof rawEntry.lastSortableUniqueId !== "string") {
|
|
187
|
+
return { error: "lastSortableUniqueId must be a string" };
|
|
188
|
+
}
|
|
189
|
+
if (rawEntry.lastSortableUniqueId !== "") {
|
|
190
|
+
try {
|
|
191
|
+
assertSortableUniqueId(rawEntry.lastSortableUniqueId);
|
|
192
|
+
}
|
|
193
|
+
catch {
|
|
194
|
+
return { error: "lastSortableUniqueId must be a 30-digit SortableUniqueId" };
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
entries.push({ tag: rawEntry.tag, lastSortableUniqueId: rawEntry.lastSortableUniqueId });
|
|
198
|
+
}
|
|
199
|
+
if (new Set(entries.map((entry) => entry.tag)).size !== entries.length) {
|
|
200
|
+
return { error: "consistency tag values must be unique" };
|
|
201
|
+
}
|
|
202
|
+
return { value: entries };
|
|
203
|
+
}
|
|
204
|
+
function acquireFrom(value, tag) {
|
|
205
|
+
const epoch = epochFrom(value);
|
|
206
|
+
if (epoch.value === undefined || !isObject(value)) {
|
|
207
|
+
return { error: epoch.error ?? "Invalid acquire request" };
|
|
208
|
+
}
|
|
209
|
+
const eventTags = stringArrayFrom(value.eventTags, "eventTags");
|
|
210
|
+
const consistencyTags = consistencyTagsFrom(value.consistencyTags);
|
|
211
|
+
if (eventTags.value === undefined || consistencyTags.value === undefined) {
|
|
212
|
+
return { error: eventTags.error ?? consistencyTags.error ?? "Invalid acquire request" };
|
|
213
|
+
}
|
|
214
|
+
const matchingEntry = consistencyTags.value.find((entry) => entry.tag === tag);
|
|
215
|
+
if (matchingEntry !== undefined && !eventTags.value.includes(tag)) {
|
|
216
|
+
return { error: "a consistency tag must also be an event tag" };
|
|
217
|
+
}
|
|
218
|
+
return {
|
|
219
|
+
value: {
|
|
220
|
+
...epoch.value,
|
|
221
|
+
eventTags: eventTags.value,
|
|
222
|
+
consistencyTags: consistencyTags.value,
|
|
223
|
+
expectedHead: matchingEntry?.lastSortableUniqueId ?? null,
|
|
224
|
+
},
|
|
225
|
+
};
|
|
226
|
+
}
|
|
227
|
+
function reservationFrom(value) {
|
|
228
|
+
const epoch = epochFrom(value);
|
|
229
|
+
if (epoch.value === undefined || !isObject(value)) {
|
|
230
|
+
return { error: epoch.error ?? "Invalid reservation request" };
|
|
231
|
+
}
|
|
232
|
+
if (value.reservationToken !== undefined && !isNonEmptyString(value.reservationToken)) {
|
|
233
|
+
return { error: "reservationToken must be a non-empty string when present" };
|
|
234
|
+
}
|
|
235
|
+
if (value.forceTombstone !== undefined && typeof value.forceTombstone !== "boolean") {
|
|
236
|
+
return { error: "forceTombstone must be a boolean when present" };
|
|
237
|
+
}
|
|
238
|
+
return {
|
|
239
|
+
value: {
|
|
240
|
+
...epoch.value,
|
|
241
|
+
reservationToken: value.reservationToken,
|
|
242
|
+
forceTombstone: value.forceTombstone,
|
|
243
|
+
},
|
|
244
|
+
};
|
|
245
|
+
}
|
|
246
|
+
function appendFrom(value, tag) {
|
|
247
|
+
const reservation = reservationFrom(value);
|
|
248
|
+
if (reservation.value === undefined || !isObject(value) || !Array.isArray(value.candidates)) {
|
|
249
|
+
return { error: reservation.error ?? "candidates must be an array" };
|
|
250
|
+
}
|
|
251
|
+
if (value.candidates.length === 0) {
|
|
252
|
+
return { error: "an append requires at least one candidate" };
|
|
253
|
+
}
|
|
254
|
+
const candidates = [];
|
|
255
|
+
for (const rawCandidate of value.candidates) {
|
|
256
|
+
if (!isObject(rawCandidate) ||
|
|
257
|
+
!isNonEmptyString(rawCandidate.eventId) ||
|
|
258
|
+
!isNonEmptyString(rawCandidate.suid) ||
|
|
259
|
+
typeof rawCandidate.payload !== "string") {
|
|
260
|
+
return { error: "each candidate needs eventId, suid, and payload" };
|
|
261
|
+
}
|
|
262
|
+
const eventTags = stringArrayFrom(rawCandidate.eventTags, "candidate eventTags");
|
|
263
|
+
if (eventTags.value === undefined || !eventTags.value.includes(tag)) {
|
|
264
|
+
return { error: eventTags.error ?? "each candidate must include this event tag" };
|
|
265
|
+
}
|
|
266
|
+
let eventType;
|
|
267
|
+
try {
|
|
268
|
+
if (!isNonEmptyString(rawCandidate.eventType))
|
|
269
|
+
return { error: "candidate eventType is required" };
|
|
270
|
+
eventType = assertCanonicalEventType(rawCandidate.eventType).key;
|
|
271
|
+
assertSortableUniqueId(rawCandidate.suid);
|
|
272
|
+
if (!isUuidV7(rawCandidate.eventId))
|
|
273
|
+
return { error: "candidate eventId must be a UUID v7" };
|
|
274
|
+
JSON.parse(rawCandidate.payload);
|
|
275
|
+
}
|
|
276
|
+
catch (error) {
|
|
277
|
+
return { error: error instanceof Error ? error.message : "candidate identity or payload is invalid" };
|
|
278
|
+
}
|
|
279
|
+
if (rawCandidate.provenance !== "g32")
|
|
280
|
+
return { error: "candidate provenance must be g32" };
|
|
281
|
+
if (!isNonEmptyString(rawCandidate.allocatorLineageId))
|
|
282
|
+
return { error: "candidate allocatorLineageId is required" };
|
|
283
|
+
if (!isNonEmptyString(rawCandidate.timestamp) || !CANONICAL_UTC_TIMESTAMP_PATTERN.test(rawCandidate.timestamp)) {
|
|
284
|
+
return { error: "candidate timestamp must be canonical UTC" };
|
|
285
|
+
}
|
|
286
|
+
candidates.push({
|
|
287
|
+
eventId: rawCandidate.eventId,
|
|
288
|
+
suid: rawCandidate.suid,
|
|
289
|
+
payload: rawCandidate.payload,
|
|
290
|
+
eventTags: eventTags.value,
|
|
291
|
+
allocatorLineageId: rawCandidate.allocatorLineageId,
|
|
292
|
+
eventType,
|
|
293
|
+
provenance: "g32",
|
|
294
|
+
timestamp: rawCandidate.timestamp,
|
|
295
|
+
});
|
|
296
|
+
}
|
|
297
|
+
if (new Set(candidates.map((candidate) => candidate.eventId)).size !== candidates.length) {
|
|
298
|
+
return { error: "candidate eventId values must be unique" };
|
|
299
|
+
}
|
|
300
|
+
const ordered = [...candidates].sort((left, right) => left.suid < right.suid ? -1 : left.suid > right.suid ? 1 : 0);
|
|
301
|
+
if (ordered.some((candidate, index) => index > 0 && ordered[index - 1].suid === candidate.suid)) {
|
|
302
|
+
return { error: "candidate SUID values must be unique" };
|
|
303
|
+
}
|
|
304
|
+
const faultInjection = value.faultInjection;
|
|
305
|
+
if (faultInjection !== undefined && faultInjection !== "after-append-before-confirm") {
|
|
306
|
+
return { error: "unsupported faultInjection" };
|
|
307
|
+
}
|
|
308
|
+
return { value: { ...reservation.value, candidates: ordered, faultInjection } };
|
|
309
|
+
}
|
|
310
|
+
function bootstrapAppendFrom(value, tag) {
|
|
311
|
+
if (!isObject(value) || !isNonEmptyString(value.importId) || !isEpoch(value.leaseEpoch) || !isNonEmptyString(value.manifestDigest) || !isNonEmptyString(value.targetServiceId) || !Array.isArray(value.candidates) || value.candidates.length === 0)
|
|
312
|
+
return { error: "bootstrap identity and candidates are required" };
|
|
313
|
+
const candidates = [];
|
|
314
|
+
for (const candidate of value.candidates) {
|
|
315
|
+
if (!isObject(candidate) || !isNonEmptyString(candidate.eventId) || !isNonEmptyString(candidate.suid) || typeof candidate.payload !== "string")
|
|
316
|
+
return { error: "bootstrap candidate is invalid" };
|
|
317
|
+
const tags = stringArrayFrom(candidate.eventTags, "bootstrap candidate eventTags");
|
|
318
|
+
if (tags.value === undefined || !tags.value.includes(tag) || !isNonEmptyString(candidate.allocatorLineageId))
|
|
319
|
+
return { error: "bootstrap candidate must include this tag and lineage" };
|
|
320
|
+
let eventType;
|
|
321
|
+
try {
|
|
322
|
+
if (!isNonEmptyString(candidate.eventType))
|
|
323
|
+
return { error: "bootstrap candidate eventType is required" };
|
|
324
|
+
eventType = assertCanonicalEventType(candidate.eventType).key;
|
|
325
|
+
assertSortableUniqueId(candidate.suid);
|
|
326
|
+
// C# import records can predate UUID v7. This is confined to the
|
|
327
|
+
// coordinator-only bootstrap route; normal /append remains UUID v7.
|
|
328
|
+
if (!isRfc4122Uuid(candidate.eventId))
|
|
329
|
+
return { error: "bootstrap candidate eventId must be an RFC 4122 UUID" };
|
|
330
|
+
JSON.parse(candidate.payload);
|
|
331
|
+
}
|
|
332
|
+
catch (error) {
|
|
333
|
+
return { error: error instanceof Error ? error.message : "bootstrap candidate identity or payload is invalid" };
|
|
334
|
+
}
|
|
335
|
+
if (candidate.provenance !== "g32")
|
|
336
|
+
return { error: "bootstrap candidate provenance must be g32" };
|
|
337
|
+
if (!isNonEmptyString(candidate.timestamp) || !CANONICAL_UTC_TIMESTAMP_PATTERN.test(candidate.timestamp))
|
|
338
|
+
return { error: "bootstrap candidate timestamp must be canonical UTC" };
|
|
339
|
+
candidates.push({ eventId: candidate.eventId, suid: candidate.suid, payload: candidate.payload, eventTags: tags.value, allocatorLineageId: candidate.allocatorLineageId, eventType, provenance: "g32", timestamp: candidate.timestamp });
|
|
340
|
+
}
|
|
341
|
+
const ordered = [...candidates].sort((a, b) => a.suid < b.suid ? -1 : a.suid > b.suid ? 1 : 0);
|
|
342
|
+
if (ordered.some((candidate, index) => index > 0 && ordered[index - 1].suid === candidate.suid))
|
|
343
|
+
return { error: "bootstrap SUID values must be unique" };
|
|
344
|
+
return { value: { importId: value.importId, leaseEpoch: value.leaseEpoch, manifestDigest: value.manifestDigest, targetServiceId: value.targetServiceId, candidates: ordered } };
|
|
345
|
+
}
|
|
346
|
+
function fenceFrom(value) {
|
|
347
|
+
const epoch = epochFrom(value);
|
|
348
|
+
if (epoch.value === undefined || !isObject(value) || !isNonEmptyString(value.reason)) {
|
|
349
|
+
return { error: epoch.error ?? "reason is required" };
|
|
350
|
+
}
|
|
351
|
+
return { value: { ...epoch.value, reason: value.reason } };
|
|
352
|
+
}
|
|
353
|
+
function repairScopeItemFrom(value, tag) {
|
|
354
|
+
if (!isObject(value) ||
|
|
355
|
+
!isNonEmptyString(value.attemptId) ||
|
|
356
|
+
!isNonEmptyString(value.eventId) ||
|
|
357
|
+
!isNonEmptyString(value.suid) ||
|
|
358
|
+
typeof value.payload !== "string" ||
|
|
359
|
+
!isNonEmptyString(value.allocatorLineageId) ||
|
|
360
|
+
!isNonEmptyString(value.eventType) ||
|
|
361
|
+
value.provenance !== "g32" ||
|
|
362
|
+
!isNonEmptyString(value.timestamp)) {
|
|
363
|
+
return { error: "each repair scope item needs attemptId, eventId, suid, and payload" };
|
|
364
|
+
}
|
|
365
|
+
const eventTags = stringArrayFrom(value.eventTags, "repair eventTags");
|
|
366
|
+
if (eventTags.value === undefined || !eventTags.value.includes(tag)) {
|
|
367
|
+
return { error: eventTags.error ?? "each repair scope item must include this tag" };
|
|
368
|
+
}
|
|
369
|
+
try {
|
|
370
|
+
assertSortableUniqueId(value.suid);
|
|
371
|
+
if (!isUuidV7(value.eventId))
|
|
372
|
+
throw new Error("event id");
|
|
373
|
+
assertCanonicalEventType(value.eventType);
|
|
374
|
+
JSON.parse(value.payload);
|
|
375
|
+
}
|
|
376
|
+
catch {
|
|
377
|
+
return { error: "repair scope item has invalid G32 event identity" };
|
|
378
|
+
}
|
|
379
|
+
return {
|
|
380
|
+
value: {
|
|
381
|
+
attemptId: value.attemptId,
|
|
382
|
+
eventId: value.eventId,
|
|
383
|
+
suid: value.suid,
|
|
384
|
+
payload: value.payload,
|
|
385
|
+
eventTags: eventTags.value,
|
|
386
|
+
allocatorLineageId: value.allocatorLineageId,
|
|
387
|
+
eventType: value.eventType,
|
|
388
|
+
provenance: "g32",
|
|
389
|
+
timestamp: value.timestamp,
|
|
390
|
+
},
|
|
391
|
+
};
|
|
392
|
+
}
|
|
393
|
+
function repairScopeFrom(value, tag) {
|
|
394
|
+
if (!Array.isArray(value) || value.length === 0) {
|
|
395
|
+
return { error: "repair scope must be a non-empty array" };
|
|
396
|
+
}
|
|
397
|
+
const scope = [];
|
|
398
|
+
for (const raw of value) {
|
|
399
|
+
const parsed = repairScopeItemFrom(raw, tag);
|
|
400
|
+
if (parsed.value === undefined) {
|
|
401
|
+
return { error: parsed.error };
|
|
402
|
+
}
|
|
403
|
+
scope.push(parsed.value);
|
|
404
|
+
}
|
|
405
|
+
const keys = scope.map(repairScopeKey);
|
|
406
|
+
if (new Set(keys).size !== keys.length) {
|
|
407
|
+
return { error: "repair scope items must be unique" };
|
|
408
|
+
}
|
|
409
|
+
return { value: sortRepairScope(scope) };
|
|
410
|
+
}
|
|
411
|
+
function repairLeaseFrom(value) {
|
|
412
|
+
if (!isObject(value) || !isNonEmptyString(value.owner) || !isEpoch(value.epoch)) {
|
|
413
|
+
return { error: "repair owner and non-negative safe-integer epoch are required" };
|
|
414
|
+
}
|
|
415
|
+
return { value: { owner: value.owner, epoch: value.epoch } };
|
|
416
|
+
}
|
|
417
|
+
function repairAcquireFrom(value, tag) {
|
|
418
|
+
if (!isObject(value) || !isNonEmptyString(value.owner)) {
|
|
419
|
+
return { error: "repair owner is required" };
|
|
420
|
+
}
|
|
421
|
+
const leaseMs = value.leaseMs ?? DEFAULT_REPAIR_LEASE_MS;
|
|
422
|
+
if (!isSafeInteger(leaseMs) || leaseMs <= 0 || leaseMs > MAX_REPAIR_LEASE_MS) {
|
|
423
|
+
return { error: `leaseMs must be a positive safe integer no greater than ${MAX_REPAIR_LEASE_MS}` };
|
|
424
|
+
}
|
|
425
|
+
const scope = repairScopeFrom(value.scope, tag);
|
|
426
|
+
return scope.value === undefined
|
|
427
|
+
? { error: scope.error }
|
|
428
|
+
: { value: { owner: value.owner, leaseMs, scope: scope.value } };
|
|
429
|
+
}
|
|
430
|
+
function repairScopeUnionFrom(value, tag) {
|
|
431
|
+
const lease = repairLeaseFrom(value);
|
|
432
|
+
if (lease.value === undefined || !isObject(value)) {
|
|
433
|
+
return { error: lease.error };
|
|
434
|
+
}
|
|
435
|
+
const scope = repairScopeFrom(value.scope, tag);
|
|
436
|
+
return scope.value === undefined ? { error: scope.error } : { value: { ...lease.value, scope: scope.value } };
|
|
437
|
+
}
|
|
438
|
+
function repairApplyFrom(value, tag) {
|
|
439
|
+
const lease = repairLeaseFrom(value);
|
|
440
|
+
if (lease.value === undefined || !isObject(value)) {
|
|
441
|
+
return { error: lease.error };
|
|
442
|
+
}
|
|
443
|
+
const item = repairScopeItemFrom(value.item, tag);
|
|
444
|
+
return item.value === undefined ? { error: item.error } : { value: { ...lease.value, item: item.value } };
|
|
445
|
+
}
|
|
446
|
+
function repairAuditFrom(value, tag) {
|
|
447
|
+
const apply = repairApplyFrom(value, tag);
|
|
448
|
+
if (apply.value === undefined || !isObject(value) || !isNonEmptyString(value.actor)) {
|
|
449
|
+
return { error: apply.error ?? "audit actor is required" };
|
|
450
|
+
}
|
|
451
|
+
return { value: { ...apply.value, actor: value.actor } };
|
|
452
|
+
}
|
|
453
|
+
function repairClearFrom(value) {
|
|
454
|
+
const lease = repairLeaseFrom(value);
|
|
455
|
+
if (lease.value === undefined ||
|
|
456
|
+
!isObject(value) ||
|
|
457
|
+
!isNonEmptyString(value.attemptId) ||
|
|
458
|
+
!isNonNegativeInteger(value.scopeVersion)) {
|
|
459
|
+
return { error: lease.error ?? "attemptId and scopeVersion are required" };
|
|
460
|
+
}
|
|
461
|
+
return { value: { ...lease.value, attemptId: value.attemptId, scopeVersion: value.scopeVersion } };
|
|
462
|
+
}
|
|
463
|
+
function outboxPendingFrom(value) {
|
|
464
|
+
if (!isObject(value)) {
|
|
465
|
+
return { error: "outbox pending request must be an object" };
|
|
466
|
+
}
|
|
467
|
+
const nowMs = value.nowMs ?? Date.now();
|
|
468
|
+
if (!isSafeInteger(nowMs)) {
|
|
469
|
+
return { error: "nowMs must be a safe integer when present" };
|
|
470
|
+
}
|
|
471
|
+
const limit = value.limit;
|
|
472
|
+
if (limit !== undefined && (!isNonNegativeInteger(limit) || limit < 1 || limit > OBLIGATION_ALARM_BATCH_LIMIT)) {
|
|
473
|
+
return { error: `limit must be an integer in [1, ${OBLIGATION_ALARM_BATCH_LIMIT}] when present` };
|
|
474
|
+
}
|
|
475
|
+
const force = value.force;
|
|
476
|
+
if (force !== undefined && typeof force !== "boolean") {
|
|
477
|
+
return { error: "force must be boolean when present" };
|
|
478
|
+
}
|
|
479
|
+
return { value: { nowMs, limit, force } };
|
|
480
|
+
}
|
|
481
|
+
function outboxMarkFrom(value) {
|
|
482
|
+
if (!isObject(value) || !Array.isArray(value.deliveries) || value.deliveries.length === 0) {
|
|
483
|
+
return { error: "deliveries must be a non-empty array" };
|
|
484
|
+
}
|
|
485
|
+
const deliveries = [];
|
|
486
|
+
for (const raw of value.deliveries) {
|
|
487
|
+
if (!isObject(raw) ||
|
|
488
|
+
!isNonEmptyString(raw.attemptId) ||
|
|
489
|
+
!isNonEmptyString(raw.eventId) ||
|
|
490
|
+
!isNonEmptyString(raw.suid) ||
|
|
491
|
+
typeof raw.payload !== "string" ||
|
|
492
|
+
!isSafeInteger(raw.enqueuedAt)) {
|
|
493
|
+
return { error: "each delivery needs attemptId, eventId, suid, payload, and enqueuedAt" };
|
|
494
|
+
}
|
|
495
|
+
if (!isNonEmptyString(raw.eventType) || raw.provenance !== "g32" || !isNonEmptyString(raw.timestamp) || !isNonEmptyString(raw.allocatorLineageId))
|
|
496
|
+
return { error: "delivery must carry the complete G32 identity" };
|
|
497
|
+
if (!isObject(raw.completeness)) {
|
|
498
|
+
return { error: "delivery must carry a readable global receipt/membership join proof" };
|
|
499
|
+
}
|
|
500
|
+
const completeness = raw.completeness;
|
|
501
|
+
const obligationSequence = completeness.obligationSequence;
|
|
502
|
+
const memberships = completeness.localCommittedMembership;
|
|
503
|
+
if (!isNonEmptyString(completeness.eventDigest) || !isSafeInteger(obligationSequence) || obligationSequence < 1 ||
|
|
504
|
+
!Array.isArray(memberships) || memberships.length !== 1 || !isObject(memberships[0]) ||
|
|
505
|
+
memberships[0].eventId !== raw.eventId || typeof memberships[0].tag !== "string") {
|
|
506
|
+
return { error: "delivery must carry a readable global receipt/membership join proof" };
|
|
507
|
+
}
|
|
508
|
+
const membership = memberships[0];
|
|
509
|
+
const membershipTag = membership.tag;
|
|
510
|
+
if (typeof membershipTag !== "string")
|
|
511
|
+
return { error: "delivery must carry a readable global receipt/membership join proof" };
|
|
512
|
+
deliveries.push({
|
|
513
|
+
attemptId: raw.attemptId,
|
|
514
|
+
eventId: raw.eventId,
|
|
515
|
+
suid: raw.suid,
|
|
516
|
+
payload: raw.payload,
|
|
517
|
+
enqueuedAt: raw.enqueuedAt,
|
|
518
|
+
eventType: raw.eventType,
|
|
519
|
+
provenance: "g32",
|
|
520
|
+
timestamp: raw.timestamp,
|
|
521
|
+
allocatorLineageId: raw.allocatorLineageId,
|
|
522
|
+
completeness: {
|
|
523
|
+
eventDigest: completeness.eventDigest,
|
|
524
|
+
obligationSequence,
|
|
525
|
+
canonicalBytesBase64: typeof completeness.canonicalBytesBase64 === "string" ? completeness.canonicalBytesBase64 : "",
|
|
526
|
+
declaredTagSet: Array.isArray(completeness.declaredTagSet)
|
|
527
|
+
? completeness.declaredTagSet.filter((tag) => typeof tag === "string")
|
|
528
|
+
: [],
|
|
529
|
+
localCommittedMembership: [{
|
|
530
|
+
serviceId: typeof membership.serviceId === "string" ? membership.serviceId : "",
|
|
531
|
+
eventId: raw.eventId,
|
|
532
|
+
tag: membershipTag,
|
|
533
|
+
}],
|
|
534
|
+
},
|
|
535
|
+
});
|
|
536
|
+
}
|
|
537
|
+
if (new Set(deliveries.map(outboxRowKey)).size !== deliveries.length) {
|
|
538
|
+
return { error: "deliveries must be unique" };
|
|
539
|
+
}
|
|
540
|
+
const nowMs = value.nowMs ?? Date.now();
|
|
541
|
+
if (!isSafeInteger(nowMs)) {
|
|
542
|
+
return { error: "nowMs must be a safe integer when present" };
|
|
543
|
+
}
|
|
544
|
+
return { value: { deliveries, nowMs } };
|
|
545
|
+
}
|
|
546
|
+
function g44SourceScanFrom(value) {
|
|
547
|
+
if (!isObject(value) ||
|
|
548
|
+
!isNonEmptyString(value.serviceId) || !isNonEmptyString(value.tag) ||
|
|
549
|
+
!isSafeInteger(value.upperBoundSequence) || value.upperBoundSequence < 0 ||
|
|
550
|
+
!isSafeInteger(value.afterSequence) || value.afterSequence < 0 ||
|
|
551
|
+
!isSafeInteger(value.limit) || value.limit < 1 || value.limit > 256) {
|
|
552
|
+
return { error: "source scan needs serviceId, tag, and a bounded compound cursor" };
|
|
553
|
+
}
|
|
554
|
+
return {
|
|
555
|
+
value: {
|
|
556
|
+
serviceId: value.serviceId,
|
|
557
|
+
tag: value.tag,
|
|
558
|
+
upperBoundSequence: value.upperBoundSequence,
|
|
559
|
+
afterSequence: value.afterSequence,
|
|
560
|
+
limit: value.limit,
|
|
561
|
+
},
|
|
562
|
+
};
|
|
563
|
+
}
|
|
564
|
+
function g46TagStateSourceFrom(value) {
|
|
565
|
+
if (!isObject(value) || !isNonEmptyString(value.tag) || typeof value.cursor !== "string" ||
|
|
566
|
+
!isSafeInteger(value.limit) || value.limit < 1 || value.limit > 256 ||
|
|
567
|
+
(value.through !== undefined && typeof value.through !== "string")) {
|
|
568
|
+
return { error: "G46 source read needs tag, cursor, optional through, and a bounded limit" };
|
|
569
|
+
}
|
|
570
|
+
try {
|
|
571
|
+
if (value.cursor !== "")
|
|
572
|
+
assertSortableUniqueId(value.cursor);
|
|
573
|
+
if (value.through !== undefined && value.through !== "")
|
|
574
|
+
assertSortableUniqueId(value.through);
|
|
575
|
+
}
|
|
576
|
+
catch {
|
|
577
|
+
return { error: "G46 source cursor and frontier must be SortableUniqueIds" };
|
|
578
|
+
}
|
|
579
|
+
return {
|
|
580
|
+
value: {
|
|
581
|
+
tag: value.tag,
|
|
582
|
+
cursor: value.cursor,
|
|
583
|
+
limit: value.limit,
|
|
584
|
+
...(value.through === undefined ? {} : { through: value.through }),
|
|
585
|
+
},
|
|
586
|
+
};
|
|
587
|
+
}
|
|
588
|
+
function epochFor(entries, attemptId) {
|
|
589
|
+
return entries.find((entry) => entry.attemptId === attemptId)?.epoch;
|
|
590
|
+
}
|
|
591
|
+
function withMaxEpoch(entries, attemptId, epoch) {
|
|
592
|
+
const existing = epochFor(entries, attemptId);
|
|
593
|
+
if (existing === undefined) {
|
|
594
|
+
return [...entries, { attemptId, epoch }];
|
|
595
|
+
}
|
|
596
|
+
if (existing >= epoch) {
|
|
597
|
+
return entries;
|
|
598
|
+
}
|
|
599
|
+
return entries.map((entry) => (entry.attemptId === attemptId ? { ...entry, epoch } : entry));
|
|
600
|
+
}
|
|
601
|
+
function fenceFor(entries, reason, attemptId) {
|
|
602
|
+
return entries.find((entry) => entry.reason === reason && entry.attemptId === attemptId);
|
|
603
|
+
}
|
|
604
|
+
function withFence(entries, fence) {
|
|
605
|
+
const existing = fenceFor(entries, fence.reason, fence.attemptId);
|
|
606
|
+
if (existing === undefined) {
|
|
607
|
+
return [...entries, fence];
|
|
608
|
+
}
|
|
609
|
+
return entries.map((entry) => entry.reason === fence.reason && entry.attemptId === fence.attemptId ? fence : entry);
|
|
610
|
+
}
|
|
611
|
+
function withoutFence(entries, reason, attemptId) {
|
|
612
|
+
return entries.filter((entry) => entry.reason !== reason || entry.attemptId !== attemptId);
|
|
613
|
+
}
|
|
614
|
+
function outboxRowKey(row) {
|
|
615
|
+
return `${row.attemptId}\u0000${row.eventId}\u0000${row.suid}\u0000${row.payload}\u0000${row.eventType}\u0000${row.provenance}\u0000${row.timestamp}\u0000${row.allocatorLineageId}`;
|
|
616
|
+
}
|
|
617
|
+
function repairScopeKey(item) {
|
|
618
|
+
return `${item.attemptId}\u0000${item.eventId}\u0000${item.suid}`;
|
|
619
|
+
}
|
|
620
|
+
function sortRepairScope(scope) {
|
|
621
|
+
return [...scope].sort((left, right) => {
|
|
622
|
+
if (left.suid !== right.suid) {
|
|
623
|
+
return left.suid < right.suid ? -1 : 1;
|
|
624
|
+
}
|
|
625
|
+
const leftKey = repairScopeKey(left);
|
|
626
|
+
const rightKey = repairScopeKey(right);
|
|
627
|
+
return leftKey < rightKey ? -1 : leftKey > rightKey ? 1 : 0;
|
|
628
|
+
});
|
|
629
|
+
}
|
|
630
|
+
function unionRepairScope(existing, incoming) {
|
|
631
|
+
const byKey = new Map(existing.map((item) => [repairScopeKey(item), item]));
|
|
632
|
+
for (const item of incoming) {
|
|
633
|
+
const key = repairScopeKey(item);
|
|
634
|
+
const previous = byKey.get(key);
|
|
635
|
+
if (previous !== undefined && (previous.payload !== item.payload ||
|
|
636
|
+
previous.eventTags.join("\u0000") !== item.eventTags.join("\u0000"))) {
|
|
637
|
+
// A scope cannot silently change the durable candidate it authorizes.
|
|
638
|
+
throw new Error("Repair scope candidate identity conflict");
|
|
639
|
+
}
|
|
640
|
+
byKey.set(key, item);
|
|
641
|
+
}
|
|
642
|
+
return sortRepairScope([...byKey.values()]);
|
|
643
|
+
}
|
|
644
|
+
function repairScopeContains(scope, item) {
|
|
645
|
+
return scope.some((candidate) => repairScopeKey(candidate) === repairScopeKey(item) &&
|
|
646
|
+
candidate.payload === item.payload &&
|
|
647
|
+
candidate.eventTags.join("\u0000") === item.eventTags.join("\u0000"));
|
|
648
|
+
}
|
|
649
|
+
function repairFactsDefault() {
|
|
650
|
+
return { resolutions: [], audits: [] };
|
|
651
|
+
}
|
|
652
|
+
function repairResolutionFor(facts, item) {
|
|
653
|
+
return facts.resolutions.find((resolution) => resolution.attemptId === item.attemptId &&
|
|
654
|
+
resolution.eventId === item.eventId &&
|
|
655
|
+
resolution.suid === item.suid);
|
|
656
|
+
}
|
|
657
|
+
function repairAuditFor(facts, item) {
|
|
658
|
+
return facts.audits.find((audit) => audit.attemptId === item.attemptId && audit.eventId === item.eventId && audit.suid === item.suid);
|
|
659
|
+
}
|
|
660
|
+
function hasSegmentRotationFence(record) {
|
|
661
|
+
return record.fences.some((fence) => fence.reason === SEGMENT_ROTATION_FENCE_REASON);
|
|
662
|
+
}
|
|
663
|
+
function repairLeaseRejection(record, owner, epoch) {
|
|
664
|
+
if (hasSegmentRotationFence(record)) {
|
|
665
|
+
return "segment_rotation_fence_held";
|
|
666
|
+
}
|
|
667
|
+
if (record.repairOwner !== owner) {
|
|
668
|
+
return "repair_owner_mismatch";
|
|
669
|
+
}
|
|
670
|
+
if (record.highestRepairEpoch !== epoch) {
|
|
671
|
+
return epoch < record.highestRepairEpoch ? "stale_repair_epoch" : "repair_epoch_mismatch";
|
|
672
|
+
}
|
|
673
|
+
if (record.repairLeaseUntil === null || logicalNow(record) >= record.repairLeaseUntil) {
|
|
674
|
+
return "repair_lease_expired";
|
|
675
|
+
}
|
|
676
|
+
return undefined;
|
|
677
|
+
}
|
|
678
|
+
function latestFenceEpoch(record, reason, attemptId) {
|
|
679
|
+
const epochs = [
|
|
680
|
+
fenceFor(record.fences, reason, attemptId)?.epoch,
|
|
681
|
+
fenceFor(record.clearedFences, reason, attemptId)?.epoch,
|
|
682
|
+
].filter((epoch) => epoch !== undefined);
|
|
683
|
+
return epochs.length === 0 ? undefined : Math.max(...epochs);
|
|
684
|
+
}
|
|
685
|
+
function fenceEpochRejection(record, reason, attemptId, epoch) {
|
|
686
|
+
const latest = latestFenceEpoch(record, reason, attemptId);
|
|
687
|
+
return latest !== undefined && epoch < latest ? "stale_epoch" : undefined;
|
|
688
|
+
}
|
|
689
|
+
function newRecord(tag) {
|
|
690
|
+
const timestamp = nowIso();
|
|
691
|
+
return {
|
|
692
|
+
schemaVersion: 3,
|
|
693
|
+
tag,
|
|
694
|
+
head: "",
|
|
695
|
+
activeReservation: null,
|
|
696
|
+
alarmDueAt: null,
|
|
697
|
+
events: [],
|
|
698
|
+
outbox: [],
|
|
699
|
+
highestEpoch: [],
|
|
700
|
+
sealedEpoch: [],
|
|
701
|
+
tombstones: [],
|
|
702
|
+
confirmations: [],
|
|
703
|
+
fences: [],
|
|
704
|
+
clearedFences: [],
|
|
705
|
+
bootstrapAdmission: null,
|
|
706
|
+
repairOwner: null,
|
|
707
|
+
repairLeaseUntil: null,
|
|
708
|
+
highestRepairEpoch: 0,
|
|
709
|
+
repairScope: [],
|
|
710
|
+
repairScopeVersion: 0,
|
|
711
|
+
clockOffsetMs: 0,
|
|
712
|
+
clockNowMs: null,
|
|
713
|
+
version: 0,
|
|
714
|
+
createdAt: timestamp,
|
|
715
|
+
updatedAt: timestamp,
|
|
716
|
+
};
|
|
717
|
+
}
|
|
718
|
+
/** No pre-G32 Tag state is migrated or interpreted after the cutover. */
|
|
719
|
+
function requireG32TagRecord(record) {
|
|
720
|
+
if (record.schemaVersion !== 3) {
|
|
721
|
+
throw new Error("G32 Tag Durable Object requires a fresh cutover namespace");
|
|
722
|
+
}
|
|
723
|
+
for (const event of record.events) {
|
|
724
|
+
assertSortableUniqueId(event.suid);
|
|
725
|
+
if (!isRfc4122Uuid(event.eventId) || event.provenance !== "g32") {
|
|
726
|
+
throw new Error("G32 Tag Durable Object refuses a legacy durable event");
|
|
727
|
+
}
|
|
728
|
+
}
|
|
729
|
+
return record;
|
|
730
|
+
}
|
|
731
|
+
function changed(record, updates) {
|
|
732
|
+
return {
|
|
733
|
+
...record,
|
|
734
|
+
...updates,
|
|
735
|
+
version: record.version + 1,
|
|
736
|
+
updatedAt: nowIso(),
|
|
737
|
+
};
|
|
738
|
+
}
|
|
739
|
+
function logicalNow(record) {
|
|
740
|
+
return record.clockNowMs ?? Date.now() + record.clockOffsetMs;
|
|
741
|
+
}
|
|
742
|
+
function expireReservation(record) {
|
|
743
|
+
const active = record.activeReservation;
|
|
744
|
+
if (active === null || active.expiresAt > logicalNow(record)) {
|
|
745
|
+
return { record, expired: false };
|
|
746
|
+
}
|
|
747
|
+
return {
|
|
748
|
+
record: { ...record, activeReservation: null, alarmDueAt: null },
|
|
749
|
+
expired: true,
|
|
750
|
+
};
|
|
751
|
+
}
|
|
752
|
+
function advanceEpoch(record, attemptId, epoch) {
|
|
753
|
+
return {
|
|
754
|
+
...record,
|
|
755
|
+
highestEpoch: withMaxEpoch(record.highestEpoch, attemptId, epoch),
|
|
756
|
+
};
|
|
757
|
+
}
|
|
758
|
+
function basicEpochRejection(record, attemptId, epoch) {
|
|
759
|
+
const highest = epochFor(record.highestEpoch, attemptId);
|
|
760
|
+
return highest !== undefined && epoch < highest ? "stale_epoch" : undefined;
|
|
761
|
+
}
|
|
762
|
+
function operationEpochRejection(record, attemptId, epoch) {
|
|
763
|
+
const stale = basicEpochRejection(record, attemptId, epoch);
|
|
764
|
+
if (stale !== undefined) {
|
|
765
|
+
return stale;
|
|
766
|
+
}
|
|
767
|
+
const tombstone = epochFor(record.tombstones, attemptId);
|
|
768
|
+
if (tombstone !== undefined && epoch <= tombstone) {
|
|
769
|
+
return "tombstoned_epoch";
|
|
770
|
+
}
|
|
771
|
+
const sealed = epochFor(record.sealedEpoch, attemptId);
|
|
772
|
+
if (sealed !== undefined && epoch <= sealed) {
|
|
773
|
+
return "sealed_epoch";
|
|
774
|
+
}
|
|
775
|
+
return undefined;
|
|
776
|
+
}
|
|
777
|
+
function candidateIsExactDuplicate(record, attemptId, candidate) {
|
|
778
|
+
return record.events.some((event) => event.attemptId === attemptId &&
|
|
779
|
+
event.eventId === candidate.eventId &&
|
|
780
|
+
event.suid === candidate.suid &&
|
|
781
|
+
event.payload === candidate.payload &&
|
|
782
|
+
event.eventType === candidate.eventType &&
|
|
783
|
+
event.provenance === candidate.provenance &&
|
|
784
|
+
event.timestamp === candidate.timestamp);
|
|
785
|
+
}
|
|
786
|
+
function batchIsExactDuplicate(record, input) {
|
|
787
|
+
return input.candidates.every((candidate) => candidateIsExactDuplicate(record, input.attemptId, candidate));
|
|
788
|
+
}
|
|
789
|
+
function hasEventConflict(record, candidates) {
|
|
790
|
+
return candidates.some((candidate) => record.events.some((event) => event.eventId === candidate.eventId && !(event.suid === candidate.suid &&
|
|
791
|
+
event.payload === candidate.payload &&
|
|
792
|
+
event.eventType === candidate.eventType &&
|
|
793
|
+
event.provenance === candidate.provenance &&
|
|
794
|
+
event.timestamp === candidate.timestamp)));
|
|
795
|
+
}
|
|
796
|
+
function monotonicityViolation(head, candidates) {
|
|
797
|
+
let previous = head;
|
|
798
|
+
for (const candidate of candidates) {
|
|
799
|
+
if (candidate.suid <= previous) {
|
|
800
|
+
return true;
|
|
801
|
+
}
|
|
802
|
+
previous = candidate.suid;
|
|
803
|
+
}
|
|
804
|
+
return false;
|
|
805
|
+
}
|
|
806
|
+
/** Retained control-plane observability from G3; it never substitutes for the gate. */
|
|
807
|
+
function inspectFenceGate(record, attemptId) {
|
|
808
|
+
return {
|
|
809
|
+
checked: true,
|
|
810
|
+
activeFenceCount: record.fences.filter((fence) => fence.attemptId === attemptId).length,
|
|
811
|
+
};
|
|
812
|
+
}
|
|
813
|
+
/** The gate is closed by any durable fence, regardless of its owner. */
|
|
814
|
+
function isFenceGateClosed(record) {
|
|
815
|
+
return record.fences.length > 0;
|
|
816
|
+
}
|
|
817
|
+
function fenceGateRejected() {
|
|
818
|
+
return {
|
|
819
|
+
status: 500,
|
|
820
|
+
body: {
|
|
821
|
+
error: "Tag writes are unavailable while a durable repair fence is held",
|
|
822
|
+
code: "internal_error",
|
|
823
|
+
},
|
|
824
|
+
};
|
|
825
|
+
}
|
|
826
|
+
/**
|
|
827
|
+
* Rotation is intentionally only a fence-level interface in SDT-G5. A
|
|
828
|
+
* partial repair cannot jump an earlier rotation fence, and a held partial
|
|
829
|
+
* repair lease prevents a new rotation start. SDT-G6 owns rotation itself.
|
|
830
|
+
*/
|
|
831
|
+
function overlappingFenceReason(record, reason) {
|
|
832
|
+
if (reason === PARTIAL_WRITE_FENCE_REASON && record.fences.some((fence) => fence.reason === SEGMENT_ROTATION_FENCE_REASON)) {
|
|
833
|
+
return SEGMENT_ROTATION_FENCE_REASON;
|
|
834
|
+
}
|
|
835
|
+
if (reason === SEGMENT_ROTATION_FENCE_REASON && record.fences.some((fence) => fence.reason === PARTIAL_WRITE_FENCE_REASON)) {
|
|
836
|
+
return PARTIAL_WRITE_FENCE_REASON;
|
|
837
|
+
}
|
|
838
|
+
return undefined;
|
|
839
|
+
}
|
|
840
|
+
export class TagDurableObject {
|
|
841
|
+
ctx;
|
|
842
|
+
env;
|
|
843
|
+
nativeTracing;
|
|
844
|
+
/** Constructor-scoped observation only; never persisted or used for control. */
|
|
845
|
+
activation = new DurableObjectActivation();
|
|
846
|
+
/**
|
|
847
|
+
* Direct unit seams intentionally omit `storage.sql`. They exercise
|
|
848
|
+
* transport behavior only; deployed and Miniflare Tag DOs always use the
|
|
849
|
+
* normalized SQL path. Keeping this ephemeral fallback out of durable KV
|
|
850
|
+
* prevents a TAG_KEY record from remaining a second source of truth.
|
|
851
|
+
*/
|
|
852
|
+
fallbackRecord;
|
|
853
|
+
fallbackOutboxDeliveries = [];
|
|
854
|
+
/** Ephemeral test/evidence seam; never exposed as an HTTP route or stored. */
|
|
855
|
+
g43SqlMeasurement;
|
|
856
|
+
/** Exercises alarm crash boundaries without adding a production control path. */
|
|
857
|
+
g43SchedulerFault;
|
|
858
|
+
constructor(ctx, env, nativeTracing = noOpNativeTracing) {
|
|
859
|
+
this.ctx = ctx;
|
|
860
|
+
this.env = env;
|
|
861
|
+
this.nativeTracing = nativeTracing;
|
|
862
|
+
const sql = this.sqlStorage();
|
|
863
|
+
if (sql !== undefined)
|
|
864
|
+
initializeTagSqlSchema(sql);
|
|
865
|
+
}
|
|
866
|
+
async fetch(request) {
|
|
867
|
+
// Flip before this handler performs its first await.
|
|
868
|
+
const activation = this.activation.beginHandler();
|
|
869
|
+
const observation = beginDurableObjectHandlerObservation("TAG", activation);
|
|
870
|
+
const url = new URL(request.url);
|
|
871
|
+
const tag = url.searchParams.get("__tag");
|
|
872
|
+
if (!isNonEmptyString(tag)) {
|
|
873
|
+
return error(400, "tag_identity_required", "Tag identity is required");
|
|
874
|
+
}
|
|
875
|
+
const serviceId = url.searchParams.get("__serviceId");
|
|
876
|
+
if (request.method === "GET" && url.pathname === "/head-facts") {
|
|
877
|
+
return this.traceCommitReadActor(request, tag, serviceId, activation, observation, async () => {
|
|
878
|
+
observation.markFirstStorageRead();
|
|
879
|
+
try {
|
|
880
|
+
const facts = this.readHeadFacts(tag);
|
|
881
|
+
return facts === undefined
|
|
882
|
+
? error(404, "tag_not_found", "Tag has no durable state yet")
|
|
883
|
+
: json(facts);
|
|
884
|
+
}
|
|
885
|
+
catch (caught) {
|
|
886
|
+
if (caught instanceof TagIdentityConflict) {
|
|
887
|
+
return error(409, "tag_identity_conflict", "Tag Durable Object identity changed");
|
|
888
|
+
}
|
|
889
|
+
throw caught;
|
|
890
|
+
}
|
|
891
|
+
});
|
|
892
|
+
}
|
|
893
|
+
if (request.method === "GET" && url.pathname === "/state") {
|
|
894
|
+
return this.traceCommitReadActor(request, tag, serviceId, activation, observation, async () => {
|
|
895
|
+
observation.markFirstStorageRead();
|
|
896
|
+
const record = this.readStoredRecord(tag);
|
|
897
|
+
if (record === undefined) {
|
|
898
|
+
return error(404, "tag_not_found", "Tag has no durable state yet");
|
|
899
|
+
}
|
|
900
|
+
return record.tag === tag
|
|
901
|
+
? json(requireG32TagRecord(record))
|
|
902
|
+
: error(409, "tag_identity_conflict", "Tag Durable Object identity changed");
|
|
903
|
+
});
|
|
904
|
+
}
|
|
905
|
+
if (request.method === "GET" && url.pathname === "/repair/facts") {
|
|
906
|
+
return this.repairFacts(tag);
|
|
907
|
+
}
|
|
908
|
+
if (request.method === "POST" && url.pathname === "/debug/alarm") {
|
|
909
|
+
const record = await this.runAlarm();
|
|
910
|
+
return record === undefined ? error(404, "tag_not_found", "Tag has no durable state yet") : json(record);
|
|
911
|
+
}
|
|
912
|
+
if (request.method === "POST" && url.pathname === "/acquire") {
|
|
913
|
+
return this.traceCommitActor(request, tag, serviceId, activation, observation, (body) => this.acquire(tag, body, observation));
|
|
914
|
+
}
|
|
915
|
+
if (request.method === "POST" && url.pathname === "/cancel") {
|
|
916
|
+
return this.traceCommitActor(request, tag, serviceId, activation, observation, (body) => this.cancel(tag, body, observation));
|
|
917
|
+
}
|
|
918
|
+
if (request.method === "POST" && url.pathname === "/seal") {
|
|
919
|
+
return this.traceCommitActor(request, tag, serviceId, activation, observation, (body) => this.seal(tag, body, observation));
|
|
920
|
+
}
|
|
921
|
+
if (request.method === "POST" && url.pathname === "/append") {
|
|
922
|
+
return this.traceCommitActor(request, tag, serviceId, activation, observation, (body) => this.append(tag, body, serviceId, url.searchParams.get("__domainDeliveryClass") ?? undefined, observation));
|
|
923
|
+
}
|
|
924
|
+
const body = await this.jsonBody(request);
|
|
925
|
+
if (body === undefined) {
|
|
926
|
+
return error(400, "malformed_tag_request", "Request body must be JSON");
|
|
927
|
+
}
|
|
928
|
+
// The portable runtime cannot rely on the Cloudflare native-RPC class
|
|
929
|
+
// brand, so TagStateDO reaches this *direct DO* adapter by stub.fetch.
|
|
930
|
+
// It is deliberately not routed from the Worker and calls the one G43
|
|
931
|
+
// bounded source method below; /state and the linear rebuild stay out of
|
|
932
|
+
// the TagState call universe.
|
|
933
|
+
if (request.method === "POST" && url.pathname === "/__internal/g46/tag-state-incremental") {
|
|
934
|
+
if (request.headers.get("x-sdt-g46-source-read") !== "1") {
|
|
935
|
+
return error(403, "g46_source_read_forbidden", "G46 source reads require the TagState adapter");
|
|
936
|
+
}
|
|
937
|
+
const parsed = g46TagStateSourceFrom(body);
|
|
938
|
+
if (parsed.value === undefined || parsed.value.tag !== tag) {
|
|
939
|
+
return error(400, "g46_source_read_invalid", parsed.error ?? "G46 source read is invalid");
|
|
940
|
+
}
|
|
941
|
+
try {
|
|
942
|
+
return json(await this.g43TagStateIncrementalCatchUp(parsed.value));
|
|
943
|
+
}
|
|
944
|
+
catch (caught) {
|
|
945
|
+
if (caught instanceof TagIdentityConflict) {
|
|
946
|
+
return error(409, "tag_identity_conflict", "Tag Durable Object identity changed");
|
|
947
|
+
}
|
|
948
|
+
return error(503, "g46_source_read_unavailable", caught instanceof Error ? caught.message : "G46 source read is unavailable");
|
|
949
|
+
}
|
|
950
|
+
}
|
|
951
|
+
// This is an internal Tag-to-scanner seam, not a Worker route or a
|
|
952
|
+
// role-facing API. The scanner fixes its vector cursor from D1 before
|
|
953
|
+
// calling it; Queue state and public delivery endpoints never enter it.
|
|
954
|
+
if (request.method === "POST" && url.pathname === "/__internal/g44/source-obligations") {
|
|
955
|
+
if (request.headers.get("x-sdt-g44-source-scan") !== "1") {
|
|
956
|
+
return error(403, "g44_source_scan_forbidden", "G44 source scan requires the internal scanner seam");
|
|
957
|
+
}
|
|
958
|
+
const parsed = g44SourceScanFrom(body);
|
|
959
|
+
if (parsed.value === undefined || parsed.value.tag !== tag || parsed.value.serviceId !== serviceId) {
|
|
960
|
+
return error(400, "g44_source_scan_invalid", parsed.error ?? "G44 source scan identity is invalid");
|
|
961
|
+
}
|
|
962
|
+
try {
|
|
963
|
+
return json(await this.g44ReadSourceObligations(parsed.value));
|
|
964
|
+
}
|
|
965
|
+
catch (failure) {
|
|
966
|
+
return error(503, "g44_source_scan_unavailable", failure instanceof Error ? failure.message : "G44 source scan is unavailable");
|
|
967
|
+
}
|
|
968
|
+
}
|
|
969
|
+
if (request.method === "POST" && url.pathname === "/bootstrap/admit") {
|
|
970
|
+
return this.bootstrapAppend(tag, body, url.searchParams.get("__serviceId"));
|
|
971
|
+
}
|
|
972
|
+
if (request.method === "POST" && url.pathname === "/bootstrap/close") {
|
|
973
|
+
return this.closeBootstrap(tag, body);
|
|
974
|
+
}
|
|
975
|
+
if (request.method === "POST" && url.pathname === "/outbox/pending") {
|
|
976
|
+
return this.pendingOutbox(tag, url.searchParams.get("__serviceId"), body);
|
|
977
|
+
}
|
|
978
|
+
if (request.method === "POST" && url.pathname === "/outbox/mark-delivered") {
|
|
979
|
+
return this.markOutboxDelivered(tag, body);
|
|
980
|
+
}
|
|
981
|
+
if (request.method === "POST" && url.pathname === "/confirm") {
|
|
982
|
+
return this.confirm(tag, body);
|
|
983
|
+
}
|
|
984
|
+
if (request.method === "POST" && url.pathname === "/fence/install") {
|
|
985
|
+
return this.installFence(tag, body);
|
|
986
|
+
}
|
|
987
|
+
if (request.method === "POST" && url.pathname === "/fence/clear") {
|
|
988
|
+
return this.clearFence(tag, body);
|
|
989
|
+
}
|
|
990
|
+
if (request.method === "POST" && url.pathname === "/repair/acquire") {
|
|
991
|
+
return this.acquireRepairLease(tag, body);
|
|
992
|
+
}
|
|
993
|
+
if (request.method === "POST" && url.pathname === "/repair/renew") {
|
|
994
|
+
return this.renewRepairLease(tag, body);
|
|
995
|
+
}
|
|
996
|
+
if (request.method === "POST" && url.pathname === "/repair/scope-union") {
|
|
997
|
+
return this.unionRepairScope(tag, body);
|
|
998
|
+
}
|
|
999
|
+
if (request.method === "POST" && url.pathname === "/repair/apply") {
|
|
1000
|
+
return this.applyRepair(tag, body);
|
|
1001
|
+
}
|
|
1002
|
+
if (request.method === "POST" && url.pathname === "/repair/audit") {
|
|
1003
|
+
return this.auditRepair(tag, body);
|
|
1004
|
+
}
|
|
1005
|
+
if (request.method === "POST" && url.pathname === "/repair/clear") {
|
|
1006
|
+
return this.clearRepairFence(tag, body);
|
|
1007
|
+
}
|
|
1008
|
+
if (request.method === "POST" && url.pathname === "/debug/clock") {
|
|
1009
|
+
return this.setClockOffset(tag, body);
|
|
1010
|
+
}
|
|
1011
|
+
if (request.method === "POST" && url.pathname === "/__internal/g77/inspect-target") {
|
|
1012
|
+
if (!isNonEmptyString(serviceId)) {
|
|
1013
|
+
return error(400, "service_identity_required", "Service identity is required");
|
|
1014
|
+
}
|
|
1015
|
+
return this.inspectG77Target(serviceId, tag, body);
|
|
1016
|
+
}
|
|
1017
|
+
if (request.method === "POST" && url.pathname === "/__internal/g77/fence-absent-target") {
|
|
1018
|
+
return this.fenceAbsentG77Target(tag, body);
|
|
1019
|
+
}
|
|
1020
|
+
if (request.method === "POST" && url.pathname === "/__internal/g77/capabilities") {
|
|
1021
|
+
return json({ inspectTarget: true, fenceAbsentTarget: true });
|
|
1022
|
+
}
|
|
1023
|
+
return error(404, "tag_route_not_found", "Tag route was not found");
|
|
1024
|
+
}
|
|
1025
|
+
/** Identity-bound installed/absent inspection for G77 issuance reconciliation. */
|
|
1026
|
+
inspectG77Target(serviceId, tag, body) {
|
|
1027
|
+
if (!isObject(body) || !isNonEmptyString(body.eventId) || !isNonEmptyString(body.suid)) {
|
|
1028
|
+
return error(400, "invalid_inspect_target", "eventId and suid are required");
|
|
1029
|
+
}
|
|
1030
|
+
const pinnedWriterEpoch = isNonNegativeInteger(body.pinnedWriterEpoch) ? body.pinnedWriterEpoch : 0;
|
|
1031
|
+
const attemptId = isNonEmptyString(body.attemptId) ? body.attemptId : undefined;
|
|
1032
|
+
const sql = this.sqlStorage();
|
|
1033
|
+
if (sql === undefined)
|
|
1034
|
+
return error(503, "tag_sql_unavailable", "Tag SQL storage is unavailable");
|
|
1035
|
+
const identity = sql.exec("SELECT tag FROM tag_identity WHERE singleton = 1").toArray()[0];
|
|
1036
|
+
if (identity === undefined) {
|
|
1037
|
+
if (attemptId !== undefined) {
|
|
1038
|
+
return json({ inspectionStatus: "absent-never-contacted" });
|
|
1039
|
+
}
|
|
1040
|
+
return error(409, "target_not_terminal", "Target is not terminal under pinned writer authority");
|
|
1041
|
+
}
|
|
1042
|
+
if (sqlString(identity.tag, "tag_identity.tag") !== tag) {
|
|
1043
|
+
return error(409, "tag_identity_mismatch", "Tag identity does not match the inspected tag");
|
|
1044
|
+
}
|
|
1045
|
+
const event = sql.exec(`
|
|
1046
|
+
SELECT attempt_id, event_id, suid FROM tag_event
|
|
1047
|
+
WHERE service_id = ? AND event_id = ?
|
|
1048
|
+
`, serviceId, body.eventId).toArray()[0];
|
|
1049
|
+
if (event !== undefined &&
|
|
1050
|
+
sqlString(event.event_id, "tag_event.event_id") === body.eventId &&
|
|
1051
|
+
sqlString(event.suid, "tag_event.suid") === body.suid) {
|
|
1052
|
+
const obligation = sql.exec(`
|
|
1053
|
+
SELECT event_digest FROM tag_outbox_obligation
|
|
1054
|
+
WHERE service_id = ? AND event_id = ? ORDER BY obligation_sequence ASC LIMIT 1
|
|
1055
|
+
`, serviceId, body.eventId).toArray()[0];
|
|
1056
|
+
if (obligation !== undefined) {
|
|
1057
|
+
const obligationDigest = sqlString(obligation.event_digest, "tag_outbox_obligation.event_digest");
|
|
1058
|
+
return json({
|
|
1059
|
+
terminalStatus: "installed-and-covered",
|
|
1060
|
+
eventId: body.eventId,
|
|
1061
|
+
obligationDigest,
|
|
1062
|
+
});
|
|
1063
|
+
}
|
|
1064
|
+
return error(409, "target_not_terminal", "Target append is in flight under pinned writer authority");
|
|
1065
|
+
}
|
|
1066
|
+
if (attemptId !== undefined) {
|
|
1067
|
+
const tombstone = sql.exec("SELECT epoch FROM tag_tombstone WHERE attempt_id = ?", attemptId).toArray()[0];
|
|
1068
|
+
const tombstoneEpoch = tombstone === undefined ? undefined : sqlNumber(tombstone.epoch, "tag_tombstone.epoch");
|
|
1069
|
+
if (tombstoneEpoch !== undefined && pinnedWriterEpoch <= tombstoneEpoch) {
|
|
1070
|
+
return json({ terminalStatus: "absent-and-irrevocably-fenced", tombstoneEpoch });
|
|
1071
|
+
}
|
|
1072
|
+
const reservation = sql.exec("SELECT attempt_id, expires_at FROM tag_reservation WHERE singleton = 1").toArray()[0];
|
|
1073
|
+
if (reservation !== undefined &&
|
|
1074
|
+
sqlString(reservation.attempt_id, "tag_reservation.attempt_id") === attemptId) {
|
|
1075
|
+
return error(409, "target_not_terminal", "Active reservation holds writer authority");
|
|
1076
|
+
}
|
|
1077
|
+
const epoch = sql.exec("SELECT attempt_id FROM tag_epoch WHERE attempt_id = ?", attemptId).toArray()[0];
|
|
1078
|
+
if (epoch !== undefined) {
|
|
1079
|
+
return json({ inspectionStatus: "absent-but-unfenced" });
|
|
1080
|
+
}
|
|
1081
|
+
return json({ inspectionStatus: "absent-never-contacted" });
|
|
1082
|
+
}
|
|
1083
|
+
return error(409, "target_not_terminal", "Target is not terminal under pinned writer authority");
|
|
1084
|
+
}
|
|
1085
|
+
/** Reconciler-only fence for never-contacted targets; creates durable tombstone state. */
|
|
1086
|
+
async fenceAbsentG77Target(tag, body) {
|
|
1087
|
+
const parsed = reservationFrom(body);
|
|
1088
|
+
if (parsed.value === undefined) {
|
|
1089
|
+
return error(400, "invalid_fence_target", parsed.error ?? "attemptId and epoch are required");
|
|
1090
|
+
}
|
|
1091
|
+
this.ensureSqlTag(tag);
|
|
1092
|
+
const result = await this.cancelSql(tag, { ...parsed.value, forceTombstone: true });
|
|
1093
|
+
return result.status >= 200 && result.status < 300
|
|
1094
|
+
? json(result.body)
|
|
1095
|
+
: error(result.status, "fence_target_rejected", "Absent target fence was rejected");
|
|
1096
|
+
}
|
|
1097
|
+
async alarm() {
|
|
1098
|
+
this.activation.beginHandler();
|
|
1099
|
+
await this.runAlarm();
|
|
1100
|
+
}
|
|
1101
|
+
/**
|
|
1102
|
+
* Structural-measurement seam used around a real handler request. The
|
|
1103
|
+
* production wire cannot start it, and it neither changes control flow nor
|
|
1104
|
+
* persists instrumentation. Completion is intentionally separate so the
|
|
1105
|
+
* caller can first consume the response body (the specified boundary).
|
|
1106
|
+
*/
|
|
1107
|
+
beginG43SqlMeasurement() {
|
|
1108
|
+
if (this.g43SqlMeasurement !== undefined)
|
|
1109
|
+
throw new Error("G43 SQL measurement is already active");
|
|
1110
|
+
this.g43SqlMeasurement = new G43SqlMeasurement();
|
|
1111
|
+
}
|
|
1112
|
+
completeG43SqlMeasurement() {
|
|
1113
|
+
const measurement = this.g43SqlMeasurement;
|
|
1114
|
+
if (measurement === undefined)
|
|
1115
|
+
throw new Error("G43 SQL measurement is not active");
|
|
1116
|
+
this.g43SqlMeasurement = undefined;
|
|
1117
|
+
return measurement.complete();
|
|
1118
|
+
}
|
|
1119
|
+
/** Test-only fault seam for AC6's transactional re-arm crash boundaries. */
|
|
1120
|
+
setG43SchedulerFaultForTest(fault) {
|
|
1121
|
+
this.g43SchedulerFault = fault;
|
|
1122
|
+
}
|
|
1123
|
+
/**
|
|
1124
|
+
* Bounded source seam for the normal incremental TagState source read. Its
|
|
1125
|
+
* direct-DO adapter above is transport-only; this method remains the one
|
|
1126
|
+
* authority for cursor range reads and frozen-frontier completion.
|
|
1127
|
+
*/
|
|
1128
|
+
async g43ReadAfter(sortableUniqueId, limit, through) {
|
|
1129
|
+
if (!Number.isSafeInteger(limit) || limit < 1)
|
|
1130
|
+
throw new Error("G43 readAfter limit must be a positive safe integer");
|
|
1131
|
+
if (sortableUniqueId !== "")
|
|
1132
|
+
assertSortableUniqueId(sortableUniqueId);
|
|
1133
|
+
if (through !== undefined && through !== "")
|
|
1134
|
+
assertSortableUniqueId(through);
|
|
1135
|
+
if (through !== undefined && sortableUniqueId !== "" && through !== "" && compareSortableUniqueId(sortableUniqueId, through) > 0) {
|
|
1136
|
+
throw new Error("G43 readAfter cursor cannot exceed its frozen frontier");
|
|
1137
|
+
}
|
|
1138
|
+
const sql = this.sqlStorage();
|
|
1139
|
+
if (sql === undefined)
|
|
1140
|
+
return [];
|
|
1141
|
+
if (through === "")
|
|
1142
|
+
return [];
|
|
1143
|
+
return sql.exec(through === undefined ? TAG_READ_AFTER_SQL : TAG_READ_AFTER_THROUGH_SQL, ...(through === undefined ? [sortableUniqueId, limit] : [sortableUniqueId, through, limit]))
|
|
1144
|
+
.toArray()
|
|
1145
|
+
.map((row) => sqlJson(row.event_json, "tag_event.event_json"));
|
|
1146
|
+
}
|
|
1147
|
+
/**
|
|
1148
|
+
* The bounded normal projection source operation: it reads only events
|
|
1149
|
+
* newer than the supplied checkpoint and folds the fixed result window in
|
|
1150
|
+
* memory. No existing history is reread.
|
|
1151
|
+
*/
|
|
1152
|
+
async g43TagStateIncrementalCatchUp(input) {
|
|
1153
|
+
if (!Number.isSafeInteger(input.limit) || input.limit < 1) {
|
|
1154
|
+
throw new Error("G43 TagState incremental limit must be a positive safe integer");
|
|
1155
|
+
}
|
|
1156
|
+
if (input.cursor !== "")
|
|
1157
|
+
assertSortableUniqueId(input.cursor);
|
|
1158
|
+
if (input.through !== undefined && input.through !== "")
|
|
1159
|
+
assertSortableUniqueId(input.through);
|
|
1160
|
+
// Always read scalar head facts at this boundary. The first page uses the
|
|
1161
|
+
// checked head as its immutable frontier; later pages keep checking the
|
|
1162
|
+
// same identity/control/head invariant without moving that frontier as
|
|
1163
|
+
// concurrent appends advance the current Tag head.
|
|
1164
|
+
const facts = this.readHeadFacts(input.tag);
|
|
1165
|
+
if (facts === undefined && input.through !== undefined && input.through !== "") {
|
|
1166
|
+
throw new Error("G43 TagState source disappeared before its frozen frontier completed");
|
|
1167
|
+
}
|
|
1168
|
+
const through = input.through ?? facts?.head ?? "";
|
|
1169
|
+
if (facts !== undefined && through !== "" && facts.head !== "" && compareSortableUniqueId(facts.head, through) < 0) {
|
|
1170
|
+
throw new Error("G43 TagState frozen frontier is ahead of Tag head facts");
|
|
1171
|
+
}
|
|
1172
|
+
if (through !== "" && input.cursor !== "" && compareSortableUniqueId(input.cursor, through) > 0) {
|
|
1173
|
+
throw new Error("G43 TagState cursor cannot exceed its frozen frontier");
|
|
1174
|
+
}
|
|
1175
|
+
const events = await this.g43ReadAfter(input.cursor, input.limit, through);
|
|
1176
|
+
let lastSortableUniqueId = input.cursor;
|
|
1177
|
+
for (const event of events)
|
|
1178
|
+
lastSortableUniqueId = event.suid;
|
|
1179
|
+
const complete = events.length < input.limit || lastSortableUniqueId === through;
|
|
1180
|
+
return {
|
|
1181
|
+
events,
|
|
1182
|
+
lastSortableUniqueId,
|
|
1183
|
+
through,
|
|
1184
|
+
completeThrough: complete ? through : null,
|
|
1185
|
+
};
|
|
1186
|
+
}
|
|
1187
|
+
/** Full replay remains intentionally linear and is measurement-only info. */
|
|
1188
|
+
async g43TagStateRebuild() {
|
|
1189
|
+
const sql = this.sqlStorage();
|
|
1190
|
+
if (sql === undefined)
|
|
1191
|
+
return [];
|
|
1192
|
+
return sql.exec("SELECT event_json FROM tag_event ORDER BY suid ASC")
|
|
1193
|
+
.toArray()
|
|
1194
|
+
.map((row) => sqlJson(row.event_json, "tag_event.event_json"));
|
|
1195
|
+
}
|
|
1196
|
+
/**
|
|
1197
|
+
* Internal source-authority scanner for the G43 obligation seam. This is
|
|
1198
|
+
* intentionally a DO RPC surface rather than a Worker route: it adds no
|
|
1199
|
+
* role-facing HTTP API, and it does not share Queue/doorbell failure paths.
|
|
1200
|
+
*/
|
|
1201
|
+
async g43ScanSourceObligations(tag, nowMs) {
|
|
1202
|
+
if (!isSafeInteger(nowMs))
|
|
1203
|
+
throw new Error("G43 source scan time must be a safe integer");
|
|
1204
|
+
const response = await this.scanOutboxObligations(tag, nowMs);
|
|
1205
|
+
if (!response.ok)
|
|
1206
|
+
throw new Error(`G43 source scan failed with ${response.status}`);
|
|
1207
|
+
return response.json();
|
|
1208
|
+
}
|
|
1209
|
+
/**
|
|
1210
|
+
* G44's private source scanner RPC. It exposes only the normalized source
|
|
1211
|
+
* obligation table, never Queue state or a role-facing Worker route. The
|
|
1212
|
+
* caller fixes `upperBoundSequence` from the source partition registry
|
|
1213
|
+
* before paging, so later commits cannot move a scan frontier mid-pass.
|
|
1214
|
+
*/
|
|
1215
|
+
async g44ReadSourceObligations(input) {
|
|
1216
|
+
if (!isNonEmptyString(input.serviceId) || !isNonEmptyString(input.tag) ||
|
|
1217
|
+
!isSafeInteger(input.upperBoundSequence) || input.upperBoundSequence < 0 ||
|
|
1218
|
+
!isSafeInteger(input.afterSequence) || input.afterSequence < 0 ||
|
|
1219
|
+
!isSafeInteger(input.limit) || input.limit < 1 || input.limit > 256) {
|
|
1220
|
+
throw new Error("g44_source_scan_invalid_cursor");
|
|
1221
|
+
}
|
|
1222
|
+
const sql = this.sqlStorage();
|
|
1223
|
+
if (sql === undefined)
|
|
1224
|
+
throw new Error("g44_source_scan_sql_unavailable");
|
|
1225
|
+
const identity = sql.exec("SELECT tag FROM tag_identity WHERE singleton = 1").toArray()[0];
|
|
1226
|
+
if (identity === undefined || sqlString(identity.tag, "tag_identity.tag") !== input.tag) {
|
|
1227
|
+
throw new Error("g44_source_scan_partition_unreadable");
|
|
1228
|
+
}
|
|
1229
|
+
const maxRow = sql.exec("SELECT MAX(obligation_sequence) AS sequence FROM tag_outbox_obligation WHERE service_id = ?", input.serviceId).toArray()[0];
|
|
1230
|
+
const observedMaxSequence = maxRow === undefined || maxRow.sequence === null
|
|
1231
|
+
? 0
|
|
1232
|
+
: sqlNumber(maxRow.sequence, "tag_outbox_obligation.max_sequence");
|
|
1233
|
+
if (observedMaxSequence < input.upperBoundSequence) {
|
|
1234
|
+
throw new Error("g44_source_scan_snapshot_unreadable");
|
|
1235
|
+
}
|
|
1236
|
+
const rawRows = sql.exec(`
|
|
1237
|
+
SELECT obligation_sequence, event_id, event_digest, canonical_bytes,
|
|
1238
|
+
declared_tag_set_json, local_committed_membership_json, status
|
|
1239
|
+
FROM tag_outbox_obligation
|
|
1240
|
+
WHERE service_id = ?
|
|
1241
|
+
AND obligation_sequence > ?
|
|
1242
|
+
AND obligation_sequence <= ?
|
|
1243
|
+
ORDER BY obligation_sequence ASC
|
|
1244
|
+
LIMIT ?
|
|
1245
|
+
`, input.serviceId, input.afterSequence, input.upperBoundSequence, input.limit + 1).toArray();
|
|
1246
|
+
const hasMore = rawRows.length > input.limit;
|
|
1247
|
+
const rows = rawRows.slice(0, input.limit).map((row) => ({
|
|
1248
|
+
obligationSequence: sqlNumber(row.obligation_sequence, "tag_outbox_obligation.obligation_sequence"),
|
|
1249
|
+
eventId: sqlString(row.event_id, "tag_outbox_obligation.event_id"),
|
|
1250
|
+
eventDigest: sqlString(row.event_digest, "tag_outbox_obligation.event_digest"),
|
|
1251
|
+
canonicalBytesBase64: arrayBufferBase64(sqlArrayBuffer(row.canonical_bytes, "tag_outbox_obligation.canonical_bytes")),
|
|
1252
|
+
declaredTagSet: sqlJson(row.declared_tag_set_json, "tag_outbox_obligation.declared_tag_set_json"),
|
|
1253
|
+
localCommittedMembership: sqlJson(row.local_committed_membership_json, "tag_outbox_obligation.local_committed_membership_json"),
|
|
1254
|
+
status: sqlString(row.status, "tag_outbox_obligation.status"),
|
|
1255
|
+
}));
|
|
1256
|
+
return {
|
|
1257
|
+
serviceId: input.serviceId,
|
|
1258
|
+
tag: input.tag,
|
|
1259
|
+
upperBoundSequence: input.upperBoundSequence,
|
|
1260
|
+
observedMaxSequence,
|
|
1261
|
+
afterSequence: input.afterSequence,
|
|
1262
|
+
rows,
|
|
1263
|
+
hasMore,
|
|
1264
|
+
};
|
|
1265
|
+
}
|
|
1266
|
+
/**
|
|
1267
|
+
* G44's source acknowledgement guard. A transport success or an unjoined
|
|
1268
|
+
* receipt is deliberately insufficient: the global event, local membership
|
|
1269
|
+
* and exact source-obligation receipt must all be readable together.
|
|
1270
|
+
*/
|
|
1271
|
+
async globalReceiptMatches(delivery) {
|
|
1272
|
+
// The generic runtime's SQLite test harness is not a global D1 event
|
|
1273
|
+
// array at all. Every actual G32/G44 source has the `dcb_events` shape;
|
|
1274
|
+
// only that authority may turn a source mark into an acknowledgement.
|
|
1275
|
+
// A partially migrated G32 array still throws from this probe and is
|
|
1276
|
+
// therefore fail-closed in the caller below.
|
|
1277
|
+
if (!(await this.hasG44GlobalArrayAuthority()))
|
|
1278
|
+
return true;
|
|
1279
|
+
const database = this.env.D1;
|
|
1280
|
+
if (database === undefined)
|
|
1281
|
+
return false;
|
|
1282
|
+
const membership = delivery.completeness.localCommittedMembership[0];
|
|
1283
|
+
if (membership === undefined)
|
|
1284
|
+
return false;
|
|
1285
|
+
const row = await database.prepare(`SELECT 1 AS joined
|
|
1286
|
+
FROM serialized_dcb_global_receipts AS receipt
|
|
1287
|
+
JOIN serialized_dcb_global_memberships AS membership
|
|
1288
|
+
ON membership.service_id = receipt.service_id
|
|
1289
|
+
AND membership.event_id = receipt.event_id
|
|
1290
|
+
AND membership.partition_tag = receipt.membership_tag
|
|
1291
|
+
AND membership.event_digest = receipt.event_digest
|
|
1292
|
+
JOIN dcb_events AS event
|
|
1293
|
+
ON event."ServiceId" = receipt.service_id
|
|
1294
|
+
AND event."Id" = receipt.event_id
|
|
1295
|
+
AND event."EventDigest" = receipt.event_digest
|
|
1296
|
+
WHERE receipt.service_id = ?
|
|
1297
|
+
AND receipt.partition_tag = ?
|
|
1298
|
+
AND receipt.obligation_sequence = ?
|
|
1299
|
+
AND receipt.event_id = ?
|
|
1300
|
+
AND receipt.event_digest = ?
|
|
1301
|
+
AND receipt.membership_tag = ?
|
|
1302
|
+
AND membership.partition_tag = ?`).bind(membership.serviceId, membership.tag, delivery.completeness.obligationSequence, delivery.eventId, delivery.completeness.eventDigest, membership.tag, membership.tag).first();
|
|
1303
|
+
return row !== null && row !== undefined && row.joined === 1;
|
|
1304
|
+
}
|
|
1305
|
+
/**
|
|
1306
|
+
* The first append on a Tag must make its source partition enumerable before
|
|
1307
|
+
* the local event transaction starts. Once the local marker is registered,
|
|
1308
|
+
* later appends never consult or await D1 for this fact; Queue admission is
|
|
1309
|
+
* responsible for advancing the registered obligation sequence.
|
|
1310
|
+
*/
|
|
1311
|
+
async ensureSourcePartitionBeforeFirstAppend(tag, serviceId) {
|
|
1312
|
+
if (this.sourcePartitionRegistrationStatus(tag, serviceId) === "registered")
|
|
1313
|
+
return "existing";
|
|
1314
|
+
// A generic/local composition can use the SQL-backed Tag without wiring
|
|
1315
|
+
// the G44 completeness store. That is the pre-G65 path: there is no
|
|
1316
|
+
// registration work to wait for, and the durable local append remains
|
|
1317
|
+
// authoritative. Only a binding that proves the G44 global-array schema
|
|
1318
|
+
// is present enters the first-partition refusal contract below.
|
|
1319
|
+
if (this.env.D1 === undefined)
|
|
1320
|
+
return "unconfigured";
|
|
1321
|
+
if (g44GlobalArrayAuthorityResultByD1.get(this.env.D1) === false)
|
|
1322
|
+
return "unconfigured";
|
|
1323
|
+
const attempt = await this.boundedDerivedWrite(() => this.registerSourcePartition(tag, serviceId, 0), G65_SOURCE_REGISTRATION_BUDGET_MS);
|
|
1324
|
+
if (attempt.status !== "completed") {
|
|
1325
|
+
throw new PartitionRegistrationUnavailableError(attempt.status === "timeout" ? "timeout" : attempt.error);
|
|
1326
|
+
}
|
|
1327
|
+
if (!attempt.value)
|
|
1328
|
+
return "unconfigured";
|
|
1329
|
+
await this.markSourcePartitionRegistration(tag, serviceId, "registered", undefined, undefined, 0);
|
|
1330
|
+
return "newly-registered";
|
|
1331
|
+
}
|
|
1332
|
+
sourcePartitionRegistrationStatus(tag, serviceId) {
|
|
1333
|
+
const sql = this.sqlStorage();
|
|
1334
|
+
if (sql === undefined)
|
|
1335
|
+
return undefined;
|
|
1336
|
+
const row = sql.exec(`
|
|
1337
|
+
SELECT status
|
|
1338
|
+
FROM tag_source_partition_registration
|
|
1339
|
+
WHERE service_id = ? AND partition_tag = ?
|
|
1340
|
+
`, serviceId, tag).toArray()[0];
|
|
1341
|
+
return row === undefined ? undefined : sqlString(row.status, "tag_source_partition_registration.status");
|
|
1342
|
+
}
|
|
1343
|
+
/**
|
|
1344
|
+
* Refreshes the already-authorized source row with the committed local
|
|
1345
|
+
* obligation sequence without putting D1 back on the response path. The
|
|
1346
|
+
* first-append authority check above is the only registration operation that
|
|
1347
|
+
* may refuse a commit; this post-append watermark is Queue/retry recovery
|
|
1348
|
+
* work and is deliberately handed to waitUntil.
|
|
1349
|
+
*/
|
|
1350
|
+
scheduleSourcePartitionWatermark(tag, serviceId) {
|
|
1351
|
+
const refresh = this.retrySourcePartitionRegistration(tag, serviceId, true).catch((failure) => {
|
|
1352
|
+
console.warn("source_partition_registration", {
|
|
1353
|
+
status: "degraded",
|
|
1354
|
+
reason: "source_partition_registration_exhausted",
|
|
1355
|
+
attempts: G65_SOURCE_REGISTRATION_MAX_ATTEMPTS,
|
|
1356
|
+
error: String(failure),
|
|
1357
|
+
});
|
|
1358
|
+
});
|
|
1359
|
+
this.ctx.waitUntil(refresh);
|
|
1360
|
+
}
|
|
1361
|
+
async retrySourcePartitionRegistration(tag, serviceId, preserveRegistered = false) {
|
|
1362
|
+
let lastFailure;
|
|
1363
|
+
for (let attempt = 1; attempt <= G65_SOURCE_REGISTRATION_MAX_ATTEMPTS; attempt += 1) {
|
|
1364
|
+
const result = await this.boundedDerivedWrite(() => this.registerSourcePartition(tag, serviceId), G65_SOURCE_REGISTRATION_BUDGET_MS);
|
|
1365
|
+
if (result.status === "completed") {
|
|
1366
|
+
// A D1 binding without the G44 global-array schema is an explicitly
|
|
1367
|
+
// unconfigured completeness store, not a failed registration. Do not
|
|
1368
|
+
// create pending retry state for that composition. The local marker is
|
|
1369
|
+
// still recorded after the durable append so the scheduler retains its
|
|
1370
|
+
// pre-G65 source bookkeeping; it is not a D1 registration or a
|
|
1371
|
+
// response-path wait.
|
|
1372
|
+
if (!result.value) {
|
|
1373
|
+
await this.markSourcePartitionRegistration(tag, serviceId, "registered");
|
|
1374
|
+
return;
|
|
1375
|
+
}
|
|
1376
|
+
await this.markSourcePartitionRegistration(tag, serviceId, "registered");
|
|
1377
|
+
return;
|
|
1378
|
+
}
|
|
1379
|
+
lastFailure = result.status === "timeout" ? "timeout" : String(result.error);
|
|
1380
|
+
if (attempt < G65_SOURCE_REGISTRATION_MAX_ATTEMPTS) {
|
|
1381
|
+
await new Promise((resolve) => setTimeout(resolve, G65_SOURCE_REGISTRATION_RETRY_DELAY_MS * attempt));
|
|
1382
|
+
}
|
|
1383
|
+
}
|
|
1384
|
+
if (!preserveRegistered || this.sourcePartitionRegistrationStatus(tag, serviceId) !== "registered") {
|
|
1385
|
+
await this.markSourcePartitionRegistration(tag, serviceId, "pending", lastFailure ?? "unknown", Date.now() + G65_SOURCE_REGISTRATION_RETRY_DELAY_MS * G65_SOURCE_REGISTRATION_MAX_ATTEMPTS);
|
|
1386
|
+
}
|
|
1387
|
+
throw new Error(`source_partition_registration_failed:${lastFailure ?? "unknown"}`);
|
|
1388
|
+
}
|
|
1389
|
+
async markSourcePartitionRegistration(tag, serviceId, status, lastError, nextAttemptAt, lastObligationSequence = 0) {
|
|
1390
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
1391
|
+
const sql = this.sqlStorage();
|
|
1392
|
+
if (sql === undefined)
|
|
1393
|
+
return;
|
|
1394
|
+
sql.exec(`
|
|
1395
|
+
INSERT INTO tag_source_partition_registration
|
|
1396
|
+
(service_id, partition_tag, last_obligation_sequence, status, next_attempt_at, attempt_count, last_error)
|
|
1397
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
1398
|
+
ON CONFLICT (service_id, partition_tag) DO UPDATE SET
|
|
1399
|
+
last_obligation_sequence = MAX(
|
|
1400
|
+
tag_source_partition_registration.last_obligation_sequence,
|
|
1401
|
+
excluded.last_obligation_sequence
|
|
1402
|
+
),
|
|
1403
|
+
status = excluded.status,
|
|
1404
|
+
next_attempt_at = excluded.next_attempt_at,
|
|
1405
|
+
attempt_count = excluded.attempt_count,
|
|
1406
|
+
last_error = excluded.last_error
|
|
1407
|
+
`, serviceId, tag, lastObligationSequence, status, status === "pending" ? (nextAttemptAt ?? Date.now()) : null, status === "pending" ? 1 : 0, status === "pending" ? (lastError ?? null) : null);
|
|
1408
|
+
const dueAt = this.nextSqlAlarmDue();
|
|
1409
|
+
if (dueAt === null)
|
|
1410
|
+
await txn.deleteAlarm();
|
|
1411
|
+
else
|
|
1412
|
+
await txn.setAlarm(dueAt);
|
|
1413
|
+
});
|
|
1414
|
+
}
|
|
1415
|
+
async registerSourcePartition(tag, serviceId, requestedSequence) {
|
|
1416
|
+
// G44 is the D1 global-array implementation, not a rollout or
|
|
1417
|
+
// mixed-version mode. A non-G32 D1 harness has no global `dcb_events`
|
|
1418
|
+
// array at all; every actual G32/G44 D1-backed Tag commit registers its
|
|
1419
|
+
// partition, and a partially migrated G32 array fails closed below.
|
|
1420
|
+
const database = this.env.D1;
|
|
1421
|
+
if (database === undefined)
|
|
1422
|
+
return false;
|
|
1423
|
+
if (!(await this.hasG44GlobalArrayAuthority()))
|
|
1424
|
+
return false;
|
|
1425
|
+
const sql = this.sqlStorage();
|
|
1426
|
+
if (sql === undefined) {
|
|
1427
|
+
throw new Error("g44_source_partition_registry_requires_sql_tag");
|
|
1428
|
+
}
|
|
1429
|
+
const max = requestedSequence === undefined
|
|
1430
|
+
? sql.exec("SELECT MAX(obligation_sequence) AS sequence FROM tag_outbox_obligation WHERE service_id = ?", serviceId).toArray()[0]
|
|
1431
|
+
: undefined;
|
|
1432
|
+
const sequence = requestedSequence ??
|
|
1433
|
+
(max === undefined || max.sequence === null ? undefined : sqlNumber(max.sequence, "tag_outbox_obligation.max_sequence"));
|
|
1434
|
+
if (sequence === undefined)
|
|
1435
|
+
return true;
|
|
1436
|
+
await database.prepare(`INSERT INTO serialized_dcb_source_partitions
|
|
1437
|
+
(service_id, partition_tag, last_obligation_sequence, registered_at)
|
|
1438
|
+
VALUES (?, ?, ?, ?)
|
|
1439
|
+
ON CONFLICT (service_id, partition_tag) DO UPDATE
|
|
1440
|
+
SET last_obligation_sequence = MAX(
|
|
1441
|
+
serialized_dcb_source_partitions.last_obligation_sequence,
|
|
1442
|
+
excluded.last_obligation_sequence
|
|
1443
|
+
),
|
|
1444
|
+
registered_at = excluded.registered_at`).bind(serviceId, tag, sequence, Date.now()).run();
|
|
1445
|
+
return true;
|
|
1446
|
+
}
|
|
1447
|
+
async hasG44GlobalArrayAuthority() {
|
|
1448
|
+
const database = this.env.D1;
|
|
1449
|
+
if (database === undefined)
|
|
1450
|
+
return false;
|
|
1451
|
+
// The generic root Worker has a pre-G32 local D1 harness. It is not the
|
|
1452
|
+
// `dcb_events` global array. Cache this immutable schema probe by binding,
|
|
1453
|
+
// rather than repeating it once for every independent Tag DO.
|
|
1454
|
+
let authority = g44GlobalArrayAuthorityByD1.get(database);
|
|
1455
|
+
if (authority === undefined) {
|
|
1456
|
+
authority = this.readG44GlobalArrayAuthority(database);
|
|
1457
|
+
g44GlobalArrayAuthorityByD1.set(database, authority);
|
|
1458
|
+
}
|
|
1459
|
+
try {
|
|
1460
|
+
const result = await authority;
|
|
1461
|
+
g44GlobalArrayAuthorityResultByD1.set(database, result);
|
|
1462
|
+
return result;
|
|
1463
|
+
}
|
|
1464
|
+
catch (error) {
|
|
1465
|
+
if (g44GlobalArrayAuthorityByD1.get(database) === authority) {
|
|
1466
|
+
g44GlobalArrayAuthorityByD1.delete(database);
|
|
1467
|
+
}
|
|
1468
|
+
throw error;
|
|
1469
|
+
}
|
|
1470
|
+
}
|
|
1471
|
+
async readG44GlobalArrayAuthority(database) {
|
|
1472
|
+
try {
|
|
1473
|
+
// `dcb_events.EventDigest` is introduced by G44's ordinary migration.
|
|
1474
|
+
// The root unit-test Worker has a distinct pre-G32 D1 schema and no
|
|
1475
|
+
// global-array completeness store; it is not a configured source
|
|
1476
|
+
// registry. A missing table or column therefore selects the unchanged
|
|
1477
|
+
// local-append path. Any other error, including a configured store that
|
|
1478
|
+
// hangs or fails, is deliberately surfaced to the bounded first-write
|
|
1479
|
+
// refusal path.
|
|
1480
|
+
await database.prepare('SELECT "EventDigest" FROM dcb_events WHERE 1 = 0').all();
|
|
1481
|
+
return true;
|
|
1482
|
+
}
|
|
1483
|
+
catch (error) {
|
|
1484
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1485
|
+
if (/no such (?:table|column):\s*(?:dcb_events|EventDigest)/i.test(message))
|
|
1486
|
+
return false;
|
|
1487
|
+
throw error;
|
|
1488
|
+
}
|
|
1489
|
+
}
|
|
1490
|
+
async traceCommitActor(request, tag, serviceId, activation, observation, callback) {
|
|
1491
|
+
// S16 identity and the mutation use one parsed body. Reading a clone for
|
|
1492
|
+
// attribution and the original for behavior creates a second DO request
|
|
1493
|
+
// stream that can race response teardown in a parallel workerd pool.
|
|
1494
|
+
let body;
|
|
1495
|
+
return enterNativeActorHandleSpan(this.nativeTracing, { actorClass: "TAG", actorKey: `tag:${serviceId}:${tag}`, activation, observation }, async () => {
|
|
1496
|
+
body = await this.jsonBody(request);
|
|
1497
|
+
const attemptId = isObject(body) && isNonEmptyString(body.attemptId) ? body.attemptId : undefined;
|
|
1498
|
+
return attemptId === undefined || !isNonEmptyString(serviceId)
|
|
1499
|
+
? undefined
|
|
1500
|
+
: { attemptId, serviceId };
|
|
1501
|
+
}, async () => body === undefined
|
|
1502
|
+
? error(400, "malformed_tag_request", "Request body must be JSON")
|
|
1503
|
+
: callback(body));
|
|
1504
|
+
}
|
|
1505
|
+
/**
|
|
1506
|
+
* Internal Tag reads keep their Cloudflare parent solely from active async
|
|
1507
|
+
* context; G30 does not add a correlation header or alter either /state or
|
|
1508
|
+
* /head-facts just to label telemetry.
|
|
1509
|
+
*/
|
|
1510
|
+
async traceCommitReadActor(request, tag, serviceId, activation, observation, callback) {
|
|
1511
|
+
return enterNativeActorHandleSpan(this.nativeTracing, { actorClass: "TAG", actorKey: `tag:${serviceId}:${tag}`, activation, observation }, async () => undefined, callback);
|
|
1512
|
+
}
|
|
1513
|
+
async jsonBody(request) {
|
|
1514
|
+
try {
|
|
1515
|
+
return await request.json();
|
|
1516
|
+
}
|
|
1517
|
+
catch {
|
|
1518
|
+
return undefined;
|
|
1519
|
+
}
|
|
1520
|
+
}
|
|
1521
|
+
sqlStorage() {
|
|
1522
|
+
const sql = hasTagSqlStorage(this.ctx.storage) ? this.ctx.storage.sql : undefined;
|
|
1523
|
+
return sql === undefined ? undefined : this.g43SqlMeasurement?.instrument(sql) ?? sql;
|
|
1524
|
+
}
|
|
1525
|
+
/**
|
|
1526
|
+
* Rehydrate the legacy response shape only at a state/read boundary. The
|
|
1527
|
+
* append path below deliberately does not call this method: it reads the
|
|
1528
|
+
* head and candidate identities by index instead of deserializing history.
|
|
1529
|
+
*/
|
|
1530
|
+
readStoredRecord(tag) {
|
|
1531
|
+
const sql = this.sqlStorage();
|
|
1532
|
+
if (sql === undefined) {
|
|
1533
|
+
if (this.fallbackRecord !== undefined && this.fallbackRecord.tag !== tag) {
|
|
1534
|
+
throw new Error("Tag Durable Object identity changed");
|
|
1535
|
+
}
|
|
1536
|
+
return this.fallbackRecord;
|
|
1537
|
+
}
|
|
1538
|
+
const identity = sql.exec("SELECT tag, created_at FROM tag_identity WHERE singleton = 1").toArray()[0];
|
|
1539
|
+
if (identity === undefined)
|
|
1540
|
+
return undefined;
|
|
1541
|
+
if (sqlString(identity.tag, "tag_identity.tag") !== tag) {
|
|
1542
|
+
throw new Error("Tag Durable Object identity changed");
|
|
1543
|
+
}
|
|
1544
|
+
const control = sql.exec("SELECT * FROM tag_control WHERE singleton = 1").toArray()[0];
|
|
1545
|
+
if (control === undefined)
|
|
1546
|
+
throw new Error("Tag SQL identity exists without control state");
|
|
1547
|
+
const head = sql.exec("SELECT head_suid FROM tag_head WHERE singleton = 1").toArray()[0];
|
|
1548
|
+
if (head !== undefined && sqlString(head.head_suid, "tag_head.head_suid") !== sqlString(control.head_suid, "tag_control.head_suid")) {
|
|
1549
|
+
throw new Error("Tag SQL head/control mismatch");
|
|
1550
|
+
}
|
|
1551
|
+
const reservationRow = sql.exec("SELECT * FROM tag_reservation WHERE singleton = 1").toArray()[0];
|
|
1552
|
+
const activeReservation = reservationRow === undefined
|
|
1553
|
+
? null
|
|
1554
|
+
: {
|
|
1555
|
+
attemptId: sqlString(reservationRow.attempt_id, "tag_reservation.attempt_id"),
|
|
1556
|
+
epoch: sqlNumber(reservationRow.epoch, "tag_reservation.epoch"),
|
|
1557
|
+
token: sqlString(reservationRow.reservation_token, "tag_reservation.reservation_token"),
|
|
1558
|
+
expectedHead: sqlString(reservationRow.expected_head, "tag_reservation.expected_head"),
|
|
1559
|
+
expiresAt: sqlNumber(reservationRow.expires_at, "tag_reservation.expires_at"),
|
|
1560
|
+
alarmDueAt: sqlNumber(reservationRow.alarm_due_at, "tag_reservation.alarm_due_at"),
|
|
1561
|
+
};
|
|
1562
|
+
const events = sql.exec("SELECT event_json FROM tag_event ORDER BY suid ASC").toArray().map((row) => sqlJson(row.event_json, "tag_event.event_json"));
|
|
1563
|
+
const outbox = sql.exec(`
|
|
1564
|
+
SELECT attempt_id, event_id, suid, payload, allocator_lineage_id, event_type, provenance, timestamp
|
|
1565
|
+
FROM tag_outbox_obligation
|
|
1566
|
+
ORDER BY obligation_sequence ASC
|
|
1567
|
+
`).toArray().map((row) => ({
|
|
1568
|
+
attemptId: sqlString(row.attempt_id, "tag_outbox_obligation.attempt_id"),
|
|
1569
|
+
eventId: sqlString(row.event_id, "tag_outbox_obligation.event_id"),
|
|
1570
|
+
suid: sqlString(row.suid, "tag_outbox_obligation.suid"),
|
|
1571
|
+
payload: sqlString(row.payload, "tag_outbox_obligation.payload"),
|
|
1572
|
+
allocatorLineageId: sqlString(row.allocator_lineage_id, "tag_outbox_obligation.allocator_lineage_id"),
|
|
1573
|
+
eventType: sqlString(row.event_type, "tag_outbox_obligation.event_type"),
|
|
1574
|
+
provenance: sqlString(row.provenance, "tag_outbox_obligation.provenance"),
|
|
1575
|
+
timestamp: sqlString(row.timestamp, "tag_outbox_obligation.timestamp"),
|
|
1576
|
+
}));
|
|
1577
|
+
const epochRows = sql.exec("SELECT * FROM tag_epoch ORDER BY attempt_id ASC").toArray();
|
|
1578
|
+
const highestEpoch = epochRows.map((row) => ({
|
|
1579
|
+
attemptId: sqlString(row.attempt_id, "tag_epoch.attempt_id"),
|
|
1580
|
+
epoch: sqlNumber(row.highest_epoch, "tag_epoch.highest_epoch"),
|
|
1581
|
+
}));
|
|
1582
|
+
const sealedEpoch = epochRows.flatMap((row) => {
|
|
1583
|
+
const epoch = sqlNullableNumber(row.sealed_epoch, "tag_epoch.sealed_epoch");
|
|
1584
|
+
return epoch === null ? [] : [{ attemptId: sqlString(row.attempt_id, "tag_epoch.attempt_id"), epoch }];
|
|
1585
|
+
});
|
|
1586
|
+
const confirmations = sql.exec(`
|
|
1587
|
+
SELECT attempt_id, epoch FROM tag_commit_receipt
|
|
1588
|
+
WHERE reservation_confirmed = 1
|
|
1589
|
+
ORDER BY attempt_id ASC
|
|
1590
|
+
`).toArray().map((row) => ({
|
|
1591
|
+
attemptId: sqlString(row.attempt_id, "tag_commit_receipt.attempt_id"),
|
|
1592
|
+
epoch: sqlNumber(row.epoch, "tag_commit_receipt.epoch"),
|
|
1593
|
+
}));
|
|
1594
|
+
const tombstones = sql.exec("SELECT attempt_id, epoch FROM tag_tombstone ORDER BY attempt_id ASC").toArray().map((row) => ({
|
|
1595
|
+
attemptId: sqlString(row.attempt_id, "tag_tombstone.attempt_id"),
|
|
1596
|
+
epoch: sqlNumber(row.epoch, "tag_tombstone.epoch"),
|
|
1597
|
+
}));
|
|
1598
|
+
const fences = sql.exec("SELECT reason, attempt_id, epoch FROM tag_fence ORDER BY reason, attempt_id").toArray().map((row) => ({
|
|
1599
|
+
reason: sqlString(row.reason, "tag_fence.reason"),
|
|
1600
|
+
attemptId: sqlString(row.attempt_id, "tag_fence.attempt_id"),
|
|
1601
|
+
epoch: sqlNumber(row.epoch, "tag_fence.epoch"),
|
|
1602
|
+
}));
|
|
1603
|
+
const clearedFences = sql.exec("SELECT reason, attempt_id, epoch FROM tag_cleared_fence ORDER BY reason, attempt_id").toArray().map((row) => ({
|
|
1604
|
+
reason: sqlString(row.reason, "tag_cleared_fence.reason"),
|
|
1605
|
+
attemptId: sqlString(row.attempt_id, "tag_cleared_fence.attempt_id"),
|
|
1606
|
+
epoch: sqlNumber(row.epoch, "tag_cleared_fence.epoch"),
|
|
1607
|
+
}));
|
|
1608
|
+
const bootstrap = sql.exec("SELECT * FROM tag_bootstrap_admission WHERE singleton = 1").toArray()[0];
|
|
1609
|
+
const bootstrapAdmission = bootstrap === undefined ? null : {
|
|
1610
|
+
importId: sqlString(bootstrap.import_id, "tag_bootstrap_admission.import_id"),
|
|
1611
|
+
leaseEpoch: sqlNumber(bootstrap.lease_epoch, "tag_bootstrap_admission.lease_epoch"),
|
|
1612
|
+
manifestDigest: sqlString(bootstrap.manifest_digest, "tag_bootstrap_admission.manifest_digest"),
|
|
1613
|
+
targetServiceId: sqlString(bootstrap.target_service_id, "tag_bootstrap_admission.target_service_id"),
|
|
1614
|
+
closed: sqlNumber(bootstrap.closed, "tag_bootstrap_admission.closed") === 1,
|
|
1615
|
+
};
|
|
1616
|
+
// The in-memory scope is canonicalized by SUID, then the identity key.
|
|
1617
|
+
// The SQL representation stores the full item as JSON, so reproduce that
|
|
1618
|
+
// order rather than substituting the table's composite-key order.
|
|
1619
|
+
const repairScope = sql.exec(`
|
|
1620
|
+
SELECT item_json FROM tag_repair_scope
|
|
1621
|
+
ORDER BY json_extract(item_json, '$.suid') ASC, attempt_id ASC, event_id ASC
|
|
1622
|
+
`).toArray()
|
|
1623
|
+
.map((row) => sqlJson(row.item_json, "tag_repair_scope.item_json"));
|
|
1624
|
+
return {
|
|
1625
|
+
schemaVersion: sqlNumber(control.schema_version, "tag_control.schema_version"),
|
|
1626
|
+
tag,
|
|
1627
|
+
head: sqlString(control.head_suid, "tag_control.head_suid"),
|
|
1628
|
+
activeReservation,
|
|
1629
|
+
// `alarmDueAt` is retained as the reservation response/state fact. The
|
|
1630
|
+
// actual durable alarm is the scheduler minimum and is intentionally
|
|
1631
|
+
// inspected through the scheduler seam rather than this legacy shape.
|
|
1632
|
+
alarmDueAt: activeReservation?.alarmDueAt ?? null,
|
|
1633
|
+
events,
|
|
1634
|
+
outbox,
|
|
1635
|
+
highestEpoch,
|
|
1636
|
+
sealedEpoch,
|
|
1637
|
+
tombstones,
|
|
1638
|
+
confirmations,
|
|
1639
|
+
fences,
|
|
1640
|
+
clearedFences,
|
|
1641
|
+
bootstrapAdmission,
|
|
1642
|
+
repairOwner: sqlNullableString(control.repair_owner, "tag_control.repair_owner"),
|
|
1643
|
+
repairLeaseUntil: sqlNullableNumber(control.repair_lease_until, "tag_control.repair_lease_until"),
|
|
1644
|
+
highestRepairEpoch: sqlNumber(control.highest_repair_epoch, "tag_control.highest_repair_epoch"),
|
|
1645
|
+
repairScope,
|
|
1646
|
+
repairScopeVersion: sqlNumber(control.repair_scope_version, "tag_control.repair_scope_version"),
|
|
1647
|
+
clockOffsetMs: sqlNumber(control.clock_offset_ms, "tag_control.clock_offset_ms"),
|
|
1648
|
+
clockNowMs: sqlNullableNumber(control.clock_now_ms, "tag_control.clock_now_ms"),
|
|
1649
|
+
version: sqlNumber(control.version, "tag_control.version"),
|
|
1650
|
+
createdAt: sqlString(control.created_at, "tag_control.created_at"),
|
|
1651
|
+
updatedAt: sqlString(control.updated_at, "tag_control.updated_at"),
|
|
1652
|
+
};
|
|
1653
|
+
}
|
|
1654
|
+
/**
|
|
1655
|
+
* Reads the scalar commit-response facts without rehydrating `tag_event`.
|
|
1656
|
+
* Keep the identity and control/head consistency checks at this boundary:
|
|
1657
|
+
* an internal caller is not allowed to turn an identity change into a
|
|
1658
|
+
* plausible but stale response.
|
|
1659
|
+
*/
|
|
1660
|
+
readHeadFacts(tag) {
|
|
1661
|
+
const sql = this.sqlStorage();
|
|
1662
|
+
if (sql === undefined) {
|
|
1663
|
+
if (this.fallbackRecord !== undefined && this.fallbackRecord.tag !== tag) {
|
|
1664
|
+
throw new TagIdentityConflict();
|
|
1665
|
+
}
|
|
1666
|
+
return this.fallbackRecord === undefined
|
|
1667
|
+
? undefined
|
|
1668
|
+
: {
|
|
1669
|
+
head: this.fallbackRecord.head,
|
|
1670
|
+
version: this.fallbackRecord.version,
|
|
1671
|
+
updatedAt: this.fallbackRecord.updatedAt,
|
|
1672
|
+
};
|
|
1673
|
+
}
|
|
1674
|
+
const identity = sql.exec("SELECT tag FROM tag_identity WHERE singleton = 1").toArray()[0];
|
|
1675
|
+
if (identity === undefined)
|
|
1676
|
+
return undefined;
|
|
1677
|
+
if (sqlString(identity.tag, "tag_identity.tag") !== tag)
|
|
1678
|
+
throw new TagIdentityConflict();
|
|
1679
|
+
const control = sql.exec(`
|
|
1680
|
+
SELECT head_suid, version, updated_at
|
|
1681
|
+
FROM tag_control
|
|
1682
|
+
WHERE singleton = 1
|
|
1683
|
+
`).toArray()[0];
|
|
1684
|
+
if (control === undefined)
|
|
1685
|
+
throw new Error("Tag SQL identity exists without control state");
|
|
1686
|
+
const head = sql.exec("SELECT head_suid FROM tag_head WHERE singleton = 1").toArray()[0];
|
|
1687
|
+
const controlHead = sqlString(control.head_suid, "tag_control.head_suid");
|
|
1688
|
+
if (head !== undefined && sqlString(head.head_suid, "tag_head.head_suid") !== controlHead) {
|
|
1689
|
+
throw new Error("Tag SQL head/control mismatch");
|
|
1690
|
+
}
|
|
1691
|
+
return {
|
|
1692
|
+
head: controlHead,
|
|
1693
|
+
version: sqlNumber(control.version, "tag_control.version"),
|
|
1694
|
+
updatedAt: sqlString(control.updated_at, "tag_control.updated_at"),
|
|
1695
|
+
};
|
|
1696
|
+
}
|
|
1697
|
+
async obligationArtifact(event, serviceId, tag) {
|
|
1698
|
+
const declaredTagSet = canonicalDeclaredTagSet(event.eventTags);
|
|
1699
|
+
const payloadBytes = new TextEncoder().encode(event.payload);
|
|
1700
|
+
const digestInput = {
|
|
1701
|
+
serviceId,
|
|
1702
|
+
eventId: event.eventId,
|
|
1703
|
+
sortableUniqueId: event.suid,
|
|
1704
|
+
eventType: event.eventType,
|
|
1705
|
+
timestamp: event.timestamp,
|
|
1706
|
+
allocatorLineageId: event.allocatorLineageId,
|
|
1707
|
+
attemptId: event.attemptId,
|
|
1708
|
+
declaredTagSet,
|
|
1709
|
+
// TagEvent.payload is the exact persisted UTF-8 JSON text in the
|
|
1710
|
+
// current G32 record. The digest encoder accepts this byte view rather
|
|
1711
|
+
// than decoding/re-serializing it, which keeps the raw-byte rule at the
|
|
1712
|
+
// boundary where persisted text becomes an obligation artifact.
|
|
1713
|
+
payload: payloadBytes,
|
|
1714
|
+
};
|
|
1715
|
+
return {
|
|
1716
|
+
canonicalBytes: copyArrayBuffer(eventDigestBytes(digestInput)),
|
|
1717
|
+
eventDigest: await eventDigestHex(digestInput),
|
|
1718
|
+
declaredTagSet: JSON.stringify(declaredTagSet),
|
|
1719
|
+
localMembership: JSON.stringify([{ serviceId, eventId: event.eventId, tag }]),
|
|
1720
|
+
};
|
|
1721
|
+
}
|
|
1722
|
+
ensureSqlTag(tag) {
|
|
1723
|
+
const sql = this.sqlStorage();
|
|
1724
|
+
if (sql === undefined)
|
|
1725
|
+
throw new Error("Tag SQL storage is unavailable");
|
|
1726
|
+
const existing = sql.exec("SELECT tag FROM tag_identity WHERE singleton = 1").toArray()[0];
|
|
1727
|
+
if (existing !== undefined) {
|
|
1728
|
+
if (sqlString(existing.tag, "tag_identity.tag") !== tag)
|
|
1729
|
+
throw new Error("Tag Durable Object identity changed");
|
|
1730
|
+
return;
|
|
1731
|
+
}
|
|
1732
|
+
const record = newRecord(tag);
|
|
1733
|
+
sql.exec("INSERT INTO tag_identity (singleton, tag, created_at) VALUES (1, ?, ?)", tag, record.createdAt);
|
|
1734
|
+
sql.exec(`
|
|
1735
|
+
INSERT INTO tag_control (
|
|
1736
|
+
singleton, schema_version, head_suid, clock_offset_ms, clock_now_ms,
|
|
1737
|
+
version, repair_owner, repair_lease_until, highest_repair_epoch,
|
|
1738
|
+
repair_scope_version, created_at, updated_at
|
|
1739
|
+
) VALUES (1, 3, '', 0, NULL, 0, NULL, NULL, 0, 0, ?, ?)
|
|
1740
|
+
`, record.createdAt, record.updatedAt);
|
|
1741
|
+
sql.exec("INSERT INTO tag_head (singleton, service_id, head_suid) VALUES (1, '', '')");
|
|
1742
|
+
}
|
|
1743
|
+
writeCommittedSqlEvent(sql, serviceId, event) {
|
|
1744
|
+
sql.exec(`
|
|
1745
|
+
INSERT INTO tag_event (
|
|
1746
|
+
service_id, event_id, attempt_id, suid, payload, event_tags_json,
|
|
1747
|
+
allocator_lineage_id, event_type, provenance, timestamp, event_json
|
|
1748
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1749
|
+
`, serviceId, event.eventId, event.attemptId, event.suid, event.payload, JSON.stringify(event.eventTags), event.allocatorLineageId, event.eventType, event.provenance, event.timestamp, JSON.stringify(event));
|
|
1750
|
+
}
|
|
1751
|
+
writeCommittedSqlMembership(sql, serviceId, eventId, tag, committedAt) {
|
|
1752
|
+
sql.exec(`
|
|
1753
|
+
INSERT INTO tag_committed_membership (service_id, event_id, tag, committed_at)
|
|
1754
|
+
VALUES (?, ?, ?, ?)
|
|
1755
|
+
`, serviceId, eventId, tag, committedAt);
|
|
1756
|
+
}
|
|
1757
|
+
writeCommittedSqlObligation(sql, serviceId, tag, event, artifact, trackSourcePartitionRegistration = true) {
|
|
1758
|
+
sql.exec(`
|
|
1759
|
+
INSERT INTO tag_outbox_obligation (
|
|
1760
|
+
service_id, event_id, attempt_id, suid, payload, allocator_lineage_id,
|
|
1761
|
+
event_type, provenance, timestamp, canonical_bytes, event_digest,
|
|
1762
|
+
declared_tag_set_json, local_committed_membership_json, status,
|
|
1763
|
+
next_attempt_at, attempt_count, enqueued_at, acknowledged_at, last_error
|
|
1764
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending', ?, 0, NULL, NULL, NULL)
|
|
1765
|
+
`, serviceId, event.eventId, event.attemptId, event.suid, event.payload, event.allocatorLineageId, event.eventType, event.provenance, event.timestamp, artifact.canonicalBytes, artifact.eventDigest, artifact.declaredTagSet, artifact.localMembership, Date.now());
|
|
1766
|
+
if (trackSourcePartitionRegistration) {
|
|
1767
|
+
this.upsertSourcePartitionRegistration(sql, serviceId, tag, event.eventId);
|
|
1768
|
+
}
|
|
1769
|
+
}
|
|
1770
|
+
upsertSourcePartitionRegistration(sql, serviceId, tag, eventId) {
|
|
1771
|
+
const obligation = sql.exec(`
|
|
1772
|
+
SELECT obligation_sequence
|
|
1773
|
+
FROM tag_outbox_obligation
|
|
1774
|
+
WHERE service_id = ? AND event_id = ?
|
|
1775
|
+
ORDER BY obligation_sequence DESC
|
|
1776
|
+
LIMIT 1
|
|
1777
|
+
`, serviceId, eventId).one();
|
|
1778
|
+
const sequence = sqlNumber(obligation.obligation_sequence, "tag_outbox_obligation.obligation_sequence");
|
|
1779
|
+
sql.exec(`
|
|
1780
|
+
INSERT INTO tag_source_partition_registration
|
|
1781
|
+
(service_id, partition_tag, last_obligation_sequence, status, next_attempt_at, attempt_count, last_error)
|
|
1782
|
+
VALUES (?, ?, ?, 'pending', ?, 0, NULL)
|
|
1783
|
+
ON CONFLICT (service_id, partition_tag) DO UPDATE SET
|
|
1784
|
+
last_obligation_sequence = MAX(
|
|
1785
|
+
tag_source_partition_registration.last_obligation_sequence,
|
|
1786
|
+
excluded.last_obligation_sequence
|
|
1787
|
+
),
|
|
1788
|
+
status = CASE
|
|
1789
|
+
WHEN tag_source_partition_registration.status = 'registered' THEN 'registered'
|
|
1790
|
+
ELSE 'pending'
|
|
1791
|
+
END,
|
|
1792
|
+
next_attempt_at = CASE
|
|
1793
|
+
WHEN tag_source_partition_registration.status = 'registered' THEN NULL
|
|
1794
|
+
ELSE excluded.next_attempt_at
|
|
1795
|
+
END,
|
|
1796
|
+
attempt_count = CASE
|
|
1797
|
+
WHEN tag_source_partition_registration.status = 'registered' THEN 0
|
|
1798
|
+
ELSE 0
|
|
1799
|
+
END,
|
|
1800
|
+
last_error = CASE
|
|
1801
|
+
WHEN tag_source_partition_registration.status = 'registered' THEN NULL
|
|
1802
|
+
ELSE NULL
|
|
1803
|
+
END
|
|
1804
|
+
`, serviceId, tag, sequence, Date.now());
|
|
1805
|
+
}
|
|
1806
|
+
writeCommittedSqlHead(sql, serviceId, head, version, committedAt) {
|
|
1807
|
+
sql.exec("UPDATE tag_control SET head_suid = ?, version = ?, updated_at = ? WHERE singleton = 1", head, version, committedAt);
|
|
1808
|
+
sql.exec("UPDATE tag_head SET service_id = ?, head_suid = ? WHERE singleton = 1", serviceId, head);
|
|
1809
|
+
}
|
|
1810
|
+
writeCommittedSqlReceipt(sql, input, committedAt, eventCount, head, confirmsReservation) {
|
|
1811
|
+
sql.exec(`
|
|
1812
|
+
INSERT INTO tag_commit_receipt
|
|
1813
|
+
(attempt_id, epoch, committed_at, committed_event_count, head_suid, reservation_confirmed)
|
|
1814
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
1815
|
+
ON CONFLICT(attempt_id, epoch) DO UPDATE SET
|
|
1816
|
+
committed_at = excluded.committed_at,
|
|
1817
|
+
committed_event_count = excluded.committed_event_count,
|
|
1818
|
+
head_suid = excluded.head_suid,
|
|
1819
|
+
reservation_confirmed = excluded.reservation_confirmed
|
|
1820
|
+
`, input.attemptId, input.epoch, committedAt, eventCount, head, confirmsReservation ? 1 : 0);
|
|
1821
|
+
}
|
|
1822
|
+
/**
|
|
1823
|
+
* The normal append path deliberately probes only the indexed rows it needs
|
|
1824
|
+
* (identity/control, this attempt, this reservation, and these candidates).
|
|
1825
|
+
* It never calls `readStoredRecord`, so history size cannot turn append into
|
|
1826
|
+
* a whole-array deserialize/copy/serialize operation.
|
|
1827
|
+
*/
|
|
1828
|
+
async appendSql(tag, input, suppliedServiceId, trackSourcePartitionRegistration) {
|
|
1829
|
+
const serviceId = suppliedServiceId ?? "";
|
|
1830
|
+
const events = input.candidates.map((candidate) => ({
|
|
1831
|
+
attemptId: input.attemptId,
|
|
1832
|
+
eventId: candidate.eventId,
|
|
1833
|
+
suid: candidate.suid,
|
|
1834
|
+
payload: candidate.payload,
|
|
1835
|
+
eventTags: candidate.eventTags,
|
|
1836
|
+
allocatorLineageId: candidate.allocatorLineageId,
|
|
1837
|
+
eventType: candidate.eventType,
|
|
1838
|
+
provenance: candidate.provenance,
|
|
1839
|
+
timestamp: candidate.timestamp,
|
|
1840
|
+
}));
|
|
1841
|
+
const artifacts = await Promise.all(events.map((event) => this.obligationArtifact(event, serviceId, tag)));
|
|
1842
|
+
const result = await this.ctx.storage.transaction(async (txn) => {
|
|
1843
|
+
const sql = this.sqlStorage();
|
|
1844
|
+
if (sql === undefined)
|
|
1845
|
+
throw new Error("Tag SQL storage is unavailable");
|
|
1846
|
+
this.ensureSqlTag(tag);
|
|
1847
|
+
const control = sql.exec("SELECT * FROM tag_control WHERE singleton = 1").one();
|
|
1848
|
+
let head = sqlString(control.head_suid, "tag_control.head_suid");
|
|
1849
|
+
let version = sqlNumber(control.version, "tag_control.version");
|
|
1850
|
+
const existingCandidates = input.candidates.map((candidate, index) => {
|
|
1851
|
+
const event = sql.exec(`
|
|
1852
|
+
SELECT attempt_id, event_id, suid, payload, event_type, provenance, timestamp
|
|
1853
|
+
FROM tag_event WHERE service_id = ? AND event_id = ?
|
|
1854
|
+
`, serviceId, candidate.eventId).toArray()[0];
|
|
1855
|
+
const obligation = event === undefined ? undefined : sql.exec(`
|
|
1856
|
+
SELECT event_digest FROM tag_outbox_obligation
|
|
1857
|
+
WHERE service_id = ? AND event_id = ? ORDER BY obligation_sequence ASC LIMIT 1
|
|
1858
|
+
`, serviceId, candidate.eventId).toArray()[0];
|
|
1859
|
+
const exact = event !== undefined &&
|
|
1860
|
+
sqlString(event.attempt_id, "tag_event.attempt_id") === input.attemptId &&
|
|
1861
|
+
sqlString(event.event_id, "tag_event.event_id") === candidate.eventId &&
|
|
1862
|
+
sqlString(event.suid, "tag_event.suid") === candidate.suid &&
|
|
1863
|
+
sqlString(event.payload, "tag_event.payload") === candidate.payload &&
|
|
1864
|
+
sqlString(event.event_type, "tag_event.event_type") === candidate.eventType &&
|
|
1865
|
+
sqlString(event.provenance, "tag_event.provenance") === candidate.provenance &&
|
|
1866
|
+
sqlString(event.timestamp, "tag_event.timestamp") === candidate.timestamp &&
|
|
1867
|
+
(obligation === undefined || sqlString(obligation.event_digest, "tag_outbox_obligation.event_digest") === artifacts[index].eventDigest);
|
|
1868
|
+
return { event, obligation, exact };
|
|
1869
|
+
});
|
|
1870
|
+
// Exact duplicate replay precedes all epoch/token checks by contract.
|
|
1871
|
+
if (existingCandidates.every(({ exact }) => exact)) {
|
|
1872
|
+
return { status: 200, body: { status: "duplicate", version } };
|
|
1873
|
+
}
|
|
1874
|
+
const active = sql.exec("SELECT * FROM tag_reservation WHERE singleton = 1").toArray()[0];
|
|
1875
|
+
const now = sqlNullableNumber(control.clock_now_ms, "tag_control.clock_now_ms") ??
|
|
1876
|
+
Date.now() + sqlNumber(control.clock_offset_ms, "tag_control.clock_offset_ms");
|
|
1877
|
+
let reservation = active;
|
|
1878
|
+
if (reservation !== undefined && sqlNumber(reservation.expires_at, "tag_reservation.expires_at") <= now) {
|
|
1879
|
+
sql.exec("DELETE FROM tag_reservation WHERE singleton = 1");
|
|
1880
|
+
version += 1;
|
|
1881
|
+
sql.exec("UPDATE tag_control SET version = ?, updated_at = ? WHERE singleton = 1", version, nowIso());
|
|
1882
|
+
reservation = undefined;
|
|
1883
|
+
}
|
|
1884
|
+
const epoch = sql.exec("SELECT highest_epoch, sealed_epoch FROM tag_epoch WHERE attempt_id = ?", input.attemptId).toArray()[0];
|
|
1885
|
+
const highest = epoch === undefined ? undefined : sqlNumber(epoch.highest_epoch, "tag_epoch.highest_epoch");
|
|
1886
|
+
const sealed = epoch === undefined ? null : sqlNullableNumber(epoch.sealed_epoch, "tag_epoch.sealed_epoch");
|
|
1887
|
+
const tombstone = sql.exec("SELECT epoch FROM tag_tombstone WHERE attempt_id = ?", input.attemptId).toArray()[0];
|
|
1888
|
+
const tombstoneEpoch = tombstone === undefined ? undefined : sqlNumber(tombstone.epoch, "tag_tombstone.epoch");
|
|
1889
|
+
const epochError = highest !== undefined && input.epoch < highest
|
|
1890
|
+
? "stale_epoch"
|
|
1891
|
+
: tombstoneEpoch !== undefined && input.epoch <= tombstoneEpoch
|
|
1892
|
+
? "tombstoned_epoch"
|
|
1893
|
+
: sealed !== null && input.epoch <= sealed
|
|
1894
|
+
? "sealed_epoch"
|
|
1895
|
+
: undefined;
|
|
1896
|
+
if (epochError !== undefined) {
|
|
1897
|
+
await this.rearmScheduler(txn);
|
|
1898
|
+
return { ...rejected(epochError), body: { ...rejected(epochError).body, version } };
|
|
1899
|
+
}
|
|
1900
|
+
const fenceCount = sql.exec("SELECT COUNT(*) AS count FROM tag_fence").one();
|
|
1901
|
+
if (sqlNumber(fenceCount.count, "tag_fence.count") > 0) {
|
|
1902
|
+
await this.rearmScheduler(txn);
|
|
1903
|
+
return fenceGateRejected();
|
|
1904
|
+
}
|
|
1905
|
+
const confirmsReservation = reservation !== undefined &&
|
|
1906
|
+
sqlString(reservation.attempt_id, "tag_reservation.attempt_id") === input.attemptId &&
|
|
1907
|
+
sqlNumber(reservation.epoch, "tag_reservation.epoch") === input.epoch &&
|
|
1908
|
+
sqlString(reservation.reservation_token, "tag_reservation.reservation_token") === input.reservationToken;
|
|
1909
|
+
if (reservation !== undefined && !confirmsReservation) {
|
|
1910
|
+
await this.rearmScheduler(txn);
|
|
1911
|
+
return {
|
|
1912
|
+
...rejected("reservation_token_required"),
|
|
1913
|
+
body: { ...rejected("reservation_token_required").body, version },
|
|
1914
|
+
};
|
|
1915
|
+
}
|
|
1916
|
+
if (confirmsReservation && reservation !== undefined &&
|
|
1917
|
+
sqlString(reservation.expected_head, "tag_reservation.expected_head") !== head) {
|
|
1918
|
+
await this.rearmScheduler(txn);
|
|
1919
|
+
return {
|
|
1920
|
+
...rejected("consistency_head_mismatch"),
|
|
1921
|
+
body: { ...rejected("consistency_head_mismatch").body, version },
|
|
1922
|
+
};
|
|
1923
|
+
}
|
|
1924
|
+
if (existingCandidates.some(({ event, obligation }, index) => event !== undefined && obligation !== undefined &&
|
|
1925
|
+
sqlString(obligation.event_digest, "tag_outbox_obligation.event_digest") !== artifacts[index].eventDigest)) {
|
|
1926
|
+
await this.rearmScheduler(txn);
|
|
1927
|
+
return {
|
|
1928
|
+
...rejected("event_digest_conflict"),
|
|
1929
|
+
body: { ...rejected("event_digest_conflict").body, version },
|
|
1930
|
+
};
|
|
1931
|
+
}
|
|
1932
|
+
if (existingCandidates.some(({ event }) => event !== undefined)) {
|
|
1933
|
+
await this.rearmScheduler(txn);
|
|
1934
|
+
return {
|
|
1935
|
+
...rejected("event_conflict"),
|
|
1936
|
+
body: { ...rejected("event_conflict").body, version },
|
|
1937
|
+
};
|
|
1938
|
+
}
|
|
1939
|
+
for (const candidate of input.candidates) {
|
|
1940
|
+
if (candidate.suid <= head) {
|
|
1941
|
+
await this.rearmScheduler(txn);
|
|
1942
|
+
return {
|
|
1943
|
+
...rejected("non_monotonic_suid"),
|
|
1944
|
+
body: { ...rejected("non_monotonic_suid").body, version },
|
|
1945
|
+
};
|
|
1946
|
+
}
|
|
1947
|
+
head = candidate.suid;
|
|
1948
|
+
}
|
|
1949
|
+
const nextHighest = Math.max(highest ?? input.epoch, input.epoch);
|
|
1950
|
+
sql.exec(`
|
|
1951
|
+
INSERT INTO tag_epoch (attempt_id, highest_epoch, sealed_epoch, confirmation_epoch)
|
|
1952
|
+
VALUES (?, ?, NULL, NULL)
|
|
1953
|
+
ON CONFLICT(attempt_id) DO UPDATE SET highest_epoch = MAX(highest_epoch, excluded.highest_epoch)
|
|
1954
|
+
`, input.attemptId, nextHighest);
|
|
1955
|
+
const committedAt = nowIso();
|
|
1956
|
+
const hopFacts = [];
|
|
1957
|
+
for (let index = 0; index < events.length; index += 1) {
|
|
1958
|
+
const event = events[index];
|
|
1959
|
+
const artifact = artifacts[index];
|
|
1960
|
+
this.writeCommittedSqlEvent(sql, serviceId, event);
|
|
1961
|
+
this.writeCommittedSqlMembership(sql, serviceId, event.eventId, tag, committedAt);
|
|
1962
|
+
const obligationWrittenAt = Date.now();
|
|
1963
|
+
this.writeCommittedSqlObligation(sql, serviceId, tag, event, artifact, trackSourcePartitionRegistration);
|
|
1964
|
+
hopFacts.push({
|
|
1965
|
+
eventId: event.eventId,
|
|
1966
|
+
suid: event.suid,
|
|
1967
|
+
attemptId: event.attemptId,
|
|
1968
|
+
tag,
|
|
1969
|
+
// This is the existing durable tag_commit_receipt clock fact,
|
|
1970
|
+
// shared by membership/head/receipt writes in this transaction.
|
|
1971
|
+
tagAppendCommittedAt: Date.parse(committedAt),
|
|
1972
|
+
obligationWrittenAt,
|
|
1973
|
+
});
|
|
1974
|
+
}
|
|
1975
|
+
this.writeCommittedSqlHead(sql, serviceId, head, version + 1, committedAt);
|
|
1976
|
+
if (input.faultInjection === "after-append-before-confirm") {
|
|
1977
|
+
throw new AppendTransactionFault("Simulated interruption before atomic commit receipt");
|
|
1978
|
+
}
|
|
1979
|
+
this.writeCommittedSqlReceipt(sql, input, committedAt, events.length, head, confirmsReservation);
|
|
1980
|
+
if (confirmsReservation)
|
|
1981
|
+
sql.exec("DELETE FROM tag_reservation WHERE singleton = 1");
|
|
1982
|
+
await this.rearmScheduler(txn);
|
|
1983
|
+
return {
|
|
1984
|
+
status: 201,
|
|
1985
|
+
body: {
|
|
1986
|
+
status: "appended",
|
|
1987
|
+
fenceGate: { checked: true, activeFenceCount: 0 },
|
|
1988
|
+
version: version + 1,
|
|
1989
|
+
},
|
|
1990
|
+
hopFacts,
|
|
1991
|
+
};
|
|
1992
|
+
});
|
|
1993
|
+
return result;
|
|
1994
|
+
}
|
|
1995
|
+
/**
|
|
1996
|
+
* SQL-native reservation transition. This deliberately reads only the
|
|
1997
|
+
* control, epoch, tombstone, fence and reservation rows that participate in
|
|
1998
|
+
* the transition; it never rehydrates tag_event history just to reserve.
|
|
1999
|
+
*/
|
|
2000
|
+
async acquireSql(tag, input) {
|
|
2001
|
+
return this.ctx.storage.transaction(async (txn) => {
|
|
2002
|
+
const sql = this.sqlStorage();
|
|
2003
|
+
if (sql === undefined)
|
|
2004
|
+
throw new Error("Tag SQL storage is unavailable");
|
|
2005
|
+
this.ensureSqlTag(tag);
|
|
2006
|
+
const control = sql.exec("SELECT * FROM tag_control WHERE singleton = 1").one();
|
|
2007
|
+
let version = sqlNumber(control.version, "tag_control.version");
|
|
2008
|
+
const now = sqlNullableNumber(control.clock_now_ms, "tag_control.clock_now_ms") ??
|
|
2009
|
+
Date.now() + sqlNumber(control.clock_offset_ms, "tag_control.clock_offset_ms");
|
|
2010
|
+
let reservation = sql.exec("SELECT * FROM tag_reservation WHERE singleton = 1").toArray()[0];
|
|
2011
|
+
const epoch = sql.exec("SELECT highest_epoch, sealed_epoch FROM tag_epoch WHERE attempt_id = ?", input.attemptId).toArray()[0];
|
|
2012
|
+
const tombstone = sql.exec("SELECT epoch FROM tag_tombstone WHERE attempt_id = ?", input.attemptId).toArray()[0];
|
|
2013
|
+
const highest = epoch === undefined ? undefined : sqlNumber(epoch.highest_epoch, "tag_epoch.highest_epoch");
|
|
2014
|
+
const sealed = epoch === undefined ? null : sqlNullableNumber(epoch.sealed_epoch, "tag_epoch.sealed_epoch");
|
|
2015
|
+
const tombstoneEpoch = tombstone === undefined ? undefined : sqlNumber(tombstone.epoch, "tag_tombstone.epoch");
|
|
2016
|
+
const epochError = highest !== undefined && input.epoch < highest
|
|
2017
|
+
? "stale_epoch"
|
|
2018
|
+
: tombstoneEpoch !== undefined && input.epoch <= tombstoneEpoch
|
|
2019
|
+
? "tombstoned_epoch"
|
|
2020
|
+
: sealed !== null && input.epoch <= sealed
|
|
2021
|
+
? "sealed_epoch"
|
|
2022
|
+
: undefined;
|
|
2023
|
+
if (epochError !== undefined) {
|
|
2024
|
+
await this.rearmScheduler(txn);
|
|
2025
|
+
return { ...rejected(epochError), body: { ...rejected(epochError).body, version } };
|
|
2026
|
+
}
|
|
2027
|
+
// Preserve the established ordering: an expired reservation is cleaned
|
|
2028
|
+
// after its epoch has been validated, before head/fence/reservation use.
|
|
2029
|
+
if (reservation !== undefined && sqlNumber(reservation.expires_at, "tag_reservation.expires_at") <= now) {
|
|
2030
|
+
sql.exec("DELETE FROM tag_reservation WHERE singleton = 1");
|
|
2031
|
+
version += 1;
|
|
2032
|
+
sql.exec("UPDATE tag_control SET version = ?, updated_at = ? WHERE singleton = 1", version, nowIso());
|
|
2033
|
+
reservation = undefined;
|
|
2034
|
+
}
|
|
2035
|
+
const observed = input.expectedHead !== null;
|
|
2036
|
+
const isEventTag = input.eventTags.includes(tag);
|
|
2037
|
+
if (!observed || !isEventTag) {
|
|
2038
|
+
await this.rearmScheduler(txn);
|
|
2039
|
+
return { status: 200, body: { status: "omitted", reservation: null, fenceGateChecked: false, version } };
|
|
2040
|
+
}
|
|
2041
|
+
const fence = sql.exec("SELECT COUNT(*) AS count FROM tag_fence").one();
|
|
2042
|
+
if (sqlNumber(fence.count, "tag_fence.count") > 0) {
|
|
2043
|
+
await this.rearmScheduler(txn);
|
|
2044
|
+
return fenceGateRejected();
|
|
2045
|
+
}
|
|
2046
|
+
const head = sqlString(control.head_suid, "tag_control.head_suid");
|
|
2047
|
+
const assertedEmpty = input.expectedHead === "";
|
|
2048
|
+
if (assertedEmpty) {
|
|
2049
|
+
const eventCount = sql.exec("SELECT COUNT(*) AS count FROM tag_event").one();
|
|
2050
|
+
if (sqlNumber(eventCount.count, "tag_event.count") > 0 || head !== "") {
|
|
2051
|
+
await this.rearmScheduler(txn);
|
|
2052
|
+
return {
|
|
2053
|
+
...rejected(ASSERT_EMPTY_CONFLICT_REASON),
|
|
2054
|
+
body: { ...rejected(ASSERT_EMPTY_CONFLICT_REASON).body, version },
|
|
2055
|
+
};
|
|
2056
|
+
}
|
|
2057
|
+
}
|
|
2058
|
+
if (head !== input.expectedHead) {
|
|
2059
|
+
await this.rearmScheduler(txn);
|
|
2060
|
+
return {
|
|
2061
|
+
...rejected("consistency_head_mismatch"),
|
|
2062
|
+
body: { ...rejected("consistency_head_mismatch").body, version },
|
|
2063
|
+
};
|
|
2064
|
+
}
|
|
2065
|
+
if (reservation !== undefined) {
|
|
2066
|
+
if (sqlString(reservation.attempt_id, "tag_reservation.attempt_id") === input.attemptId &&
|
|
2067
|
+
sqlNumber(reservation.epoch, "tag_reservation.epoch") === input.epoch) {
|
|
2068
|
+
const activeFenceCount = sqlNumber(sql.exec("SELECT COUNT(*) AS count FROM tag_fence WHERE attempt_id = ?", input.attemptId).one().count, "tag_fence.attempt_count");
|
|
2069
|
+
await this.rearmScheduler(txn);
|
|
2070
|
+
return {
|
|
2071
|
+
status: 200,
|
|
2072
|
+
body: {
|
|
2073
|
+
status: "reserved",
|
|
2074
|
+
reservation: {
|
|
2075
|
+
attemptId: input.attemptId,
|
|
2076
|
+
epoch: input.epoch,
|
|
2077
|
+
token: sqlString(reservation.reservation_token, "tag_reservation.reservation_token"),
|
|
2078
|
+
expectedHead: sqlString(reservation.expected_head, "tag_reservation.expected_head"),
|
|
2079
|
+
expiresAt: sqlNumber(reservation.expires_at, "tag_reservation.expires_at"),
|
|
2080
|
+
alarmDueAt: sqlNumber(reservation.alarm_due_at, "tag_reservation.alarm_due_at"),
|
|
2081
|
+
},
|
|
2082
|
+
fenceGate: { checked: true, activeFenceCount },
|
|
2083
|
+
version,
|
|
2084
|
+
},
|
|
2085
|
+
};
|
|
2086
|
+
}
|
|
2087
|
+
await this.rearmScheduler(txn);
|
|
2088
|
+
return {
|
|
2089
|
+
...rejected(assertedEmpty ? ASSERT_EMPTY_CONFLICT_REASON : "active_reservation_conflict"),
|
|
2090
|
+
body: {
|
|
2091
|
+
...rejected(assertedEmpty ? ASSERT_EMPTY_CONFLICT_REASON : "active_reservation_conflict").body,
|
|
2092
|
+
version,
|
|
2093
|
+
},
|
|
2094
|
+
};
|
|
2095
|
+
}
|
|
2096
|
+
const reservationToken = crypto.randomUUID();
|
|
2097
|
+
const reservationFact = {
|
|
2098
|
+
attemptId: input.attemptId,
|
|
2099
|
+
epoch: input.epoch,
|
|
2100
|
+
token: reservationToken,
|
|
2101
|
+
expectedHead: input.expectedHead,
|
|
2102
|
+
expiresAt: now + RESERVATION_WINDOW_MS,
|
|
2103
|
+
alarmDueAt: Date.now() + RESERVATION_WINDOW_MS,
|
|
2104
|
+
};
|
|
2105
|
+
sql.exec(`
|
|
2106
|
+
INSERT INTO tag_epoch (attempt_id, highest_epoch, sealed_epoch, confirmation_epoch)
|
|
2107
|
+
VALUES (?, ?, NULL, NULL)
|
|
2108
|
+
ON CONFLICT(attempt_id) DO UPDATE SET highest_epoch = MAX(highest_epoch, excluded.highest_epoch)
|
|
2109
|
+
`, input.attemptId, input.epoch);
|
|
2110
|
+
if (assertedEmpty) {
|
|
2111
|
+
// An asserted-empty reservation is the prepare step of the ordinary
|
|
2112
|
+
// first write. It must not manufacture a version before the first
|
|
2113
|
+
// committed event; append therefore returns version 1 just like the
|
|
2114
|
+
// unobserved first-write path.
|
|
2115
|
+
sql.exec("UPDATE tag_control SET updated_at = ? WHERE singleton = 1", nowIso());
|
|
2116
|
+
}
|
|
2117
|
+
else {
|
|
2118
|
+
version += 1;
|
|
2119
|
+
sql.exec("UPDATE tag_control SET version = ?, updated_at = ? WHERE singleton = 1", version, nowIso());
|
|
2120
|
+
}
|
|
2121
|
+
sql.exec(`
|
|
2122
|
+
INSERT INTO tag_reservation
|
|
2123
|
+
(singleton, attempt_id, epoch, reservation_token, expected_head, expires_at, alarm_due_at)
|
|
2124
|
+
VALUES (1, ?, ?, ?, ?, ?, ?)
|
|
2125
|
+
`, reservationFact.attemptId, reservationFact.epoch, reservationFact.token, reservationFact.expectedHead, reservationFact.expiresAt, reservationFact.alarmDueAt);
|
|
2126
|
+
await this.rearmScheduler(txn);
|
|
2127
|
+
return {
|
|
2128
|
+
status: 201,
|
|
2129
|
+
body: { status: "reserved", reservation: reservationFact, fenceGate: { checked: true, activeFenceCount: 0 }, version },
|
|
2130
|
+
};
|
|
2131
|
+
});
|
|
2132
|
+
}
|
|
2133
|
+
/**
|
|
2134
|
+
* SQL-native cancel transition. It only mutates reservation/epoch/tombstone
|
|
2135
|
+
* control rows, so a cancellation cannot hide a committed event, membership
|
|
2136
|
+
* or obligation and does not rewrite event history.
|
|
2137
|
+
*/
|
|
2138
|
+
async cancelSql(tag, input) {
|
|
2139
|
+
return this.ctx.storage.transaction(async (txn) => {
|
|
2140
|
+
const sql = this.sqlStorage();
|
|
2141
|
+
if (sql === undefined)
|
|
2142
|
+
throw new Error("Tag SQL storage is unavailable");
|
|
2143
|
+
const identity = sql.exec("SELECT tag FROM tag_identity WHERE singleton = 1").toArray()[0];
|
|
2144
|
+
if (identity === undefined) {
|
|
2145
|
+
return { ...rejected("reservation_token_required"), body: { ...rejected("reservation_token_required").body, version: 0 } };
|
|
2146
|
+
}
|
|
2147
|
+
if (sqlString(identity.tag, "tag_identity.tag") !== tag)
|
|
2148
|
+
throw new Error("Tag Durable Object identity changed");
|
|
2149
|
+
const control = sql.exec("SELECT * FROM tag_control WHERE singleton = 1").one();
|
|
2150
|
+
let version = sqlNumber(control.version, "tag_control.version");
|
|
2151
|
+
const now = sqlNullableNumber(control.clock_now_ms, "tag_control.clock_now_ms") ??
|
|
2152
|
+
Date.now() + sqlNumber(control.clock_offset_ms, "tag_control.clock_offset_ms");
|
|
2153
|
+
let reservation = sql.exec("SELECT * FROM tag_reservation WHERE singleton = 1").toArray()[0];
|
|
2154
|
+
if (reservation !== undefined && sqlNumber(reservation.expires_at, "tag_reservation.expires_at") <= now) {
|
|
2155
|
+
sql.exec("DELETE FROM tag_reservation WHERE singleton = 1");
|
|
2156
|
+
version += 1;
|
|
2157
|
+
sql.exec("UPDATE tag_control SET version = ?, updated_at = ? WHERE singleton = 1", version, nowIso());
|
|
2158
|
+
reservation = undefined;
|
|
2159
|
+
}
|
|
2160
|
+
const tombstone = sql.exec("SELECT epoch FROM tag_tombstone WHERE attempt_id = ?", input.attemptId).toArray()[0];
|
|
2161
|
+
const tombstoneEpoch = tombstone === undefined ? undefined : sqlNumber(tombstone.epoch, "tag_tombstone.epoch");
|
|
2162
|
+
const holdsReservation = reservation !== undefined &&
|
|
2163
|
+
sqlString(reservation.attempt_id, "tag_reservation.attempt_id") === input.attemptId &&
|
|
2164
|
+
sqlNumber(reservation.epoch, "tag_reservation.epoch") === input.epoch;
|
|
2165
|
+
if (input.forceTombstone === true) {
|
|
2166
|
+
if (!holdsReservation && tombstoneEpoch !== undefined && tombstoneEpoch >= input.epoch) {
|
|
2167
|
+
sql.exec("UPDATE tag_epoch SET sealed_epoch = ? WHERE attempt_id = ? AND (sealed_epoch IS NULL OR sealed_epoch < ?)", MAX_EPOCH, input.attemptId, MAX_EPOCH);
|
|
2168
|
+
await this.rearmScheduler(txn);
|
|
2169
|
+
return { status: 200, body: { status: "cancelled", idempotent: true, version } };
|
|
2170
|
+
}
|
|
2171
|
+
// Force-tombstone records the pinned epoch for inspect/reconciler evidence
|
|
2172
|
+
// and seals at MAX_EPOCH so no higher writer generation can append.
|
|
2173
|
+
sql.exec(`
|
|
2174
|
+
INSERT INTO tag_epoch (attempt_id, highest_epoch, sealed_epoch, confirmation_epoch)
|
|
2175
|
+
VALUES (?, ?, NULL, NULL)
|
|
2176
|
+
ON CONFLICT(attempt_id) DO UPDATE SET highest_epoch = MAX(highest_epoch, excluded.highest_epoch)
|
|
2177
|
+
`, input.attemptId, input.epoch);
|
|
2178
|
+
sql.exec("UPDATE tag_epoch SET sealed_epoch = ? WHERE attempt_id = ?", MAX_EPOCH, input.attemptId);
|
|
2179
|
+
sql.exec(`
|
|
2180
|
+
INSERT INTO tag_tombstone (attempt_id, epoch) VALUES (?, ?)
|
|
2181
|
+
ON CONFLICT(attempt_id) DO UPDATE SET epoch = MAX(epoch, excluded.epoch)
|
|
2182
|
+
`, input.attemptId, input.epoch);
|
|
2183
|
+
if (holdsReservation)
|
|
2184
|
+
sql.exec("DELETE FROM tag_reservation WHERE singleton = 1");
|
|
2185
|
+
version += 1;
|
|
2186
|
+
sql.exec("UPDATE tag_control SET version = ?, updated_at = ? WHERE singleton = 1", version, nowIso());
|
|
2187
|
+
await this.rearmScheduler(txn);
|
|
2188
|
+
return { status: 200, body: { status: "cancelled", idempotent: false, version } };
|
|
2189
|
+
}
|
|
2190
|
+
if (!holdsReservation && tombstoneEpoch === input.epoch) {
|
|
2191
|
+
await this.rearmScheduler(txn);
|
|
2192
|
+
return { status: 200, body: { status: "cancelled", idempotent: true, version } };
|
|
2193
|
+
}
|
|
2194
|
+
const epoch = sql.exec("SELECT highest_epoch, sealed_epoch FROM tag_epoch WHERE attempt_id = ?", input.attemptId).toArray()[0];
|
|
2195
|
+
const highest = epoch === undefined ? undefined : sqlNumber(epoch.highest_epoch, "tag_epoch.highest_epoch");
|
|
2196
|
+
const sealed = epoch === undefined ? null : sqlNullableNumber(epoch.sealed_epoch, "tag_epoch.sealed_epoch");
|
|
2197
|
+
const epochError = highest !== undefined && input.epoch < highest
|
|
2198
|
+
? "stale_epoch"
|
|
2199
|
+
: tombstoneEpoch !== undefined && input.epoch <= tombstoneEpoch
|
|
2200
|
+
? "tombstoned_epoch"
|
|
2201
|
+
: sealed !== null && input.epoch <= sealed
|
|
2202
|
+
? "sealed_epoch"
|
|
2203
|
+
: undefined;
|
|
2204
|
+
if (epochError !== undefined) {
|
|
2205
|
+
await this.rearmScheduler(txn);
|
|
2206
|
+
return { ...rejected(epochError), body: { ...rejected(epochError).body, version } };
|
|
2207
|
+
}
|
|
2208
|
+
if (!holdsReservation || reservation === undefined) {
|
|
2209
|
+
await this.rearmScheduler(txn);
|
|
2210
|
+
return {
|
|
2211
|
+
...rejected("reservation_token_required"),
|
|
2212
|
+
body: { ...rejected("reservation_token_required").body, version },
|
|
2213
|
+
};
|
|
2214
|
+
}
|
|
2215
|
+
if (sqlString(reservation.reservation_token, "tag_reservation.reservation_token") !== input.reservationToken) {
|
|
2216
|
+
await this.rearmScheduler(txn);
|
|
2217
|
+
return {
|
|
2218
|
+
...rejected("reservation_token_required"),
|
|
2219
|
+
body: { ...rejected("reservation_token_required").body, version },
|
|
2220
|
+
};
|
|
2221
|
+
}
|
|
2222
|
+
sql.exec(`
|
|
2223
|
+
INSERT INTO tag_epoch (attempt_id, highest_epoch, sealed_epoch, confirmation_epoch)
|
|
2224
|
+
VALUES (?, ?, NULL, NULL)
|
|
2225
|
+
ON CONFLICT(attempt_id) DO UPDATE SET highest_epoch = MAX(highest_epoch, excluded.highest_epoch)
|
|
2226
|
+
`, input.attemptId, input.epoch);
|
|
2227
|
+
sql.exec(`
|
|
2228
|
+
INSERT INTO tag_tombstone (attempt_id, epoch) VALUES (?, ?)
|
|
2229
|
+
ON CONFLICT(attempt_id) DO UPDATE SET epoch = MAX(epoch, excluded.epoch)
|
|
2230
|
+
`, input.attemptId, input.epoch);
|
|
2231
|
+
sql.exec("DELETE FROM tag_reservation WHERE singleton = 1");
|
|
2232
|
+
version += 1;
|
|
2233
|
+
sql.exec("UPDATE tag_control SET version = ?, updated_at = ? WHERE singleton = 1", version, nowIso());
|
|
2234
|
+
await this.rearmScheduler(txn);
|
|
2235
|
+
return { status: 200, body: { status: "cancelled", idempotent: false, version } };
|
|
2236
|
+
});
|
|
2237
|
+
}
|
|
2238
|
+
async writeSqlRecord(record, requestedServiceId = "") {
|
|
2239
|
+
const sql = this.sqlStorage();
|
|
2240
|
+
if (sql === undefined)
|
|
2241
|
+
throw new Error("Tag SQL storage is unavailable");
|
|
2242
|
+
const existingIdentity = sql.exec("SELECT tag FROM tag_identity WHERE singleton = 1").toArray()[0];
|
|
2243
|
+
if (existingIdentity === undefined) {
|
|
2244
|
+
sql.exec("INSERT INTO tag_identity (singleton, tag, created_at) VALUES (1, ?, ?)", record.tag, record.createdAt);
|
|
2245
|
+
}
|
|
2246
|
+
else if (sqlString(existingIdentity.tag, "tag_identity.tag") !== record.tag) {
|
|
2247
|
+
throw new Error("Tag Durable Object identity changed");
|
|
2248
|
+
}
|
|
2249
|
+
sql.exec(`
|
|
2250
|
+
INSERT INTO tag_control (
|
|
2251
|
+
singleton, schema_version, head_suid, clock_offset_ms, clock_now_ms,
|
|
2252
|
+
version, repair_owner, repair_lease_until, highest_repair_epoch,
|
|
2253
|
+
repair_scope_version, created_at, updated_at
|
|
2254
|
+
) VALUES (1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
2255
|
+
ON CONFLICT(singleton) DO UPDATE SET
|
|
2256
|
+
schema_version = excluded.schema_version,
|
|
2257
|
+
head_suid = excluded.head_suid,
|
|
2258
|
+
clock_offset_ms = excluded.clock_offset_ms,
|
|
2259
|
+
clock_now_ms = excluded.clock_now_ms,
|
|
2260
|
+
version = excluded.version,
|
|
2261
|
+
repair_owner = excluded.repair_owner,
|
|
2262
|
+
repair_lease_until = excluded.repair_lease_until,
|
|
2263
|
+
highest_repair_epoch = excluded.highest_repair_epoch,
|
|
2264
|
+
repair_scope_version = excluded.repair_scope_version,
|
|
2265
|
+
updated_at = excluded.updated_at
|
|
2266
|
+
`, record.schemaVersion, record.head, record.clockOffsetMs, record.clockNowMs, record.version, record.repairOwner, record.repairLeaseUntil, record.highestRepairEpoch, record.repairScopeVersion, record.createdAt, record.updatedAt);
|
|
2267
|
+
const existingHead = sql.exec("SELECT service_id FROM tag_head WHERE singleton = 1").toArray()[0];
|
|
2268
|
+
const serviceId = requestedServiceId || (existingHead === undefined ? "" : sqlString(existingHead.service_id, "tag_head.service_id")) || record.bootstrapAdmission?.targetServiceId || "";
|
|
2269
|
+
sql.exec(`
|
|
2270
|
+
INSERT INTO tag_head (singleton, service_id, head_suid) VALUES (1, ?, ?)
|
|
2271
|
+
ON CONFLICT(singleton) DO UPDATE SET service_id = excluded.service_id, head_suid = excluded.head_suid
|
|
2272
|
+
`, serviceId, record.head);
|
|
2273
|
+
sql.exec("DELETE FROM tag_reservation WHERE singleton = 1");
|
|
2274
|
+
if (record.activeReservation !== null) {
|
|
2275
|
+
const reservation = record.activeReservation;
|
|
2276
|
+
sql.exec(`
|
|
2277
|
+
INSERT INTO tag_reservation
|
|
2278
|
+
(singleton, attempt_id, epoch, reservation_token, expected_head, expires_at, alarm_due_at)
|
|
2279
|
+
VALUES (1, ?, ?, ?, ?, ?, ?)
|
|
2280
|
+
`, reservation.attemptId, reservation.epoch, reservation.token, reservation.expectedHead, reservation.expiresAt, reservation.alarmDueAt);
|
|
2281
|
+
}
|
|
2282
|
+
sql.exec("DELETE FROM tag_epoch");
|
|
2283
|
+
const epochs = new Map();
|
|
2284
|
+
for (const entry of record.highestEpoch)
|
|
2285
|
+
epochs.set(entry.attemptId, { highest: entry.epoch, sealed: null, confirmation: null });
|
|
2286
|
+
for (const entry of record.sealedEpoch) {
|
|
2287
|
+
const prior = epochs.get(entry.attemptId) ?? { highest: entry.epoch, sealed: null, confirmation: null };
|
|
2288
|
+
prior.highest = Math.max(prior.highest, entry.epoch);
|
|
2289
|
+
prior.sealed = entry.epoch;
|
|
2290
|
+
epochs.set(entry.attemptId, prior);
|
|
2291
|
+
}
|
|
2292
|
+
for (const entry of record.confirmations) {
|
|
2293
|
+
const prior = epochs.get(entry.attemptId) ?? { highest: entry.epoch, sealed: null, confirmation: null };
|
|
2294
|
+
prior.highest = Math.max(prior.highest, entry.epoch);
|
|
2295
|
+
prior.confirmation = entry.epoch;
|
|
2296
|
+
epochs.set(entry.attemptId, prior);
|
|
2297
|
+
}
|
|
2298
|
+
for (const [attemptId, epoch] of epochs) {
|
|
2299
|
+
sql.exec("INSERT INTO tag_epoch (attempt_id, highest_epoch, sealed_epoch, confirmation_epoch) VALUES (?, ?, ?, ?)", attemptId, epoch.highest, epoch.sealed, epoch.confirmation);
|
|
2300
|
+
}
|
|
2301
|
+
sql.exec("DELETE FROM tag_tombstone");
|
|
2302
|
+
for (const tombstone of record.tombstones) {
|
|
2303
|
+
sql.exec("INSERT INTO tag_tombstone (attempt_id, epoch) VALUES (?, ?)", tombstone.attemptId, tombstone.epoch);
|
|
2304
|
+
}
|
|
2305
|
+
sql.exec("DELETE FROM tag_fence");
|
|
2306
|
+
for (const fence of record.fences)
|
|
2307
|
+
sql.exec("INSERT INTO tag_fence (reason, attempt_id, epoch) VALUES (?, ?, ?)", fence.reason, fence.attemptId, fence.epoch);
|
|
2308
|
+
sql.exec("DELETE FROM tag_cleared_fence");
|
|
2309
|
+
for (const fence of record.clearedFences)
|
|
2310
|
+
sql.exec("INSERT INTO tag_cleared_fence (reason, attempt_id, epoch) VALUES (?, ?, ?)", fence.reason, fence.attemptId, fence.epoch);
|
|
2311
|
+
sql.exec("DELETE FROM tag_bootstrap_admission WHERE singleton = 1");
|
|
2312
|
+
if (record.bootstrapAdmission !== null) {
|
|
2313
|
+
const admission = record.bootstrapAdmission;
|
|
2314
|
+
sql.exec(`
|
|
2315
|
+
INSERT INTO tag_bootstrap_admission
|
|
2316
|
+
(singleton, import_id, lease_epoch, manifest_digest, target_service_id, closed)
|
|
2317
|
+
VALUES (1, ?, ?, ?, ?, ?)
|
|
2318
|
+
`, admission.importId, admission.leaseEpoch, admission.manifestDigest, admission.targetServiceId, admission.closed ? 1 : 0);
|
|
2319
|
+
}
|
|
2320
|
+
sql.exec("DELETE FROM tag_repair_scope");
|
|
2321
|
+
for (const item of record.repairScope) {
|
|
2322
|
+
sql.exec("INSERT INTO tag_repair_scope (attempt_id, event_id, item_json) VALUES (?, ?, ?)", item.attemptId, item.eventId, JSON.stringify(item));
|
|
2323
|
+
}
|
|
2324
|
+
const knownEvents = new Map(sql.exec("SELECT service_id, event_id FROM tag_event").toArray().map((row) => [
|
|
2325
|
+
sqlString(row.event_id, "tag_event.event_id"),
|
|
2326
|
+
sqlString(row.service_id, "tag_event.service_id"),
|
|
2327
|
+
]));
|
|
2328
|
+
const eventByOutboxIdentity = new Map(record.events.map((event) => [
|
|
2329
|
+
outboxRowKey(event),
|
|
2330
|
+
event,
|
|
2331
|
+
]));
|
|
2332
|
+
for (const event of record.events) {
|
|
2333
|
+
const eventServiceId = knownEvents.get(event.eventId) ?? serviceId;
|
|
2334
|
+
if (!knownEvents.has(event.eventId)) {
|
|
2335
|
+
sql.exec(`
|
|
2336
|
+
INSERT INTO tag_event (
|
|
2337
|
+
service_id, event_id, attempt_id, suid, payload, event_tags_json,
|
|
2338
|
+
allocator_lineage_id, event_type, provenance, timestamp, event_json
|
|
2339
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
2340
|
+
`, eventServiceId, event.eventId, event.attemptId, event.suid, event.payload, JSON.stringify(event.eventTags), event.allocatorLineageId, event.eventType, event.provenance, event.timestamp, JSON.stringify(event));
|
|
2341
|
+
}
|
|
2342
|
+
sql.exec(`
|
|
2343
|
+
INSERT OR IGNORE INTO tag_committed_membership (service_id, event_id, tag, committed_at)
|
|
2344
|
+
VALUES (?, ?, ?, ?)
|
|
2345
|
+
`, eventServiceId, event.eventId, record.tag, record.updatedAt);
|
|
2346
|
+
}
|
|
2347
|
+
for (const outbox of record.outbox) {
|
|
2348
|
+
const event = eventByOutboxIdentity.get(outboxRowKey(outbox));
|
|
2349
|
+
if (event === undefined)
|
|
2350
|
+
continue;
|
|
2351
|
+
const eventServiceId = knownEvents.get(event.eventId) ?? serviceId;
|
|
2352
|
+
const artifact = await this.obligationArtifact(event, eventServiceId, record.tag);
|
|
2353
|
+
sql.exec(`
|
|
2354
|
+
INSERT OR IGNORE INTO tag_outbox_obligation (
|
|
2355
|
+
service_id, event_id, attempt_id, suid, payload, allocator_lineage_id,
|
|
2356
|
+
event_type, provenance, timestamp, canonical_bytes, event_digest,
|
|
2357
|
+
declared_tag_set_json, local_committed_membership_json, status,
|
|
2358
|
+
next_attempt_at, attempt_count, enqueued_at, acknowledged_at, last_error
|
|
2359
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending', ?, 0, NULL, NULL, NULL)
|
|
2360
|
+
`, eventServiceId, outbox.eventId, outbox.attemptId, outbox.suid, outbox.payload, outbox.allocatorLineageId, outbox.eventType, outbox.provenance, outbox.timestamp, artifact.canonicalBytes, artifact.eventDigest, artifact.declaredTagSet, artifact.localMembership, Date.now());
|
|
2361
|
+
this.upsertSourcePartitionRegistration(sql, eventServiceId, record.tag, outbox.eventId);
|
|
2362
|
+
}
|
|
2363
|
+
for (const confirmation of record.confirmations) {
|
|
2364
|
+
sql.exec(`
|
|
2365
|
+
INSERT OR IGNORE INTO tag_commit_receipt
|
|
2366
|
+
(attempt_id, epoch, committed_at, committed_event_count, head_suid, reservation_confirmed)
|
|
2367
|
+
VALUES (?, ?, ?, ?, ?, 1)
|
|
2368
|
+
`, confirmation.attemptId, confirmation.epoch, record.updatedAt, record.events.filter((event) => event.attemptId === confirmation.attemptId).length, record.head);
|
|
2369
|
+
}
|
|
2370
|
+
}
|
|
2371
|
+
nextSqlAlarmDue() {
|
|
2372
|
+
const sql = this.sqlStorage();
|
|
2373
|
+
if (sql === undefined)
|
|
2374
|
+
return this.fallbackRecord?.alarmDueAt ?? null;
|
|
2375
|
+
const row = sql.exec(`
|
|
2376
|
+
SELECT MIN(due_at) AS due_at FROM (
|
|
2377
|
+
SELECT alarm_due_at AS due_at FROM tag_reservation
|
|
2378
|
+
UNION ALL
|
|
2379
|
+
SELECT next_attempt_at AS due_at FROM tag_outbox_obligation
|
|
2380
|
+
WHERE status = 'pending'
|
|
2381
|
+
UNION ALL
|
|
2382
|
+
SELECT next_attempt_at AS due_at FROM tag_source_partition_registration
|
|
2383
|
+
WHERE status = 'pending'
|
|
2384
|
+
)
|
|
2385
|
+
`).toArray()[0];
|
|
2386
|
+
return row === undefined ? null : sqlNullableNumber(row.due_at, "due_at");
|
|
2387
|
+
}
|
|
2388
|
+
async recordFor(txn, tag) {
|
|
2389
|
+
void txn;
|
|
2390
|
+
const existing = this.readStoredRecord(tag);
|
|
2391
|
+
return { record: existing === undefined ? newRecord(tag) : requireG32TagRecord(existing), exists: existing !== undefined };
|
|
2392
|
+
}
|
|
2393
|
+
async write(txn, record, serviceId = "") {
|
|
2394
|
+
if (this.sqlStorage() === undefined) {
|
|
2395
|
+
this.fallbackRecord = record;
|
|
2396
|
+
}
|
|
2397
|
+
else {
|
|
2398
|
+
await this.writeSqlRecord(record, serviceId);
|
|
2399
|
+
}
|
|
2400
|
+
const dueAt = this.nextSqlAlarmDue();
|
|
2401
|
+
if (dueAt === null) {
|
|
2402
|
+
await txn.deleteAlarm();
|
|
2403
|
+
}
|
|
2404
|
+
else {
|
|
2405
|
+
await txn.setAlarm(dueAt);
|
|
2406
|
+
}
|
|
2407
|
+
}
|
|
2408
|
+
async commit(txn, record, updates, serviceId = "") {
|
|
2409
|
+
const updated = changed(record, updates);
|
|
2410
|
+
await this.write(txn, updated, serviceId);
|
|
2411
|
+
return updated;
|
|
2412
|
+
}
|
|
2413
|
+
async commitExpiryIfNeeded(txn, expiry) {
|
|
2414
|
+
return expiry.expired ? this.commit(txn, expiry.record, {}) : expiry.record;
|
|
2415
|
+
}
|
|
2416
|
+
async acquire(tag, body, observation) {
|
|
2417
|
+
const parsed = acquireFrom(body, tag);
|
|
2418
|
+
if (parsed.value === undefined) {
|
|
2419
|
+
return error(400, "invalid_tag_acquire", parsed.error ?? "Invalid acquire request");
|
|
2420
|
+
}
|
|
2421
|
+
const input = parsed.value;
|
|
2422
|
+
observation?.markFirstStorageRead();
|
|
2423
|
+
if (this.sqlStorage() !== undefined) {
|
|
2424
|
+
const result = await this.acquireSql(tag, input);
|
|
2425
|
+
return json(result.body, result.status);
|
|
2426
|
+
}
|
|
2427
|
+
const result = await this.ctx.storage.transaction(async (txn) => {
|
|
2428
|
+
const loaded = await this.recordFor(txn, tag);
|
|
2429
|
+
let record = loaded.record;
|
|
2430
|
+
const observed = input.expectedHead !== null;
|
|
2431
|
+
const isEventTag = input.eventTags.includes(tag);
|
|
2432
|
+
const epochError = operationEpochRejection(record, input.attemptId, input.epoch);
|
|
2433
|
+
if (epochError !== undefined) {
|
|
2434
|
+
return { ...rejected(epochError), body: { ...rejected(epochError).body, version: record.version } };
|
|
2435
|
+
}
|
|
2436
|
+
// Acquire ordering is deliberate: epoch, expired cleanup, fence, then
|
|
2437
|
+
// authoritative head comparison/reservation creation.
|
|
2438
|
+
const expiry = expireReservation(record);
|
|
2439
|
+
record = await this.commitExpiryIfNeeded(txn, expiry);
|
|
2440
|
+
if (!observed || !isEventTag) {
|
|
2441
|
+
return {
|
|
2442
|
+
status: 200,
|
|
2443
|
+
body: { status: "omitted", reservation: null, fenceGateChecked: false, version: record.version },
|
|
2444
|
+
};
|
|
2445
|
+
}
|
|
2446
|
+
if (isFenceGateClosed(record)) {
|
|
2447
|
+
return fenceGateRejected();
|
|
2448
|
+
}
|
|
2449
|
+
const assertedEmpty = input.expectedHead === "";
|
|
2450
|
+
if (assertedEmpty && (record.events.length > 0 || record.head !== "")) {
|
|
2451
|
+
return {
|
|
2452
|
+
...rejected(ASSERT_EMPTY_CONFLICT_REASON),
|
|
2453
|
+
body: { ...rejected(ASSERT_EMPTY_CONFLICT_REASON).body, version: record.version },
|
|
2454
|
+
};
|
|
2455
|
+
}
|
|
2456
|
+
if (record.head !== input.expectedHead) {
|
|
2457
|
+
return {
|
|
2458
|
+
...rejected("consistency_head_mismatch"),
|
|
2459
|
+
body: { ...rejected("consistency_head_mismatch").body, version: record.version },
|
|
2460
|
+
};
|
|
2461
|
+
}
|
|
2462
|
+
const active = record.activeReservation;
|
|
2463
|
+
if (active !== null) {
|
|
2464
|
+
if (active.attemptId === input.attemptId && active.epoch === input.epoch) {
|
|
2465
|
+
return {
|
|
2466
|
+
status: 200,
|
|
2467
|
+
body: {
|
|
2468
|
+
status: "reserved",
|
|
2469
|
+
reservation: record.activeReservation,
|
|
2470
|
+
fenceGate: inspectFenceGate(record, input.attemptId),
|
|
2471
|
+
version: record.version,
|
|
2472
|
+
},
|
|
2473
|
+
};
|
|
2474
|
+
}
|
|
2475
|
+
return {
|
|
2476
|
+
...rejected(assertedEmpty ? ASSERT_EMPTY_CONFLICT_REASON : "active_reservation_conflict"),
|
|
2477
|
+
body: {
|
|
2478
|
+
...rejected(assertedEmpty ? ASSERT_EMPTY_CONFLICT_REASON : "active_reservation_conflict").body,
|
|
2479
|
+
version: record.version,
|
|
2480
|
+
},
|
|
2481
|
+
};
|
|
2482
|
+
}
|
|
2483
|
+
const fenceGate = inspectFenceGate(record, input.attemptId);
|
|
2484
|
+
record = advanceEpoch(record, input.attemptId, input.epoch);
|
|
2485
|
+
const now = logicalNow(record);
|
|
2486
|
+
const reservation = {
|
|
2487
|
+
attemptId: input.attemptId,
|
|
2488
|
+
epoch: input.epoch,
|
|
2489
|
+
token: crypto.randomUUID(),
|
|
2490
|
+
expectedHead: input.expectedHead,
|
|
2491
|
+
expiresAt: now + RESERVATION_WINDOW_MS,
|
|
2492
|
+
alarmDueAt: Date.now() + RESERVATION_WINDOW_MS,
|
|
2493
|
+
};
|
|
2494
|
+
const updated = assertedEmpty
|
|
2495
|
+
? {
|
|
2496
|
+
...record,
|
|
2497
|
+
activeReservation: reservation,
|
|
2498
|
+
alarmDueAt: reservation.alarmDueAt,
|
|
2499
|
+
updatedAt: nowIso(),
|
|
2500
|
+
}
|
|
2501
|
+
: await this.commit(txn, record, {
|
|
2502
|
+
activeReservation: reservation,
|
|
2503
|
+
alarmDueAt: reservation.alarmDueAt,
|
|
2504
|
+
});
|
|
2505
|
+
if (assertedEmpty)
|
|
2506
|
+
await this.write(txn, updated, "");
|
|
2507
|
+
return {
|
|
2508
|
+
status: 201,
|
|
2509
|
+
body: { status: "reserved", reservation, fenceGate, version: updated.version },
|
|
2510
|
+
};
|
|
2511
|
+
});
|
|
2512
|
+
return json(result.body, result.status);
|
|
2513
|
+
}
|
|
2514
|
+
async cancel(tag, body, observation) {
|
|
2515
|
+
const parsed = reservationFrom(body);
|
|
2516
|
+
if (parsed.value === undefined) {
|
|
2517
|
+
return error(400, "invalid_tag_cancel", parsed.error ?? "Invalid cancel request");
|
|
2518
|
+
}
|
|
2519
|
+
const input = parsed.value;
|
|
2520
|
+
observation?.markFirstStorageRead();
|
|
2521
|
+
if (this.sqlStorage() !== undefined) {
|
|
2522
|
+
const result = await this.cancelSql(tag, input);
|
|
2523
|
+
return json(result.body, result.status);
|
|
2524
|
+
}
|
|
2525
|
+
const result = await this.ctx.storage.transaction(async (txn) => {
|
|
2526
|
+
const loaded = await this.recordFor(txn, tag);
|
|
2527
|
+
const expiry = expireReservation(loaded.record);
|
|
2528
|
+
let record = expiry.record;
|
|
2529
|
+
const tombstone = epochFor(record.tombstones, input.attemptId);
|
|
2530
|
+
const holdsReservation = record.activeReservation?.attemptId === input.attemptId &&
|
|
2531
|
+
record.activeReservation?.epoch === input.epoch;
|
|
2532
|
+
if (input.forceTombstone === true) {
|
|
2533
|
+
if (!holdsReservation && tombstone !== undefined && tombstone >= input.epoch) {
|
|
2534
|
+
record = await this.commitExpiryIfNeeded(txn, expiry);
|
|
2535
|
+
const sealedEpoch = withMaxEpoch(record.sealedEpoch, input.attemptId, MAX_EPOCH);
|
|
2536
|
+
if (sealedEpoch !== record.sealedEpoch) {
|
|
2537
|
+
record = await this.commit(txn, record, { sealedEpoch });
|
|
2538
|
+
}
|
|
2539
|
+
return { status: 200, body: { status: "cancelled", idempotent: true, version: record.version } };
|
|
2540
|
+
}
|
|
2541
|
+
// Force-tombstone seals all writer epochs for the attempt while leaving
|
|
2542
|
+
// unrelated active reservations untouched.
|
|
2543
|
+
record = advanceEpoch(record, input.attemptId, input.epoch);
|
|
2544
|
+
const updated = await this.commit(txn, record, {
|
|
2545
|
+
activeReservation: holdsReservation ? null : record.activeReservation,
|
|
2546
|
+
alarmDueAt: holdsReservation ? null : record.alarmDueAt,
|
|
2547
|
+
tombstones: withMaxEpoch(record.tombstones, input.attemptId, input.epoch),
|
|
2548
|
+
sealedEpoch: withMaxEpoch(record.sealedEpoch, input.attemptId, MAX_EPOCH),
|
|
2549
|
+
});
|
|
2550
|
+
return { status: 200, body: { status: "cancelled", idempotent: false, version: updated.version } };
|
|
2551
|
+
}
|
|
2552
|
+
if (!holdsReservation && tombstone === input.epoch) {
|
|
2553
|
+
record = await this.commitExpiryIfNeeded(txn, expiry);
|
|
2554
|
+
return { status: 200, body: { status: "cancelled", idempotent: true, version: record.version } };
|
|
2555
|
+
}
|
|
2556
|
+
const epochError = operationEpochRejection(record, input.attemptId, input.epoch);
|
|
2557
|
+
if (epochError !== undefined) {
|
|
2558
|
+
record = await this.commitExpiryIfNeeded(txn, expiry);
|
|
2559
|
+
return { ...rejected(epochError), body: { ...rejected(epochError).body, version: record.version } };
|
|
2560
|
+
}
|
|
2561
|
+
const active = record.activeReservation;
|
|
2562
|
+
if (active === null ||
|
|
2563
|
+
active.attemptId !== input.attemptId ||
|
|
2564
|
+
active.epoch !== input.epoch ||
|
|
2565
|
+
active.token !== input.reservationToken) {
|
|
2566
|
+
record = await this.commitExpiryIfNeeded(txn, expiry);
|
|
2567
|
+
return {
|
|
2568
|
+
...rejected("reservation_token_required"),
|
|
2569
|
+
body: { ...rejected("reservation_token_required").body, version: record.version },
|
|
2570
|
+
};
|
|
2571
|
+
}
|
|
2572
|
+
record = advanceEpoch(record, input.attemptId, input.epoch);
|
|
2573
|
+
const updated = await this.commit(txn, record, {
|
|
2574
|
+
activeReservation: null,
|
|
2575
|
+
alarmDueAt: null,
|
|
2576
|
+
tombstones: withMaxEpoch(record.tombstones, input.attemptId, input.epoch),
|
|
2577
|
+
});
|
|
2578
|
+
return { status: 200, body: { status: "cancelled", idempotent: false, version: updated.version } };
|
|
2579
|
+
});
|
|
2580
|
+
return json(result.body, result.status);
|
|
2581
|
+
}
|
|
2582
|
+
async seal(tag, body, observation) {
|
|
2583
|
+
const parsed = epochFrom(body);
|
|
2584
|
+
if (parsed.value === undefined) {
|
|
2585
|
+
return error(400, "invalid_tag_seal", parsed.error ?? "Invalid seal request");
|
|
2586
|
+
}
|
|
2587
|
+
const input = parsed.value;
|
|
2588
|
+
observation?.markFirstStorageRead();
|
|
2589
|
+
const result = await this.ctx.storage.transaction(async (txn) => {
|
|
2590
|
+
const loaded = await this.recordFor(txn, tag);
|
|
2591
|
+
const expiry = expireReservation(loaded.record);
|
|
2592
|
+
let record = expiry.record;
|
|
2593
|
+
const highest = epochFor(record.highestEpoch, input.attemptId);
|
|
2594
|
+
if (highest !== undefined && input.epoch < highest) {
|
|
2595
|
+
record = await this.commitExpiryIfNeeded(txn, expiry);
|
|
2596
|
+
return { status: 200, body: { status: "seal-ignored", highestEpoch: highest, version: record.version } };
|
|
2597
|
+
}
|
|
2598
|
+
const previouslySealed = epochFor(record.sealedEpoch, input.attemptId);
|
|
2599
|
+
record = advanceEpoch(record, input.attemptId, input.epoch);
|
|
2600
|
+
let activeReservation = record.activeReservation;
|
|
2601
|
+
let tombstones = record.tombstones;
|
|
2602
|
+
if (activeReservation !== null &&
|
|
2603
|
+
activeReservation.attemptId === input.attemptId &&
|
|
2604
|
+
activeReservation.epoch <= input.epoch) {
|
|
2605
|
+
activeReservation = null;
|
|
2606
|
+
tombstones = withMaxEpoch(tombstones, input.attemptId, input.epoch);
|
|
2607
|
+
}
|
|
2608
|
+
const sealedEpoch = withMaxEpoch(record.sealedEpoch, input.attemptId, input.epoch);
|
|
2609
|
+
const changedBySeal = previouslySealed === undefined || previouslySealed < input.epoch || activeReservation !== record.activeReservation;
|
|
2610
|
+
if (!changedBySeal) {
|
|
2611
|
+
record = await this.commitExpiryIfNeeded(txn, expiry);
|
|
2612
|
+
return { status: 200, body: { status: "sealed", highestEpoch: input.epoch, version: record.version } };
|
|
2613
|
+
}
|
|
2614
|
+
const updated = await this.commit(txn, record, {
|
|
2615
|
+
sealedEpoch,
|
|
2616
|
+
tombstones,
|
|
2617
|
+
activeReservation,
|
|
2618
|
+
alarmDueAt: activeReservation === null ? null : activeReservation.alarmDueAt,
|
|
2619
|
+
});
|
|
2620
|
+
return { status: 200, body: { status: "sealed", highestEpoch: input.epoch, version: updated.version } };
|
|
2621
|
+
});
|
|
2622
|
+
return json(result.body, result.status);
|
|
2623
|
+
}
|
|
2624
|
+
/**
|
|
2625
|
+
* Coordinator-only import admission. This is deliberately not a flag on
|
|
2626
|
+
* /append: it never creates an outbox row, never accepts a reservation, and
|
|
2627
|
+
* closes permanently once the coordinator reaches READY.
|
|
2628
|
+
*/
|
|
2629
|
+
async bootstrapAppend(tag, body, serviceId) {
|
|
2630
|
+
const parsed = bootstrapAppendFrom(body, tag);
|
|
2631
|
+
if (parsed.value === undefined)
|
|
2632
|
+
return error(400, "invalid_bootstrap_admission", parsed.error ?? "Invalid bootstrap admission");
|
|
2633
|
+
if (serviceId !== parsed.value.targetServiceId)
|
|
2634
|
+
return error(409, "bootstrap_service_scope_mismatch", "Bootstrap target service does not match tag scope");
|
|
2635
|
+
const input = parsed.value;
|
|
2636
|
+
const result = await this.ctx.storage.transaction(async (txn) => {
|
|
2637
|
+
const loaded = await this.recordFor(txn, tag);
|
|
2638
|
+
const record = loaded.record;
|
|
2639
|
+
if (record.bootstrapAdmission?.closed)
|
|
2640
|
+
return rejected("bootstrap_closed_permanently");
|
|
2641
|
+
if (record.activeReservation !== null || record.fences.length > 0 || record.outbox.length > 0)
|
|
2642
|
+
return rejected("bootstrap_requires_empty_tag");
|
|
2643
|
+
const admission = record.bootstrapAdmission ?? { importId: input.importId, leaseEpoch: input.leaseEpoch, manifestDigest: input.manifestDigest, targetServiceId: input.targetServiceId, closed: false };
|
|
2644
|
+
if (admission.importId !== input.importId || admission.leaseEpoch !== input.leaseEpoch || admission.manifestDigest !== input.manifestDigest || admission.targetServiceId !== input.targetServiceId)
|
|
2645
|
+
return rejected("bootstrap_fencing_or_manifest_mismatch");
|
|
2646
|
+
const exact = input.candidates.every((candidate) => record.events.some((event) => event.eventId === candidate.eventId && event.suid === candidate.suid && event.payload === candidate.payload && event.eventTags.join("\u0000") === candidate.eventTags.join("\u0000") && event.eventType === candidate.eventType && event.provenance === candidate.provenance && event.timestamp === candidate.timestamp));
|
|
2647
|
+
if (exact)
|
|
2648
|
+
return { status: 200, body: { status: "duplicate", version: record.version } };
|
|
2649
|
+
// A coordinator may send several bounded chunks for one tag. The
|
|
2650
|
+
// admission identity is fixed above; only duplicate replay, conflicting
|
|
2651
|
+
// identity, or non-monotonic continuation can be rejected here.
|
|
2652
|
+
if (hasEventConflict(record, input.candidates) || monotonicityViolation(record.head, input.candidates))
|
|
2653
|
+
return rejected("bootstrap_identity_or_order_conflict");
|
|
2654
|
+
const events = input.candidates.map((candidate) => ({ attemptId: `bootstrap:${input.importId}`, eventId: candidate.eventId, suid: candidate.suid, payload: candidate.payload, eventTags: candidate.eventTags, allocatorLineageId: candidate.allocatorLineageId, eventType: candidate.eventType, provenance: candidate.provenance, timestamp: candidate.timestamp }));
|
|
2655
|
+
const updated = await this.commit(txn, record, { bootstrapAdmission: admission, head: input.candidates.at(-1).suid, events: [...record.events, ...events] }, input.targetServiceId);
|
|
2656
|
+
return { status: 201, body: { status: "bootstrap_admitted", version: updated.version } };
|
|
2657
|
+
});
|
|
2658
|
+
return json(result.body, result.status);
|
|
2659
|
+
}
|
|
2660
|
+
async closeBootstrap(tag, body) {
|
|
2661
|
+
if (!isObject(body) || !isNonEmptyString(body.importId) || !isEpoch(body.leaseEpoch))
|
|
2662
|
+
return error(400, "invalid_bootstrap_close", "bootstrap importId and leaseEpoch are required");
|
|
2663
|
+
const result = await this.ctx.storage.transaction(async (txn) => {
|
|
2664
|
+
const loaded = await this.recordFor(txn, tag);
|
|
2665
|
+
const record = loaded.record;
|
|
2666
|
+
const admission = record.bootstrapAdmission;
|
|
2667
|
+
if (admission === null || admission.importId !== body.importId || admission.leaseEpoch !== body.leaseEpoch)
|
|
2668
|
+
return rejected("bootstrap_fencing_or_manifest_mismatch");
|
|
2669
|
+
if (admission.closed)
|
|
2670
|
+
return { status: 200, body: { status: "closed", version: record.version } };
|
|
2671
|
+
const updated = await this.commit(txn, record, { bootstrapAdmission: { ...admission, closed: true } });
|
|
2672
|
+
return { status: 200, body: { status: "closed", version: updated.version } };
|
|
2673
|
+
});
|
|
2674
|
+
return json(result.body, result.status);
|
|
2675
|
+
}
|
|
2676
|
+
directDoorbellPreflight(domainDeliveryClass) {
|
|
2677
|
+
const domainClass = domainDeliveryClass === undefined
|
|
2678
|
+
? undefined
|
|
2679
|
+
: readDomainDeliveryClass({ DOMAIN_DELIVERY_CLASS: domainDeliveryClass });
|
|
2680
|
+
const config = readDirectDoorbellConfig(this.env, domainClass);
|
|
2681
|
+
const preflight = preflightDirectDoorbell(config, config.allowedViews.length);
|
|
2682
|
+
if (preflight.status === "fail-fast") {
|
|
2683
|
+
throw new Error(`direct_doorbell_preflight_failed:${preflight.reason}`);
|
|
2684
|
+
}
|
|
2685
|
+
if (config.deliveryClass === "immediate-preferred" &&
|
|
2686
|
+
config.enabled &&
|
|
2687
|
+
this.env.DOWNSTREAM_DOORBELL === undefined) {
|
|
2688
|
+
if (config.degradation === "fail-fast") {
|
|
2689
|
+
throw new Error("direct_doorbell_capability_missing");
|
|
2690
|
+
}
|
|
2691
|
+
}
|
|
2692
|
+
return preflight;
|
|
2693
|
+
}
|
|
2694
|
+
async append(tag, body, serviceId, domainDeliveryClass, observation) {
|
|
2695
|
+
const parsed = appendFrom(body, tag);
|
|
2696
|
+
if (parsed.value === undefined) {
|
|
2697
|
+
return error(400, "invalid_tag_append", parsed.error ?? "Invalid append request");
|
|
2698
|
+
}
|
|
2699
|
+
const input = parsed.value;
|
|
2700
|
+
try {
|
|
2701
|
+
const doorbellPreflight = this.directDoorbellPreflight(domainDeliveryClass);
|
|
2702
|
+
observation?.markFirstStorageRead();
|
|
2703
|
+
// A real SQLite-backed DO takes the indexed transaction below. The
|
|
2704
|
+
// fallback only exists for deliberately minimal transport seams that do
|
|
2705
|
+
// not provide `storage.sql`; it is never a second persisted record
|
|
2706
|
+
// format in a deployed Tag DO.
|
|
2707
|
+
if (this.sqlStorage() !== undefined) {
|
|
2708
|
+
const sourcePartitionWasRegistered = serviceId !== null && serviceId.length > 0 &&
|
|
2709
|
+
this.sourcePartitionRegistrationStatus(tag, serviceId) === "registered";
|
|
2710
|
+
const sourcePartitionRegistration = serviceId !== null && serviceId.length > 0
|
|
2711
|
+
? await this.ensureSourcePartitionBeforeFirstAppend(tag, serviceId)
|
|
2712
|
+
: "unconfigured";
|
|
2713
|
+
const result = await this.appendSql(tag, input, serviceId, sourcePartitionRegistration !== "unconfigured");
|
|
2714
|
+
for (const fact of result.hopFacts ?? []) {
|
|
2715
|
+
this.scheduleDurableHop({
|
|
2716
|
+
stage: "tag-append-committed",
|
|
2717
|
+
serviceId: serviceId ?? "",
|
|
2718
|
+
eventId: fact.eventId,
|
|
2719
|
+
suid: fact.suid,
|
|
2720
|
+
attemptId: fact.attemptId,
|
|
2721
|
+
partitionTag: fact.tag,
|
|
2722
|
+
observedAt: fact.tagAppendCommittedAt,
|
|
2723
|
+
});
|
|
2724
|
+
this.scheduleDurableHop({
|
|
2725
|
+
stage: "outbox-obligation-written",
|
|
2726
|
+
serviceId: serviceId ?? "",
|
|
2727
|
+
eventId: fact.eventId,
|
|
2728
|
+
suid: fact.suid,
|
|
2729
|
+
attemptId: fact.attemptId,
|
|
2730
|
+
partitionTag: fact.tag,
|
|
2731
|
+
observedAt: fact.obligationWrittenAt,
|
|
2732
|
+
});
|
|
2733
|
+
}
|
|
2734
|
+
if ((result.status === 201 || result.status === 200) &&
|
|
2735
|
+
serviceId !== null && serviceId.length > 0 &&
|
|
2736
|
+
sourcePartitionRegistration !== "unconfigured") {
|
|
2737
|
+
this.scheduleSourcePartitionWatermark(tag, serviceId);
|
|
2738
|
+
}
|
|
2739
|
+
const response = json(result.body, result.status);
|
|
2740
|
+
if (result.status === 201 &&
|
|
2741
|
+
serviceId !== null && serviceId.length > 0 &&
|
|
2742
|
+
this.env.AUTO_DRAIN_OUTBOX === "true" &&
|
|
2743
|
+
(this.env.DOWNSTREAM_QUEUE !== undefined ||
|
|
2744
|
+
(doorbellPreflight.status === "ready" && this.env.DOWNSTREAM_DOORBELL !== undefined))) {
|
|
2745
|
+
const directRows = doorbellPreflight.status === "ready" && this.env.DOWNSTREAM_DOORBELL !== undefined
|
|
2746
|
+
? await this.directDeliveryBeforeResponse(tag, serviceId)
|
|
2747
|
+
: undefined;
|
|
2748
|
+
// An explicitly unconfigured completeness store is the pre-G65
|
|
2749
|
+
// local composition: it has no global-admission authority to
|
|
2750
|
+
// consult. Keep the committed response and the ordinary Queue
|
|
2751
|
+
// fallback on that path without introducing a synchronous D1
|
|
2752
|
+
// attempt merely because a non-authoritative local D1 binding is
|
|
2753
|
+
// present. Configured G44 stores retain the bounded admission
|
|
2754
|
+
// attempt and header contract above.
|
|
2755
|
+
if (sourcePartitionRegistration !== "unconfigured" || sourcePartitionWasRegistered) {
|
|
2756
|
+
withGlobalAdmission(response, await this.globalAdmissionBeforeResponse(tag, serviceId, directRows));
|
|
2757
|
+
}
|
|
2758
|
+
this.startAutoDrainBeforeResponse(tag, serviceId, domainDeliveryClass, directRows);
|
|
2759
|
+
}
|
|
2760
|
+
return response;
|
|
2761
|
+
}
|
|
2762
|
+
const result = await this.ctx.storage.transaction(async (txn) => {
|
|
2763
|
+
const loaded = await this.recordFor(txn, tag);
|
|
2764
|
+
let record = loaded.record;
|
|
2765
|
+
// Required order begins here: exact duplicate is before any epoch or token check.
|
|
2766
|
+
if (batchIsExactDuplicate(record, input)) {
|
|
2767
|
+
return { status: 200, body: { status: "duplicate", version: record.version } };
|
|
2768
|
+
}
|
|
2769
|
+
const expiry = expireReservation(record);
|
|
2770
|
+
record = expiry.record;
|
|
2771
|
+
const epochError = operationEpochRejection(record, input.attemptId, input.epoch);
|
|
2772
|
+
if (epochError !== undefined) {
|
|
2773
|
+
record = await this.commitExpiryIfNeeded(txn, expiry);
|
|
2774
|
+
return { ...rejected(epochError), body: { ...rejected(epochError).body, version: record.version } };
|
|
2775
|
+
}
|
|
2776
|
+
// Exact duplicates are intentionally above the epoch check. Every
|
|
2777
|
+
// non-duplicate append checks the set gate before ownership/token or
|
|
2778
|
+
// content guards, so a fence cannot be bypassed by a stale caller.
|
|
2779
|
+
if (isFenceGateClosed(record)) {
|
|
2780
|
+
await this.commitExpiryIfNeeded(txn, expiry);
|
|
2781
|
+
return fenceGateRejected();
|
|
2782
|
+
}
|
|
2783
|
+
const active = record.activeReservation;
|
|
2784
|
+
const confirmsReservation = active !== null &&
|
|
2785
|
+
active.attemptId === input.attemptId &&
|
|
2786
|
+
active.epoch === input.epoch &&
|
|
2787
|
+
active.token === input.reservationToken;
|
|
2788
|
+
if (active !== null &&
|
|
2789
|
+
(active.attemptId !== input.attemptId ||
|
|
2790
|
+
active.epoch !== input.epoch ||
|
|
2791
|
+
active.token !== input.reservationToken)) {
|
|
2792
|
+
record = await this.commitExpiryIfNeeded(txn, expiry);
|
|
2793
|
+
return {
|
|
2794
|
+
...rejected("reservation_token_required"),
|
|
2795
|
+
body: { ...rejected("reservation_token_required").body, version: record.version },
|
|
2796
|
+
};
|
|
2797
|
+
}
|
|
2798
|
+
if (hasEventConflict(record, input.candidates)) {
|
|
2799
|
+
record = await this.commitExpiryIfNeeded(txn, expiry);
|
|
2800
|
+
return {
|
|
2801
|
+
...rejected("event_conflict"),
|
|
2802
|
+
body: { ...rejected("event_conflict").body, version: record.version },
|
|
2803
|
+
};
|
|
2804
|
+
}
|
|
2805
|
+
if (monotonicityViolation(record.head, input.candidates)) {
|
|
2806
|
+
record = await this.commitExpiryIfNeeded(txn, expiry);
|
|
2807
|
+
return {
|
|
2808
|
+
...rejected("non_monotonic_suid"),
|
|
2809
|
+
body: { ...rejected("non_monotonic_suid").body, version: record.version },
|
|
2810
|
+
};
|
|
2811
|
+
}
|
|
2812
|
+
const fenceGate = inspectFenceGate(record, input.attemptId);
|
|
2813
|
+
record = advanceEpoch(record, input.attemptId, input.epoch);
|
|
2814
|
+
const appendedEvents = input.candidates.map((candidate) => ({
|
|
2815
|
+
attemptId: input.attemptId,
|
|
2816
|
+
eventId: candidate.eventId,
|
|
2817
|
+
suid: candidate.suid,
|
|
2818
|
+
payload: candidate.payload,
|
|
2819
|
+
eventTags: candidate.eventTags,
|
|
2820
|
+
allocatorLineageId: candidate.allocatorLineageId,
|
|
2821
|
+
eventType: candidate.eventType,
|
|
2822
|
+
provenance: candidate.provenance,
|
|
2823
|
+
timestamp: candidate.timestamp,
|
|
2824
|
+
}));
|
|
2825
|
+
const outboxRows = input.candidates.map((candidate) => ({
|
|
2826
|
+
attemptId: input.attemptId,
|
|
2827
|
+
eventId: candidate.eventId,
|
|
2828
|
+
suid: candidate.suid,
|
|
2829
|
+
payload: candidate.payload,
|
|
2830
|
+
allocatorLineageId: candidate.allocatorLineageId,
|
|
2831
|
+
eventType: candidate.eventType,
|
|
2832
|
+
provenance: candidate.provenance,
|
|
2833
|
+
timestamp: candidate.timestamp,
|
|
2834
|
+
}));
|
|
2835
|
+
const head = input.candidates[input.candidates.length - 1].suid;
|
|
2836
|
+
const appendedOnly = changed(record, {
|
|
2837
|
+
head,
|
|
2838
|
+
events: [...record.events, ...appendedEvents],
|
|
2839
|
+
outbox: [...record.outbox, ...outboxRows],
|
|
2840
|
+
});
|
|
2841
|
+
await this.write(txn, appendedOnly, serviceId ?? "");
|
|
2842
|
+
if (input.faultInjection === "after-append-before-confirm") {
|
|
2843
|
+
throw new AppendTransactionFault("Simulated interruption before reservation confirmation");
|
|
2844
|
+
}
|
|
2845
|
+
const updated = changed(record, {
|
|
2846
|
+
head,
|
|
2847
|
+
events: [...record.events, ...appendedEvents],
|
|
2848
|
+
outbox: [...record.outbox, ...outboxRows],
|
|
2849
|
+
activeReservation: null,
|
|
2850
|
+
alarmDueAt: null,
|
|
2851
|
+
confirmations: confirmsReservation
|
|
2852
|
+
? withMaxEpoch(record.confirmations, input.attemptId, input.epoch)
|
|
2853
|
+
: record.confirmations,
|
|
2854
|
+
});
|
|
2855
|
+
await this.write(txn, updated, serviceId ?? "");
|
|
2856
|
+
return {
|
|
2857
|
+
status: 201,
|
|
2858
|
+
body: { status: "appended", fenceGate, version: updated.version },
|
|
2859
|
+
};
|
|
2860
|
+
});
|
|
2861
|
+
const response = json(result.body, result.status);
|
|
2862
|
+
if (result.status === 201 &&
|
|
2863
|
+
serviceId !== null && serviceId.length > 0 &&
|
|
2864
|
+
this.env.AUTO_DRAIN_OUTBOX === "true" &&
|
|
2865
|
+
(this.env.DOWNSTREAM_QUEUE !== undefined ||
|
|
2866
|
+
(doorbellPreflight.status === "ready" && this.env.DOWNSTREAM_DOORBELL !== undefined))) {
|
|
2867
|
+
const directRows = doorbellPreflight.status === "ready" && this.env.DOWNSTREAM_DOORBELL !== undefined
|
|
2868
|
+
? await this.directDeliveryBeforeResponse(tag, serviceId)
|
|
2869
|
+
: undefined;
|
|
2870
|
+
withGlobalAdmission(response, await this.globalAdmissionBeforeResponse(tag, serviceId, directRows));
|
|
2871
|
+
this.startAutoDrainBeforeResponse(tag, serviceId, domainDeliveryClass, directRows);
|
|
2872
|
+
}
|
|
2873
|
+
return response;
|
|
2874
|
+
}
|
|
2875
|
+
catch (failure) {
|
|
2876
|
+
if (failure instanceof PartitionRegistrationUnavailableError) {
|
|
2877
|
+
return error(503, "partition_registration_unavailable", "Source partition registration is unavailable; retry the commit.", true);
|
|
2878
|
+
}
|
|
2879
|
+
if (failure instanceof AppendTransactionFault) {
|
|
2880
|
+
return error(503, "simulated_append_crash", "Simulated interruption before atomic append and confirmation commit");
|
|
2881
|
+
}
|
|
2882
|
+
return error(500, "tag_append_failure", "Tag append could not be persisted");
|
|
2883
|
+
}
|
|
2884
|
+
}
|
|
2885
|
+
/**
|
|
2886
|
+
* Claim and run the opted-in direct unsafe path after the append transaction
|
|
2887
|
+
* has committed, but before the 201 response is returned. The rows are
|
|
2888
|
+
* handed to the Queue drain afterwards so the durable Queue remains the
|
|
2889
|
+
* owner of global admission, ordering, retries, and DLQ recovery.
|
|
2890
|
+
*
|
|
2891
|
+
* A pending-row read can fail independently of the already committed
|
|
2892
|
+
* append. In that case the caller falls back to the ordinary Queue drain;
|
|
2893
|
+
* it never turns a durable acceptance into a response failure.
|
|
2894
|
+
*/
|
|
2895
|
+
async directDeliveryBeforeResponse(tag, serviceId) {
|
|
2896
|
+
let rows;
|
|
2897
|
+
const attempt = await this.boundedDerivedWrite(async () => {
|
|
2898
|
+
const pending = await this.pendingOutbox(tag, serviceId, { nowMs: Date.now(), force: true });
|
|
2899
|
+
if (!pending.ok)
|
|
2900
|
+
throw new Error(`direct outbox pending read failed with ${pending.status}`);
|
|
2901
|
+
const body = await pending.json();
|
|
2902
|
+
rows = body.rows ?? [];
|
|
2903
|
+
await this.deliverDirectRows(rows);
|
|
2904
|
+
});
|
|
2905
|
+
if (attempt.status === "timeout") {
|
|
2906
|
+
// Rows already claimed by pendingOutbox are handed to the Queue without
|
|
2907
|
+
// another direct attempt. If the pending read itself hung, rows stays
|
|
2908
|
+
// undefined and the unchanged scheduler-backed drain rereads it later.
|
|
2909
|
+
console.warn("direct_doorbell_before_response", {
|
|
2910
|
+
status: "queued-degraded",
|
|
2911
|
+
reason: "direct_doorbell_before_response_timeout",
|
|
2912
|
+
budgetMs: G65_DERIVED_WRITE_BUDGET_MS,
|
|
2913
|
+
});
|
|
2914
|
+
return rows;
|
|
2915
|
+
}
|
|
2916
|
+
if (attempt.status === "failed") {
|
|
2917
|
+
console.warn("direct_doorbell_before_response", {
|
|
2918
|
+
status: "queued-degraded",
|
|
2919
|
+
reason: "direct_doorbell_before_response_failed",
|
|
2920
|
+
error: String(attempt.error),
|
|
2921
|
+
});
|
|
2922
|
+
return rows;
|
|
2923
|
+
}
|
|
2924
|
+
return rows;
|
|
2925
|
+
}
|
|
2926
|
+
/**
|
|
2927
|
+
* Attempt the same idempotent D1 admission used by the Queue consumer.
|
|
2928
|
+
* This is a derived write only: the Tag event, outbox obligation, and local
|
|
2929
|
+
* receipt are already durable, and the Queue remains the guarantee.
|
|
2930
|
+
*/
|
|
2931
|
+
async globalAdmissionBeforeResponse(tag, serviceId, preloadedRows) {
|
|
2932
|
+
let rows = preloadedRows;
|
|
2933
|
+
const admissionStartedAt = Date.now();
|
|
2934
|
+
const completedAtByEventId = new Map();
|
|
2935
|
+
const attempt = await this.boundedDerivedWrite(async () => {
|
|
2936
|
+
if (rows === undefined) {
|
|
2937
|
+
const pending = await this.pendingOutbox(tag, serviceId, { nowMs: Date.now(), force: true });
|
|
2938
|
+
if (!pending.ok)
|
|
2939
|
+
throw new Error(`global admission pending read failed with ${pending.status}`);
|
|
2940
|
+
const body = await pending.json();
|
|
2941
|
+
rows = body.rows ?? [];
|
|
2942
|
+
}
|
|
2943
|
+
if (rows.length === 0)
|
|
2944
|
+
return;
|
|
2945
|
+
if (this.env.D1 === undefined)
|
|
2946
|
+
throw new Error("global admission D1 binding is unavailable");
|
|
2947
|
+
const store = new D1EventStore(this.env.D1);
|
|
2948
|
+
await store.initialize();
|
|
2949
|
+
for (const row of rows) {
|
|
2950
|
+
const outcome = await store.recordDelivery(row, Date.now(), "fast");
|
|
2951
|
+
if (outcome.outcome !== "stored") {
|
|
2952
|
+
throw new Error(`global admission rejected:${outcome.outcome}`);
|
|
2953
|
+
}
|
|
2954
|
+
completedAtByEventId.set(row.eventId, Date.now());
|
|
2955
|
+
}
|
|
2956
|
+
});
|
|
2957
|
+
const admissionFinishedAt = Date.now();
|
|
2958
|
+
const admissionOutcome = attempt.status === "timeout"
|
|
2959
|
+
? "unknown"
|
|
2960
|
+
: attempt.status === "failed"
|
|
2961
|
+
? "not-admitted"
|
|
2962
|
+
: "admitted";
|
|
2963
|
+
for (const row of rows ?? []) {
|
|
2964
|
+
this.scheduleG65AdmissionAttempt({
|
|
2965
|
+
serviceId: row.serviceId,
|
|
2966
|
+
eventId: row.eventId,
|
|
2967
|
+
suid: row.suid,
|
|
2968
|
+
attemptId: row.attemptId,
|
|
2969
|
+
partitionTag: row.tag,
|
|
2970
|
+
deliverySource: "fast",
|
|
2971
|
+
admissionStartedAt,
|
|
2972
|
+
admissionFinishedAt,
|
|
2973
|
+
outcome: completedAtByEventId.has(row.eventId) ? "admitted" : admissionOutcome,
|
|
2974
|
+
globalCompletionObservedAt: completedAtByEventId.get(row.eventId) ?? null,
|
|
2975
|
+
});
|
|
2976
|
+
}
|
|
2977
|
+
if (attempt.status === "timeout") {
|
|
2978
|
+
console.warn("global_admission_before_response", {
|
|
2979
|
+
status: "unknown",
|
|
2980
|
+
reason: "global_admission_before_response_timeout",
|
|
2981
|
+
budgetMs: G65_DERIVED_WRITE_BUDGET_MS,
|
|
2982
|
+
});
|
|
2983
|
+
return "unknown";
|
|
2984
|
+
}
|
|
2985
|
+
if (attempt.status === "failed") {
|
|
2986
|
+
console.warn("global_admission_before_response", {
|
|
2987
|
+
status: "not-admitted",
|
|
2988
|
+
reason: "global_admission_before_response_failed",
|
|
2989
|
+
error: String(attempt.error),
|
|
2990
|
+
});
|
|
2991
|
+
return "not-admitted";
|
|
2992
|
+
}
|
|
2993
|
+
return "admitted";
|
|
2994
|
+
}
|
|
2995
|
+
scheduleG65AdmissionAttempt(input) {
|
|
2996
|
+
const database = this.env.D1;
|
|
2997
|
+
if (database === undefined)
|
|
2998
|
+
return;
|
|
2999
|
+
this.ctx.waitUntil(recordG65AdmissionAttempt(database, input).catch(() => undefined));
|
|
3000
|
+
}
|
|
3001
|
+
/**
|
|
3002
|
+
* Bound a derived attempt without cancelling or making its eventual result
|
|
3003
|
+
* part of commit semantics. The rejection branch is attached immediately
|
|
3004
|
+
* so a late D1/doorbell failure cannot become an unhandled rejection.
|
|
3005
|
+
*/
|
|
3006
|
+
async boundedDerivedWrite(operation, budgetMs = G65_DERIVED_WRITE_BUDGET_MS) {
|
|
3007
|
+
let timer;
|
|
3008
|
+
const operationResult = Promise.resolve()
|
|
3009
|
+
.then(operation)
|
|
3010
|
+
.then((value) => ({ status: "completed", value }), (error) => ({ status: "failed", error }));
|
|
3011
|
+
const timeout = new Promise((resolve) => {
|
|
3012
|
+
timer = setTimeout(() => resolve({ status: "timeout" }), budgetMs);
|
|
3013
|
+
});
|
|
3014
|
+
try {
|
|
3015
|
+
return await Promise.race([operationResult, timeout]);
|
|
3016
|
+
}
|
|
3017
|
+
finally {
|
|
3018
|
+
if (timer !== undefined)
|
|
3019
|
+
clearTimeout(timer);
|
|
3020
|
+
}
|
|
3021
|
+
}
|
|
3022
|
+
/**
|
|
3023
|
+
* The append transaction has durably committed the Tag event, outbox
|
|
3024
|
+
* obligation, and receipt before this handoff is started. A direct delivery
|
|
3025
|
+
* has already run when `preclaimedRows` is present; the retained promise
|
|
3026
|
+
* then performs only the unchanged Queue submission for those same bytes.
|
|
3027
|
+
* With no preclaimed rows this remains the existing waitUntil-backed drain,
|
|
3028
|
+
* including its direct-before-Queue behavior for alarms and fallback paths.
|
|
3029
|
+
*/
|
|
3030
|
+
startAutoDrainBeforeResponse(tag, serviceId, domainDeliveryClass, preclaimedRows) {
|
|
3031
|
+
const drain = this.autoDrainOutbox(tag, serviceId, domainDeliveryClass, undefined, preclaimedRows, preclaimedRows !== undefined).catch(() => undefined);
|
|
3032
|
+
this.ctx.waitUntil(drain);
|
|
3033
|
+
}
|
|
3034
|
+
async deliverDirectRows(rows) {
|
|
3035
|
+
for (const row of rows) {
|
|
3036
|
+
// `row` is the object created by pendingOutbox. The bytes are captured
|
|
3037
|
+
// once for correlation/equality evidence and the exact same envelope is
|
|
3038
|
+
// handed to both transports; no reduced TagOutboxRow is reconstructed.
|
|
3039
|
+
const envelopeBytes = downstreamEnvelopeBytes(row);
|
|
3040
|
+
const correlationId = deliveryCorrelationId(row, "fast");
|
|
3041
|
+
try {
|
|
3042
|
+
const delivered = await this.env.DOWNSTREAM_DOORBELL.deliver(row);
|
|
3043
|
+
const resultCorrelationId = typeof delivered === "object" && delivered !== null &&
|
|
3044
|
+
"correlationId" in delivered &&
|
|
3045
|
+
typeof delivered.correlationId === "string"
|
|
3046
|
+
? delivered.correlationId
|
|
3047
|
+
: correlationId;
|
|
3048
|
+
if (typeof delivered === "object" && delivered !== null &&
|
|
3049
|
+
"fastDisposition" in delivered &&
|
|
3050
|
+
delivered.fastDisposition === "failed") {
|
|
3051
|
+
console.warn("direct_doorbell_status", {
|
|
3052
|
+
status: "failed",
|
|
3053
|
+
reason: "receiver_delivery_failed",
|
|
3054
|
+
correlationId: resultCorrelationId,
|
|
3055
|
+
envelopeBytes,
|
|
3056
|
+
});
|
|
3057
|
+
throw new Error("doorbell_receiver_delivery_failed");
|
|
3058
|
+
}
|
|
3059
|
+
console.log("direct_doorbell_delivery", {
|
|
3060
|
+
correlationId: resultCorrelationId,
|
|
3061
|
+
envelopeBytes,
|
|
3062
|
+
status: "success",
|
|
3063
|
+
});
|
|
3064
|
+
}
|
|
3065
|
+
catch (error) {
|
|
3066
|
+
// A direct failure is explicit and the durable Queue is the backstop.
|
|
3067
|
+
// Replaying the same envelope is safe because the core is
|
|
3068
|
+
// receipt-idempotent; this is not a silent capability downgrade.
|
|
3069
|
+
console.warn("direct_doorbell_status", {
|
|
3070
|
+
status: "queued-degraded",
|
|
3071
|
+
reason: "direct_doorbell_delivery_failed",
|
|
3072
|
+
correlationId,
|
|
3073
|
+
envelopeBytes,
|
|
3074
|
+
failureKind: classifyDirectDoorbellFailure(error),
|
|
3075
|
+
error: String(error),
|
|
3076
|
+
});
|
|
3077
|
+
}
|
|
3078
|
+
}
|
|
3079
|
+
}
|
|
3080
|
+
async autoDrainOutbox(tag, serviceId, domainDeliveryClass, limit, preclaimedRows, skipDirect = false) {
|
|
3081
|
+
const domainClass = domainDeliveryClass === undefined
|
|
3082
|
+
? undefined
|
|
3083
|
+
: readDomainDeliveryClass({ DOMAIN_DELIVERY_CLASS: domainDeliveryClass });
|
|
3084
|
+
const config = readDirectDoorbellConfig(this.env, domainClass);
|
|
3085
|
+
const preflight = preflightDirectDoorbell(config, config.allowedViews.length);
|
|
3086
|
+
if (preflight.status === "fail-fast") {
|
|
3087
|
+
throw new Error(`direct_doorbell_preflight_failed:${preflight.reason}`);
|
|
3088
|
+
}
|
|
3089
|
+
const rows = preclaimedRows ?? await (async () => {
|
|
3090
|
+
const pending = await this.pendingOutbox(tag, serviceId, { nowMs: Date.now(), limit });
|
|
3091
|
+
if (!pending.ok) {
|
|
3092
|
+
throw new Error(`automatic outbox pending read failed with ${pending.status}`);
|
|
3093
|
+
}
|
|
3094
|
+
const body = await pending.json();
|
|
3095
|
+
return body.rows ?? [];
|
|
3096
|
+
})();
|
|
3097
|
+
const directReady = preflight.status === "ready" &&
|
|
3098
|
+
config.deliveryClass === "immediate-preferred" &&
|
|
3099
|
+
config.enabled &&
|
|
3100
|
+
this.env.DOWNSTREAM_DOORBELL !== undefined;
|
|
3101
|
+
const queue = this.env.DOWNSTREAM_QUEUE;
|
|
3102
|
+
for (const row of rows) {
|
|
3103
|
+
// `row` is the object created by pendingOutbox. The bytes are captured
|
|
3104
|
+
// once for correlation/equality evidence and the exact same envelope is
|
|
3105
|
+
// handed to both transports; no reduced TagOutboxRow is reconstructed.
|
|
3106
|
+
const envelopeBytes = downstreamEnvelopeBytes(row);
|
|
3107
|
+
const correlationId = deliveryCorrelationId(row, "fast");
|
|
3108
|
+
if (directReady && !skipDirect)
|
|
3109
|
+
await this.deliverDirectRows([row]);
|
|
3110
|
+
if (!directReady && config.deliveryClass === "immediate-preferred" && config.enabled) {
|
|
3111
|
+
console.warn("direct_doorbell_status", {
|
|
3112
|
+
status: "queued-degraded",
|
|
3113
|
+
reason: "direct_doorbell_not_ready",
|
|
3114
|
+
correlationId,
|
|
3115
|
+
envelopeBytes,
|
|
3116
|
+
});
|
|
3117
|
+
}
|
|
3118
|
+
if (queue !== undefined) {
|
|
3119
|
+
try {
|
|
3120
|
+
await queue.send(row, { contentType: "json" });
|
|
3121
|
+
this.scheduleDurableHop({
|
|
3122
|
+
stage: "queue-send-returned",
|
|
3123
|
+
serviceId,
|
|
3124
|
+
eventId: row.eventId,
|
|
3125
|
+
suid: row.suid,
|
|
3126
|
+
attemptId: row.attemptId,
|
|
3127
|
+
partitionTag: tag,
|
|
3128
|
+
transport: "queue",
|
|
3129
|
+
observedAt: Date.now(),
|
|
3130
|
+
});
|
|
3131
|
+
}
|
|
3132
|
+
catch (failure) {
|
|
3133
|
+
// A failed source-to-sink handoff is itself source state. Do not
|
|
3134
|
+
// let one poison row skip a later due row or suppress re-arming.
|
|
3135
|
+
await this.recordOutboxFailure(tag, row, failure);
|
|
3136
|
+
}
|
|
3137
|
+
}
|
|
3138
|
+
}
|
|
3139
|
+
}
|
|
3140
|
+
/** G60 writes are observation-only and stay outside the append/drain result. */
|
|
3141
|
+
scheduleDurableHop(input) {
|
|
3142
|
+
if (this.env.D1 === undefined)
|
|
3143
|
+
return;
|
|
3144
|
+
this.ctx.waitUntil(recordDurableHop(this.env.D1, input).catch(() => undefined));
|
|
3145
|
+
}
|
|
3146
|
+
async confirm(tag, body) {
|
|
3147
|
+
const parsed = reservationFrom(body);
|
|
3148
|
+
if (parsed.value === undefined) {
|
|
3149
|
+
return error(400, "invalid_tag_confirm", parsed.error ?? "Invalid confirm request");
|
|
3150
|
+
}
|
|
3151
|
+
const input = parsed.value;
|
|
3152
|
+
const result = await this.ctx.storage.transaction(async (txn) => {
|
|
3153
|
+
const loaded = await this.recordFor(txn, tag);
|
|
3154
|
+
const expiry = expireReservation(loaded.record);
|
|
3155
|
+
let record = expiry.record;
|
|
3156
|
+
if (epochFor(record.confirmations, input.attemptId) === input.epoch) {
|
|
3157
|
+
record = await this.commitExpiryIfNeeded(txn, expiry);
|
|
3158
|
+
return { status: 200, body: { status: "confirmed", idempotent: true, version: record.version } };
|
|
3159
|
+
}
|
|
3160
|
+
const epochError = operationEpochRejection(record, input.attemptId, input.epoch);
|
|
3161
|
+
if (epochError !== undefined) {
|
|
3162
|
+
record = await this.commitExpiryIfNeeded(txn, expiry);
|
|
3163
|
+
return { ...rejected(epochError), body: { ...rejected(epochError).body, version: record.version } };
|
|
3164
|
+
}
|
|
3165
|
+
const active = record.activeReservation;
|
|
3166
|
+
if (active !== null &&
|
|
3167
|
+
active.attemptId === input.attemptId &&
|
|
3168
|
+
active.epoch === input.epoch &&
|
|
3169
|
+
active.token === input.reservationToken) {
|
|
3170
|
+
record = await this.commitExpiryIfNeeded(txn, expiry);
|
|
3171
|
+
return {
|
|
3172
|
+
...rejected("append_required_for_confirmation"),
|
|
3173
|
+
body: { ...rejected("append_required_for_confirmation").body, version: record.version },
|
|
3174
|
+
};
|
|
3175
|
+
}
|
|
3176
|
+
record = await this.commitExpiryIfNeeded(txn, expiry);
|
|
3177
|
+
return {
|
|
3178
|
+
...rejected("confirmation_not_found"),
|
|
3179
|
+
body: { ...rejected("confirmation_not_found").body, version: record.version },
|
|
3180
|
+
};
|
|
3181
|
+
});
|
|
3182
|
+
return json(result.body, result.status);
|
|
3183
|
+
}
|
|
3184
|
+
async installFence(tag, body) {
|
|
3185
|
+
const parsed = fenceFrom(body);
|
|
3186
|
+
if (parsed.value === undefined) {
|
|
3187
|
+
return error(400, "invalid_tag_fence", parsed.error ?? "Invalid fence request");
|
|
3188
|
+
}
|
|
3189
|
+
const input = parsed.value;
|
|
3190
|
+
const result = await this.ctx.storage.transaction(async (txn) => {
|
|
3191
|
+
const loaded = await this.recordFor(txn, tag);
|
|
3192
|
+
const expiry = expireReservation(loaded.record);
|
|
3193
|
+
let record = expiry.record;
|
|
3194
|
+
const existing = fenceFor(record.fences, input.reason, input.attemptId);
|
|
3195
|
+
if (existing?.epoch === input.epoch) {
|
|
3196
|
+
record = await this.commitExpiryIfNeeded(txn, expiry);
|
|
3197
|
+
return { status: 200, body: { status: "fence-installed", idempotent: true, version: record.version } };
|
|
3198
|
+
}
|
|
3199
|
+
const epochError = fenceEpochRejection(record, input.reason, input.attemptId, input.epoch);
|
|
3200
|
+
if (epochError !== undefined) {
|
|
3201
|
+
record = await this.commitExpiryIfNeeded(txn, expiry);
|
|
3202
|
+
return { ...rejected(epochError), body: { ...rejected(epochError).body, version: record.version } };
|
|
3203
|
+
}
|
|
3204
|
+
const overlap = overlappingFenceReason(record, input.reason);
|
|
3205
|
+
if (overlap !== undefined) {
|
|
3206
|
+
record = await this.commitExpiryIfNeeded(txn, expiry);
|
|
3207
|
+
return {
|
|
3208
|
+
...rejected("fence_overlap_blocked"),
|
|
3209
|
+
body: {
|
|
3210
|
+
...rejected("fence_overlap_blocked").body,
|
|
3211
|
+
error: `Fence authorization is blocked by ${overlap}`,
|
|
3212
|
+
version: record.version,
|
|
3213
|
+
},
|
|
3214
|
+
};
|
|
3215
|
+
}
|
|
3216
|
+
record = advanceEpoch(record, input.attemptId, input.epoch);
|
|
3217
|
+
const fence = { reason: input.reason, attemptId: input.attemptId, epoch: input.epoch };
|
|
3218
|
+
const updated = await this.commit(txn, record, {
|
|
3219
|
+
fences: withFence(record.fences, fence),
|
|
3220
|
+
clearedFences: withoutFence(record.clearedFences, input.reason, input.attemptId),
|
|
3221
|
+
});
|
|
3222
|
+
return { status: 201, body: { status: "fence-installed", idempotent: false, version: updated.version } };
|
|
3223
|
+
});
|
|
3224
|
+
return json(result.body, result.status);
|
|
3225
|
+
}
|
|
3226
|
+
async clearFence(tag, body) {
|
|
3227
|
+
const parsed = fenceFrom(body);
|
|
3228
|
+
if (parsed.value === undefined) {
|
|
3229
|
+
return error(400, "invalid_tag_fence", parsed.error ?? "Invalid fence request");
|
|
3230
|
+
}
|
|
3231
|
+
const input = parsed.value;
|
|
3232
|
+
const result = await this.ctx.storage.transaction(async (txn) => {
|
|
3233
|
+
const loaded = await this.recordFor(txn, tag);
|
|
3234
|
+
const expiry = expireReservation(loaded.record);
|
|
3235
|
+
let record = expiry.record;
|
|
3236
|
+
const existing = fenceFor(record.fences, input.reason, input.attemptId);
|
|
3237
|
+
const alreadyCleared = fenceFor(record.clearedFences, input.reason, input.attemptId);
|
|
3238
|
+
if (alreadyCleared !== undefined && alreadyCleared.epoch >= input.epoch) {
|
|
3239
|
+
record = await this.commitExpiryIfNeeded(txn, expiry);
|
|
3240
|
+
return { status: 200, body: { status: "fence-cleared", idempotent: true, version: record.version } };
|
|
3241
|
+
}
|
|
3242
|
+
const epochError = existing !== undefined && input.epoch < existing.epoch
|
|
3243
|
+
? "stale_epoch"
|
|
3244
|
+
: fenceEpochRejection(record, input.reason, input.attemptId, input.epoch);
|
|
3245
|
+
if (epochError !== undefined) {
|
|
3246
|
+
record = await this.commitExpiryIfNeeded(txn, expiry);
|
|
3247
|
+
return { ...rejected(epochError), body: { ...rejected(epochError).body, version: record.version } };
|
|
3248
|
+
}
|
|
3249
|
+
record = advanceEpoch(record, input.attemptId, input.epoch);
|
|
3250
|
+
const fence = { reason: input.reason, attemptId: input.attemptId, epoch: input.epoch };
|
|
3251
|
+
const updated = await this.commit(txn, record, {
|
|
3252
|
+
fences: withoutFence(record.fences, input.reason, input.attemptId),
|
|
3253
|
+
clearedFences: withFence(record.clearedFences, fence),
|
|
3254
|
+
});
|
|
3255
|
+
return { status: 200, body: { status: "fence-cleared", idempotent: false, version: updated.version } };
|
|
3256
|
+
});
|
|
3257
|
+
return json(result.body, result.status);
|
|
3258
|
+
}
|
|
3259
|
+
/**
|
|
3260
|
+
* Returns Tag-side repair facts only. Journal observations are deliberately
|
|
3261
|
+
* absent: they are progress telemetry and can never grant repair authority.
|
|
3262
|
+
*/
|
|
3263
|
+
async repairFacts(tag) {
|
|
3264
|
+
const result = await this.ctx.storage.transaction(async (txn) => {
|
|
3265
|
+
const record = this.readStoredRecord(tag);
|
|
3266
|
+
if (record === undefined) {
|
|
3267
|
+
return { status: 404, body: { error: "Tag has no durable state yet", code: "tag_not_found" } };
|
|
3268
|
+
}
|
|
3269
|
+
const normalized = requireG32TagRecord(record);
|
|
3270
|
+
if (normalized.tag !== tag) {
|
|
3271
|
+
return { status: 409, body: { error: "Tag Durable Object identity changed", code: "tag_identity_conflict" } };
|
|
3272
|
+
}
|
|
3273
|
+
const facts = (await txn.get(REPAIR_FACTS_KEY)) ?? repairFactsDefault();
|
|
3274
|
+
return {
|
|
3275
|
+
status: 200,
|
|
3276
|
+
body: {
|
|
3277
|
+
tag: normalized.tag,
|
|
3278
|
+
head: normalized.head,
|
|
3279
|
+
version: normalized.version,
|
|
3280
|
+
events: normalized.events,
|
|
3281
|
+
outbox: normalized.outbox,
|
|
3282
|
+
fences: normalized.fences,
|
|
3283
|
+
clearedFences: normalized.clearedFences,
|
|
3284
|
+
repairOwner: normalized.repairOwner,
|
|
3285
|
+
repairLeaseUntil: normalized.repairLeaseUntil,
|
|
3286
|
+
highestRepairEpoch: normalized.highestRepairEpoch,
|
|
3287
|
+
repairScope: normalized.repairScope,
|
|
3288
|
+
repairScopeVersion: normalized.repairScopeVersion,
|
|
3289
|
+
facts,
|
|
3290
|
+
},
|
|
3291
|
+
};
|
|
3292
|
+
});
|
|
3293
|
+
return json(result.body, result.status);
|
|
3294
|
+
}
|
|
3295
|
+
/**
|
|
3296
|
+
* Internal outbox handoff: enqueuedAt is persisted before any Queue send so
|
|
3297
|
+
* a crash after send but before delivered marking replays the same clock
|
|
3298
|
+
* fact. It never changes the immutable event/outbox row.
|
|
3299
|
+
*/
|
|
3300
|
+
async pendingOutbox(tag, serviceId, body) {
|
|
3301
|
+
const parsed = outboxPendingFrom(body);
|
|
3302
|
+
if (parsed.value === undefined) {
|
|
3303
|
+
return error(400, "invalid_outbox_pending", parsed.error ?? "Invalid outbox pending request");
|
|
3304
|
+
}
|
|
3305
|
+
if (!isNonEmptyString(serviceId)) {
|
|
3306
|
+
return error(400, "outbox_service_identity_required", "Outbox delivery requires a service identity");
|
|
3307
|
+
}
|
|
3308
|
+
const input = parsed.value;
|
|
3309
|
+
const result = await this.ctx.storage.transaction(async (txn) => {
|
|
3310
|
+
if (this.sqlStorage() !== undefined) {
|
|
3311
|
+
return this.pendingSqlOutbox(txn, tag, serviceId, input.nowMs, input.limit, input.force === true);
|
|
3312
|
+
}
|
|
3313
|
+
const record = this.readStoredRecord(tag);
|
|
3314
|
+
if (record === undefined) {
|
|
3315
|
+
return { status: 404, body: { error: "Tag has no durable state yet", code: "tag_not_found" } };
|
|
3316
|
+
}
|
|
3317
|
+
if (record.tag !== tag) {
|
|
3318
|
+
return { status: 409, body: { error: "Tag Durable Object identity changed", code: "tag_identity_conflict" } };
|
|
3319
|
+
}
|
|
3320
|
+
const deliveries = (await txn.get(OUTBOX_DELIVERIES_KEY)) ?? this.fallbackOutboxDeliveries;
|
|
3321
|
+
const byRow = new Map(deliveries.map((delivery) => [outboxRowKey(delivery), delivery]));
|
|
3322
|
+
let changedDeliveries = deliveries;
|
|
3323
|
+
const rows = [];
|
|
3324
|
+
for (const row of record.outbox) {
|
|
3325
|
+
if (input.limit !== undefined && rows.length >= input.limit)
|
|
3326
|
+
break;
|
|
3327
|
+
const key = outboxRowKey(row);
|
|
3328
|
+
let delivery = byRow.get(key);
|
|
3329
|
+
if (delivery === undefined) {
|
|
3330
|
+
delivery = { ...row, enqueuedAt: input.nowMs, deliveredAt: null };
|
|
3331
|
+
changedDeliveries = [...changedDeliveries, delivery];
|
|
3332
|
+
byRow.set(key, delivery);
|
|
3333
|
+
}
|
|
3334
|
+
if (delivery.deliveredAt !== null) {
|
|
3335
|
+
continue;
|
|
3336
|
+
}
|
|
3337
|
+
const event = record.events.find((candidate) => candidate.attemptId === row.attemptId &&
|
|
3338
|
+
candidate.eventId === row.eventId &&
|
|
3339
|
+
candidate.suid === row.suid &&
|
|
3340
|
+
candidate.payload === row.payload &&
|
|
3341
|
+
candidate.eventType === row.eventType &&
|
|
3342
|
+
candidate.provenance === row.provenance &&
|
|
3343
|
+
candidate.timestamp === row.timestamp);
|
|
3344
|
+
if (event === undefined) {
|
|
3345
|
+
return {
|
|
3346
|
+
status: 409,
|
|
3347
|
+
body: { error: "Outbox row has no matching durable event", code: "outbox_event_missing" },
|
|
3348
|
+
};
|
|
3349
|
+
}
|
|
3350
|
+
const metadata = serializedEventMetadata(row.eventId);
|
|
3351
|
+
const artifact = await this.obligationArtifact(event, serviceId, tag);
|
|
3352
|
+
rows.push({
|
|
3353
|
+
version: 1,
|
|
3354
|
+
serviceId,
|
|
3355
|
+
tag,
|
|
3356
|
+
attemptId: row.attemptId,
|
|
3357
|
+
eventId: row.eventId,
|
|
3358
|
+
suid: row.suid,
|
|
3359
|
+
payload: row.payload,
|
|
3360
|
+
allocatorLineageId: row.allocatorLineageId,
|
|
3361
|
+
eventTags: event.eventTags,
|
|
3362
|
+
eventType: event.eventType,
|
|
3363
|
+
provenance: "g32",
|
|
3364
|
+
timestamp: event.timestamp,
|
|
3365
|
+
causationId: metadata.causationId,
|
|
3366
|
+
correlationId: metadata.correlationId,
|
|
3367
|
+
executedUser: metadata.executedUser,
|
|
3368
|
+
enqueuedAt: delivery.enqueuedAt,
|
|
3369
|
+
completeness: {
|
|
3370
|
+
canonicalBytesBase64: arrayBufferBase64(artifact.canonicalBytes),
|
|
3371
|
+
eventDigest: artifact.eventDigest,
|
|
3372
|
+
declaredTagSet: JSON.parse(artifact.declaredTagSet),
|
|
3373
|
+
localCommittedMembership: JSON.parse(artifact.localMembership),
|
|
3374
|
+
// The non-SQL fallback is a transport-only test seam. Its local
|
|
3375
|
+
// ordinal is still explicit so it cannot masquerade as a global
|
|
3376
|
+
// sequence during receipt verification.
|
|
3377
|
+
obligationSequence: record.outbox.findIndex((candidate) => outboxRowKey(candidate) === outboxRowKey(row)) + 1,
|
|
3378
|
+
},
|
|
3379
|
+
});
|
|
3380
|
+
}
|
|
3381
|
+
if (changedDeliveries !== deliveries) {
|
|
3382
|
+
this.fallbackOutboxDeliveries = changedDeliveries;
|
|
3383
|
+
await txn.put(OUTBOX_DELIVERIES_KEY, changedDeliveries);
|
|
3384
|
+
}
|
|
3385
|
+
return { status: 200, body: { rows } };
|
|
3386
|
+
});
|
|
3387
|
+
return json(result.body, result.status);
|
|
3388
|
+
}
|
|
3389
|
+
/**
|
|
3390
|
+
* Source-side obligation scanner. It deliberately does not call the Queue,
|
|
3391
|
+
* the doorbell, `pendingOutbox`, or any detector. An obligation therefore
|
|
3392
|
+
* remains visible when delivery is disabled, throws, or has not started.
|
|
3393
|
+
*/
|
|
3394
|
+
async scanOutboxObligations(tag, nowMs) {
|
|
3395
|
+
const result = await this.ctx.storage.transaction(async (txn) => {
|
|
3396
|
+
const sql = this.sqlStorage();
|
|
3397
|
+
if (sql === undefined) {
|
|
3398
|
+
const record = this.readStoredRecord(tag);
|
|
3399
|
+
if (record === undefined) {
|
|
3400
|
+
return { status: 404, body: { error: "Tag has no durable state yet", code: "tag_not_found" } };
|
|
3401
|
+
}
|
|
3402
|
+
const deliveries = (await txn.get(OUTBOX_DELIVERIES_KEY)) ?? this.fallbackOutboxDeliveries;
|
|
3403
|
+
const delivered = new Set(deliveries
|
|
3404
|
+
.filter((delivery) => delivery.deliveredAt !== null)
|
|
3405
|
+
.map((delivery) => outboxRowKey(delivery)));
|
|
3406
|
+
const findings = record.outbox
|
|
3407
|
+
.filter((row) => !delivered.has(outboxRowKey(row)))
|
|
3408
|
+
.map((row, index) => ({
|
|
3409
|
+
code: "tag_outbox_obligation_unacknowledged",
|
|
3410
|
+
obligationSequence: index + 1,
|
|
3411
|
+
attemptId: row.attemptId,
|
|
3412
|
+
eventId: row.eventId,
|
|
3413
|
+
status: "pending",
|
|
3414
|
+
nextAttemptAt: nowMs,
|
|
3415
|
+
source: "tag_outbox_obligation",
|
|
3416
|
+
}));
|
|
3417
|
+
return {
|
|
3418
|
+
status: 200,
|
|
3419
|
+
body: {
|
|
3420
|
+
status: "scan-complete",
|
|
3421
|
+
source: "tag_outbox_obligation",
|
|
3422
|
+
scannedAt: nowMs,
|
|
3423
|
+
pendingCount: findings.length,
|
|
3424
|
+
findings,
|
|
3425
|
+
},
|
|
3426
|
+
};
|
|
3427
|
+
}
|
|
3428
|
+
const identity = sql.exec("SELECT tag FROM tag_identity WHERE singleton = 1").toArray()[0];
|
|
3429
|
+
if (identity === undefined) {
|
|
3430
|
+
return { status: 404, body: { error: "Tag has no durable state yet", code: "tag_not_found" } };
|
|
3431
|
+
}
|
|
3432
|
+
if (sqlString(identity.tag, "tag_identity.tag") !== tag) {
|
|
3433
|
+
return { status: 409, body: { error: "Tag Durable Object identity changed", code: "tag_identity_conflict" } };
|
|
3434
|
+
}
|
|
3435
|
+
const findings = sql.exec(`
|
|
3436
|
+
SELECT obligation_sequence, attempt_id, event_id, status, next_attempt_at, attempt_count
|
|
3437
|
+
FROM tag_outbox_obligation
|
|
3438
|
+
WHERE status <> 'acknowledged'
|
|
3439
|
+
ORDER BY obligation_sequence ASC
|
|
3440
|
+
`).toArray().map((row) => ({
|
|
3441
|
+
code: "tag_outbox_obligation_unacknowledged",
|
|
3442
|
+
obligationSequence: sqlNumber(row.obligation_sequence, "tag_outbox_obligation.obligation_sequence"),
|
|
3443
|
+
attemptId: sqlString(row.attempt_id, "tag_outbox_obligation.attempt_id"),
|
|
3444
|
+
eventId: sqlString(row.event_id, "tag_outbox_obligation.event_id"),
|
|
3445
|
+
status: sqlString(row.status, "tag_outbox_obligation.status"),
|
|
3446
|
+
nextAttemptAt: sqlNumber(row.next_attempt_at, "tag_outbox_obligation.next_attempt_at"),
|
|
3447
|
+
attemptCount: sqlNumber(row.attempt_count, "tag_outbox_obligation.attempt_count"),
|
|
3448
|
+
source: "tag_outbox_obligation",
|
|
3449
|
+
}));
|
|
3450
|
+
return {
|
|
3451
|
+
status: 200,
|
|
3452
|
+
body: {
|
|
3453
|
+
status: "scan-complete",
|
|
3454
|
+
source: "tag_outbox_obligation",
|
|
3455
|
+
scannedAt: nowMs,
|
|
3456
|
+
pendingCount: findings.length,
|
|
3457
|
+
findings,
|
|
3458
|
+
},
|
|
3459
|
+
};
|
|
3460
|
+
});
|
|
3461
|
+
return json(result.body, result.status);
|
|
3462
|
+
}
|
|
3463
|
+
/** Record a retry or terminal poison disposition without changing the source event. */
|
|
3464
|
+
async recordOutboxFailure(tag, delivery, failure) {
|
|
3465
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3466
|
+
const sql = this.sqlStorage();
|
|
3467
|
+
if (sql === undefined) {
|
|
3468
|
+
// Minimal direct seams have no persistent SQL store. They intentionally
|
|
3469
|
+
// remain transport-only and cannot become a durable obligation source.
|
|
3470
|
+
return;
|
|
3471
|
+
}
|
|
3472
|
+
const identity = sql.exec("SELECT tag FROM tag_identity WHERE singleton = 1").toArray()[0];
|
|
3473
|
+
if (identity === undefined || sqlString(identity.tag, "tag_identity.tag") !== tag) {
|
|
3474
|
+
throw new Error("Cannot record an outbox failure for a different tag identity");
|
|
3475
|
+
}
|
|
3476
|
+
const obligation = sql.exec(`
|
|
3477
|
+
SELECT obligation_sequence, attempt_count, status
|
|
3478
|
+
FROM tag_outbox_obligation
|
|
3479
|
+
WHERE attempt_id = ? AND event_id = ? AND suid = ? AND payload = ?
|
|
3480
|
+
AND allocator_lineage_id = ? AND event_type = ? AND provenance = ? AND timestamp = ?
|
|
3481
|
+
`, delivery.attemptId, delivery.eventId, delivery.suid, delivery.payload, delivery.allocatorLineageId, delivery.eventType, delivery.provenance, delivery.timestamp).toArray()[0];
|
|
3482
|
+
if (obligation === undefined || sqlString(obligation.status, "tag_outbox_obligation.status") === "acknowledged") {
|
|
3483
|
+
await this.rearmScheduler(txn);
|
|
3484
|
+
return;
|
|
3485
|
+
}
|
|
3486
|
+
const attempts = sqlNumber(obligation.attempt_count, "tag_outbox_obligation.attempt_count");
|
|
3487
|
+
const status = attempts >= OBLIGATION_MAX_ATTEMPTS ? "poison" : "pending";
|
|
3488
|
+
const message = String(failure).slice(0, 256);
|
|
3489
|
+
sql.exec(`
|
|
3490
|
+
UPDATE tag_outbox_obligation
|
|
3491
|
+
SET status = ?, next_attempt_at = ?, last_error = ?
|
|
3492
|
+
WHERE obligation_sequence = ?
|
|
3493
|
+
`, status, Date.now() + OBLIGATION_RETRY_MS, message, sqlNumber(obligation.obligation_sequence, "tag_outbox_obligation.obligation_sequence"));
|
|
3494
|
+
await this.rearmScheduler(txn);
|
|
3495
|
+
});
|
|
3496
|
+
}
|
|
3497
|
+
/** Marks only a previously enqueued row; repeating a mark is harmless. */
|
|
3498
|
+
async markOutboxDelivered(tag, body) {
|
|
3499
|
+
const parsed = outboxMarkFrom(body);
|
|
3500
|
+
if (parsed.value === undefined) {
|
|
3501
|
+
return error(400, "invalid_outbox_delivery_mark", parsed.error ?? "Invalid outbox delivery mark");
|
|
3502
|
+
}
|
|
3503
|
+
const input = parsed.value;
|
|
3504
|
+
try {
|
|
3505
|
+
const joined = await Promise.all(input.deliveries.map((delivery) => this.globalReceiptMatches(delivery)));
|
|
3506
|
+
if (joined.some((value) => !value)) {
|
|
3507
|
+
return error(409, "outbox_global_receipt_unverified", "Global receipt and membership must be read back before source acknowledgement");
|
|
3508
|
+
}
|
|
3509
|
+
}
|
|
3510
|
+
catch {
|
|
3511
|
+
// Do not turn an unavailable/partial global read into an acknowledgement.
|
|
3512
|
+
return error(503, "outbox_global_receipt_unavailable", "Global receipt verification is unavailable");
|
|
3513
|
+
}
|
|
3514
|
+
const result = await this.ctx.storage.transaction(async (txn) => {
|
|
3515
|
+
if (this.sqlStorage() !== undefined) {
|
|
3516
|
+
return this.markSqlOutboxDelivered(txn, tag, input);
|
|
3517
|
+
}
|
|
3518
|
+
const record = this.readStoredRecord(tag);
|
|
3519
|
+
if (record === undefined) {
|
|
3520
|
+
return { status: 404, body: { error: "Tag has no durable state yet", code: "tag_not_found" } };
|
|
3521
|
+
}
|
|
3522
|
+
if (record.tag !== tag) {
|
|
3523
|
+
return { status: 409, body: { error: "Tag Durable Object identity changed", code: "tag_identity_conflict" } };
|
|
3524
|
+
}
|
|
3525
|
+
const deliveries = (await txn.get(OUTBOX_DELIVERIES_KEY)) ?? this.fallbackOutboxDeliveries;
|
|
3526
|
+
const requested = new Map(input.deliveries.map((delivery) => [outboxRowKey(delivery), delivery]));
|
|
3527
|
+
for (const [key, delivery] of requested) {
|
|
3528
|
+
if (!record.outbox.some((row) => outboxRowKey(row) === key)) {
|
|
3529
|
+
return { status: 409, body: { error: "Outbox row is not durable", code: "outbox_row_not_found" } };
|
|
3530
|
+
}
|
|
3531
|
+
const persisted = deliveries.find((candidate) => outboxRowKey(candidate) === key);
|
|
3532
|
+
if (persisted === undefined || persisted.enqueuedAt !== delivery.enqueuedAt) {
|
|
3533
|
+
return { status: 409, body: { error: "Outbox row was not enqueued", code: "outbox_delivery_not_enqueued" } };
|
|
3534
|
+
}
|
|
3535
|
+
}
|
|
3536
|
+
let marked = 0;
|
|
3537
|
+
const updated = deliveries.map((delivery) => {
|
|
3538
|
+
const requestedDelivery = requested.get(outboxRowKey(delivery));
|
|
3539
|
+
if (requestedDelivery === undefined || delivery.deliveredAt !== null) {
|
|
3540
|
+
return delivery;
|
|
3541
|
+
}
|
|
3542
|
+
marked += 1;
|
|
3543
|
+
return { ...delivery, deliveredAt: input.nowMs };
|
|
3544
|
+
});
|
|
3545
|
+
if (marked > 0) {
|
|
3546
|
+
this.fallbackOutboxDeliveries = updated;
|
|
3547
|
+
await txn.put(OUTBOX_DELIVERIES_KEY, updated);
|
|
3548
|
+
}
|
|
3549
|
+
return { status: 200, body: { marked, idempotent: marked === 0 } };
|
|
3550
|
+
});
|
|
3551
|
+
return json(result.body, result.status);
|
|
3552
|
+
}
|
|
3553
|
+
/**
|
|
3554
|
+
* The delivery handoff reads its universe from `tag_outbox_obligation`.
|
|
3555
|
+
* It does not consult the Queue, a sink receipt, or a runner's expected
|
|
3556
|
+
* count. Reserving the retry timestamp before the external handoff lets the
|
|
3557
|
+
* single durable scheduler retry a crash/throw without treating delivery as
|
|
3558
|
+
* the authority for obligation existence.
|
|
3559
|
+
*/
|
|
3560
|
+
async pendingSqlOutbox(txn, tag, serviceId, nowMs, limit, force = false) {
|
|
3561
|
+
const sql = this.sqlStorage();
|
|
3562
|
+
if (sql === undefined)
|
|
3563
|
+
throw new Error("Tag SQL storage is unavailable");
|
|
3564
|
+
const identity = sql.exec("SELECT tag FROM tag_identity WHERE singleton = 1").toArray()[0];
|
|
3565
|
+
if (identity === undefined)
|
|
3566
|
+
return { status: 404, body: { error: "Tag has no durable state yet", code: "tag_not_found" } };
|
|
3567
|
+
if (sqlString(identity.tag, "tag_identity.tag") !== tag) {
|
|
3568
|
+
return { status: 409, body: { error: "Tag Durable Object identity changed", code: "tag_identity_conflict" } };
|
|
3569
|
+
}
|
|
3570
|
+
const pending = sql.exec(`
|
|
3571
|
+
SELECT
|
|
3572
|
+
o.obligation_sequence, o.service_id, o.event_id, o.attempt_id, o.suid,
|
|
3573
|
+
o.payload, o.allocator_lineage_id, o.event_type, o.provenance,
|
|
3574
|
+
o.timestamp, o.enqueued_at, o.canonical_bytes, o.event_digest,
|
|
3575
|
+
o.declared_tag_set_json, o.local_committed_membership_json,
|
|
3576
|
+
e.event_tags_json
|
|
3577
|
+
FROM tag_outbox_obligation AS o
|
|
3578
|
+
JOIN tag_event AS e ON e.service_id = o.service_id AND e.event_id = o.event_id
|
|
3579
|
+
WHERE o.status = 'pending' AND (? = 1 OR o.next_attempt_at <= ?)
|
|
3580
|
+
ORDER BY o.obligation_sequence ASC
|
|
3581
|
+
LIMIT ?
|
|
3582
|
+
`, force ? 1 : 0, nowMs, limit ?? Number.MAX_SAFE_INTEGER).toArray();
|
|
3583
|
+
const rows = [];
|
|
3584
|
+
for (const obligation of pending) {
|
|
3585
|
+
const storedServiceId = sqlString(obligation.service_id, "tag_outbox_obligation.service_id");
|
|
3586
|
+
if (storedServiceId !== serviceId) {
|
|
3587
|
+
return { status: 409, body: { error: "Outbox service identity changed", code: "outbox_service_identity_conflict" } };
|
|
3588
|
+
}
|
|
3589
|
+
const sequence = sqlNumber(obligation.obligation_sequence, "tag_outbox_obligation.obligation_sequence");
|
|
3590
|
+
const enqueuedAt = sqlNullableNumber(obligation.enqueued_at, "tag_outbox_obligation.enqueued_at") ?? nowMs;
|
|
3591
|
+
sql.exec(`
|
|
3592
|
+
UPDATE tag_outbox_obligation
|
|
3593
|
+
SET enqueued_at = ?, attempt_count = attempt_count + 1, next_attempt_at = ?
|
|
3594
|
+
WHERE obligation_sequence = ? AND status = 'pending'
|
|
3595
|
+
`, enqueuedAt, nowMs + OBLIGATION_RETRY_MS, sequence);
|
|
3596
|
+
const eventId = sqlString(obligation.event_id, "tag_outbox_obligation.event_id");
|
|
3597
|
+
const metadata = serializedEventMetadata(eventId);
|
|
3598
|
+
rows.push({
|
|
3599
|
+
version: 1,
|
|
3600
|
+
serviceId,
|
|
3601
|
+
tag,
|
|
3602
|
+
attemptId: sqlString(obligation.attempt_id, "tag_outbox_obligation.attempt_id"),
|
|
3603
|
+
eventId,
|
|
3604
|
+
suid: sqlString(obligation.suid, "tag_outbox_obligation.suid"),
|
|
3605
|
+
payload: sqlString(obligation.payload, "tag_outbox_obligation.payload"),
|
|
3606
|
+
allocatorLineageId: sqlString(obligation.allocator_lineage_id, "tag_outbox_obligation.allocator_lineage_id"),
|
|
3607
|
+
eventTags: sqlJson(obligation.event_tags_json, "tag_event.event_tags_json"),
|
|
3608
|
+
eventType: sqlString(obligation.event_type, "tag_outbox_obligation.event_type"),
|
|
3609
|
+
provenance: sqlString(obligation.provenance, "tag_outbox_obligation.provenance"),
|
|
3610
|
+
timestamp: sqlString(obligation.timestamp, "tag_outbox_obligation.timestamp"),
|
|
3611
|
+
causationId: metadata.causationId,
|
|
3612
|
+
correlationId: metadata.correlationId,
|
|
3613
|
+
executedUser: metadata.executedUser,
|
|
3614
|
+
enqueuedAt,
|
|
3615
|
+
completeness: {
|
|
3616
|
+
canonicalBytesBase64: arrayBufferBase64(sqlArrayBuffer(obligation.canonical_bytes, "tag_outbox_obligation.canonical_bytes")),
|
|
3617
|
+
eventDigest: sqlString(obligation.event_digest, "tag_outbox_obligation.event_digest"),
|
|
3618
|
+
declaredTagSet: sqlJson(obligation.declared_tag_set_json, "tag_outbox_obligation.declared_tag_set_json"),
|
|
3619
|
+
localCommittedMembership: sqlJson(obligation.local_committed_membership_json, "tag_outbox_obligation.local_committed_membership_json"),
|
|
3620
|
+
obligationSequence: sequence,
|
|
3621
|
+
},
|
|
3622
|
+
});
|
|
3623
|
+
}
|
|
3624
|
+
await this.rearmScheduler(txn);
|
|
3625
|
+
return { status: 200, body: { rows } };
|
|
3626
|
+
}
|
|
3627
|
+
async markSqlOutboxDelivered(txn, tag, input) {
|
|
3628
|
+
const sql = this.sqlStorage();
|
|
3629
|
+
if (sql === undefined)
|
|
3630
|
+
throw new Error("Tag SQL storage is unavailable");
|
|
3631
|
+
const identity = sql.exec("SELECT tag FROM tag_identity WHERE singleton = 1").toArray()[0];
|
|
3632
|
+
if (identity === undefined)
|
|
3633
|
+
return { status: 404, body: { error: "Tag has no durable state yet", code: "tag_not_found" } };
|
|
3634
|
+
if (sqlString(identity.tag, "tag_identity.tag") !== tag) {
|
|
3635
|
+
return { status: 409, body: { error: "Tag Durable Object identity changed", code: "tag_identity_conflict" } };
|
|
3636
|
+
}
|
|
3637
|
+
let marked = 0;
|
|
3638
|
+
for (const delivery of input.deliveries) {
|
|
3639
|
+
const obligation = sql.exec(`
|
|
3640
|
+
SELECT obligation_sequence, enqueued_at, status, canonical_bytes, event_digest,
|
|
3641
|
+
declared_tag_set_json, local_committed_membership_json
|
|
3642
|
+
FROM tag_outbox_obligation
|
|
3643
|
+
WHERE attempt_id = ? AND event_id = ? AND suid = ? AND payload = ?
|
|
3644
|
+
AND allocator_lineage_id = ? AND event_type = ? AND provenance = ? AND timestamp = ?
|
|
3645
|
+
`, delivery.attemptId, delivery.eventId, delivery.suid, delivery.payload, delivery.allocatorLineageId, delivery.eventType, delivery.provenance, delivery.timestamp).toArray()[0];
|
|
3646
|
+
if (obligation === undefined) {
|
|
3647
|
+
return { status: 409, body: { error: "Outbox row is not durable", code: "outbox_row_not_found" } };
|
|
3648
|
+
}
|
|
3649
|
+
if (sqlNullableNumber(obligation.enqueued_at, "tag_outbox_obligation.enqueued_at") !== delivery.enqueuedAt) {
|
|
3650
|
+
return { status: 409, body: { error: "Outbox row was not enqueued", code: "outbox_delivery_not_enqueued" } };
|
|
3651
|
+
}
|
|
3652
|
+
const localMembership = sqlJson(obligation.local_committed_membership_json, "tag_outbox_obligation.local_committed_membership_json");
|
|
3653
|
+
const sourceFactsMatch = sqlString(obligation.event_digest, "tag_outbox_obligation.event_digest") === delivery.completeness.eventDigest &&
|
|
3654
|
+
arrayBufferBase64(sqlArrayBuffer(obligation.canonical_bytes, "tag_outbox_obligation.canonical_bytes")) === delivery.completeness.canonicalBytesBase64 &&
|
|
3655
|
+
JSON.stringify(sqlJson(obligation.declared_tag_set_json, "tag_outbox_obligation.declared_tag_set_json")) === JSON.stringify(delivery.completeness.declaredTagSet) &&
|
|
3656
|
+
JSON.stringify(localMembership) === JSON.stringify(delivery.completeness.localCommittedMembership) &&
|
|
3657
|
+
sqlNumber(obligation.obligation_sequence, "tag_outbox_obligation.obligation_sequence") === delivery.completeness.obligationSequence;
|
|
3658
|
+
if (!sourceFactsMatch) {
|
|
3659
|
+
return { status: 409, body: { error: "Outbox source facts do not match the durable obligation", code: "outbox_receipt_join_mismatch" } };
|
|
3660
|
+
}
|
|
3661
|
+
if (sqlString(obligation.status, "tag_outbox_obligation.status") === "acknowledged")
|
|
3662
|
+
continue;
|
|
3663
|
+
sql.exec(`
|
|
3664
|
+
UPDATE tag_outbox_obligation
|
|
3665
|
+
SET status = 'acknowledged', acknowledged_at = ?, last_error = NULL
|
|
3666
|
+
WHERE obligation_sequence = ?
|
|
3667
|
+
`, input.nowMs, sqlNumber(obligation.obligation_sequence, "tag_outbox_obligation.obligation_sequence"));
|
|
3668
|
+
marked += 1;
|
|
3669
|
+
}
|
|
3670
|
+
await this.rearmScheduler(txn);
|
|
3671
|
+
return { status: 200, body: { marked, idempotent: marked === 0 } };
|
|
3672
|
+
}
|
|
3673
|
+
async rearmScheduler(txn) {
|
|
3674
|
+
if (this.g43SchedulerFault === "before-rearm")
|
|
3675
|
+
throw new Error("G43 scheduler crash before re-arm");
|
|
3676
|
+
const dueAt = this.nextSqlAlarmDue();
|
|
3677
|
+
if (dueAt === null)
|
|
3678
|
+
await txn.deleteAlarm();
|
|
3679
|
+
else
|
|
3680
|
+
await txn.setAlarm(dueAt);
|
|
3681
|
+
if (this.g43SchedulerFault === "after-rearm")
|
|
3682
|
+
throw new Error("G43 scheduler crash after re-arm");
|
|
3683
|
+
}
|
|
3684
|
+
async acquireRepairLease(tag, body) {
|
|
3685
|
+
const parsed = repairAcquireFrom(body, tag);
|
|
3686
|
+
if (parsed.value === undefined) {
|
|
3687
|
+
return error(400, "invalid_repair_acquire", parsed.error ?? "Invalid repair acquire request");
|
|
3688
|
+
}
|
|
3689
|
+
const input = parsed.value;
|
|
3690
|
+
const result = await this.ctx.storage.transaction(async (txn) => {
|
|
3691
|
+
const loaded = await this.recordFor(txn, tag);
|
|
3692
|
+
const record = loaded.record;
|
|
3693
|
+
if (hasSegmentRotationFence(record)) {
|
|
3694
|
+
return repairRejected("segment_rotation_fence_held");
|
|
3695
|
+
}
|
|
3696
|
+
if (input.scope.some((item) => fenceFor(record.fences, PARTIAL_WRITE_FENCE_REASON, item.attemptId) === undefined)) {
|
|
3697
|
+
return repairRejected("partial_write_fence_required");
|
|
3698
|
+
}
|
|
3699
|
+
const leaseIsLive = record.repairOwner !== null &&
|
|
3700
|
+
record.repairLeaseUntil !== null &&
|
|
3701
|
+
logicalNow(record) < record.repairLeaseUntil;
|
|
3702
|
+
if (leaseIsLive) {
|
|
3703
|
+
return repairRejected("repair_lease_live");
|
|
3704
|
+
}
|
|
3705
|
+
if (record.highestRepairEpoch >= MAX_EPOCH) {
|
|
3706
|
+
return repairRejected("repair_epoch_exhausted", 422);
|
|
3707
|
+
}
|
|
3708
|
+
let scope;
|
|
3709
|
+
try {
|
|
3710
|
+
scope = unionRepairScope(record.repairScope, input.scope);
|
|
3711
|
+
}
|
|
3712
|
+
catch {
|
|
3713
|
+
return repairRejected("repair_scope_identity_conflict");
|
|
3714
|
+
}
|
|
3715
|
+
const epoch = record.highestRepairEpoch + 1;
|
|
3716
|
+
const updated = await this.commit(txn, record, {
|
|
3717
|
+
repairOwner: input.owner,
|
|
3718
|
+
repairLeaseUntil: logicalNow(record) + input.leaseMs,
|
|
3719
|
+
highestRepairEpoch: epoch,
|
|
3720
|
+
repairScope: scope,
|
|
3721
|
+
repairScopeVersion: record.repairScopeVersion + 1,
|
|
3722
|
+
});
|
|
3723
|
+
return {
|
|
3724
|
+
status: 201,
|
|
3725
|
+
body: {
|
|
3726
|
+
status: "repair-lease-acquired",
|
|
3727
|
+
owner: updated.repairOwner,
|
|
3728
|
+
epoch: updated.highestRepairEpoch,
|
|
3729
|
+
leaseUntil: updated.repairLeaseUntil,
|
|
3730
|
+
scopeVersion: updated.repairScopeVersion,
|
|
3731
|
+
},
|
|
3732
|
+
};
|
|
3733
|
+
});
|
|
3734
|
+
return json(result.body, result.status);
|
|
3735
|
+
}
|
|
3736
|
+
async renewRepairLease(tag, body) {
|
|
3737
|
+
const parsed = repairLeaseFrom(body);
|
|
3738
|
+
if (parsed.value === undefined) {
|
|
3739
|
+
return error(400, "invalid_repair_renew", parsed.error ?? "Invalid repair renew request");
|
|
3740
|
+
}
|
|
3741
|
+
const input = parsed.value;
|
|
3742
|
+
const result = await this.ctx.storage.transaction(async (txn) => {
|
|
3743
|
+
const loaded = await this.recordFor(txn, tag);
|
|
3744
|
+
const record = loaded.record;
|
|
3745
|
+
const leaseError = repairLeaseRejection(record, input.owner, input.epoch);
|
|
3746
|
+
if (leaseError !== undefined) {
|
|
3747
|
+
return repairRejected(leaseError);
|
|
3748
|
+
}
|
|
3749
|
+
const updated = await this.commit(txn, record, {
|
|
3750
|
+
repairLeaseUntil: logicalNow(record) + DEFAULT_REPAIR_LEASE_MS,
|
|
3751
|
+
});
|
|
3752
|
+
return {
|
|
3753
|
+
status: 200,
|
|
3754
|
+
body: {
|
|
3755
|
+
status: "repair-lease-renewed",
|
|
3756
|
+
owner: updated.repairOwner,
|
|
3757
|
+
epoch: updated.highestRepairEpoch,
|
|
3758
|
+
leaseUntil: updated.repairLeaseUntil,
|
|
3759
|
+
scopeVersion: updated.repairScopeVersion,
|
|
3760
|
+
},
|
|
3761
|
+
};
|
|
3762
|
+
});
|
|
3763
|
+
return json(result.body, result.status);
|
|
3764
|
+
}
|
|
3765
|
+
async unionRepairScope(tag, body) {
|
|
3766
|
+
const parsed = repairScopeUnionFrom(body, tag);
|
|
3767
|
+
if (parsed.value === undefined) {
|
|
3768
|
+
return error(400, "invalid_repair_scope_union", parsed.error ?? "Invalid repair scope union request");
|
|
3769
|
+
}
|
|
3770
|
+
const input = parsed.value;
|
|
3771
|
+
const result = await this.ctx.storage.transaction(async (txn) => {
|
|
3772
|
+
const loaded = await this.recordFor(txn, tag);
|
|
3773
|
+
const record = loaded.record;
|
|
3774
|
+
const leaseError = repairLeaseRejection(record, input.owner, input.epoch);
|
|
3775
|
+
if (leaseError !== undefined) {
|
|
3776
|
+
return repairRejected(leaseError);
|
|
3777
|
+
}
|
|
3778
|
+
if (input.scope.some((item) => fenceFor(record.fences, PARTIAL_WRITE_FENCE_REASON, item.attemptId) === undefined)) {
|
|
3779
|
+
return repairRejected("partial_write_fence_required");
|
|
3780
|
+
}
|
|
3781
|
+
let scope;
|
|
3782
|
+
try {
|
|
3783
|
+
scope = unionRepairScope(record.repairScope, input.scope);
|
|
3784
|
+
}
|
|
3785
|
+
catch {
|
|
3786
|
+
return repairRejected("repair_scope_identity_conflict");
|
|
3787
|
+
}
|
|
3788
|
+
if (scope.length === record.repairScope.length) {
|
|
3789
|
+
return {
|
|
3790
|
+
status: 200,
|
|
3791
|
+
body: { status: "repair-scope-unchanged", scopeVersion: record.repairScopeVersion },
|
|
3792
|
+
};
|
|
3793
|
+
}
|
|
3794
|
+
const updated = await this.commit(txn, record, {
|
|
3795
|
+
repairScope: scope,
|
|
3796
|
+
repairScopeVersion: record.repairScopeVersion + 1,
|
|
3797
|
+
});
|
|
3798
|
+
return {
|
|
3799
|
+
status: 200,
|
|
3800
|
+
body: { status: "repair-scope-unioned", scopeVersion: updated.repairScopeVersion, scope: updated.repairScope },
|
|
3801
|
+
};
|
|
3802
|
+
});
|
|
3803
|
+
return json(result.body, result.status);
|
|
3804
|
+
}
|
|
3805
|
+
async applyRepair(tag, body) {
|
|
3806
|
+
const parsed = repairApplyFrom(body, tag);
|
|
3807
|
+
if (parsed.value === undefined) {
|
|
3808
|
+
return error(400, "invalid_repair_apply", parsed.error ?? "Invalid repair apply request");
|
|
3809
|
+
}
|
|
3810
|
+
const input = parsed.value;
|
|
3811
|
+
const result = await this.ctx.storage.transaction(async (txn) => {
|
|
3812
|
+
const loaded = await this.recordFor(txn, tag);
|
|
3813
|
+
const record = loaded.record;
|
|
3814
|
+
const leaseError = repairLeaseRejection(record, input.owner, input.epoch);
|
|
3815
|
+
if (leaseError !== undefined) {
|
|
3816
|
+
return repairRejected(leaseError);
|
|
3817
|
+
}
|
|
3818
|
+
if (!repairScopeContains(record.repairScope, input.item)) {
|
|
3819
|
+
return repairRejected("repair_scope_required");
|
|
3820
|
+
}
|
|
3821
|
+
if (fenceFor(record.fences, PARTIAL_WRITE_FENCE_REASON, input.item.attemptId) === undefined) {
|
|
3822
|
+
return repairRejected("partial_write_fence_required");
|
|
3823
|
+
}
|
|
3824
|
+
const facts = (await txn.get(REPAIR_FACTS_KEY)) ?? repairFactsDefault();
|
|
3825
|
+
const prior = repairResolutionFor(facts, input.item);
|
|
3826
|
+
if (prior !== undefined) {
|
|
3827
|
+
return {
|
|
3828
|
+
status: 200,
|
|
3829
|
+
body: { status: prior.branch, idempotent: true, head: record.head, version: record.version },
|
|
3830
|
+
};
|
|
3831
|
+
}
|
|
3832
|
+
const existing = record.events.find((event) => event.eventId === input.item.eventId);
|
|
3833
|
+
let branch;
|
|
3834
|
+
let updatedRecord;
|
|
3835
|
+
if (existing !== undefined) {
|
|
3836
|
+
branch = existing.payload === input.item.payload && existing.suid === input.item.suid
|
|
3837
|
+
? "ROLLED_FORWARD"
|
|
3838
|
+
: "FAILED_CLOSED";
|
|
3839
|
+
}
|
|
3840
|
+
else if (record.head < input.item.suid) {
|
|
3841
|
+
branch = "ROLLED_FORWARD";
|
|
3842
|
+
updatedRecord = changed(record, {
|
|
3843
|
+
head: input.item.suid,
|
|
3844
|
+
events: [...record.events, {
|
|
3845
|
+
attemptId: input.item.attemptId,
|
|
3846
|
+
eventId: input.item.eventId,
|
|
3847
|
+
suid: input.item.suid,
|
|
3848
|
+
payload: input.item.payload,
|
|
3849
|
+
eventTags: input.item.eventTags,
|
|
3850
|
+
allocatorLineageId: input.item.allocatorLineageId,
|
|
3851
|
+
eventType: input.item.eventType,
|
|
3852
|
+
provenance: input.item.provenance,
|
|
3853
|
+
timestamp: input.item.timestamp,
|
|
3854
|
+
}],
|
|
3855
|
+
outbox: [...record.outbox, {
|
|
3856
|
+
attemptId: input.item.attemptId,
|
|
3857
|
+
eventId: input.item.eventId,
|
|
3858
|
+
suid: input.item.suid,
|
|
3859
|
+
payload: input.item.payload,
|
|
3860
|
+
allocatorLineageId: input.item.allocatorLineageId,
|
|
3861
|
+
eventType: input.item.eventType,
|
|
3862
|
+
provenance: input.item.provenance,
|
|
3863
|
+
timestamp: input.item.timestamp,
|
|
3864
|
+
}],
|
|
3865
|
+
});
|
|
3866
|
+
}
|
|
3867
|
+
else if (record.head > input.item.suid) {
|
|
3868
|
+
// The exclusion audit is stored separately so Branch B cannot advance
|
|
3869
|
+
// head or version merely by recording its durable decision.
|
|
3870
|
+
branch = "EXCLUDED_AUDITED";
|
|
3871
|
+
}
|
|
3872
|
+
else {
|
|
3873
|
+
branch = "FAILED_CLOSED";
|
|
3874
|
+
}
|
|
3875
|
+
const resolution = {
|
|
3876
|
+
attemptId: input.item.attemptId,
|
|
3877
|
+
eventId: input.item.eventId,
|
|
3878
|
+
suid: input.item.suid,
|
|
3879
|
+
branch,
|
|
3880
|
+
epoch: input.epoch,
|
|
3881
|
+
owner: input.owner,
|
|
3882
|
+
recordedAt: nowIso(),
|
|
3883
|
+
};
|
|
3884
|
+
await txn.put(REPAIR_FACTS_KEY, {
|
|
3885
|
+
...facts,
|
|
3886
|
+
resolutions: [...facts.resolutions, resolution],
|
|
3887
|
+
});
|
|
3888
|
+
if (updatedRecord !== undefined) {
|
|
3889
|
+
await this.write(txn, updatedRecord);
|
|
3890
|
+
}
|
|
3891
|
+
return {
|
|
3892
|
+
status: 200,
|
|
3893
|
+
body: { status: branch, idempotent: false, head: (updatedRecord ?? record).head, version: (updatedRecord ?? record).version },
|
|
3894
|
+
};
|
|
3895
|
+
});
|
|
3896
|
+
return json(result.body, result.status);
|
|
3897
|
+
}
|
|
3898
|
+
async auditRepair(tag, body) {
|
|
3899
|
+
const parsed = repairAuditFrom(body, tag);
|
|
3900
|
+
if (parsed.value === undefined) {
|
|
3901
|
+
return error(400, "invalid_repair_audit", parsed.error ?? "Invalid repair audit request");
|
|
3902
|
+
}
|
|
3903
|
+
const input = parsed.value;
|
|
3904
|
+
const result = await this.ctx.storage.transaction(async (txn) => {
|
|
3905
|
+
const loaded = await this.recordFor(txn, tag);
|
|
3906
|
+
const record = loaded.record;
|
|
3907
|
+
const leaseError = repairLeaseRejection(record, input.owner, input.epoch);
|
|
3908
|
+
if (leaseError !== undefined) {
|
|
3909
|
+
return repairRejected(leaseError);
|
|
3910
|
+
}
|
|
3911
|
+
const facts = (await txn.get(REPAIR_FACTS_KEY)) ?? repairFactsDefault();
|
|
3912
|
+
const resolution = repairResolutionFor(facts, input.item);
|
|
3913
|
+
if (resolution === undefined || resolution.branch === "FAILED_CLOSED") {
|
|
3914
|
+
return repairRejected("repair_resolution_not_auditable");
|
|
3915
|
+
}
|
|
3916
|
+
if (resolution.branch === "ROLLED_FORWARD" && !record.outbox.some((row) => row.attemptId === input.item.attemptId &&
|
|
3917
|
+
row.eventId === input.item.eventId &&
|
|
3918
|
+
row.suid === input.item.suid &&
|
|
3919
|
+
row.payload === input.item.payload)) {
|
|
3920
|
+
return repairRejected("durable_outbox_required");
|
|
3921
|
+
}
|
|
3922
|
+
const prior = repairAuditFor(facts, input.item);
|
|
3923
|
+
if (prior !== undefined) {
|
|
3924
|
+
return { status: 200, body: { status: "repair-audited", idempotent: true, branch: prior.branch } };
|
|
3925
|
+
}
|
|
3926
|
+
const audit = {
|
|
3927
|
+
attemptId: input.item.attemptId,
|
|
3928
|
+
eventId: input.item.eventId,
|
|
3929
|
+
suid: input.item.suid,
|
|
3930
|
+
branch: resolution.branch,
|
|
3931
|
+
actor: input.actor,
|
|
3932
|
+
epoch: input.epoch,
|
|
3933
|
+
owner: input.owner,
|
|
3934
|
+
recordedAt: nowIso(),
|
|
3935
|
+
};
|
|
3936
|
+
await txn.put(REPAIR_FACTS_KEY, { ...facts, audits: [...facts.audits, audit] });
|
|
3937
|
+
return { status: 200, body: { status: "repair-audited", idempotent: false, branch: audit.branch } };
|
|
3938
|
+
});
|
|
3939
|
+
return json(result.body, result.status);
|
|
3940
|
+
}
|
|
3941
|
+
async clearRepairFence(tag, body) {
|
|
3942
|
+
const parsed = repairClearFrom(body);
|
|
3943
|
+
if (parsed.value === undefined) {
|
|
3944
|
+
return error(400, "invalid_repair_clear", parsed.error ?? "Invalid repair clear request");
|
|
3945
|
+
}
|
|
3946
|
+
const input = parsed.value;
|
|
3947
|
+
const result = await this.ctx.storage.transaction(async (txn) => {
|
|
3948
|
+
const loaded = await this.recordFor(txn, tag);
|
|
3949
|
+
const record = loaded.record;
|
|
3950
|
+
const leaseError = repairLeaseRejection(record, input.owner, input.epoch);
|
|
3951
|
+
if (leaseError !== undefined) {
|
|
3952
|
+
return repairRejected(leaseError);
|
|
3953
|
+
}
|
|
3954
|
+
if (record.repairScopeVersion !== input.scopeVersion) {
|
|
3955
|
+
return repairRejected("repair_scope_snapshot_changed");
|
|
3956
|
+
}
|
|
3957
|
+
const exactFence = fenceFor(record.fences, PARTIAL_WRITE_FENCE_REASON, input.attemptId);
|
|
3958
|
+
if (exactFence === undefined) {
|
|
3959
|
+
return { status: 200, body: { status: "repair-fence-cleared", idempotent: true, version: record.version } };
|
|
3960
|
+
}
|
|
3961
|
+
const facts = (await txn.get(REPAIR_FACTS_KEY)) ?? repairFactsDefault();
|
|
3962
|
+
const attemptScope = record.repairScope.filter((item) => item.attemptId === input.attemptId);
|
|
3963
|
+
if (attemptScope.length === 0 || attemptScope.some((item) => {
|
|
3964
|
+
const resolution = repairResolutionFor(facts, item);
|
|
3965
|
+
return resolution === undefined || resolution.branch === "FAILED_CLOSED" || repairAuditFor(facts, item) === undefined;
|
|
3966
|
+
})) {
|
|
3967
|
+
return repairRejected("repair_scope_not_durably_resolved");
|
|
3968
|
+
}
|
|
3969
|
+
const updated = await this.commit(txn, record, {
|
|
3970
|
+
fences: withoutFence(record.fences, PARTIAL_WRITE_FENCE_REASON, input.attemptId),
|
|
3971
|
+
clearedFences: withFence(record.clearedFences, exactFence),
|
|
3972
|
+
});
|
|
3973
|
+
return { status: 200, body: { status: "repair-fence-cleared", idempotent: false, version: updated.version } };
|
|
3974
|
+
});
|
|
3975
|
+
return json(result.body, result.status);
|
|
3976
|
+
}
|
|
3977
|
+
async setClockOffset(tag, body) {
|
|
3978
|
+
if (!isObject(body)) {
|
|
3979
|
+
return error(400, "invalid_tag_clock", "offsetMs or nowMs must be a safe integer");
|
|
3980
|
+
}
|
|
3981
|
+
const nowMs = body.nowMs;
|
|
3982
|
+
const offsetMs = body.offsetMs;
|
|
3983
|
+
if (!isSafeInteger(nowMs) && !isSafeInteger(offsetMs)) {
|
|
3984
|
+
return error(400, "invalid_tag_clock", "offsetMs or nowMs must be a safe integer");
|
|
3985
|
+
}
|
|
3986
|
+
const result = await this.ctx.storage.transaction(async (txn) => {
|
|
3987
|
+
const loaded = await this.recordFor(txn, tag);
|
|
3988
|
+
const updated = await this.commit(txn, loaded.record, {
|
|
3989
|
+
clockOffsetMs: isSafeInteger(offsetMs) ? offsetMs : loaded.record.clockOffsetMs,
|
|
3990
|
+
clockNowMs: isSafeInteger(nowMs) ? nowMs : loaded.record.clockNowMs,
|
|
3991
|
+
});
|
|
3992
|
+
return {
|
|
3993
|
+
status: 200,
|
|
3994
|
+
body: {
|
|
3995
|
+
status: "clock-set",
|
|
3996
|
+
offsetMs: updated.clockOffsetMs,
|
|
3997
|
+
nowMs: updated.clockNowMs,
|
|
3998
|
+
version: updated.version,
|
|
3999
|
+
},
|
|
4000
|
+
};
|
|
4001
|
+
});
|
|
4002
|
+
return json(result.body, result.status);
|
|
4003
|
+
}
|
|
4004
|
+
async runAlarm() {
|
|
4005
|
+
const due = await this.ctx.storage.transaction(async (txn) => {
|
|
4006
|
+
const sql = this.sqlStorage();
|
|
4007
|
+
if (sql === undefined) {
|
|
4008
|
+
const storedTag = this.fallbackRecord?.tag;
|
|
4009
|
+
if (storedTag === undefined)
|
|
4010
|
+
return { record: undefined, retryDue: false };
|
|
4011
|
+
const record = this.readStoredRecord(storedTag);
|
|
4012
|
+
if (record === undefined)
|
|
4013
|
+
return { record: undefined, retryDue: false };
|
|
4014
|
+
const expiry = expireReservation(record);
|
|
4015
|
+
return {
|
|
4016
|
+
record: expiry.expired ? await this.commit(txn, expiry.record, {}) : record,
|
|
4017
|
+
retryDue: false,
|
|
4018
|
+
};
|
|
4019
|
+
}
|
|
4020
|
+
const identity = sql.exec("SELECT tag FROM tag_identity WHERE singleton = 1").toArray()[0];
|
|
4021
|
+
if (identity === undefined)
|
|
4022
|
+
return { record: undefined, retryDue: false };
|
|
4023
|
+
const tag = sqlString(identity.tag, "tag_identity.tag");
|
|
4024
|
+
const control = sql.exec("SELECT * FROM tag_control WHERE singleton = 1").one();
|
|
4025
|
+
const now = sqlNullableNumber(control.clock_now_ms, "tag_control.clock_now_ms") ??
|
|
4026
|
+
Date.now() + sqlNumber(control.clock_offset_ms, "tag_control.clock_offset_ms");
|
|
4027
|
+
const reservation = sql.exec("SELECT expires_at FROM tag_reservation WHERE singleton = 1").toArray()[0];
|
|
4028
|
+
if (reservation !== undefined && sqlNumber(reservation.expires_at, "tag_reservation.expires_at") <= now) {
|
|
4029
|
+
sql.exec("DELETE FROM tag_reservation WHERE singleton = 1");
|
|
4030
|
+
sql.exec("UPDATE tag_control SET version = ?, updated_at = ? WHERE singleton = 1", sqlNumber(control.version, "tag_control.version") + 1, nowIso());
|
|
4031
|
+
}
|
|
4032
|
+
const service = sql.exec("SELECT service_id FROM tag_head WHERE singleton = 1").toArray()[0];
|
|
4033
|
+
const retryDue = sqlNumber(sql.exec(`
|
|
4034
|
+
SELECT COUNT(*) AS count FROM tag_outbox_obligation
|
|
4035
|
+
WHERE status = 'pending' AND next_attempt_at <= ?
|
|
4036
|
+
`, Date.now()).one().count, "tag_outbox_obligation.count") > 0;
|
|
4037
|
+
const sourceRegistration = sql.exec(`
|
|
4038
|
+
SELECT service_id, partition_tag
|
|
4039
|
+
FROM tag_source_partition_registration
|
|
4040
|
+
WHERE status = 'pending' AND next_attempt_at <= ?
|
|
4041
|
+
ORDER BY next_attempt_at ASC, service_id COLLATE BINARY ASC, partition_tag COLLATE BINARY ASC
|
|
4042
|
+
LIMIT 1
|
|
4043
|
+
`, Date.now()).toArray()[0];
|
|
4044
|
+
await this.rearmScheduler(txn);
|
|
4045
|
+
return {
|
|
4046
|
+
record: this.readStoredRecord(tag),
|
|
4047
|
+
tag,
|
|
4048
|
+
serviceId: service === undefined ? undefined : sqlString(service.service_id, "tag_head.service_id"),
|
|
4049
|
+
retryDue,
|
|
4050
|
+
sourceRegistrationDue: sourceRegistration === undefined ? undefined : {
|
|
4051
|
+
serviceId: sqlString(sourceRegistration.service_id, "tag_source_partition_registration.service_id"),
|
|
4052
|
+
tag: sqlString(sourceRegistration.partition_tag, "tag_source_partition_registration.partition_tag"),
|
|
4053
|
+
},
|
|
4054
|
+
};
|
|
4055
|
+
});
|
|
4056
|
+
if (due.sourceRegistrationDue !== undefined) {
|
|
4057
|
+
await this.retrySourcePartitionRegistration(due.sourceRegistrationDue.tag, due.sourceRegistrationDue.serviceId).catch(() => undefined);
|
|
4058
|
+
}
|
|
4059
|
+
if (due.retryDue &&
|
|
4060
|
+
this.env.AUTO_DRAIN_OUTBOX === "true" &&
|
|
4061
|
+
due.tag !== undefined &&
|
|
4062
|
+
isNonEmptyString(due.serviceId)) {
|
|
4063
|
+
// Delivery is outside the storage transaction. Each selected obligation
|
|
4064
|
+
// already has a retry timestamp, and per-row failures cannot starve
|
|
4065
|
+
// sibling rows or reservation expiry.
|
|
4066
|
+
await this.autoDrainOutbox(due.tag, due.serviceId, undefined, OBLIGATION_ALARM_BATCH_LIMIT).catch(() => undefined);
|
|
4067
|
+
}
|
|
4068
|
+
return due.record;
|
|
4069
|
+
}
|
|
4070
|
+
}
|
|
4071
|
+
export { MAX_EPOCH };
|