@sekiban/dcb-runtime 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +86 -0
- package/README.md +13 -0
- package/dist/allocator/AllocatorDurableObject.d.ts +36 -0
- package/dist/allocator/AllocatorDurableObject.js +658 -0
- package/dist/allocator/IssuanceLedger.d.ts +188 -0
- package/dist/allocator/IssuanceLedger.js +608 -0
- package/dist/allocator/IssuanceReconciler.d.ts +41 -0
- package/dist/allocator/IssuanceReconciler.js +191 -0
- package/dist/allocator/OrderClock.d.ts +41 -0
- package/dist/allocator/OrderClock.js +92 -0
- package/dist/allocator/SortableUniqueId.d.ts +46 -0
- package/dist/allocator/SortableUniqueId.js +156 -0
- package/dist/allocator/types.d.ts +52 -0
- package/dist/allocator/types.js +1 -0
- package/dist/bootstrap/BootstrapCoordinatorDurableObject.d.ts +28 -0
- package/dist/bootstrap/BootstrapCoordinatorDurableObject.js +357 -0
- package/dist/bootstrap/BootstrapStoreAdapter.d.ts +51 -0
- package/dist/bootstrap/BootstrapStoreAdapter.js +162 -0
- package/dist/bootstrap/OperatorBootstrap.d.ts +20 -0
- package/dist/bootstrap/OperatorBootstrap.js +104 -0
- package/dist/bootstrap/manifest.d.ts +9 -0
- package/dist/bootstrap/manifest.js +115 -0
- package/dist/bootstrap/types.d.ts +75 -0
- package/dist/bootstrap/types.js +1 -0
- package/dist/cli/OperatorRepairCli.d.ts +11 -0
- package/dist/cli/OperatorRepairCli.js +108 -0
- package/dist/cloudflare.d.ts +164 -0
- package/dist/cloudflare.js +19487 -0
- package/dist/commit/CommitWorker.d.ts +101 -0
- package/dist/commit/CommitWorker.js +921 -0
- package/dist/commit/types.d.ts +25 -0
- package/dist/commit/types.js +13 -0
- package/dist/completeness/GlobalCompletenessReconciler.d.ts +77 -0
- package/dist/completeness/GlobalCompletenessReconciler.js +572 -0
- package/dist/completeness/types.d.ts +85 -0
- package/dist/completeness/types.js +6 -0
- package/dist/composition.d.ts +123 -0
- package/dist/composition.js +237 -0
- package/dist/cosmos.d.ts +11 -0
- package/dist/cosmos.js +1411 -0
- package/dist/d1-mv.d.ts +8 -0
- package/dist/d1-mv.js +2001 -0
- package/dist/d1.d.ts +16 -0
- package/dist/d1.js +3418 -0
- package/dist/diagnostics/G60DurableHop.d.ts +99 -0
- package/dist/diagnostics/G60DurableHop.js +248 -0
- package/dist/diagnostics/G65Admission.d.ts +23 -0
- package/dist/diagnostics/G65Admission.js +43 -0
- package/dist/diagnostics/G65DirectRing.d.ts +33 -0
- package/dist/diagnostics/G65DirectRing.js +85 -0
- package/dist/diagnostics/G69AdmissionAttempt.d.ts +43 -0
- package/dist/diagnostics/G69AdmissionAttempt.js +22 -0
- package/dist/downstream/DeliveryCore.d.ts +127 -0
- package/dist/downstream/DeliveryCore.js +304 -0
- package/dist/downstream/Doorbell.d.ts +43 -0
- package/dist/downstream/Doorbell.js +137 -0
- package/dist/downstream/DownstreamAdapter.d.ts +52 -0
- package/dist/downstream/DownstreamAdapter.js +299 -0
- package/dist/downstream/ExclusionLookup.d.ts +36 -0
- package/dist/downstream/ExclusionLookup.js +50 -0
- package/dist/downstream/InconsistencyDetector.d.ts +27 -0
- package/dist/downstream/InconsistencyDetector.js +82 -0
- package/dist/downstream/OutboxDrain.d.ts +35 -0
- package/dist/downstream/OutboxDrain.js +126 -0
- package/dist/downstream/types.d.ts +77 -0
- package/dist/downstream/types.js +64 -0
- package/dist/eventIdentity.d.ts +30 -0
- package/dist/eventIdentity.js +49 -0
- package/dist/eventRecord.d.ts +26 -0
- package/dist/eventRecord.js +53 -0
- package/dist/g42-probe.d.ts +9 -0
- package/dist/g42-probe.js +9 -0
- package/dist/http/SerializedQueryWorker.d.ts +58 -0
- package/dist/http/SerializedQueryWorker.js +497 -0
- package/dist/http/commitResponse.d.ts +46 -0
- package/dist/http/commitResponse.js +80 -0
- package/dist/http/testServiceId.d.ts +17 -0
- package/dist/http/testServiceId.js +14 -0
- package/dist/index.d.ts +112 -0
- package/dist/index.js +19926 -0
- package/dist/journal/JournalDurableObject.d.ts +30 -0
- package/dist/journal/JournalDurableObject.js +481 -0
- package/dist/journal/JournalFirstTouchProbe.d.ts +190 -0
- package/dist/journal/JournalFirstTouchProbe.js +375 -0
- package/dist/journal/types.d.ts +116 -0
- package/dist/journal/types.js +52 -0
- package/dist/mv/MaterializedViewCatchUp.d.ts +60 -0
- package/dist/mv/MaterializedViewCatchUp.js +353 -0
- package/dist/mv/MaterializedViewRuntime.d.ts +80 -0
- package/dist/mv/MaterializedViewRuntime.js +328 -0
- package/dist/mv/MaterializedViewStore.d.ts +257 -0
- package/dist/mv/MaterializedViewStore.js +875 -0
- package/dist/mv/UnsafeWindowMaterializedView.d.ts +112 -0
- package/dist/mv/UnsafeWindowMaterializedView.js +495 -0
- package/dist/mv/index.d.ts +8 -0
- package/dist/mv/index.js +2357 -0
- package/dist/projection/LiveProjectionWorker.d.ts +86 -0
- package/dist/projection/LiveProjectionWorker.js +240 -0
- package/dist/projection/ProjectionRuntime.d.ts +113 -0
- package/dist/projection/ProjectionRuntime.js +346 -0
- package/dist/projection/ProjectorRegistry.d.ts +51 -0
- package/dist/projection/ProjectorRegistry.js +116 -0
- package/dist/query/ProjectionQueryStore.d.ts +102 -0
- package/dist/query/ProjectionQueryStore.js +185 -0
- package/dist/query/QueryRegistry.d.ts +22 -0
- package/dist/query/QueryRegistry.js +36 -0
- package/dist/read/SerializedReadWorker.d.ts +36 -0
- package/dist/read/SerializedReadWorker.js +225 -0
- package/dist/repair/RepairWorker.d.ts +72 -0
- package/dist/repair/RepairWorker.js +461 -0
- package/dist/safeWindow.d.ts +20 -0
- package/dist/safeWindow.js +36 -0
- package/dist/scope/ControlRouteScope.d.ts +22 -0
- package/dist/scope/ControlRouteScope.js +36 -0
- package/dist/scope/ScopeName.d.ts +34 -0
- package/dist/scope/ScopeName.js +67 -0
- package/dist/service/ServiceIdentityProvider.d.ts +35 -0
- package/dist/service/ServiceIdentityProvider.js +56 -0
- package/dist/store/CosmosEventStore.d.ts +126 -0
- package/dist/store/CosmosEventStore.js +950 -0
- package/dist/store/D1EventStore.d.ts +111 -0
- package/dist/store/D1EventStore.js +1111 -0
- package/dist/store/PostgresEventStore.d.ts +35 -0
- package/dist/store/PostgresEventStore.js +811 -0
- package/dist/store/provider.d.ts +25 -0
- package/dist/store/provider.js +32 -0
- package/dist/store/types.d.ts +205 -0
- package/dist/store/types.js +18 -0
- package/dist/tag/EventDigest.d.ts +40 -0
- package/dist/tag/EventDigest.js +158 -0
- package/dist/tag/TagDurableObject.d.ts +286 -0
- package/dist/tag/TagDurableObject.js +4071 -0
- package/dist/tag/TagSqlMeasurement.d.ts +26 -0
- package/dist/tag/TagSqlMeasurement.js +44 -0
- package/dist/tag/TagSqlSchema.d.ts +29 -0
- package/dist/tag/TagSqlSchema.js +211 -0
- package/dist/tag/types.d.ts +173 -0
- package/dist/tag/types.js +3 -0
- package/dist/tagstate/TagStateDurableObject.d.ts +57 -0
- package/dist/tagstate/TagStateDurableObject.js +499 -0
- package/dist/trace/AttributionRatio.d.ts +12 -0
- package/dist/trace/AttributionRatio.js +68 -0
- package/dist/trace/CloudflareTracing.d.ts +7 -0
- package/dist/trace/CloudflareTracing.js +9 -0
- package/dist/trace/CommitTrace.d.ts +405 -0
- package/dist/trace/CommitTrace.js +1008 -0
- package/dist/trace/CommitTraceConsoleSink.d.ts +68 -0
- package/dist/trace/CommitTraceConsoleSink.js +114 -0
- package/dist/trace/CommitTraceVerifier.d.ts +22 -0
- package/dist/trace/CommitTraceVerifier.js +295 -0
- package/dist/trace/ObservationStream.d.ts +118 -0
- package/dist/trace/ObservationStream.js +148 -0
- package/package.json +71 -0
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
export const JOURNAL_STATES = [
|
|
2
|
+
"ADMITTED",
|
|
3
|
+
"RESERVED",
|
|
4
|
+
"ALLOCATED",
|
|
5
|
+
"WRITING",
|
|
6
|
+
"SEALING",
|
|
7
|
+
"COMPLETE",
|
|
8
|
+
"REFUSED",
|
|
9
|
+
"FAILED",
|
|
10
|
+
"PARTIAL",
|
|
11
|
+
"ABANDONED",
|
|
12
|
+
];
|
|
13
|
+
export const TERMINAL_JOURNAL_STATES = new Set([
|
|
14
|
+
"COMPLETE",
|
|
15
|
+
"REFUSED",
|
|
16
|
+
"FAILED",
|
|
17
|
+
"PARTIAL",
|
|
18
|
+
"ABANDONED",
|
|
19
|
+
]);
|
|
20
|
+
/**
|
|
21
|
+
* Persisted fault points used by the commit-worker recovery tests. They are
|
|
22
|
+
* consumed one at a time, so an at-least-once alarm can resume safely.
|
|
23
|
+
*/
|
|
24
|
+
export const ALARM_FAULT_POINTS = [
|
|
25
|
+
"handler-entry",
|
|
26
|
+
"after-rearm-before-seal",
|
|
27
|
+
"after-partial-seal",
|
|
28
|
+
"after-full-seal-before-requery",
|
|
29
|
+
"after-fence-before-cancel",
|
|
30
|
+
"before-outcome-cas",
|
|
31
|
+
];
|
|
32
|
+
export function isTerminalState(state) {
|
|
33
|
+
return TERMINAL_JOURNAL_STATES.has(state);
|
|
34
|
+
}
|
|
35
|
+
export function isJournalState(value) {
|
|
36
|
+
return typeof value === "string" && JOURNAL_STATES.includes(value);
|
|
37
|
+
}
|
|
38
|
+
export function isAllowedTransition(from, to) {
|
|
39
|
+
const allowed = {
|
|
40
|
+
ADMITTED: ["RESERVED", "ALLOCATED", "SEALING", "ABANDONED"],
|
|
41
|
+
RESERVED: ["ALLOCATED", "SEALING", "REFUSED", "FAILED", "ABANDONED"],
|
|
42
|
+
ALLOCATED: ["WRITING", "SEALING", "COMPLETE"],
|
|
43
|
+
WRITING: ["SEALING", "COMPLETE"],
|
|
44
|
+
SEALING: ["COMPLETE", "REFUSED", "FAILED", "PARTIAL"],
|
|
45
|
+
COMPLETE: [],
|
|
46
|
+
REFUSED: [],
|
|
47
|
+
FAILED: [],
|
|
48
|
+
PARTIAL: [],
|
|
49
|
+
ABANDONED: [],
|
|
50
|
+
};
|
|
51
|
+
return allowed[from].includes(to);
|
|
52
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import type { MaterializedViewRowMaterializer } from "@sekiban/dcb-core";
|
|
2
|
+
import type { ProjectionStore, StoredEvent } from "../store/types";
|
|
3
|
+
import { type MaterializedViewApplyResult, type MaterializedViewInstance } from "./MaterializedViewStore";
|
|
4
|
+
import type { D1MaterializedViewStore } from "./MaterializedViewStore";
|
|
5
|
+
export interface MaterializedViewCatchUpResult {
|
|
6
|
+
readonly instance: MaterializedViewInstance;
|
|
7
|
+
readonly dynamicLagBoundMs: number;
|
|
8
|
+
readonly safeWindowMs: number;
|
|
9
|
+
readonly advancedSourceEvents: number;
|
|
10
|
+
readonly appliedEvents: number;
|
|
11
|
+
/** Bounded late-lower detector query cost for this catch-up pass. */
|
|
12
|
+
readonly lateLowerQueryDurationMs: number;
|
|
13
|
+
readonly indeterminate: boolean;
|
|
14
|
+
/** The first source event withheld by the current safe-lane boundary. */
|
|
15
|
+
readonly deferredEventSuid: string | null;
|
|
16
|
+
readonly deferredEventLastArrivedAt: number | null;
|
|
17
|
+
readonly deferredDeadlineAt: number | null;
|
|
18
|
+
readonly stopReason: "safe_window_fence" | "safe_window_ceiling" | "frontier_fence" | null;
|
|
19
|
+
}
|
|
20
|
+
export interface MaterializedViewCatchUpHooks {
|
|
21
|
+
/** Test-only crash boundary; never part of production persistence. */
|
|
22
|
+
readonly beforeApply?: (event: StoredEvent) => Promise<void> | void;
|
|
23
|
+
/** Test-only crash boundary after the atomic MV call. */
|
|
24
|
+
readonly afterApply?: (event: StoredEvent, result: MaterializedViewApplyResult) => Promise<void> | void;
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* An optional scanner-proven high-water mark for a safe-lane pass. `null`
|
|
28
|
+
* means that no FULL snapshot exists yet, so this pass may perform GC but
|
|
29
|
+
* cannot advance a source checkpoint. `undefined` retains the existing
|
|
30
|
+
* ungated caller behaviour.
|
|
31
|
+
*/
|
|
32
|
+
export interface MaterializedViewCatchUpOptions {
|
|
33
|
+
readonly maximumSuid?: string | null;
|
|
34
|
+
/** Run the bounded late-lower detector only from scheduled maintenance. */
|
|
35
|
+
readonly runOrderingDetector?: boolean;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* SafeWindow-aware row materializer runtime. It keeps the memory projection
|
|
39
|
+
* runtime untouched while allowing the same event source to feed a separate
|
|
40
|
+
* D1 MV database.
|
|
41
|
+
*/
|
|
42
|
+
export declare class MaterializedViewCatchUpRuntime {
|
|
43
|
+
private readonly source;
|
|
44
|
+
private readonly materializedViews;
|
|
45
|
+
constructor(source: ProjectionStore, materializedViews: D1MaterializedViewStore);
|
|
46
|
+
build<TEvent = StoredEvent>(serviceId: string, materializer: MaterializedViewRowMaterializer<TEvent>, nowMs: number, hooks?: MaterializedViewCatchUpHooks, options?: MaterializedViewCatchUpOptions): Promise<MaterializedViewCatchUpResult>;
|
|
47
|
+
follow<TEvent = StoredEvent>(serviceId: string, materializer: MaterializedViewRowMaterializer<TEvent>, nowMs: number, hooks?: MaterializedViewCatchUpHooks, options?: MaterializedViewCatchUpOptions): Promise<MaterializedViewCatchUpResult>;
|
|
48
|
+
rebuild<TEvent = StoredEvent>(serviceId: string, materializer: MaterializedViewRowMaterializer<TEvent>, nowMs: number, rebuildId?: string, hooks?: MaterializedViewCatchUpHooks, options?: MaterializedViewCatchUpOptions): Promise<MaterializedViewCatchUpResult & {
|
|
49
|
+
readonly candidateGeneration: number;
|
|
50
|
+
readonly rebuildId?: string;
|
|
51
|
+
}>;
|
|
52
|
+
promote(serviceId: string, materializer: MaterializedViewRowMaterializer, candidateGeneration: number, nowMs: number): Promise<MaterializedViewInstance>;
|
|
53
|
+
read(serviceId: string, materializer: MaterializedViewRowMaterializer): Promise<{
|
|
54
|
+
readonly instance: MaterializedViewInstance;
|
|
55
|
+
readonly rows: Awaited<ReturnType<D1MaterializedViewStore["readRows"]>>;
|
|
56
|
+
} | undefined>;
|
|
57
|
+
private followGeneration;
|
|
58
|
+
private detectLateLowerSuid;
|
|
59
|
+
private assertSourceBatchOrder;
|
|
60
|
+
}
|
|
@@ -0,0 +1,353 @@
|
|
|
1
|
+
import { safeWindowCeilingExceeded, safeWindowMs } from "../projection/ProjectionRuntime";
|
|
2
|
+
import { assertSortableUniqueId } from "../allocator/SortableUniqueId";
|
|
3
|
+
import { MaterializedViewCasError, MaterializedViewStoreError, } from "./MaterializedViewStore";
|
|
4
|
+
const MAX_CAS_RETRIES = 8;
|
|
5
|
+
/**
|
|
6
|
+
* SafeWindow-aware row materializer runtime. It keeps the memory projection
|
|
7
|
+
* runtime untouched while allowing the same event source to feed a separate
|
|
8
|
+
* D1 MV database.
|
|
9
|
+
*/
|
|
10
|
+
export class MaterializedViewCatchUpRuntime {
|
|
11
|
+
source;
|
|
12
|
+
materializedViews;
|
|
13
|
+
constructor(source, materializedViews) {
|
|
14
|
+
this.source = source;
|
|
15
|
+
this.materializedViews = materializedViews;
|
|
16
|
+
}
|
|
17
|
+
async build(serviceId, materializer, nowMs, hooks = {}, options = {}) {
|
|
18
|
+
const existing = await this.materializedViews.readActive(serviceId, materializer.id);
|
|
19
|
+
if (existing !== undefined) {
|
|
20
|
+
throw new MaterializedViewStoreError("create-active", "MV_INSTANCE_EXISTS", "Active materialized view already exists");
|
|
21
|
+
}
|
|
22
|
+
await this.materializedViews.createActive({
|
|
23
|
+
serviceId,
|
|
24
|
+
viewId: materializer.id,
|
|
25
|
+
generation: 0,
|
|
26
|
+
definitionVersion: materializer.version,
|
|
27
|
+
updatedAt: nowMs,
|
|
28
|
+
});
|
|
29
|
+
return this.follow(serviceId, materializer, nowMs, hooks, options);
|
|
30
|
+
}
|
|
31
|
+
async follow(serviceId, materializer, nowMs, hooks = {}, options = {}) {
|
|
32
|
+
const active = await this.materializedViews.readActive(serviceId, materializer.id);
|
|
33
|
+
if (active === undefined) {
|
|
34
|
+
throw new MaterializedViewStoreError("apply", "MV_INSTANCE_MISSING", "Materialized-view must be built before follow");
|
|
35
|
+
}
|
|
36
|
+
return this.followGeneration(serviceId, materializer, active.generation, nowMs, "apply", hooks, options);
|
|
37
|
+
}
|
|
38
|
+
async rebuild(serviceId, materializer, nowMs, rebuildId, hooks = {}, options = {}) {
|
|
39
|
+
const effectiveRebuildId = rebuildId ?? `g69-rebuild-${crypto.randomUUID()}`;
|
|
40
|
+
const sourceBefore = await this.source.readAllEvents(serviceId, "");
|
|
41
|
+
const sourceBeforeProof = await sourceHistoryProof(sourceBefore);
|
|
42
|
+
const candidate = await this.materializedViews.beginRebuild({
|
|
43
|
+
serviceId,
|
|
44
|
+
viewId: materializer.id,
|
|
45
|
+
definitionVersion: materializer.version,
|
|
46
|
+
updatedAt: nowMs,
|
|
47
|
+
});
|
|
48
|
+
const result = await this.followGeneration(serviceId, materializer, candidate.generation, nowMs, "apply", hooks, options);
|
|
49
|
+
const sourceAfter = await this.source.readAllEvents(serviceId, "");
|
|
50
|
+
const sourceAfterProof = await sourceHistoryProof(sourceAfter);
|
|
51
|
+
if (result.stopReason === null &&
|
|
52
|
+
!result.indeterminate &&
|
|
53
|
+
result.advancedSourceEvents === sourceBefore.length &&
|
|
54
|
+
sourceBeforeProof.digest === sourceAfterProof.digest) {
|
|
55
|
+
await this.materializedViews.markGenerationRebuilt({
|
|
56
|
+
serviceId,
|
|
57
|
+
viewId: materializer.id,
|
|
58
|
+
generation: candidate.generation,
|
|
59
|
+
verifiedAt: nowMs,
|
|
60
|
+
rebuildId: effectiveRebuildId,
|
|
61
|
+
sourceEventCount: sourceBeforeProof.eventIds.length,
|
|
62
|
+
sourceEventIds: sourceBeforeProof.eventIds,
|
|
63
|
+
sourceSuids: sourceBeforeProof.suids,
|
|
64
|
+
sourceMaxSuid: sourceBeforeProof.maxSuid,
|
|
65
|
+
sourceHistoryDigest: sourceBeforeProof.digest,
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
return { ...result, candidateGeneration: candidate.generation, rebuildId: effectiveRebuildId };
|
|
69
|
+
}
|
|
70
|
+
async promote(serviceId, materializer, candidateGeneration, nowMs) {
|
|
71
|
+
const active = await this.materializedViews.readActive(serviceId, materializer.id);
|
|
72
|
+
return this.materializedViews.promoteGeneration({
|
|
73
|
+
serviceId,
|
|
74
|
+
viewId: materializer.id,
|
|
75
|
+
candidateGeneration,
|
|
76
|
+
expectedActiveGeneration: active?.generation ?? null,
|
|
77
|
+
updatedAt: nowMs,
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
async read(serviceId, materializer) {
|
|
81
|
+
const instance = await this.materializedViews.readActive(serviceId, materializer.id);
|
|
82
|
+
if (instance === undefined)
|
|
83
|
+
return undefined;
|
|
84
|
+
return { instance, rows: await this.materializedViews.readRows(serviceId, materializer.id, instance.generation) };
|
|
85
|
+
}
|
|
86
|
+
async followGeneration(serviceId, materializer, generation, nowMs, _operation, hooks, options) {
|
|
87
|
+
for (let attempt = 0; attempt < MAX_CAS_RETRIES; attempt += 1) {
|
|
88
|
+
const instance = await this.materializedViews.readInstance(serviceId, materializer.id, generation);
|
|
89
|
+
if (instance === undefined) {
|
|
90
|
+
throw new MaterializedViewStoreError("apply", "MV_INSTANCE_MISSING", "Materialized-view generation is missing");
|
|
91
|
+
}
|
|
92
|
+
// `readAllEvents(serviceId, checkpoint)` is empty both for a caught-up
|
|
93
|
+
// source and for a reset source that is now behind the checkpoint. Read
|
|
94
|
+
// the source head separately so the latter can never freeze silently.
|
|
95
|
+
const sourceForHead = await this.source.readAllEvents(serviceId, "");
|
|
96
|
+
for (const event of sourceForHead)
|
|
97
|
+
assertSortableUniqueId(event.suid);
|
|
98
|
+
const storeMaxSuid = sourceForHead.reduce((maximum, event) => compareSuid(maximum, event.suid) >= 0 ? maximum : event.suid, "");
|
|
99
|
+
if (instance.lastSuid.length > 0 && compareSuid(storeMaxSuid, instance.lastSuid) < 0) {
|
|
100
|
+
await this.materializedViews.recordCheckpointAhead({
|
|
101
|
+
serviceId,
|
|
102
|
+
viewId: materializer.id,
|
|
103
|
+
generation,
|
|
104
|
+
checkpointSuid: instance.lastSuid,
|
|
105
|
+
storeMaxSuid,
|
|
106
|
+
observedAt: nowMs,
|
|
107
|
+
});
|
|
108
|
+
throw new MaterializedViewStoreError("apply", "MV_STORE_OPERATION_FAILED", "CHECKPOINT_AHEAD: materialized-view checkpoint is ahead of the source store; rebuild and promote a generation");
|
|
109
|
+
}
|
|
110
|
+
const dynamicLagBoundMs = await this.source.currentLagBound(serviceId, nowMs);
|
|
111
|
+
const windowMs = safeWindowMs(dynamicLagBoundMs);
|
|
112
|
+
if (safeWindowCeilingExceeded(dynamicLagBoundMs)) {
|
|
113
|
+
return {
|
|
114
|
+
instance,
|
|
115
|
+
dynamicLagBoundMs,
|
|
116
|
+
safeWindowMs: windowMs,
|
|
117
|
+
advancedSourceEvents: 0,
|
|
118
|
+
appliedEvents: 0,
|
|
119
|
+
lateLowerQueryDurationMs: 0,
|
|
120
|
+
indeterminate: true,
|
|
121
|
+
deferredEventSuid: null,
|
|
122
|
+
deferredEventLastArrivedAt: null,
|
|
123
|
+
deferredDeadlineAt: null,
|
|
124
|
+
stopReason: "safe_window_ceiling",
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
const sourceEvents = await this.source.readAllEvents(serviceId, instance.lastSuid);
|
|
128
|
+
await this.assertSourceBatchOrder(serviceId, materializer.id, generation, instance.lastSuid, sourceEvents);
|
|
129
|
+
let lateLowerQueryDurationMs = 0;
|
|
130
|
+
if (options.runOrderingDetector === true) {
|
|
131
|
+
lateLowerQueryDurationMs = await this.detectLateLowerSuid(serviceId, materializer.id, generation, instance.lastSuid, instance.updatedAt);
|
|
132
|
+
}
|
|
133
|
+
let current = instance;
|
|
134
|
+
let advancedSourceEvents = 0;
|
|
135
|
+
let appliedEvents = 0;
|
|
136
|
+
let conflicted = false;
|
|
137
|
+
for (const event of sourceEvents) {
|
|
138
|
+
// A G44 BLOCK tick may drain only through the last FULL snapshot's
|
|
139
|
+
// high-water mark. Do not skip an earlier row or treat a later one
|
|
140
|
+
// as a new safe frontier; either would violate the source ordering
|
|
141
|
+
// proof that protects the materialized safe lane.
|
|
142
|
+
if (options.maximumSuid === null || (options.maximumSuid !== undefined && compareSuid(event.suid, options.maximumSuid) > 0)) {
|
|
143
|
+
return {
|
|
144
|
+
instance: current,
|
|
145
|
+
dynamicLagBoundMs,
|
|
146
|
+
safeWindowMs: windowMs,
|
|
147
|
+
advancedSourceEvents,
|
|
148
|
+
appliedEvents,
|
|
149
|
+
lateLowerQueryDurationMs,
|
|
150
|
+
indeterminate: false,
|
|
151
|
+
deferredEventSuid: event.suid,
|
|
152
|
+
deferredEventLastArrivedAt: event.lastArrivedAt,
|
|
153
|
+
deferredDeadlineAt: null,
|
|
154
|
+
stopReason: "frontier_fence",
|
|
155
|
+
};
|
|
156
|
+
}
|
|
157
|
+
// Read-only SafeWindow rule: do not skip the first unsafe event or
|
|
158
|
+
// process later events ahead of a late lower SUID.
|
|
159
|
+
if (event.lastArrivedAt > nowMs - windowMs) {
|
|
160
|
+
return {
|
|
161
|
+
instance: current,
|
|
162
|
+
dynamicLagBoundMs,
|
|
163
|
+
safeWindowMs: windowMs,
|
|
164
|
+
advancedSourceEvents,
|
|
165
|
+
appliedEvents,
|
|
166
|
+
lateLowerQueryDurationMs,
|
|
167
|
+
indeterminate: false,
|
|
168
|
+
deferredEventSuid: event.suid,
|
|
169
|
+
deferredEventLastArrivedAt: event.lastArrivedAt,
|
|
170
|
+
deferredDeadlineAt: event.lastArrivedAt + windowMs,
|
|
171
|
+
stopReason: "safe_window_fence",
|
|
172
|
+
};
|
|
173
|
+
}
|
|
174
|
+
await hooks.beforeApply?.(event);
|
|
175
|
+
const mutations = materializer.plan(event);
|
|
176
|
+
let result;
|
|
177
|
+
try {
|
|
178
|
+
result = await this.materializedViews.applyMutationsAndAdvanceCheckpoint({
|
|
179
|
+
serviceId,
|
|
180
|
+
viewId: materializer.id,
|
|
181
|
+
generation,
|
|
182
|
+
// The durable schema represents an unadvanced checkpoint as an
|
|
183
|
+
// empty string, while the atomic apply port reserves `null` for
|
|
184
|
+
// that state so all non-null ingress values are full G32 SUIDs.
|
|
185
|
+
expectedLastSuid: current.lastSuid.length === 0 ? null : current.lastSuid,
|
|
186
|
+
lastSuid: event.suid,
|
|
187
|
+
definitionVersion: materializer.version,
|
|
188
|
+
updatedAt: nowMs,
|
|
189
|
+
mutations,
|
|
190
|
+
});
|
|
191
|
+
}
|
|
192
|
+
catch (error) {
|
|
193
|
+
if (error instanceof MaterializedViewCasError) {
|
|
194
|
+
conflicted = true;
|
|
195
|
+
break;
|
|
196
|
+
}
|
|
197
|
+
throw error;
|
|
198
|
+
}
|
|
199
|
+
current = result.instance;
|
|
200
|
+
advancedSourceEvents += 1;
|
|
201
|
+
appliedEvents += 1;
|
|
202
|
+
// A safe checkpoint alone cannot retire unsafe marker state. The
|
|
203
|
+
// exact matching receipt is observed in its own guarded transaction.
|
|
204
|
+
const unsafe = this.materializedViews.unsafeWindow();
|
|
205
|
+
if (await unsafe.hasTargetReceipt(serviceId, materializer.id, event.eventId, event.suid)) {
|
|
206
|
+
await unsafe.observeSafeReceipt(serviceId, materializer.id, generation, event.eventId, event.suid);
|
|
207
|
+
}
|
|
208
|
+
await hooks.afterApply?.(event, result);
|
|
209
|
+
}
|
|
210
|
+
if (!conflicted) {
|
|
211
|
+
// This runs even when sourceEvents is empty: a previous unsafe write
|
|
212
|
+
// may only become collectable after the safe checkpoint is observed.
|
|
213
|
+
await this.materializedViews.collectUnsafeGarbage(serviceId, materializer.id, generation, materializer.version, current.lastSuid);
|
|
214
|
+
return {
|
|
215
|
+
instance: current,
|
|
216
|
+
dynamicLagBoundMs,
|
|
217
|
+
safeWindowMs: windowMs,
|
|
218
|
+
advancedSourceEvents,
|
|
219
|
+
appliedEvents,
|
|
220
|
+
lateLowerQueryDurationMs,
|
|
221
|
+
indeterminate: false,
|
|
222
|
+
deferredEventSuid: null,
|
|
223
|
+
deferredEventLastArrivedAt: null,
|
|
224
|
+
deferredDeadlineAt: null,
|
|
225
|
+
stopReason: null,
|
|
226
|
+
};
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
throw new MaterializedViewStoreError("apply", "MV_CAS_MISMATCH", "Materialized-view catch-up did not converge after concurrent updates");
|
|
230
|
+
}
|
|
231
|
+
async detectLateLowerSuid(serviceId, viewId, generation, priorSuid, checkpointUpdatedAt) {
|
|
232
|
+
let lateLowerQueryDurationMs = 0;
|
|
233
|
+
if (priorSuid.length > 0) {
|
|
234
|
+
const detectorStartedAt = Date.now();
|
|
235
|
+
const evidence = this.source.findLateLowerSuidEvidence === undefined
|
|
236
|
+
? undefined
|
|
237
|
+
: await this.source.findLateLowerSuidEvidence(serviceId, priorSuid, checkpointUpdatedAt, generation);
|
|
238
|
+
const lateLower = evidence === undefined
|
|
239
|
+
? await this.source.findLateLowerSuid?.(serviceId, priorSuid, checkpointUpdatedAt)
|
|
240
|
+
: evidence.kind === "late-lower-suid" ? evidence.event : undefined;
|
|
241
|
+
lateLowerQueryDurationMs = evidence?.durationMs ?? Math.max(0, Date.now() - detectorStartedAt);
|
|
242
|
+
if (evidence?.kind === "unknown") {
|
|
243
|
+
const incident = {
|
|
244
|
+
serviceId,
|
|
245
|
+
identityKey: `ORDERING_DETECTOR_UNKNOWN|${serviceId}|${viewId}|${generation}|${priorSuid}|${checkpointUpdatedAt}`,
|
|
246
|
+
classification: "ORDERING_DETECTOR_UNKNOWN",
|
|
247
|
+
suid: priorSuid,
|
|
248
|
+
observedAt: checkpointUpdatedAt,
|
|
249
|
+
};
|
|
250
|
+
// The detector has not proved a bounded refusal condition. Persist an
|
|
251
|
+
// alarm-only incident and continue the existing safe-lane path; an
|
|
252
|
+
// unknown clock/provenance observation must never be promoted to a
|
|
253
|
+
// fabricated ordering violation or a guessed quarantine.
|
|
254
|
+
await this.source.appendDeliveryIncident(incident);
|
|
255
|
+
console.error("SDT-G69_ORDERING_DETECTOR_UNKNOWN", JSON.stringify({
|
|
256
|
+
...incident,
|
|
257
|
+
reason: evidence.reason ?? "unknown",
|
|
258
|
+
}));
|
|
259
|
+
}
|
|
260
|
+
if (lateLower !== undefined) {
|
|
261
|
+
const incident = {
|
|
262
|
+
serviceId,
|
|
263
|
+
identityKey: `LATE_LOWER_SUID|${serviceId}|${viewId}|${priorSuid}|${lateLower.suid}|${lateLower.eventId}`,
|
|
264
|
+
classification: "ORDER_VIOLATION",
|
|
265
|
+
suid: lateLower.suid,
|
|
266
|
+
eventId: lateLower.eventId,
|
|
267
|
+
incomingEventId: lateLower.eventId,
|
|
268
|
+
observedAt: lateLower.lastArrivedAt,
|
|
269
|
+
};
|
|
270
|
+
await this.source.appendDeliveryIncident(incident);
|
|
271
|
+
await this.materializedViews.recordOrderingQuarantine({
|
|
272
|
+
serviceId,
|
|
273
|
+
viewId,
|
|
274
|
+
generation,
|
|
275
|
+
checkpointSuid: priorSuid,
|
|
276
|
+
lateSuid: lateLower.suid,
|
|
277
|
+
eventId: lateLower.eventId,
|
|
278
|
+
classification: "LATE_LOWER_SUID",
|
|
279
|
+
observedAt: lateLower.lastArrivedAt,
|
|
280
|
+
});
|
|
281
|
+
console.error("SDT-G69_ORDERING_QUARANTINE", JSON.stringify({
|
|
282
|
+
serviceId,
|
|
283
|
+
viewId,
|
|
284
|
+
generation,
|
|
285
|
+
checkpointSuid: priorSuid,
|
|
286
|
+
lateSuid: lateLower.suid,
|
|
287
|
+
eventId: lateLower.eventId,
|
|
288
|
+
classification: "LATE_LOWER_SUID",
|
|
289
|
+
}));
|
|
290
|
+
throw new MaterializedViewStoreError("apply", "MV_ORDERING_QUARANTINED", "Materialized-view safe lane is quarantined for an ordering violation; rebuild and promote a generation");
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
return lateLowerQueryDurationMs;
|
|
294
|
+
}
|
|
295
|
+
async assertSourceBatchOrder(serviceId, viewId, generation, priorSuid, events) {
|
|
296
|
+
let previous = priorSuid;
|
|
297
|
+
for (const event of events) {
|
|
298
|
+
if (compareSuid(previous, event.suid) >= 0) {
|
|
299
|
+
const incident = {
|
|
300
|
+
serviceId,
|
|
301
|
+
identityKey: `ORDER_VIOLATION|${serviceId}|${viewId}|${previous}|${event.suid}|${event.eventId}`,
|
|
302
|
+
classification: "ORDER_VIOLATION",
|
|
303
|
+
suid: event.suid,
|
|
304
|
+
eventId: event.eventId,
|
|
305
|
+
incomingEventId: event.eventId,
|
|
306
|
+
observedAt: event.lastArrivedAt,
|
|
307
|
+
};
|
|
308
|
+
await this.source.appendDeliveryIncident(incident);
|
|
309
|
+
await this.materializedViews.recordOrderingQuarantine({
|
|
310
|
+
serviceId,
|
|
311
|
+
viewId,
|
|
312
|
+
generation,
|
|
313
|
+
checkpointSuid: previous,
|
|
314
|
+
lateSuid: event.suid,
|
|
315
|
+
eventId: event.eventId,
|
|
316
|
+
classification: "ORDER_VIOLATION",
|
|
317
|
+
observedAt: event.lastArrivedAt,
|
|
318
|
+
});
|
|
319
|
+
console.error("SDT-G69_ORDERING_QUARANTINE", JSON.stringify({
|
|
320
|
+
serviceId,
|
|
321
|
+
viewId,
|
|
322
|
+
generation,
|
|
323
|
+
checkpointSuid: previous,
|
|
324
|
+
lateSuid: event.suid,
|
|
325
|
+
eventId: event.eventId,
|
|
326
|
+
classification: "ORDER_VIOLATION",
|
|
327
|
+
}));
|
|
328
|
+
throw new MaterializedViewStoreError("apply", "MV_ORDERING_QUARANTINED", "Materialized-view safe lane is quarantined for an ordering violation; rebuild and promote a generation");
|
|
329
|
+
}
|
|
330
|
+
previous = event.suid;
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
async function sourceHistoryProof(events) {
|
|
335
|
+
const eventIds = events.map((event) => event.eventId);
|
|
336
|
+
const suids = events.map((event) => event.suid);
|
|
337
|
+
const maxSuid = events.reduce((maximum, event) => maximum === "" || compareSuid(event.suid, maximum) > 0 ? event.suid : maximum, "");
|
|
338
|
+
const canonical = events.map((event) => `${event.eventId}\u0000${event.suid}`).join("\n");
|
|
339
|
+
const digestBytes = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(canonical));
|
|
340
|
+
const digest = [...new Uint8Array(digestBytes)].map((value) => value.toString(16).padStart(2, "0")).join("");
|
|
341
|
+
return { eventIds, suids, maxSuid, digest };
|
|
342
|
+
}
|
|
343
|
+
function compareSuid(left, right) {
|
|
344
|
+
const leftBytes = new TextEncoder().encode(left);
|
|
345
|
+
const rightBytes = new TextEncoder().encode(right);
|
|
346
|
+
const shared = Math.min(leftBytes.length, rightBytes.length);
|
|
347
|
+
for (let index = 0; index < shared; index += 1) {
|
|
348
|
+
const difference = leftBytes[index] - rightBytes[index];
|
|
349
|
+
if (difference !== 0)
|
|
350
|
+
return difference;
|
|
351
|
+
}
|
|
352
|
+
return leftBytes.length - rightBytes.length;
|
|
353
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import type { ProjectionCheckpoint, ProjectionStore, StoredEvent } from "../store/types";
|
|
2
|
+
export type MaterializedViewOperation = "build" | "follow" | "rebuild" | "reset" | "promote";
|
|
3
|
+
export type MaterializedViewErrorCode = "MV_DEFINITION_INVALID" | "MV_BUILD_ALREADY_EXISTS" | "MV_BUILD_CONFLICT" | "MV_FOLLOW_CHECKPOINT_UNAVAILABLE" | "MV_CHECKPOINT_CONFLICT" | "MV_REBUILD_ID_INVALID" | "MV_REBUILD_CANDIDATE_EXISTS" | "MV_REBUILD_CONFLICT" | "MV_RESET_MISSING" | "MV_RESET_NOTHING_TO_RESET" | "MV_RESET_CONFLICT" | "MV_PROMOTE_CANDIDATE_MISSING" | "MV_PROMOTE_NOTHING_TO_PROMOTE" | "MV_PROMOTE_CONFLICT" | "MV_STATE_INVALID" | "MV_OPERATION_FAILED" | "MV_SOURCE_NOT_STRICTLY_SUID_ORDERED";
|
|
4
|
+
/** A failed change operation is never represented as a successful no-op. */
|
|
5
|
+
export declare class MaterializedViewOperationError extends Error {
|
|
6
|
+
readonly operation: MaterializedViewOperation;
|
|
7
|
+
readonly code: MaterializedViewErrorCode;
|
|
8
|
+
constructor(operation: MaterializedViewOperation, code: MaterializedViewErrorCode, message: string);
|
|
9
|
+
}
|
|
10
|
+
/** A deploy-time TypeScript definition; materialized views do not extend V1 HTTP wire. */
|
|
11
|
+
export interface MaterializedViewDefinition<State> {
|
|
12
|
+
readonly id: string;
|
|
13
|
+
initialState(): State;
|
|
14
|
+
apply(state: State, event: StoredEvent): State;
|
|
15
|
+
serializeState(state: State): string;
|
|
16
|
+
deserializeState(stateJson: string): State;
|
|
17
|
+
version(state: State): number;
|
|
18
|
+
}
|
|
19
|
+
export interface MaterializedViewSnapshot<State> {
|
|
20
|
+
serviceId: string;
|
|
21
|
+
viewId: string;
|
|
22
|
+
checkpoint: ProjectionCheckpoint;
|
|
23
|
+
state: State;
|
|
24
|
+
}
|
|
25
|
+
export interface MaterializedViewFollowResult<State> extends MaterializedViewSnapshot<State> {
|
|
26
|
+
dynamicLagBoundMs: number;
|
|
27
|
+
safeWindowMs: number;
|
|
28
|
+
advancedSourceEvents: number;
|
|
29
|
+
appliedEvents: number;
|
|
30
|
+
}
|
|
31
|
+
export interface MaterializedViewRebuildResult<State> {
|
|
32
|
+
candidateId: string;
|
|
33
|
+
result: MaterializedViewFollowResult<State>;
|
|
34
|
+
}
|
|
35
|
+
export interface MaterializedViewFollowHooks {
|
|
36
|
+
/** Test-only crash boundary before durable state/checkpoint advancement. */
|
|
37
|
+
beforeCheckpoint?(event: StoredEvent, checkpoint: ProjectionCheckpoint): Promise<void> | void;
|
|
38
|
+
/** Test-only crash boundary after durable state/checkpoint advancement. */
|
|
39
|
+
afterCheckpoint?(event: StoredEvent, checkpoint: ProjectionCheckpoint): Promise<void> | void;
|
|
40
|
+
}
|
|
41
|
+
/** Stable durable namespace for the promoted materialized view. */
|
|
42
|
+
export declare function materializedViewId(definitionId: string): string;
|
|
43
|
+
/** Durable namespace for a rebuild candidate that has not yet been promoted. */
|
|
44
|
+
export declare function materializedViewCandidateId(definitionId: string, rebuildId: string): string;
|
|
45
|
+
/**
|
|
46
|
+
* A materialized view consumes the same read-only global SUID source and
|
|
47
|
+
* SafeWindow as a live projection. Its state and source position share one
|
|
48
|
+
* durable checkpoint transaction, so a restart cannot double-apply a source
|
|
49
|
+
* event that has already changed state.
|
|
50
|
+
*/
|
|
51
|
+
export declare class MaterializedViewRuntime {
|
|
52
|
+
private readonly store;
|
|
53
|
+
constructor(store: ProjectionStore);
|
|
54
|
+
read<State>(serviceId: string, definition: MaterializedViewDefinition<State>): Promise<MaterializedViewSnapshot<State> | undefined>;
|
|
55
|
+
/** Creates the initial durable state then follows through the SafeWindow. */
|
|
56
|
+
build<State>(serviceId: string, definition: MaterializedViewDefinition<State>, nowMs: number, hooks?: MaterializedViewFollowHooks): Promise<MaterializedViewFollowResult<State>>;
|
|
57
|
+
/** Continues the promoted durable view from its own global SUID checkpoint. */
|
|
58
|
+
follow<State>(serviceId: string, definition: MaterializedViewDefinition<State>, nowMs: number, hooks?: MaterializedViewFollowHooks): Promise<MaterializedViewFollowResult<State>>;
|
|
59
|
+
/**
|
|
60
|
+
* Builds a separate durable candidate from the beginning of the global
|
|
61
|
+
* source. The caller must explicitly promote it; no rebuild silently swaps
|
|
62
|
+
* production state.
|
|
63
|
+
*/
|
|
64
|
+
rebuild<State>(serviceId: string, definition: MaterializedViewDefinition<State>, rebuildId: string, nowMs: number, hooks?: MaterializedViewFollowHooks): Promise<MaterializedViewRebuildResult<State>>;
|
|
65
|
+
/** Resets the promoted view to a durable initial state, never a no-op. */
|
|
66
|
+
reset<State>(serviceId: string, definition: MaterializedViewDefinition<State>, nowMs: number): Promise<MaterializedViewSnapshot<State>>;
|
|
67
|
+
/** Promotes a previously rebuilt durable candidate with a compare-and-swap. */
|
|
68
|
+
promote<State>(serviceId: string, definition: MaterializedViewDefinition<State>, rebuildId: string, nowMs: number): Promise<MaterializedViewSnapshot<State>>;
|
|
69
|
+
private ensureInitialCheckpoint;
|
|
70
|
+
private followCheckpoint;
|
|
71
|
+
private checkedViewId;
|
|
72
|
+
private checkedDefinitionId;
|
|
73
|
+
private stateFrom;
|
|
74
|
+
private initialState;
|
|
75
|
+
private apply;
|
|
76
|
+
private checkpointFor;
|
|
77
|
+
private snapshot;
|
|
78
|
+
private failure;
|
|
79
|
+
private typedFailure;
|
|
80
|
+
}
|