@voidhash/mimic-effect 1.0.0-beta.7 → 1.0.0-beta.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +94 -94
- package/dist/DocumentInstance.cjs +257 -0
- package/dist/DocumentInstance.d.cts +74 -0
- package/dist/DocumentInstance.d.cts.map +1 -0
- package/dist/DocumentInstance.d.mts +74 -0
- package/dist/DocumentInstance.d.mts.map +1 -0
- package/dist/DocumentInstance.mjs +258 -0
- package/dist/DocumentInstance.mjs.map +1 -0
- package/dist/MimicClusterServerEngine.cjs +19 -203
- package/dist/MimicClusterServerEngine.d.cts.map +1 -1
- package/dist/MimicClusterServerEngine.d.mts.map +1 -1
- package/dist/MimicClusterServerEngine.mjs +24 -208
- package/dist/MimicClusterServerEngine.mjs.map +1 -1
- package/dist/MimicServerEngine.cjs +72 -10
- package/dist/MimicServerEngine.d.cts +12 -7
- package/dist/MimicServerEngine.d.cts.map +1 -1
- package/dist/MimicServerEngine.d.mts +12 -7
- package/dist/MimicServerEngine.d.mts.map +1 -1
- package/dist/MimicServerEngine.mjs +74 -12
- package/dist/MimicServerEngine.mjs.map +1 -1
- package/dist/Protocol.d.cts +1 -1
- package/dist/Protocol.d.mts +1 -1
- package/dist/index.cjs +2 -4
- package/dist/index.d.cts +2 -2
- package/dist/index.d.mts +2 -2
- package/dist/index.mjs +2 -2
- package/package.json +3 -3
- package/src/DocumentInstance.ts +516 -0
- package/src/MimicClusterServerEngine.ts +40 -357
- package/src/MimicServerEngine.ts +172 -36
- package/src/index.ts +3 -4
- package/tests/DocumentInstance.test.ts +669 -0
- package/dist/DocumentManager.cjs +0 -299
- package/dist/DocumentManager.d.cts +0 -67
- package/dist/DocumentManager.d.cts.map +0 -1
- package/dist/DocumentManager.d.mts +0 -67
- package/dist/DocumentManager.d.mts.map +0 -1
- package/dist/DocumentManager.mjs +0 -297
- package/dist/DocumentManager.mjs.map +0 -1
- package/src/DocumentManager.ts +0 -616
- package/tests/DocumentManager.test.ts +0 -335
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { ColdStorageError, HotStorageError } from "./Errors.mjs";
|
|
2
|
+
import { ServerBroadcast } from "./Protocol.mjs";
|
|
3
|
+
import { ColdStorage } from "./ColdStorage.mjs";
|
|
4
|
+
import { HotStorage } from "./HotStorage.mjs";
|
|
5
|
+
import { Duration, Effect, PubSub } from "effect";
|
|
6
|
+
import { Primitive, Transaction } from "@voidhash/mimic";
|
|
7
|
+
import { ServerDocument } from "@voidhash/mimic/server";
|
|
8
|
+
|
|
9
|
+
//#region src/DocumentInstance.d.ts
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Result of submitting a transaction
|
|
13
|
+
*/
|
|
14
|
+
type SubmitResult = {
|
|
15
|
+
readonly success: true;
|
|
16
|
+
readonly version: number;
|
|
17
|
+
} | {
|
|
18
|
+
readonly success: false;
|
|
19
|
+
readonly reason: string;
|
|
20
|
+
};
|
|
21
|
+
/**
|
|
22
|
+
* Configuration for a DocumentInstance
|
|
23
|
+
*/
|
|
24
|
+
interface DocumentInstanceConfig<TSchema extends Primitive.AnyPrimitive> {
|
|
25
|
+
readonly schema: TSchema;
|
|
26
|
+
readonly initial?: Primitive.InferSetInput<TSchema> | ((ctx: {
|
|
27
|
+
documentId: string;
|
|
28
|
+
}) => Effect.Effect<Primitive.InferSetInput<TSchema>>);
|
|
29
|
+
readonly maxTransactionHistory: number;
|
|
30
|
+
readonly snapshot: {
|
|
31
|
+
readonly interval: Duration.Duration;
|
|
32
|
+
readonly transactionThreshold: number;
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Snapshot tracking state
|
|
37
|
+
*/
|
|
38
|
+
interface SnapshotTracking {
|
|
39
|
+
readonly lastSnapshotVersion: number;
|
|
40
|
+
readonly lastSnapshotTime: number;
|
|
41
|
+
readonly transactionsSinceSnapshot: number;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* A DocumentInstance manages a single document's lifecycle
|
|
45
|
+
*/
|
|
46
|
+
interface DocumentInstance<TSchema extends Primitive.AnyPrimitive> {
|
|
47
|
+
/** The underlying ServerDocument */
|
|
48
|
+
readonly document: ServerDocument.ServerDocument<TSchema>;
|
|
49
|
+
/** PubSub for broadcasting messages to subscribers */
|
|
50
|
+
readonly pubsub: PubSub.PubSub<ServerBroadcast>;
|
|
51
|
+
/** Current snapshot tracking state */
|
|
52
|
+
readonly getSnapshotTracking: Effect.Effect<SnapshotTracking>;
|
|
53
|
+
/** Submit a transaction */
|
|
54
|
+
readonly submit: (transaction: Transaction.Transaction) => Effect.Effect<SubmitResult, ColdStorageError | HotStorageError>;
|
|
55
|
+
/** Save a snapshot to cold storage */
|
|
56
|
+
readonly saveSnapshot: () => Effect.Effect<void, ColdStorageError | HotStorageError>;
|
|
57
|
+
/** Check if snapshot should be triggered and save if needed */
|
|
58
|
+
readonly checkSnapshotTriggers: () => Effect.Effect<void, ColdStorageError | HotStorageError>;
|
|
59
|
+
/** Update last activity time (for external tracking) */
|
|
60
|
+
readonly touch: () => Effect.Effect<void>;
|
|
61
|
+
/** Get current document version */
|
|
62
|
+
readonly getVersion: () => number;
|
|
63
|
+
/** Get document snapshot */
|
|
64
|
+
readonly getSnapshot: () => {
|
|
65
|
+
state: unknown;
|
|
66
|
+
version: number;
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
declare const DocumentInstance: {
|
|
70
|
+
make: <TSchema extends Primitive.AnyPrimitive>(documentId: string, config: DocumentInstanceConfig<TSchema>, coldStorage: ColdStorage, hotStorage: HotStorage) => Effect.Effect<DocumentInstance<TSchema>, ColdStorageError | HotStorageError>;
|
|
71
|
+
};
|
|
72
|
+
//#endregion
|
|
73
|
+
export { DocumentInstance, SubmitResult };
|
|
74
|
+
//# sourceMappingURL=DocumentInstance.d.mts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"DocumentInstance.d.mts","names":[],"sources":["../src/DocumentInstance.ts"],"sourcesContent":[],"mappings":";;;;;;;;;;;;AAkDA;AASiB,KA/BL,YAAA,GA+BqB;EAAiB,SAAU,OAAA,EAAA,IAAA;EAET,SAAA,OAAA,EAAA,MAAA;CAA9B,GAAA;EAEY,SAAA,OAAA,EAAA,KAAA;EAAd,SAAO,MAAA,EAAA,MAAA;CAEoB;;;;AAE2C,UAhCxE,sBAgCwE,CAAA,gBAhCjC,SAAA,CAAU,YAgCuB,CAAA,CAAA;EAAmB,SAAA,MAAA,EA/BzF,OA+ByF;EAA/C,SAAO,OAAA,CAAA,EA7B9D,SAAA,CAAU,aA6BoD,CA7BtC,OA6BsC,CAAA,GAAA,CAAA,CAAA,GAAA,EAAA;IAEjB,UAAA,EAAA,MAAA;EAAmB,CAAA,EAAA,GA9B9B,MAAA,CAAO,MA8BuB,CA9BhB,SAAA,CAAU,aA8BM,CA9BQ,OA8BR,CAAA,CAAA,CAAA;EAAvC,SAAO,qBAAA,EAAA,MAAA;EAEsB,SAAA,QAAA,EAAA;IAAmB,SAAA,QAAA,EA7BxD,QAAA,CAAS,QA6B+C;IAAhC,SAAA,oBAAA,EAAA,MAAA;EAEvB,CAAA;;AAwbxB;;;AAjaU,UA9CO,gBAAA,CA8CP;EACK,SAAA,mBAAA,EAAA,MAAA;EACD,SAAA,gBAAA,EAAA,MAAA;EACoB,SAAA,yBAAA,EAAA,MAAA;;;;;AAAlB,UAxCC,gBAwCD,CAAA,gBAxCkC,SAAA,CAAU,YAwC5C,CAAA,CAAA;;qBAtCK,cAAA,CAAe,eAAe;;mBAEhC,MAAA,CAAO,OAAO;;gCAED,MAAA,CAAO,OAAO;;iCAEb,WAAA,CAAY,gBAAgB,MAAA,CAAO,OAAO,cAAc,mBAAmB;;+BAE7E,MAAA,CAAO,aAAa,mBAAmB;;wCAE9B,MAAA,CAAO,aAAa,mBAAmB;;wBAEvD,MAAA,CAAO;;;;;;;;;cAwblB;yBAnawB,SAAA,CAAU,0CAErC,uBAAuB,uBAClB,yBACD,eACX,MAAA,CAAO,OAAO,iBAAiB,UAAU,mBAAmB"}
|
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
import { documentsActive, documentsCreated, documentsRestored, storageSnapshotLatency, storageSnapshots, storageVersionGaps, storageWalAppends, transactionsLatency, transactionsProcessed, transactionsRejected, walAppendFailures } from "./Metrics.mjs";
|
|
2
|
+
import { Duration, Effect, Metric, PubSub, Ref } from "effect";
|
|
3
|
+
import { Document } from "@voidhash/mimic";
|
|
4
|
+
import { ServerDocument } from "@voidhash/mimic/server";
|
|
5
|
+
|
|
6
|
+
//#region src/DocumentInstance.ts
|
|
7
|
+
/**
|
|
8
|
+
* @voidhash/mimic-effect - DocumentInstance
|
|
9
|
+
*
|
|
10
|
+
* Manages the lifecycle of a single document including:
|
|
11
|
+
* - Restoration from storage (cold storage + WAL replay)
|
|
12
|
+
* - Transaction submission with WAL persistence
|
|
13
|
+
* - Snapshot saving and trigger checking
|
|
14
|
+
*
|
|
15
|
+
* Used by both MimicServerEngine (single-node) and MimicClusterServerEngine (clustered).
|
|
16
|
+
*/
|
|
17
|
+
/**
|
|
18
|
+
* Create a DocumentInstance for a single document.
|
|
19
|
+
*
|
|
20
|
+
* This handles:
|
|
21
|
+
* - Loading from cold storage or computing initial state
|
|
22
|
+
* - Persisting initial state immediately (crash safety)
|
|
23
|
+
* - Replaying WAL entries
|
|
24
|
+
* - Transaction submission with WAL persistence
|
|
25
|
+
* - Snapshot saving
|
|
26
|
+
*/
|
|
27
|
+
const make = (documentId, config, coldStorage, hotStorage) => Effect.gen(function* () {
|
|
28
|
+
const SCHEMA_VERSION = 1;
|
|
29
|
+
const storedDoc = yield* coldStorage.load(documentId);
|
|
30
|
+
let initialState;
|
|
31
|
+
let initial;
|
|
32
|
+
let initialVersion = 0;
|
|
33
|
+
if (storedDoc) {
|
|
34
|
+
initialState = storedDoc.state;
|
|
35
|
+
initialVersion = storedDoc.version;
|
|
36
|
+
} else initial = yield* computeInitialState(config, documentId);
|
|
37
|
+
const pubsub = yield* PubSub.unbounded();
|
|
38
|
+
const lastSnapshotVersionRef = yield* Ref.make(initialVersion);
|
|
39
|
+
const lastSnapshotTimeRef = yield* Ref.make(Date.now());
|
|
40
|
+
const transactionsSinceSnapshotRef = yield* Ref.make(0);
|
|
41
|
+
const lastActivityTimeRef = yield* Ref.make(Date.now());
|
|
42
|
+
const document = ServerDocument.make({
|
|
43
|
+
schema: config.schema,
|
|
44
|
+
initial,
|
|
45
|
+
initialState,
|
|
46
|
+
initialVersion,
|
|
47
|
+
maxTransactionHistory: config.maxTransactionHistory,
|
|
48
|
+
onBroadcast: (message) => {
|
|
49
|
+
Effect.runSync(PubSub.publish(pubsub, {
|
|
50
|
+
type: "transaction",
|
|
51
|
+
transaction: message.transaction,
|
|
52
|
+
version: message.version
|
|
53
|
+
}));
|
|
54
|
+
},
|
|
55
|
+
onRejection: (transactionId, reason) => {
|
|
56
|
+
Effect.runSync(PubSub.publish(pubsub, {
|
|
57
|
+
type: "error",
|
|
58
|
+
transactionId,
|
|
59
|
+
reason
|
|
60
|
+
}));
|
|
61
|
+
}
|
|
62
|
+
});
|
|
63
|
+
if (!storedDoc) {
|
|
64
|
+
const initialStoredDoc = createStoredDocument(document.get(), 0, SCHEMA_VERSION);
|
|
65
|
+
yield* coldStorage.save(documentId, initialStoredDoc);
|
|
66
|
+
yield* Effect.logDebug("Initial state persisted to cold storage", { documentId });
|
|
67
|
+
}
|
|
68
|
+
const walEntries = yield* hotStorage.getEntries(documentId, initialVersion);
|
|
69
|
+
yield* verifyWalContinuity(documentId, walEntries, initialVersion);
|
|
70
|
+
yield* replayWalEntries(documentId, document, walEntries);
|
|
71
|
+
if (storedDoc) yield* Metric.increment(documentsRestored);
|
|
72
|
+
else yield* Metric.increment(documentsCreated);
|
|
73
|
+
yield* Metric.incrementBy(documentsActive, 1);
|
|
74
|
+
const getSnapshotTracking = Effect.gen(function* () {
|
|
75
|
+
return {
|
|
76
|
+
lastSnapshotVersion: yield* Ref.get(lastSnapshotVersionRef),
|
|
77
|
+
lastSnapshotTime: yield* Ref.get(lastSnapshotTimeRef),
|
|
78
|
+
transactionsSinceSnapshot: yield* Ref.get(transactionsSinceSnapshotRef)
|
|
79
|
+
};
|
|
80
|
+
});
|
|
81
|
+
const saveSnapshot = Effect.fn("document.snapshot.save")(function* () {
|
|
82
|
+
var _baseSnapshot$version;
|
|
83
|
+
const targetVersion = document.getVersion();
|
|
84
|
+
if (targetVersion <= (yield* Ref.get(lastSnapshotVersionRef))) return;
|
|
85
|
+
const snapshotStartTime = Date.now();
|
|
86
|
+
const baseSnapshot = yield* coldStorage.load(documentId);
|
|
87
|
+
const baseVersion = (_baseSnapshot$version = baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.version) !== null && _baseSnapshot$version !== void 0 ? _baseSnapshot$version : 0;
|
|
88
|
+
const baseState = baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.state;
|
|
89
|
+
const walEntries$1 = yield* hotStorage.getEntries(documentId, baseVersion);
|
|
90
|
+
const snapshotResult = computeSnapshotState(config.schema, baseState, walEntries$1, targetVersion);
|
|
91
|
+
if (!snapshotResult) return;
|
|
92
|
+
const currentLastSnapshot = yield* Ref.get(lastSnapshotVersionRef);
|
|
93
|
+
if (snapshotResult.version <= currentLastSnapshot) return;
|
|
94
|
+
const storedDoc$1 = createStoredDocument(snapshotResult.state, snapshotResult.version, SCHEMA_VERSION);
|
|
95
|
+
yield* coldStorage.save(documentId, storedDoc$1);
|
|
96
|
+
const snapshotDuration = Date.now() - snapshotStartTime;
|
|
97
|
+
yield* Metric.increment(storageSnapshots);
|
|
98
|
+
yield* Metric.update(storageSnapshotLatency, snapshotDuration);
|
|
99
|
+
yield* Ref.set(lastSnapshotVersionRef, snapshotResult.version);
|
|
100
|
+
yield* Ref.set(lastSnapshotTimeRef, Date.now());
|
|
101
|
+
yield* Ref.set(transactionsSinceSnapshotRef, 0);
|
|
102
|
+
yield* Effect.catchAll(hotStorage.truncate(documentId, snapshotResult.version), (e) => Effect.logWarning("WAL truncate failed - will retry on next snapshot", {
|
|
103
|
+
documentId,
|
|
104
|
+
version: snapshotResult.version,
|
|
105
|
+
error: e
|
|
106
|
+
}));
|
|
107
|
+
});
|
|
108
|
+
const checkSnapshotTriggers = Effect.fn("document.snapshot.check-triggers")(function* () {
|
|
109
|
+
if (shouldTriggerSnapshot(yield* Ref.get(transactionsSinceSnapshotRef), yield* Ref.get(lastSnapshotTimeRef), config.snapshot)) yield* saveSnapshot();
|
|
110
|
+
});
|
|
111
|
+
return {
|
|
112
|
+
document,
|
|
113
|
+
pubsub,
|
|
114
|
+
getSnapshotTracking,
|
|
115
|
+
submit: Effect.fn("document.transaction.submit")(function* (transaction) {
|
|
116
|
+
const submitStartTime = Date.now();
|
|
117
|
+
yield* Ref.set(lastActivityTimeRef, Date.now());
|
|
118
|
+
const validation = document.validate(transaction);
|
|
119
|
+
if (!validation.valid) {
|
|
120
|
+
yield* Metric.increment(transactionsRejected);
|
|
121
|
+
const latency$1 = Date.now() - submitStartTime;
|
|
122
|
+
yield* Metric.update(transactionsLatency, latency$1);
|
|
123
|
+
return {
|
|
124
|
+
success: false,
|
|
125
|
+
reason: validation.reason
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
const walEntry = {
|
|
129
|
+
transaction,
|
|
130
|
+
version: validation.nextVersion,
|
|
131
|
+
timestamp: Date.now()
|
|
132
|
+
};
|
|
133
|
+
const appendResult = yield* Effect.either(hotStorage.appendWithCheck(documentId, walEntry, validation.nextVersion));
|
|
134
|
+
if (appendResult._tag === "Left") {
|
|
135
|
+
yield* Effect.logError("WAL append failed", {
|
|
136
|
+
documentId,
|
|
137
|
+
version: validation.nextVersion,
|
|
138
|
+
error: appendResult.left
|
|
139
|
+
});
|
|
140
|
+
yield* Metric.increment(walAppendFailures);
|
|
141
|
+
const latency$1 = Date.now() - submitStartTime;
|
|
142
|
+
yield* Metric.update(transactionsLatency, latency$1);
|
|
143
|
+
return {
|
|
144
|
+
success: false,
|
|
145
|
+
reason: "Storage unavailable. Please retry."
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
document.apply(transaction);
|
|
149
|
+
const latency = Date.now() - submitStartTime;
|
|
150
|
+
yield* Metric.update(transactionsLatency, latency);
|
|
151
|
+
yield* Metric.increment(transactionsProcessed);
|
|
152
|
+
yield* Metric.increment(storageWalAppends);
|
|
153
|
+
yield* Ref.update(transactionsSinceSnapshotRef, (n) => n + 1);
|
|
154
|
+
yield* checkSnapshotTriggers();
|
|
155
|
+
return {
|
|
156
|
+
success: true,
|
|
157
|
+
version: validation.nextVersion
|
|
158
|
+
};
|
|
159
|
+
}),
|
|
160
|
+
saveSnapshot,
|
|
161
|
+
checkSnapshotTriggers,
|
|
162
|
+
touch: Effect.fn("document.touch")(function* () {
|
|
163
|
+
yield* Ref.set(lastActivityTimeRef, Date.now());
|
|
164
|
+
}),
|
|
165
|
+
getVersion: () => document.getVersion(),
|
|
166
|
+
getSnapshot: () => document.getSnapshot()
|
|
167
|
+
};
|
|
168
|
+
});
|
|
169
|
+
/**
|
|
170
|
+
* Compute initial state for a new document.
|
|
171
|
+
*/
|
|
172
|
+
const computeInitialState = (config, documentId) => {
|
|
173
|
+
if (config.initial === void 0) return Effect.succeed(void 0);
|
|
174
|
+
if (typeof config.initial === "function") return config.initial({ documentId });
|
|
175
|
+
return Effect.succeed(config.initial);
|
|
176
|
+
};
|
|
177
|
+
/**
|
|
178
|
+
* Verify WAL continuity and log warnings for any gaps.
|
|
179
|
+
*/
|
|
180
|
+
const verifyWalContinuity = Effect.fn("document.wal.verify")(function* (documentId, walEntries, baseVersion) {
|
|
181
|
+
if (walEntries.length === 0) return;
|
|
182
|
+
const firstWalVersion = walEntries[0].version;
|
|
183
|
+
const expectedFirst = baseVersion + 1;
|
|
184
|
+
if (firstWalVersion !== expectedFirst) {
|
|
185
|
+
yield* Effect.logWarning("WAL version gap detected", {
|
|
186
|
+
documentId,
|
|
187
|
+
snapshotVersion: baseVersion,
|
|
188
|
+
firstWalVersion,
|
|
189
|
+
expectedFirst
|
|
190
|
+
});
|
|
191
|
+
yield* Metric.increment(storageVersionGaps);
|
|
192
|
+
}
|
|
193
|
+
for (let i = 1; i < walEntries.length; i++) {
|
|
194
|
+
const prev = walEntries[i - 1].version;
|
|
195
|
+
const curr = walEntries[i].version;
|
|
196
|
+
if (curr !== prev + 1) yield* Effect.logWarning("WAL internal gap detected", {
|
|
197
|
+
documentId,
|
|
198
|
+
previousVersion: prev,
|
|
199
|
+
currentVersion: curr
|
|
200
|
+
});
|
|
201
|
+
}
|
|
202
|
+
});
|
|
203
|
+
/**
|
|
204
|
+
* Replay WAL entries onto a ServerDocument.
|
|
205
|
+
*/
|
|
206
|
+
const replayWalEntries = Effect.fn("document.wal.replay")(function* (documentId, document, walEntries) {
|
|
207
|
+
for (const entry of walEntries) {
|
|
208
|
+
const result = document.submit(entry.transaction);
|
|
209
|
+
if (!result.success) yield* Effect.logWarning("Skipping corrupted WAL entry", {
|
|
210
|
+
documentId,
|
|
211
|
+
version: entry.version,
|
|
212
|
+
reason: result.reason
|
|
213
|
+
});
|
|
214
|
+
}
|
|
215
|
+
});
|
|
216
|
+
/**
|
|
217
|
+
* Compute snapshot state by replaying WAL entries on a base state.
|
|
218
|
+
*/
|
|
219
|
+
const computeSnapshotState = (schema, baseState, walEntries, targetVersion) => {
|
|
220
|
+
const relevantEntries = walEntries.filter((e) => e.version <= targetVersion);
|
|
221
|
+
if (relevantEntries.length === 0 && baseState === void 0) return;
|
|
222
|
+
let snapshotState = baseState;
|
|
223
|
+
for (const entry of relevantEntries) {
|
|
224
|
+
const tempDoc = Document.make(schema, { initialState: snapshotState });
|
|
225
|
+
tempDoc.apply(entry.transaction.ops);
|
|
226
|
+
snapshotState = tempDoc.get();
|
|
227
|
+
}
|
|
228
|
+
if (snapshotState === void 0) return;
|
|
229
|
+
const snapshotVersion = relevantEntries.length > 0 ? relevantEntries[relevantEntries.length - 1].version : 0;
|
|
230
|
+
return {
|
|
231
|
+
state: snapshotState,
|
|
232
|
+
version: snapshotVersion
|
|
233
|
+
};
|
|
234
|
+
};
|
|
235
|
+
/**
|
|
236
|
+
* Check if a snapshot should be triggered.
|
|
237
|
+
*/
|
|
238
|
+
const shouldTriggerSnapshot = (transactionsSinceSnapshot, lastSnapshotTime, config) => {
|
|
239
|
+
const now = Date.now();
|
|
240
|
+
const intervalMs = Duration.toMillis(config.interval);
|
|
241
|
+
if (transactionsSinceSnapshot >= config.transactionThreshold) return true;
|
|
242
|
+
if (now - lastSnapshotTime >= intervalMs) return true;
|
|
243
|
+
return false;
|
|
244
|
+
};
|
|
245
|
+
/**
|
|
246
|
+
* Create a StoredDocument for persistence.
|
|
247
|
+
*/
|
|
248
|
+
const createStoredDocument = (state, version, schemaVersion) => ({
|
|
249
|
+
state,
|
|
250
|
+
version,
|
|
251
|
+
schemaVersion,
|
|
252
|
+
savedAt: Date.now()
|
|
253
|
+
});
|
|
254
|
+
const DocumentInstance = { make };
|
|
255
|
+
|
|
256
|
+
//#endregion
|
|
257
|
+
export { DocumentInstance };
|
|
258
|
+
//# sourceMappingURL=DocumentInstance.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"DocumentInstance.mjs","names":["initialState: Primitive.InferState<TSchema> | undefined","initial: Primitive.InferSetInput<TSchema> | undefined","Metrics.documentsRestored","Metrics.documentsCreated","Metrics.documentsActive","walEntries","storedDoc","Metrics.storageSnapshots","Metrics.storageSnapshotLatency","Metrics.transactionsRejected","latency","Metrics.transactionsLatency","walEntry: WalEntry","Metrics.walAppendFailures","Metrics.transactionsProcessed","Metrics.storageWalAppends","Metrics.storageVersionGaps","snapshotState: Primitive.InferState<TSchema> | undefined"],"sources":["../src/DocumentInstance.ts"],"sourcesContent":["/**\n * @voidhash/mimic-effect - DocumentInstance\n *\n * Manages the lifecycle of a single document including:\n * - Restoration from storage (cold storage + WAL replay)\n * - Transaction submission with WAL persistence\n * - Snapshot saving and trigger checking\n *\n * Used by both MimicServerEngine (single-node) and MimicClusterServerEngine (clustered).\n */\nimport { Duration, Effect, Metric, PubSub, Ref } from \"effect\";\nimport { Document, type Primitive, type Transaction } from \"@voidhash/mimic\";\nimport { ServerDocument } from \"@voidhash/mimic/server\";\nimport type { StoredDocument, WalEntry } from \"./Types\";\nimport type { ServerBroadcast } from \"./Protocol\";\nimport type { ColdStorage } from \"./ColdStorage\";\nimport type { HotStorage } from \"./HotStorage\";\nimport type { ColdStorageError } from \"./Errors\";\nimport type { HotStorageError } from \"./Errors\";\nimport * as Metrics from \"./Metrics\";\n\n// =============================================================================\n// Types\n// =============================================================================\n\n/**\n * Result of submitting a transaction\n */\nexport type SubmitResult =\n | { readonly success: true; readonly version: number }\n | { readonly success: false; readonly reason: string };\n\n/**\n * Configuration for a DocumentInstance\n */\nexport interface DocumentInstanceConfig<TSchema extends Primitive.AnyPrimitive> {\n readonly schema: TSchema;\n readonly initial?:\n | Primitive.InferSetInput<TSchema>\n | ((ctx: { documentId: string }) => Effect.Effect<Primitive.InferSetInput<TSchema>>);\n readonly maxTransactionHistory: number;\n readonly snapshot: {\n readonly interval: Duration.Duration;\n readonly transactionThreshold: number;\n };\n}\n\n/**\n * Snapshot tracking state\n */\nexport interface SnapshotTracking {\n readonly lastSnapshotVersion: number;\n readonly lastSnapshotTime: number;\n readonly transactionsSinceSnapshot: number;\n}\n\n/**\n * A DocumentInstance manages a single document's lifecycle\n */\nexport interface DocumentInstance<TSchema extends Primitive.AnyPrimitive> {\n /** The underlying ServerDocument */\n readonly document: ServerDocument.ServerDocument<TSchema>;\n /** PubSub for broadcasting messages to subscribers */\n readonly pubsub: PubSub.PubSub<ServerBroadcast>;\n /** Current snapshot tracking state */\n readonly getSnapshotTracking: Effect.Effect<SnapshotTracking>;\n /** Submit a transaction */\n readonly submit: (transaction: Transaction.Transaction) => Effect.Effect<SubmitResult, ColdStorageError | HotStorageError>;\n /** Save a snapshot to cold storage */\n readonly saveSnapshot: () => Effect.Effect<void, ColdStorageError | HotStorageError>;\n /** Check if snapshot should be triggered and save if needed */\n readonly checkSnapshotTriggers: () => Effect.Effect<void, ColdStorageError | HotStorageError>;\n /** Update last activity time (for external tracking) */\n readonly touch: () => Effect.Effect<void>;\n /** Get current document version */\n readonly getVersion: () => number;\n /** Get document snapshot */\n readonly getSnapshot: () => { state: unknown; version: number };\n}\n\n// =============================================================================\n// Factory\n// =============================================================================\n\n/**\n * Create a DocumentInstance for a single document.\n *\n * This handles:\n * - Loading from cold storage or computing initial state\n * - Persisting initial state immediately (crash safety)\n * - Replaying WAL entries\n * - Transaction submission with WAL persistence\n * - Snapshot saving\n */\nexport const make = <TSchema extends Primitive.AnyPrimitive>(\n documentId: string,\n config: DocumentInstanceConfig<TSchema>,\n coldStorage: ColdStorage,\n hotStorage: HotStorage\n): Effect.Effect<DocumentInstance<TSchema>, ColdStorageError | HotStorageError> =>\n Effect.gen(function* () {\n // Current schema version (hard-coded to 1 for now)\n const SCHEMA_VERSION = 1;\n\n // 1. Load snapshot from ColdStorage\n const storedDoc = yield* coldStorage.load(documentId);\n\n // Track initial values - only one will be set:\n // - initialState: raw state from storage (already in internal format)\n // - initial: computed from config (needs conversion to state format)\n let initialState: Primitive.InferState<TSchema> | undefined;\n let initial: Primitive.InferSetInput<TSchema> | undefined;\n let initialVersion = 0;\n\n if (storedDoc) {\n // Loading from storage - state is already in internal format\n initialState = storedDoc.state as Primitive.InferState<TSchema>;\n initialVersion = storedDoc.version;\n } else {\n // New document - compute initial value (set input format)\n initial = yield* computeInitialState(config, documentId);\n }\n\n // 2. Create PubSub for broadcasting\n const pubsub = yield* PubSub.unbounded<ServerBroadcast>();\n\n // 3. Create refs for tracking\n const lastSnapshotVersionRef = yield* Ref.make(initialVersion);\n const lastSnapshotTimeRef = yield* Ref.make(Date.now());\n const transactionsSinceSnapshotRef = yield* Ref.make(0);\n const lastActivityTimeRef = yield* Ref.make(Date.now());\n\n // 4. Create ServerDocument with callbacks\n const document = ServerDocument.make({\n schema: config.schema,\n initial,\n initialState,\n initialVersion,\n maxTransactionHistory: config.maxTransactionHistory,\n onBroadcast: (message: ServerDocument.TransactionMessage) => {\n Effect.runSync(\n PubSub.publish(pubsub, {\n type: \"transaction\",\n transaction: message.transaction,\n version: message.version,\n })\n );\n },\n onRejection: (transactionId: string, reason: string) => {\n Effect.runSync(\n PubSub.publish(pubsub, {\n type: \"error\",\n transactionId,\n reason,\n })\n );\n },\n });\n\n // 5. If this is a new document, immediately save to cold storage\n // This ensures the initial state is durable before any transactions are accepted.\n if (!storedDoc) {\n const initialStoredDoc = createStoredDocument(document.get(), 0, SCHEMA_VERSION);\n yield* coldStorage.save(documentId, initialStoredDoc);\n yield* Effect.logDebug(\"Initial state persisted to cold storage\", { documentId });\n }\n\n // 6. Load WAL entries\n const walEntries = yield* hotStorage.getEntries(documentId, initialVersion);\n\n // 7. Verify WAL continuity (warning only, non-blocking)\n yield* verifyWalContinuity(documentId, walEntries, initialVersion);\n\n // 8. Replay WAL entries\n yield* replayWalEntries(documentId, document, walEntries);\n\n // Track metrics\n if (storedDoc) {\n yield* Metric.increment(Metrics.documentsRestored);\n } else {\n yield* Metric.increment(Metrics.documentsCreated);\n }\n yield* Metric.incrementBy(Metrics.documentsActive, 1);\n\n // ==========================================================================\n // Instance Methods\n // ==========================================================================\n\n const getSnapshotTracking = Effect.gen(function* () {\n return {\n lastSnapshotVersion: yield* Ref.get(lastSnapshotVersionRef),\n lastSnapshotTime: yield* Ref.get(lastSnapshotTimeRef),\n transactionsSinceSnapshot: yield* Ref.get(transactionsSinceSnapshotRef),\n };\n });\n\n const saveSnapshot = Effect.fn(\"document.snapshot.save\")(function* () {\n const targetVersion = document.getVersion();\n const lastSnapshotVersion = yield* Ref.get(lastSnapshotVersionRef);\n\n // Idempotency check: skip if already snapshotted at this version\n if (targetVersion <= lastSnapshotVersion) {\n return;\n }\n\n const snapshotStartTime = Date.now();\n\n // Load base snapshot from cold storage\n const baseSnapshot = yield* coldStorage.load(documentId);\n const baseVersion = baseSnapshot?.version ?? 0;\n const baseState = baseSnapshot?.state as Primitive.InferState<TSchema> | undefined;\n\n // Load WAL entries from base to target\n const walEntries = yield* hotStorage.getEntries(documentId, baseVersion);\n\n // Compute snapshot state by replaying WAL on base\n const snapshotResult = computeSnapshotState(\n config.schema,\n baseState,\n walEntries,\n targetVersion\n );\n\n if (!snapshotResult) {\n return;\n }\n\n // Re-check before saving (in case another snapshot completed while we were working)\n const currentLastSnapshot = yield* Ref.get(lastSnapshotVersionRef);\n if (snapshotResult.version <= currentLastSnapshot) {\n return;\n }\n\n const storedDoc = createStoredDocument(\n snapshotResult.state,\n snapshotResult.version,\n SCHEMA_VERSION\n );\n\n // Save to ColdStorage\n yield* coldStorage.save(documentId, storedDoc);\n\n // Track snapshot metrics\n const snapshotDuration = Date.now() - snapshotStartTime;\n yield* Metric.increment(Metrics.storageSnapshots);\n yield* Metric.update(Metrics.storageSnapshotLatency, snapshotDuration);\n\n // Update tracking BEFORE truncate (for idempotency on retry)\n yield* Ref.set(lastSnapshotVersionRef, snapshotResult.version);\n yield* Ref.set(lastSnapshotTimeRef, Date.now());\n yield* Ref.set(transactionsSinceSnapshotRef, 0);\n\n // Truncate WAL - non-fatal, will be retried on next snapshot\n yield* Effect.catchAll(hotStorage.truncate(documentId, snapshotResult.version), (e) =>\n Effect.logWarning(\"WAL truncate failed - will retry on next snapshot\", {\n documentId,\n version: snapshotResult.version,\n error: e,\n })\n );\n });\n\n const checkSnapshotTriggers = Effect.fn(\"document.snapshot.check-triggers\")(function* () {\n const txCount = yield* Ref.get(transactionsSinceSnapshotRef);\n const lastTime = yield* Ref.get(lastSnapshotTimeRef);\n\n if (shouldTriggerSnapshot(txCount, lastTime, config.snapshot)) {\n yield* saveSnapshot();\n }\n });\n\n const submit = Effect.fn(\"document.transaction.submit\")(function* (\n transaction: Transaction.Transaction\n ) {\n const submitStartTime = Date.now();\n\n // Update activity time\n yield* Ref.set(lastActivityTimeRef, Date.now());\n\n // Phase 1: Validate (no side effects)\n const validation = document.validate(transaction);\n\n if (!validation.valid) {\n yield* Metric.increment(Metrics.transactionsRejected);\n const latency = Date.now() - submitStartTime;\n yield* Metric.update(Metrics.transactionsLatency, latency);\n\n return {\n success: false as const,\n reason: validation.reason,\n };\n }\n\n // Phase 2: Append to WAL with gap check (BEFORE state mutation)\n const walEntry: WalEntry = {\n transaction,\n version: validation.nextVersion,\n timestamp: Date.now(),\n };\n\n const appendResult = yield* Effect.either(\n hotStorage.appendWithCheck(documentId, walEntry, validation.nextVersion)\n );\n\n if (appendResult._tag === \"Left\") {\n yield* Effect.logError(\"WAL append failed\", {\n documentId,\n version: validation.nextVersion,\n error: appendResult.left,\n });\n yield* Metric.increment(Metrics.walAppendFailures);\n\n const latency = Date.now() - submitStartTime;\n yield* Metric.update(Metrics.transactionsLatency, latency);\n\n return {\n success: false as const,\n reason: \"Storage unavailable. Please retry.\",\n };\n }\n\n // Phase 3: Apply (state mutation + broadcast)\n document.apply(transaction);\n\n // Track metrics\n const latency = Date.now() - submitStartTime;\n yield* Metric.update(Metrics.transactionsLatency, latency);\n yield* Metric.increment(Metrics.transactionsProcessed);\n yield* Metric.increment(Metrics.storageWalAppends);\n\n // Increment transaction count\n yield* Ref.update(transactionsSinceSnapshotRef, (n) => n + 1);\n\n // Check snapshot triggers\n yield* checkSnapshotTriggers();\n\n return {\n success: true as const,\n version: validation.nextVersion,\n };\n });\n\n const touch = Effect.fn(\"document.touch\")(function* () {\n yield* Ref.set(lastActivityTimeRef, Date.now());\n });\n\n return {\n document,\n pubsub,\n getSnapshotTracking,\n submit,\n saveSnapshot,\n checkSnapshotTriggers,\n touch,\n getVersion: () => document.getVersion(),\n getSnapshot: () => document.getSnapshot(),\n };\n });\n\n// =============================================================================\n// Helper Functions\n// =============================================================================\n\n/**\n * Compute initial state for a new document.\n */\nconst computeInitialState = <TSchema extends Primitive.AnyPrimitive>(\n config: DocumentInstanceConfig<TSchema>,\n documentId: string\n): Effect.Effect<Primitive.InferSetInput<TSchema> | undefined> => {\n if (config.initial === undefined) {\n return Effect.succeed(undefined);\n }\n\n if (typeof config.initial === \"function\") {\n return (config.initial as (ctx: { documentId: string }) => Effect.Effect<Primitive.InferSetInput<TSchema>>)({\n documentId,\n });\n }\n\n return Effect.succeed(config.initial as Primitive.InferSetInput<TSchema>);\n};\n\n/**\n * Verify WAL continuity and log warnings for any gaps.\n */\nconst verifyWalContinuity = Effect.fn(\"document.wal.verify\")(function* (\n documentId: string,\n walEntries: readonly WalEntry[],\n baseVersion: number\n) {\n if (walEntries.length === 0) {\n return;\n }\n\n const firstWalVersion = walEntries[0]!.version;\n const expectedFirst = baseVersion + 1;\n\n if (firstWalVersion !== expectedFirst) {\n yield* Effect.logWarning(\"WAL version gap detected\", {\n documentId,\n snapshotVersion: baseVersion,\n firstWalVersion,\n expectedFirst,\n });\n yield* Metric.increment(Metrics.storageVersionGaps);\n }\n\n for (let i = 1; i < walEntries.length; i++) {\n const prev = walEntries[i - 1]!.version;\n const curr = walEntries[i]!.version;\n if (curr !== prev + 1) {\n yield* Effect.logWarning(\"WAL internal gap detected\", {\n documentId,\n previousVersion: prev,\n currentVersion: curr,\n });\n }\n }\n});\n\n/**\n * Replay WAL entries onto a ServerDocument.\n */\nconst replayWalEntries = Effect.fn(\"document.wal.replay\")(function* (\n documentId: string,\n document: ServerDocument.ServerDocument<Primitive.AnyPrimitive>,\n walEntries: readonly WalEntry[]\n) {\n for (const entry of walEntries) {\n const result = document.submit(entry.transaction);\n if (!result.success) {\n yield* Effect.logWarning(\"Skipping corrupted WAL entry\", {\n documentId,\n version: entry.version,\n reason: result.reason,\n });\n }\n }\n});\n\n/**\n * Compute snapshot state by replaying WAL entries on a base state.\n */\nconst computeSnapshotState = <TSchema extends Primitive.AnyPrimitive>(\n schema: TSchema,\n baseState: Primitive.InferState<TSchema> | undefined,\n walEntries: readonly WalEntry[],\n targetVersion: number\n): { state: Primitive.InferState<TSchema>; version: number } | undefined => {\n const relevantEntries = walEntries.filter((e) => e.version <= targetVersion);\n\n if (relevantEntries.length === 0 && baseState === undefined) {\n return undefined;\n }\n\n let snapshotState: Primitive.InferState<TSchema> | undefined = baseState;\n for (const entry of relevantEntries) {\n const tempDoc = Document.make(schema, { initialState: snapshotState });\n tempDoc.apply(entry.transaction.ops);\n snapshotState = tempDoc.get();\n }\n\n if (snapshotState === undefined) {\n return undefined;\n }\n\n const snapshotVersion =\n relevantEntries.length > 0 ? relevantEntries[relevantEntries.length - 1]!.version : 0;\n\n return { state: snapshotState, version: snapshotVersion };\n};\n\n/**\n * Check if a snapshot should be triggered.\n */\nconst shouldTriggerSnapshot = (\n transactionsSinceSnapshot: number,\n lastSnapshotTime: number,\n config: { interval: Duration.Duration; transactionThreshold: number }\n): boolean => {\n const now = Date.now();\n const intervalMs = Duration.toMillis(config.interval);\n\n if (transactionsSinceSnapshot >= config.transactionThreshold) {\n return true;\n }\n\n if (now - lastSnapshotTime >= intervalMs) {\n return true;\n }\n\n return false;\n};\n\n/**\n * Create a StoredDocument for persistence.\n */\nconst createStoredDocument = (\n state: unknown,\n version: number,\n schemaVersion: number\n): StoredDocument => ({\n state,\n version,\n schemaVersion,\n savedAt: Date.now(),\n});\n\n// =============================================================================\n// Re-export namespace\n// =============================================================================\n\nexport const DocumentInstance = {\n make,\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AA8FA,MAAa,QACX,YACA,QACA,aACA,eAEA,OAAO,IAAI,aAAa;CAEtB,MAAM,iBAAiB;CAGvB,MAAM,YAAY,OAAO,YAAY,KAAK,WAAW;CAKrD,IAAIA;CACJ,IAAIC;CACJ,IAAI,iBAAiB;AAErB,KAAI,WAAW;AAEb,iBAAe,UAAU;AACzB,mBAAiB,UAAU;OAG3B,WAAU,OAAO,oBAAoB,QAAQ,WAAW;CAI1D,MAAM,SAAS,OAAO,OAAO,WAA4B;CAGzD,MAAM,yBAAyB,OAAO,IAAI,KAAK,eAAe;CAC9D,MAAM,sBAAsB,OAAO,IAAI,KAAK,KAAK,KAAK,CAAC;CACvD,MAAM,+BAA+B,OAAO,IAAI,KAAK,EAAE;CACvD,MAAM,sBAAsB,OAAO,IAAI,KAAK,KAAK,KAAK,CAAC;CAGvD,MAAM,WAAW,eAAe,KAAK;EACnC,QAAQ,OAAO;EACf;EACA;EACA;EACA,uBAAuB,OAAO;EAC9B,cAAc,YAA+C;AAC3D,UAAO,QACL,OAAO,QAAQ,QAAQ;IACrB,MAAM;IACN,aAAa,QAAQ;IACrB,SAAS,QAAQ;IAClB,CAAC,CACH;;EAEH,cAAc,eAAuB,WAAmB;AACtD,UAAO,QACL,OAAO,QAAQ,QAAQ;IACrB,MAAM;IACN;IACA;IACD,CAAC,CACH;;EAEJ,CAAC;AAIF,KAAI,CAAC,WAAW;EACd,MAAM,mBAAmB,qBAAqB,SAAS,KAAK,EAAE,GAAG,eAAe;AAChF,SAAO,YAAY,KAAK,YAAY,iBAAiB;AACrD,SAAO,OAAO,SAAS,2CAA2C,EAAE,YAAY,CAAC;;CAInF,MAAM,aAAa,OAAO,WAAW,WAAW,YAAY,eAAe;AAG3E,QAAO,oBAAoB,YAAY,YAAY,eAAe;AAGlE,QAAO,iBAAiB,YAAY,UAAU,WAAW;AAGzD,KAAI,UACF,QAAO,OAAO,UAAUC,kBAA0B;KAElD,QAAO,OAAO,UAAUC,iBAAyB;AAEnD,QAAO,OAAO,YAAYC,iBAAyB,EAAE;CAMrD,MAAM,sBAAsB,OAAO,IAAI,aAAa;AAClD,SAAO;GACL,qBAAqB,OAAO,IAAI,IAAI,uBAAuB;GAC3D,kBAAkB,OAAO,IAAI,IAAI,oBAAoB;GACrD,2BAA2B,OAAO,IAAI,IAAI,6BAA6B;GACxE;GACD;CAEF,MAAM,eAAe,OAAO,GAAG,yBAAyB,CAAC,aAAa;;EACpE,MAAM,gBAAgB,SAAS,YAAY;AAI3C,MAAI,kBAHwB,OAAO,IAAI,IAAI,uBAAuB,EAIhE;EAGF,MAAM,oBAAoB,KAAK,KAAK;EAGpC,MAAM,eAAe,OAAO,YAAY,KAAK,WAAW;EACxD,MAAM,mGAAc,aAAc,gFAAW;EAC7C,MAAM,wEAAY,aAAc;EAGhC,MAAMC,eAAa,OAAO,WAAW,WAAW,YAAY,YAAY;EAGxE,MAAM,iBAAiB,qBACrB,OAAO,QACP,WACAA,cACA,cACD;AAED,MAAI,CAAC,eACH;EAIF,MAAM,sBAAsB,OAAO,IAAI,IAAI,uBAAuB;AAClE,MAAI,eAAe,WAAW,oBAC5B;EAGF,MAAMC,cAAY,qBAChB,eAAe,OACf,eAAe,SACf,eACD;AAGD,SAAO,YAAY,KAAK,YAAYA,YAAU;EAG9C,MAAM,mBAAmB,KAAK,KAAK,GAAG;AACtC,SAAO,OAAO,UAAUC,iBAAyB;AACjD,SAAO,OAAO,OAAOC,wBAAgC,iBAAiB;AAGtE,SAAO,IAAI,IAAI,wBAAwB,eAAe,QAAQ;AAC9D,SAAO,IAAI,IAAI,qBAAqB,KAAK,KAAK,CAAC;AAC/C,SAAO,IAAI,IAAI,8BAA8B,EAAE;AAG/C,SAAO,OAAO,SAAS,WAAW,SAAS,YAAY,eAAe,QAAQ,GAAG,MAC/E,OAAO,WAAW,qDAAqD;GACrE;GACA,SAAS,eAAe;GACxB,OAAO;GACR,CAAC,CACH;GACD;CAEF,MAAM,wBAAwB,OAAO,GAAG,mCAAmC,CAAC,aAAa;AAIvF,MAAI,sBAHY,OAAO,IAAI,IAAI,6BAA6B,EAC3C,OAAO,IAAI,IAAI,oBAAoB,EAEP,OAAO,SAAS,CAC3D,QAAO,cAAc;GAEvB;AA6EF,QAAO;EACL;EACA;EACA;EACA,QA/Ea,OAAO,GAAG,8BAA8B,CAAC,WACtD,aACA;GACA,MAAM,kBAAkB,KAAK,KAAK;AAGlC,UAAO,IAAI,IAAI,qBAAqB,KAAK,KAAK,CAAC;GAG/C,MAAM,aAAa,SAAS,SAAS,YAAY;AAEjD,OAAI,CAAC,WAAW,OAAO;AACrB,WAAO,OAAO,UAAUC,qBAA6B;IACrD,MAAMC,YAAU,KAAK,KAAK,GAAG;AAC7B,WAAO,OAAO,OAAOC,qBAA6BD,UAAQ;AAE1D,WAAO;KACL,SAAS;KACT,QAAQ,WAAW;KACpB;;GAIH,MAAME,WAAqB;IACzB;IACA,SAAS,WAAW;IACpB,WAAW,KAAK,KAAK;IACtB;GAED,MAAM,eAAe,OAAO,OAAO,OACjC,WAAW,gBAAgB,YAAY,UAAU,WAAW,YAAY,CACzE;AAED,OAAI,aAAa,SAAS,QAAQ;AAChC,WAAO,OAAO,SAAS,qBAAqB;KAC1C;KACA,SAAS,WAAW;KACpB,OAAO,aAAa;KACrB,CAAC;AACF,WAAO,OAAO,UAAUC,kBAA0B;IAElD,MAAMH,YAAU,KAAK,KAAK,GAAG;AAC7B,WAAO,OAAO,OAAOC,qBAA6BD,UAAQ;AAE1D,WAAO;KACL,SAAS;KACT,QAAQ;KACT;;AAIH,YAAS,MAAM,YAAY;GAG3B,MAAM,UAAU,KAAK,KAAK,GAAG;AAC7B,UAAO,OAAO,OAAOC,qBAA6B,QAAQ;AAC1D,UAAO,OAAO,UAAUG,sBAA8B;AACtD,UAAO,OAAO,UAAUC,kBAA0B;AAGlD,UAAO,IAAI,OAAO,+BAA+B,MAAM,IAAI,EAAE;AAG7D,UAAO,uBAAuB;AAE9B,UAAO;IACL,SAAS;IACT,SAAS,WAAW;IACrB;IACD;EAWA;EACA;EACA,OAXY,OAAO,GAAG,iBAAiB,CAAC,aAAa;AACrD,UAAO,IAAI,IAAI,qBAAqB,KAAK,KAAK,CAAC;IAC/C;EAUA,kBAAkB,SAAS,YAAY;EACvC,mBAAmB,SAAS,aAAa;EAC1C;EACD;;;;AASJ,MAAM,uBACJ,QACA,eACgE;AAChE,KAAI,OAAO,YAAY,OACrB,QAAO,OAAO,QAAQ,OAAU;AAGlC,KAAI,OAAO,OAAO,YAAY,WAC5B,QAAQ,OAAO,QAA6F,EAC1G,YACD,CAAC;AAGJ,QAAO,OAAO,QAAQ,OAAO,QAA4C;;;;;AAM3E,MAAM,sBAAsB,OAAO,GAAG,sBAAsB,CAAC,WAC3D,YACA,YACA,aACA;AACA,KAAI,WAAW,WAAW,EACxB;CAGF,MAAM,kBAAkB,WAAW,GAAI;CACvC,MAAM,gBAAgB,cAAc;AAEpC,KAAI,oBAAoB,eAAe;AACrC,SAAO,OAAO,WAAW,4BAA4B;GACnD;GACA,iBAAiB;GACjB;GACA;GACD,CAAC;AACF,SAAO,OAAO,UAAUC,mBAA2B;;AAGrD,MAAK,IAAI,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK;EAC1C,MAAM,OAAO,WAAW,IAAI,GAAI;EAChC,MAAM,OAAO,WAAW,GAAI;AAC5B,MAAI,SAAS,OAAO,EAClB,QAAO,OAAO,WAAW,6BAA6B;GACpD;GACA,iBAAiB;GACjB,gBAAgB;GACjB,CAAC;;EAGN;;;;AAKF,MAAM,mBAAmB,OAAO,GAAG,sBAAsB,CAAC,WACxD,YACA,UACA,YACA;AACA,MAAK,MAAM,SAAS,YAAY;EAC9B,MAAM,SAAS,SAAS,OAAO,MAAM,YAAY;AACjD,MAAI,CAAC,OAAO,QACV,QAAO,OAAO,WAAW,gCAAgC;GACvD;GACA,SAAS,MAAM;GACf,QAAQ,OAAO;GAChB,CAAC;;EAGN;;;;AAKF,MAAM,wBACJ,QACA,WACA,YACA,kBAC0E;CAC1E,MAAM,kBAAkB,WAAW,QAAQ,MAAM,EAAE,WAAW,cAAc;AAE5E,KAAI,gBAAgB,WAAW,KAAK,cAAc,OAChD;CAGF,IAAIC,gBAA2D;AAC/D,MAAK,MAAM,SAAS,iBAAiB;EACnC,MAAM,UAAU,SAAS,KAAK,QAAQ,EAAE,cAAc,eAAe,CAAC;AACtE,UAAQ,MAAM,MAAM,YAAY,IAAI;AACpC,kBAAgB,QAAQ,KAAK;;AAG/B,KAAI,kBAAkB,OACpB;CAGF,MAAM,kBACJ,gBAAgB,SAAS,IAAI,gBAAgB,gBAAgB,SAAS,GAAI,UAAU;AAEtF,QAAO;EAAE,OAAO;EAAe,SAAS;EAAiB;;;;;AAM3D,MAAM,yBACJ,2BACA,kBACA,WACY;CACZ,MAAM,MAAM,KAAK,KAAK;CACtB,MAAM,aAAa,SAAS,SAAS,OAAO,SAAS;AAErD,KAAI,6BAA6B,OAAO,qBACtC,QAAO;AAGT,KAAI,MAAM,oBAAoB,WAC5B,QAAO;AAGT,QAAO;;;;;AAMT,MAAM,wBACJ,OACA,SACA,mBACoB;CACpB;CACA;CACA;CACA,SAAS,KAAK,KAAK;CACpB;AAMD,MAAa,mBAAmB,EAC9B,MACD"}
|
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
const require_ColdStorage = require('./ColdStorage.cjs');
|
|
2
2
|
const require_HotStorage = require('./HotStorage.cjs');
|
|
3
3
|
const require_Metrics = require('./Metrics.cjs');
|
|
4
|
+
const require_DocumentInstance = require('./DocumentInstance.cjs');
|
|
4
5
|
const require_objectSpread2 = require('./_virtual/_@oxc-project_runtime@0.103.0/helpers/objectSpread2.cjs');
|
|
5
6
|
const require_MimicServerEngine = require('./MimicServerEngine.cjs');
|
|
6
7
|
let effect = require("effect");
|
|
7
|
-
let _voidhash_mimic = require("@voidhash/mimic");
|
|
8
|
-
let _voidhash_mimic_server = require("@voidhash/mimic/server");
|
|
9
8
|
let _effect_cluster = require("@effect/cluster");
|
|
10
9
|
let _effect_rpc = require("@effect/rpc");
|
|
11
10
|
|
|
@@ -133,224 +132,41 @@ const encodeTransaction = (tx) => {
|
|
|
133
132
|
*/
|
|
134
133
|
const createEntityHandler = (config, coldStorage, hotStorage) => effect.Effect.fn("cluster.entity.handler.create")(function* () {
|
|
135
134
|
const documentId = (yield* _effect_cluster.Entity.CurrentAddress).entityId;
|
|
136
|
-
const
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
};
|
|
142
|
-
const storedDoc = yield* coldStorage.load(documentId).pipe(effect.Effect.orDie);
|
|
143
|
-
let initialState;
|
|
144
|
-
let initialVersion = 0;
|
|
145
|
-
if (storedDoc) {
|
|
146
|
-
initialState = storedDoc.state;
|
|
147
|
-
initialVersion = storedDoc.version;
|
|
148
|
-
} else initialState = yield* computeInitialState();
|
|
149
|
-
const broadcastPubSub = yield* effect.PubSub.unbounded();
|
|
135
|
+
const instance = yield* require_DocumentInstance.DocumentInstance.make(documentId, {
|
|
136
|
+
schema: config.schema,
|
|
137
|
+
initial: config.initial,
|
|
138
|
+
maxTransactionHistory: config.maxTransactionHistory,
|
|
139
|
+
snapshot: config.snapshot
|
|
140
|
+
}, coldStorage, hotStorage).pipe(effect.Effect.orDie);
|
|
150
141
|
const presencePubSub = yield* effect.PubSub.unbounded();
|
|
151
142
|
const stateRef = yield* effect.Ref.make({
|
|
152
|
-
|
|
153
|
-
broadcastPubSub,
|
|
143
|
+
instance,
|
|
154
144
|
presences: effect.HashMap.empty(),
|
|
155
|
-
presencePubSub
|
|
156
|
-
lastSnapshotVersion: initialVersion,
|
|
157
|
-
lastSnapshotTime: Date.now(),
|
|
158
|
-
transactionsSinceSnapshot: 0
|
|
145
|
+
presencePubSub
|
|
159
146
|
});
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
initialState,
|
|
163
|
-
initialVersion,
|
|
164
|
-
maxTransactionHistory: config.maxTransactionHistory,
|
|
165
|
-
onBroadcast: (message) => {
|
|
166
|
-
effect.Effect.runSync(effect.PubSub.publish(broadcastPubSub, {
|
|
167
|
-
type: "transaction",
|
|
168
|
-
transaction: message.transaction,
|
|
169
|
-
version: message.version
|
|
170
|
-
}));
|
|
171
|
-
},
|
|
172
|
-
onRejection: (transactionId, reason) => {
|
|
173
|
-
effect.Effect.runSync(effect.PubSub.publish(broadcastPubSub, {
|
|
174
|
-
type: "error",
|
|
175
|
-
transactionId,
|
|
176
|
-
reason
|
|
177
|
-
}));
|
|
178
|
-
}
|
|
179
|
-
});
|
|
180
|
-
yield* effect.Ref.update(stateRef, (s) => require_objectSpread2._objectSpread2(require_objectSpread2._objectSpread2({}, s), {}, { document }));
|
|
181
|
-
const walEntries = yield* hotStorage.getEntries(documentId, initialVersion).pipe(effect.Effect.orDie);
|
|
182
|
-
if (walEntries.length > 0) {
|
|
183
|
-
const firstWalVersion = walEntries[0].version;
|
|
184
|
-
const expectedFirst = initialVersion + 1;
|
|
185
|
-
if (firstWalVersion !== expectedFirst) {
|
|
186
|
-
yield* effect.Effect.logWarning("WAL version gap detected", {
|
|
187
|
-
documentId,
|
|
188
|
-
snapshotVersion: initialVersion,
|
|
189
|
-
firstWalVersion,
|
|
190
|
-
expectedFirst
|
|
191
|
-
});
|
|
192
|
-
yield* effect.Metric.increment(require_Metrics.storageVersionGaps);
|
|
193
|
-
}
|
|
194
|
-
for (let i = 1; i < walEntries.length; i++) {
|
|
195
|
-
const prev = walEntries[i - 1].version;
|
|
196
|
-
const curr = walEntries[i].version;
|
|
197
|
-
if (curr !== prev + 1) yield* effect.Effect.logWarning("WAL internal gap detected", {
|
|
198
|
-
documentId,
|
|
199
|
-
previousVersion: prev,
|
|
200
|
-
currentVersion: curr
|
|
201
|
-
});
|
|
202
|
-
}
|
|
203
|
-
}
|
|
204
|
-
for (const entry of walEntries) {
|
|
205
|
-
const result = document.submit(entry.transaction);
|
|
206
|
-
if (!result.success) yield* effect.Effect.logWarning("Skipping corrupted WAL entry", {
|
|
207
|
-
documentId,
|
|
208
|
-
version: entry.version,
|
|
209
|
-
reason: result.reason
|
|
210
|
-
});
|
|
211
|
-
}
|
|
212
|
-
if (storedDoc) yield* effect.Metric.increment(require_Metrics.documentsRestored);
|
|
213
|
-
else yield* effect.Metric.increment(require_Metrics.documentsCreated);
|
|
214
|
-
yield* effect.Metric.incrementBy(require_Metrics.documentsActive, 1);
|
|
215
|
-
/**
|
|
216
|
-
* Save snapshot to ColdStorage derived from WAL entries.
|
|
217
|
-
* This ensures snapshots are always based on durable WAL data.
|
|
218
|
-
* Idempotent: skips save if already snapshotted at target version.
|
|
219
|
-
* Truncate failures are non-fatal and will be retried on next snapshot.
|
|
220
|
-
*/
|
|
221
|
-
const saveSnapshot = effect.Effect.fn("cluster.document.snapshot.save")(function* (targetVersion) {
|
|
222
|
-
var _baseSnapshot$version;
|
|
223
|
-
if (targetVersion <= (yield* effect.Ref.get(stateRef)).lastSnapshotVersion) return;
|
|
224
|
-
const snapshotStartTime = Date.now();
|
|
225
|
-
const baseSnapshotResult = yield* effect.Effect.either(coldStorage.load(documentId));
|
|
226
|
-
if (baseSnapshotResult._tag === "Left") {
|
|
227
|
-
yield* effect.Effect.logError("Failed to load base snapshot for WAL replay", {
|
|
228
|
-
documentId,
|
|
229
|
-
error: baseSnapshotResult.left
|
|
230
|
-
});
|
|
231
|
-
return;
|
|
232
|
-
}
|
|
233
|
-
const baseSnapshot = baseSnapshotResult.right;
|
|
234
|
-
const baseVersion = (_baseSnapshot$version = baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.version) !== null && _baseSnapshot$version !== void 0 ? _baseSnapshot$version : 0;
|
|
235
|
-
const baseState = baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.state;
|
|
236
|
-
const walEntriesResult = yield* effect.Effect.either(hotStorage.getEntries(documentId, baseVersion));
|
|
237
|
-
if (walEntriesResult._tag === "Left") {
|
|
238
|
-
yield* effect.Effect.logError("Failed to load WAL entries for snapshot", {
|
|
239
|
-
documentId,
|
|
240
|
-
error: walEntriesResult.left
|
|
241
|
-
});
|
|
242
|
-
return;
|
|
243
|
-
}
|
|
244
|
-
const relevantEntries = walEntriesResult.right.filter((e) => e.version <= targetVersion);
|
|
245
|
-
if (relevantEntries.length === 0 && !baseSnapshot) return;
|
|
246
|
-
let snapshotState = baseState;
|
|
247
|
-
for (const entry of relevantEntries) {
|
|
248
|
-
const tempDoc = _voidhash_mimic.Document.make(config.schema, { initialState: snapshotState });
|
|
249
|
-
tempDoc.apply(entry.transaction.ops);
|
|
250
|
-
snapshotState = tempDoc.get();
|
|
251
|
-
}
|
|
252
|
-
if (snapshotState === void 0) return;
|
|
253
|
-
const snapshotVersion = relevantEntries.length > 0 ? relevantEntries[relevantEntries.length - 1].version : baseVersion;
|
|
254
|
-
if (snapshotVersion <= (yield* effect.Ref.get(stateRef)).lastSnapshotVersion) return;
|
|
255
|
-
const storedDocument = {
|
|
256
|
-
state: snapshotState,
|
|
257
|
-
version: snapshotVersion,
|
|
258
|
-
schemaVersion: SCHEMA_VERSION,
|
|
259
|
-
savedAt: Date.now()
|
|
260
|
-
};
|
|
261
|
-
yield* effect.Effect.catchAll(coldStorage.save(documentId, storedDocument), (e) => effect.Effect.logError("Failed to save snapshot", {
|
|
262
|
-
documentId,
|
|
263
|
-
error: e
|
|
264
|
-
}));
|
|
265
|
-
const snapshotDuration = Date.now() - snapshotStartTime;
|
|
266
|
-
yield* effect.Metric.increment(require_Metrics.storageSnapshots);
|
|
267
|
-
yield* effect.Metric.update(require_Metrics.storageSnapshotLatency, snapshotDuration);
|
|
268
|
-
yield* effect.Ref.update(stateRef, (s) => require_objectSpread2._objectSpread2(require_objectSpread2._objectSpread2({}, s), {}, {
|
|
269
|
-
lastSnapshotVersion: snapshotVersion,
|
|
270
|
-
lastSnapshotTime: Date.now(),
|
|
271
|
-
transactionsSinceSnapshot: 0
|
|
272
|
-
}));
|
|
273
|
-
yield* effect.Effect.catchAll(hotStorage.truncate(documentId, snapshotVersion), (e) => effect.Effect.logWarning("WAL truncate failed - will retry on next snapshot", {
|
|
147
|
+
yield* effect.Effect.addFinalizer(() => effect.Effect.fn("cluster.entity.finalize")(function* () {
|
|
148
|
+
yield* effect.Effect.catchAll(instance.saveSnapshot(), (e) => effect.Effect.logError("Failed to save snapshot during entity finalization", {
|
|
274
149
|
documentId,
|
|
275
|
-
version: snapshotVersion,
|
|
276
150
|
error: e
|
|
277
151
|
}));
|
|
278
|
-
});
|
|
279
|
-
/**
|
|
280
|
-
* Check if snapshot should be triggered
|
|
281
|
-
*/
|
|
282
|
-
const checkSnapshotTriggers = effect.Effect.fn("cluster.document.snapshot.check-triggers")(function* () {
|
|
283
|
-
const state = yield* effect.Ref.get(stateRef);
|
|
284
|
-
const now = Date.now();
|
|
285
|
-
const currentVersion = state.document.getVersion();
|
|
286
|
-
const intervalMs = effect.Duration.toMillis(config.snapshot.interval);
|
|
287
|
-
const threshold = config.snapshot.transactionThreshold;
|
|
288
|
-
if (state.transactionsSinceSnapshot >= threshold) {
|
|
289
|
-
yield* saveSnapshot(currentVersion);
|
|
290
|
-
return;
|
|
291
|
-
}
|
|
292
|
-
if (now - state.lastSnapshotTime >= intervalMs) {
|
|
293
|
-
yield* saveSnapshot(currentVersion);
|
|
294
|
-
return;
|
|
295
|
-
}
|
|
296
|
-
});
|
|
297
|
-
yield* effect.Effect.addFinalizer(() => effect.Effect.fn("cluster.entity.finalize")(function* () {
|
|
298
|
-
yield* saveSnapshot((yield* effect.Ref.get(stateRef)).document.getVersion());
|
|
299
152
|
yield* effect.Metric.incrementBy(require_Metrics.documentsActive, -1);
|
|
300
153
|
yield* effect.Metric.increment(require_Metrics.documentsEvicted);
|
|
301
154
|
yield* effect.Effect.logDebug("Entity finalized", { documentId });
|
|
302
155
|
})());
|
|
303
156
|
return {
|
|
304
157
|
Submit: effect.Effect.fn("cluster.document.transaction.submit")(function* ({ payload }) {
|
|
305
|
-
const submitStartTime = Date.now();
|
|
306
|
-
const state = yield* effect.Ref.get(stateRef);
|
|
307
158
|
const transaction = decodeTransaction(payload.transaction);
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
yield* effect.Metric.update(require_Metrics.transactionsLatency, latency$1);
|
|
313
|
-
return {
|
|
314
|
-
success: false,
|
|
315
|
-
reason: validation.reason
|
|
316
|
-
};
|
|
317
|
-
}
|
|
318
|
-
const walEntry = {
|
|
319
|
-
transaction,
|
|
320
|
-
version: validation.nextVersion,
|
|
321
|
-
timestamp: Date.now()
|
|
322
|
-
};
|
|
323
|
-
const appendResult = yield* effect.Effect.either(hotStorage.appendWithCheck(documentId, walEntry, validation.nextVersion));
|
|
324
|
-
if (appendResult._tag === "Left") {
|
|
325
|
-
yield* effect.Effect.logError("WAL append failed", {
|
|
326
|
-
documentId,
|
|
327
|
-
version: validation.nextVersion,
|
|
328
|
-
error: appendResult.left
|
|
329
|
-
});
|
|
330
|
-
yield* effect.Metric.increment(require_Metrics.walAppendFailures);
|
|
331
|
-
const latency$1 = Date.now() - submitStartTime;
|
|
332
|
-
yield* effect.Metric.update(require_Metrics.transactionsLatency, latency$1);
|
|
333
|
-
return {
|
|
334
|
-
success: false,
|
|
335
|
-
reason: "Storage unavailable. Please retry."
|
|
336
|
-
};
|
|
337
|
-
}
|
|
338
|
-
state.document.apply(transaction);
|
|
339
|
-
const latency = Date.now() - submitStartTime;
|
|
340
|
-
yield* effect.Metric.update(require_Metrics.transactionsLatency, latency);
|
|
341
|
-
yield* effect.Metric.increment(require_Metrics.transactionsProcessed);
|
|
342
|
-
yield* effect.Metric.increment(require_Metrics.storageWalAppends);
|
|
343
|
-
yield* effect.Ref.update(stateRef, (s) => require_objectSpread2._objectSpread2(require_objectSpread2._objectSpread2({}, s), {}, { transactionsSinceSnapshot: s.transactionsSinceSnapshot + 1 }));
|
|
344
|
-
yield* checkSnapshotTriggers();
|
|
345
|
-
return {
|
|
346
|
-
success: true,
|
|
347
|
-
version: validation.nextVersion
|
|
348
|
-
};
|
|
159
|
+
return yield* instance.submit(transaction).pipe(effect.Effect.catchAll((error) => effect.Effect.succeed({
|
|
160
|
+
success: false,
|
|
161
|
+
reason: `Storage error: ${String(error)}`
|
|
162
|
+
})));
|
|
349
163
|
}),
|
|
350
164
|
GetSnapshot: effect.Effect.fn("cluster.document.snapshot.get")(function* () {
|
|
351
|
-
return
|
|
165
|
+
return instance.getSnapshot();
|
|
166
|
+
}),
|
|
167
|
+
Touch: effect.Effect.fn("cluster.document.touch")(function* () {
|
|
168
|
+
yield* instance.touch();
|
|
352
169
|
}),
|
|
353
|
-
Touch: effect.Effect.fn("cluster.document.touch")(function* () {}),
|
|
354
170
|
SetPresence: effect.Effect.fn("cluster.presence.set")(function* ({ payload }) {
|
|
355
171
|
const { connectionId, entry } = payload;
|
|
356
172
|
yield* effect.Ref.update(stateRef, (s) => require_objectSpread2._objectSpread2(require_objectSpread2._objectSpread2({}, s), {}, { presences: effect.HashMap.set(s.presences, connectionId, entry) }));
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MimicClusterServerEngine.d.cts","names":[],"sources":["../src/MimicClusterServerEngine.ts"],"sourcesContent":[],"mappings":";;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"MimicClusterServerEngine.d.cts","names":[],"sources":["../src/MimicClusterServerEngine.ts"],"sourcesContent":[],"mappings":";;;;;;;;;;;cA8nBa;yBA3JwB,SAAA,CAAU,sBACrC,+BAA+B,aACtC,KAAA,CAAM,MACP,6BAEA,iBAAiB,gBAAgB,sBAAsB,QAAA,CAAS"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MimicClusterServerEngine.d.mts","names":[],"sources":["../src/MimicClusterServerEngine.ts"],"sourcesContent":[],"mappings":";;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"MimicClusterServerEngine.d.mts","names":[],"sources":["../src/MimicClusterServerEngine.ts"],"sourcesContent":[],"mappings":";;;;;;;;;;;cA8nBa;yBA3JwB,SAAA,CAAU,sBACrC,+BAA+B,aACtC,KAAA,CAAM,MACP,6BAEA,iBAAiB,gBAAgB,sBAAsB,QAAA,CAAS"}
|