@voidhash/mimic-effect 1.0.0-beta.6 → 1.0.0-beta.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +41 -41
- package/dist/ColdStorage.cjs +9 -5
- package/dist/ColdStorage.d.cts.map +1 -1
- package/dist/ColdStorage.d.mts.map +1 -1
- package/dist/ColdStorage.mjs +9 -5
- package/dist/ColdStorage.mjs.map +1 -1
- package/dist/DocumentInstance.cjs +255 -0
- package/dist/DocumentInstance.d.cts +74 -0
- package/dist/DocumentInstance.d.cts.map +1 -0
- package/dist/DocumentInstance.d.mts +74 -0
- package/dist/DocumentInstance.d.mts.map +1 -0
- package/dist/DocumentInstance.mjs +256 -0
- package/dist/DocumentInstance.mjs.map +1 -0
- package/dist/HotStorage.cjs +17 -13
- package/dist/HotStorage.d.cts.map +1 -1
- package/dist/HotStorage.d.mts.map +1 -1
- package/dist/HotStorage.mjs +17 -13
- package/dist/HotStorage.mjs.map +1 -1
- package/dist/MimicClusterServerEngine.cjs +31 -215
- package/dist/MimicClusterServerEngine.d.cts.map +1 -1
- package/dist/MimicClusterServerEngine.d.mts.map +1 -1
- package/dist/MimicClusterServerEngine.mjs +36 -220
- package/dist/MimicClusterServerEngine.mjs.map +1 -1
- package/dist/MimicServer.cjs +19 -19
- package/dist/MimicServer.d.cts.map +1 -1
- package/dist/MimicServer.d.mts.map +1 -1
- package/dist/MimicServer.mjs +19 -19
- package/dist/MimicServer.mjs.map +1 -1
- package/dist/MimicServerEngine.cjs +71 -9
- package/dist/MimicServerEngine.d.cts +12 -7
- package/dist/MimicServerEngine.d.cts.map +1 -1
- package/dist/MimicServerEngine.d.mts +12 -7
- package/dist/MimicServerEngine.d.mts.map +1 -1
- package/dist/MimicServerEngine.mjs +73 -11
- package/dist/MimicServerEngine.mjs.map +1 -1
- package/dist/PresenceManager.cjs +5 -5
- package/dist/PresenceManager.d.cts.map +1 -1
- package/dist/PresenceManager.d.mts.map +1 -1
- package/dist/PresenceManager.mjs +5 -5
- package/dist/PresenceManager.mjs.map +1 -1
- package/dist/Protocol.d.cts +1 -1
- package/dist/Protocol.d.mts +1 -1
- package/dist/index.cjs +2 -4
- package/dist/index.d.cts +2 -2
- package/dist/index.d.mts +2 -2
- package/dist/index.mjs +2 -2
- package/dist/testing/types.d.cts +3 -3
- package/package.json +3 -3
- package/src/ColdStorage.ts +21 -12
- package/src/DocumentInstance.ts +510 -0
- package/src/HotStorage.ts +75 -58
- package/src/MimicClusterServerEngine.ts +93 -398
- package/src/MimicServer.ts +83 -75
- package/src/MimicServerEngine.ts +170 -34
- package/src/PresenceManager.ts +44 -34
- package/src/index.ts +3 -4
- package/tests/DocumentInstance.test.ts +669 -0
- package/dist/DocumentManager.cjs +0 -299
- package/dist/DocumentManager.d.cts +0 -67
- package/dist/DocumentManager.d.cts.map +0 -1
- package/dist/DocumentManager.d.mts +0 -67
- package/dist/DocumentManager.d.mts.map +0 -1
- package/dist/DocumentManager.mjs +0 -297
- package/dist/DocumentManager.mjs.map +0 -1
- package/src/DocumentManager.ts +0 -614
- package/tests/DocumentManager.test.ts +0 -335
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
const require_ColdStorage = require('./ColdStorage.cjs');
|
|
2
2
|
const require_HotStorage = require('./HotStorage.cjs');
|
|
3
3
|
const require_Metrics = require('./Metrics.cjs');
|
|
4
|
+
const require_DocumentInstance = require('./DocumentInstance.cjs');
|
|
4
5
|
const require_objectSpread2 = require('./_virtual/_@oxc-project_runtime@0.103.0/helpers/objectSpread2.cjs');
|
|
5
6
|
const require_MimicServerEngine = require('./MimicServerEngine.cjs');
|
|
6
7
|
let effect = require("effect");
|
|
7
|
-
let _voidhash_mimic = require("@voidhash/mimic");
|
|
8
|
-
let _voidhash_mimic_server = require("@voidhash/mimic/server");
|
|
9
8
|
let _effect_cluster = require("@effect/cluster");
|
|
10
9
|
let _effect_rpc = require("@effect/rpc");
|
|
11
10
|
|
|
@@ -131,227 +130,44 @@ const encodeTransaction = (tx) => {
|
|
|
131
130
|
/**
|
|
132
131
|
* Create the entity handler for MimicDocument
|
|
133
132
|
*/
|
|
134
|
-
const createEntityHandler = (config, coldStorage, hotStorage) => effect.Effect.
|
|
133
|
+
const createEntityHandler = (config, coldStorage, hotStorage) => effect.Effect.fn("cluster.entity.handler.create")(function* () {
|
|
135
134
|
const documentId = (yield* _effect_cluster.Entity.CurrentAddress).entityId;
|
|
136
|
-
const
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
};
|
|
142
|
-
const storedDoc = yield* coldStorage.load(documentId).pipe(effect.Effect.orDie);
|
|
143
|
-
let initialState;
|
|
144
|
-
let initialVersion = 0;
|
|
145
|
-
if (storedDoc) {
|
|
146
|
-
initialState = storedDoc.state;
|
|
147
|
-
initialVersion = storedDoc.version;
|
|
148
|
-
} else initialState = yield* computeInitialState();
|
|
149
|
-
const broadcastPubSub = yield* effect.PubSub.unbounded();
|
|
135
|
+
const instance = yield* require_DocumentInstance.DocumentInstance.make(documentId, {
|
|
136
|
+
schema: config.schema,
|
|
137
|
+
initial: config.initial,
|
|
138
|
+
maxTransactionHistory: config.maxTransactionHistory,
|
|
139
|
+
snapshot: config.snapshot
|
|
140
|
+
}, coldStorage, hotStorage).pipe(effect.Effect.orDie);
|
|
150
141
|
const presencePubSub = yield* effect.PubSub.unbounded();
|
|
151
142
|
const stateRef = yield* effect.Ref.make({
|
|
152
|
-
|
|
153
|
-
broadcastPubSub,
|
|
143
|
+
instance,
|
|
154
144
|
presences: effect.HashMap.empty(),
|
|
155
|
-
presencePubSub
|
|
156
|
-
lastSnapshotVersion: initialVersion,
|
|
157
|
-
lastSnapshotTime: Date.now(),
|
|
158
|
-
transactionsSinceSnapshot: 0
|
|
145
|
+
presencePubSub
|
|
159
146
|
});
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
initialState,
|
|
163
|
-
initialVersion,
|
|
164
|
-
maxTransactionHistory: config.maxTransactionHistory,
|
|
165
|
-
onBroadcast: (message) => {
|
|
166
|
-
effect.Effect.runSync(effect.PubSub.publish(broadcastPubSub, {
|
|
167
|
-
type: "transaction",
|
|
168
|
-
transaction: message.transaction,
|
|
169
|
-
version: message.version
|
|
170
|
-
}));
|
|
171
|
-
},
|
|
172
|
-
onRejection: (transactionId, reason) => {
|
|
173
|
-
effect.Effect.runSync(effect.PubSub.publish(broadcastPubSub, {
|
|
174
|
-
type: "error",
|
|
175
|
-
transactionId,
|
|
176
|
-
reason
|
|
177
|
-
}));
|
|
178
|
-
}
|
|
179
|
-
});
|
|
180
|
-
yield* effect.Ref.update(stateRef, (s) => require_objectSpread2._objectSpread2(require_objectSpread2._objectSpread2({}, s), {}, { document }));
|
|
181
|
-
const walEntries = yield* hotStorage.getEntries(documentId, initialVersion).pipe(effect.Effect.orDie);
|
|
182
|
-
if (walEntries.length > 0) {
|
|
183
|
-
const firstWalVersion = walEntries[0].version;
|
|
184
|
-
const expectedFirst = initialVersion + 1;
|
|
185
|
-
if (firstWalVersion !== expectedFirst) {
|
|
186
|
-
yield* effect.Effect.logWarning("WAL version gap detected", {
|
|
187
|
-
documentId,
|
|
188
|
-
snapshotVersion: initialVersion,
|
|
189
|
-
firstWalVersion,
|
|
190
|
-
expectedFirst
|
|
191
|
-
});
|
|
192
|
-
yield* effect.Metric.increment(require_Metrics.storageVersionGaps);
|
|
193
|
-
}
|
|
194
|
-
for (let i = 1; i < walEntries.length; i++) {
|
|
195
|
-
const prev = walEntries[i - 1].version;
|
|
196
|
-
const curr = walEntries[i].version;
|
|
197
|
-
if (curr !== prev + 1) yield* effect.Effect.logWarning("WAL internal gap detected", {
|
|
198
|
-
documentId,
|
|
199
|
-
previousVersion: prev,
|
|
200
|
-
currentVersion: curr
|
|
201
|
-
});
|
|
202
|
-
}
|
|
203
|
-
}
|
|
204
|
-
for (const entry of walEntries) {
|
|
205
|
-
const result = document.submit(entry.transaction);
|
|
206
|
-
if (!result.success) yield* effect.Effect.logWarning("Skipping corrupted WAL entry", {
|
|
207
|
-
documentId,
|
|
208
|
-
version: entry.version,
|
|
209
|
-
reason: result.reason
|
|
210
|
-
});
|
|
211
|
-
}
|
|
212
|
-
if (storedDoc) yield* effect.Metric.increment(require_Metrics.documentsRestored);
|
|
213
|
-
else yield* effect.Metric.increment(require_Metrics.documentsCreated);
|
|
214
|
-
yield* effect.Metric.incrementBy(require_Metrics.documentsActive, 1);
|
|
215
|
-
/**
|
|
216
|
-
* Save snapshot to ColdStorage derived from WAL entries.
|
|
217
|
-
* This ensures snapshots are always based on durable WAL data.
|
|
218
|
-
* Idempotent: skips save if already snapshotted at target version.
|
|
219
|
-
* Truncate failures are non-fatal and will be retried on next snapshot.
|
|
220
|
-
*/
|
|
221
|
-
const saveSnapshot = (targetVersion) => effect.Effect.gen(function* () {
|
|
222
|
-
var _baseSnapshot$version;
|
|
223
|
-
if (targetVersion <= (yield* effect.Ref.get(stateRef)).lastSnapshotVersion) return;
|
|
224
|
-
const snapshotStartTime = Date.now();
|
|
225
|
-
const baseSnapshotResult = yield* effect.Effect.either(coldStorage.load(documentId));
|
|
226
|
-
if (baseSnapshotResult._tag === "Left") {
|
|
227
|
-
yield* effect.Effect.logError("Failed to load base snapshot for WAL replay", {
|
|
228
|
-
documentId,
|
|
229
|
-
error: baseSnapshotResult.left
|
|
230
|
-
});
|
|
231
|
-
return;
|
|
232
|
-
}
|
|
233
|
-
const baseSnapshot = baseSnapshotResult.right;
|
|
234
|
-
const baseVersion = (_baseSnapshot$version = baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.version) !== null && _baseSnapshot$version !== void 0 ? _baseSnapshot$version : 0;
|
|
235
|
-
const baseState = baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.state;
|
|
236
|
-
const walEntriesResult = yield* effect.Effect.either(hotStorage.getEntries(documentId, baseVersion));
|
|
237
|
-
if (walEntriesResult._tag === "Left") {
|
|
238
|
-
yield* effect.Effect.logError("Failed to load WAL entries for snapshot", {
|
|
239
|
-
documentId,
|
|
240
|
-
error: walEntriesResult.left
|
|
241
|
-
});
|
|
242
|
-
return;
|
|
243
|
-
}
|
|
244
|
-
const relevantEntries = walEntriesResult.right.filter((e) => e.version <= targetVersion);
|
|
245
|
-
if (relevantEntries.length === 0 && !baseSnapshot) return;
|
|
246
|
-
let snapshotState = baseState;
|
|
247
|
-
for (const entry of relevantEntries) {
|
|
248
|
-
const tempDoc = _voidhash_mimic.Document.make(config.schema, { initialState: snapshotState });
|
|
249
|
-
tempDoc.apply(entry.transaction.ops);
|
|
250
|
-
snapshotState = tempDoc.get();
|
|
251
|
-
}
|
|
252
|
-
if (snapshotState === void 0) return;
|
|
253
|
-
const snapshotVersion = relevantEntries.length > 0 ? relevantEntries[relevantEntries.length - 1].version : baseVersion;
|
|
254
|
-
if (snapshotVersion <= (yield* effect.Ref.get(stateRef)).lastSnapshotVersion) return;
|
|
255
|
-
const storedDocument = {
|
|
256
|
-
state: snapshotState,
|
|
257
|
-
version: snapshotVersion,
|
|
258
|
-
schemaVersion: SCHEMA_VERSION,
|
|
259
|
-
savedAt: Date.now()
|
|
260
|
-
};
|
|
261
|
-
yield* effect.Effect.catchAll(coldStorage.save(documentId, storedDocument), (e) => effect.Effect.logError("Failed to save snapshot", {
|
|
262
|
-
documentId,
|
|
263
|
-
error: e
|
|
264
|
-
}));
|
|
265
|
-
const snapshotDuration = Date.now() - snapshotStartTime;
|
|
266
|
-
yield* effect.Metric.increment(require_Metrics.storageSnapshots);
|
|
267
|
-
yield* effect.Metric.update(require_Metrics.storageSnapshotLatency, snapshotDuration);
|
|
268
|
-
yield* effect.Ref.update(stateRef, (s) => require_objectSpread2._objectSpread2(require_objectSpread2._objectSpread2({}, s), {}, {
|
|
269
|
-
lastSnapshotVersion: snapshotVersion,
|
|
270
|
-
lastSnapshotTime: Date.now(),
|
|
271
|
-
transactionsSinceSnapshot: 0
|
|
272
|
-
}));
|
|
273
|
-
yield* effect.Effect.catchAll(hotStorage.truncate(documentId, snapshotVersion), (e) => effect.Effect.logWarning("WAL truncate failed - will retry on next snapshot", {
|
|
147
|
+
yield* effect.Effect.addFinalizer(() => effect.Effect.fn("cluster.entity.finalize")(function* () {
|
|
148
|
+
yield* effect.Effect.catchAll(instance.saveSnapshot(), (e) => effect.Effect.logError("Failed to save snapshot during entity finalization", {
|
|
274
149
|
documentId,
|
|
275
|
-
version: snapshotVersion,
|
|
276
150
|
error: e
|
|
277
151
|
}));
|
|
278
|
-
});
|
|
279
|
-
/**
|
|
280
|
-
* Check if snapshot should be triggered
|
|
281
|
-
*/
|
|
282
|
-
const checkSnapshotTriggers = effect.Effect.gen(function* () {
|
|
283
|
-
const state = yield* effect.Ref.get(stateRef);
|
|
284
|
-
const now = Date.now();
|
|
285
|
-
const currentVersion = state.document.getVersion();
|
|
286
|
-
const intervalMs = effect.Duration.toMillis(config.snapshot.interval);
|
|
287
|
-
const threshold = config.snapshot.transactionThreshold;
|
|
288
|
-
if (state.transactionsSinceSnapshot >= threshold) {
|
|
289
|
-
yield* saveSnapshot(currentVersion);
|
|
290
|
-
return;
|
|
291
|
-
}
|
|
292
|
-
if (now - state.lastSnapshotTime >= intervalMs) {
|
|
293
|
-
yield* saveSnapshot(currentVersion);
|
|
294
|
-
return;
|
|
295
|
-
}
|
|
296
|
-
});
|
|
297
|
-
yield* effect.Effect.addFinalizer(() => effect.Effect.gen(function* () {
|
|
298
|
-
yield* saveSnapshot((yield* effect.Ref.get(stateRef)).document.getVersion());
|
|
299
152
|
yield* effect.Metric.incrementBy(require_Metrics.documentsActive, -1);
|
|
300
153
|
yield* effect.Metric.increment(require_Metrics.documentsEvicted);
|
|
301
154
|
yield* effect.Effect.logDebug("Entity finalized", { documentId });
|
|
302
|
-
}));
|
|
155
|
+
})());
|
|
303
156
|
return {
|
|
304
|
-
Submit: effect.Effect.
|
|
305
|
-
const submitStartTime = Date.now();
|
|
306
|
-
const state = yield* effect.Ref.get(stateRef);
|
|
157
|
+
Submit: effect.Effect.fn("cluster.document.transaction.submit")(function* ({ payload }) {
|
|
307
158
|
const transaction = decodeTransaction(payload.transaction);
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
yield* effect.Metric.update(require_Metrics.transactionsLatency, latency$1);
|
|
313
|
-
return {
|
|
314
|
-
success: false,
|
|
315
|
-
reason: validation.reason
|
|
316
|
-
};
|
|
317
|
-
}
|
|
318
|
-
const walEntry = {
|
|
319
|
-
transaction,
|
|
320
|
-
version: validation.nextVersion,
|
|
321
|
-
timestamp: Date.now()
|
|
322
|
-
};
|
|
323
|
-
const appendResult = yield* effect.Effect.either(hotStorage.appendWithCheck(documentId, walEntry, validation.nextVersion));
|
|
324
|
-
if (appendResult._tag === "Left") {
|
|
325
|
-
yield* effect.Effect.logError("WAL append failed", {
|
|
326
|
-
documentId,
|
|
327
|
-
version: validation.nextVersion,
|
|
328
|
-
error: appendResult.left
|
|
329
|
-
});
|
|
330
|
-
yield* effect.Metric.increment(require_Metrics.walAppendFailures);
|
|
331
|
-
const latency$1 = Date.now() - submitStartTime;
|
|
332
|
-
yield* effect.Metric.update(require_Metrics.transactionsLatency, latency$1);
|
|
333
|
-
return {
|
|
334
|
-
success: false,
|
|
335
|
-
reason: "Storage unavailable. Please retry."
|
|
336
|
-
};
|
|
337
|
-
}
|
|
338
|
-
state.document.apply(transaction);
|
|
339
|
-
const latency = Date.now() - submitStartTime;
|
|
340
|
-
yield* effect.Metric.update(require_Metrics.transactionsLatency, latency);
|
|
341
|
-
yield* effect.Metric.increment(require_Metrics.transactionsProcessed);
|
|
342
|
-
yield* effect.Metric.increment(require_Metrics.storageWalAppends);
|
|
343
|
-
yield* effect.Ref.update(stateRef, (s) => require_objectSpread2._objectSpread2(require_objectSpread2._objectSpread2({}, s), {}, { transactionsSinceSnapshot: s.transactionsSinceSnapshot + 1 }));
|
|
344
|
-
yield* checkSnapshotTriggers;
|
|
345
|
-
return {
|
|
346
|
-
success: true,
|
|
347
|
-
version: validation.nextVersion
|
|
348
|
-
};
|
|
159
|
+
return yield* instance.submit(transaction).pipe(effect.Effect.catchAll((error) => effect.Effect.succeed({
|
|
160
|
+
success: false,
|
|
161
|
+
reason: `Storage error: ${String(error)}`
|
|
162
|
+
})));
|
|
349
163
|
}),
|
|
350
|
-
GetSnapshot: effect.Effect.
|
|
351
|
-
return
|
|
164
|
+
GetSnapshot: effect.Effect.fn("cluster.document.snapshot.get")(function* () {
|
|
165
|
+
return instance.getSnapshot();
|
|
352
166
|
}),
|
|
353
|
-
Touch: effect.Effect.
|
|
354
|
-
|
|
167
|
+
Touch: effect.Effect.fn("cluster.document.touch")(function* () {
|
|
168
|
+
yield* instance.touch();
|
|
169
|
+
}),
|
|
170
|
+
SetPresence: effect.Effect.fn("cluster.presence.set")(function* ({ payload }) {
|
|
355
171
|
const { connectionId, entry } = payload;
|
|
356
172
|
yield* effect.Ref.update(stateRef, (s) => require_objectSpread2._objectSpread2(require_objectSpread2._objectSpread2({}, s), {}, { presences: effect.HashMap.set(s.presences, connectionId, entry) }));
|
|
357
173
|
yield* effect.Metric.increment(require_Metrics.presenceUpdates);
|
|
@@ -365,7 +181,7 @@ const createEntityHandler = (config, coldStorage, hotStorage) => effect.Effect.g
|
|
|
365
181
|
};
|
|
366
182
|
yield* effect.PubSub.publish(state.presencePubSub, event);
|
|
367
183
|
}),
|
|
368
|
-
RemovePresence: effect.Effect.
|
|
184
|
+
RemovePresence: effect.Effect.fn("cluster.presence.remove")(function* ({ payload }) {
|
|
369
185
|
const { connectionId } = payload;
|
|
370
186
|
const state = yield* effect.Ref.get(stateRef);
|
|
371
187
|
if (!effect.HashMap.has(state.presences, connectionId)) return;
|
|
@@ -377,20 +193,20 @@ const createEntityHandler = (config, coldStorage, hotStorage) => effect.Effect.g
|
|
|
377
193
|
};
|
|
378
194
|
yield* effect.PubSub.publish(state.presencePubSub, event);
|
|
379
195
|
}),
|
|
380
|
-
GetPresenceSnapshot: effect.Effect.
|
|
196
|
+
GetPresenceSnapshot: effect.Effect.fn("cluster.presence.snapshot.get")(function* () {
|
|
381
197
|
const state = yield* effect.Ref.get(stateRef);
|
|
382
198
|
const presences = {};
|
|
383
199
|
for (const [id, entry] of state.presences) presences[id] = entry;
|
|
384
200
|
return { presences };
|
|
385
201
|
})
|
|
386
202
|
};
|
|
387
|
-
});
|
|
203
|
+
})();
|
|
388
204
|
var SubscriptionStoreTag = class extends effect.Context.Tag("@voidhash/mimic-effect/SubscriptionStore")() {};
|
|
389
|
-
const subscriptionStoreLayer = effect.Layer.effect(SubscriptionStoreTag, effect.Effect.
|
|
205
|
+
const subscriptionStoreLayer = effect.Layer.effect(SubscriptionStoreTag, effect.Effect.fn("cluster.subscriptions.layer.create")(function* () {
|
|
390
206
|
const documentPubSubs = yield* effect.Ref.make(effect.HashMap.empty());
|
|
391
207
|
const presencePubSubs = yield* effect.Ref.make(effect.HashMap.empty());
|
|
392
208
|
return {
|
|
393
|
-
getOrCreatePubSub:
|
|
209
|
+
getOrCreatePubSub: effect.Effect.fn("cluster.subscriptions.pubsub.get-or-create")(function* (documentId) {
|
|
394
210
|
const current = yield* effect.Ref.get(documentPubSubs);
|
|
395
211
|
const existing = effect.HashMap.get(current, documentId);
|
|
396
212
|
if (existing._tag === "Some") return existing.value;
|
|
@@ -398,7 +214,7 @@ const subscriptionStoreLayer = effect.Layer.effect(SubscriptionStoreTag, effect.
|
|
|
398
214
|
yield* effect.Ref.update(documentPubSubs, (map) => effect.HashMap.set(map, documentId, pubsub));
|
|
399
215
|
return pubsub;
|
|
400
216
|
}),
|
|
401
|
-
getOrCreatePresencePubSub:
|
|
217
|
+
getOrCreatePresencePubSub: effect.Effect.fn("cluster.subscriptions.presence-pubsub.get-or-create")(function* (documentId) {
|
|
402
218
|
const current = yield* effect.Ref.get(presencePubSubs);
|
|
403
219
|
const existing = effect.HashMap.get(current, documentId);
|
|
404
220
|
if (existing._tag === "Some") return existing.value;
|
|
@@ -407,7 +223,7 @@ const subscriptionStoreLayer = effect.Layer.effect(SubscriptionStoreTag, effect.
|
|
|
407
223
|
return pubsub;
|
|
408
224
|
})
|
|
409
225
|
};
|
|
410
|
-
}));
|
|
226
|
+
})());
|
|
411
227
|
/**
|
|
412
228
|
* Create a MimicClusterServerEngine layer.
|
|
413
229
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MimicClusterServerEngine.d.cts","names":[],"sources":["../src/MimicClusterServerEngine.ts"],"sourcesContent":[],"mappings":";;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"MimicClusterServerEngine.d.cts","names":[],"sources":["../src/MimicClusterServerEngine.ts"],"sourcesContent":[],"mappings":";;;;;;;;;;;cA8nBa;yBA3JwB,SAAA,CAAU,sBACrC,+BAA+B,aACtC,KAAA,CAAM,MACP,6BAEA,iBAAiB,gBAAgB,sBAAsB,QAAA,CAAS"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MimicClusterServerEngine.d.mts","names":[],"sources":["../src/MimicClusterServerEngine.ts"],"sourcesContent":[],"mappings":";;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"MimicClusterServerEngine.d.mts","names":[],"sources":["../src/MimicClusterServerEngine.ts"],"sourcesContent":[],"mappings":";;;;;;;;;;;cA8nBa;yBA3JwB,SAAA,CAAU,sBACrC,+BAA+B,aACtC,KAAA,CAAM,MACP,6BAEA,iBAAiB,gBAAgB,sBAAsB,QAAA,CAAS"}
|
|
@@ -1,12 +1,11 @@
|
|
|
1
1
|
import { __require } from "./_virtual/rolldown_runtime.mjs";
|
|
2
2
|
import { ColdStorageTag } from "./ColdStorage.mjs";
|
|
3
3
|
import { HotStorageTag } from "./HotStorage.mjs";
|
|
4
|
-
import { documentsActive,
|
|
4
|
+
import { documentsActive, documentsEvicted, presenceActive, presenceUpdates } from "./Metrics.mjs";
|
|
5
|
+
import { DocumentInstance } from "./DocumentInstance.mjs";
|
|
5
6
|
import { _objectSpread2 } from "./_virtual/_@oxc-project_runtime@0.103.0/helpers/objectSpread2.mjs";
|
|
6
7
|
import { MimicServerEngineTag } from "./MimicServerEngine.mjs";
|
|
7
8
|
import { Context, Duration, Effect, HashMap, Layer, Metric, PubSub, Ref, Schema, Stream } from "effect";
|
|
8
|
-
import { Document } from "@voidhash/mimic";
|
|
9
|
-
import { ServerDocument } from "@voidhash/mimic/server";
|
|
10
9
|
import { Entity } from "@effect/cluster";
|
|
11
10
|
import { Rpc } from "@effect/rpc";
|
|
12
11
|
|
|
@@ -119,240 +118,57 @@ const resolveClusterConfig = (config) => {
|
|
|
119
118
|
* Decode an encoded transaction to a Transaction object
|
|
120
119
|
*/
|
|
121
120
|
const decodeTransaction = (encoded) => {
|
|
122
|
-
const { Transaction
|
|
123
|
-
return Transaction
|
|
121
|
+
const { Transaction } = __require("@voidhash/mimic");
|
|
122
|
+
return Transaction.decode(encoded);
|
|
124
123
|
};
|
|
125
124
|
/**
|
|
126
125
|
* Encode a Transaction to wire format
|
|
127
126
|
*/
|
|
128
127
|
const encodeTransaction = (tx) => {
|
|
129
|
-
const { Transaction
|
|
130
|
-
return Transaction
|
|
128
|
+
const { Transaction } = __require("@voidhash/mimic");
|
|
129
|
+
return Transaction.encode(tx);
|
|
131
130
|
};
|
|
132
131
|
/**
|
|
133
132
|
* Create the entity handler for MimicDocument
|
|
134
133
|
*/
|
|
135
|
-
const createEntityHandler = (config, coldStorage, hotStorage) => Effect.
|
|
134
|
+
const createEntityHandler = (config, coldStorage, hotStorage) => Effect.fn("cluster.entity.handler.create")(function* () {
|
|
136
135
|
const documentId = (yield* Entity.CurrentAddress).entityId;
|
|
137
|
-
const
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
};
|
|
143
|
-
const storedDoc = yield* coldStorage.load(documentId).pipe(Effect.orDie);
|
|
144
|
-
let initialState;
|
|
145
|
-
let initialVersion = 0;
|
|
146
|
-
if (storedDoc) {
|
|
147
|
-
initialState = storedDoc.state;
|
|
148
|
-
initialVersion = storedDoc.version;
|
|
149
|
-
} else initialState = yield* computeInitialState();
|
|
150
|
-
const broadcastPubSub = yield* PubSub.unbounded();
|
|
136
|
+
const instance = yield* DocumentInstance.make(documentId, {
|
|
137
|
+
schema: config.schema,
|
|
138
|
+
initial: config.initial,
|
|
139
|
+
maxTransactionHistory: config.maxTransactionHistory,
|
|
140
|
+
snapshot: config.snapshot
|
|
141
|
+
}, coldStorage, hotStorage).pipe(Effect.orDie);
|
|
151
142
|
const presencePubSub = yield* PubSub.unbounded();
|
|
152
143
|
const stateRef = yield* Ref.make({
|
|
153
|
-
|
|
154
|
-
broadcastPubSub,
|
|
144
|
+
instance,
|
|
155
145
|
presences: HashMap.empty(),
|
|
156
|
-
presencePubSub
|
|
157
|
-
lastSnapshotVersion: initialVersion,
|
|
158
|
-
lastSnapshotTime: Date.now(),
|
|
159
|
-
transactionsSinceSnapshot: 0
|
|
146
|
+
presencePubSub
|
|
160
147
|
});
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
initialState,
|
|
164
|
-
initialVersion,
|
|
165
|
-
maxTransactionHistory: config.maxTransactionHistory,
|
|
166
|
-
onBroadcast: (message) => {
|
|
167
|
-
Effect.runSync(PubSub.publish(broadcastPubSub, {
|
|
168
|
-
type: "transaction",
|
|
169
|
-
transaction: message.transaction,
|
|
170
|
-
version: message.version
|
|
171
|
-
}));
|
|
172
|
-
},
|
|
173
|
-
onRejection: (transactionId, reason) => {
|
|
174
|
-
Effect.runSync(PubSub.publish(broadcastPubSub, {
|
|
175
|
-
type: "error",
|
|
176
|
-
transactionId,
|
|
177
|
-
reason
|
|
178
|
-
}));
|
|
179
|
-
}
|
|
180
|
-
});
|
|
181
|
-
yield* Ref.update(stateRef, (s) => _objectSpread2(_objectSpread2({}, s), {}, { document }));
|
|
182
|
-
const walEntries = yield* hotStorage.getEntries(documentId, initialVersion).pipe(Effect.orDie);
|
|
183
|
-
if (walEntries.length > 0) {
|
|
184
|
-
const firstWalVersion = walEntries[0].version;
|
|
185
|
-
const expectedFirst = initialVersion + 1;
|
|
186
|
-
if (firstWalVersion !== expectedFirst) {
|
|
187
|
-
yield* Effect.logWarning("WAL version gap detected", {
|
|
188
|
-
documentId,
|
|
189
|
-
snapshotVersion: initialVersion,
|
|
190
|
-
firstWalVersion,
|
|
191
|
-
expectedFirst
|
|
192
|
-
});
|
|
193
|
-
yield* Metric.increment(storageVersionGaps);
|
|
194
|
-
}
|
|
195
|
-
for (let i = 1; i < walEntries.length; i++) {
|
|
196
|
-
const prev = walEntries[i - 1].version;
|
|
197
|
-
const curr = walEntries[i].version;
|
|
198
|
-
if (curr !== prev + 1) yield* Effect.logWarning("WAL internal gap detected", {
|
|
199
|
-
documentId,
|
|
200
|
-
previousVersion: prev,
|
|
201
|
-
currentVersion: curr
|
|
202
|
-
});
|
|
203
|
-
}
|
|
204
|
-
}
|
|
205
|
-
for (const entry of walEntries) {
|
|
206
|
-
const result = document.submit(entry.transaction);
|
|
207
|
-
if (!result.success) yield* Effect.logWarning("Skipping corrupted WAL entry", {
|
|
208
|
-
documentId,
|
|
209
|
-
version: entry.version,
|
|
210
|
-
reason: result.reason
|
|
211
|
-
});
|
|
212
|
-
}
|
|
213
|
-
if (storedDoc) yield* Metric.increment(documentsRestored);
|
|
214
|
-
else yield* Metric.increment(documentsCreated);
|
|
215
|
-
yield* Metric.incrementBy(documentsActive, 1);
|
|
216
|
-
/**
|
|
217
|
-
* Save snapshot to ColdStorage derived from WAL entries.
|
|
218
|
-
* This ensures snapshots are always based on durable WAL data.
|
|
219
|
-
* Idempotent: skips save if already snapshotted at target version.
|
|
220
|
-
* Truncate failures are non-fatal and will be retried on next snapshot.
|
|
221
|
-
*/
|
|
222
|
-
const saveSnapshot = (targetVersion) => Effect.gen(function* () {
|
|
223
|
-
var _baseSnapshot$version;
|
|
224
|
-
if (targetVersion <= (yield* Ref.get(stateRef)).lastSnapshotVersion) return;
|
|
225
|
-
const snapshotStartTime = Date.now();
|
|
226
|
-
const baseSnapshotResult = yield* Effect.either(coldStorage.load(documentId));
|
|
227
|
-
if (baseSnapshotResult._tag === "Left") {
|
|
228
|
-
yield* Effect.logError("Failed to load base snapshot for WAL replay", {
|
|
229
|
-
documentId,
|
|
230
|
-
error: baseSnapshotResult.left
|
|
231
|
-
});
|
|
232
|
-
return;
|
|
233
|
-
}
|
|
234
|
-
const baseSnapshot = baseSnapshotResult.right;
|
|
235
|
-
const baseVersion = (_baseSnapshot$version = baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.version) !== null && _baseSnapshot$version !== void 0 ? _baseSnapshot$version : 0;
|
|
236
|
-
const baseState = baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.state;
|
|
237
|
-
const walEntriesResult = yield* Effect.either(hotStorage.getEntries(documentId, baseVersion));
|
|
238
|
-
if (walEntriesResult._tag === "Left") {
|
|
239
|
-
yield* Effect.logError("Failed to load WAL entries for snapshot", {
|
|
240
|
-
documentId,
|
|
241
|
-
error: walEntriesResult.left
|
|
242
|
-
});
|
|
243
|
-
return;
|
|
244
|
-
}
|
|
245
|
-
const relevantEntries = walEntriesResult.right.filter((e) => e.version <= targetVersion);
|
|
246
|
-
if (relevantEntries.length === 0 && !baseSnapshot) return;
|
|
247
|
-
let snapshotState = baseState;
|
|
248
|
-
for (const entry of relevantEntries) {
|
|
249
|
-
const tempDoc = Document.make(config.schema, { initialState: snapshotState });
|
|
250
|
-
tempDoc.apply(entry.transaction.ops);
|
|
251
|
-
snapshotState = tempDoc.get();
|
|
252
|
-
}
|
|
253
|
-
if (snapshotState === void 0) return;
|
|
254
|
-
const snapshotVersion = relevantEntries.length > 0 ? relevantEntries[relevantEntries.length - 1].version : baseVersion;
|
|
255
|
-
if (snapshotVersion <= (yield* Ref.get(stateRef)).lastSnapshotVersion) return;
|
|
256
|
-
const storedDocument = {
|
|
257
|
-
state: snapshotState,
|
|
258
|
-
version: snapshotVersion,
|
|
259
|
-
schemaVersion: SCHEMA_VERSION,
|
|
260
|
-
savedAt: Date.now()
|
|
261
|
-
};
|
|
262
|
-
yield* Effect.catchAll(coldStorage.save(documentId, storedDocument), (e) => Effect.logError("Failed to save snapshot", {
|
|
263
|
-
documentId,
|
|
264
|
-
error: e
|
|
265
|
-
}));
|
|
266
|
-
const snapshotDuration = Date.now() - snapshotStartTime;
|
|
267
|
-
yield* Metric.increment(storageSnapshots);
|
|
268
|
-
yield* Metric.update(storageSnapshotLatency, snapshotDuration);
|
|
269
|
-
yield* Ref.update(stateRef, (s) => _objectSpread2(_objectSpread2({}, s), {}, {
|
|
270
|
-
lastSnapshotVersion: snapshotVersion,
|
|
271
|
-
lastSnapshotTime: Date.now(),
|
|
272
|
-
transactionsSinceSnapshot: 0
|
|
273
|
-
}));
|
|
274
|
-
yield* Effect.catchAll(hotStorage.truncate(documentId, snapshotVersion), (e) => Effect.logWarning("WAL truncate failed - will retry on next snapshot", {
|
|
148
|
+
yield* Effect.addFinalizer(() => Effect.fn("cluster.entity.finalize")(function* () {
|
|
149
|
+
yield* Effect.catchAll(instance.saveSnapshot(), (e) => Effect.logError("Failed to save snapshot during entity finalization", {
|
|
275
150
|
documentId,
|
|
276
|
-
version: snapshotVersion,
|
|
277
151
|
error: e
|
|
278
152
|
}));
|
|
279
|
-
});
|
|
280
|
-
/**
|
|
281
|
-
* Check if snapshot should be triggered
|
|
282
|
-
*/
|
|
283
|
-
const checkSnapshotTriggers = Effect.gen(function* () {
|
|
284
|
-
const state = yield* Ref.get(stateRef);
|
|
285
|
-
const now = Date.now();
|
|
286
|
-
const currentVersion = state.document.getVersion();
|
|
287
|
-
const intervalMs = Duration.toMillis(config.snapshot.interval);
|
|
288
|
-
const threshold = config.snapshot.transactionThreshold;
|
|
289
|
-
if (state.transactionsSinceSnapshot >= threshold) {
|
|
290
|
-
yield* saveSnapshot(currentVersion);
|
|
291
|
-
return;
|
|
292
|
-
}
|
|
293
|
-
if (now - state.lastSnapshotTime >= intervalMs) {
|
|
294
|
-
yield* saveSnapshot(currentVersion);
|
|
295
|
-
return;
|
|
296
|
-
}
|
|
297
|
-
});
|
|
298
|
-
yield* Effect.addFinalizer(() => Effect.gen(function* () {
|
|
299
|
-
yield* saveSnapshot((yield* Ref.get(stateRef)).document.getVersion());
|
|
300
153
|
yield* Metric.incrementBy(documentsActive, -1);
|
|
301
154
|
yield* Metric.increment(documentsEvicted);
|
|
302
155
|
yield* Effect.logDebug("Entity finalized", { documentId });
|
|
303
|
-
}));
|
|
156
|
+
})());
|
|
304
157
|
return {
|
|
305
|
-
Submit: Effect.
|
|
306
|
-
const submitStartTime = Date.now();
|
|
307
|
-
const state = yield* Ref.get(stateRef);
|
|
158
|
+
Submit: Effect.fn("cluster.document.transaction.submit")(function* ({ payload }) {
|
|
308
159
|
const transaction = decodeTransaction(payload.transaction);
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
yield* Metric.update(transactionsLatency, latency$1);
|
|
314
|
-
return {
|
|
315
|
-
success: false,
|
|
316
|
-
reason: validation.reason
|
|
317
|
-
};
|
|
318
|
-
}
|
|
319
|
-
const walEntry = {
|
|
320
|
-
transaction,
|
|
321
|
-
version: validation.nextVersion,
|
|
322
|
-
timestamp: Date.now()
|
|
323
|
-
};
|
|
324
|
-
const appendResult = yield* Effect.either(hotStorage.appendWithCheck(documentId, walEntry, validation.nextVersion));
|
|
325
|
-
if (appendResult._tag === "Left") {
|
|
326
|
-
yield* Effect.logError("WAL append failed", {
|
|
327
|
-
documentId,
|
|
328
|
-
version: validation.nextVersion,
|
|
329
|
-
error: appendResult.left
|
|
330
|
-
});
|
|
331
|
-
yield* Metric.increment(walAppendFailures);
|
|
332
|
-
const latency$1 = Date.now() - submitStartTime;
|
|
333
|
-
yield* Metric.update(transactionsLatency, latency$1);
|
|
334
|
-
return {
|
|
335
|
-
success: false,
|
|
336
|
-
reason: "Storage unavailable. Please retry."
|
|
337
|
-
};
|
|
338
|
-
}
|
|
339
|
-
state.document.apply(transaction);
|
|
340
|
-
const latency = Date.now() - submitStartTime;
|
|
341
|
-
yield* Metric.update(transactionsLatency, latency);
|
|
342
|
-
yield* Metric.increment(transactionsProcessed);
|
|
343
|
-
yield* Metric.increment(storageWalAppends);
|
|
344
|
-
yield* Ref.update(stateRef, (s) => _objectSpread2(_objectSpread2({}, s), {}, { transactionsSinceSnapshot: s.transactionsSinceSnapshot + 1 }));
|
|
345
|
-
yield* checkSnapshotTriggers;
|
|
346
|
-
return {
|
|
347
|
-
success: true,
|
|
348
|
-
version: validation.nextVersion
|
|
349
|
-
};
|
|
160
|
+
return yield* instance.submit(transaction).pipe(Effect.catchAll((error) => Effect.succeed({
|
|
161
|
+
success: false,
|
|
162
|
+
reason: `Storage error: ${String(error)}`
|
|
163
|
+
})));
|
|
350
164
|
}),
|
|
351
|
-
GetSnapshot: Effect.
|
|
352
|
-
return
|
|
165
|
+
GetSnapshot: Effect.fn("cluster.document.snapshot.get")(function* () {
|
|
166
|
+
return instance.getSnapshot();
|
|
353
167
|
}),
|
|
354
|
-
Touch: Effect.
|
|
355
|
-
|
|
168
|
+
Touch: Effect.fn("cluster.document.touch")(function* () {
|
|
169
|
+
yield* instance.touch();
|
|
170
|
+
}),
|
|
171
|
+
SetPresence: Effect.fn("cluster.presence.set")(function* ({ payload }) {
|
|
356
172
|
const { connectionId, entry } = payload;
|
|
357
173
|
yield* Ref.update(stateRef, (s) => _objectSpread2(_objectSpread2({}, s), {}, { presences: HashMap.set(s.presences, connectionId, entry) }));
|
|
358
174
|
yield* Metric.increment(presenceUpdates);
|
|
@@ -366,7 +182,7 @@ const createEntityHandler = (config, coldStorage, hotStorage) => Effect.gen(func
|
|
|
366
182
|
};
|
|
367
183
|
yield* PubSub.publish(state.presencePubSub, event);
|
|
368
184
|
}),
|
|
369
|
-
RemovePresence: Effect.
|
|
185
|
+
RemovePresence: Effect.fn("cluster.presence.remove")(function* ({ payload }) {
|
|
370
186
|
const { connectionId } = payload;
|
|
371
187
|
const state = yield* Ref.get(stateRef);
|
|
372
188
|
if (!HashMap.has(state.presences, connectionId)) return;
|
|
@@ -378,20 +194,20 @@ const createEntityHandler = (config, coldStorage, hotStorage) => Effect.gen(func
|
|
|
378
194
|
};
|
|
379
195
|
yield* PubSub.publish(state.presencePubSub, event);
|
|
380
196
|
}),
|
|
381
|
-
GetPresenceSnapshot: Effect.
|
|
197
|
+
GetPresenceSnapshot: Effect.fn("cluster.presence.snapshot.get")(function* () {
|
|
382
198
|
const state = yield* Ref.get(stateRef);
|
|
383
199
|
const presences = {};
|
|
384
200
|
for (const [id, entry] of state.presences) presences[id] = entry;
|
|
385
201
|
return { presences };
|
|
386
202
|
})
|
|
387
203
|
};
|
|
388
|
-
});
|
|
204
|
+
})();
|
|
389
205
|
var SubscriptionStoreTag = class extends Context.Tag("@voidhash/mimic-effect/SubscriptionStore")() {};
|
|
390
|
-
const subscriptionStoreLayer = Layer.effect(SubscriptionStoreTag, Effect.
|
|
206
|
+
const subscriptionStoreLayer = Layer.effect(SubscriptionStoreTag, Effect.fn("cluster.subscriptions.layer.create")(function* () {
|
|
391
207
|
const documentPubSubs = yield* Ref.make(HashMap.empty());
|
|
392
208
|
const presencePubSubs = yield* Ref.make(HashMap.empty());
|
|
393
209
|
return {
|
|
394
|
-
getOrCreatePubSub:
|
|
210
|
+
getOrCreatePubSub: Effect.fn("cluster.subscriptions.pubsub.get-or-create")(function* (documentId) {
|
|
395
211
|
const current = yield* Ref.get(documentPubSubs);
|
|
396
212
|
const existing = HashMap.get(current, documentId);
|
|
397
213
|
if (existing._tag === "Some") return existing.value;
|
|
@@ -399,7 +215,7 @@ const subscriptionStoreLayer = Layer.effect(SubscriptionStoreTag, Effect.gen(fun
|
|
|
399
215
|
yield* Ref.update(documentPubSubs, (map) => HashMap.set(map, documentId, pubsub));
|
|
400
216
|
return pubsub;
|
|
401
217
|
}),
|
|
402
|
-
getOrCreatePresencePubSub:
|
|
218
|
+
getOrCreatePresencePubSub: Effect.fn("cluster.subscriptions.presence-pubsub.get-or-create")(function* (documentId) {
|
|
403
219
|
const current = yield* Ref.get(presencePubSubs);
|
|
404
220
|
const existing = HashMap.get(current, documentId);
|
|
405
221
|
if (existing._tag === "Some") return existing.value;
|
|
@@ -408,7 +224,7 @@ const subscriptionStoreLayer = Layer.effect(SubscriptionStoreTag, Effect.gen(fun
|
|
|
408
224
|
return pubsub;
|
|
409
225
|
})
|
|
410
226
|
};
|
|
411
|
-
}));
|
|
227
|
+
})());
|
|
412
228
|
/**
|
|
413
229
|
* Create a MimicClusterServerEngine layer.
|
|
414
230
|
*
|