@voidhash/mimic-effect 1.0.0-beta.1 → 1.0.0-beta.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. package/.turbo/turbo-build.log +116 -74
  2. package/dist/ColdStorage.cjs +9 -5
  3. package/dist/ColdStorage.d.cts.map +1 -1
  4. package/dist/ColdStorage.d.mts.map +1 -1
  5. package/dist/ColdStorage.mjs +9 -5
  6. package/dist/ColdStorage.mjs.map +1 -1
  7. package/dist/DocumentInstance.cjs +263 -0
  8. package/dist/DocumentInstance.d.cts +78 -0
  9. package/dist/DocumentInstance.d.cts.map +1 -0
  10. package/dist/DocumentInstance.d.mts +78 -0
  11. package/dist/DocumentInstance.d.mts.map +1 -0
  12. package/dist/DocumentInstance.mjs +264 -0
  13. package/dist/DocumentInstance.mjs.map +1 -0
  14. package/dist/Errors.cjs +10 -1
  15. package/dist/Errors.d.cts +18 -3
  16. package/dist/Errors.d.cts.map +1 -1
  17. package/dist/Errors.d.mts +18 -3
  18. package/dist/Errors.d.mts.map +1 -1
  19. package/dist/Errors.mjs +9 -1
  20. package/dist/Errors.mjs.map +1 -1
  21. package/dist/HotStorage.cjs +39 -12
  22. package/dist/HotStorage.d.cts +17 -1
  23. package/dist/HotStorage.d.cts.map +1 -1
  24. package/dist/HotStorage.d.mts +17 -1
  25. package/dist/HotStorage.d.mts.map +1 -1
  26. package/dist/HotStorage.mjs +39 -12
  27. package/dist/HotStorage.mjs.map +1 -1
  28. package/dist/Metrics.cjs +29 -1
  29. package/dist/Metrics.d.cts +5 -0
  30. package/dist/Metrics.d.cts.map +1 -1
  31. package/dist/Metrics.d.mts +5 -0
  32. package/dist/Metrics.d.mts.map +1 -1
  33. package/dist/Metrics.mjs +26 -1
  34. package/dist/Metrics.mjs.map +1 -1
  35. package/dist/MimicClusterServerEngine.cjs +44 -139
  36. package/dist/MimicClusterServerEngine.d.cts.map +1 -1
  37. package/dist/MimicClusterServerEngine.d.mts +1 -1
  38. package/dist/MimicClusterServerEngine.d.mts.map +1 -1
  39. package/dist/MimicClusterServerEngine.mjs +46 -141
  40. package/dist/MimicClusterServerEngine.mjs.map +1 -1
  41. package/dist/MimicServer.cjs +20 -20
  42. package/dist/MimicServer.d.cts.map +1 -1
  43. package/dist/MimicServer.d.mts.map +1 -1
  44. package/dist/MimicServer.mjs +20 -20
  45. package/dist/MimicServer.mjs.map +1 -1
  46. package/dist/MimicServerEngine.cjs +92 -11
  47. package/dist/MimicServerEngine.d.cts +12 -4
  48. package/dist/MimicServerEngine.d.cts.map +1 -1
  49. package/dist/MimicServerEngine.d.mts +12 -4
  50. package/dist/MimicServerEngine.d.mts.map +1 -1
  51. package/dist/MimicServerEngine.mjs +94 -13
  52. package/dist/MimicServerEngine.mjs.map +1 -1
  53. package/dist/PresenceManager.cjs +5 -5
  54. package/dist/PresenceManager.d.cts.map +1 -1
  55. package/dist/PresenceManager.d.mts.map +1 -1
  56. package/dist/PresenceManager.mjs +5 -5
  57. package/dist/PresenceManager.mjs.map +1 -1
  58. package/dist/Protocol.d.cts +1 -1
  59. package/dist/Protocol.d.mts +1 -1
  60. package/dist/Types.d.cts +9 -2
  61. package/dist/Types.d.cts.map +1 -1
  62. package/dist/Types.d.mts +9 -2
  63. package/dist/Types.d.mts.map +1 -1
  64. package/dist/index.cjs +5 -6
  65. package/dist/index.d.cts +3 -3
  66. package/dist/index.d.mts +3 -3
  67. package/dist/index.mjs +3 -3
  68. package/dist/testing/ColdStorageTestSuite.cjs +508 -0
  69. package/dist/testing/ColdStorageTestSuite.d.cts +36 -0
  70. package/dist/testing/ColdStorageTestSuite.d.cts.map +1 -0
  71. package/dist/testing/ColdStorageTestSuite.d.mts +36 -0
  72. package/dist/testing/ColdStorageTestSuite.d.mts.map +1 -0
  73. package/dist/testing/ColdStorageTestSuite.mjs +508 -0
  74. package/dist/testing/ColdStorageTestSuite.mjs.map +1 -0
  75. package/dist/testing/FailingStorage.cjs +162 -0
  76. package/dist/testing/FailingStorage.d.cts +43 -0
  77. package/dist/testing/FailingStorage.d.cts.map +1 -0
  78. package/dist/testing/FailingStorage.d.mts +43 -0
  79. package/dist/testing/FailingStorage.d.mts.map +1 -0
  80. package/dist/testing/FailingStorage.mjs +163 -0
  81. package/dist/testing/FailingStorage.mjs.map +1 -0
  82. package/dist/testing/HotStorageTestSuite.cjs +820 -0
  83. package/dist/testing/HotStorageTestSuite.d.cts +42 -0
  84. package/dist/testing/HotStorageTestSuite.d.cts.map +1 -0
  85. package/dist/testing/HotStorageTestSuite.d.mts +42 -0
  86. package/dist/testing/HotStorageTestSuite.d.mts.map +1 -0
  87. package/dist/testing/HotStorageTestSuite.mjs +820 -0
  88. package/dist/testing/HotStorageTestSuite.mjs.map +1 -0
  89. package/dist/testing/StorageIntegrationTestSuite.cjs +487 -0
  90. package/dist/testing/StorageIntegrationTestSuite.d.cts +37 -0
  91. package/dist/testing/StorageIntegrationTestSuite.d.cts.map +1 -0
  92. package/dist/testing/StorageIntegrationTestSuite.d.mts +37 -0
  93. package/dist/testing/StorageIntegrationTestSuite.d.mts.map +1 -0
  94. package/dist/testing/StorageIntegrationTestSuite.mjs +487 -0
  95. package/dist/testing/StorageIntegrationTestSuite.mjs.map +1 -0
  96. package/dist/testing/assertions.cjs +117 -0
  97. package/dist/testing/assertions.mjs +112 -0
  98. package/dist/testing/assertions.mjs.map +1 -0
  99. package/dist/testing/index.cjs +14 -0
  100. package/dist/testing/index.d.cts +6 -0
  101. package/dist/testing/index.d.mts +6 -0
  102. package/dist/testing/index.mjs +7 -0
  103. package/dist/testing/types.cjs +15 -0
  104. package/dist/testing/types.d.cts +90 -0
  105. package/dist/testing/types.d.cts.map +1 -0
  106. package/dist/testing/types.d.mts +90 -0
  107. package/dist/testing/types.d.mts.map +1 -0
  108. package/dist/testing/types.mjs +16 -0
  109. package/dist/testing/types.mjs.map +1 -0
  110. package/package.json +8 -3
  111. package/src/ColdStorage.ts +21 -12
  112. package/src/DocumentInstance.ts +527 -0
  113. package/src/Errors.ts +15 -1
  114. package/src/HotStorage.ts +115 -24
  115. package/src/Metrics.ts +30 -0
  116. package/src/MimicClusterServerEngine.ts +120 -275
  117. package/src/MimicServer.ts +83 -75
  118. package/src/MimicServerEngine.ts +230 -30
  119. package/src/PresenceManager.ts +44 -34
  120. package/src/Types.ts +9 -2
  121. package/src/index.ts +5 -35
  122. package/src/testing/ColdStorageTestSuite.ts +589 -0
  123. package/src/testing/FailingStorage.ts +338 -0
  124. package/src/testing/HotStorageTestSuite.ts +1105 -0
  125. package/src/testing/StorageIntegrationTestSuite.ts +736 -0
  126. package/src/testing/assertions.ts +188 -0
  127. package/src/testing/index.ts +83 -0
  128. package/src/testing/types.ts +100 -0
  129. package/tests/ColdStorage.test.ts +8 -120
  130. package/tests/DocumentInstance.test.ts +669 -0
  131. package/tests/HotStorage.test.ts +7 -126
  132. package/tests/StorageIntegration.test.ts +259 -0
  133. package/tsdown.config.ts +1 -1
  134. package/dist/DocumentManager.cjs +0 -229
  135. package/dist/DocumentManager.d.cts +0 -59
  136. package/dist/DocumentManager.d.cts.map +0 -1
  137. package/dist/DocumentManager.d.mts +0 -59
  138. package/dist/DocumentManager.d.mts.map +0 -1
  139. package/dist/DocumentManager.mjs +0 -227
  140. package/dist/DocumentManager.mjs.map +0 -1
  141. package/src/DocumentManager.ts +0 -506
  142. package/tests/DocumentManager.test.ts +0 -335
@@ -0,0 +1,263 @@
1
+ const require_Metrics = require('./Metrics.cjs');
2
+ let effect = require("effect");
3
+ let _voidhash_mimic = require("@voidhash/mimic");
4
+ let _voidhash_mimic_server = require("@voidhash/mimic/server");
5
+
6
+ //#region src/DocumentInstance.ts
7
+ /**
8
+ * @voidhash/mimic-effect - DocumentInstance
9
+ *
10
+ * Manages the lifecycle of a single document including:
11
+ * - Restoration from storage (cold storage + WAL replay)
12
+ * - Transaction submission with WAL persistence
13
+ * - Snapshot saving and trigger checking
14
+ *
15
+ * Used by both MimicServerEngine (single-node) and MimicClusterServerEngine (clustered).
16
+ */
17
+ /**
18
+ * Create a DocumentInstance for a single document.
19
+ *
20
+ * This handles:
21
+ * - Loading from cold storage or computing initial state
22
+ * - Persisting initial state immediately (crash safety)
23
+ * - Replaying WAL entries
24
+ * - Transaction submission with WAL persistence
25
+ * - Snapshot saving
26
+ */
27
+ const make = (documentId, config, coldStorage, hotStorage) => effect.Effect.gen(function* () {
28
+ const SCHEMA_VERSION = 1;
29
+ const storedDoc = yield* coldStorage.load(documentId);
30
+ let initialState;
31
+ let initial;
32
+ let initialVersion = 0;
33
+ if (storedDoc) {
34
+ initialState = storedDoc.state;
35
+ initialVersion = storedDoc.version;
36
+ } else initial = yield* computeInitialState(config, documentId);
37
+ const pubsub = yield* effect.PubSub.unbounded();
38
+ const lastSnapshotVersionRef = yield* effect.Ref.make(initialVersion);
39
+ const lastSnapshotTimeRef = yield* effect.Ref.make(Date.now());
40
+ const transactionsSinceSnapshotRef = yield* effect.Ref.make(0);
41
+ const lastActivityTimeRef = yield* effect.Ref.make(Date.now());
42
+ const document = _voidhash_mimic_server.ServerDocument.make({
43
+ schema: config.schema,
44
+ initial,
45
+ initialState,
46
+ initialVersion,
47
+ maxTransactionHistory: config.maxTransactionHistory,
48
+ onBroadcast: (message) => {
49
+ effect.Effect.runSync(effect.PubSub.publish(pubsub, {
50
+ type: "transaction",
51
+ transaction: message.transaction,
52
+ version: message.version
53
+ }));
54
+ },
55
+ onRejection: (transactionId, reason) => {
56
+ effect.Effect.runSync(effect.PubSub.publish(pubsub, {
57
+ type: "error",
58
+ transactionId,
59
+ reason
60
+ }));
61
+ }
62
+ });
63
+ if (!storedDoc) {
64
+ const initialStoredDoc = createStoredDocument(document.get(), 0, SCHEMA_VERSION);
65
+ yield* coldStorage.save(documentId, initialStoredDoc);
66
+ yield* effect.Effect.logDebug("Initial state persisted to cold storage", { documentId });
67
+ }
68
+ const walEntries = yield* hotStorage.getEntries(documentId, initialVersion);
69
+ yield* verifyWalContinuity(documentId, walEntries, initialVersion);
70
+ yield* replayWalEntries(documentId, document, walEntries);
71
+ if (storedDoc) yield* effect.Metric.increment(require_Metrics.documentsRestored);
72
+ else yield* effect.Metric.increment(require_Metrics.documentsCreated);
73
+ yield* effect.Metric.incrementBy(require_Metrics.documentsActive, 1);
74
+ const getSnapshotTracking = effect.Effect.gen(function* () {
75
+ return {
76
+ lastSnapshotVersion: yield* effect.Ref.get(lastSnapshotVersionRef),
77
+ lastSnapshotTime: yield* effect.Ref.get(lastSnapshotTimeRef),
78
+ transactionsSinceSnapshot: yield* effect.Ref.get(transactionsSinceSnapshotRef)
79
+ };
80
+ });
81
+ const saveSnapshot = effect.Effect.fn("document.snapshot.save")(function* () {
82
+ var _baseSnapshot$version;
83
+ const targetVersion = document.getVersion();
84
+ if (targetVersion <= (yield* effect.Ref.get(lastSnapshotVersionRef))) return;
85
+ const snapshotStartTime = Date.now();
86
+ const baseSnapshot = yield* coldStorage.load(documentId);
87
+ const baseVersion = (_baseSnapshot$version = baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.version) !== null && _baseSnapshot$version !== void 0 ? _baseSnapshot$version : 0;
88
+ const baseState = baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.state;
89
+ const walEntries$1 = yield* hotStorage.getEntries(documentId, baseVersion);
90
+ const snapshotResult = computeSnapshotState(config.schema, baseState, walEntries$1, targetVersion);
91
+ if (!snapshotResult) return;
92
+ const currentLastSnapshot = yield* effect.Ref.get(lastSnapshotVersionRef);
93
+ if (snapshotResult.version <= currentLastSnapshot) return;
94
+ const storedDoc$1 = createStoredDocument(snapshotResult.state, snapshotResult.version, SCHEMA_VERSION);
95
+ yield* coldStorage.save(documentId, storedDoc$1);
96
+ const snapshotDuration = Date.now() - snapshotStartTime;
97
+ yield* effect.Metric.increment(require_Metrics.storageSnapshots);
98
+ yield* effect.Metric.update(require_Metrics.storageSnapshotLatency, snapshotDuration);
99
+ yield* effect.Ref.set(lastSnapshotVersionRef, snapshotResult.version);
100
+ yield* effect.Ref.set(lastSnapshotTimeRef, Date.now());
101
+ yield* effect.Ref.set(transactionsSinceSnapshotRef, 0);
102
+ yield* effect.Effect.catchAll(hotStorage.truncate(documentId, snapshotResult.version), (e) => effect.Effect.logWarning("WAL truncate failed - will retry on next snapshot", {
103
+ documentId,
104
+ version: snapshotResult.version,
105
+ error: e
106
+ }));
107
+ });
108
+ const checkSnapshotTriggers = effect.Effect.fn("document.snapshot.check-triggers")(function* () {
109
+ if (shouldTriggerSnapshot(yield* effect.Ref.get(transactionsSinceSnapshotRef), yield* effect.Ref.get(lastSnapshotTimeRef), config.snapshot)) yield* saveSnapshot();
110
+ });
111
+ const submit = effect.Effect.fn("document.transaction.submit")(function* (transaction) {
112
+ const submitStartTime = Date.now();
113
+ yield* effect.Ref.set(lastActivityTimeRef, Date.now());
114
+ const validation = document.validate(transaction);
115
+ if (!validation.valid) {
116
+ yield* effect.Metric.increment(require_Metrics.transactionsRejected);
117
+ const latency$1 = Date.now() - submitStartTime;
118
+ yield* effect.Metric.update(require_Metrics.transactionsLatency, latency$1);
119
+ return {
120
+ success: false,
121
+ reason: validation.reason
122
+ };
123
+ }
124
+ const walEntry = {
125
+ transaction,
126
+ version: validation.nextVersion,
127
+ timestamp: Date.now()
128
+ };
129
+ const appendResult = yield* effect.Effect.either(hotStorage.appendWithCheck(documentId, walEntry, validation.nextVersion));
130
+ if (appendResult._tag === "Left") {
131
+ yield* effect.Effect.logError("WAL append failed", {
132
+ documentId,
133
+ version: validation.nextVersion,
134
+ error: appendResult.left
135
+ });
136
+ yield* effect.Metric.increment(require_Metrics.walAppendFailures);
137
+ const latency$1 = Date.now() - submitStartTime;
138
+ yield* effect.Metric.update(require_Metrics.transactionsLatency, latency$1);
139
+ return {
140
+ success: false,
141
+ reason: "Storage unavailable. Please retry."
142
+ };
143
+ }
144
+ document.apply(transaction);
145
+ const latency = Date.now() - submitStartTime;
146
+ yield* effect.Metric.update(require_Metrics.transactionsLatency, latency);
147
+ yield* effect.Metric.increment(require_Metrics.transactionsProcessed);
148
+ yield* effect.Metric.increment(require_Metrics.storageWalAppends);
149
+ yield* effect.Ref.update(transactionsSinceSnapshotRef, (n) => n + 1);
150
+ yield* checkSnapshotTriggers();
151
+ return {
152
+ success: true,
153
+ version: validation.nextVersion
154
+ };
155
+ });
156
+ const touch = effect.Effect.fn("document.touch")(function* () {
157
+ yield* effect.Ref.set(lastActivityTimeRef, Date.now());
158
+ });
159
+ const needsSnapshot = () => effect.Effect.map(effect.Ref.get(transactionsSinceSnapshotRef), (n) => n > 0);
160
+ const getLastActivityTime = () => effect.Ref.get(lastActivityTimeRef);
161
+ return {
162
+ document,
163
+ pubsub,
164
+ getSnapshotTracking,
165
+ submit,
166
+ saveSnapshot,
167
+ checkSnapshotTriggers,
168
+ touch,
169
+ getVersion: () => document.getVersion(),
170
+ getSnapshot: () => document.getSnapshot(),
171
+ needsSnapshot,
172
+ getLastActivityTime
173
+ };
174
+ });
175
+ /**
176
+ * Compute initial state for a new document.
177
+ */
178
+ const computeInitialState = (config, documentId) => {
179
+ if (config.initial === void 0) return effect.Effect.succeed(void 0);
180
+ if (typeof config.initial === "function") return config.initial({ documentId });
181
+ return effect.Effect.succeed(config.initial);
182
+ };
183
+ /**
184
+ * Verify WAL continuity and log warnings for any gaps.
185
+ */
186
+ const verifyWalContinuity = effect.Effect.fn("document.wal.verify")(function* (documentId, walEntries, baseVersion) {
187
+ if (walEntries.length === 0) return;
188
+ const firstWalVersion = walEntries[0].version;
189
+ const expectedFirst = baseVersion + 1;
190
+ if (firstWalVersion !== expectedFirst) {
191
+ yield* effect.Effect.logWarning("WAL version gap detected", {
192
+ documentId,
193
+ snapshotVersion: baseVersion,
194
+ firstWalVersion,
195
+ expectedFirst
196
+ });
197
+ yield* effect.Metric.increment(require_Metrics.storageVersionGaps);
198
+ }
199
+ for (let i = 1; i < walEntries.length; i++) {
200
+ const prev = walEntries[i - 1].version;
201
+ const curr = walEntries[i].version;
202
+ if (curr !== prev + 1) yield* effect.Effect.logWarning("WAL internal gap detected", {
203
+ documentId,
204
+ previousVersion: prev,
205
+ currentVersion: curr
206
+ });
207
+ }
208
+ });
209
+ /**
210
+ * Replay WAL entries onto a ServerDocument.
211
+ */
212
+ const replayWalEntries = effect.Effect.fn("document.wal.replay")(function* (documentId, document, walEntries) {
213
+ for (const entry of walEntries) {
214
+ const result = document.submit(entry.transaction);
215
+ if (!result.success) yield* effect.Effect.logWarning("Skipping corrupted WAL entry", {
216
+ documentId,
217
+ version: entry.version,
218
+ reason: result.reason
219
+ });
220
+ }
221
+ });
222
+ /**
223
+ * Compute snapshot state by replaying WAL entries on a base state.
224
+ */
225
+ const computeSnapshotState = (schema, baseState, walEntries, targetVersion) => {
226
+ const relevantEntries = walEntries.filter((e) => e.version <= targetVersion);
227
+ if (relevantEntries.length === 0 && baseState === void 0) return;
228
+ let snapshotState = baseState;
229
+ for (const entry of relevantEntries) {
230
+ const tempDoc = _voidhash_mimic.Document.make(schema, { initialState: snapshotState });
231
+ tempDoc.apply(entry.transaction.ops);
232
+ snapshotState = tempDoc.get();
233
+ }
234
+ if (snapshotState === void 0) return;
235
+ const snapshotVersion = relevantEntries.length > 0 ? relevantEntries[relevantEntries.length - 1].version : 0;
236
+ return {
237
+ state: snapshotState,
238
+ version: snapshotVersion
239
+ };
240
+ };
241
+ /**
242
+ * Check if a snapshot should be triggered.
243
+ */
244
+ const shouldTriggerSnapshot = (transactionsSinceSnapshot, lastSnapshotTime, config) => {
245
+ const now = Date.now();
246
+ const intervalMs = effect.Duration.toMillis(config.interval);
247
+ if (transactionsSinceSnapshot >= config.transactionThreshold) return true;
248
+ if (now - lastSnapshotTime >= intervalMs) return true;
249
+ return false;
250
+ };
251
+ /**
252
+ * Create a StoredDocument for persistence.
253
+ */
254
+ const createStoredDocument = (state, version, schemaVersion) => ({
255
+ state,
256
+ version,
257
+ schemaVersion,
258
+ savedAt: Date.now()
259
+ });
260
+ const DocumentInstance = { make };
261
+
262
+ //#endregion
263
+ exports.DocumentInstance = DocumentInstance;
@@ -0,0 +1,78 @@
1
+ import { ColdStorageError, HotStorageError } from "./Errors.cjs";
2
+ import { ServerBroadcast } from "./Protocol.cjs";
3
+ import { ColdStorage } from "./ColdStorage.cjs";
4
+ import { HotStorage } from "./HotStorage.cjs";
5
+ import { Duration, Effect, PubSub } from "effect";
6
+ import { Primitive, Transaction } from "@voidhash/mimic";
7
+ import { ServerDocument } from "@voidhash/mimic/server";
8
+
9
+ //#region src/DocumentInstance.d.ts
10
+
11
+ /**
12
+ * Result of submitting a transaction
13
+ */
14
+ type SubmitResult = {
15
+ readonly success: true;
16
+ readonly version: number;
17
+ } | {
18
+ readonly success: false;
19
+ readonly reason: string;
20
+ };
21
+ /**
22
+ * Configuration for a DocumentInstance
23
+ */
24
+ interface DocumentInstanceConfig<TSchema extends Primitive.AnyPrimitive> {
25
+ readonly schema: TSchema;
26
+ readonly initial?: Primitive.InferSetInput<TSchema> | ((ctx: {
27
+ documentId: string;
28
+ }) => Effect.Effect<Primitive.InferSetInput<TSchema>>);
29
+ readonly maxTransactionHistory: number;
30
+ readonly snapshot: {
31
+ readonly interval: Duration.Duration;
32
+ readonly transactionThreshold: number;
33
+ };
34
+ }
35
+ /**
36
+ * Snapshot tracking state
37
+ */
38
+ interface SnapshotTracking {
39
+ readonly lastSnapshotVersion: number;
40
+ readonly lastSnapshotTime: number;
41
+ readonly transactionsSinceSnapshot: number;
42
+ }
43
+ /**
44
+ * A DocumentInstance manages a single document's lifecycle
45
+ */
46
+ interface DocumentInstance<TSchema extends Primitive.AnyPrimitive> {
47
+ /** The underlying ServerDocument */
48
+ readonly document: ServerDocument.ServerDocument<TSchema>;
49
+ /** PubSub for broadcasting messages to subscribers */
50
+ readonly pubsub: PubSub.PubSub<ServerBroadcast>;
51
+ /** Current snapshot tracking state */
52
+ readonly getSnapshotTracking: Effect.Effect<SnapshotTracking>;
53
+ /** Submit a transaction */
54
+ readonly submit: (transaction: Transaction.Transaction) => Effect.Effect<SubmitResult, ColdStorageError | HotStorageError>;
55
+ /** Save a snapshot to cold storage */
56
+ readonly saveSnapshot: () => Effect.Effect<void, ColdStorageError | HotStorageError>;
57
+ /** Check if snapshot should be triggered and save if needed */
58
+ readonly checkSnapshotTriggers: () => Effect.Effect<void, ColdStorageError | HotStorageError>;
59
+ /** Update last activity time (for external tracking) */
60
+ readonly touch: () => Effect.Effect<void>;
61
+ /** Get current document version */
62
+ readonly getVersion: () => number;
63
+ /** Get document snapshot */
64
+ readonly getSnapshot: () => {
65
+ state: unknown;
66
+ version: number;
67
+ };
68
+ /** Check if document has unsnapshot transactions that need persisting */
69
+ readonly needsSnapshot: () => Effect.Effect<boolean>;
70
+ /** Get the last activity timestamp for idle detection */
71
+ readonly getLastActivityTime: () => Effect.Effect<number>;
72
+ }
73
+ declare const DocumentInstance: {
74
+ make: <TSchema extends Primitive.AnyPrimitive>(documentId: string, config: DocumentInstanceConfig<TSchema>, coldStorage: ColdStorage, hotStorage: HotStorage) => Effect.Effect<DocumentInstance<TSchema>, ColdStorageError | HotStorageError>;
75
+ };
76
+ //#endregion
77
+ export { DocumentInstance, SubmitResult };
78
+ //# sourceMappingURL=DocumentInstance.d.cts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"DocumentInstance.d.cts","names":[],"sources":["../src/DocumentInstance.ts"],"sourcesContent":[],"mappings":";;;;;;;;;;;;AAkDA;AASiB,KA/BL,YAAA,GA+BqB;EAAiB,SAAU,OAAA,EAAA,IAAA;EAET,SAAA,OAAA,EAAA,MAAA;CAA9B,GAAA;EAEY,SAAA,OAAA,EAAA,KAAA;EAAd,SAAO,MAAA,EAAA,MAAA;CAEoB;;;;AAE2C,UAhCxE,sBAgCwE,CAAA,gBAhCjC,SAAA,CAAU,YAgCuB,CAAA,CAAA;EAAmB,SAAA,MAAA,EA/BzF,OA+ByF;EAA/C,SAAO,OAAA,CAAA,EA7B9D,SAAA,CAAU,aA6BoD,CA7BtC,OA6BsC,CAAA,GAAA,CAAA,CAAA,GAAA,EAAA;IAEjB,UAAA,EAAA,MAAA;EAAmB,CAAA,EAAA,GA9B9B,MAAA,CAAO,MA8BuB,CA9BhB,SAAA,CAAU,aA8BM,CA9BQ,OA8BR,CAAA,CAAA,CAAA;EAAvC,SAAO,qBAAA,EAAA,MAAA;EAEsB,SAAA,QAAA,EAAA;IAAmB,SAAA,QAAA,EA7BxD,QAAA,CAAS,QA6B+C;IAAhC,SAAA,oBAAA,EAAA,MAAA;EAEvB,CAAA;;;;AAmcxB;AA1aqC,UAhDpB,gBAAA,CAgD8B;EAEd,SAAA,mBAAA,EAAA,MAAA;EAAvB,SAAA,gBAAA,EAAA,MAAA;EACK,SAAA,yBAAA,EAAA,MAAA;;;;;AAEgD,UA5C9C,gBA4C8C,CAAA,gBA5Cb,SAAA,CAAU,YA4CG,CAAA,CAAA;EAA5D;EAAa,SAAA,QAAA,EA1CK,cAAA,CAAe,cA0CpB,CA1CmC,OA0CnC,CAAA;;mBAxCG,MAAA,CAAO,OAAO;;gCAED,MAAA,CAAO,OAAO;;iCAEb,WAAA,CAAY,gBAAgB,MAAA,CAAO,OAAO,cAAc,mBAAmB;;+BAE7E,MAAA,CAAO,aAAa,mBAAmB;;wCAE9B,MAAA,CAAO,aAAa,mBAAmB;;wBAEvD,MAAA,CAAO;;;;;;;;;gCAMC,MAAA,CAAO;;sCAED,MAAA,CAAO;;cA2bhC;yBA1awB,SAAA,CAAU,0CAErC,uBAAuB,uBAClB,yBACD,eACX,MAAA,CAAO,OAAO,iBAAiB,UAAU,mBAAmB"}
@@ -0,0 +1,78 @@
1
+ import { ColdStorageError, HotStorageError } from "./Errors.mjs";
2
+ import { ServerBroadcast } from "./Protocol.mjs";
3
+ import { ColdStorage } from "./ColdStorage.mjs";
4
+ import { HotStorage } from "./HotStorage.mjs";
5
+ import { Duration, Effect, PubSub } from "effect";
6
+ import { Primitive, Transaction } from "@voidhash/mimic";
7
+ import { ServerDocument } from "@voidhash/mimic/server";
8
+
9
+ //#region src/DocumentInstance.d.ts
10
+
11
+ /**
12
+ * Result of submitting a transaction
13
+ */
14
+ type SubmitResult = {
15
+ readonly success: true;
16
+ readonly version: number;
17
+ } | {
18
+ readonly success: false;
19
+ readonly reason: string;
20
+ };
21
+ /**
22
+ * Configuration for a DocumentInstance
23
+ */
24
+ interface DocumentInstanceConfig<TSchema extends Primitive.AnyPrimitive> {
25
+ readonly schema: TSchema;
26
+ readonly initial?: Primitive.InferSetInput<TSchema> | ((ctx: {
27
+ documentId: string;
28
+ }) => Effect.Effect<Primitive.InferSetInput<TSchema>>);
29
+ readonly maxTransactionHistory: number;
30
+ readonly snapshot: {
31
+ readonly interval: Duration.Duration;
32
+ readonly transactionThreshold: number;
33
+ };
34
+ }
35
+ /**
36
+ * Snapshot tracking state
37
+ */
38
+ interface SnapshotTracking {
39
+ readonly lastSnapshotVersion: number;
40
+ readonly lastSnapshotTime: number;
41
+ readonly transactionsSinceSnapshot: number;
42
+ }
43
+ /**
44
+ * A DocumentInstance manages a single document's lifecycle
45
+ */
46
+ interface DocumentInstance<TSchema extends Primitive.AnyPrimitive> {
47
+ /** The underlying ServerDocument */
48
+ readonly document: ServerDocument.ServerDocument<TSchema>;
49
+ /** PubSub for broadcasting messages to subscribers */
50
+ readonly pubsub: PubSub.PubSub<ServerBroadcast>;
51
+ /** Current snapshot tracking state */
52
+ readonly getSnapshotTracking: Effect.Effect<SnapshotTracking>;
53
+ /** Submit a transaction */
54
+ readonly submit: (transaction: Transaction.Transaction) => Effect.Effect<SubmitResult, ColdStorageError | HotStorageError>;
55
+ /** Save a snapshot to cold storage */
56
+ readonly saveSnapshot: () => Effect.Effect<void, ColdStorageError | HotStorageError>;
57
+ /** Check if snapshot should be triggered and save if needed */
58
+ readonly checkSnapshotTriggers: () => Effect.Effect<void, ColdStorageError | HotStorageError>;
59
+ /** Update last activity time (for external tracking) */
60
+ readonly touch: () => Effect.Effect<void>;
61
+ /** Get current document version */
62
+ readonly getVersion: () => number;
63
+ /** Get document snapshot */
64
+ readonly getSnapshot: () => {
65
+ state: unknown;
66
+ version: number;
67
+ };
68
+ /** Check if document has unsnapshot transactions that need persisting */
69
+ readonly needsSnapshot: () => Effect.Effect<boolean>;
70
+ /** Get the last activity timestamp for idle detection */
71
+ readonly getLastActivityTime: () => Effect.Effect<number>;
72
+ }
73
+ declare const DocumentInstance: {
74
+ make: <TSchema extends Primitive.AnyPrimitive>(documentId: string, config: DocumentInstanceConfig<TSchema>, coldStorage: ColdStorage, hotStorage: HotStorage) => Effect.Effect<DocumentInstance<TSchema>, ColdStorageError | HotStorageError>;
75
+ };
76
+ //#endregion
77
+ export { DocumentInstance, SubmitResult };
78
+ //# sourceMappingURL=DocumentInstance.d.mts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"DocumentInstance.d.mts","names":[],"sources":["../src/DocumentInstance.ts"],"sourcesContent":[],"mappings":";;;;;;;;;;;;AAkDA;AASiB,KA/BL,YAAA,GA+BqB;EAAiB,SAAU,OAAA,EAAA,IAAA;EAET,SAAA,OAAA,EAAA,MAAA;CAA9B,GAAA;EAEY,SAAA,OAAA,EAAA,KAAA;EAAd,SAAO,MAAA,EAAA,MAAA;CAEoB;;;;AAE2C,UAhCxE,sBAgCwE,CAAA,gBAhCjC,SAAA,CAAU,YAgCuB,CAAA,CAAA;EAAmB,SAAA,MAAA,EA/BzF,OA+ByF;EAA/C,SAAO,OAAA,CAAA,EA7B9D,SAAA,CAAU,aA6BoD,CA7BtC,OA6BsC,CAAA,GAAA,CAAA,CAAA,GAAA,EAAA;IAEjB,UAAA,EAAA,MAAA;EAAmB,CAAA,EAAA,GA9B9B,MAAA,CAAO,MA8BuB,CA9BhB,SAAA,CAAU,aA8BM,CA9BQ,OA8BR,CAAA,CAAA,CAAA;EAAvC,SAAO,qBAAA,EAAA,MAAA;EAEsB,SAAA,QAAA,EAAA;IAAmB,SAAA,QAAA,EA7BxD,QAAA,CAAS,QA6B+C;IAAhC,SAAA,oBAAA,EAAA,MAAA;EAEvB,CAAA;;;;AAmcxB;AA1aqC,UAhDpB,gBAAA,CAgD8B;EAEd,SAAA,mBAAA,EAAA,MAAA;EAAvB,SAAA,gBAAA,EAAA,MAAA;EACK,SAAA,yBAAA,EAAA,MAAA;;;;;AAEgD,UA5C9C,gBA4C8C,CAAA,gBA5Cb,SAAA,CAAU,YA4CG,CAAA,CAAA;EAA5D;EAAa,SAAA,QAAA,EA1CK,cAAA,CAAe,cA0CpB,CA1CmC,OA0CnC,CAAA;;mBAxCG,MAAA,CAAO,OAAO;;gCAED,MAAA,CAAO,OAAO;;iCAEb,WAAA,CAAY,gBAAgB,MAAA,CAAO,OAAO,cAAc,mBAAmB;;+BAE7E,MAAA,CAAO,aAAa,mBAAmB;;wCAE9B,MAAA,CAAO,aAAa,mBAAmB;;wBAEvD,MAAA,CAAO;;;;;;;;;gCAMC,MAAA,CAAO;;sCAED,MAAA,CAAO;;cA2bhC;yBA1awB,SAAA,CAAU,0CAErC,uBAAuB,uBAClB,yBACD,eACX,MAAA,CAAO,OAAO,iBAAiB,UAAU,mBAAmB"}
@@ -0,0 +1,264 @@
1
+ import { documentsActive, documentsCreated, documentsRestored, storageSnapshotLatency, storageSnapshots, storageVersionGaps, storageWalAppends, transactionsLatency, transactionsProcessed, transactionsRejected, walAppendFailures } from "./Metrics.mjs";
2
+ import { Duration, Effect, Metric, PubSub, Ref } from "effect";
3
+ import { Document } from "@voidhash/mimic";
4
+ import { ServerDocument } from "@voidhash/mimic/server";
5
+
6
+ //#region src/DocumentInstance.ts
7
+ /**
8
+ * @voidhash/mimic-effect - DocumentInstance
9
+ *
10
+ * Manages the lifecycle of a single document including:
11
+ * - Restoration from storage (cold storage + WAL replay)
12
+ * - Transaction submission with WAL persistence
13
+ * - Snapshot saving and trigger checking
14
+ *
15
+ * Used by both MimicServerEngine (single-node) and MimicClusterServerEngine (clustered).
16
+ */
17
+ /**
18
+ * Create a DocumentInstance for a single document.
19
+ *
20
+ * This handles:
21
+ * - Loading from cold storage or computing initial state
22
+ * - Persisting initial state immediately (crash safety)
23
+ * - Replaying WAL entries
24
+ * - Transaction submission with WAL persistence
25
+ * - Snapshot saving
26
+ */
27
+ const make = (documentId, config, coldStorage, hotStorage) => Effect.gen(function* () {
28
+ const SCHEMA_VERSION = 1;
29
+ const storedDoc = yield* coldStorage.load(documentId);
30
+ let initialState;
31
+ let initial;
32
+ let initialVersion = 0;
33
+ if (storedDoc) {
34
+ initialState = storedDoc.state;
35
+ initialVersion = storedDoc.version;
36
+ } else initial = yield* computeInitialState(config, documentId);
37
+ const pubsub = yield* PubSub.unbounded();
38
+ const lastSnapshotVersionRef = yield* Ref.make(initialVersion);
39
+ const lastSnapshotTimeRef = yield* Ref.make(Date.now());
40
+ const transactionsSinceSnapshotRef = yield* Ref.make(0);
41
+ const lastActivityTimeRef = yield* Ref.make(Date.now());
42
+ const document = ServerDocument.make({
43
+ schema: config.schema,
44
+ initial,
45
+ initialState,
46
+ initialVersion,
47
+ maxTransactionHistory: config.maxTransactionHistory,
48
+ onBroadcast: (message) => {
49
+ Effect.runSync(PubSub.publish(pubsub, {
50
+ type: "transaction",
51
+ transaction: message.transaction,
52
+ version: message.version
53
+ }));
54
+ },
55
+ onRejection: (transactionId, reason) => {
56
+ Effect.runSync(PubSub.publish(pubsub, {
57
+ type: "error",
58
+ transactionId,
59
+ reason
60
+ }));
61
+ }
62
+ });
63
+ if (!storedDoc) {
64
+ const initialStoredDoc = createStoredDocument(document.get(), 0, SCHEMA_VERSION);
65
+ yield* coldStorage.save(documentId, initialStoredDoc);
66
+ yield* Effect.logDebug("Initial state persisted to cold storage", { documentId });
67
+ }
68
+ const walEntries = yield* hotStorage.getEntries(documentId, initialVersion);
69
+ yield* verifyWalContinuity(documentId, walEntries, initialVersion);
70
+ yield* replayWalEntries(documentId, document, walEntries);
71
+ if (storedDoc) yield* Metric.increment(documentsRestored);
72
+ else yield* Metric.increment(documentsCreated);
73
+ yield* Metric.incrementBy(documentsActive, 1);
74
+ const getSnapshotTracking = Effect.gen(function* () {
75
+ return {
76
+ lastSnapshotVersion: yield* Ref.get(lastSnapshotVersionRef),
77
+ lastSnapshotTime: yield* Ref.get(lastSnapshotTimeRef),
78
+ transactionsSinceSnapshot: yield* Ref.get(transactionsSinceSnapshotRef)
79
+ };
80
+ });
81
+ const saveSnapshot = Effect.fn("document.snapshot.save")(function* () {
82
+ var _baseSnapshot$version;
83
+ const targetVersion = document.getVersion();
84
+ if (targetVersion <= (yield* Ref.get(lastSnapshotVersionRef))) return;
85
+ const snapshotStartTime = Date.now();
86
+ const baseSnapshot = yield* coldStorage.load(documentId);
87
+ const baseVersion = (_baseSnapshot$version = baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.version) !== null && _baseSnapshot$version !== void 0 ? _baseSnapshot$version : 0;
88
+ const baseState = baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.state;
89
+ const walEntries$1 = yield* hotStorage.getEntries(documentId, baseVersion);
90
+ const snapshotResult = computeSnapshotState(config.schema, baseState, walEntries$1, targetVersion);
91
+ if (!snapshotResult) return;
92
+ const currentLastSnapshot = yield* Ref.get(lastSnapshotVersionRef);
93
+ if (snapshotResult.version <= currentLastSnapshot) return;
94
+ const storedDoc$1 = createStoredDocument(snapshotResult.state, snapshotResult.version, SCHEMA_VERSION);
95
+ yield* coldStorage.save(documentId, storedDoc$1);
96
+ const snapshotDuration = Date.now() - snapshotStartTime;
97
+ yield* Metric.increment(storageSnapshots);
98
+ yield* Metric.update(storageSnapshotLatency, snapshotDuration);
99
+ yield* Ref.set(lastSnapshotVersionRef, snapshotResult.version);
100
+ yield* Ref.set(lastSnapshotTimeRef, Date.now());
101
+ yield* Ref.set(transactionsSinceSnapshotRef, 0);
102
+ yield* Effect.catchAll(hotStorage.truncate(documentId, snapshotResult.version), (e) => Effect.logWarning("WAL truncate failed - will retry on next snapshot", {
103
+ documentId,
104
+ version: snapshotResult.version,
105
+ error: e
106
+ }));
107
+ });
108
+ const checkSnapshotTriggers = Effect.fn("document.snapshot.check-triggers")(function* () {
109
+ if (shouldTriggerSnapshot(yield* Ref.get(transactionsSinceSnapshotRef), yield* Ref.get(lastSnapshotTimeRef), config.snapshot)) yield* saveSnapshot();
110
+ });
111
+ const submit = Effect.fn("document.transaction.submit")(function* (transaction) {
112
+ const submitStartTime = Date.now();
113
+ yield* Ref.set(lastActivityTimeRef, Date.now());
114
+ const validation = document.validate(transaction);
115
+ if (!validation.valid) {
116
+ yield* Metric.increment(transactionsRejected);
117
+ const latency$1 = Date.now() - submitStartTime;
118
+ yield* Metric.update(transactionsLatency, latency$1);
119
+ return {
120
+ success: false,
121
+ reason: validation.reason
122
+ };
123
+ }
124
+ const walEntry = {
125
+ transaction,
126
+ version: validation.nextVersion,
127
+ timestamp: Date.now()
128
+ };
129
+ const appendResult = yield* Effect.either(hotStorage.appendWithCheck(documentId, walEntry, validation.nextVersion));
130
+ if (appendResult._tag === "Left") {
131
+ yield* Effect.logError("WAL append failed", {
132
+ documentId,
133
+ version: validation.nextVersion,
134
+ error: appendResult.left
135
+ });
136
+ yield* Metric.increment(walAppendFailures);
137
+ const latency$1 = Date.now() - submitStartTime;
138
+ yield* Metric.update(transactionsLatency, latency$1);
139
+ return {
140
+ success: false,
141
+ reason: "Storage unavailable. Please retry."
142
+ };
143
+ }
144
+ document.apply(transaction);
145
+ const latency = Date.now() - submitStartTime;
146
+ yield* Metric.update(transactionsLatency, latency);
147
+ yield* Metric.increment(transactionsProcessed);
148
+ yield* Metric.increment(storageWalAppends);
149
+ yield* Ref.update(transactionsSinceSnapshotRef, (n) => n + 1);
150
+ yield* checkSnapshotTriggers();
151
+ return {
152
+ success: true,
153
+ version: validation.nextVersion
154
+ };
155
+ });
156
+ const touch = Effect.fn("document.touch")(function* () {
157
+ yield* Ref.set(lastActivityTimeRef, Date.now());
158
+ });
159
+ const needsSnapshot = () => Effect.map(Ref.get(transactionsSinceSnapshotRef), (n) => n > 0);
160
+ const getLastActivityTime = () => Ref.get(lastActivityTimeRef);
161
+ return {
162
+ document,
163
+ pubsub,
164
+ getSnapshotTracking,
165
+ submit,
166
+ saveSnapshot,
167
+ checkSnapshotTriggers,
168
+ touch,
169
+ getVersion: () => document.getVersion(),
170
+ getSnapshot: () => document.getSnapshot(),
171
+ needsSnapshot,
172
+ getLastActivityTime
173
+ };
174
+ });
175
+ /**
176
+ * Compute initial state for a new document.
177
+ */
178
+ const computeInitialState = (config, documentId) => {
179
+ if (config.initial === void 0) return Effect.succeed(void 0);
180
+ if (typeof config.initial === "function") return config.initial({ documentId });
181
+ return Effect.succeed(config.initial);
182
+ };
183
+ /**
184
+ * Verify WAL continuity and log warnings for any gaps.
185
+ */
186
+ const verifyWalContinuity = Effect.fn("document.wal.verify")(function* (documentId, walEntries, baseVersion) {
187
+ if (walEntries.length === 0) return;
188
+ const firstWalVersion = walEntries[0].version;
189
+ const expectedFirst = baseVersion + 1;
190
+ if (firstWalVersion !== expectedFirst) {
191
+ yield* Effect.logWarning("WAL version gap detected", {
192
+ documentId,
193
+ snapshotVersion: baseVersion,
194
+ firstWalVersion,
195
+ expectedFirst
196
+ });
197
+ yield* Metric.increment(storageVersionGaps);
198
+ }
199
+ for (let i = 1; i < walEntries.length; i++) {
200
+ const prev = walEntries[i - 1].version;
201
+ const curr = walEntries[i].version;
202
+ if (curr !== prev + 1) yield* Effect.logWarning("WAL internal gap detected", {
203
+ documentId,
204
+ previousVersion: prev,
205
+ currentVersion: curr
206
+ });
207
+ }
208
+ });
209
+ /**
210
+ * Replay WAL entries onto a ServerDocument.
211
+ */
212
+ const replayWalEntries = Effect.fn("document.wal.replay")(function* (documentId, document, walEntries) {
213
+ for (const entry of walEntries) {
214
+ const result = document.submit(entry.transaction);
215
+ if (!result.success) yield* Effect.logWarning("Skipping corrupted WAL entry", {
216
+ documentId,
217
+ version: entry.version,
218
+ reason: result.reason
219
+ });
220
+ }
221
+ });
222
+ /**
223
+ * Compute snapshot state by replaying WAL entries on a base state.
224
+ */
225
+ const computeSnapshotState = (schema, baseState, walEntries, targetVersion) => {
226
+ const relevantEntries = walEntries.filter((e) => e.version <= targetVersion);
227
+ if (relevantEntries.length === 0 && baseState === void 0) return;
228
+ let snapshotState = baseState;
229
+ for (const entry of relevantEntries) {
230
+ const tempDoc = Document.make(schema, { initialState: snapshotState });
231
+ tempDoc.apply(entry.transaction.ops);
232
+ snapshotState = tempDoc.get();
233
+ }
234
+ if (snapshotState === void 0) return;
235
+ const snapshotVersion = relevantEntries.length > 0 ? relevantEntries[relevantEntries.length - 1].version : 0;
236
+ return {
237
+ state: snapshotState,
238
+ version: snapshotVersion
239
+ };
240
+ };
241
+ /**
242
+ * Check if a snapshot should be triggered.
243
+ */
244
+ const shouldTriggerSnapshot = (transactionsSinceSnapshot, lastSnapshotTime, config) => {
245
+ const now = Date.now();
246
+ const intervalMs = Duration.toMillis(config.interval);
247
+ if (transactionsSinceSnapshot >= config.transactionThreshold) return true;
248
+ if (now - lastSnapshotTime >= intervalMs) return true;
249
+ return false;
250
+ };
251
+ /**
252
+ * Create a StoredDocument for persistence.
253
+ */
254
+ const createStoredDocument = (state, version, schemaVersion) => ({
255
+ state,
256
+ version,
257
+ schemaVersion,
258
+ savedAt: Date.now()
259
+ });
260
+ const DocumentInstance = { make };
261
+
262
+ //#endregion
263
+ export { DocumentInstance };
264
+ //# sourceMappingURL=DocumentInstance.mjs.map