@fluidframework/id-compressor 3.2.0 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/api-report/id-compressor.legacy.beta.api.md +0 -49
- package/dist/idCompressor.d.ts +69 -19
- package/dist/idCompressor.d.ts.map +1 -1
- package/dist/idCompressor.js +365 -57
- package/dist/idCompressor.js.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -1
- package/dist/legacy.d.ts +1 -12
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/test/idCompressor.perf.spec.js.map +1 -1
- package/dist/test/idCompressor.spec.js.map +1 -1
- package/dist/test/idCompressorTestUtilities.d.ts.map +1 -1
- package/dist/test/idCompressorTestUtilities.js +8 -3
- package/dist/test/idCompressorTestUtilities.js.map +1 -1
- package/dist/test/sharding.fuzz.spec.d.ts +6 -0
- package/dist/test/sharding.fuzz.spec.d.ts.map +1 -0
- package/dist/test/sharding.fuzz.spec.js.map +1 -0
- package/dist/test/sharding.spec.d.ts +6 -0
- package/dist/test/sharding.spec.d.ts.map +1 -0
- package/dist/test/sharding.spec.js.map +1 -0
- package/dist/test/snapshots/summary.spec.js.map +1 -1
- package/dist/test/types/validateIdCompressorPrevious.generated.js.map +1 -1
- package/dist/types/idCompressor.d.ts +108 -2
- package/dist/types/idCompressor.d.ts.map +1 -1
- package/dist/types/idCompressor.js +15 -0
- package/dist/types/idCompressor.js.map +1 -1
- package/dist/types/index.d.ts +2 -1
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/index.js +3 -0
- package/dist/types/index.js.map +1 -1
- package/dist/types/persisted-types/0.0.1.d.ts +4 -4
- package/dist/types/persisted-types/0.0.1.js.map +1 -1
- package/dist/utilities.d.ts +1 -1
- package/dist/utilities.js +1 -1
- package/dist/utilities.js.map +1 -1
- package/lib/idCompressor.d.ts +69 -19
- package/lib/idCompressor.d.ts.map +1 -1
- package/lib/idCompressor.js +365 -57
- package/lib/idCompressor.js.map +1 -1
- package/lib/index.d.ts +2 -1
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +1 -0
- package/lib/index.js.map +1 -1
- package/lib/legacy.d.ts +1 -12
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/test/idCompressor.perf.spec.js.map +1 -1
- package/lib/test/idCompressor.spec.js.map +1 -1
- package/lib/test/idCompressorTestUtilities.d.ts.map +1 -1
- package/lib/test/idCompressorTestUtilities.js +8 -3
- package/lib/test/idCompressorTestUtilities.js.map +1 -1
- package/lib/test/sharding.fuzz.spec.d.ts +6 -0
- package/lib/test/sharding.fuzz.spec.d.ts.map +1 -0
- package/lib/test/sharding.fuzz.spec.js.map +1 -0
- package/lib/test/sharding.spec.d.ts +6 -0
- package/lib/test/sharding.spec.d.ts.map +1 -0
- package/lib/test/sharding.spec.js.map +1 -0
- package/lib/test/snapshots/summary.spec.js.map +1 -1
- package/lib/test/types/validateIdCompressorPrevious.generated.js.map +1 -1
- package/lib/types/idCompressor.d.ts +108 -2
- package/lib/types/idCompressor.d.ts.map +1 -1
- package/lib/types/idCompressor.js +14 -1
- package/lib/types/idCompressor.js.map +1 -1
- package/lib/types/index.d.ts +2 -1
- package/lib/types/index.d.ts.map +1 -1
- package/lib/types/index.js +1 -1
- package/lib/types/index.js.map +1 -1
- package/lib/types/persisted-types/0.0.1.d.ts +4 -4
- package/lib/types/persisted-types/0.0.1.js.map +1 -1
- package/lib/utilities.d.ts +1 -1
- package/lib/utilities.js +1 -1
- package/lib/utilities.js.map +1 -1
- package/package.json +42 -9
- package/src/idCompressor.ts +497 -73
- package/src/index.ts +3 -0
- package/src/packageVersion.ts +1 -1
- package/src/test/idCompressorTestUtilities.ts +16 -3
- package/src/test/snapshots/v2/compressor-with-finalized-range-from-one-client +1 -0
- package/src/test/snapshots/v2/compressor-with-finalized-range-from-one-client-client2 +1 -0
- package/src/test/snapshots/v2/compressors-with-finalized-ranges-from-two-clients +1 -0
- package/src/test/snapshots/v2/compressors-with-finalized-ranges-from-two-clients-client2 +1 -0
- package/src/test/snapshots/v2/empty-compressor-summary +1 -0
- package/src/test/snapshots/v2/expansion-semantics +1 -0
- package/src/test/snapshots/v2/expansion-semantics-client2 +1 -0
- package/src/test/snapshots/v3/compressor-with-finalized-range-from-one-client +1 -0
- package/src/test/snapshots/v3/compressor-with-finalized-range-from-one-client-client2 +1 -0
- package/src/test/snapshots/v3/compressors-with-finalized-ranges-from-two-clients +1 -0
- package/src/test/snapshots/v3/compressors-with-finalized-ranges-from-two-clients-client2 +1 -0
- package/src/test/snapshots/v3/empty-compressor-summary +1 -0
- package/src/test/snapshots/v3/expansion-semantics +1 -0
- package/src/test/snapshots/v3/expansion-semantics-client2 +1 -0
- package/src/test/types/validateIdCompressorPrevious.generated.ts +13 -1
- package/src/types/idCompressor.ts +119 -2
- package/src/types/index.ts +10 -1
- package/src/types/persisted-types/0.0.1.ts +4 -4
- package/src/utilities.ts +1 -1
package/dist/idCompressor.js
CHANGED
|
@@ -17,23 +17,21 @@ const identifiers_js_1 = require("./identifiers.js");
|
|
|
17
17
|
const persistanceUtilities_js_1 = require("./persistanceUtilities.js");
|
|
18
18
|
const sessionSpaceNormalizer_js_1 = require("./sessionSpaceNormalizer.js");
|
|
19
19
|
const sessions_js_1 = require("./sessions.js");
|
|
20
|
+
const index_js_1 = require("./types/index.js");
|
|
20
21
|
const utilities_js_1 = require("./utilities.js");
|
|
21
|
-
/**
|
|
22
|
-
* The version of IdCompressor that is currently persisted.
|
|
23
|
-
* This should not be changed without careful consideration to compatibility.
|
|
24
|
-
*/
|
|
25
|
-
const currentWrittenVersion = 2;
|
|
26
22
|
function rangeFinalizationError(expectedStart, actualStart) {
|
|
27
23
|
return new internal_2.LoggingError("Ranges finalized out of order", {
|
|
28
24
|
expectedStart,
|
|
29
25
|
actualStart,
|
|
30
26
|
});
|
|
31
27
|
}
|
|
28
|
+
const MAX_STRIDE_LENGTH = 1000000;
|
|
32
29
|
/**
|
|
33
30
|
* See {@link IIdCompressor} and {@link IIdCompressorCore}
|
|
34
31
|
*/
|
|
35
32
|
class IdCompressor {
|
|
36
33
|
logger;
|
|
34
|
+
writeVersion;
|
|
37
35
|
/**
|
|
38
36
|
* Max allowed initial cluster size.
|
|
39
37
|
*/
|
|
@@ -44,11 +42,19 @@ class IdCompressor {
|
|
|
44
42
|
normalizer = new sessionSpaceNormalizer_js_1.SessionSpaceNormalizer();
|
|
45
43
|
// The number of IDs generated by the local session
|
|
46
44
|
localGenCount = 0;
|
|
45
|
+
/**
|
|
46
|
+
* The sharding state of this compressor, if it has been sharded or is a shard itself.
|
|
47
|
+
* When defined, this compressor operates in "sharding mode" where:
|
|
48
|
+
* - IDs are generated using stride-based allocation (no eager finals)
|
|
49
|
+
* - The normalizer backfills all IDs in each stride cycle
|
|
50
|
+
* - Normal operation resumes when all children are disposed
|
|
51
|
+
*/
|
|
52
|
+
shardingState;
|
|
47
53
|
// #endregion
|
|
48
54
|
// #region Final state
|
|
49
55
|
/**
|
|
50
56
|
* The gen count to be annotated on the range returned by the next call to `takeNextCreationRange`.
|
|
51
|
-
* This is advanced to `
|
|
57
|
+
* This is advanced to `localGenCount` + 1 each time it is called.
|
|
52
58
|
* On the other hand, when `resetUnfinalizedCreationRange` is called,
|
|
53
59
|
* this is moved back to the start of the unfinalized range, to ensure those IDs are included in the next range.
|
|
54
60
|
*/
|
|
@@ -72,8 +78,15 @@ class IdCompressor {
|
|
|
72
78
|
// The ongoing ghost session, if one exists.
|
|
73
79
|
ongoingGhostSession;
|
|
74
80
|
// #endregion
|
|
75
|
-
constructor(localSessionIdOrDeserialized, logger
|
|
81
|
+
constructor(localSessionIdOrDeserialized, logger,
|
|
82
|
+
/**
|
|
83
|
+
* The version this compressor will write out.
|
|
84
|
+
* This applies to both `serialize` and creation range functions.
|
|
85
|
+
* This number also determines what features are available (e.g. sharding support starts at V3).
|
|
86
|
+
*/
|
|
87
|
+
writeVersion) {
|
|
76
88
|
this.logger = logger;
|
|
89
|
+
this.writeVersion = writeVersion;
|
|
77
90
|
if (typeof localSessionIdOrDeserialized === "string") {
|
|
78
91
|
this.localSessionId = localSessionIdOrDeserialized;
|
|
79
92
|
this.localSession = this.sessions.getOrCreate(localSessionIdOrDeserialized);
|
|
@@ -101,29 +114,38 @@ class IdCompressor {
|
|
|
101
114
|
this.ongoingGhostSession.cluster.count++;
|
|
102
115
|
return (0, sessions_js_1.lastFinalizedFinal)(this.ongoingGhostSession.cluster);
|
|
103
116
|
}
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
// Space in the cluster: eager final
|
|
120
|
-
return (lastCluster.baseFinalId +
|
|
121
|
-
clusterOffset);
|
|
122
|
-
}
|
|
123
|
-
// No space in the cluster, return next local
|
|
117
|
+
// In sharding mode, always generate local IDs with stride-based allocation.
|
|
118
|
+
// Eager finals are disabled because shards backfill IDs not in their stride cycle.
|
|
119
|
+
// If another shard generated them as eager finals, ingesting IDs from another shard
|
|
120
|
+
// would fail depending on timing.
|
|
121
|
+
if (this.shardingState !== undefined) {
|
|
122
|
+
this.telemetryLocalIdCount++;
|
|
123
|
+
const { currentStride } = this.shardingState;
|
|
124
|
+
this.normalizer.addLocalRange(this.localGenCount + 1, currentStride);
|
|
125
|
+
this.localGenCount += currentStride;
|
|
126
|
+
return (0, utilities_js_1.localIdFromGenCount)(this.localGenCount);
|
|
127
|
+
}
|
|
128
|
+
// Normal ID generation
|
|
129
|
+
this.localGenCount++;
|
|
130
|
+
const lastCluster = this.localSession.getLastCluster();
|
|
131
|
+
if (lastCluster === undefined) {
|
|
124
132
|
this.telemetryLocalIdCount++;
|
|
125
133
|
return this.generateNextLocalId();
|
|
126
134
|
}
|
|
135
|
+
// If there exists a cluster of final IDs already claimed by the local session that still has room in it,
|
|
136
|
+
// it is known prior to range sequencing what a local ID's corresponding final ID will be.
|
|
137
|
+
// In this case, it is safe to return the final ID immediately. This is guaranteed to be safe because
|
|
138
|
+
// any op that the local session sends that contains one of those final IDs are guaranteed to arrive to
|
|
139
|
+
// collaborators *after* the one containing the creation range.
|
|
140
|
+
const clusterOffset = this.localGenCount - (0, utilities_js_1.genCountFromLocalId)(lastCluster.baseLocalId);
|
|
141
|
+
if (lastCluster.capacity > clusterOffset) {
|
|
142
|
+
this.telemetryEagerFinalIdCount++;
|
|
143
|
+
// Space in the cluster: eager final
|
|
144
|
+
return (lastCluster.baseFinalId + clusterOffset);
|
|
145
|
+
}
|
|
146
|
+
// No space in the cluster, return next local
|
|
147
|
+
this.telemetryLocalIdCount++;
|
|
148
|
+
return this.generateNextLocalId();
|
|
127
149
|
}
|
|
128
150
|
generateDocumentUniqueId() {
|
|
129
151
|
const id = this.generateCompressedId();
|
|
@@ -141,6 +163,9 @@ class IdCompressor {
|
|
|
141
163
|
* {@inheritdoc IIdCompressorCore.beginGhostSession}
|
|
142
164
|
*/
|
|
143
165
|
beginGhostSession(ghostSessionId, ghostSessionCallback) {
|
|
166
|
+
if (this.shardingState) {
|
|
167
|
+
throw new Error("Cannot start a ghost session while sharded.");
|
|
168
|
+
}
|
|
144
169
|
this.startGhostSession(ghostSessionId);
|
|
145
170
|
try {
|
|
146
171
|
ghostSessionCallback();
|
|
@@ -149,12 +174,177 @@ class IdCompressor {
|
|
|
149
174
|
this.ongoingGhostSession = undefined;
|
|
150
175
|
}
|
|
151
176
|
}
|
|
177
|
+
/**
|
|
178
|
+
* {@inheritdoc IIdCompressorCore.shard}
|
|
179
|
+
*/
|
|
180
|
+
shard(newShardCount) {
|
|
181
|
+
if (!Number.isSafeInteger(newShardCount) || newShardCount <= 0) {
|
|
182
|
+
throw new Error("Shard count must be a positive safe integer");
|
|
183
|
+
}
|
|
184
|
+
if (this.ongoingGhostSession) {
|
|
185
|
+
throw new Error("Cannot shard during ghost session");
|
|
186
|
+
}
|
|
187
|
+
if (this.writeVersion < index_js_1.SerializationVersion.V3) {
|
|
188
|
+
throw new Error(`Sharding requires document version ${index_js_1.SerializationVersion.V3} or higher, but this document is version ${this.writeVersion}`);
|
|
189
|
+
}
|
|
190
|
+
const currentStride = this.shardingState?.currentStride ?? 1;
|
|
191
|
+
const newStride = currentStride * (newShardCount + 1);
|
|
192
|
+
// ID usage is exponential in stride count. Limit to practical size to avoid overflow.
|
|
193
|
+
if (newStride > MAX_STRIDE_LENGTH) {
|
|
194
|
+
throw new Error("Sharding limit reached.");
|
|
195
|
+
}
|
|
196
|
+
(0, internal_1.assert)(Number.isSafeInteger(newStride), 0xd52 /* Shard stride must be a safe integer */);
|
|
197
|
+
if (this.shardingState === undefined) {
|
|
198
|
+
// First time sharding - initialize state
|
|
199
|
+
this.shardingState = {
|
|
200
|
+
currentStride: newStride,
|
|
201
|
+
originalStride: 1,
|
|
202
|
+
activeChildIds: new Set(),
|
|
203
|
+
};
|
|
204
|
+
}
|
|
205
|
+
else {
|
|
206
|
+
this.shardingState.currentStride = newStride;
|
|
207
|
+
}
|
|
208
|
+
// Cache to avoid repeating per child
|
|
209
|
+
const serialized = this.serialize(true);
|
|
210
|
+
const { localGenCount: parentGenCount } = this;
|
|
211
|
+
const shards = [];
|
|
212
|
+
// Children are positioned at consecutive positions within the parents current stride cycle
|
|
213
|
+
// e.g. if parent owns the "evens" space (-2, -4, -6, ...) and is sharded into 3 (two children)
|
|
214
|
+
// while at -8, then the child shards will start at -10 and -12 with a stride of 6
|
|
215
|
+
for (let i = 1; i <= newShardCount; i++) {
|
|
216
|
+
const childShardId = (0, utilities_js_1.createSessionId)();
|
|
217
|
+
this.shardingState.activeChildIds.add(childShardId);
|
|
218
|
+
const childLocalGenCount = parentGenCount + i * currentStride;
|
|
219
|
+
// This code results in a double roundtrip, but is probably plenty fast enough for scenarios that need sharding,
|
|
220
|
+
// as they necessarily have process boundaries to deal with.
|
|
221
|
+
// If this ever became an issue we can implement a fast clone of compressors
|
|
222
|
+
const child = deserializeIdCompressor(serialized, this.writeVersion);
|
|
223
|
+
const genCountJump = childLocalGenCount - parentGenCount;
|
|
224
|
+
(0, internal_1.assert)(child.localGenCount === parentGenCount && genCountJump > 0, 0xd53 /* Child offsets incorrectly calculated. */);
|
|
225
|
+
child.normalizer.addLocalRange(child.localGenCount + 1, genCountJump);
|
|
226
|
+
child.localGenCount = childLocalGenCount;
|
|
227
|
+
child.shardingState = {
|
|
228
|
+
currentStride: newStride,
|
|
229
|
+
originalStride: newStride,
|
|
230
|
+
activeChildIds: new Set(),
|
|
231
|
+
shardId: childShardId,
|
|
232
|
+
};
|
|
233
|
+
shards.push(child.serialize(true));
|
|
234
|
+
}
|
|
235
|
+
return shards;
|
|
236
|
+
}
|
|
237
|
+
/**
|
|
238
|
+
* {@inheritdoc IIdCompressorCore.synchronizeWithShard}
|
|
239
|
+
*/
|
|
240
|
+
synchronizeWithShard(syncToken) {
|
|
241
|
+
const isNowLeaf = this.synchronizeChild(syncToken);
|
|
242
|
+
// A disposal token additionally reclaims the child's ID space. If reclaiming the last child
|
|
243
|
+
// returns this compressor to a leaf and it is the root (originalStride === 1), exit sharding mode.
|
|
244
|
+
if (syncToken.disposed) {
|
|
245
|
+
(0, internal_1.assert)(this.shardingState !== undefined, 0xd54 /* Must be sharded */);
|
|
246
|
+
if (isNowLeaf && this.shardingState.originalStride === 1) {
|
|
247
|
+
this.shardingState = undefined;
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
synchronizeChild(syncToken) {
|
|
252
|
+
const childDisposed = syncToken.disposed;
|
|
253
|
+
if (this.writeVersion < index_js_1.SerializationVersion.V3) {
|
|
254
|
+
throw new Error(`Sharding requires document version ${index_js_1.SerializationVersion.V3} or higher, but this document is version ${this.writeVersion}`);
|
|
255
|
+
}
|
|
256
|
+
if (this.shardingState === undefined) {
|
|
257
|
+
throw new Error("Must be in sharding mode.");
|
|
258
|
+
}
|
|
259
|
+
const childShardId = syncToken.shardId;
|
|
260
|
+
const { activeChildIds, originalStride } = this.shardingState;
|
|
261
|
+
// Verify this child belongs to us
|
|
262
|
+
if (!activeChildIds.has(childShardId)) {
|
|
263
|
+
throw new Error(`Cannot synchronize with child with ID ${childShardId}: not in active children set`);
|
|
264
|
+
}
|
|
265
|
+
// Only disposal removes the child from the active set. A plain synchronization leaves the
|
|
266
|
+
// child active so it can continue generating IDs and be synchronized with again later.
|
|
267
|
+
if (childDisposed) {
|
|
268
|
+
activeChildIds.delete(childShardId);
|
|
269
|
+
}
|
|
270
|
+
const isLeaf = activeChildIds.size === 0;
|
|
271
|
+
if (isLeaf && childDisposed) {
|
|
272
|
+
this.shardingState.currentStride = originalStride;
|
|
273
|
+
}
|
|
274
|
+
// Read currentStride after the possible stride reset above, so that a leaf-making disposal
|
|
275
|
+
// realigns the parent on its original (reclaimed) stride rather than the wider sharded stride.
|
|
276
|
+
const { currentStride } = this.shardingState;
|
|
277
|
+
const childGenCount = syncToken.localGenCount;
|
|
278
|
+
// Realign parent to next position in its sequence if child is ahead
|
|
279
|
+
if (childGenCount > this.localGenCount) {
|
|
280
|
+
// Find next aligned position in parent's sequence after child's genCount
|
|
281
|
+
const distance = childGenCount - this.localGenCount + 1;
|
|
282
|
+
const steps = Math.ceil(distance / currentStride);
|
|
283
|
+
const count = steps * currentStride;
|
|
284
|
+
this.normalizer.addLocalRange(this.localGenCount + 1, count);
|
|
285
|
+
this.localGenCount += count;
|
|
286
|
+
}
|
|
287
|
+
return isLeaf;
|
|
288
|
+
}
|
|
289
|
+
/**
|
|
290
|
+
* {@inheritdoc IIdCompressorCore.dispose}
|
|
291
|
+
*/
|
|
292
|
+
disposeShard() {
|
|
293
|
+
if (this.shardingState === undefined) {
|
|
294
|
+
return undefined;
|
|
295
|
+
}
|
|
296
|
+
else {
|
|
297
|
+
if (this.shardingState.activeChildIds.size > 0) {
|
|
298
|
+
throw new Error("Cannot dispose a shard with active children.");
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
// A disposal token is a sync token that additionally signals reclamation of this shard's ID space.
|
|
302
|
+
const token = this.makeShardToken(true);
|
|
303
|
+
// Make this compressor unusable by replacing all methods with throwing stubs
|
|
304
|
+
// This is a one-time mutation that avoids runtime checks in every method
|
|
305
|
+
makeCompressorUnusable(this);
|
|
306
|
+
return token;
|
|
307
|
+
}
|
|
308
|
+
/**
|
|
309
|
+
* {@inheritdoc IIdCompressorCore.getShardSyncToken}
|
|
310
|
+
*/
|
|
311
|
+
getShardSyncToken() {
|
|
312
|
+
if (this.shardingState === undefined) {
|
|
313
|
+
return undefined;
|
|
314
|
+
}
|
|
315
|
+
return this.makeShardToken(false);
|
|
316
|
+
}
|
|
317
|
+
/**
|
|
318
|
+
* Builds a token describing this shard's current progress.
|
|
319
|
+
* @param disposed - Whether the token also signals disposal (reclamation of this shard's ID space).
|
|
320
|
+
*/
|
|
321
|
+
makeShardToken(disposed) {
|
|
322
|
+
(0, internal_1.assert)(this.shardingState !== undefined, 0xd55 /* Compressor is not sharded. */);
|
|
323
|
+
// Root cannot produce a token (shardId is undefined for root)
|
|
324
|
+
if (this.shardingState.shardId === undefined) {
|
|
325
|
+
throw new Error("Cannot get a token for a root compressor - only shards can be disposed");
|
|
326
|
+
}
|
|
327
|
+
const token = {
|
|
328
|
+
localGenCount: this.localGenCount,
|
|
329
|
+
shardId: this.shardingState.shardId,
|
|
330
|
+
disposed,
|
|
331
|
+
};
|
|
332
|
+
return token;
|
|
333
|
+
}
|
|
152
334
|
generateNextLocalId() {
|
|
153
335
|
// Must tell the normalizer that we generated a local ID
|
|
154
336
|
this.normalizer.addLocalRange(this.localGenCount, 1);
|
|
155
337
|
return (0, utilities_js_1.localIdFromGenCount)(this.localGenCount);
|
|
156
338
|
}
|
|
157
339
|
takeNextCreationRange() {
|
|
340
|
+
// Child (non-root) shards must not submit their own creation ranges. All shards in a shard tree share a
|
|
341
|
+
// single session, and only the root shard is responsible for finalizing that session's IDs with the server.
|
|
342
|
+
// A child's generated IDs are reconciled with the root via shard synchronization tokens (see
|
|
343
|
+
// synchronizeWithShard / disposeShard), not by independently sending ranges, which would overlap and
|
|
344
|
+
// conflict with the root's ranges for the shared session.
|
|
345
|
+
if (this.shardingState?.shardId !== undefined) {
|
|
346
|
+
throw new Error("Cannot take a creation range from a non-root shard.");
|
|
347
|
+
}
|
|
158
348
|
(0, internal_1.assert)(!this.ongoingGhostSession, 0x8a6 /* IdCompressor should not be operated normally when in a ghost session */);
|
|
159
349
|
const count = this.localGenCount - (this.nextRangeBaseGenCount - 1);
|
|
160
350
|
if (count === 0) {
|
|
@@ -443,24 +633,36 @@ class IdCompressor {
|
|
|
443
633
|
sessionIndex++;
|
|
444
634
|
}
|
|
445
635
|
}
|
|
636
|
+
const sizeOfUuid = 2;
|
|
637
|
+
const shardingStateSize = hasLocalState && this.writeVersion >= index_js_1.SerializationVersion.V3
|
|
638
|
+
? this.shardingState === undefined
|
|
639
|
+
? 1 // Boolean: no sharding state
|
|
640
|
+
: 1 + // Boolean: has sharding state
|
|
641
|
+
2 + // Numbers: currentStride, originalStride (Float64)
|
|
642
|
+
1 + // Number: activeChildIds count (Float64)
|
|
643
|
+
this.shardingState.activeChildIds.size * sizeOfUuid + // UUIDs: each is 128-bit
|
|
644
|
+
1 + // Boolean: hasShardId
|
|
645
|
+
(this.shardingState.shardId === undefined ? 0 : sizeOfUuid) // UUID: 128-bit if present
|
|
646
|
+
: 0;
|
|
446
647
|
const localStateSize = hasLocalState
|
|
447
648
|
? 1 + // generated ID count
|
|
448
649
|
1 + // next range base genCount
|
|
449
650
|
1 + // count of normalizer pairs
|
|
450
|
-
normalizer.idRanges.size * 2 // pairs
|
|
651
|
+
normalizer.idRanges.size * 2 + // pairs
|
|
652
|
+
shardingStateSize // sharding state if present
|
|
451
653
|
: 0;
|
|
452
654
|
// Layout size, in 8 byte increments
|
|
453
655
|
const totalSize = 1 + // version
|
|
454
656
|
1 + // hasLocalState
|
|
455
657
|
1 + // session count
|
|
456
658
|
1 + // cluster count
|
|
457
|
-
sessionIndexMap.size *
|
|
659
|
+
sessionIndexMap.size * sizeOfUuid + // session IDs
|
|
458
660
|
finalSpace.clusters.length * 3 + // clusters: (sessionIndex, capacity, count)[]
|
|
459
661
|
localStateSize; // local state, if present
|
|
460
662
|
const serializedFloat = new Float64Array(totalSize);
|
|
461
663
|
const serializedUint = new BigUint64Array(serializedFloat.buffer);
|
|
462
664
|
let index = 0;
|
|
463
|
-
index = (0, persistanceUtilities_js_1.writeNumber)(serializedFloat, index,
|
|
665
|
+
index = (0, persistanceUtilities_js_1.writeNumber)(serializedFloat, index, this.writeVersion);
|
|
464
666
|
index = (0, persistanceUtilities_js_1.writeBoolean)(serializedFloat, index, hasLocalState);
|
|
465
667
|
index = (0, persistanceUtilities_js_1.writeNumber)(serializedFloat, index, sessionIndexMap.size);
|
|
466
668
|
index = (0, persistanceUtilities_js_1.writeNumber)(serializedFloat, index, finalSpace.clusters.length);
|
|
@@ -480,6 +682,24 @@ class IdCompressor {
|
|
|
480
682
|
index = (0, persistanceUtilities_js_1.writeNumber)(serializedFloat, index, leadingGenCount);
|
|
481
683
|
index = (0, persistanceUtilities_js_1.writeNumber)(serializedFloat, index, count);
|
|
482
684
|
}
|
|
685
|
+
// Write sharding state (version 3+)
|
|
686
|
+
if (this.writeVersion >= index_js_1.SerializationVersion.V3) {
|
|
687
|
+
index = (0, persistanceUtilities_js_1.writeBoolean)(serializedFloat, index, this.shardingState !== undefined);
|
|
688
|
+
if (this.shardingState !== undefined) {
|
|
689
|
+
index = (0, persistanceUtilities_js_1.writeNumber)(serializedFloat, index, this.shardingState.currentStride);
|
|
690
|
+
index = (0, persistanceUtilities_js_1.writeNumber)(serializedFloat, index, this.shardingState.originalStride);
|
|
691
|
+
// Write activeChildIds Set as count + array of NumericUuids
|
|
692
|
+
index = (0, persistanceUtilities_js_1.writeNumber)(serializedFloat, index, this.shardingState.activeChildIds.size);
|
|
693
|
+
for (const childId of this.shardingState.activeChildIds) {
|
|
694
|
+
index = (0, persistanceUtilities_js_1.writeNumericUuid)(serializedUint, index, (0, utilities_js_1.numericUuidFromStableId)(childId));
|
|
695
|
+
}
|
|
696
|
+
// Write shardId (optional)
|
|
697
|
+
index = (0, persistanceUtilities_js_1.writeBoolean)(serializedFloat, index, this.shardingState.shardId !== undefined);
|
|
698
|
+
if (this.shardingState.shardId !== undefined) {
|
|
699
|
+
index = (0, persistanceUtilities_js_1.writeNumericUuid)(serializedUint, index, (0, utilities_js_1.numericUuidFromStableId)(this.shardingState.shardId));
|
|
700
|
+
}
|
|
701
|
+
}
|
|
702
|
+
}
|
|
483
703
|
}
|
|
484
704
|
(0, internal_1.assert)(index === totalSize, 0x75b /* Serialized size was incorrectly calculated. */);
|
|
485
705
|
logger?.sendTelemetryEvent({
|
|
@@ -491,27 +711,29 @@ class IdCompressor {
|
|
|
491
711
|
return (0, client_utils_1.bufferToString)(serializedFloat.buffer, "base64");
|
|
492
712
|
}
|
|
493
713
|
static deserialize(params) {
|
|
494
|
-
const { serialized, newSessionId, logger } = params;
|
|
714
|
+
const { serialized, newSessionId: sessionId, logger, requestedWriteVersion } = params;
|
|
495
715
|
const buffer = (0, client_utils_1.stringToBuffer)(serialized, "base64");
|
|
496
716
|
const index = {
|
|
497
717
|
index: 0,
|
|
498
718
|
bufferFloat: new Float64Array(buffer),
|
|
499
719
|
bufferUint: new BigUint64Array(buffer),
|
|
500
720
|
};
|
|
501
|
-
const
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
case 2:
|
|
507
|
-
|
|
721
|
+
const serializedVersion = (0, persistanceUtilities_js_1.readNumber)(index);
|
|
722
|
+
// If requested version is < the serialized version, we must write the version serialized
|
|
723
|
+
// to avoid losing data.
|
|
724
|
+
const writeVersion = Math.max(requestedWriteVersion, serializedVersion);
|
|
725
|
+
switch (serializedVersion) {
|
|
726
|
+
case 2:
|
|
727
|
+
case 3: {
|
|
728
|
+
break;
|
|
508
729
|
}
|
|
509
730
|
default: {
|
|
731
|
+
if (serializedVersion === 1) {
|
|
732
|
+
throw new Error("IdCompressor version 1.0 is no longer supported.");
|
|
733
|
+
}
|
|
510
734
|
throw new Error("Unknown IdCompressor serialized version.");
|
|
511
735
|
}
|
|
512
736
|
}
|
|
513
|
-
}
|
|
514
|
-
static deserialize2_0(index, sessionId, logger) {
|
|
515
737
|
const hasLocalState = (0, persistanceUtilities_js_1.readBoolean)(index);
|
|
516
738
|
const sessionCount = (0, persistanceUtilities_js_1.readNumber)(index);
|
|
517
739
|
const clusterCount = (0, persistanceUtilities_js_1.readNumber)(index);
|
|
@@ -532,7 +754,7 @@ class IdCompressor {
|
|
|
532
754
|
const numeric = (0, persistanceUtilities_js_1.readNumericUuid)(index);
|
|
533
755
|
sessions.push([numeric, new sessions_js_1.Session(numeric)]);
|
|
534
756
|
}
|
|
535
|
-
const compressor = new IdCompressor(new sessions_js_1.Sessions(sessions), logger);
|
|
757
|
+
const compressor = new IdCompressor(new sessions_js_1.Sessions(sessions), logger, writeVersion);
|
|
536
758
|
// Clusters
|
|
537
759
|
let baseFinalId = 0;
|
|
538
760
|
for (let i = 0; i < clusterCount; i++) {
|
|
@@ -554,6 +776,34 @@ class IdCompressor {
|
|
|
554
776
|
for (let i = 0; i < normalizerCount; i++) {
|
|
555
777
|
compressor.normalizer.addLocalRange((0, persistanceUtilities_js_1.readNumber)(index), (0, persistanceUtilities_js_1.readNumber)(index));
|
|
556
778
|
}
|
|
779
|
+
// Sharding state (version 3+)
|
|
780
|
+
if (serializedVersion >= 3) {
|
|
781
|
+
const hasShardingState = (0, persistanceUtilities_js_1.readBoolean)(index);
|
|
782
|
+
if (hasShardingState) {
|
|
783
|
+
const currentStride = (0, persistanceUtilities_js_1.readNumber)(index);
|
|
784
|
+
const originalStride = (0, persistanceUtilities_js_1.readNumber)(index);
|
|
785
|
+
// Read activeChildIds Set
|
|
786
|
+
const activeChildCount = (0, persistanceUtilities_js_1.readNumber)(index);
|
|
787
|
+
const activeChildIds = new Set();
|
|
788
|
+
for (let i = 0; i < activeChildCount; i++) {
|
|
789
|
+
const childIdNumeric = (0, persistanceUtilities_js_1.readNumericUuid)(index);
|
|
790
|
+
// Safe cast: these were serialized as SessionIds
|
|
791
|
+
activeChildIds.add((0, utilities_js_1.stableIdFromNumericUuid)(childIdNumeric));
|
|
792
|
+
}
|
|
793
|
+
// Read shardId (optional)
|
|
794
|
+
const hasShardId = (0, persistanceUtilities_js_1.readBoolean)(index);
|
|
795
|
+
const shardId = hasShardId
|
|
796
|
+
? (0, utilities_js_1.stableIdFromNumericUuid)((0, persistanceUtilities_js_1.readNumericUuid)(index))
|
|
797
|
+
: undefined;
|
|
798
|
+
// Build sharding state object, conditionally including shardId
|
|
799
|
+
compressor.shardingState = {
|
|
800
|
+
currentStride,
|
|
801
|
+
originalStride,
|
|
802
|
+
activeChildIds,
|
|
803
|
+
...(shardId !== undefined && { shardId }),
|
|
804
|
+
};
|
|
805
|
+
}
|
|
806
|
+
}
|
|
557
807
|
}
|
|
558
808
|
(0, internal_1.assert)(index.index === index.bufferFloat.length, 0x75f /* Failed to read entire serialized compressor. */);
|
|
559
809
|
return compressor;
|
|
@@ -567,44 +817,102 @@ class IdCompressor {
|
|
|
567
817
|
this.localGenCount !== other.localGenCount)) {
|
|
568
818
|
return false;
|
|
569
819
|
}
|
|
820
|
+
// Shard ownership affects future local ID allocation, even when the generated IDs match.
|
|
821
|
+
// It is not part of finalized state, so ignore it when comparing only that state.
|
|
822
|
+
if (includeLocalState) {
|
|
823
|
+
const state = this.shardingState;
|
|
824
|
+
const otherState = other.shardingState;
|
|
825
|
+
if (state === undefined || otherState === undefined) {
|
|
826
|
+
// Two unsharded compressors match here; a sharded and an unsharded compressor do not.
|
|
827
|
+
if (state !== otherState) {
|
|
828
|
+
return false;
|
|
829
|
+
}
|
|
830
|
+
}
|
|
831
|
+
else {
|
|
832
|
+
if (state.currentStride !== otherState.currentStride ||
|
|
833
|
+
state.originalStride !== otherState.originalStride ||
|
|
834
|
+
state.shardId !== otherState.shardId ||
|
|
835
|
+
state.activeChildIds.size !== otherState.activeChildIds.size) {
|
|
836
|
+
return false;
|
|
837
|
+
}
|
|
838
|
+
// Child ownership is an unordered set. Equal sizes and membership imply equal sets,
|
|
839
|
+
// regardless of insertion order or whether the state was deserialized.
|
|
840
|
+
for (const childId of state.activeChildIds) {
|
|
841
|
+
if (!otherState.activeChildIds.has(childId)) {
|
|
842
|
+
return false;
|
|
843
|
+
}
|
|
844
|
+
}
|
|
845
|
+
}
|
|
846
|
+
}
|
|
570
847
|
return (this.sessions.equals(other.sessions, includeLocalState) &&
|
|
571
848
|
this.finalSpace.equals(other.finalSpace));
|
|
572
849
|
}
|
|
573
850
|
}
|
|
574
851
|
exports.IdCompressor = IdCompressor;
|
|
575
|
-
function createIdCompressor(
|
|
852
|
+
function createIdCompressor(sessionIdOrDocumentVersion, writeVersionOrLogger, loggerOrUndefined) {
|
|
576
853
|
let localSessionId;
|
|
854
|
+
let writeVersion;
|
|
577
855
|
let logger;
|
|
578
|
-
if (
|
|
579
|
-
|
|
856
|
+
if (typeof sessionIdOrDocumentVersion === "string") {
|
|
857
|
+
// Called with sessionId, writeVersion, logger?
|
|
858
|
+
localSessionId = sessionIdOrDocumentVersion;
|
|
859
|
+
writeVersion = writeVersionOrLogger;
|
|
860
|
+
logger = loggerOrUndefined;
|
|
580
861
|
}
|
|
581
862
|
else {
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
else {
|
|
587
|
-
localSessionId = (0, utilities_js_1.createSessionId)();
|
|
588
|
-
logger = sessionIdOrLogger;
|
|
589
|
-
}
|
|
863
|
+
// Called with writeVersion, logger?
|
|
864
|
+
localSessionId = (0, utilities_js_1.createSessionId)();
|
|
865
|
+
writeVersion = sessionIdOrDocumentVersion;
|
|
866
|
+
logger = writeVersionOrLogger;
|
|
590
867
|
}
|
|
591
|
-
const compressor = new IdCompressor(localSessionId, logger === undefined ? undefined : (0, internal_2.createChildLogger)({ logger }));
|
|
868
|
+
const compressor = new IdCompressor(localSessionId, logger === undefined ? undefined : (0, internal_2.createChildLogger)({ logger }), writeVersion);
|
|
592
869
|
return compressor;
|
|
593
870
|
}
|
|
594
|
-
function deserializeIdCompressor(serialized,
|
|
595
|
-
if (typeof
|
|
871
|
+
function deserializeIdCompressor(serialized, sessionIdOrDocumentVersion, writeVersionOrLogger, loggerOrUndefined) {
|
|
872
|
+
if (typeof sessionIdOrDocumentVersion === "string") {
|
|
873
|
+
// Called with (serialized, sessionId, writeVersion, logger?)
|
|
596
874
|
return IdCompressor.deserialize({
|
|
597
875
|
serialized: serialized,
|
|
598
876
|
logger: (0, internal_2.extractTelemetryLoggerExt)(loggerOrUndefined),
|
|
599
|
-
newSessionId:
|
|
877
|
+
newSessionId: sessionIdOrDocumentVersion,
|
|
878
|
+
requestedWriteVersion: writeVersionOrLogger,
|
|
600
879
|
});
|
|
601
880
|
}
|
|
602
881
|
(0, internal_1.assert)(loggerOrUndefined === undefined, 0xc2d /* logger would be in sessionIdOrLogger in this codepath */);
|
|
882
|
+
// Called with (serialized, writeVersion, logger?)
|
|
603
883
|
return IdCompressor.deserialize({
|
|
604
884
|
serialized: serialized,
|
|
605
|
-
logger: (0, internal_2.extractTelemetryLoggerExt)(
|
|
885
|
+
logger: (0, internal_2.extractTelemetryLoggerExt)(writeVersionOrLogger),
|
|
886
|
+
requestedWriteVersion: sessionIdOrDocumentVersion,
|
|
606
887
|
});
|
|
607
888
|
}
|
|
889
|
+
/**
|
|
890
|
+
* Throws an error indicating the compressor has been disposed.
|
|
891
|
+
*/
|
|
892
|
+
const throwDisposed = () => {
|
|
893
|
+
throw new Error("This compressor has been disposed via disposeShard() and can no longer be used.");
|
|
894
|
+
};
|
|
895
|
+
/**
|
|
896
|
+
* Makes an IdCompressor instance unusable by replacing all its methods with functions that throw.
|
|
897
|
+
* This is called once when disposeShard() is invoked to make the compressor permanently unusable.
|
|
898
|
+
* @param compressor - The compressor instance to make unusable
|
|
899
|
+
*/
|
|
900
|
+
function makeCompressorUnusable(compressor) {
|
|
901
|
+
// Get all method names from the prototype (where class methods live)
|
|
902
|
+
const proto = Object.getPrototypeOf(compressor);
|
|
903
|
+
for (const key of Reflect.ownKeys(proto)) {
|
|
904
|
+
if (key !== "constructor" && typeof Reflect.get(proto, key) === "function") {
|
|
905
|
+
// Shadow the method only on this instance so other compressors remain usable.
|
|
906
|
+
// Omitting `enumerable` keeps new overrides non-enumerable, like class methods.
|
|
907
|
+
// Reflection supports string and symbol keys without adding an index signature.
|
|
908
|
+
Object.defineProperty(compressor, key, {
|
|
909
|
+
value: throwDisposed,
|
|
910
|
+
configurable: true,
|
|
911
|
+
writable: true,
|
|
912
|
+
});
|
|
913
|
+
}
|
|
914
|
+
}
|
|
915
|
+
}
|
|
608
916
|
function serializeIdCompressor(compressor, withSession) {
|
|
609
917
|
const core = toIdCompressorWithCore(compressor);
|
|
610
918
|
return withSession ? core.serialize(true) : core.serialize(false);
|