flatpack-json 9.6.4 → 9.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/CompactStorage.d.mts +10 -0
- package/dist/CompactStorage.mjs +7 -0
- package/dist/Flatpack.d.mts +2 -86
- package/dist/Flatpack.mjs +5 -610
- package/dist/FlatpackData.d.mts +24 -0
- package/dist/FlatpackData.mjs +122 -0
- package/dist/FlatpackV1.d.mts +100 -0
- package/dist/FlatpackV1.mjs +642 -0
- package/dist/FlatpackV2.d.mts +16 -0
- package/dist/FlatpackV2.mjs +41 -0
- package/dist/RefCounter.d.mts +17 -0
- package/dist/RefCounter.mjs +42 -0
- package/dist/RefElements.d.mts +2 -2
- package/dist/Trie.d.mts +37 -7
- package/dist/Trie.mjs +95 -4
- package/dist/WeakCache.d.mts +15 -0
- package/dist/WeakCache.mjs +41 -0
- package/dist/flatpackUtil.d.mts +1 -2
- package/dist/flatpacked.d.mts +14 -0
- package/dist/flatpacked.mjs +129 -0
- package/dist/optimizeFlatpacked.mjs +54 -71
- package/dist/proxy.mjs +0 -1
- package/dist/storage.d.mts +3 -64
- package/dist/storage.mjs +31 -439
- package/dist/storageV1.d.mts +67 -0
- package/dist/storageV1.mjs +445 -0
- package/dist/storageV2.d.mts +70 -0
- package/dist/storageV2.mjs +451 -0
- package/dist/stringTable.d.mts +39 -0
- package/dist/stringTable.mjs +213 -0
- package/dist/stringify.d.mts +6 -1
- package/dist/stringify.mjs +31 -2
- package/dist/types.d.mts +63 -15
- package/dist/types.mjs +6 -2
- package/dist/unpack.d.mts +1 -1
- package/dist/unpack.mjs +70 -31
- package/dist/unpackedAnnotation.d.mts +23 -0
- package/dist/unpackedAnnotation.mjs +32 -0
- package/package.json +5 -5
|
@@ -0,0 +1,451 @@
|
|
|
1
|
+
import { CompactStorage } from './CompactStorage.mjs';
|
|
2
|
+
import { FlatpackData } from './FlatpackData.mjs';
|
|
3
|
+
import { extractObjectKeyAndValueIndexes, extractObjectKeyAndValueIndexesFrom, getFlatpackedRootIdx, } from './flatpacked.mjs';
|
|
4
|
+
import { optimizeFlatpacked } from './optimizeFlatpacked.mjs';
|
|
5
|
+
import { RefCounter } from './RefCounter.mjs';
|
|
6
|
+
import { Trie } from './Trie.mjs';
|
|
7
|
+
import { dataHeaderV2_0, ElementType } from './types.mjs';
|
|
8
|
+
import { extractUnpackedAnnotation } from './unpackedAnnotation.mjs';
|
|
9
|
+
import { WeakCache } from './WeakCache.mjs';
|
|
10
|
+
const collator = new Intl.Collator('en', {
|
|
11
|
+
usage: 'sort',
|
|
12
|
+
numeric: true,
|
|
13
|
+
sensitivity: 'variant',
|
|
14
|
+
caseFirst: 'upper',
|
|
15
|
+
ignorePunctuation: false,
|
|
16
|
+
});
|
|
17
|
+
const compare = collator.compare;
|
|
18
|
+
export class CompactStorageV2 extends CompactStorage {
|
|
19
|
+
data;
|
|
20
|
+
stringTable;
|
|
21
|
+
dedupe = true;
|
|
22
|
+
sortKeys = true;
|
|
23
|
+
emptyObjIdx = 0;
|
|
24
|
+
/**
|
|
25
|
+
* Cache of primitives and objects that have been added to the data.
|
|
26
|
+
*/
|
|
27
|
+
cache = new WeakCache([[undefined, 0]]);
|
|
28
|
+
/**
|
|
29
|
+
* Set of indexes that have been referenced by other indexes.
|
|
30
|
+
*/
|
|
31
|
+
referencedFromCache = new RefCounter();
|
|
32
|
+
/**
|
|
33
|
+
* Cache of arrays that have been deduped.
|
|
34
|
+
* The key is a hash of the array elements as a function of the index of the element.
|
|
35
|
+
*/
|
|
36
|
+
cachedArrays = new Map();
|
|
37
|
+
cachedElementsTrie = new Trie();
|
|
38
|
+
unpackMetaData;
|
|
39
|
+
used = new Set();
|
|
40
|
+
constructor(options) {
|
|
41
|
+
super(options);
|
|
42
|
+
this.dedupe = options?.dedupe ?? true;
|
|
43
|
+
this.sortKeys = options?.sortKeys || this.dedupe;
|
|
44
|
+
this.unpackMetaData = undefined;
|
|
45
|
+
this.data = new FlatpackData(undefined);
|
|
46
|
+
this.stringTable = this.data.stringTable;
|
|
47
|
+
if (options?.meta) {
|
|
48
|
+
this.useFlatpackMetaData(options.meta);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
primitiveToIdx(value) {
|
|
52
|
+
if (typeof value === 'string')
|
|
53
|
+
return this.stringToIdx(value);
|
|
54
|
+
if (typeof value === 'bigint')
|
|
55
|
+
return this.bigintToIdx(value);
|
|
56
|
+
if (value === undefined)
|
|
57
|
+
return 0;
|
|
58
|
+
const found = this.#getFromCacheAndReference(value);
|
|
59
|
+
if (found !== undefined && this.data.get(found) === value) {
|
|
60
|
+
return found;
|
|
61
|
+
}
|
|
62
|
+
return this.#setElement(value);
|
|
63
|
+
}
|
|
64
|
+
addStringPrimitive(value) {
|
|
65
|
+
return -this.stringTable.add(value);
|
|
66
|
+
}
|
|
67
|
+
stringToIdx(value) {
|
|
68
|
+
const found = this.#getFromCacheAndReference(value);
|
|
69
|
+
if (found !== undefined) {
|
|
70
|
+
return found;
|
|
71
|
+
}
|
|
72
|
+
return this.addStringPrimitive(value);
|
|
73
|
+
}
|
|
74
|
+
objSetToIdx(value) {
|
|
75
|
+
const found = this.#getFromCacheAndReference(value);
|
|
76
|
+
if (found !== undefined) {
|
|
77
|
+
return found;
|
|
78
|
+
}
|
|
79
|
+
const idx = this.#reserverForValue(value);
|
|
80
|
+
const keys = [...value];
|
|
81
|
+
const k = this.createUniqueKeys(keys, false);
|
|
82
|
+
const element = [ElementType.Set, k];
|
|
83
|
+
return this.storeElement(value, idx, element);
|
|
84
|
+
}
|
|
85
|
+
createUniqueKeys(keys, cacheValue = true) {
|
|
86
|
+
let k = this.arrToIdx(keys, cacheValue);
|
|
87
|
+
const elementKeys = this.data.get(k);
|
|
88
|
+
const uniqueKeys = new Set(elementKeys.slice(1));
|
|
89
|
+
if (uniqueKeys.size !== keys.length) {
|
|
90
|
+
// one or more of the keys got deduped. We need to duplicate it.
|
|
91
|
+
uniqueKeys.clear();
|
|
92
|
+
const indexes = elementKeys.slice(1).map((idx) => {
|
|
93
|
+
if (uniqueKeys.has(idx)) {
|
|
94
|
+
return this.data.duplicateIndex(idx);
|
|
95
|
+
}
|
|
96
|
+
uniqueKeys.add(idx);
|
|
97
|
+
return idx;
|
|
98
|
+
});
|
|
99
|
+
k = this.createArrayElementFromIndexValues(this.data.reserve(), indexes);
|
|
100
|
+
}
|
|
101
|
+
return k;
|
|
102
|
+
}
|
|
103
|
+
objMapToIdx(value) {
|
|
104
|
+
const found = this.#getFromCacheAndReference(value);
|
|
105
|
+
if (found !== undefined) {
|
|
106
|
+
return found;
|
|
107
|
+
}
|
|
108
|
+
const idx = this.#reserverForValue(value);
|
|
109
|
+
const entries = [...value.entries()];
|
|
110
|
+
const k = this.createUniqueKeys(entries.map(([key]) => key), false);
|
|
111
|
+
const v = this.arrToIdx(entries.map(([, value]) => value), false);
|
|
112
|
+
const element = [ElementType.Map, k, v];
|
|
113
|
+
return this.storeElement(value, idx, element);
|
|
114
|
+
}
|
|
115
|
+
objRegExpToIdx(value) {
|
|
116
|
+
const found = this.#getFromCacheAndReference(value);
|
|
117
|
+
if (found !== undefined) {
|
|
118
|
+
return found;
|
|
119
|
+
}
|
|
120
|
+
const idx = this.#reserverForValue(value);
|
|
121
|
+
const element = [
|
|
122
|
+
ElementType.RegExp,
|
|
123
|
+
this.stringToIdx(value.source),
|
|
124
|
+
this.stringToIdx(value.flags),
|
|
125
|
+
];
|
|
126
|
+
return this.storeElement(value, idx, element);
|
|
127
|
+
}
|
|
128
|
+
objDateToIdx(value) {
|
|
129
|
+
const found = this.#getFromCacheAndReference(value);
|
|
130
|
+
if (found !== undefined) {
|
|
131
|
+
return found;
|
|
132
|
+
}
|
|
133
|
+
const idx = this.#reserverForValue(value);
|
|
134
|
+
const element = [ElementType.Date, value.getTime()];
|
|
135
|
+
return this.storeElement(value, idx, element);
|
|
136
|
+
}
|
|
137
|
+
bigintToIdx(value) {
|
|
138
|
+
const found = this.#getFromCacheAndReference(value);
|
|
139
|
+
if (found !== undefined) {
|
|
140
|
+
return found;
|
|
141
|
+
}
|
|
142
|
+
const idx = this.#reserverForValue(value);
|
|
143
|
+
const element = [
|
|
144
|
+
ElementType.BigInt,
|
|
145
|
+
this.primitiveToIdx(value <= Number.MAX_SAFE_INTEGER && value >= -Number.MAX_SAFE_INTEGER
|
|
146
|
+
? Number(value)
|
|
147
|
+
: value.toString()),
|
|
148
|
+
];
|
|
149
|
+
return this.storeElement(value, idx, element);
|
|
150
|
+
}
|
|
151
|
+
objToIdx(value) {
|
|
152
|
+
const found = this.#getFromCacheAndReference(value);
|
|
153
|
+
if (found !== undefined) {
|
|
154
|
+
return found;
|
|
155
|
+
}
|
|
156
|
+
if (isObjectWrapper(value)) {
|
|
157
|
+
const idx = this.data.add({});
|
|
158
|
+
this.#cacheValue(value, idx);
|
|
159
|
+
const element = [ElementType.Object, 0, this.valueToIdx(value.valueOf())];
|
|
160
|
+
return this.storeElement(value, idx, element);
|
|
161
|
+
}
|
|
162
|
+
const entries = Object.entries(value);
|
|
163
|
+
if (!entries.length) {
|
|
164
|
+
if (this.emptyObjIdx) {
|
|
165
|
+
return this.emptyObjIdx;
|
|
166
|
+
}
|
|
167
|
+
const idx = this.data.add({});
|
|
168
|
+
this.emptyObjIdx = idx;
|
|
169
|
+
return idx;
|
|
170
|
+
}
|
|
171
|
+
if (this.sortKeys) {
|
|
172
|
+
entries.sort(([a], [b]) => compare(a, b));
|
|
173
|
+
}
|
|
174
|
+
const idx = this.#reserverForValue(value);
|
|
175
|
+
const kvp = this.#getObjectKeyValueIndexesFromMetaData(idx);
|
|
176
|
+
const k = this.objectKeysOrValuesToIdx(entries.map(([key]) => key), kvp?.[0]);
|
|
177
|
+
const v = this.objectKeysOrValuesToIdx(entries.map(([, value]) => value), kvp?.[1]);
|
|
178
|
+
const element = [ElementType.Object, k, v];
|
|
179
|
+
return this.storeElement(value, idx, element);
|
|
180
|
+
}
|
|
181
|
+
objectKeysOrValuesToIdx(keys, idx) {
|
|
182
|
+
const element = [ElementType.Array, ...this.mapValuesToIndexes(keys)];
|
|
183
|
+
const cachedIdx = this.cacheElement(element, idx);
|
|
184
|
+
const useIdx = idx ?? cachedIdx;
|
|
185
|
+
this.data.set(useIdx, element);
|
|
186
|
+
return useIdx;
|
|
187
|
+
}
|
|
188
|
+
mapValuesToIndexes(values) {
|
|
189
|
+
return values.map((value) => this.valueToIdx(value));
|
|
190
|
+
}
|
|
191
|
+
/**
|
|
192
|
+
* Store an element in the data array, using the cache if deduplication is enabled.
|
|
193
|
+
* @param value - the value being stored, used for caching purposes.
|
|
194
|
+
* @param idx - the index at which to store the element if it is not found in the cache.
|
|
195
|
+
* @param element - the element to store if it is not found in the cache.
|
|
196
|
+
* @returns the index at which the element is stored.
|
|
197
|
+
*/
|
|
198
|
+
storeElement(value, idx, element) {
|
|
199
|
+
const useIdx = this.dedupe ? this.cacheElement(element, idx) : idx;
|
|
200
|
+
if (useIdx !== idx) {
|
|
201
|
+
this.data.delete(idx);
|
|
202
|
+
return this.#cacheValue(value, useIdx);
|
|
203
|
+
}
|
|
204
|
+
this.data.set(idx, element);
|
|
205
|
+
return idx;
|
|
206
|
+
}
|
|
207
|
+
cacheElement(element, elemIdx) {
|
|
208
|
+
const foundIdx = this.cachedElementsTrie.get(element);
|
|
209
|
+
if (foundIdx === undefined) {
|
|
210
|
+
const idx = elemIdx ?? this.data.reserve();
|
|
211
|
+
this.cachedElementsTrie.set(element, idx);
|
|
212
|
+
return idx;
|
|
213
|
+
}
|
|
214
|
+
if (elemIdx && this.referencedFromCache.isReferenced(elemIdx)) {
|
|
215
|
+
if (!this.referencedFromCache.isReferenced(foundIdx)) {
|
|
216
|
+
this.cachedElementsTrie.set(element, elemIdx);
|
|
217
|
+
}
|
|
218
|
+
return elemIdx;
|
|
219
|
+
}
|
|
220
|
+
const foundElement = this.data.get(foundIdx);
|
|
221
|
+
if (!this.referencedFromCache.isReferenced(foundIdx) && !isArrayEqual(foundElement, element)) {
|
|
222
|
+
const idx = elemIdx ?? this.data.reserve();
|
|
223
|
+
this.cachedElementsTrie.set(element, idx);
|
|
224
|
+
return idx;
|
|
225
|
+
}
|
|
226
|
+
return foundIdx;
|
|
227
|
+
}
|
|
228
|
+
stashArray(idx, element) {
|
|
229
|
+
const indexHash = simpleHash(element);
|
|
230
|
+
let found = this.cachedArrays.get(indexHash);
|
|
231
|
+
if (!found) {
|
|
232
|
+
found = [];
|
|
233
|
+
this.cachedArrays.set(indexHash, found);
|
|
234
|
+
}
|
|
235
|
+
// It is possible for an array to have a circular reference to itself (possibly through a nested object).
|
|
236
|
+
// In that case, we want to treat it as a unique array and not dedupe
|
|
237
|
+
// it with other arrays that have the same content.
|
|
238
|
+
if (this.referencedFromCache.isReferenced(idx)) {
|
|
239
|
+
found.push(idx);
|
|
240
|
+
return idx;
|
|
241
|
+
}
|
|
242
|
+
const foundIdx = found.find((entryIdx) => isArrayEqual(this.data.get(entryIdx), element));
|
|
243
|
+
if (foundIdx) {
|
|
244
|
+
return foundIdx;
|
|
245
|
+
}
|
|
246
|
+
found.push(idx);
|
|
247
|
+
return idx;
|
|
248
|
+
}
|
|
249
|
+
createArrayElementFromIndexValues(idx, indexValues) {
|
|
250
|
+
const element = [ElementType.Array, ...indexValues];
|
|
251
|
+
const useIdx = this.dedupe ? this.stashArray(idx, element) : idx;
|
|
252
|
+
if (useIdx !== idx) {
|
|
253
|
+
this.data.delete(idx);
|
|
254
|
+
this.data.markUsed(useIdx);
|
|
255
|
+
return useIdx;
|
|
256
|
+
}
|
|
257
|
+
this.data.set(idx, element);
|
|
258
|
+
return idx;
|
|
259
|
+
}
|
|
260
|
+
/**
|
|
261
|
+
* Convert an array to an index.
|
|
262
|
+
* @param value - The array to convert to an index.
|
|
263
|
+
* @param cacheValue - Whether to cache the value.
|
|
264
|
+
* @returns the index of the array.
|
|
265
|
+
*/
|
|
266
|
+
arrToIdx(value, cacheValue = true) {
|
|
267
|
+
const found = this.#getFromCacheAndReference(value);
|
|
268
|
+
if (found !== undefined) {
|
|
269
|
+
return found;
|
|
270
|
+
}
|
|
271
|
+
const idx = this.#reserverForValue(value, cacheValue);
|
|
272
|
+
const useIdx = this.createArrayElementFromIndexValues(idx, value.map((idx) => this.valueToIdx(idx)));
|
|
273
|
+
if (cacheValue) {
|
|
274
|
+
this.cache.set(value, useIdx);
|
|
275
|
+
}
|
|
276
|
+
return useIdx;
|
|
277
|
+
}
|
|
278
|
+
valueToIdx(value) {
|
|
279
|
+
if (value === null) {
|
|
280
|
+
return this.primitiveToIdx(value);
|
|
281
|
+
}
|
|
282
|
+
if (typeof value === 'object') {
|
|
283
|
+
if (value instanceof Set) {
|
|
284
|
+
return this.objSetToIdx(value);
|
|
285
|
+
}
|
|
286
|
+
if (value instanceof Map) {
|
|
287
|
+
return this.objMapToIdx(value);
|
|
288
|
+
}
|
|
289
|
+
if (value instanceof RegExp) {
|
|
290
|
+
return this.objRegExpToIdx(value);
|
|
291
|
+
}
|
|
292
|
+
if (Array.isArray(value)) {
|
|
293
|
+
return this.arrToIdx(value);
|
|
294
|
+
}
|
|
295
|
+
if (value instanceof Date) {
|
|
296
|
+
return this.objDateToIdx(value);
|
|
297
|
+
}
|
|
298
|
+
return this.objToIdx(value);
|
|
299
|
+
}
|
|
300
|
+
return this.primitiveToIdx(value);
|
|
301
|
+
}
|
|
302
|
+
#reserverForValue(value, cacheValue = true) {
|
|
303
|
+
const valueIdx = this.#getAvailableValueIndexFromAnnotation(value);
|
|
304
|
+
const idx = valueIdx ?? this.data.reserve();
|
|
305
|
+
this.data.markUsed(idx);
|
|
306
|
+
if (cacheValue) {
|
|
307
|
+
this.cache.set(value, idx);
|
|
308
|
+
}
|
|
309
|
+
return idx;
|
|
310
|
+
}
|
|
311
|
+
#getAvailableValueIndexFromAnnotation(value) {
|
|
312
|
+
const idx = this.#getValueIndexFromAnnotation(value);
|
|
313
|
+
return idx !== undefined && !this.data.isUsed(idx) ? idx : undefined;
|
|
314
|
+
}
|
|
315
|
+
#getValueIndexFromAnnotation(value) {
|
|
316
|
+
const annotation = extractUnpackedAnnotation(value);
|
|
317
|
+
if (!annotation || annotation.meta !== this.unpackMetaData)
|
|
318
|
+
return undefined;
|
|
319
|
+
return annotation.index;
|
|
320
|
+
}
|
|
321
|
+
#getSrcElementFromMetaData(idx) {
|
|
322
|
+
const flatpack = this.unpackMetaData?.flatpack;
|
|
323
|
+
return flatpack?.[idx];
|
|
324
|
+
}
|
|
325
|
+
#getObjectKeyValueIndexesFromMetaData(idx) {
|
|
326
|
+
const kvp = extractObjectKeyAndValueIndexesFrom(this.#getSrcElementFromMetaData(idx));
|
|
327
|
+
if (!kvp)
|
|
328
|
+
return undefined;
|
|
329
|
+
const [k, v] = kvp;
|
|
330
|
+
const key = (k && this.data.claimOwnership(k, idx) === idx && k) || undefined;
|
|
331
|
+
const value = (v && this.data.claimOwnership(v, idx) === idx && v) || undefined;
|
|
332
|
+
return [key, value];
|
|
333
|
+
}
|
|
334
|
+
#setElement(value) {
|
|
335
|
+
const idx = this.data.add(value);
|
|
336
|
+
this.#cacheValue(value, idx);
|
|
337
|
+
return idx;
|
|
338
|
+
}
|
|
339
|
+
/**
|
|
340
|
+
* Reset things in a way that allows for reuse.
|
|
341
|
+
*/
|
|
342
|
+
softReset() {
|
|
343
|
+
this.cache.clear();
|
|
344
|
+
this.cache.set(undefined, 0);
|
|
345
|
+
this.cachedArrays.clear();
|
|
346
|
+
this.cachedElementsTrie.clear();
|
|
347
|
+
this.referencedFromCache.clear();
|
|
348
|
+
this.used.clear();
|
|
349
|
+
this.data = new FlatpackData([dataHeaderV2_0, this.stringTable.build()]);
|
|
350
|
+
this.stringTable = this.data.stringTable;
|
|
351
|
+
this.emptyObjIdx = 0;
|
|
352
|
+
}
|
|
353
|
+
useFlatpackMetaData(data) {
|
|
354
|
+
if (!data || data.flatpack[0] !== dataHeaderV2_0) {
|
|
355
|
+
return;
|
|
356
|
+
}
|
|
357
|
+
this.softReset();
|
|
358
|
+
this.unpackMetaData = data;
|
|
359
|
+
this.data = new FlatpackData(data.flatpack);
|
|
360
|
+
this.stringTable = this.data.stringTable;
|
|
361
|
+
this.initFromFlatpackData();
|
|
362
|
+
// Clear the referenced indexes since we don't want to treat them as referenced when we re-pack the data.
|
|
363
|
+
this.referencedFromCache.clear();
|
|
364
|
+
}
|
|
365
|
+
/**
|
|
366
|
+
* Cache the primitives from the unpacked data to improve performance when re-packing the same data.
|
|
367
|
+
*/
|
|
368
|
+
initFromFlatpackData() {
|
|
369
|
+
const data = this.data.flatpack;
|
|
370
|
+
const rootIndex = getFlatpackedRootIdx(data);
|
|
371
|
+
const idxOfObjectKeysAndValues = new Set();
|
|
372
|
+
for (let i = rootIndex; i < data.length; ++i) {
|
|
373
|
+
const objKeysAndValues = extractObjectKeyAndValueIndexes(data[i]);
|
|
374
|
+
for (const idx of objKeysAndValues) {
|
|
375
|
+
idxOfObjectKeysAndValues.add(idx);
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
for (let i = rootIndex + 1; i < data.length; ++i) {
|
|
379
|
+
this.useFlattenedElement(data[i], i, idxOfObjectKeysAndValues);
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
useFlattenedElement(element, index, idxOfObjectKeysAndValues) {
|
|
383
|
+
// Primitives and null are cached directly.
|
|
384
|
+
if (!element || typeof element !== 'object') {
|
|
385
|
+
this.cache.set(element, index);
|
|
386
|
+
return;
|
|
387
|
+
}
|
|
388
|
+
if (!Array.isArray(element))
|
|
389
|
+
return;
|
|
390
|
+
// Arrays are cached based on their content, so we need to handle them separately.
|
|
391
|
+
if (!idxOfObjectKeysAndValues.has(index) && element[0] === ElementType.Array) {
|
|
392
|
+
this.stashArray(index, element);
|
|
393
|
+
return;
|
|
394
|
+
}
|
|
395
|
+
// Other objects are cached based on their identity, so we can cache them directly.
|
|
396
|
+
this.cachedElementsTrie.set(element, index);
|
|
397
|
+
}
|
|
398
|
+
#cacheValue(value, idx) {
|
|
399
|
+
this.cache.set(value, idx);
|
|
400
|
+
return idx;
|
|
401
|
+
}
|
|
402
|
+
#getFromCacheAndReference(value) {
|
|
403
|
+
const found = this.cache.get(value);
|
|
404
|
+
if (found !== undefined) {
|
|
405
|
+
this.data.markUsed(found);
|
|
406
|
+
this.#addReference(found);
|
|
407
|
+
}
|
|
408
|
+
return found;
|
|
409
|
+
}
|
|
410
|
+
#addReference(idx) {
|
|
411
|
+
this.referencedFromCache.add(idx);
|
|
412
|
+
}
|
|
413
|
+
toJSON(json) {
|
|
414
|
+
this.softReset();
|
|
415
|
+
const annotation = extractUnpackedAnnotation(json);
|
|
416
|
+
this.useFlatpackMetaData(this.unpackMetaData ?? annotation?.meta);
|
|
417
|
+
const lastIdx = this.valueToIdx(json);
|
|
418
|
+
if (lastIdx < 0) {
|
|
419
|
+
this.data.add([ElementType.String, lastIdx]);
|
|
420
|
+
}
|
|
421
|
+
const data = this.data.finalize();
|
|
422
|
+
return this.options?.optimize ? optimizeFlatpacked(data) : data;
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
function isArrayEqual(a, b) {
|
|
426
|
+
if (!Array.isArray(a) || !Array.isArray(b))
|
|
427
|
+
return false;
|
|
428
|
+
if (a === b)
|
|
429
|
+
return true;
|
|
430
|
+
if (a.length !== b.length)
|
|
431
|
+
return false;
|
|
432
|
+
for (let i = 0; i < a.length; i++) {
|
|
433
|
+
if (a[i] !== b[i])
|
|
434
|
+
return false;
|
|
435
|
+
}
|
|
436
|
+
return true;
|
|
437
|
+
}
|
|
438
|
+
function simpleHash(values) {
|
|
439
|
+
let hash = Math.sqrt(values.length);
|
|
440
|
+
for (const value of values) {
|
|
441
|
+
hash += value * value;
|
|
442
|
+
}
|
|
443
|
+
return hash;
|
|
444
|
+
}
|
|
445
|
+
function isObjectWrapper(value) {
|
|
446
|
+
return (typeof value === 'object' &&
|
|
447
|
+
value !== null &&
|
|
448
|
+
typeof value.valueOf === 'function' &&
|
|
449
|
+
value.valueOf() !== value);
|
|
450
|
+
}
|
|
451
|
+
//# sourceMappingURL=storageV2.mjs.map
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import type { StringTableElement } from './types.mjs';
|
|
2
|
+
export declare class StringTable {
|
|
3
|
+
#private;
|
|
4
|
+
readonly stringTableElement: StringTableElement;
|
|
5
|
+
constructor(stringTableElement: StringTableElement);
|
|
6
|
+
get(index: number): string | undefined;
|
|
7
|
+
entries(): Iterable<[number, string]>;
|
|
8
|
+
values(): Iterable<string>;
|
|
9
|
+
get size(): number;
|
|
10
|
+
}
|
|
11
|
+
export declare class StringTableBuilder {
|
|
12
|
+
#private;
|
|
13
|
+
splitStrings: boolean;
|
|
14
|
+
tokenRegex: RegExp;
|
|
15
|
+
constructor(stringTableElement?: StringTableElement);
|
|
16
|
+
set splitIntoTokensWhenAdding(value: boolean);
|
|
17
|
+
get splitIntoTokensWhenAdding(): boolean;
|
|
18
|
+
add(str: string): number;
|
|
19
|
+
getIndex(str: string): number | undefined;
|
|
20
|
+
get(index: number): string | undefined;
|
|
21
|
+
/**
|
|
22
|
+
* Increments the reference count for the given index.
|
|
23
|
+
* @param index - The index of the string in the string table. The absolute value is used.
|
|
24
|
+
* @returns the new reference count for the string at the given index.
|
|
25
|
+
*/
|
|
26
|
+
addRef(index: number): number;
|
|
27
|
+
getRefCount(index: number): number;
|
|
28
|
+
clearUnusedEntries(): this;
|
|
29
|
+
tokenizeAllEntries(): this;
|
|
30
|
+
/**
|
|
31
|
+
* Sorts the entries in the string table by reference count, with the most referenced strings first.
|
|
32
|
+
* This can help reduce the size of the string table when serialized, as more frequently used strings
|
|
33
|
+
* will have smaller indexes.
|
|
34
|
+
* @returns a map of old indexes to new indexes after sorting. The index 0 is always mapped to itself.
|
|
35
|
+
*/
|
|
36
|
+
sortEntriesByRefCount(): Map<number, number>;
|
|
37
|
+
build(): StringTableElement;
|
|
38
|
+
}
|
|
39
|
+
//# sourceMappingURL=stringTable.d.mts.map
|
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
import assert from 'node:assert';
|
|
2
|
+
import { ElementType } from './types.mjs';
|
|
3
|
+
export class StringTable {
|
|
4
|
+
stringTableElement;
|
|
5
|
+
constructor(stringTableElement) {
|
|
6
|
+
this.stringTableElement = stringTableElement;
|
|
7
|
+
}
|
|
8
|
+
get(index) {
|
|
9
|
+
if (!index)
|
|
10
|
+
return '';
|
|
11
|
+
index = index < 0 ? -index : index;
|
|
12
|
+
if (index >= this.stringTableElement.length)
|
|
13
|
+
return undefined;
|
|
14
|
+
return this.#getCompoundString(index);
|
|
15
|
+
}
|
|
16
|
+
*entries() {
|
|
17
|
+
for (let i = 1; i < this.stringTableElement.length; i++) {
|
|
18
|
+
yield [i, this.#getCompoundString(i)];
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
*values() {
|
|
22
|
+
for (const entry of this.entries()) {
|
|
23
|
+
yield entry[1];
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
get size() {
|
|
27
|
+
return this.stringTableElement.length;
|
|
28
|
+
}
|
|
29
|
+
#getCompoundString(index, visited = new Set()) {
|
|
30
|
+
if (visited.has(index)) {
|
|
31
|
+
throw new Error(`Circular reference in string table at index ${index}`);
|
|
32
|
+
}
|
|
33
|
+
const entry = this.stringTableElement[index];
|
|
34
|
+
if (typeof entry === 'string') {
|
|
35
|
+
return entry;
|
|
36
|
+
}
|
|
37
|
+
if (Array.isArray(entry)) {
|
|
38
|
+
visited.add(index);
|
|
39
|
+
const value = entry.map((i) => this.#getCompoundString(i, visited)).join('');
|
|
40
|
+
visited.delete(index);
|
|
41
|
+
return value;
|
|
42
|
+
}
|
|
43
|
+
throw new Error(`Invalid string table entry at index ${index}`);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
const tokenRegex = /\w+/g;
|
|
47
|
+
export class StringTableBuilder {
|
|
48
|
+
splitStrings = false;
|
|
49
|
+
#stringToIndex = new Map();
|
|
50
|
+
#entries = [{ value: '', entry: '', refCount: 0 }];
|
|
51
|
+
#availableIndexes = [];
|
|
52
|
+
tokenRegex = tokenRegex;
|
|
53
|
+
#splitIntoTokens = false;
|
|
54
|
+
constructor(stringTableElement) {
|
|
55
|
+
if (!stringTableElement)
|
|
56
|
+
return;
|
|
57
|
+
const st = new StringTable(stringTableElement);
|
|
58
|
+
for (const [idx, value] of st.entries()) {
|
|
59
|
+
if (!idx)
|
|
60
|
+
continue;
|
|
61
|
+
const entry = stringTableElement[idx];
|
|
62
|
+
this.#entries[idx] = { value, entry, refCount: 0 };
|
|
63
|
+
if (Array.isArray(entry) && !entry.length) {
|
|
64
|
+
this.#availableIndexes.push(idx);
|
|
65
|
+
continue;
|
|
66
|
+
}
|
|
67
|
+
if (!this.#stringToIndex.has(value)) {
|
|
68
|
+
this.#stringToIndex.set(value, idx);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
set splitIntoTokensWhenAdding(value) {
|
|
73
|
+
this.#splitIntoTokens = value;
|
|
74
|
+
}
|
|
75
|
+
get splitIntoTokensWhenAdding() {
|
|
76
|
+
return this.#splitIntoTokens;
|
|
77
|
+
}
|
|
78
|
+
add(str) {
|
|
79
|
+
const found = this.#stringToIndex.get(str);
|
|
80
|
+
if (found !== undefined) {
|
|
81
|
+
const entry = this.#entries[found];
|
|
82
|
+
entry.refCount++;
|
|
83
|
+
return found;
|
|
84
|
+
}
|
|
85
|
+
str ||= '';
|
|
86
|
+
return this.#append(str);
|
|
87
|
+
}
|
|
88
|
+
getIndex(str) {
|
|
89
|
+
return this.#stringToIndex.get(str);
|
|
90
|
+
}
|
|
91
|
+
get(index) {
|
|
92
|
+
const entry = this.#getEntry(index);
|
|
93
|
+
return entry?.value;
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Increments the reference count for the given index.
|
|
97
|
+
* @param index - The index of the string in the string table. The absolute value is used.
|
|
98
|
+
* @returns the new reference count for the string at the given index.
|
|
99
|
+
*/
|
|
100
|
+
addRef(index) {
|
|
101
|
+
const entry = this.#getEntryCheckBounds(index);
|
|
102
|
+
const count = ++entry.refCount;
|
|
103
|
+
if (count === 1 && Array.isArray(entry.entry)) {
|
|
104
|
+
entry.entry.forEach((i) => this.addRef(i));
|
|
105
|
+
}
|
|
106
|
+
return count;
|
|
107
|
+
}
|
|
108
|
+
getRefCount(index) {
|
|
109
|
+
const entry = this.#getEntryCheckBounds(index);
|
|
110
|
+
return entry.refCount;
|
|
111
|
+
}
|
|
112
|
+
#getEntry(index) {
|
|
113
|
+
index = index < 0 ? -index : index;
|
|
114
|
+
return this.#entries[index];
|
|
115
|
+
}
|
|
116
|
+
#getEntryCheckBounds(index) {
|
|
117
|
+
const entry = this.#getEntry(index);
|
|
118
|
+
if (!entry) {
|
|
119
|
+
throw new Error(`Invalid string table index: ${index}`);
|
|
120
|
+
}
|
|
121
|
+
return entry;
|
|
122
|
+
}
|
|
123
|
+
#append(str) {
|
|
124
|
+
const found = this.#stringToIndex.get(str);
|
|
125
|
+
if (found !== undefined) {
|
|
126
|
+
return found;
|
|
127
|
+
}
|
|
128
|
+
const entry = { value: str, entry: str, refCount: 1 };
|
|
129
|
+
const idx = this.#availableIndexes.shift() ?? this.#entries.length;
|
|
130
|
+
this.#entries[idx] = entry;
|
|
131
|
+
this.#stringToIndex.set(str, idx);
|
|
132
|
+
if (this.#splitIntoTokens) {
|
|
133
|
+
this.#splitEntryIntoTokens(entry);
|
|
134
|
+
}
|
|
135
|
+
return idx;
|
|
136
|
+
}
|
|
137
|
+
#splitEntryIntoTokens(entry) {
|
|
138
|
+
if (Array.isArray(entry.entry))
|
|
139
|
+
return;
|
|
140
|
+
if (!entry.value)
|
|
141
|
+
return;
|
|
142
|
+
const regex = new RegExp(this.tokenRegex);
|
|
143
|
+
const indexes = [...entry.value.matchAll(regex)].flatMap((match) => [
|
|
144
|
+
match.index,
|
|
145
|
+
match.index + match[0].length,
|
|
146
|
+
]);
|
|
147
|
+
if (!indexes.length)
|
|
148
|
+
return;
|
|
149
|
+
if (indexes.length === 2 && indexes[0] === 0 && indexes[1] === entry.value.length) {
|
|
150
|
+
return;
|
|
151
|
+
}
|
|
152
|
+
const subEntries = [];
|
|
153
|
+
entry.entry = subEntries;
|
|
154
|
+
indexes.push(entry.value.length);
|
|
155
|
+
let lastIndex = 0;
|
|
156
|
+
for (const index of indexes) {
|
|
157
|
+
if (index === lastIndex)
|
|
158
|
+
continue;
|
|
159
|
+
const value = entry.value.slice(lastIndex, index);
|
|
160
|
+
subEntries.push(this.add(value));
|
|
161
|
+
lastIndex = index;
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
clearUnusedEntries() {
|
|
165
|
+
for (let i = 1; i < this.#entries.length; i++) {
|
|
166
|
+
const entry = this.#entries[i];
|
|
167
|
+
if (entry.refCount > 0)
|
|
168
|
+
continue;
|
|
169
|
+
if (this.#stringToIndex.get(entry.value) === i) {
|
|
170
|
+
this.#stringToIndex.delete(entry.value);
|
|
171
|
+
}
|
|
172
|
+
this.#entries[i] = { value: '', entry: [], refCount: 0 };
|
|
173
|
+
this.#availableIndexes.push(i);
|
|
174
|
+
}
|
|
175
|
+
return this;
|
|
176
|
+
}
|
|
177
|
+
tokenizeAllEntries() {
|
|
178
|
+
for (const entry of this.#entries) {
|
|
179
|
+
this.#splitEntryIntoTokens(entry);
|
|
180
|
+
}
|
|
181
|
+
return this;
|
|
182
|
+
}
|
|
183
|
+
/**
|
|
184
|
+
* Sorts the entries in the string table by reference count, with the most referenced strings first.
|
|
185
|
+
* This can help reduce the size of the string table when serialized, as more frequently used strings
|
|
186
|
+
* will have smaller indexes.
|
|
187
|
+
* @returns a map of old indexes to new indexes after sorting. The index 0 is always mapped to itself.
|
|
188
|
+
*/
|
|
189
|
+
sortEntriesByRefCount() {
|
|
190
|
+
const mapEntryToOldIndex = new Map(this.#entries.map((entry, index) => [entry, index]));
|
|
191
|
+
const entry0 = this.#entries[0];
|
|
192
|
+
const sorted = this.#entries.sort((a, b) => a === entry0 ? -1 : b === entry0 ? 1 : b.refCount - a.refCount || getOldIndex(a) - getOldIndex(b));
|
|
193
|
+
const oldIndexToNew = new Map(sorted.map((entry, index) => [getOldIndex(entry), index]));
|
|
194
|
+
for (const entry of this.#entries) {
|
|
195
|
+
if (!Array.isArray(entry.entry))
|
|
196
|
+
continue;
|
|
197
|
+
entry.entry = entry.entry.map((i) => oldIndexToNew.get(i) ?? i);
|
|
198
|
+
}
|
|
199
|
+
for (const [str, oldIdx] of this.#stringToIndex.entries()) {
|
|
200
|
+
this.#stringToIndex.set(str, oldIndexToNew.get(oldIdx) ?? oldIdx);
|
|
201
|
+
}
|
|
202
|
+
return oldIndexToNew;
|
|
203
|
+
function getOldIndex(entry) {
|
|
204
|
+
const oldIndex = mapEntryToOldIndex.get(entry);
|
|
205
|
+
assert(oldIndex !== undefined, 'Entry not found in map');
|
|
206
|
+
return oldIndex;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
build() {
|
|
210
|
+
return [ElementType.StringTable, ...this.#entries.slice(1).map((e) => e.entry)];
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
//# sourceMappingURL=stringTable.mjs.map
|