document-dataply 0.0.14-alpha.0 → 0.0.14-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/index.js +738 -235
- package/dist/types/core/AnalysisManager.d.ts +3 -2
- package/dist/types/core/IndexManager.d.ts +8 -2
- package/dist/types/core/MetadataManager.d.ts +3 -2
- package/dist/types/core/MutationManager.d.ts +4 -2
- package/dist/types/core/QueryManager.d.ts +3 -2
- package/dist/types/core/document.d.ts +22 -0
- package/dist/types/core/documentAPI.d.ts +22 -0
- package/dist/types/utils/{DeadlineChunker.d.ts → eventLoopManager.d.ts} +1 -0
- package/package.json +3 -3
package/dist/cjs/index.js
CHANGED
|
@@ -66,7 +66,7 @@ var require_cjs = __commonJS({
|
|
|
66
66
|
AsyncMVCCStrategy: () => AsyncMVCCStrategy2,
|
|
67
67
|
AsyncMVCCTransaction: () => AsyncMVCCTransaction2,
|
|
68
68
|
BPTreeAsync: () => BPTreeAsync2,
|
|
69
|
-
BPTreeAsyncTransaction: () =>
|
|
69
|
+
BPTreeAsyncTransaction: () => BPTreeAsyncTransaction,
|
|
70
70
|
BPTreeSync: () => BPTreeSync,
|
|
71
71
|
BPTreeSyncTransaction: () => BPTreeSyncTransaction,
|
|
72
72
|
BitmapPageManager: () => BitmapPageManager,
|
|
@@ -82,6 +82,8 @@ var require_cjs = __commonJS({
|
|
|
82
82
|
IndexPageManager: () => IndexPageManager,
|
|
83
83
|
InvertedWeakMap: () => InvertedWeakMap,
|
|
84
84
|
LRUMap: () => LRUMap2,
|
|
85
|
+
Logger: () => Logger2,
|
|
86
|
+
LoggerManager: () => LoggerManager,
|
|
85
87
|
MVCCStrategy: () => MVCCStrategy2,
|
|
86
88
|
MVCCTransaction: () => MVCCTransaction2,
|
|
87
89
|
MetadataPageManager: () => MetadataPageManager,
|
|
@@ -476,11 +478,7 @@ var require_cjs = __commonJS({
|
|
|
476
478
|
*/
|
|
477
479
|
rollback() {
|
|
478
480
|
const { created, updated, deleted } = this.getResultEntries();
|
|
479
|
-
this.
|
|
480
|
-
this.deleteBuffer.clear();
|
|
481
|
-
this.createdKeys.clear();
|
|
482
|
-
this.deletedValues.clear();
|
|
483
|
-
this.originallyExisted.clear();
|
|
481
|
+
this._cleanupAll();
|
|
484
482
|
this.committed = true;
|
|
485
483
|
if (this.root !== this) {
|
|
486
484
|
this.root.activeTransactions.delete(this);
|
|
@@ -516,6 +514,19 @@ var require_cjs = __commonJS({
|
|
|
516
514
|
}
|
|
517
515
|
return Array.from(conflicts);
|
|
518
516
|
}
|
|
517
|
+
/**
|
|
518
|
+
* Cleans up all buffers and history.
|
|
519
|
+
* This method is called by the commit method.
|
|
520
|
+
*/
|
|
521
|
+
_cleanupAll() {
|
|
522
|
+
this.writeBuffer.clear();
|
|
523
|
+
this.deleteBuffer.clear();
|
|
524
|
+
this.createdKeys.clear();
|
|
525
|
+
this.deletedValues.clear();
|
|
526
|
+
this.originallyExisted.clear();
|
|
527
|
+
this.keyVersions.clear();
|
|
528
|
+
this.bufferHistory.clear();
|
|
529
|
+
}
|
|
519
530
|
/**
|
|
520
531
|
* Cleans up both deletedCache and versionIndex based on minActiveVersion.
|
|
521
532
|
* Root transactions call this after commit to reclaim memory.
|
|
@@ -707,6 +718,7 @@ var require_cjs = __commonJS({
|
|
|
707
718
|
if (this.parent) {
|
|
708
719
|
const failure = this.parent._merge(this);
|
|
709
720
|
if (failure) {
|
|
721
|
+
this.rollback();
|
|
710
722
|
return {
|
|
711
723
|
label,
|
|
712
724
|
success: false,
|
|
@@ -717,11 +729,13 @@ var require_cjs = __commonJS({
|
|
|
717
729
|
deleted
|
|
718
730
|
};
|
|
719
731
|
}
|
|
732
|
+
this._cleanupAll();
|
|
720
733
|
this.committed = true;
|
|
721
734
|
} else {
|
|
722
735
|
if (this.writeBuffer.size > 0 || this.deleteBuffer.size > 0) {
|
|
723
736
|
const failure = this._merge(this);
|
|
724
737
|
if (failure) {
|
|
738
|
+
this.rollback();
|
|
725
739
|
return {
|
|
726
740
|
label,
|
|
727
741
|
success: false,
|
|
@@ -732,13 +746,7 @@ var require_cjs = __commonJS({
|
|
|
732
746
|
deleted: []
|
|
733
747
|
};
|
|
734
748
|
}
|
|
735
|
-
this.
|
|
736
|
-
this.deleteBuffer.clear();
|
|
737
|
-
this.createdKeys.clear();
|
|
738
|
-
this.deletedValues.clear();
|
|
739
|
-
this.originallyExisted.clear();
|
|
740
|
-
this.keyVersions.clear();
|
|
741
|
-
this.bufferHistory.clear();
|
|
749
|
+
this._cleanupAll();
|
|
742
750
|
this.localVersion = 0;
|
|
743
751
|
this.snapshotVersion = this.version;
|
|
744
752
|
}
|
|
@@ -1372,6 +1380,7 @@ var require_cjs = __commonJS({
|
|
|
1372
1380
|
if (this.parent) {
|
|
1373
1381
|
const failure = await this.parent._merge(this);
|
|
1374
1382
|
if (failure) {
|
|
1383
|
+
this.rollback();
|
|
1375
1384
|
return {
|
|
1376
1385
|
label,
|
|
1377
1386
|
success: false,
|
|
@@ -1382,11 +1391,13 @@ var require_cjs = __commonJS({
|
|
|
1382
1391
|
deleted
|
|
1383
1392
|
};
|
|
1384
1393
|
}
|
|
1394
|
+
this._cleanupAll();
|
|
1385
1395
|
this.committed = true;
|
|
1386
1396
|
} else {
|
|
1387
1397
|
if (this.writeBuffer.size > 0 || this.deleteBuffer.size > 0) {
|
|
1388
1398
|
const failure = await this._merge(this);
|
|
1389
1399
|
if (failure) {
|
|
1400
|
+
this.rollback();
|
|
1390
1401
|
return {
|
|
1391
1402
|
label,
|
|
1392
1403
|
success: false,
|
|
@@ -1397,13 +1408,7 @@ var require_cjs = __commonJS({
|
|
|
1397
1408
|
deleted: []
|
|
1398
1409
|
};
|
|
1399
1410
|
}
|
|
1400
|
-
this.
|
|
1401
|
-
this.deleteBuffer.clear();
|
|
1402
|
-
this.createdKeys.clear();
|
|
1403
|
-
this.deletedValues.clear();
|
|
1404
|
-
this.originallyExisted.clear();
|
|
1405
|
-
this.keyVersions.clear();
|
|
1406
|
-
this.bufferHistory.clear();
|
|
1411
|
+
this._cleanupAll();
|
|
1407
1412
|
this.localVersion = 0;
|
|
1408
1413
|
this.snapshotVersion = this.version;
|
|
1409
1414
|
}
|
|
@@ -1985,30 +1990,22 @@ var require_cjs = __commonJS({
|
|
|
1985
1990
|
*/
|
|
1986
1991
|
_insertValueIntoLeaf(leaf, key, value) {
|
|
1987
1992
|
if (leaf.values.length) {
|
|
1988
|
-
|
|
1989
|
-
|
|
1990
|
-
if (
|
|
1991
|
-
|
|
1992
|
-
return false;
|
|
1993
|
-
}
|
|
1994
|
-
leaf.keys[i].push(key);
|
|
1995
|
-
return true;
|
|
1996
|
-
} else if (this.comparator.isLower(value, nValue)) {
|
|
1997
|
-
leaf.values.splice(i, 0, value);
|
|
1998
|
-
leaf.keys.splice(i, 0, [key]);
|
|
1999
|
-
return true;
|
|
2000
|
-
} else if (i + 1 === leaf.values.length) {
|
|
2001
|
-
leaf.values.push(value);
|
|
2002
|
-
leaf.keys.push([key]);
|
|
2003
|
-
return true;
|
|
1993
|
+
const { index, found } = this._binarySearchValues(leaf.values, value);
|
|
1994
|
+
if (found) {
|
|
1995
|
+
if (leaf.keys[index].includes(key)) {
|
|
1996
|
+
return false;
|
|
2004
1997
|
}
|
|
1998
|
+
leaf.keys[index].push(key);
|
|
1999
|
+
return true;
|
|
2005
2000
|
}
|
|
2001
|
+
leaf.values.splice(index, 0, value);
|
|
2002
|
+
leaf.keys.splice(index, 0, [key]);
|
|
2003
|
+
return true;
|
|
2006
2004
|
} else {
|
|
2007
2005
|
leaf.values = [value];
|
|
2008
2006
|
leaf.keys = [[key]];
|
|
2009
2007
|
return true;
|
|
2010
2008
|
}
|
|
2011
|
-
return false;
|
|
2012
2009
|
}
|
|
2013
2010
|
_cloneNode(node) {
|
|
2014
2011
|
return JSON.parse(JSON.stringify(node));
|
|
@@ -2299,12 +2296,8 @@ var require_cjs = __commonJS({
|
|
|
2299
2296
|
const midValue = parentNode.values[mid];
|
|
2300
2297
|
parentNode.values = parentNode.values.slice(0, mid);
|
|
2301
2298
|
parentNode.keys = parentNode.keys.slice(0, mid + 1);
|
|
2302
|
-
for (
|
|
2303
|
-
const
|
|
2304
|
-
n.parent = parentNode.id;
|
|
2305
|
-
this._updateNode(n);
|
|
2306
|
-
}
|
|
2307
|
-
for (const k2 of newSiblingNodeRecursive.keys) {
|
|
2299
|
+
for (let i = 0, len = newSiblingNodeRecursive.keys.length; i < len; i++) {
|
|
2300
|
+
const k2 = newSiblingNodeRecursive.keys[i];
|
|
2308
2301
|
const n = this._cloneNode(this.getNode(k2));
|
|
2309
2302
|
n.parent = newSiblingNodeRecursive.id;
|
|
2310
2303
|
this._updateNode(n);
|
|
@@ -2386,7 +2379,7 @@ var require_cjs = __commonJS({
|
|
|
2386
2379
|
for (let i = 0; i < len; i++) {
|
|
2387
2380
|
const nValue = node.values[i];
|
|
2388
2381
|
const keys = node.keys[i];
|
|
2389
|
-
for (let j = 0,
|
|
2382
|
+
for (let j = 0, len2 = keys.length; j < len2; j++) {
|
|
2390
2383
|
yield [keys[j], nValue];
|
|
2391
2384
|
}
|
|
2392
2385
|
}
|
|
@@ -2465,7 +2458,7 @@ var require_cjs = __commonJS({
|
|
|
2465
2458
|
while (true) {
|
|
2466
2459
|
for (let i = 0, len = node.values.length; i < len; i++) {
|
|
2467
2460
|
const keys = node.keys[i];
|
|
2468
|
-
for (let j = 0,
|
|
2461
|
+
for (let j = 0, len2 = keys.length; j < len2; j++) {
|
|
2469
2462
|
if (keys[j] === key) {
|
|
2470
2463
|
return node.values[i];
|
|
2471
2464
|
}
|
|
@@ -2574,8 +2567,16 @@ var require_cjs = __commonJS({
|
|
|
2574
2567
|
const sorted = [...entries].sort((a, b2) => this.comparator.asc(a[1], b2[1]));
|
|
2575
2568
|
let currentLeaf = null;
|
|
2576
2569
|
let modified = false;
|
|
2577
|
-
|
|
2578
|
-
|
|
2570
|
+
let cachedLeafId = null;
|
|
2571
|
+
let cachedLeafMaxValue = null;
|
|
2572
|
+
for (let i = 0, len = sorted.length; i < len; i++) {
|
|
2573
|
+
const [key, value] = sorted[i];
|
|
2574
|
+
let targetLeaf;
|
|
2575
|
+
if (cachedLeafId !== null && cachedLeafMaxValue !== null && currentLeaf !== null && (this.comparator.isLower(value, cachedLeafMaxValue) || this.comparator.isSame(value, cachedLeafMaxValue))) {
|
|
2576
|
+
targetLeaf = currentLeaf;
|
|
2577
|
+
} else {
|
|
2578
|
+
targetLeaf = this.locateLeaf(value);
|
|
2579
|
+
}
|
|
2579
2580
|
if (currentLeaf !== null && currentLeaf.id === targetLeaf.id) {
|
|
2580
2581
|
} else {
|
|
2581
2582
|
if (currentLeaf !== null && modified) {
|
|
@@ -2584,8 +2585,10 @@ var require_cjs = __commonJS({
|
|
|
2584
2585
|
currentLeaf = this._cloneNode(targetLeaf);
|
|
2585
2586
|
modified = false;
|
|
2586
2587
|
}
|
|
2588
|
+
cachedLeafId = currentLeaf.id;
|
|
2587
2589
|
const changed = this._insertValueIntoLeaf(currentLeaf, key, value);
|
|
2588
2590
|
modified = modified || changed;
|
|
2591
|
+
cachedLeafMaxValue = currentLeaf.values[currentLeaf.values.length - 1];
|
|
2589
2592
|
if (currentLeaf.values.length === this.order) {
|
|
2590
2593
|
this._updateNode(currentLeaf);
|
|
2591
2594
|
let after = this._createNode(
|
|
@@ -2606,6 +2609,8 @@ var require_cjs = __commonJS({
|
|
|
2606
2609
|
this._updateNode(after);
|
|
2607
2610
|
this._insertInParent(currentLeaf, after.values[0], after);
|
|
2608
2611
|
currentLeaf = null;
|
|
2612
|
+
cachedLeafId = null;
|
|
2613
|
+
cachedLeafMaxValue = null;
|
|
2609
2614
|
modified = false;
|
|
2610
2615
|
}
|
|
2611
2616
|
}
|
|
@@ -2613,6 +2618,85 @@ var require_cjs = __commonJS({
|
|
|
2613
2618
|
this._updateNode(currentLeaf);
|
|
2614
2619
|
}
|
|
2615
2620
|
}
|
|
2621
|
+
bulkLoad(entries) {
|
|
2622
|
+
if (entries.length === 0) return;
|
|
2623
|
+
const root = this.getNode(this.rootId);
|
|
2624
|
+
if (!root.leaf || root.values.length > 0) {
|
|
2625
|
+
throw new Error("bulkLoad can only be called on an empty tree. Use batchInsert for non-empty trees.");
|
|
2626
|
+
}
|
|
2627
|
+
const sorted = [...entries].sort((a, b2) => this.comparator.asc(a[1], b2[1]));
|
|
2628
|
+
const grouped = [];
|
|
2629
|
+
for (let i = 0, len = sorted.length; i < len; i++) {
|
|
2630
|
+
const [key, value] = sorted[i];
|
|
2631
|
+
const last = grouped[grouped.length - 1];
|
|
2632
|
+
if (last && this.comparator.isSame(last.value, value)) {
|
|
2633
|
+
if (!last.keys.includes(key)) {
|
|
2634
|
+
last.keys.push(key);
|
|
2635
|
+
}
|
|
2636
|
+
} else {
|
|
2637
|
+
grouped.push({ keys: [key], value });
|
|
2638
|
+
}
|
|
2639
|
+
}
|
|
2640
|
+
this._deleteNode(root);
|
|
2641
|
+
const maxLeafSize = this.order - 1;
|
|
2642
|
+
const leaves = [];
|
|
2643
|
+
for (let i = 0, len = grouped.length; i < len; i += maxLeafSize) {
|
|
2644
|
+
const chunk = grouped.slice(i, i + maxLeafSize);
|
|
2645
|
+
const leafKeys = chunk.map((g2) => g2.keys);
|
|
2646
|
+
const leafValues = chunk.map((g2) => g2.value);
|
|
2647
|
+
const leaf = this._createNode(
|
|
2648
|
+
true,
|
|
2649
|
+
leafKeys,
|
|
2650
|
+
leafValues,
|
|
2651
|
+
null,
|
|
2652
|
+
null,
|
|
2653
|
+
null
|
|
2654
|
+
);
|
|
2655
|
+
leaves.push(leaf);
|
|
2656
|
+
}
|
|
2657
|
+
for (let i = 0, len = leaves.length; i < len; i++) {
|
|
2658
|
+
if (i > 0) {
|
|
2659
|
+
leaves[i].prev = leaves[i - 1].id;
|
|
2660
|
+
}
|
|
2661
|
+
if (i < len - 1) {
|
|
2662
|
+
leaves[i].next = leaves[i + 1].id;
|
|
2663
|
+
}
|
|
2664
|
+
this._updateNode(leaves[i]);
|
|
2665
|
+
}
|
|
2666
|
+
let currentLevel = leaves;
|
|
2667
|
+
while (currentLevel.length > 1) {
|
|
2668
|
+
const nextLevel = [];
|
|
2669
|
+
for (let i = 0, len = currentLevel.length; i < len; i += this.order) {
|
|
2670
|
+
const children = currentLevel.slice(i, i + this.order);
|
|
2671
|
+
const childIds = children.map((c) => c.id);
|
|
2672
|
+
const separators = [];
|
|
2673
|
+
for (let j = 1, len2 = children.length; j < len2; j++) {
|
|
2674
|
+
separators.push(children[j].values[0]);
|
|
2675
|
+
}
|
|
2676
|
+
const internalNode = this._createNode(
|
|
2677
|
+
false,
|
|
2678
|
+
childIds,
|
|
2679
|
+
separators,
|
|
2680
|
+
null,
|
|
2681
|
+
null,
|
|
2682
|
+
null
|
|
2683
|
+
);
|
|
2684
|
+
for (let j = 0, len2 = children.length; j < len2; j++) {
|
|
2685
|
+
const child = children[j];
|
|
2686
|
+
child.parent = internalNode.id;
|
|
2687
|
+
this._updateNode(child);
|
|
2688
|
+
}
|
|
2689
|
+
nextLevel.push(internalNode);
|
|
2690
|
+
}
|
|
2691
|
+
currentLevel = nextLevel;
|
|
2692
|
+
}
|
|
2693
|
+
const newRoot = currentLevel[0];
|
|
2694
|
+
this._writeHead({
|
|
2695
|
+
root: newRoot.id,
|
|
2696
|
+
order: this.order,
|
|
2697
|
+
data: this.strategy.head.data
|
|
2698
|
+
});
|
|
2699
|
+
}
|
|
2616
2700
|
_deleteEntry(node, key) {
|
|
2617
2701
|
if (!node.leaf) {
|
|
2618
2702
|
let keyIndex = -1;
|
|
@@ -2716,7 +2800,8 @@ var require_cjs = __commonJS({
|
|
|
2716
2800
|
siblingNode.values.push(...node.values);
|
|
2717
2801
|
if (!siblingNode.leaf) {
|
|
2718
2802
|
const keys = siblingNode.keys;
|
|
2719
|
-
for (
|
|
2803
|
+
for (let i = 0, len = keys.length; i < len; i++) {
|
|
2804
|
+
const key2 = keys[i];
|
|
2720
2805
|
const node2 = this._cloneNode(this.getNode(key2));
|
|
2721
2806
|
node2.parent = siblingNode.id;
|
|
2722
2807
|
this._updateNode(node2);
|
|
@@ -2784,21 +2869,24 @@ var require_cjs = __commonJS({
|
|
|
2784
2869
|
this._updateNode(siblingNode);
|
|
2785
2870
|
}
|
|
2786
2871
|
if (!siblingNode.leaf) {
|
|
2787
|
-
for (
|
|
2872
|
+
for (let i = 0, len = siblingNode.keys.length; i < len; i++) {
|
|
2873
|
+
const key2 = siblingNode.keys[i];
|
|
2788
2874
|
const n = this._cloneNode(this.getNode(key2));
|
|
2789
2875
|
n.parent = siblingNode.id;
|
|
2790
2876
|
this._updateNode(n);
|
|
2791
2877
|
}
|
|
2792
2878
|
}
|
|
2793
2879
|
if (!node.leaf) {
|
|
2794
|
-
for (
|
|
2880
|
+
for (let i = 0, len = node.keys.length; i < len; i++) {
|
|
2881
|
+
const key2 = node.keys[i];
|
|
2795
2882
|
const n = this._cloneNode(this.getNode(key2));
|
|
2796
2883
|
n.parent = node.id;
|
|
2797
2884
|
this._updateNode(n);
|
|
2798
2885
|
}
|
|
2799
2886
|
}
|
|
2800
2887
|
if (!parentNode.leaf) {
|
|
2801
|
-
for (
|
|
2888
|
+
for (let i = 0, len = parentNode.keys.length; i < len; i++) {
|
|
2889
|
+
const key2 = parentNode.keys[i];
|
|
2802
2890
|
const n = this._cloneNode(this.getNode(key2));
|
|
2803
2891
|
n.parent = parentNode.id;
|
|
2804
2892
|
this._updateNode(n);
|
|
@@ -2875,8 +2963,12 @@ var require_cjs = __commonJS({
|
|
|
2875
2963
|
result = this.rootTx.commit(label);
|
|
2876
2964
|
if (result.success) {
|
|
2877
2965
|
this.rootTx.rootId = this.rootId;
|
|
2966
|
+
} else {
|
|
2967
|
+
this.mvcc.rollback();
|
|
2878
2968
|
}
|
|
2879
2969
|
}
|
|
2970
|
+
} else {
|
|
2971
|
+
this.mvcc.rollback();
|
|
2880
2972
|
}
|
|
2881
2973
|
return result;
|
|
2882
2974
|
}
|
|
@@ -2972,6 +3064,14 @@ var require_cjs = __commonJS({
|
|
|
2972
3064
|
throw new Error(`Transaction failed: ${result.error || "Commit failed due to conflict"}`);
|
|
2973
3065
|
}
|
|
2974
3066
|
}
|
|
3067
|
+
bulkLoad(entries) {
|
|
3068
|
+
const tx = this.createTransaction();
|
|
3069
|
+
tx.bulkLoad(entries);
|
|
3070
|
+
const result = tx.commit();
|
|
3071
|
+
if (!result.success) {
|
|
3072
|
+
throw new Error(`Transaction failed: ${result.error || "Commit failed due to conflict"}`);
|
|
3073
|
+
}
|
|
3074
|
+
}
|
|
2975
3075
|
};
|
|
2976
3076
|
var Ryoiki2 = class _Ryoiki2 {
|
|
2977
3077
|
readings;
|
|
@@ -3229,7 +3329,7 @@ var require_cjs = __commonJS({
|
|
|
3229
3329
|
this._free(this.writings, lockId);
|
|
3230
3330
|
}
|
|
3231
3331
|
};
|
|
3232
|
-
var
|
|
3332
|
+
var BPTreeAsyncTransaction = class extends BPTreeTransaction {
|
|
3233
3333
|
lock;
|
|
3234
3334
|
constructor(rootTx, mvccRoot, mvcc, strategy, comparator, option) {
|
|
3235
3335
|
super(
|
|
@@ -3385,12 +3485,8 @@ var require_cjs = __commonJS({
|
|
|
3385
3485
|
const midValue = parentNode.values[mid];
|
|
3386
3486
|
parentNode.values = parentNode.values.slice(0, mid);
|
|
3387
3487
|
parentNode.keys = parentNode.keys.slice(0, mid + 1);
|
|
3388
|
-
for (
|
|
3389
|
-
const
|
|
3390
|
-
n.parent = parentNode.id;
|
|
3391
|
-
await this._updateNode(n);
|
|
3392
|
-
}
|
|
3393
|
-
for (const k2 of newSiblingNodeRecursive.keys) {
|
|
3488
|
+
for (let i = 0, len = newSiblingNodeRecursive.keys.length; i < len; i++) {
|
|
3489
|
+
const k2 = newSiblingNodeRecursive.keys[i];
|
|
3394
3490
|
const n = this._cloneNode(await this.getNode(k2));
|
|
3395
3491
|
n.parent = newSiblingNodeRecursive.id;
|
|
3396
3492
|
await this._updateNode(n);
|
|
@@ -3482,7 +3578,7 @@ var require_cjs = __commonJS({
|
|
|
3482
3578
|
for (let i = 0; i < len; i++) {
|
|
3483
3579
|
const nValue = node.values[i];
|
|
3484
3580
|
const keys = node.keys[i];
|
|
3485
|
-
for (let j = 0,
|
|
3581
|
+
for (let j = 0, len2 = keys.length; j < len2; j++) {
|
|
3486
3582
|
yield [keys[j], nValue];
|
|
3487
3583
|
}
|
|
3488
3584
|
}
|
|
@@ -3560,7 +3656,7 @@ var require_cjs = __commonJS({
|
|
|
3560
3656
|
while (true) {
|
|
3561
3657
|
for (let i = 0, len = node.values.length; i < len; i++) {
|
|
3562
3658
|
const keys = node.keys[i];
|
|
3563
|
-
for (let j = 0,
|
|
3659
|
+
for (let j = 0, len2 = keys.length; j < len2; j++) {
|
|
3564
3660
|
if (keys[j] === key) {
|
|
3565
3661
|
return node.values[i];
|
|
3566
3662
|
}
|
|
@@ -3672,8 +3768,16 @@ var require_cjs = __commonJS({
|
|
|
3672
3768
|
const sorted = [...entries].sort((a, b2) => this.comparator.asc(a[1], b2[1]));
|
|
3673
3769
|
let currentLeaf = null;
|
|
3674
3770
|
let modified = false;
|
|
3675
|
-
|
|
3676
|
-
|
|
3771
|
+
let cachedLeafId = null;
|
|
3772
|
+
let cachedLeafMaxValue = null;
|
|
3773
|
+
for (let i = 0, len = sorted.length; i < len; i++) {
|
|
3774
|
+
const [key, value] = sorted[i];
|
|
3775
|
+
let targetLeaf;
|
|
3776
|
+
if (cachedLeafId !== null && cachedLeafMaxValue !== null && currentLeaf !== null && (this.comparator.isLower(value, cachedLeafMaxValue) || this.comparator.isSame(value, cachedLeafMaxValue))) {
|
|
3777
|
+
targetLeaf = currentLeaf;
|
|
3778
|
+
} else {
|
|
3779
|
+
targetLeaf = await this.locateLeaf(value);
|
|
3780
|
+
}
|
|
3677
3781
|
if (currentLeaf !== null && currentLeaf.id === targetLeaf.id) {
|
|
3678
3782
|
} else {
|
|
3679
3783
|
if (currentLeaf !== null && modified) {
|
|
@@ -3682,8 +3786,10 @@ var require_cjs = __commonJS({
|
|
|
3682
3786
|
currentLeaf = this._cloneNode(targetLeaf);
|
|
3683
3787
|
modified = false;
|
|
3684
3788
|
}
|
|
3789
|
+
cachedLeafId = currentLeaf.id;
|
|
3685
3790
|
const changed = this._insertValueIntoLeaf(currentLeaf, key, value);
|
|
3686
3791
|
modified = modified || changed;
|
|
3792
|
+
cachedLeafMaxValue = currentLeaf.values[currentLeaf.values.length - 1];
|
|
3687
3793
|
if (currentLeaf.values.length === this.order) {
|
|
3688
3794
|
await this._updateNode(currentLeaf);
|
|
3689
3795
|
let after = await this._createNode(
|
|
@@ -3704,6 +3810,8 @@ var require_cjs = __commonJS({
|
|
|
3704
3810
|
await this._updateNode(after);
|
|
3705
3811
|
await this._insertInParent(currentLeaf, after.values[0], after);
|
|
3706
3812
|
currentLeaf = null;
|
|
3813
|
+
cachedLeafId = null;
|
|
3814
|
+
cachedLeafMaxValue = null;
|
|
3707
3815
|
modified = false;
|
|
3708
3816
|
}
|
|
3709
3817
|
}
|
|
@@ -3712,6 +3820,87 @@ var require_cjs = __commonJS({
|
|
|
3712
3820
|
}
|
|
3713
3821
|
});
|
|
3714
3822
|
}
|
|
3823
|
+
async bulkLoad(entries) {
|
|
3824
|
+
if (entries.length === 0) return;
|
|
3825
|
+
return this.writeLock(0, async () => {
|
|
3826
|
+
const root = await this.getNode(this.rootId);
|
|
3827
|
+
if (!root.leaf || root.values.length > 0) {
|
|
3828
|
+
throw new Error("bulkLoad can only be called on an empty tree. Use batchInsert for non-empty trees.");
|
|
3829
|
+
}
|
|
3830
|
+
const sorted = [...entries].sort((a, b2) => this.comparator.asc(a[1], b2[1]));
|
|
3831
|
+
const grouped = [];
|
|
3832
|
+
for (let i = 0, len = sorted.length; i < len; i++) {
|
|
3833
|
+
const [key, value] = sorted[i];
|
|
3834
|
+
const last = grouped[grouped.length - 1];
|
|
3835
|
+
if (last && this.comparator.isSame(last.value, value)) {
|
|
3836
|
+
if (!last.keys.includes(key)) {
|
|
3837
|
+
last.keys.push(key);
|
|
3838
|
+
}
|
|
3839
|
+
} else {
|
|
3840
|
+
grouped.push({ keys: [key], value });
|
|
3841
|
+
}
|
|
3842
|
+
}
|
|
3843
|
+
await this._deleteNode(root);
|
|
3844
|
+
const maxLeafSize = this.order - 1;
|
|
3845
|
+
const leaves = [];
|
|
3846
|
+
for (let i = 0, len = grouped.length; i < len; i += maxLeafSize) {
|
|
3847
|
+
const chunk = grouped.slice(i, i + maxLeafSize);
|
|
3848
|
+
const leafKeys = chunk.map((g2) => g2.keys);
|
|
3849
|
+
const leafValues = chunk.map((g2) => g2.value);
|
|
3850
|
+
const leaf = await this._createNode(
|
|
3851
|
+
true,
|
|
3852
|
+
leafKeys,
|
|
3853
|
+
leafValues,
|
|
3854
|
+
null,
|
|
3855
|
+
null,
|
|
3856
|
+
null
|
|
3857
|
+
);
|
|
3858
|
+
leaves.push(leaf);
|
|
3859
|
+
}
|
|
3860
|
+
for (let i = 0, len = leaves.length; i < len; i++) {
|
|
3861
|
+
if (i > 0) {
|
|
3862
|
+
leaves[i].prev = leaves[i - 1].id;
|
|
3863
|
+
}
|
|
3864
|
+
if (i < len - 1) {
|
|
3865
|
+
leaves[i].next = leaves[i + 1].id;
|
|
3866
|
+
}
|
|
3867
|
+
await this._updateNode(leaves[i]);
|
|
3868
|
+
}
|
|
3869
|
+
let currentLevel = leaves;
|
|
3870
|
+
while (currentLevel.length > 1) {
|
|
3871
|
+
const nextLevel = [];
|
|
3872
|
+
for (let i = 0, len = currentLevel.length; i < len; i += this.order) {
|
|
3873
|
+
const children = currentLevel.slice(i, i + this.order);
|
|
3874
|
+
const childIds = children.map((c) => c.id);
|
|
3875
|
+
const separators = [];
|
|
3876
|
+
for (let j = 1, len2 = children.length; j < len2; j++) {
|
|
3877
|
+
separators.push(children[j].values[0]);
|
|
3878
|
+
}
|
|
3879
|
+
const internalNode = await this._createNode(
|
|
3880
|
+
false,
|
|
3881
|
+
childIds,
|
|
3882
|
+
separators,
|
|
3883
|
+
null,
|
|
3884
|
+
null,
|
|
3885
|
+
null
|
|
3886
|
+
);
|
|
3887
|
+
for (let j = 0, len2 = children.length; j < len2; j++) {
|
|
3888
|
+
const child = children[j];
|
|
3889
|
+
child.parent = internalNode.id;
|
|
3890
|
+
await this._updateNode(child);
|
|
3891
|
+
}
|
|
3892
|
+
nextLevel.push(internalNode);
|
|
3893
|
+
}
|
|
3894
|
+
currentLevel = nextLevel;
|
|
3895
|
+
}
|
|
3896
|
+
const newRoot = currentLevel[0];
|
|
3897
|
+
await this._writeHead({
|
|
3898
|
+
root: newRoot.id,
|
|
3899
|
+
order: this.order,
|
|
3900
|
+
data: this.strategy.head.data
|
|
3901
|
+
});
|
|
3902
|
+
});
|
|
3903
|
+
}
|
|
3715
3904
|
async _deleteEntry(node, key) {
|
|
3716
3905
|
if (!node.leaf) {
|
|
3717
3906
|
let keyIndex = -1;
|
|
@@ -3815,7 +4004,8 @@ var require_cjs = __commonJS({
|
|
|
3815
4004
|
siblingNode.values.push(...node.values);
|
|
3816
4005
|
if (!siblingNode.leaf) {
|
|
3817
4006
|
const keys = siblingNode.keys;
|
|
3818
|
-
for (
|
|
4007
|
+
for (let i = 0, len = keys.length; i < len; i++) {
|
|
4008
|
+
const key2 = keys[i];
|
|
3819
4009
|
const node2 = this._cloneNode(await this.getNode(key2));
|
|
3820
4010
|
node2.parent = siblingNode.id;
|
|
3821
4011
|
await this._updateNode(node2);
|
|
@@ -3883,21 +4073,24 @@ var require_cjs = __commonJS({
|
|
|
3883
4073
|
await this._updateNode(siblingNode);
|
|
3884
4074
|
}
|
|
3885
4075
|
if (!siblingNode.leaf) {
|
|
3886
|
-
for (
|
|
4076
|
+
for (let i = 0, len = siblingNode.keys.length; i < len; i++) {
|
|
4077
|
+
const key2 = siblingNode.keys[i];
|
|
3887
4078
|
const n = this._cloneNode(await this.getNode(key2));
|
|
3888
4079
|
n.parent = siblingNode.id;
|
|
3889
4080
|
await this._updateNode(n);
|
|
3890
4081
|
}
|
|
3891
4082
|
}
|
|
3892
4083
|
if (!node.leaf) {
|
|
3893
|
-
for (
|
|
4084
|
+
for (let i = 0, len = node.keys.length; i < len; i++) {
|
|
4085
|
+
const key2 = node.keys[i];
|
|
3894
4086
|
const n = this._cloneNode(await this.getNode(key2));
|
|
3895
4087
|
n.parent = node.id;
|
|
3896
4088
|
await this._updateNode(n);
|
|
3897
4089
|
}
|
|
3898
4090
|
}
|
|
3899
4091
|
if (!parentNode.leaf) {
|
|
3900
|
-
for (
|
|
4092
|
+
for (let i = 0, len = parentNode.keys.length; i < len; i++) {
|
|
4093
|
+
const key2 = parentNode.keys[i];
|
|
3901
4094
|
const n = this._cloneNode(await this.getNode(key2));
|
|
3902
4095
|
n.parent = parentNode.id;
|
|
3903
4096
|
await this._updateNode(n);
|
|
@@ -3976,8 +4169,12 @@ var require_cjs = __commonJS({
|
|
|
3976
4169
|
result = await this.rootTx.commit(label);
|
|
3977
4170
|
if (result.success) {
|
|
3978
4171
|
this.rootTx.rootId = this.rootId;
|
|
4172
|
+
} else {
|
|
4173
|
+
this.mvcc.rollback();
|
|
3979
4174
|
}
|
|
3980
4175
|
}
|
|
4176
|
+
} else {
|
|
4177
|
+
this.mvcc.rollback();
|
|
3981
4178
|
}
|
|
3982
4179
|
return result;
|
|
3983
4180
|
}
|
|
@@ -4018,7 +4215,7 @@ var require_cjs = __commonJS({
|
|
|
4018
4215
|
}
|
|
4019
4216
|
}
|
|
4020
4217
|
};
|
|
4021
|
-
var BPTreeAsync2 = class extends
|
|
4218
|
+
var BPTreeAsync2 = class extends BPTreeAsyncTransaction {
|
|
4022
4219
|
constructor(strategy, comparator, option) {
|
|
4023
4220
|
const mvccRoot = new AsyncMVCCTransaction(new BPTreeMVCCStrategyAsync(strategy), {
|
|
4024
4221
|
cacheCapacity: option?.capacity ?? void 0
|
|
@@ -4038,7 +4235,7 @@ var require_cjs = __commonJS({
|
|
|
4038
4235
|
*/
|
|
4039
4236
|
async createTransaction() {
|
|
4040
4237
|
const nestedTx = this.mvcc.createNested();
|
|
4041
|
-
const tx = new
|
|
4238
|
+
const tx = new BPTreeAsyncTransaction(
|
|
4042
4239
|
this,
|
|
4043
4240
|
this.mvcc,
|
|
4044
4241
|
nestedTx,
|
|
@@ -4079,6 +4276,16 @@ var require_cjs = __commonJS({
|
|
|
4079
4276
|
}
|
|
4080
4277
|
});
|
|
4081
4278
|
}
|
|
4279
|
+
async bulkLoad(entries) {
|
|
4280
|
+
return this.writeLock(1, async () => {
|
|
4281
|
+
const tx = await this.createTransaction();
|
|
4282
|
+
await tx.bulkLoad(entries);
|
|
4283
|
+
const result = await tx.commit();
|
|
4284
|
+
if (!result.success) {
|
|
4285
|
+
throw new Error(`Transaction failed: ${result.error || "Commit failed due to conflict"}`);
|
|
4286
|
+
}
|
|
4287
|
+
});
|
|
4288
|
+
}
|
|
4082
4289
|
};
|
|
4083
4290
|
var SerializeStrategy = class {
|
|
4084
4291
|
order;
|
|
@@ -5232,11 +5439,7 @@ var require_cjs = __commonJS({
|
|
|
5232
5439
|
*/
|
|
5233
5440
|
rollback() {
|
|
5234
5441
|
const { created, updated, deleted } = this.getResultEntries();
|
|
5235
|
-
this.
|
|
5236
|
-
this.deleteBuffer.clear();
|
|
5237
|
-
this.createdKeys.clear();
|
|
5238
|
-
this.deletedValues.clear();
|
|
5239
|
-
this.originallyExisted.clear();
|
|
5442
|
+
this._cleanupAll();
|
|
5240
5443
|
this.committed = true;
|
|
5241
5444
|
if (this.root !== this) {
|
|
5242
5445
|
this.root.activeTransactions.delete(this);
|
|
@@ -5272,6 +5475,19 @@ var require_cjs = __commonJS({
|
|
|
5272
5475
|
}
|
|
5273
5476
|
return Array.from(conflicts);
|
|
5274
5477
|
}
|
|
5478
|
+
/**
|
|
5479
|
+
* Cleans up all buffers and history.
|
|
5480
|
+
* This method is called by the commit method.
|
|
5481
|
+
*/
|
|
5482
|
+
_cleanupAll() {
|
|
5483
|
+
this.writeBuffer.clear();
|
|
5484
|
+
this.deleteBuffer.clear();
|
|
5485
|
+
this.createdKeys.clear();
|
|
5486
|
+
this.deletedValues.clear();
|
|
5487
|
+
this.originallyExisted.clear();
|
|
5488
|
+
this.keyVersions.clear();
|
|
5489
|
+
this.bufferHistory.clear();
|
|
5490
|
+
}
|
|
5275
5491
|
/**
|
|
5276
5492
|
* Cleans up both deletedCache and versionIndex based on minActiveVersion.
|
|
5277
5493
|
* Root transactions call this after commit to reclaim memory.
|
|
@@ -5463,6 +5679,7 @@ var require_cjs = __commonJS({
|
|
|
5463
5679
|
if (this.parent) {
|
|
5464
5680
|
const failure = this.parent._merge(this);
|
|
5465
5681
|
if (failure) {
|
|
5682
|
+
this.rollback();
|
|
5466
5683
|
return {
|
|
5467
5684
|
label,
|
|
5468
5685
|
success: false,
|
|
@@ -5473,11 +5690,13 @@ var require_cjs = __commonJS({
|
|
|
5473
5690
|
deleted
|
|
5474
5691
|
};
|
|
5475
5692
|
}
|
|
5693
|
+
this._cleanupAll();
|
|
5476
5694
|
this.committed = true;
|
|
5477
5695
|
} else {
|
|
5478
5696
|
if (this.writeBuffer.size > 0 || this.deleteBuffer.size > 0) {
|
|
5479
5697
|
const failure = this._merge(this);
|
|
5480
5698
|
if (failure) {
|
|
5699
|
+
this.rollback();
|
|
5481
5700
|
return {
|
|
5482
5701
|
label,
|
|
5483
5702
|
success: false,
|
|
@@ -5488,13 +5707,7 @@ var require_cjs = __commonJS({
|
|
|
5488
5707
|
deleted: []
|
|
5489
5708
|
};
|
|
5490
5709
|
}
|
|
5491
|
-
this.
|
|
5492
|
-
this.deleteBuffer.clear();
|
|
5493
|
-
this.createdKeys.clear();
|
|
5494
|
-
this.deletedValues.clear();
|
|
5495
|
-
this.originallyExisted.clear();
|
|
5496
|
-
this.keyVersions.clear();
|
|
5497
|
-
this.bufferHistory.clear();
|
|
5710
|
+
this._cleanupAll();
|
|
5498
5711
|
this.localVersion = 0;
|
|
5499
5712
|
this.snapshotVersion = this.version;
|
|
5500
5713
|
}
|
|
@@ -6128,6 +6341,7 @@ var require_cjs = __commonJS({
|
|
|
6128
6341
|
if (this.parent) {
|
|
6129
6342
|
const failure = await this.parent._merge(this);
|
|
6130
6343
|
if (failure) {
|
|
6344
|
+
this.rollback();
|
|
6131
6345
|
return {
|
|
6132
6346
|
label,
|
|
6133
6347
|
success: false,
|
|
@@ -6138,11 +6352,13 @@ var require_cjs = __commonJS({
|
|
|
6138
6352
|
deleted
|
|
6139
6353
|
};
|
|
6140
6354
|
}
|
|
6355
|
+
this._cleanupAll();
|
|
6141
6356
|
this.committed = true;
|
|
6142
6357
|
} else {
|
|
6143
6358
|
if (this.writeBuffer.size > 0 || this.deleteBuffer.size > 0) {
|
|
6144
6359
|
const failure = await this._merge(this);
|
|
6145
6360
|
if (failure) {
|
|
6361
|
+
this.rollback();
|
|
6146
6362
|
return {
|
|
6147
6363
|
label,
|
|
6148
6364
|
success: false,
|
|
@@ -6153,13 +6369,7 @@ var require_cjs = __commonJS({
|
|
|
6153
6369
|
deleted: []
|
|
6154
6370
|
};
|
|
6155
6371
|
}
|
|
6156
|
-
this.
|
|
6157
|
-
this.deleteBuffer.clear();
|
|
6158
|
-
this.createdKeys.clear();
|
|
6159
|
-
this.deletedValues.clear();
|
|
6160
|
-
this.originallyExisted.clear();
|
|
6161
|
-
this.keyVersions.clear();
|
|
6162
|
-
this.bufferHistory.clear();
|
|
6372
|
+
this._cleanupAll();
|
|
6163
6373
|
this.localVersion = 0;
|
|
6164
6374
|
this.snapshotVersion = this.version;
|
|
6165
6375
|
}
|
|
@@ -8065,12 +8275,14 @@ var require_cjs = __commonJS({
|
|
|
8065
8275
|
buffer;
|
|
8066
8276
|
view;
|
|
8067
8277
|
totalWrittenPages = 0;
|
|
8278
|
+
logger;
|
|
8068
8279
|
/**
|
|
8069
8280
|
* Constructor
|
|
8070
8281
|
* @param walFilePath WAL file path
|
|
8071
8282
|
* @param pageSize Page size
|
|
8283
|
+
* @param logger Logger instance
|
|
8072
8284
|
*/
|
|
8073
|
-
constructor(walFilePath, pageSize) {
|
|
8285
|
+
constructor(walFilePath, pageSize, logger) {
|
|
8074
8286
|
if ((pageSize & pageSize - 1) !== 0) {
|
|
8075
8287
|
throw new Error("Page size must be a power of 2");
|
|
8076
8288
|
}
|
|
@@ -8079,6 +8291,7 @@ var require_cjs = __commonJS({
|
|
|
8079
8291
|
this.entrySize = 4 + pageSize;
|
|
8080
8292
|
this.buffer = new Uint8Array(this.entrySize);
|
|
8081
8293
|
this.view = new DataView(this.buffer.buffer);
|
|
8294
|
+
this.logger = logger;
|
|
8082
8295
|
}
|
|
8083
8296
|
/**
|
|
8084
8297
|
* Opens the log file.
|
|
@@ -8106,11 +8319,11 @@ var require_cjs = __commonJS({
|
|
|
8106
8319
|
try {
|
|
8107
8320
|
const manager = new PageManagerFactory().getManager(data);
|
|
8108
8321
|
if (!manager.verifyChecksum(data)) {
|
|
8109
|
-
|
|
8322
|
+
this.logger.warn(`Checksum verification failed for PageID ${pageId} during recovery. Ignoring changes.`);
|
|
8110
8323
|
continue;
|
|
8111
8324
|
}
|
|
8112
8325
|
} catch (e) {
|
|
8113
|
-
|
|
8326
|
+
this.logger.warn(`Failed to verify checksum for PageID ${pageId} during recovery: ${e}. Ignoring changes.`);
|
|
8114
8327
|
continue;
|
|
8115
8328
|
}
|
|
8116
8329
|
promises.push(writePage(pageId, data));
|
|
@@ -8423,20 +8636,22 @@ var require_cjs = __commonJS({
|
|
|
8423
8636
|
* @param pageCacheCapacity 페이지 캐시 크기
|
|
8424
8637
|
* @param options 데이터플라이 옵션
|
|
8425
8638
|
*/
|
|
8426
|
-
constructor(fileHandle, pageSize, pageCacheCapacity, options) {
|
|
8639
|
+
constructor(fileHandle, pageSize, pageCacheCapacity, options, logger, walLogger) {
|
|
8427
8640
|
this.fileHandle = fileHandle;
|
|
8428
8641
|
this.pageSize = pageSize;
|
|
8429
8642
|
this.pageCacheCapacity = pageCacheCapacity;
|
|
8430
8643
|
this.options = options;
|
|
8431
8644
|
const walPath = options.wal;
|
|
8432
|
-
this.walManager = walPath ? new WALManager(walPath, pageSize) : null;
|
|
8645
|
+
this.walManager = walPath && walLogger ? new WALManager(walPath, pageSize, walLogger) : null;
|
|
8433
8646
|
this.pageManagerFactory = new PageManagerFactory();
|
|
8434
8647
|
this.pageStrategy = new PageMVCCStrategy(fileHandle, pageSize, pageCacheCapacity);
|
|
8648
|
+
this.logger = logger;
|
|
8435
8649
|
}
|
|
8436
8650
|
pageFactory = new PageManagerFactory();
|
|
8437
8651
|
walManager;
|
|
8438
8652
|
pageManagerFactory;
|
|
8439
8653
|
pageStrategy;
|
|
8654
|
+
logger;
|
|
8440
8655
|
/** 글로벌 동기화(체크포인트/커밋)를 위한 Mutex */
|
|
8441
8656
|
lockPromise = Promise.resolve();
|
|
8442
8657
|
/**
|
|
@@ -8461,7 +8676,9 @@ var require_cjs = __commonJS({
|
|
|
8461
8676
|
* Performs WAL recovery if necessary.
|
|
8462
8677
|
*/
|
|
8463
8678
|
async init() {
|
|
8679
|
+
this.logger.info("Initializing");
|
|
8464
8680
|
if (this.walManager) {
|
|
8681
|
+
this.logger.debug("WALManager found, starting recovery");
|
|
8465
8682
|
await this.walManager.recover(async (pageId, data) => {
|
|
8466
8683
|
await this.pageStrategy.write(pageId, data);
|
|
8467
8684
|
});
|
|
@@ -8633,6 +8850,7 @@ var require_cjs = __commonJS({
|
|
|
8633
8850
|
* @returns Created or reused page ID
|
|
8634
8851
|
*/
|
|
8635
8852
|
async appendNewPage(pageType = PageManager.CONSTANT.PAGE_TYPE_EMPTY, tx) {
|
|
8853
|
+
this.logger.debug(`Appending new page of type ${pageType}`);
|
|
8636
8854
|
await tx.__acquireWriteLock(0);
|
|
8637
8855
|
const metadata = await this.getMetadata(tx);
|
|
8638
8856
|
const metadataManager = this.pageFactory.getManager(metadata);
|
|
@@ -8736,6 +8954,7 @@ var require_cjs = __commonJS({
|
|
|
8736
8954
|
* @param tx Transaction
|
|
8737
8955
|
*/
|
|
8738
8956
|
async freeChain(startPageId, tx) {
|
|
8957
|
+
this.logger.debug(`Freeing chain starting at page ${startPageId}`);
|
|
8739
8958
|
let currentPageId = startPageId;
|
|
8740
8959
|
const visited = /* @__PURE__ */ new Set();
|
|
8741
8960
|
while (currentPageId !== -1 && currentPageId !== 0) {
|
|
@@ -8775,6 +8994,7 @@ var require_cjs = __commonJS({
|
|
|
8775
8994
|
*/
|
|
8776
8995
|
async commitToWAL(dirtyPages) {
|
|
8777
8996
|
if (this.walManager) {
|
|
8997
|
+
this.logger.debug(`Committing ${dirtyPages.size} pages to WAL`);
|
|
8778
8998
|
await this.walManager.prepareCommit(dirtyPages);
|
|
8779
8999
|
await this.walManager.finalizeCommit(true);
|
|
8780
9000
|
}
|
|
@@ -8786,6 +9006,7 @@ var require_cjs = __commonJS({
|
|
|
8786
9006
|
* 3. WAL 로그 파일 비우기 (Clear/Truncate)
|
|
8787
9007
|
*/
|
|
8788
9008
|
async checkpoint() {
|
|
9009
|
+
this.logger.info("Starting checkpoint");
|
|
8789
9010
|
await this.runGlobalLock(async () => {
|
|
8790
9011
|
await this.pageStrategy.flush();
|
|
8791
9012
|
await this.pageStrategy.sync();
|
|
@@ -8798,6 +9019,7 @@ var require_cjs = __commonJS({
|
|
|
8798
9019
|
* Closes the page file system.
|
|
8799
9020
|
*/
|
|
8800
9021
|
async close() {
|
|
9022
|
+
this.logger.info("Closing");
|
|
8801
9023
|
await this.checkpoint();
|
|
8802
9024
|
if (this.walManager) {
|
|
8803
9025
|
this.walManager.close();
|
|
@@ -9029,7 +9251,7 @@ var require_cjs = __commonJS({
|
|
|
9029
9251
|
}
|
|
9030
9252
|
};
|
|
9031
9253
|
var RowTableEngine = class {
|
|
9032
|
-
constructor(pfs, txContext, options) {
|
|
9254
|
+
constructor(pfs, txContext, options, logger) {
|
|
9033
9255
|
this.pfs = pfs;
|
|
9034
9256
|
this.txContext = txContext;
|
|
9035
9257
|
this.options = options;
|
|
@@ -9039,6 +9261,7 @@ var require_cjs = __commonJS({
|
|
|
9039
9261
|
this.overflowPageManager = this.factory.getManagerFromType(OverflowPageManager.CONSTANT.PAGE_TYPE_OVERFLOW);
|
|
9040
9262
|
this.rowManager = new Row();
|
|
9041
9263
|
this.keyManager = new KeyManager();
|
|
9264
|
+
this.logger = logger;
|
|
9042
9265
|
this.ridBuffer = new Uint8Array(Row.CONSTANT.SIZE_RID);
|
|
9043
9266
|
this.pageIdBuffer = new Uint8Array(DataPageManager.CONSTANT.SIZE_PAGE_ID);
|
|
9044
9267
|
this.maxBodySize = this.pfs.pageSize - DataPageManager.CONSTANT.SIZE_PAGE_HEADER;
|
|
@@ -9067,6 +9290,7 @@ var require_cjs = __commonJS({
|
|
|
9067
9290
|
maxBodySize;
|
|
9068
9291
|
ridBuffer;
|
|
9069
9292
|
pageIdBuffer;
|
|
9293
|
+
logger;
|
|
9070
9294
|
initialized = false;
|
|
9071
9295
|
/**
|
|
9072
9296
|
* Retrieves the BPTree transaction associated with the given transaction.
|
|
@@ -9101,6 +9325,7 @@ var require_cjs = __commonJS({
|
|
|
9101
9325
|
*/
|
|
9102
9326
|
async init() {
|
|
9103
9327
|
if (!this.initialized) {
|
|
9328
|
+
this.logger.info("Initializing B+ Tree");
|
|
9104
9329
|
await this.bptree.init();
|
|
9105
9330
|
this.initialized = true;
|
|
9106
9331
|
}
|
|
@@ -9148,6 +9373,7 @@ var require_cjs = __commonJS({
|
|
|
9148
9373
|
* @returns Metadata
|
|
9149
9374
|
*/
|
|
9150
9375
|
async getMetadata(tx) {
|
|
9376
|
+
this.logger.debug("Getting metadata");
|
|
9151
9377
|
if (!this.initialized) {
|
|
9152
9378
|
throw new Error("RowTableEngine instance is not initialized");
|
|
9153
9379
|
}
|
|
@@ -9173,6 +9399,7 @@ var require_cjs = __commonJS({
|
|
|
9173
9399
|
* @returns Array of PKs of the inserted data
|
|
9174
9400
|
*/
|
|
9175
9401
|
async insert(dataList, incrementRowCount, overflowForcly, tx) {
|
|
9402
|
+
this.logger.debug(`Inserting ${dataList.length} rows (overflowForcly: ${overflowForcly})`);
|
|
9176
9403
|
if (dataList.length === 0) {
|
|
9177
9404
|
return [];
|
|
9178
9405
|
}
|
|
@@ -9274,6 +9501,7 @@ var require_cjs = __commonJS({
|
|
|
9274
9501
|
* @param tx Transaction
|
|
9275
9502
|
*/
|
|
9276
9503
|
async update(pk, data, tx) {
|
|
9504
|
+
this.logger.debug(`Updating row with PK: ${pk}`);
|
|
9277
9505
|
await tx.__acquireWriteLock(0);
|
|
9278
9506
|
const rid = await this.getRidByPK(pk, tx);
|
|
9279
9507
|
if (rid === null) {
|
|
@@ -9359,6 +9587,7 @@ var require_cjs = __commonJS({
|
|
|
9359
9587
|
* @param tx Transaction
|
|
9360
9588
|
*/
|
|
9361
9589
|
async delete(pk, decrementRowCount, tx) {
|
|
9590
|
+
this.logger.debug(`Deleting row with PK: ${pk}`);
|
|
9362
9591
|
await tx.__acquireWriteLock(0);
|
|
9363
9592
|
const rid = await this.getRidByPK(pk, tx);
|
|
9364
9593
|
if (rid === null) {
|
|
@@ -9418,6 +9647,7 @@ var require_cjs = __commonJS({
|
|
|
9418
9647
|
* @returns Raw data of the row
|
|
9419
9648
|
*/
|
|
9420
9649
|
async selectByPK(pk, tx) {
|
|
9650
|
+
this.logger.debug(`Selecting row by PK: ${pk}`);
|
|
9421
9651
|
const rid = await this.getRidByPK(pk, tx);
|
|
9422
9652
|
if (rid === null) {
|
|
9423
9653
|
return null;
|
|
@@ -9432,6 +9662,7 @@ var require_cjs = __commonJS({
|
|
|
9432
9662
|
* @returns Array of raw data of the rows in the same order as input PKs
|
|
9433
9663
|
*/
|
|
9434
9664
|
async selectMany(pks, tx) {
|
|
9665
|
+
this.logger.debug(`Selecting many rows (${pks.length} PKs)`);
|
|
9435
9666
|
const collections = await this.collectItemsByPage(pks, tx);
|
|
9436
9667
|
return this.fetchRowsByRids(collections, pks.length, tx);
|
|
9437
9668
|
}
|
|
@@ -9752,10 +9983,10 @@ var require_cjs = __commonJS({
|
|
|
9752
9983
|
if (shouldTriggerCheckpoint) {
|
|
9753
9984
|
await this.pfs.checkpoint();
|
|
9754
9985
|
}
|
|
9986
|
+
} finally {
|
|
9755
9987
|
this.dirtyPages.clear();
|
|
9756
9988
|
this.undoPages.clear();
|
|
9757
9989
|
this.releaseAllLocks();
|
|
9758
|
-
} finally {
|
|
9759
9990
|
if (this._writeLockRelease) {
|
|
9760
9991
|
this._writeLockRelease();
|
|
9761
9992
|
this._writeLockRelease = null;
|
|
@@ -9810,25 +10041,99 @@ var require_cjs = __commonJS({
|
|
|
9810
10041
|
return this.storage.run(tx, callback);
|
|
9811
10042
|
}
|
|
9812
10043
|
};
|
|
10044
|
+
var Colors = {
|
|
10045
|
+
reset: "\x1B[0m",
|
|
10046
|
+
debug: "\x1B[36m",
|
|
10047
|
+
// Cyan
|
|
10048
|
+
info: "\x1B[32m",
|
|
10049
|
+
// Green
|
|
10050
|
+
warn: "\x1B[33m",
|
|
10051
|
+
// Yellow
|
|
10052
|
+
error: "\x1B[31m",
|
|
10053
|
+
// Red
|
|
10054
|
+
white: "\x1B[37m",
|
|
10055
|
+
// White
|
|
10056
|
+
grey: "\x1B[90m"
|
|
10057
|
+
// Grey
|
|
10058
|
+
};
|
|
10059
|
+
var LoggerManager = class {
|
|
10060
|
+
level;
|
|
10061
|
+
constructor(level = 2) {
|
|
10062
|
+
this.level = level;
|
|
10063
|
+
}
|
|
10064
|
+
setLevel(level) {
|
|
10065
|
+
this.level = level;
|
|
10066
|
+
}
|
|
10067
|
+
shouldLog(level) {
|
|
10068
|
+
if (this.level === 0) return false;
|
|
10069
|
+
return level >= this.level;
|
|
10070
|
+
}
|
|
10071
|
+
create(moduleName) {
|
|
10072
|
+
return new Logger2(this, moduleName);
|
|
10073
|
+
}
|
|
10074
|
+
};
|
|
10075
|
+
var Logger2 = class {
|
|
10076
|
+
constructor(parent, moduleName) {
|
|
10077
|
+
this.parent = parent;
|
|
10078
|
+
this.moduleName = moduleName;
|
|
10079
|
+
}
|
|
10080
|
+
debug(message, ...args) {
|
|
10081
|
+
if (this.parent.shouldLog(
|
|
10082
|
+
1
|
|
10083
|
+
/* Debug */
|
|
10084
|
+
)) {
|
|
10085
|
+
console.debug(`${Colors.debug}[DEBUG] [${this.moduleName}]${Colors.reset} ${Colors.white}${message}${Colors.reset}`, ...args);
|
|
10086
|
+
}
|
|
10087
|
+
}
|
|
10088
|
+
info(message, ...args) {
|
|
10089
|
+
if (this.parent.shouldLog(
|
|
10090
|
+
2
|
|
10091
|
+
/* Info */
|
|
10092
|
+
)) {
|
|
10093
|
+
console.info(`${Colors.info}[INFO] [${this.moduleName}]${Colors.reset} ${Colors.white}${message}${Colors.reset}`, ...args);
|
|
10094
|
+
}
|
|
10095
|
+
}
|
|
10096
|
+
warn(message, ...args) {
|
|
10097
|
+
if (this.parent.shouldLog(
|
|
10098
|
+
3
|
|
10099
|
+
/* Warning */
|
|
10100
|
+
)) {
|
|
10101
|
+
console.warn(`${Colors.warn}[WARN] [${this.moduleName}]${Colors.reset} ${Colors.white}${message}${Colors.reset}`, ...args);
|
|
10102
|
+
}
|
|
10103
|
+
}
|
|
10104
|
+
error(message, ...args) {
|
|
10105
|
+
if (this.parent.shouldLog(
|
|
10106
|
+
4
|
|
10107
|
+
/* Error */
|
|
10108
|
+
)) {
|
|
10109
|
+
console.error(`${Colors.error}[ERROR] [${this.moduleName}]${Colors.reset} ${Colors.white}${message}${Colors.reset}`, ...args);
|
|
10110
|
+
}
|
|
10111
|
+
}
|
|
10112
|
+
};
|
|
9813
10113
|
var DataplyAPI2 = class {
|
|
9814
10114
|
constructor(file, options) {
|
|
9815
10115
|
this.file = file;
|
|
9816
10116
|
this.hook = useHookall(this);
|
|
9817
10117
|
this.options = this.verboseOptions(options);
|
|
10118
|
+
this.loggerManager = new LoggerManager(this.options.logLevel);
|
|
10119
|
+
this.logger = this.loggerManager.create("DataplyAPI");
|
|
9818
10120
|
this.isNewlyCreated = !import_node_fs3.default.existsSync(file);
|
|
9819
10121
|
this.fileHandle = this.createOrOpen(file, this.options);
|
|
9820
10122
|
this.pfs = new PageFileSystem(
|
|
9821
10123
|
this.fileHandle,
|
|
9822
10124
|
this.options.pageSize,
|
|
9823
10125
|
this.options.pageCacheCapacity,
|
|
9824
|
-
this.options
|
|
10126
|
+
this.options,
|
|
10127
|
+
this.loggerManager.create("PageFileSystem"),
|
|
10128
|
+
options.wal ? this.loggerManager.create("WALManager") : void 0
|
|
9825
10129
|
);
|
|
9826
10130
|
this.textCodec = new TextCodec();
|
|
9827
10131
|
this.txContext = new TransactionContext();
|
|
9828
10132
|
this.lockManager = new LockManager();
|
|
9829
|
-
this.rowTableEngine = new RowTableEngine(this.pfs, this.txContext, this.options);
|
|
10133
|
+
this.rowTableEngine = new RowTableEngine(this.pfs, this.txContext, this.options, this.loggerManager.create("RowTableEngine"));
|
|
9830
10134
|
this.initialized = false;
|
|
9831
10135
|
this.txIdCounter = 0;
|
|
10136
|
+
this.logger.debug(`DataplyAPI instance created with file: ${file}`);
|
|
9832
10137
|
}
|
|
9833
10138
|
/**
|
|
9834
10139
|
* These are not the same options that were used when the database was created.
|
|
@@ -9850,6 +10155,10 @@ var require_cjs = __commonJS({
|
|
|
9850
10155
|
txContext;
|
|
9851
10156
|
/** Hook */
|
|
9852
10157
|
hook;
|
|
10158
|
+
/** Base Logger */
|
|
10159
|
+
loggerManager;
|
|
10160
|
+
/** Logger module for DataplyAPI */
|
|
10161
|
+
logger;
|
|
9853
10162
|
/** Whether the database was initialized via `init()` */
|
|
9854
10163
|
initialized;
|
|
9855
10164
|
/** Whether the database was created this time. */
|
|
@@ -9864,6 +10173,7 @@ var require_cjs = __commonJS({
|
|
|
9864
10173
|
* @returns Whether the page file is a valid Dataply file
|
|
9865
10174
|
*/
|
|
9866
10175
|
verifyFormat(fileHandle) {
|
|
10176
|
+
this.logger.debug(`Verifying format for file handle: ${fileHandle}`);
|
|
9867
10177
|
const size = MetadataPageManager.CONSTANT.OFFSET_MAGIC_STRING + MetadataPageManager.CONSTANT.MAGIC_STRING.length;
|
|
9868
10178
|
const metadataPage = new Uint8Array(size);
|
|
9869
10179
|
import_node_fs3.default.readSync(fileHandle, metadataPage, 0, size, 0);
|
|
@@ -9883,7 +10193,9 @@ var require_cjs = __commonJS({
|
|
|
9883
10193
|
pageCacheCapacity: 1e4,
|
|
9884
10194
|
pagePreallocationCount: 1e3,
|
|
9885
10195
|
wal: null,
|
|
9886
|
-
walCheckpointThreshold: 1e3
|
|
10196
|
+
walCheckpointThreshold: 1e3,
|
|
10197
|
+
logLevel: 2
|
|
10198
|
+
/* Info */
|
|
9887
10199
|
}, options);
|
|
9888
10200
|
}
|
|
9889
10201
|
/**
|
|
@@ -9894,6 +10206,7 @@ var require_cjs = __commonJS({
|
|
|
9894
10206
|
* @param fileHandle File handle
|
|
9895
10207
|
*/
|
|
9896
10208
|
initializeFile(file, fileHandle, options) {
|
|
10209
|
+
this.logger.info(`Initializing new dataply file: ${file}`);
|
|
9897
10210
|
const metadataPageManager = new MetadataPageManager();
|
|
9898
10211
|
const bitmapPageManager = new BitmapPageManager();
|
|
9899
10212
|
const dataPageManager = new DataPageManager();
|
|
@@ -9943,6 +10256,7 @@ var require_cjs = __commonJS({
|
|
|
9943
10256
|
* @returns File handle
|
|
9944
10257
|
*/
|
|
9945
10258
|
createOrOpen(file, options) {
|
|
10259
|
+
this.logger.info(`Opening dataply file: ${file}`);
|
|
9946
10260
|
let fileHandle;
|
|
9947
10261
|
if (options.pageCacheCapacity < 100) {
|
|
9948
10262
|
throw new Error("Page cache capacity must be at least 100");
|
|
@@ -9978,6 +10292,7 @@ var require_cjs = __commonJS({
|
|
|
9978
10292
|
* If not called, the dataply instance cannot be used.
|
|
9979
10293
|
*/
|
|
9980
10294
|
async init() {
|
|
10295
|
+
this.logger.info("Initializing DataplyAPI");
|
|
9981
10296
|
if (this.initialized) {
|
|
9982
10297
|
return;
|
|
9983
10298
|
}
|
|
@@ -9997,6 +10312,7 @@ var require_cjs = __commonJS({
|
|
|
9997
10312
|
* @returns Transaction object
|
|
9998
10313
|
*/
|
|
9999
10314
|
createTransaction() {
|
|
10315
|
+
this.logger.debug(`Creating transaction: ${this.txIdCounter + 1}`);
|
|
10000
10316
|
return new Transaction4(
|
|
10001
10317
|
++this.txIdCounter,
|
|
10002
10318
|
this.txContext,
|
|
@@ -10021,6 +10337,7 @@ var require_cjs = __commonJS({
|
|
|
10021
10337
|
* @returns A release function
|
|
10022
10338
|
*/
|
|
10023
10339
|
acquireWriteLock() {
|
|
10340
|
+
this.logger.debug("Acquiring write lock");
|
|
10024
10341
|
const previous = this.writeQueue;
|
|
10025
10342
|
let release;
|
|
10026
10343
|
this.writeQueue = new Promise((resolve) => {
|
|
@@ -10038,6 +10355,7 @@ var require_cjs = __commonJS({
|
|
|
10038
10355
|
* @returns The result of the callback.
|
|
10039
10356
|
*/
|
|
10040
10357
|
async runWithDefaultWrite(callback, tx) {
|
|
10358
|
+
this.logger.debug("Running with default write transaction");
|
|
10041
10359
|
if (!tx) {
|
|
10042
10360
|
const release = await this.acquireWriteLock();
|
|
10043
10361
|
const internalTx = this.createTransaction();
|
|
@@ -10061,6 +10379,7 @@ var require_cjs = __commonJS({
|
|
|
10061
10379
|
return result;
|
|
10062
10380
|
}
|
|
10063
10381
|
async runWithDefault(callback, tx) {
|
|
10382
|
+
this.logger.debug("Running with default transaction");
|
|
10064
10383
|
const isInternalTx = !tx;
|
|
10065
10384
|
if (!tx) {
|
|
10066
10385
|
tx = this.createTransaction();
|
|
@@ -10088,6 +10407,7 @@ var require_cjs = __commonJS({
|
|
|
10088
10407
|
* @returns An AsyncGenerator that yields values from the callback.
|
|
10089
10408
|
*/
|
|
10090
10409
|
async *streamWithDefault(callback, tx) {
|
|
10410
|
+
this.logger.debug("Streaming with default transaction");
|
|
10091
10411
|
const isInternalTx = !tx;
|
|
10092
10412
|
if (!tx) {
|
|
10093
10413
|
tx = this.createTransaction();
|
|
@@ -10115,6 +10435,7 @@ var require_cjs = __commonJS({
|
|
|
10115
10435
|
* @returns Metadata of the dataply.
|
|
10116
10436
|
*/
|
|
10117
10437
|
async getMetadata(tx) {
|
|
10438
|
+
this.logger.debug("Getting metadata");
|
|
10118
10439
|
if (!this.initialized) {
|
|
10119
10440
|
throw new Error("Dataply instance is not initialized");
|
|
10120
10441
|
}
|
|
@@ -10128,6 +10449,7 @@ var require_cjs = __commonJS({
|
|
|
10128
10449
|
* @returns PK of the added data
|
|
10129
10450
|
*/
|
|
10130
10451
|
async insert(data, incrementRowCount, tx) {
|
|
10452
|
+
this.logger.debug("Inserting data");
|
|
10131
10453
|
if (!this.initialized) {
|
|
10132
10454
|
throw new Error("Dataply instance is not initialized");
|
|
10133
10455
|
}
|
|
@@ -10148,6 +10470,7 @@ var require_cjs = __commonJS({
|
|
|
10148
10470
|
* @returns PK of the added data
|
|
10149
10471
|
*/
|
|
10150
10472
|
async insertAsOverflow(data, incrementRowCount, tx) {
|
|
10473
|
+
this.logger.debug("Inserting data as overflow");
|
|
10151
10474
|
if (!this.initialized) {
|
|
10152
10475
|
throw new Error("Dataply instance is not initialized");
|
|
10153
10476
|
}
|
|
@@ -10169,6 +10492,7 @@ var require_cjs = __commonJS({
|
|
|
10169
10492
|
* @returns Array of PKs of the added data
|
|
10170
10493
|
*/
|
|
10171
10494
|
async insertBatch(dataList, incrementRowCount, tx) {
|
|
10495
|
+
this.logger.debug(`Inserting batch data: ${dataList.length} items`);
|
|
10172
10496
|
if (!this.initialized) {
|
|
10173
10497
|
throw new Error("Dataply instance is not initialized");
|
|
10174
10498
|
}
|
|
@@ -10187,6 +10511,7 @@ var require_cjs = __commonJS({
|
|
|
10187
10511
|
* @param tx Transaction
|
|
10188
10512
|
*/
|
|
10189
10513
|
async update(pk, data, tx) {
|
|
10514
|
+
this.logger.debug(`Updating data for PK: ${pk}`);
|
|
10190
10515
|
if (!this.initialized) {
|
|
10191
10516
|
throw new Error("Dataply instance is not initialized");
|
|
10192
10517
|
}
|
|
@@ -10204,6 +10529,7 @@ var require_cjs = __commonJS({
|
|
|
10204
10529
|
* @param tx Transaction
|
|
10205
10530
|
*/
|
|
10206
10531
|
async delete(pk, decrementRowCount, tx) {
|
|
10532
|
+
this.logger.debug(`Deleting data for PK: ${pk}`);
|
|
10207
10533
|
if (!this.initialized) {
|
|
10208
10534
|
throw new Error("Dataply instance is not initialized");
|
|
10209
10535
|
}
|
|
@@ -10213,6 +10539,7 @@ var require_cjs = __commonJS({
|
|
|
10213
10539
|
}, tx);
|
|
10214
10540
|
}
|
|
10215
10541
|
async select(pk, asRaw = false, tx) {
|
|
10542
|
+
this.logger.debug(`Selecting data for PK: ${pk}`);
|
|
10216
10543
|
if (!this.initialized) {
|
|
10217
10544
|
throw new Error("Dataply instance is not initialized");
|
|
10218
10545
|
}
|
|
@@ -10224,6 +10551,7 @@ var require_cjs = __commonJS({
|
|
|
10224
10551
|
}, tx);
|
|
10225
10552
|
}
|
|
10226
10553
|
async selectMany(pks, asRaw = false, tx) {
|
|
10554
|
+
this.logger.debug(`Selecting many data: ${pks.length} keys`);
|
|
10227
10555
|
if (!this.initialized) {
|
|
10228
10556
|
throw new Error("Dataply instance is not initialized");
|
|
10229
10557
|
}
|
|
@@ -10240,6 +10568,7 @@ var require_cjs = __commonJS({
|
|
|
10240
10568
|
* Closes the dataply file.
|
|
10241
10569
|
*/
|
|
10242
10570
|
async close() {
|
|
10571
|
+
this.logger.info("Closing DataplyAPI");
|
|
10243
10572
|
if (!this.initialized) {
|
|
10244
10573
|
throw new Error("Dataply instance is not initialized");
|
|
10245
10574
|
}
|
|
@@ -10860,9 +11189,10 @@ var BinaryHeap = class {
|
|
|
10860
11189
|
|
|
10861
11190
|
// src/core/QueryManager.ts
|
|
10862
11191
|
var QueryManager = class {
|
|
10863
|
-
constructor(api, optimizer) {
|
|
11192
|
+
constructor(api, optimizer, logger) {
|
|
10864
11193
|
this.api = api;
|
|
10865
11194
|
this.optimizer = optimizer;
|
|
11195
|
+
this.logger = logger;
|
|
10866
11196
|
}
|
|
10867
11197
|
operatorConverters = {
|
|
10868
11198
|
equal: "primaryEqual",
|
|
@@ -11174,18 +11504,23 @@ var QueryManager = class {
|
|
|
11174
11504
|
async countDocuments(query, tx) {
|
|
11175
11505
|
return this.api.runWithDefault(async (tx2) => {
|
|
11176
11506
|
const pks = await this.getKeys(query);
|
|
11507
|
+
this.logger.debug(`Counted ${pks.length} documents matching query`);
|
|
11177
11508
|
return pks.length;
|
|
11178
11509
|
}, tx);
|
|
11179
11510
|
}
|
|
11180
11511
|
selectDocuments(query, options = {}, tx) {
|
|
11181
11512
|
for (const field of Object.keys(query)) {
|
|
11182
11513
|
if (!this.api.indexedFields.has(field)) {
|
|
11183
|
-
|
|
11514
|
+
const errorMsg = `Query field "${field}" is not indexed. Available indexed fields: ${Array.from(this.api.indexedFields).join(", ")}`;
|
|
11515
|
+
this.logger.error(errorMsg);
|
|
11516
|
+
throw new Error(errorMsg);
|
|
11184
11517
|
}
|
|
11185
11518
|
}
|
|
11186
11519
|
const orderBy = options.orderBy;
|
|
11187
11520
|
if (orderBy !== void 0 && !this.api.indexedFields.has(orderBy)) {
|
|
11188
|
-
|
|
11521
|
+
const errorMsg = `orderBy field "${orderBy}" is not indexed. Available indexed fields: ${Array.from(this.api.indexedFields).join(", ")}`;
|
|
11522
|
+
this.logger.error(errorMsg);
|
|
11523
|
+
throw new Error(errorMsg);
|
|
11189
11524
|
}
|
|
11190
11525
|
const {
|
|
11191
11526
|
limit = Infinity,
|
|
@@ -11282,6 +11617,7 @@ var QueryManager = class {
|
|
|
11282
11617
|
others,
|
|
11283
11618
|
tx2
|
|
11284
11619
|
)) {
|
|
11620
|
+
self.logger.debug(`Yielding sorted document: ${doc._id}`);
|
|
11285
11621
|
if (skippedCount < offset) {
|
|
11286
11622
|
skippedCount++;
|
|
11287
11623
|
continue;
|
|
@@ -11379,10 +11715,76 @@ var DocumentSerializeStrategyAsync = class extends import_dataply2.SerializeStra
|
|
|
11379
11715
|
}
|
|
11380
11716
|
};
|
|
11381
11717
|
|
|
11718
|
+
// src/utils/eventLoopManager.ts
|
|
11719
|
+
function yieldEventLoop() {
|
|
11720
|
+
return new Promise(setImmediate);
|
|
11721
|
+
}
|
|
11722
|
+
var DeadlineChunker = class {
|
|
11723
|
+
/**
|
|
11724
|
+
* 이벤트 루프를 막을 최대 허용 시간
|
|
11725
|
+
*/
|
|
11726
|
+
targetMs;
|
|
11727
|
+
/**
|
|
11728
|
+
* 현재 chunk size
|
|
11729
|
+
*/
|
|
11730
|
+
chunkSize;
|
|
11731
|
+
/**
|
|
11732
|
+
* Exponential Weighted Moving Average
|
|
11733
|
+
*/
|
|
11734
|
+
ewmaMs;
|
|
11735
|
+
/**
|
|
11736
|
+
* EWMA 평활화 계수
|
|
11737
|
+
*/
|
|
11738
|
+
alpha;
|
|
11739
|
+
constructor(startChunkSize = 0, targetMs = 5, alpha = 0.5) {
|
|
11740
|
+
this.chunkSize = startChunkSize;
|
|
11741
|
+
this.targetMs = targetMs;
|
|
11742
|
+
this.alpha = alpha;
|
|
11743
|
+
this.ewmaMs = null;
|
|
11744
|
+
}
|
|
11745
|
+
/**
|
|
11746
|
+
* EWMA 평활화 계수를 사용하여 평균 처리 시간을 업데이트합니다.
|
|
11747
|
+
*/
|
|
11748
|
+
updateEstimate(elapsed, count) {
|
|
11749
|
+
const msPerItem = elapsed / count;
|
|
11750
|
+
this.ewmaMs = this.ewmaMs === null ? msPerItem : this.alpha * msPerItem + (1 - this.alpha) * this.ewmaMs;
|
|
11751
|
+
}
|
|
11752
|
+
/**
|
|
11753
|
+
* 현재 chunk size를 업데이트합니다.
|
|
11754
|
+
*/
|
|
11755
|
+
nextChunkSize() {
|
|
11756
|
+
if (!this.ewmaMs || this.ewmaMs === 0) return this.chunkSize;
|
|
11757
|
+
const next = Math.floor(this.targetMs / this.ewmaMs);
|
|
11758
|
+
return Math.max(1, next);
|
|
11759
|
+
}
|
|
11760
|
+
/**
|
|
11761
|
+
* 주어진 items를 chunk로 분할하여 처리합니다.
|
|
11762
|
+
*/
|
|
11763
|
+
async processInChunks(items, processFn) {
|
|
11764
|
+
let i = 0;
|
|
11765
|
+
let len = items.length;
|
|
11766
|
+
if (this.chunkSize === 0) {
|
|
11767
|
+
this.chunkSize = Math.floor(items.length / 100 * 5);
|
|
11768
|
+
}
|
|
11769
|
+
while (i < len) {
|
|
11770
|
+
const chunk = items.slice(i, i + this.chunkSize);
|
|
11771
|
+
const count = chunk.length;
|
|
11772
|
+
const start = performance.now();
|
|
11773
|
+
await processFn(chunk);
|
|
11774
|
+
const elapsed = performance.now() - start;
|
|
11775
|
+
this.updateEstimate(elapsed, count);
|
|
11776
|
+
this.chunkSize = this.nextChunkSize();
|
|
11777
|
+
i += count;
|
|
11778
|
+
await yieldEventLoop();
|
|
11779
|
+
}
|
|
11780
|
+
}
|
|
11781
|
+
};
|
|
11782
|
+
|
|
11382
11783
|
// src/core/IndexManager.ts
|
|
11383
11784
|
var IndexManager = class {
|
|
11384
|
-
constructor(api) {
|
|
11785
|
+
constructor(api, logger) {
|
|
11385
11786
|
this.api = api;
|
|
11787
|
+
this.logger = logger;
|
|
11386
11788
|
this.trees = /* @__PURE__ */ new Map();
|
|
11387
11789
|
this.indexedFields = /* @__PURE__ */ new Set(["_id"]);
|
|
11388
11790
|
}
|
|
@@ -11484,6 +11886,7 @@ var IndexManager = class {
|
|
|
11484
11886
|
* Register an index. If called before init(), queues it.
|
|
11485
11887
|
*/
|
|
11486
11888
|
async registerIndex(name, option, tx) {
|
|
11889
|
+
this.logger.debug(`Registering index "${name}" (type: ${option.type})`);
|
|
11487
11890
|
if (!this.api.isDocInitialized) {
|
|
11488
11891
|
this.pendingCreateIndices.set(name, option);
|
|
11489
11892
|
return;
|
|
@@ -11556,7 +11959,7 @@ var IndexManager = class {
|
|
|
11556
11959
|
this.indices = metadata.indices;
|
|
11557
11960
|
this.registeredIndices.delete(name);
|
|
11558
11961
|
const fields = this.getFieldsFromConfig(config);
|
|
11559
|
-
for (let i = 0
|
|
11962
|
+
for (let i = 0, len = fields.length; i < len; i++) {
|
|
11560
11963
|
const field = fields[i];
|
|
11561
11964
|
const indexNames = this.fieldToIndices.get(field);
|
|
11562
11965
|
if (indexNames) {
|
|
@@ -11578,6 +11981,7 @@ var IndexManager = class {
|
|
|
11578
11981
|
* Backfill indices for newly created indices after data was inserted.
|
|
11579
11982
|
*/
|
|
11580
11983
|
async backfillIndices(tx) {
|
|
11984
|
+
this.logger.debug(`Starting backfill for fields: ${this.pendingBackfillFields.join(", ")}`);
|
|
11581
11985
|
return this.api.runWithDefaultWrite(async (tx2) => {
|
|
11582
11986
|
if (this.pendingBackfillFields.length === 0) {
|
|
11583
11987
|
return 0;
|
|
@@ -11587,16 +11991,14 @@ var IndexManager = class {
|
|
|
11587
11991
|
if (metadata.lastId === 0) {
|
|
11588
11992
|
return 0;
|
|
11589
11993
|
}
|
|
11590
|
-
|
|
11994
|
+
const bulkData = {};
|
|
11591
11995
|
for (const indexName of backfillTargets) {
|
|
11592
11996
|
const tree = this.trees.get(indexName);
|
|
11593
11997
|
if (tree && indexName !== "_id") {
|
|
11594
|
-
|
|
11998
|
+
bulkData[indexName] = [];
|
|
11595
11999
|
}
|
|
11596
12000
|
}
|
|
11597
12001
|
let backfilledCount = 0;
|
|
11598
|
-
let chunkCount = 0;
|
|
11599
|
-
const CHUNK_SIZE = 1e3;
|
|
11600
12002
|
const idTree = this.trees.get("_id");
|
|
11601
12003
|
if (!idTree) {
|
|
11602
12004
|
throw new Error("ID tree not found");
|
|
@@ -11610,70 +12012,145 @@ var IndexManager = class {
|
|
|
11610
12012
|
const flatDoc = this.api.flattenDocument(doc);
|
|
11611
12013
|
for (let i = 0, len = backfillTargets.length; i < len; i++) {
|
|
11612
12014
|
const indexName = backfillTargets[i];
|
|
11613
|
-
if (!(indexName in
|
|
12015
|
+
if (!(indexName in bulkData)) continue;
|
|
11614
12016
|
const config = this.registeredIndices.get(indexName);
|
|
11615
12017
|
if (!config) continue;
|
|
11616
|
-
const btx = indexTxMap[indexName];
|
|
11617
12018
|
if (config.type === "fts") {
|
|
11618
12019
|
const primaryField = this.getPrimaryField(config);
|
|
11619
12020
|
const v2 = flatDoc[primaryField];
|
|
11620
12021
|
if (v2 === void 0 || typeof v2 !== "string") continue;
|
|
11621
12022
|
const ftsConfig = this.getFtsConfig(config);
|
|
11622
12023
|
const tokens = ftsConfig ? tokenize(v2, ftsConfig) : [v2];
|
|
11623
|
-
|
|
11624
|
-
|
|
11625
|
-
const token = tokens[i2];
|
|
12024
|
+
for (let j = 0, tLen = tokens.length; j < tLen; j++) {
|
|
12025
|
+
const token = tokens[j];
|
|
11626
12026
|
const keyToInsert = this.getTokenKey(k2, token);
|
|
11627
12027
|
const entry = { k: k2, v: token };
|
|
11628
|
-
|
|
12028
|
+
bulkData[indexName].push([keyToInsert, entry]);
|
|
11629
12029
|
}
|
|
11630
|
-
await btx.batchInsert(batchInsertData);
|
|
11631
12030
|
} else {
|
|
11632
12031
|
const indexVal = this.getIndexValue(config, flatDoc);
|
|
11633
12032
|
if (indexVal === void 0) continue;
|
|
11634
12033
|
const entry = { k: k2, v: indexVal };
|
|
11635
|
-
|
|
11636
|
-
await btx.batchInsert(batchInsertData);
|
|
12034
|
+
bulkData[indexName].push([k2, entry]);
|
|
11637
12035
|
}
|
|
11638
12036
|
}
|
|
11639
12037
|
backfilledCount++;
|
|
11640
|
-
chunkCount++;
|
|
11641
|
-
if (chunkCount >= CHUNK_SIZE) {
|
|
11642
|
-
try {
|
|
11643
|
-
for (const btx of Object.values(indexTxMap)) {
|
|
11644
|
-
await btx.commit();
|
|
11645
|
-
}
|
|
11646
|
-
} catch (err) {
|
|
11647
|
-
for (const btx of Object.values(indexTxMap)) {
|
|
11648
|
-
await btx.rollback();
|
|
11649
|
-
}
|
|
11650
|
-
throw err;
|
|
11651
|
-
}
|
|
11652
|
-
for (const indexName of backfillTargets) {
|
|
11653
|
-
const tree = this.trees.get(indexName);
|
|
11654
|
-
if (tree && indexName !== "_id") {
|
|
11655
|
-
indexTxMap[indexName] = await tree.createTransaction();
|
|
11656
|
-
}
|
|
11657
|
-
}
|
|
11658
|
-
chunkCount = 0;
|
|
11659
|
-
}
|
|
11660
12038
|
}
|
|
11661
|
-
|
|
12039
|
+
for (const indexName of backfillTargets) {
|
|
12040
|
+
const tree = this.trees.get(indexName);
|
|
12041
|
+
if (!tree || indexName === "_id") continue;
|
|
12042
|
+
const entries = bulkData[indexName];
|
|
12043
|
+
if (!entries || entries.length === 0) continue;
|
|
12044
|
+
const treeTx = await tree.createTransaction();
|
|
11662
12045
|
try {
|
|
11663
|
-
|
|
11664
|
-
|
|
12046
|
+
await treeTx.bulkLoad(entries);
|
|
12047
|
+
const res = await treeTx.commit();
|
|
12048
|
+
if (!res.success) {
|
|
12049
|
+
await treeTx.rollback();
|
|
12050
|
+
throw res.error;
|
|
11665
12051
|
}
|
|
11666
12052
|
} catch (err) {
|
|
11667
|
-
|
|
11668
|
-
await btx.rollback();
|
|
11669
|
-
}
|
|
12053
|
+
await treeTx.rollback();
|
|
11670
12054
|
throw err;
|
|
11671
12055
|
}
|
|
12056
|
+
await yieldEventLoop();
|
|
11672
12057
|
}
|
|
11673
12058
|
this.pendingBackfillFields = [];
|
|
11674
12059
|
return backfilledCount;
|
|
11675
12060
|
}, tx);
|
|
11676
12061
|
}
|
|
12062
|
+
/**
|
|
12063
|
+
* Rebuild specified indices by clearing existing tree data and rebuilding via bulkLoad.
|
|
12064
|
+
* If no index names are provided, all indices (except _id) are rebuilt.
|
|
12065
|
+
*/
|
|
12066
|
+
async rebuildIndices(indexNames, tx) {
|
|
12067
|
+
const targets = indexNames ?? [...this.registeredIndices.keys()].filter((n) => n !== "_id");
|
|
12068
|
+
for (const name of targets) {
|
|
12069
|
+
if (name === "_id") {
|
|
12070
|
+
throw new Error('Cannot rebuild the "_id" index.');
|
|
12071
|
+
}
|
|
12072
|
+
if (!this.registeredIndices.has(name)) {
|
|
12073
|
+
throw new Error(`Index "${name}" does not exist.`);
|
|
12074
|
+
}
|
|
12075
|
+
}
|
|
12076
|
+
if (targets.length === 0) return 0;
|
|
12077
|
+
this.logger.debug(`Starting rebuild for indices: ${targets.join(", ")}`);
|
|
12078
|
+
return this.api.runWithDefaultWrite(async (tx2) => {
|
|
12079
|
+
const metadata = await this.api.getDocumentInnerMetadata(tx2);
|
|
12080
|
+
if (metadata.lastId === 0) return 0;
|
|
12081
|
+
const bulkData = {};
|
|
12082
|
+
for (const indexName of targets) {
|
|
12083
|
+
bulkData[indexName] = [];
|
|
12084
|
+
}
|
|
12085
|
+
const idTree = this.trees.get("_id");
|
|
12086
|
+
if (!idTree) throw new Error("ID tree not found");
|
|
12087
|
+
let docCount = 0;
|
|
12088
|
+
const stream = idTree.whereStream({ primaryGte: { v: 0 } });
|
|
12089
|
+
for await (const [k2] of stream) {
|
|
12090
|
+
const doc = await this.api.getDocument(k2, tx2);
|
|
12091
|
+
if (!doc) continue;
|
|
12092
|
+
const flatDoc = this.api.flattenDocument(doc);
|
|
12093
|
+
for (let i = 0, len = targets.length; i < len; i++) {
|
|
12094
|
+
const indexName = targets[i];
|
|
12095
|
+
const config = this.registeredIndices.get(indexName);
|
|
12096
|
+
if (!config) continue;
|
|
12097
|
+
if (config.type === "fts") {
|
|
12098
|
+
const primaryField = this.getPrimaryField(config);
|
|
12099
|
+
const v2 = flatDoc[primaryField];
|
|
12100
|
+
if (v2 === void 0 || typeof v2 !== "string") continue;
|
|
12101
|
+
const ftsConfig = this.getFtsConfig(config);
|
|
12102
|
+
const tokens = ftsConfig ? tokenize(v2, ftsConfig) : [v2];
|
|
12103
|
+
for (let j = 0, tLen = tokens.length; j < tLen; j++) {
|
|
12104
|
+
const token = tokens[j];
|
|
12105
|
+
const keyToInsert = this.getTokenKey(k2, token);
|
|
12106
|
+
bulkData[indexName].push([keyToInsert, { k: k2, v: token }]);
|
|
12107
|
+
}
|
|
12108
|
+
} else {
|
|
12109
|
+
const indexVal = this.getIndexValue(config, flatDoc);
|
|
12110
|
+
if (indexVal === void 0) continue;
|
|
12111
|
+
bulkData[indexName].push([k2, { k: k2, v: indexVal }]);
|
|
12112
|
+
}
|
|
12113
|
+
}
|
|
12114
|
+
docCount++;
|
|
12115
|
+
}
|
|
12116
|
+
const perIndexCapacity = Math.floor(this.api.options.pageCacheCapacity / Object.keys(this.api.indices).length);
|
|
12117
|
+
for (const indexName of targets) {
|
|
12118
|
+
metadata.indices[indexName][0] = -1;
|
|
12119
|
+
await this.api.updateDocumentInnerMetadata(metadata, tx2);
|
|
12120
|
+
const tree = new import_dataply3.BPTreeAsync(
|
|
12121
|
+
new DocumentSerializeStrategyAsync(
|
|
12122
|
+
this.api.rowTableEngine.order,
|
|
12123
|
+
this.api,
|
|
12124
|
+
this.api.txContext,
|
|
12125
|
+
indexName
|
|
12126
|
+
),
|
|
12127
|
+
this.api.comparator,
|
|
12128
|
+
{
|
|
12129
|
+
capacity: perIndexCapacity
|
|
12130
|
+
}
|
|
12131
|
+
);
|
|
12132
|
+
await tree.init();
|
|
12133
|
+
this.trees.set(indexName, tree);
|
|
12134
|
+
const entries = bulkData[indexName];
|
|
12135
|
+
if (entries.length > 0) {
|
|
12136
|
+
const treeTx = await tree.createTransaction();
|
|
12137
|
+
try {
|
|
12138
|
+
await treeTx.bulkLoad(entries);
|
|
12139
|
+
const res = await treeTx.commit();
|
|
12140
|
+
if (!res.success) {
|
|
12141
|
+
await treeTx.rollback();
|
|
12142
|
+
throw res.error;
|
|
12143
|
+
}
|
|
12144
|
+
} catch (err) {
|
|
12145
|
+
await treeTx.rollback();
|
|
12146
|
+
throw err;
|
|
12147
|
+
}
|
|
12148
|
+
await yieldEventLoop();
|
|
12149
|
+
}
|
|
12150
|
+
}
|
|
12151
|
+
return docCount;
|
|
12152
|
+
}, tx);
|
|
12153
|
+
}
|
|
11677
12154
|
/**
|
|
11678
12155
|
* Convert CreateIndexOption to IndexMetaConfig for metadata storage.
|
|
11679
12156
|
*/
|
|
@@ -11779,72 +12256,19 @@ async function catchPromise(promise) {
|
|
|
11779
12256
|
return promise.then((res) => [void 0, res]).catch((reason) => [reason]);
|
|
11780
12257
|
}
|
|
11781
12258
|
|
|
11782
|
-
// src/utils/DeadlineChunker.ts
|
|
11783
|
-
var DeadlineChunker = class {
|
|
11784
|
-
/**
|
|
11785
|
-
* 이벤트 루프를 막을 최대 허용 시간
|
|
11786
|
-
*/
|
|
11787
|
-
targetMs;
|
|
11788
|
-
/**
|
|
11789
|
-
* 현재 chunk size
|
|
11790
|
-
*/
|
|
11791
|
-
chunkSize;
|
|
11792
|
-
/**
|
|
11793
|
-
* Exponential Weighted Moving Average
|
|
11794
|
-
*/
|
|
11795
|
-
ewmaMs;
|
|
11796
|
-
/**
|
|
11797
|
-
* EWMA 평활화 계수
|
|
11798
|
-
*/
|
|
11799
|
-
alpha;
|
|
11800
|
-
constructor(startChunkSize = 0, targetMs = 5, alpha = 0.5) {
|
|
11801
|
-
this.chunkSize = startChunkSize;
|
|
11802
|
-
this.targetMs = targetMs;
|
|
11803
|
-
this.alpha = alpha;
|
|
11804
|
-
this.ewmaMs = null;
|
|
11805
|
-
}
|
|
11806
|
-
/**
|
|
11807
|
-
* EWMA 평활화 계수를 사용하여 평균 처리 시간을 업데이트합니다.
|
|
11808
|
-
*/
|
|
11809
|
-
updateEstimate(elapsed, count) {
|
|
11810
|
-
const msPerItem = elapsed / count;
|
|
11811
|
-
this.ewmaMs = this.ewmaMs === null ? msPerItem : this.alpha * msPerItem + (1 - this.alpha) * this.ewmaMs;
|
|
11812
|
-
}
|
|
11813
|
-
/**
|
|
11814
|
-
* 현재 chunk size를 업데이트합니다.
|
|
11815
|
-
*/
|
|
11816
|
-
nextChunkSize() {
|
|
11817
|
-
if (!this.ewmaMs || this.ewmaMs === 0) return this.chunkSize;
|
|
11818
|
-
const next = Math.floor(this.targetMs / this.ewmaMs);
|
|
11819
|
-
return Math.max(1, next);
|
|
11820
|
-
}
|
|
11821
|
-
/**
|
|
11822
|
-
* 주어진 items를 chunk로 분할하여 처리합니다.
|
|
11823
|
-
*/
|
|
11824
|
-
async processInChunks(items, processFn) {
|
|
11825
|
-
let i = 0;
|
|
11826
|
-
let len = items.length;
|
|
11827
|
-
if (this.chunkSize === 0) {
|
|
11828
|
-
this.chunkSize = Math.floor(items.length / 100 * 5);
|
|
11829
|
-
}
|
|
11830
|
-
while (i < len) {
|
|
11831
|
-
const chunk = items.slice(i, i + this.chunkSize);
|
|
11832
|
-
const count = chunk.length;
|
|
11833
|
-
const start = performance.now();
|
|
11834
|
-
await processFn(chunk);
|
|
11835
|
-
const elapsed = performance.now() - start;
|
|
11836
|
-
this.updateEstimate(elapsed, count);
|
|
11837
|
-
this.chunkSize = this.nextChunkSize();
|
|
11838
|
-
i += count;
|
|
11839
|
-
await new Promise(setImmediate);
|
|
11840
|
-
}
|
|
11841
|
-
}
|
|
11842
|
-
};
|
|
11843
|
-
|
|
11844
12259
|
// src/core/MutationManager.ts
|
|
11845
12260
|
var MutationManager = class {
|
|
11846
|
-
constructor(api) {
|
|
12261
|
+
constructor(api, logger) {
|
|
11847
12262
|
this.api = api;
|
|
12263
|
+
this.logger = logger;
|
|
12264
|
+
}
|
|
12265
|
+
async isTreeEmpty(tree, tx) {
|
|
12266
|
+
try {
|
|
12267
|
+
const root = await tree.getNode(tree.rootId, tx);
|
|
12268
|
+
return root.leaf && root.values.length === 0;
|
|
12269
|
+
} catch {
|
|
12270
|
+
return true;
|
|
12271
|
+
}
|
|
11848
12272
|
}
|
|
11849
12273
|
async insertDocumentInternal(document, tx) {
|
|
11850
12274
|
const metadata = await this.api.getDocumentInnerMetadata(tx);
|
|
@@ -11887,16 +12311,19 @@ var MutationManager = class {
|
|
|
11887
12311
|
}
|
|
11888
12312
|
}
|
|
11889
12313
|
await this.api.analysisManager.notifyInsert([flattenDocument], tx2);
|
|
12314
|
+
this.logger.debug(`Inserted single document with ID: ${dataplyDocument._id}`);
|
|
11890
12315
|
return dataplyDocument._id;
|
|
11891
12316
|
}, tx);
|
|
11892
12317
|
}
|
|
11893
12318
|
async insertBatchDocuments(documents, tx) {
|
|
11894
12319
|
return this.api.runWithDefaultWrite(async (tx2) => {
|
|
12320
|
+
this.logger.debug(`Batch inserting ${documents.length} documents`);
|
|
11895
12321
|
const metadata = await this.api.getDocumentInnerMetadata(tx2);
|
|
11896
|
-
|
|
12322
|
+
let startId = metadata.lastId + 1;
|
|
11897
12323
|
metadata.lastId += documents.length;
|
|
11898
12324
|
await this.api.updateDocumentInnerMetadata(metadata, tx2);
|
|
11899
|
-
const ids =
|
|
12325
|
+
const ids = new Float64Array(documents.length);
|
|
12326
|
+
const pks = new Float64Array(documents.length);
|
|
11900
12327
|
const dataplyDocuments = [];
|
|
11901
12328
|
const flattenedData = [];
|
|
11902
12329
|
for (let i = 0, len = documents.length; i < len; i++) {
|
|
@@ -11908,17 +12335,15 @@ var MutationManager = class {
|
|
|
11908
12335
|
dataplyDocuments.push(stringified);
|
|
11909
12336
|
const flattenDocument = this.api.flattenDocument(dataplyDocument);
|
|
11910
12337
|
flattenedData.push({ pk: -1, data: flattenDocument });
|
|
11911
|
-
ids
|
|
12338
|
+
ids[i] = id;
|
|
11912
12339
|
}
|
|
11913
|
-
const
|
|
11914
|
-
|
|
11915
|
-
|
|
11916
|
-
|
|
11917
|
-
|
|
11918
|
-
});
|
|
11919
|
-
for (let i = 0, len = pks.length; i < len; i++) {
|
|
11920
|
-
flattenedData[i].pk = pks[i];
|
|
12340
|
+
const res = await this.api.insertBatch(dataplyDocuments, true, tx2);
|
|
12341
|
+
for (let i = 0, len = res.length; i < len; i++) {
|
|
12342
|
+
const index = i;
|
|
12343
|
+
pks[index] = res[i];
|
|
12344
|
+
flattenedData[index].pk = res[i];
|
|
11921
12345
|
}
|
|
12346
|
+
await yieldEventLoop();
|
|
11922
12347
|
for (const [indexName, config] of this.api.indexManager.registeredIndices) {
|
|
11923
12348
|
const tree = this.api.trees.get(indexName);
|
|
11924
12349
|
if (!tree) continue;
|
|
@@ -11945,34 +12370,39 @@ var MutationManager = class {
|
|
|
11945
12370
|
batchInsertData.push([item.pk, { k: item.pk, v: indexVal }]);
|
|
11946
12371
|
}
|
|
11947
12372
|
}
|
|
11948
|
-
const
|
|
11949
|
-
|
|
11950
|
-
|
|
11951
|
-
Math.max(initMaxSize, Math.floor(batchInsertData.length / 100 * 5))
|
|
11952
|
-
);
|
|
11953
|
-
const chunker = new DeadlineChunker(initChunkSize);
|
|
11954
|
-
await chunker.processInChunks(batchInsertData, async (chunk) => {
|
|
11955
|
-
const [error] = await catchPromise(treeTx.batchInsert(chunk));
|
|
12373
|
+
const isEmptyTree = await this.isTreeEmpty(tree, tx2);
|
|
12374
|
+
if (isEmptyTree) {
|
|
12375
|
+
const [error] = await catchPromise(treeTx.bulkLoad(batchInsertData));
|
|
11956
12376
|
if (error) {
|
|
11957
12377
|
throw error;
|
|
11958
12378
|
}
|
|
11959
|
-
}
|
|
11960
|
-
|
|
11961
|
-
|
|
12379
|
+
} else {
|
|
12380
|
+
const [error] = await catchPromise(treeTx.batchInsert(batchInsertData));
|
|
12381
|
+
if (error) {
|
|
12382
|
+
throw error;
|
|
12383
|
+
}
|
|
12384
|
+
}
|
|
12385
|
+
const res2 = await treeTx.commit();
|
|
12386
|
+
if (!res2.success) {
|
|
11962
12387
|
await treeTx.rollback();
|
|
11963
|
-
|
|
12388
|
+
this.logger.error(`Failed to commit batch insert for index ${indexName}: ${res2.error}`);
|
|
12389
|
+
throw res2.error;
|
|
11964
12390
|
}
|
|
12391
|
+
await yieldEventLoop();
|
|
11965
12392
|
}
|
|
11966
12393
|
const flatDocs = [];
|
|
11967
12394
|
for (let i = 0, len = flattenedData.length; i < len; i++) {
|
|
11968
12395
|
flatDocs.push(flattenedData[i].data);
|
|
11969
12396
|
}
|
|
11970
12397
|
await this.api.analysisManager.notifyInsert(flatDocs, tx2);
|
|
11971
|
-
|
|
12398
|
+
await yieldEventLoop();
|
|
12399
|
+
this.logger.debug(`Successfully batch inserted ${documents.length} documents`);
|
|
12400
|
+
return Array.from(ids);
|
|
11972
12401
|
}, tx);
|
|
11973
12402
|
}
|
|
11974
12403
|
async updateInternal(query, computeUpdatedDoc, tx) {
|
|
11975
12404
|
const pks = await this.api.queryManager.getKeys(query);
|
|
12405
|
+
this.logger.debug(`Found ${pks.length} documents to update`);
|
|
11976
12406
|
let updatedCount = 0;
|
|
11977
12407
|
const updatePairs = [];
|
|
11978
12408
|
const treeTxs = /* @__PURE__ */ new Map();
|
|
@@ -12021,9 +12451,11 @@ var MutationManager = class {
|
|
|
12021
12451
|
await treeTx.batchInsert([[pk, { k: pk, v: newIndexVal }]]);
|
|
12022
12452
|
}
|
|
12023
12453
|
}
|
|
12454
|
+
await yieldEventLoop();
|
|
12024
12455
|
}
|
|
12025
12456
|
updatePairs.push({ oldDocument: oldFlatDoc, newDocument: newFlatDoc });
|
|
12026
12457
|
await this.api.update(pk, JSON.stringify(updatedDoc), tx);
|
|
12458
|
+
await yieldEventLoop();
|
|
12027
12459
|
updatedCount++;
|
|
12028
12460
|
}
|
|
12029
12461
|
for (const [indexName, treeTx] of treeTxs) {
|
|
@@ -12032,10 +12464,14 @@ var MutationManager = class {
|
|
|
12032
12464
|
for (const rollbackTx of treeTxs.values()) {
|
|
12033
12465
|
rollbackTx.rollback();
|
|
12034
12466
|
}
|
|
12467
|
+
await yieldEventLoop();
|
|
12468
|
+
this.logger.error(`Failed to commit update for index ${indexName}: ${result.error}`);
|
|
12035
12469
|
throw result.error;
|
|
12036
12470
|
}
|
|
12037
12471
|
}
|
|
12038
12472
|
await this.api.analysisManager.notifyUpdate(updatePairs, tx);
|
|
12473
|
+
await yieldEventLoop();
|
|
12474
|
+
this.logger.debug(`Successfully updated ${updatedCount} documents`);
|
|
12039
12475
|
return updatedCount;
|
|
12040
12476
|
}
|
|
12041
12477
|
async fullUpdate(query, newRecord, tx) {
|
|
@@ -12059,6 +12495,7 @@ var MutationManager = class {
|
|
|
12059
12495
|
async deleteDocuments(query, tx) {
|
|
12060
12496
|
return this.api.runWithDefaultWrite(async (tx2) => {
|
|
12061
12497
|
const pks = await this.api.queryManager.getKeys(query);
|
|
12498
|
+
this.logger.debug(`Found ${pks.length} documents to delete`);
|
|
12062
12499
|
let deletedCount = 0;
|
|
12063
12500
|
const deletedFlatDocs = [];
|
|
12064
12501
|
for (let i = 0, len = pks.length; i < len; i++) {
|
|
@@ -12083,12 +12520,16 @@ var MutationManager = class {
|
|
|
12083
12520
|
if (indexVal === void 0) continue;
|
|
12084
12521
|
await tree.delete(pk, { k: pk, v: indexVal });
|
|
12085
12522
|
}
|
|
12523
|
+
await yieldEventLoop();
|
|
12086
12524
|
}
|
|
12087
12525
|
deletedFlatDocs.push(flatDoc);
|
|
12088
12526
|
await this.api.delete(pk, true, tx2);
|
|
12527
|
+
await yieldEventLoop();
|
|
12089
12528
|
deletedCount++;
|
|
12090
12529
|
}
|
|
12091
12530
|
await this.api.analysisManager.notifyDelete(deletedFlatDocs, tx2);
|
|
12531
|
+
await yieldEventLoop();
|
|
12532
|
+
this.logger.debug(`Successfully deleted ${deletedCount} documents`);
|
|
12092
12533
|
return deletedCount;
|
|
12093
12534
|
}, tx);
|
|
12094
12535
|
}
|
|
@@ -12096,8 +12537,9 @@ var MutationManager = class {
|
|
|
12096
12537
|
|
|
12097
12538
|
// src/core/MetadataManager.ts
|
|
12098
12539
|
var MetadataManager = class {
|
|
12099
|
-
constructor(api) {
|
|
12540
|
+
constructor(api, logger) {
|
|
12100
12541
|
this.api = api;
|
|
12542
|
+
this.logger = logger;
|
|
12101
12543
|
}
|
|
12102
12544
|
async getDocumentMetadata(tx) {
|
|
12103
12545
|
const metadata = await this.api.getMetadata(tx);
|
|
@@ -12125,6 +12567,7 @@ var MetadataManager = class {
|
|
|
12125
12567
|
return JSON.parse(row);
|
|
12126
12568
|
}
|
|
12127
12569
|
async updateDocumentInnerMetadata(metadata, tx) {
|
|
12570
|
+
this.logger.debug(`Updating document inner metadata: version ${metadata.version}`);
|
|
12128
12571
|
await this.api.update(1, JSON.stringify(metadata), tx);
|
|
12129
12572
|
}
|
|
12130
12573
|
async migration(version, callback, tx) {
|
|
@@ -12134,6 +12577,7 @@ var MetadataManager = class {
|
|
|
12134
12577
|
if (currentVersion < version) {
|
|
12135
12578
|
await callback(tx2);
|
|
12136
12579
|
const freshMetadata = await this.getDocumentInnerMetadata(tx2);
|
|
12580
|
+
this.logger.info(`Migration applied successfully to schemeVersion ${version}`);
|
|
12137
12581
|
freshMetadata.schemeVersion = version;
|
|
12138
12582
|
freshMetadata.updatedAt = Date.now();
|
|
12139
12583
|
await this.updateDocumentInnerMetadata(freshMetadata, tx2);
|
|
@@ -12970,9 +13414,10 @@ var BuiltinAnalysisProviders = [
|
|
|
12970
13414
|
|
|
12971
13415
|
// src/core/AnalysisManager.ts
|
|
12972
13416
|
var AnalysisManager = class {
|
|
12973
|
-
constructor(api, schedule, sampleSize) {
|
|
13417
|
+
constructor(api, schedule, sampleSize, logger) {
|
|
12974
13418
|
this.api = api;
|
|
12975
13419
|
this.sampleSize = sampleSize;
|
|
13420
|
+
this.logger = logger;
|
|
12976
13421
|
this.cron = new E(schedule, async () => {
|
|
12977
13422
|
if (this.flushing) return;
|
|
12978
13423
|
await this.api.flushAnalysis();
|
|
@@ -13101,6 +13546,7 @@ var AnalysisManager = class {
|
|
|
13101
13546
|
if (this.flushing) {
|
|
13102
13547
|
throw new Error("Cannot flush analysis while analysis is already flushing.");
|
|
13103
13548
|
}
|
|
13549
|
+
this.logger.debug("Starting analysis data flush");
|
|
13104
13550
|
this.flushing = true;
|
|
13105
13551
|
for (const [name, provider] of this.providers) {
|
|
13106
13552
|
if (provider instanceof IntervalAnalysisProvider) {
|
|
@@ -13108,6 +13554,7 @@ var AnalysisManager = class {
|
|
|
13108
13554
|
}
|
|
13109
13555
|
}
|
|
13110
13556
|
this.flushing = false;
|
|
13557
|
+
this.logger.debug("Finished analysis data flush");
|
|
13111
13558
|
}
|
|
13112
13559
|
/**
|
|
13113
13560
|
* Get the analysis header row.
|
|
@@ -13235,32 +13682,39 @@ var DocumentDataplyAPI = class extends import_dataply4.DataplyAPI {
|
|
|
13235
13682
|
constructor(file, options) {
|
|
13236
13683
|
super(file, options);
|
|
13237
13684
|
this.optimizer = new Optimizer(this);
|
|
13238
|
-
this.queryManager = new QueryManager(this, this.optimizer);
|
|
13239
|
-
this.indexManager = new IndexManager(this);
|
|
13240
|
-
this.mutationManager = new MutationManager(this);
|
|
13241
|
-
this.metadataManager = new MetadataManager(this);
|
|
13685
|
+
this.queryManager = new QueryManager(this, this.optimizer, this.loggerManager.create("document-dataply:query"));
|
|
13686
|
+
this.indexManager = new IndexManager(this, this.loggerManager.create("document-dataply:index"));
|
|
13687
|
+
this.mutationManager = new MutationManager(this, this.loggerManager.create("document-dataply:mutation"));
|
|
13688
|
+
this.metadataManager = new MetadataManager(this, this.loggerManager.create("document-dataply:metadata"));
|
|
13242
13689
|
this.documentFormatter = new DocumentFormatter();
|
|
13243
13690
|
this.analysisManager = new AnalysisManager(
|
|
13244
13691
|
this,
|
|
13245
13692
|
options.analysisSchedule ?? "* */1 * * *",
|
|
13246
|
-
options.analysisSampleSize ?? 1e3
|
|
13693
|
+
options.analysisSampleSize ?? 1e3,
|
|
13694
|
+
this.loggerManager.create("document-dataply:analysis")
|
|
13247
13695
|
);
|
|
13248
13696
|
this.hook.onceAfter("close", async () => {
|
|
13697
|
+
this.logger.info("DocumentDataplyAPI closing");
|
|
13249
13698
|
this.analysisManager.close();
|
|
13250
13699
|
});
|
|
13251
13700
|
this.hook.onceAfter("init", async (tx, isNewlyCreated) => {
|
|
13701
|
+
this.logger.info(`Initializing document database. New creation: ${isNewlyCreated}`);
|
|
13252
13702
|
if (isNewlyCreated) {
|
|
13253
13703
|
await this.initializeDocumentFile(tx);
|
|
13704
|
+
this.logger.debug("Initialized document file format");
|
|
13254
13705
|
}
|
|
13255
13706
|
if (!await this.verifyDocumentFile(tx)) {
|
|
13707
|
+
this.logger.error("Document metadata verification failed");
|
|
13256
13708
|
throw new Error("Document metadata verification failed");
|
|
13257
13709
|
}
|
|
13258
13710
|
const metadata = await this.getDocumentInnerMetadata(tx);
|
|
13259
13711
|
await this.indexManager.initializeIndices(metadata, isNewlyCreated, tx);
|
|
13712
|
+
this.logger.debug(`Indices initialized. Total indices: ${Object.keys(metadata.indices).length}`);
|
|
13260
13713
|
this.analysisManager.registerBuiltinProviders();
|
|
13261
13714
|
await this.analysisManager.initializeProviders(tx);
|
|
13262
13715
|
this.analysisManager.triggerCron();
|
|
13263
13716
|
this._initialized = true;
|
|
13717
|
+
this.logger.info("Document database fully initialized");
|
|
13264
13718
|
return tx;
|
|
13265
13719
|
});
|
|
13266
13720
|
}
|
|
@@ -13279,6 +13733,22 @@ var DocumentDataplyAPI = class extends import_dataply4.DataplyAPI {
|
|
|
13279
13733
|
get indexedFields() {
|
|
13280
13734
|
return this.indexManager.indexedFields;
|
|
13281
13735
|
}
|
|
13736
|
+
/**
|
|
13737
|
+
* Method for reliably processing large batches (chunks) of data.
|
|
13738
|
+
* Processes data in fragments to prevent blocking the event loop while handling large volumes of data.
|
|
13739
|
+
* @param items The items to process
|
|
13740
|
+
* @param callback The callback to process each chunk
|
|
13741
|
+
* @param options The options for the chunk splitter
|
|
13742
|
+
* @param options.firstChunkSize The size of the first chunk. Subsequent chunk sizes are determined based on this value and the processing time. If not specified or set to 0, 5% of the total data is used as the initial value.
|
|
13743
|
+
* @param options.alpha A value that determines the weight given to recent processing times. Higher values weigh recent times more heavily. Lower values are recommended for stability, but excessively low values may impact performance. Default is 0.5.
|
|
13744
|
+
* @returns The processed items
|
|
13745
|
+
*/
|
|
13746
|
+
processInChunks(items, callback, options) {
|
|
13747
|
+
this.logger.debug(`Processing ${items.length} items in chunks`);
|
|
13748
|
+
const firstChunkSize = options?.firstChunkSize ?? 0;
|
|
13749
|
+
const alpha = options?.alpha ?? 0.5;
|
|
13750
|
+
return new DeadlineChunker(firstChunkSize, alpha).processInChunks(items, callback);
|
|
13751
|
+
}
|
|
13282
13752
|
/**
|
|
13283
13753
|
* Register an index.
|
|
13284
13754
|
* @param name The name of the index
|
|
@@ -13318,6 +13788,16 @@ var DocumentDataplyAPI = class extends import_dataply4.DataplyAPI {
|
|
|
13318
13788
|
async backfillIndices(tx) {
|
|
13319
13789
|
return this.indexManager.backfillIndices(tx);
|
|
13320
13790
|
}
|
|
13791
|
+
/**
|
|
13792
|
+
* Rebuild specified indices by clearing existing tree data and rebuilding via bulkLoad.
|
|
13793
|
+
* If no index names are provided, all indices (except _id) are rebuilt.
|
|
13794
|
+
* @param indexNames Optional array of index names to rebuild
|
|
13795
|
+
* @param tx Optional transaction
|
|
13796
|
+
* @returns The number of documents processed
|
|
13797
|
+
*/
|
|
13798
|
+
async rebuildIndices(indexNames, tx) {
|
|
13799
|
+
return this.indexManager.rebuildIndices(indexNames, tx);
|
|
13800
|
+
}
|
|
13321
13801
|
/**
|
|
13322
13802
|
* Flush all interval analysis providers, forcing statistics to be recalculated.
|
|
13323
13803
|
* Call this after bulk inserts or periodically to keep statistics fresh.
|
|
@@ -13518,6 +13998,19 @@ var DocumentDataply = class _DocumentDataply {
|
|
|
13518
13998
|
constructor(file, options) {
|
|
13519
13999
|
this.api = new DocumentDataplyAPI(file, options ?? {});
|
|
13520
14000
|
}
|
|
14001
|
+
/**
|
|
14002
|
+
* Method for reliably processing large batches (chunks) of data.
|
|
14003
|
+
* Processes data in fragments to prevent blocking the event loop while handling large volumes of data.
|
|
14004
|
+
* @param items The items to process
|
|
14005
|
+
* @param callback The callback to process each chunk
|
|
14006
|
+
* @param options The options for the chunk splitter
|
|
14007
|
+
* @param options.firstChunkSize The size of the first chunk. Subsequent chunk sizes are determined based on this value and the processing time. If not specified or set to 0, 5% of the total data is used as the initial value.
|
|
14008
|
+
* @param options.alpha A value that determines the weight given to recent processing times. Higher values weigh recent times more heavily. Lower values are recommended for stability, but excessively low values may impact performance. Default is 0.5.
|
|
14009
|
+
* @returns The processed items
|
|
14010
|
+
*/
|
|
14011
|
+
processInChunks(items, callback, options) {
|
|
14012
|
+
return this.api.processInChunks(items, callback, options);
|
|
14013
|
+
}
|
|
13521
14014
|
/**
|
|
13522
14015
|
* Create a named index on the database.
|
|
13523
14016
|
* Can be called before or after init().
|
|
@@ -13542,6 +14035,16 @@ var DocumentDataply = class _DocumentDataply {
|
|
|
13542
14035
|
await this.api.dropIndex(name, tx);
|
|
13543
14036
|
return this;
|
|
13544
14037
|
}
|
|
14038
|
+
/**
|
|
14039
|
+
* Rebuild specified indices by clearing existing data and reconstructing via bulkLoad.
|
|
14040
|
+
* If no index names are provided, rebuilds all indices except '_id'.
|
|
14041
|
+
* @param indexNames Optional array of index names to rebuild
|
|
14042
|
+
* @param tx Optional transaction
|
|
14043
|
+
* @returns The number of documents processed
|
|
14044
|
+
*/
|
|
14045
|
+
async rebuildIndices(indexNames, tx) {
|
|
14046
|
+
return this.api.rebuildIndices(indexNames, tx);
|
|
14047
|
+
}
|
|
13545
14048
|
/**
|
|
13546
14049
|
* Initialize the document database
|
|
13547
14050
|
*/
|