@retaw106/mmc-pi 2.2.0 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +2 -2
- package/dist/config-init.js +2 -2
- package/dist/{index-v6scmxvt.js → index-1p318k54.js} +1 -1
- package/dist/{index-ny303arh.js → index-3qawqvws.js} +266 -32
- package/dist/{index-dx9vhsy8.js → index-4ktngyya.js} +1 -1
- package/dist/{index-54cgx9gm.js → index-83p1tc98.js} +6 -2
- package/dist/{index-j0jjtcwm.js → index-f7z4jc1f.js} +1447 -216
- package/dist/index.js +21969 -17257
- package/dist/{message-time-backfill-a3p9r5rh.js → message-time-backfill-mpks152d.js} +3 -3
- package/dist/{read-session-chunk-ktp56z65.js → read-session-chunk-vhejzv1k.js} +3 -3
- package/dist/{safe-notification-target-hn43s7bc.js → safe-notification-target-5z28m0n9.js} +2 -2
- package/dist/web/main.js +396 -23
- package/package.json +13 -11
|
@@ -1,12 +1,14 @@
|
|
|
1
|
+
import {
|
|
2
|
+
getTagIdentityWriteGeneration,
|
|
3
|
+
Database,
|
|
4
|
+
getSqliteDatabasePath
|
|
5
|
+
} from "./index-3qawqvws.js";
|
|
1
6
|
import {
|
|
2
7
|
getHarness,
|
|
3
8
|
harnessOwnsOpenCodeStore,
|
|
4
9
|
log,
|
|
5
10
|
sessionLog
|
|
6
|
-
} from "./index-
|
|
7
|
-
import {
|
|
8
|
-
Database
|
|
9
|
-
} from "./index-ny303arh.js";
|
|
11
|
+
} from "./index-83p1tc98.js";
|
|
10
12
|
|
|
11
13
|
// ../plugin/src/hooks/magic-context/tokenizer-calibration-seeds.json
|
|
12
14
|
var tokenizer_calibration_seeds_default = [
|
|
@@ -571,7 +573,7 @@ import { existsSync, readFileSync, realpathSync, statSync } from "node:fs";
|
|
|
571
573
|
import { createRequire } from "node:module";
|
|
572
574
|
import { homedir } from "node:os";
|
|
573
575
|
import { dirname, join, resolve } from "node:path";
|
|
574
|
-
import { pathToFileURL } from "node:url";
|
|
576
|
+
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
575
577
|
|
|
576
578
|
// ../plugin/src/shared/commit-detection.ts
|
|
577
579
|
var HASH_HEX = "[0-9a-f]{7,12}";
|
|
@@ -593,6 +595,272 @@ function removeSystemReminders(text) {
|
|
|
593
595
|
return text.replace(/<system-reminder>[\s\S]*?<\/system-reminder>/gi, "").trim();
|
|
594
596
|
}
|
|
595
597
|
|
|
598
|
+
// ../plugin/src/hooks/magic-context/token-count-exact.ts
|
|
599
|
+
var NO_RANK = 4294967295;
|
|
600
|
+
var LONG_PIECE_BYTES = 256;
|
|
601
|
+
var MERGE_CACHE_GENERATION = 50000;
|
|
602
|
+
function hasInternals(tokenizer) {
|
|
603
|
+
const value = tokenizer;
|
|
604
|
+
return !!value && value.patternRegex instanceof RegExp && value.patternRegex.global && typeof value.stringRankEncoder === "object" && value.stringRankEncoder !== null && Array.isArray(value.binaryFirstByteIndex);
|
|
605
|
+
}
|
|
606
|
+
var textEncoder = new TextEncoder;
|
|
607
|
+
var textDecoder = new TextDecoder("utf-8");
|
|
608
|
+
function isValidUtf8(bytes) {
|
|
609
|
+
let index = 0;
|
|
610
|
+
while (index < bytes.length) {
|
|
611
|
+
const first = bytes[index];
|
|
612
|
+
let size = 0;
|
|
613
|
+
let codePoint = 0;
|
|
614
|
+
if (first <= 127) {
|
|
615
|
+
size = 1;
|
|
616
|
+
codePoint = first;
|
|
617
|
+
} else if ((first & 224) === 192) {
|
|
618
|
+
size = 2;
|
|
619
|
+
codePoint = first & 31;
|
|
620
|
+
if (first <= 193)
|
|
621
|
+
return false;
|
|
622
|
+
} else if ((first & 240) === 224) {
|
|
623
|
+
size = 3;
|
|
624
|
+
codePoint = first & 15;
|
|
625
|
+
} else if ((first & 248) === 240) {
|
|
626
|
+
size = 4;
|
|
627
|
+
codePoint = first & 7;
|
|
628
|
+
if (first > 244)
|
|
629
|
+
return false;
|
|
630
|
+
} else
|
|
631
|
+
return false;
|
|
632
|
+
if (index + size > bytes.length)
|
|
633
|
+
return false;
|
|
634
|
+
for (let offset = 1;offset < size; offset += 1) {
|
|
635
|
+
const byte = bytes[index + offset];
|
|
636
|
+
if ((byte & 192) !== 128)
|
|
637
|
+
return false;
|
|
638
|
+
codePoint = codePoint << 6 | byte & 63;
|
|
639
|
+
}
|
|
640
|
+
if (size === 2 && codePoint < 128)
|
|
641
|
+
return false;
|
|
642
|
+
if (size === 3 && codePoint < 2048)
|
|
643
|
+
return false;
|
|
644
|
+
if (size === 4 && codePoint < 65536)
|
|
645
|
+
return false;
|
|
646
|
+
if (codePoint >= 55296 && codePoint <= 57343)
|
|
647
|
+
return false;
|
|
648
|
+
if (codePoint > 1114111)
|
|
649
|
+
return false;
|
|
650
|
+
index += size;
|
|
651
|
+
}
|
|
652
|
+
return true;
|
|
653
|
+
}
|
|
654
|
+
function binarySearchBytes(sorted, key) {
|
|
655
|
+
let low = 0;
|
|
656
|
+
let high = sorted.length - 1;
|
|
657
|
+
while (low <= high) {
|
|
658
|
+
const mid = low + high >>> 1;
|
|
659
|
+
const midKey = sorted[mid][0];
|
|
660
|
+
let compare = 0;
|
|
661
|
+
const shared = Math.min(midKey.length, key.length);
|
|
662
|
+
for (let index = 0;index < shared; index += 1) {
|
|
663
|
+
compare = midKey[index] - key[index];
|
|
664
|
+
if (compare !== 0)
|
|
665
|
+
break;
|
|
666
|
+
}
|
|
667
|
+
if (compare === 0)
|
|
668
|
+
compare = midKey.length - key.length;
|
|
669
|
+
if (compare === 0)
|
|
670
|
+
return sorted[mid][1];
|
|
671
|
+
if (compare < 0)
|
|
672
|
+
low = mid + 1;
|
|
673
|
+
else
|
|
674
|
+
high = mid - 1;
|
|
675
|
+
}
|
|
676
|
+
return -1;
|
|
677
|
+
}
|
|
678
|
+
function rankOf(tokenizer, slice) {
|
|
679
|
+
if (isValidUtf8(slice)) {
|
|
680
|
+
const rank = tokenizer.stringRankEncoder[textDecoder.decode(slice)];
|
|
681
|
+
if (rank !== undefined)
|
|
682
|
+
return rank;
|
|
683
|
+
}
|
|
684
|
+
const bucket = tokenizer.binaryFirstByteIndex[slice[0]];
|
|
685
|
+
if (bucket !== null && bucket !== undefined) {
|
|
686
|
+
const rank = binarySearchBytes(bucket, slice);
|
|
687
|
+
if (rank !== -1)
|
|
688
|
+
return rank;
|
|
689
|
+
}
|
|
690
|
+
return NO_RANK;
|
|
691
|
+
}
|
|
692
|
+
function mergeByScanning(tokenizer, bytes) {
|
|
693
|
+
const starts = [];
|
|
694
|
+
const ranks = [];
|
|
695
|
+
const pairRank = (index) => {
|
|
696
|
+
const end = starts[index + 2];
|
|
697
|
+
return end === undefined ? NO_RANK : rankOf(tokenizer, bytes.subarray(starts[index], end));
|
|
698
|
+
};
|
|
699
|
+
for (let index = 0;index <= bytes.length; index += 1)
|
|
700
|
+
starts.push(index);
|
|
701
|
+
for (let index = 0;index < starts.length; index += 1)
|
|
702
|
+
ranks.push(pairRank(index));
|
|
703
|
+
while (starts.length > 1) {
|
|
704
|
+
let minRank = NO_RANK;
|
|
705
|
+
let minIndex = -1;
|
|
706
|
+
for (let index = 0;index < ranks.length - 1; index += 1) {
|
|
707
|
+
if (ranks[index] < minRank) {
|
|
708
|
+
minRank = ranks[index];
|
|
709
|
+
minIndex = index;
|
|
710
|
+
}
|
|
711
|
+
}
|
|
712
|
+
if (minIndex === -1)
|
|
713
|
+
break;
|
|
714
|
+
starts.splice(minIndex + 1, 1);
|
|
715
|
+
ranks.splice(minIndex, 1);
|
|
716
|
+
ranks[minIndex] = pairRank(minIndex);
|
|
717
|
+
if (minIndex > 0)
|
|
718
|
+
ranks[minIndex - 1] = pairRank(minIndex - 1);
|
|
719
|
+
}
|
|
720
|
+
const tokens = [];
|
|
721
|
+
for (let index = 0;index < starts.length - 1; index += 1) {
|
|
722
|
+
const rank = rankOf(tokenizer, bytes.subarray(starts[index], starts[index + 1]));
|
|
723
|
+
if (rank !== NO_RANK)
|
|
724
|
+
tokens.push(rank);
|
|
725
|
+
}
|
|
726
|
+
return tokens;
|
|
727
|
+
}
|
|
728
|
+
function mergeWithHeap(tokenizer, bytes) {
|
|
729
|
+
const length = bytes.length;
|
|
730
|
+
const next = new Int32Array(length + 1);
|
|
731
|
+
const prev = new Int32Array(length + 1);
|
|
732
|
+
const alive = new Uint8Array(length + 1).fill(1);
|
|
733
|
+
const rank = new Float64Array(length + 1).fill(NO_RANK);
|
|
734
|
+
for (let index = 0;index <= length; index += 1) {
|
|
735
|
+
next[index] = index + 1;
|
|
736
|
+
prev[index] = index - 1;
|
|
737
|
+
}
|
|
738
|
+
const heapRank = [];
|
|
739
|
+
const heapAt = [];
|
|
740
|
+
const less = (a, b) => heapRank[a] < heapRank[b] || heapRank[a] === heapRank[b] && heapAt[a] < heapAt[b];
|
|
741
|
+
const swap = (a, b) => {
|
|
742
|
+
[heapRank[a], heapRank[b]] = [heapRank[b], heapRank[a]];
|
|
743
|
+
[heapAt[a], heapAt[b]] = [heapAt[b], heapAt[a]];
|
|
744
|
+
};
|
|
745
|
+
const push = (value, at) => {
|
|
746
|
+
heapRank.push(value);
|
|
747
|
+
heapAt.push(at);
|
|
748
|
+
let child = heapRank.length - 1;
|
|
749
|
+
while (child > 0) {
|
|
750
|
+
const parent = child - 1 >> 1;
|
|
751
|
+
if (!less(child, parent))
|
|
752
|
+
break;
|
|
753
|
+
swap(child, parent);
|
|
754
|
+
child = parent;
|
|
755
|
+
}
|
|
756
|
+
};
|
|
757
|
+
const pop = () => {
|
|
758
|
+
const last = heapRank.length - 1;
|
|
759
|
+
swap(0, last);
|
|
760
|
+
heapRank.pop();
|
|
761
|
+
heapAt.pop();
|
|
762
|
+
let parent = 0;
|
|
763
|
+
for (;; ) {
|
|
764
|
+
const left = parent * 2 + 1;
|
|
765
|
+
const right = left + 1;
|
|
766
|
+
let smallest = parent;
|
|
767
|
+
if (left < heapRank.length && less(left, smallest))
|
|
768
|
+
smallest = left;
|
|
769
|
+
if (right < heapRank.length && less(right, smallest))
|
|
770
|
+
smallest = right;
|
|
771
|
+
if (smallest === parent)
|
|
772
|
+
break;
|
|
773
|
+
swap(parent, smallest);
|
|
774
|
+
parent = smallest;
|
|
775
|
+
}
|
|
776
|
+
};
|
|
777
|
+
const pairRank = (at) => {
|
|
778
|
+
const middle = next[at];
|
|
779
|
+
if (middle >= length)
|
|
780
|
+
return NO_RANK;
|
|
781
|
+
return rankOf(tokenizer, bytes.subarray(at, next[middle]));
|
|
782
|
+
};
|
|
783
|
+
const update = (at) => {
|
|
784
|
+
rank[at] = pairRank(at);
|
|
785
|
+
if (rank[at] !== NO_RANK)
|
|
786
|
+
push(rank[at], at);
|
|
787
|
+
};
|
|
788
|
+
for (let at = 0;at < length; at += 1)
|
|
789
|
+
update(at);
|
|
790
|
+
while (heapRank.length > 0) {
|
|
791
|
+
const value = heapRank[0];
|
|
792
|
+
const at = heapAt[0];
|
|
793
|
+
pop();
|
|
794
|
+
if (!alive[at] || rank[at] !== value)
|
|
795
|
+
continue;
|
|
796
|
+
const removed = next[at];
|
|
797
|
+
alive[removed] = 0;
|
|
798
|
+
next[at] = next[removed];
|
|
799
|
+
prev[next[removed]] = at;
|
|
800
|
+
update(at);
|
|
801
|
+
if (at > 0)
|
|
802
|
+
update(prev[at]);
|
|
803
|
+
}
|
|
804
|
+
const tokens = [];
|
|
805
|
+
for (let at = 0;at < length; at = next[at]) {
|
|
806
|
+
const value = rankOf(tokenizer, bytes.subarray(at, next[at]));
|
|
807
|
+
if (value !== NO_RANK)
|
|
808
|
+
tokens.push(value);
|
|
809
|
+
}
|
|
810
|
+
return tokens;
|
|
811
|
+
}
|
|
812
|
+
var mergeCaches = new WeakMap;
|
|
813
|
+
var mergedPieces = 0;
|
|
814
|
+
function mergePiece(tokenizer, piece) {
|
|
815
|
+
let cache = mergeCaches.get(tokenizer);
|
|
816
|
+
if (!cache) {
|
|
817
|
+
cache = { current: new Map, previous: new Map };
|
|
818
|
+
mergeCaches.set(tokenizer, cache);
|
|
819
|
+
}
|
|
820
|
+
let tokens = cache.current.get(piece);
|
|
821
|
+
if (tokens)
|
|
822
|
+
return tokens;
|
|
823
|
+
tokens = cache.previous.get(piece);
|
|
824
|
+
if (!tokens) {
|
|
825
|
+
const bytes = textEncoder.encode(piece);
|
|
826
|
+
mergedPieces += 1;
|
|
827
|
+
tokens = bytes.length > LONG_PIECE_BYTES ? mergeWithHeap(tokenizer, bytes) : mergeByScanning(tokenizer, bytes);
|
|
828
|
+
}
|
|
829
|
+
if (cache.current.size >= MERGE_CACHE_GENERATION) {
|
|
830
|
+
cache.previous = cache.current;
|
|
831
|
+
cache.current = new Map;
|
|
832
|
+
}
|
|
833
|
+
cache.current.set(piece, tokens);
|
|
834
|
+
return tokens;
|
|
835
|
+
}
|
|
836
|
+
function encodeTokensExactly(tokenizer, text) {
|
|
837
|
+
if (!hasInternals(tokenizer)) {
|
|
838
|
+
return tokenizer.encode(text, "all");
|
|
839
|
+
}
|
|
840
|
+
if (text.length < 10) {
|
|
841
|
+
const direct = tokenizer.stringRankEncoder[text];
|
|
842
|
+
if (direct !== undefined)
|
|
843
|
+
return [direct];
|
|
844
|
+
}
|
|
845
|
+
const tokens = [];
|
|
846
|
+
const regex = tokenizer.patternRegex;
|
|
847
|
+
regex.lastIndex = 0;
|
|
848
|
+
for (let match = regex.exec(text);match !== null; match = regex.exec(text)) {
|
|
849
|
+
const piece = match[0];
|
|
850
|
+
const direct = tokenizer.stringRankEncoder[piece];
|
|
851
|
+
if (direct !== undefined) {
|
|
852
|
+
tokens.push(direct);
|
|
853
|
+
continue;
|
|
854
|
+
}
|
|
855
|
+
for (const token of mergePiece(tokenizer, piece))
|
|
856
|
+
tokens.push(token);
|
|
857
|
+
}
|
|
858
|
+
return tokens;
|
|
859
|
+
}
|
|
860
|
+
function countTokensExactly(tokenizer, text) {
|
|
861
|
+
return encodeTokensExactly(tokenizer, text).length;
|
|
862
|
+
}
|
|
863
|
+
|
|
596
864
|
// ../plugin/src/hooks/magic-context/read-session-formatting.ts
|
|
597
865
|
var MAX_COMMITS_PER_BLOCK = 5;
|
|
598
866
|
function hasMeaningfulUserText2(parts) {
|
|
@@ -705,22 +973,30 @@ var tokenizerSerializedTableBytes;
|
|
|
705
973
|
function tokenizerPackageRoots() {
|
|
706
974
|
const cwd = process.cwd();
|
|
707
975
|
const openCodeCache = join(process.env.XDG_CACHE_HOME ?? join(homedir(), ".cache"), "opencode");
|
|
708
|
-
const roots = [cwd, openCodeCache];
|
|
709
976
|
const candidates = [];
|
|
710
|
-
|
|
977
|
+
const probeRoot = (root) => {
|
|
711
978
|
for (const packageDir of TOKENIZER_PACKAGE_DIRS) {
|
|
712
979
|
candidates.push(join(root, "node_modules", ...packageDir, "node_modules", "ai-tokenizer"));
|
|
713
980
|
}
|
|
714
981
|
candidates.push(join(root, "node_modules", "ai-tokenizer"));
|
|
982
|
+
};
|
|
983
|
+
for (const root of [cwd, openCodeCache]) {
|
|
984
|
+
probeRoot(root);
|
|
715
985
|
}
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
986
|
+
const pushAncestors = (startDir) => {
|
|
987
|
+
let ancestor = startDir;
|
|
988
|
+
while (true) {
|
|
989
|
+
candidates.push(join(ancestor, "node_modules", "ai-tokenizer"));
|
|
990
|
+
const parent = dirname(ancestor);
|
|
991
|
+
if (parent === ancestor)
|
|
992
|
+
break;
|
|
993
|
+
ancestor = parent;
|
|
994
|
+
}
|
|
995
|
+
};
|
|
996
|
+
pushAncestors(process.argv[1] ? dirname(resolve(process.argv[1])) : cwd);
|
|
997
|
+
const ownDir = dirname(fileURLToPath(new URL(import.meta.url)));
|
|
998
|
+
pushAncestors(ownDir);
|
|
999
|
+
probeRoot(join(homedir(), ".omp", "plugins"));
|
|
724
1000
|
return [...new Set(candidates)];
|
|
725
1001
|
}
|
|
726
1002
|
function packageImportTarget(value) {
|
|
@@ -838,7 +1114,7 @@ function estimateTokens(text) {
|
|
|
838
1114
|
if (!activeTokenizer)
|
|
839
1115
|
return estimateTokensHeuristically(text);
|
|
840
1116
|
try {
|
|
841
|
-
return activeTokenizer
|
|
1117
|
+
return countTokensExactly(activeTokenizer, text);
|
|
842
1118
|
} catch (error) {
|
|
843
1119
|
tokenizer = undefined;
|
|
844
1120
|
tokenizerLoadAttempted = true;
|
|
@@ -1020,15 +1296,34 @@ function isRecord(value) {
|
|
|
1020
1296
|
|
|
1021
1297
|
// ../plugin/src/shared/stable-json.ts
|
|
1022
1298
|
function stableStringify(value, seen = new WeakSet) {
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1299
|
+
const pieces = [];
|
|
1300
|
+
writeStable(value, seen, pieces);
|
|
1301
|
+
return pieces.join("");
|
|
1302
|
+
}
|
|
1303
|
+
function writeStable(value, seen, pieces) {
|
|
1304
|
+
if (value === undefined) {
|
|
1305
|
+
pieces.push("undefined");
|
|
1306
|
+
return;
|
|
1307
|
+
}
|
|
1308
|
+
if (value === null || typeof value !== "object") {
|
|
1309
|
+
pieces.push(JSON.stringify(value) ?? String(value));
|
|
1310
|
+
return;
|
|
1311
|
+
}
|
|
1312
|
+
if (seen.has(value)) {
|
|
1313
|
+
pieces.push('"[Circular]"');
|
|
1314
|
+
return;
|
|
1315
|
+
}
|
|
1029
1316
|
seen.add(value);
|
|
1030
1317
|
if (Array.isArray(value)) {
|
|
1031
|
-
|
|
1318
|
+
pieces.push("[");
|
|
1319
|
+
for (let index = 0;index < value.length; index += 1) {
|
|
1320
|
+
if (index > 0)
|
|
1321
|
+
pieces.push(",");
|
|
1322
|
+
if (index in value)
|
|
1323
|
+
writeStable(value[index], seen, pieces);
|
|
1324
|
+
}
|
|
1325
|
+
pieces.push("]");
|
|
1326
|
+
return;
|
|
1032
1327
|
}
|
|
1033
1328
|
const entries = Object.entries(value).sort(([a], [b]) => {
|
|
1034
1329
|
if (a < b)
|
|
@@ -1037,11 +1332,19 @@ function stableStringify(value, seen = new WeakSet) {
|
|
|
1037
1332
|
return 1;
|
|
1038
1333
|
return 0;
|
|
1039
1334
|
});
|
|
1040
|
-
|
|
1335
|
+
pieces.push("{");
|
|
1336
|
+
entries.forEach(([key, child], index) => {
|
|
1337
|
+
if (index > 0)
|
|
1338
|
+
pieces.push(",");
|
|
1339
|
+
pieces.push(JSON.stringify(key), ":");
|
|
1340
|
+
writeStable(child, seen, pieces);
|
|
1341
|
+
});
|
|
1342
|
+
pieces.push("}");
|
|
1041
1343
|
}
|
|
1042
1344
|
|
|
1043
1345
|
// ../plugin/src/shared/tag-owner-id.ts
|
|
1044
1346
|
var TEXT_TAG_IDENTITY_MARKER = ":mc-text-v1:";
|
|
1347
|
+
var PI_FALLBACK_PREFIXES_JSON = JSON.stringify([..."pi-msg-"].reduce((prefixes, char) => prefixes.flatMap((prefix) => /[a-z]/.test(char) ? [prefix + char, prefix + char.toUpperCase()] : [prefix + char]), [""]));
|
|
1045
1348
|
var POSITIONAL_CONTENT_SUFFIX = /:(?:p|file)\d+$/;
|
|
1046
1349
|
var CONTENT_DERIVED_TEXT_SUFFIX = /:mc-text-v1:[0-9a-f]+:[0-9a-f]+:o\d+$/;
|
|
1047
1350
|
function contentTagOwnerMessageId(contentId) {
|
|
@@ -1991,6 +2294,102 @@ function removePendingOp(db, sessionId, tagId) {
|
|
|
1991
2294
|
getRemovePendingOpStatement(db).run(sessionId, tagId);
|
|
1992
2295
|
}
|
|
1993
2296
|
|
|
2297
|
+
// ../plugin/src/features/magic-context/storage-tag-identity-revision.ts
|
|
2298
|
+
var TAG_IDENTITY_REVISION_PREFIX = "tag_identity_revision:";
|
|
2299
|
+
var TAG_REVISION_PREFIX = "tag_revision:";
|
|
2300
|
+
var TAG_REVISION_INCARNATION_KEY = "tag_revision_incarnation";
|
|
2301
|
+
var tagRevisionIncarnationSql = (value) => `CASE WHEN INSTR(${value}, ':') > 0 THEN SUBSTR(${value}, 1, INSTR(${value}, ':') - 1) ELSE '0' END`;
|
|
2302
|
+
var tagRevisionCountSql = (value) => `CAST(CASE WHEN INSTR(${value}, ':') > 0 THEN SUBSTR(${value}, INSTR(${value}, ':') + 1) ELSE ${value} END AS INTEGER)`;
|
|
2303
|
+
var ADVANCE = (delta = "1") => `CASE
|
|
2304
|
+
WHEN ${tagRevisionCountSql("value")} <= 9223372036854775807 - ${delta}
|
|
2305
|
+
THEN (${tagRevisionIncarnationSql("value")}) || ':' || CAST(${tagRevisionCountSql("value")} + ${delta} AS TEXT)
|
|
2306
|
+
ELSE NULL END`;
|
|
2307
|
+
function tagRevisionKey(sessionId) {
|
|
2308
|
+
return `${TAG_REVISION_PREFIX}${sessionId}`;
|
|
2309
|
+
}
|
|
2310
|
+
function tagIdentityRevisionKey(sessionId) {
|
|
2311
|
+
return `${TAG_IDENTITY_REVISION_PREFIX}${sessionId}`;
|
|
2312
|
+
}
|
|
2313
|
+
var BUMP = (session, prefix = TAG_IDENTITY_REVISION_PREFIX) => `INSERT INTO schema_migrations_meta (key, value)
|
|
2314
|
+
VALUES ('${prefix}' || ${session}, '1')
|
|
2315
|
+
ON CONFLICT(key) DO UPDATE SET value = ${ADVANCE()}`;
|
|
2316
|
+
var TAG_INSERT_CONFLICT_ROWS = `SELECT id, session_id, tag_number FROM tags WHERE id = NEW.id
|
|
2317
|
+
UNION SELECT id, session_id, tag_number FROM tags WHERE session_id = NEW.session_id AND tag_number = NEW.tag_number
|
|
2318
|
+
UNION SELECT id, session_id, tag_number FROM tags
|
|
2319
|
+
WHERE NEW.type = 'tool' AND NEW.tool_owner_message_id IS NOT NULL
|
|
2320
|
+
AND type = 'tool' AND tool_owner_message_id IS NOT NULL
|
|
2321
|
+
AND session_id = NEW.session_id AND message_id = NEW.message_id
|
|
2322
|
+
AND tool_owner_message_id = NEW.tool_owner_message_id`;
|
|
2323
|
+
var BUMP_INSERT_CONFLICTS = (prefix) => `INSERT INTO schema_migrations_meta (key, value)
|
|
2324
|
+
SELECT '${prefix}' || session_id, CAST(COUNT(*) AS TEXT)
|
|
2325
|
+
FROM (${TAG_INSERT_CONFLICT_ROWS}) GROUP BY session_id
|
|
2326
|
+
ON CONFLICT(key) DO UPDATE SET value = ${ADVANCE("CAST(excluded.value AS INTEGER)")}`;
|
|
2327
|
+
var TRIGGER_SQL = `CREATE TEMP TRIGGER IF NOT EXISTS mc_tag_identity_revision_au
|
|
2328
|
+
AFTER UPDATE OF id, session_id, message_id, tag_number, type, tool_owner_message_id, reasoning_token_count
|
|
2329
|
+
ON main.tags
|
|
2330
|
+
BEGIN
|
|
2331
|
+
${BUMP("OLD.session_id")};
|
|
2332
|
+
INSERT INTO schema_migrations_meta (key, value)
|
|
2333
|
+
SELECT '${TAG_IDENTITY_REVISION_PREFIX}' || NEW.session_id, '1'
|
|
2334
|
+
WHERE NEW.session_id IS NOT OLD.session_id
|
|
2335
|
+
ON CONFLICT(key) DO UPDATE SET value = ${ADVANCE()};
|
|
2336
|
+
END`;
|
|
2337
|
+
function installTagIdentityRevisionTrigger(db) {
|
|
2338
|
+
db.exec(`DROP TRIGGER IF EXISTS temp.mc_tag_identity_revision_au;
|
|
2339
|
+
DROP TRIGGER IF EXISTS temp.mc_tag_revision_bi;
|
|
2340
|
+
DROP TRIGGER IF EXISTS temp.mc_tag_revision_ai;
|
|
2341
|
+
DROP TRIGGER IF EXISTS temp.mc_tag_revision_au;
|
|
2342
|
+
DROP TRIGGER IF EXISTS temp.mc_tag_revision_ad;
|
|
2343
|
+
DROP TRIGGER IF EXISTS temp.mc_tag_revision_seed_ai;`);
|
|
2344
|
+
db.exec(`CREATE TEMP TRIGGER mc_tag_revision_seed_ai AFTER INSERT ON main.schema_migrations_meta
|
|
2345
|
+
WHEN INSTR(NEW.value, ':') = 0 AND (
|
|
2346
|
+
SUBSTR(NEW.key, 1, ${TAG_REVISION_PREFIX.length}) = '${TAG_REVISION_PREFIX}'
|
|
2347
|
+
OR SUBSTR(NEW.key, 1, ${TAG_IDENTITY_REVISION_PREFIX.length}) = '${TAG_IDENTITY_REVISION_PREFIX}')
|
|
2348
|
+
BEGIN
|
|
2349
|
+
INSERT INTO schema_migrations_meta (key, value) VALUES ('${TAG_REVISION_INCARNATION_KEY}', '1')
|
|
2350
|
+
ON CONFLICT(key) DO UPDATE SET value = CAST(CASE
|
|
2351
|
+
WHEN CAST(value AS INTEGER) < 9223372036854775807 THEN CAST(value AS INTEGER) + 1
|
|
2352
|
+
ELSE NULL END AS TEXT);
|
|
2353
|
+
UPDATE schema_migrations_meta SET value =
|
|
2354
|
+
(SELECT value FROM schema_migrations_meta WHERE key = '${TAG_REVISION_INCARNATION_KEY}')
|
|
2355
|
+
|| ':' || CAST(NEW.value AS TEXT) WHERE key = NEW.key;
|
|
2356
|
+
INSERT INTO schema_migrations_meta (key, value)
|
|
2357
|
+
SELECT '${TAG_REVISION_INCARNATION_KEY}', '1'
|
|
2358
|
+
WHERE SUBSTR(NEW.key, 1, ${TAG_REVISION_PREFIX.length}) = '${TAG_REVISION_PREFIX}'
|
|
2359
|
+
AND NOT EXISTS (SELECT 1 FROM schema_migrations_meta
|
|
2360
|
+
WHERE key = '${TAG_IDENTITY_REVISION_PREFIX}' || SUBSTR(NEW.key, ${TAG_REVISION_PREFIX.length + 1}))
|
|
2361
|
+
ON CONFLICT(key) DO UPDATE SET value = CAST(CASE
|
|
2362
|
+
WHEN CAST(value AS INTEGER) < 9223372036854775807 THEN CAST(value AS INTEGER) + 1
|
|
2363
|
+
ELSE NULL END AS TEXT);
|
|
2364
|
+
INSERT INTO schema_migrations_meta (key, value)
|
|
2365
|
+
SELECT '${TAG_IDENTITY_REVISION_PREFIX}' || SUBSTR(NEW.key, ${TAG_REVISION_PREFIX.length + 1}),
|
|
2366
|
+
(SELECT value FROM schema_migrations_meta WHERE key = '${TAG_REVISION_INCARNATION_KEY}') || ':0'
|
|
2367
|
+
WHERE SUBSTR(NEW.key, 1, ${TAG_REVISION_PREFIX.length}) = '${TAG_REVISION_PREFIX}'
|
|
2368
|
+
AND NOT EXISTS (SELECT 1 FROM schema_migrations_meta
|
|
2369
|
+
WHERE key = '${TAG_IDENTITY_REVISION_PREFIX}' || SUBSTR(NEW.key, ${TAG_REVISION_PREFIX.length + 1}));
|
|
2370
|
+
END;`);
|
|
2371
|
+
db.exec(TRIGGER_SQL);
|
|
2372
|
+
db.exec(`CREATE TEMP TRIGGER IF NOT EXISTS mc_tag_revision_bi BEFORE INSERT ON main.tags BEGIN
|
|
2373
|
+
${BUMP_INSERT_CONFLICTS(TAG_REVISION_PREFIX)};
|
|
2374
|
+
${BUMP_INSERT_CONFLICTS(TAG_IDENTITY_REVISION_PREFIX)};
|
|
2375
|
+
UPDATE session_meta SET tags_version = tags_version + 1
|
|
2376
|
+
WHERE session_id IN (SELECT session_id FROM (${TAG_INSERT_CONFLICT_ROWS}));
|
|
2377
|
+
END;`);
|
|
2378
|
+
db.exec(`CREATE TEMP TRIGGER IF NOT EXISTS mc_tag_revision_ai AFTER INSERT ON main.tags BEGIN
|
|
2379
|
+
${BUMP("NEW.session_id", TAG_REVISION_PREFIX)};
|
|
2380
|
+
END;
|
|
2381
|
+
CREATE TEMP TRIGGER IF NOT EXISTS mc_tag_revision_au AFTER UPDATE ON main.tags BEGIN
|
|
2382
|
+
${BUMP("OLD.session_id", TAG_REVISION_PREFIX)};
|
|
2383
|
+
INSERT INTO schema_migrations_meta (key, value)
|
|
2384
|
+
SELECT '${TAG_REVISION_PREFIX}' || NEW.session_id, '1'
|
|
2385
|
+
WHERE NEW.session_id IS NOT OLD.session_id
|
|
2386
|
+
ON CONFLICT(key) DO UPDATE SET value = ${ADVANCE()};
|
|
2387
|
+
END;
|
|
2388
|
+
CREATE TEMP TRIGGER IF NOT EXISTS mc_tag_revision_ad AFTER DELETE ON main.tags BEGIN
|
|
2389
|
+
${BUMP("OLD.session_id", TAG_REVISION_PREFIX)};
|
|
2390
|
+
END;`);
|
|
2391
|
+
}
|
|
2392
|
+
|
|
1994
2393
|
// ../plugin/src/features/magic-context/storage-tags.ts
|
|
1995
2394
|
var insertTagStatements = new WeakMap;
|
|
1996
2395
|
var updateTagStatusStatements = new WeakMap;
|
|
@@ -2207,8 +2606,41 @@ function getPersistedToolTagAccounting(db, sessionId, tagNumber) {
|
|
|
2207
2606
|
}
|
|
2208
2607
|
return row;
|
|
2209
2608
|
}
|
|
2210
|
-
function getAllStatusTagTokenTotalsFlat(db, sessionId, floor = 0, calibration = { proseRatio: 1, toolsRatio: 1 }, options) {
|
|
2211
|
-
const
|
|
2609
|
+
function getAllStatusTagTokenTotalsFlat(db, sessionId, floor = 0, calibration = { proseRatio: 1, toolsRatio: 1 }, options, ownerMessageIds) {
|
|
2610
|
+
const columns = "type, message_id, tool_owner_message_id, token_count, input_token_count, reasoning_token_count";
|
|
2611
|
+
const owners = ownerMessageIds ? new Set(ownerMessageIds) : undefined;
|
|
2612
|
+
let scanOrder = "id";
|
|
2613
|
+
if (owners) {
|
|
2614
|
+
const planStatement = db.prepare(`EXPLAIN QUERY PLAN SELECT ${columns} FROM tags WHERE session_id = ?${floor > 0 ? " AND tag_number >= ?" : ""}`);
|
|
2615
|
+
let plan;
|
|
2616
|
+
try {
|
|
2617
|
+
plan = planStatement.all(sessionId, ...floor > 0 ? [floor] : []);
|
|
2618
|
+
} finally {
|
|
2619
|
+
planStatement.finalize?.();
|
|
2620
|
+
}
|
|
2621
|
+
const index = plan.map((row) => /USING (?:COVERING )?INDEX (\S+)/.exec(row.detail)?.[1]).find(Boolean);
|
|
2622
|
+
if (index) {
|
|
2623
|
+
const keyStatement = db.prepare("SELECT name, coll, desc FROM pragma_index_xinfo(?) WHERE key = 1 ORDER BY seqno");
|
|
2624
|
+
let keys;
|
|
2625
|
+
try {
|
|
2626
|
+
keys = keyStatement.all(index);
|
|
2627
|
+
} finally {
|
|
2628
|
+
keyStatement.finalize?.();
|
|
2629
|
+
}
|
|
2630
|
+
scanOrder = [
|
|
2631
|
+
...keys.map((key) => `"${(key.name ?? "id").replaceAll('"', '""')}" COLLATE "${key.coll.replaceAll('"', '""')}"${key.desc ? " DESC" : ""}`),
|
|
2632
|
+
"id"
|
|
2633
|
+
].join(", ");
|
|
2634
|
+
}
|
|
2635
|
+
}
|
|
2636
|
+
const rows = owners ? db.prepare(`SELECT ${columns} FROM tags WHERE id IN (
|
|
2637
|
+
SELECT t.id FROM json_each(?) o CROSS JOIN tags t INDEXED BY idx_tags_session_message_id
|
|
2638
|
+
ON t.session_id = ? AND t.message_id >= o.value AND t.message_id < o.value || ';'
|
|
2639
|
+
UNION
|
|
2640
|
+
SELECT t.id FROM json_each(?) o CROSS JOIN tags t INDEXED BY idx_tags_pi_fallback_tool_owner
|
|
2641
|
+
ON t.session_id = ? AND t.tool_owner_message_id = o.value
|
|
2642
|
+
WHERE t.type = 'tool'
|
|
2643
|
+
) AND tag_number >= ? ORDER BY ${scanOrder}`).all(JSON.stringify([...owners]), sessionId, JSON.stringify([...owners]), sessionId, floor) : floor > 0 ? db.prepare(`SELECT type, message_id, tool_owner_message_id, token_count, input_token_count, reasoning_token_count
|
|
2212
2644
|
FROM tags
|
|
2213
2645
|
WHERE session_id = ? AND tag_number >= ?`).all(sessionId, floor) : db.prepare(`SELECT type, message_id, tool_owner_message_id, token_count, input_token_count, reasoning_token_count
|
|
2214
2646
|
FROM tags
|
|
@@ -2221,6 +2653,8 @@ function getAllStatusTagTokenTotalsFlat(db, sessionId, floor = 0, calibration =
|
|
|
2221
2653
|
if (options?.excludeToolTags === true && row.type === "tool")
|
|
2222
2654
|
continue;
|
|
2223
2655
|
const owner = ownerMessageIdForTagRow(row);
|
|
2656
|
+
if (owners && !owners.has(owner))
|
|
2657
|
+
continue;
|
|
2224
2658
|
if (row.token_count === null) {
|
|
2225
2659
|
nullMessageIds.add(owner);
|
|
2226
2660
|
totals.delete(owner);
|
|
@@ -2235,15 +2669,163 @@ function getAllStatusTagTokenTotalsFlat(db, sessionId, floor = 0, calibration =
|
|
|
2235
2669
|
return { totals, nullMessageIds };
|
|
2236
2670
|
}
|
|
2237
2671
|
function getReasoningTokenEstimatesByMessage(db, sessionId, proseRatio) {
|
|
2238
|
-
const
|
|
2239
|
-
|
|
2240
|
-
|
|
2241
|
-
|
|
2242
|
-
|
|
2243
|
-
|
|
2244
|
-
|
|
2672
|
+
const summary = readTagOwnerSummary(db, sessionId, proseRatio);
|
|
2673
|
+
return summary.reasoningEstimatesByRatio.get(proseRatio) ?? new Map;
|
|
2674
|
+
}
|
|
2675
|
+
var tagOwnerSummaries = new WeakMap;
|
|
2676
|
+
var MAX_CACHED_PROSE_RATIOS = 4;
|
|
2677
|
+
var TAG_OWNER_PAGE_LIMIT = 512;
|
|
2678
|
+
var tagOwnerRowsRead = 0;
|
|
2679
|
+
var tagOwnerShapeChecks = 0;
|
|
2680
|
+
function foldTagOwnerRow(summary, row, newRow = true) {
|
|
2681
|
+
summary.lastRowId = Math.max(summary.lastRowId, row.id);
|
|
2682
|
+
if (newRow)
|
|
2683
|
+
summary.rowCount += 1;
|
|
2684
|
+
if (row.type === "tool" && typeof row.tool_owner_message_id !== "string")
|
|
2685
|
+
summary.unownedToolRows.add(row.id);
|
|
2686
|
+
else
|
|
2687
|
+
summary.unownedToolRows.delete(row.id);
|
|
2688
|
+
const owner = row.type === "tool" ? typeof row.tool_owner_message_id === "string" ? row.tool_owner_message_id : null : typeof row.message_id === "string" ? contentTagOwnerMessageId(row.message_id) : null;
|
|
2689
|
+
if (owner) {
|
|
2690
|
+
summary.maxTagByOwner.set(owner, Math.max(summary.maxTagByOwner.get(owner) ?? 0, row.tag_number));
|
|
2691
|
+
}
|
|
2692
|
+
if (row.type !== "message" || row.reasoning_token_count === null)
|
|
2693
|
+
return;
|
|
2694
|
+
const reasoningOwner = ownerMessageIdForTagRow({
|
|
2695
|
+
type: row.type,
|
|
2696
|
+
message_id: row.message_id,
|
|
2697
|
+
tool_owner_message_id: row.tool_owner_message_id
|
|
2698
|
+
});
|
|
2699
|
+
for (const [ratio, estimates] of summary.reasoningEstimatesByRatio) {
|
|
2700
|
+
estimates.set(reasoningOwner, Math.max(estimates.get(reasoningOwner) ?? 0, row.reasoning_token_count * ratio));
|
|
2701
|
+
}
|
|
2702
|
+
}
|
|
2703
|
+
var TAG_OWNER_COLUMNS = "id, type, message_id, tool_owner_message_id, tag_number, reasoning_token_count";
|
|
2704
|
+
function connectionInTransaction(db) {
|
|
2705
|
+
try {
|
|
2706
|
+
const state = db;
|
|
2707
|
+
return state.inTransaction !== false && state.isTransaction !== false;
|
|
2708
|
+
} catch {
|
|
2709
|
+
return true;
|
|
2245
2710
|
}
|
|
2246
|
-
|
|
2711
|
+
}
|
|
2712
|
+
var TAG_OWNER_APPENDED_WITH_SHAPE_SQL = `SELECT ${TAG_OWNER_COLUMNS}, 0 AS shape, NULL AS tag_count, NULL AS max_id, NULL AS revision
|
|
2713
|
+
FROM tags WHERE id > ? AND +session_id = ?
|
|
2714
|
+
UNION ALL
|
|
2715
|
+
SELECT NULL, NULL, NULL, NULL, NULL, NULL, 1, COUNT(*), MAX(id), (SELECT value FROM schema_migrations_meta WHERE key = ?)
|
|
2716
|
+
FROM tags WHERE session_id = ?`;
|
|
2717
|
+
function readTagIdentityRevision(db, sessionId) {
|
|
2718
|
+
const row = db.prepare("SELECT value FROM schema_migrations_meta WHERE key = ?").get(tagIdentityRevisionKey(sessionId));
|
|
2719
|
+
return typeof row?.value === "string" ? row.value : null;
|
|
2720
|
+
}
|
|
2721
|
+
function captureInitialTagOwnerRevision(db, sessionId, cached, appendedRows) {
|
|
2722
|
+
if (cached.identityRevision !== null || appendedRows === 0)
|
|
2723
|
+
return true;
|
|
2724
|
+
const revision = readTagIdentityRevision(db, sessionId);
|
|
2725
|
+
if (revision !== null && !/^[1-9]\d*:0$/.test(revision))
|
|
2726
|
+
return false;
|
|
2727
|
+
cached.identityRevision = revision;
|
|
2728
|
+
return true;
|
|
2729
|
+
}
|
|
2730
|
+
function appendedSinceForeignCommit(db, sessionId, cached) {
|
|
2731
|
+
tagOwnerShapeChecks += 1;
|
|
2732
|
+
const rows = db.prepare(TAG_OWNER_APPENDED_WITH_SHAPE_SQL).all(cached.globalLastRowId, sessionId, tagIdentityRevisionKey(sessionId), sessionId);
|
|
2733
|
+
const shape = rows.find((row) => row.shape === 1);
|
|
2734
|
+
const appended = rows.filter((row) => row.shape === 0);
|
|
2735
|
+
const highest = appended.reduce((max, row) => Math.max(max, row.id), cached.lastRowId);
|
|
2736
|
+
if (!shape || (shape.revision ?? null) !== cached.identityRevision || shape.tag_count !== cached.rowCount + appended.length || (shape.max_id ?? 0) !== highest)
|
|
2737
|
+
return null;
|
|
2738
|
+
return appended;
|
|
2739
|
+
}
|
|
2740
|
+
function adoptedSinceForeignCommit(db, cached) {
|
|
2741
|
+
if (cached.unownedToolRows.size === 0)
|
|
2742
|
+
return [];
|
|
2743
|
+
return db.prepare(`SELECT ${TAG_OWNER_COLUMNS} FROM tags
|
|
2744
|
+
WHERE id IN (SELECT value FROM json_each(?)) AND tool_owner_message_id IS NOT NULL LIMIT ${TAG_OWNER_PAGE_LIMIT + 1}`).all(JSON.stringify([...cached.unownedToolRows]));
|
|
2745
|
+
}
|
|
2746
|
+
function readTagOwnerSummary(db, sessionId, proseRatio, extraRatios = []) {
|
|
2747
|
+
const inTransaction = connectionInTransaction(db);
|
|
2748
|
+
const versions = inTransaction ? null : db.prepare("SELECT data_version AS dataVersion, (SELECT schema_version FROM pragma_schema_version) AS schemaVersion FROM pragma_data_version").get();
|
|
2749
|
+
const dataVersion = versions?.dataVersion ?? -1;
|
|
2750
|
+
const schemaVersion = versions?.schemaVersion ?? -1;
|
|
2751
|
+
const generation = getTagIdentityWriteGeneration();
|
|
2752
|
+
const sessions = tagOwnerSummaries.get(db) ?? new Map;
|
|
2753
|
+
const cached = inTransaction ? undefined : sessions.get(sessionId);
|
|
2754
|
+
if (cached && (cached.generation === generation || cached.identityRevision === readTagIdentityRevision(db, sessionId)) && cached.schemaVersion === schemaVersion && (proseRatio === undefined || cached.reasoningEstimatesByRatio.has(proseRatio))) {
|
|
2755
|
+
let rows;
|
|
2756
|
+
let globalLastRowId = db.prepare("SELECT COALESCE(MAX(id), 0) AS id FROM tags").get().id;
|
|
2757
|
+
if (cached.dataVersion === dataVersion && cached.generation === generation) {
|
|
2758
|
+
const statement = db.prepare(`SELECT ${TAG_OWNER_COLUMNS} FROM tags WHERE id > ? AND +session_id = ? ORDER BY id LIMIT ${TAG_OWNER_PAGE_LIMIT + 1}`);
|
|
2759
|
+
rows = statement.all(cached.globalLastRowId, sessionId);
|
|
2760
|
+
let page = rows;
|
|
2761
|
+
while (page.length === TAG_OWNER_PAGE_LIMIT + 1) {
|
|
2762
|
+
page = statement.all(page[page.length - 1].id, sessionId);
|
|
2763
|
+
rows.push(...page);
|
|
2764
|
+
}
|
|
2765
|
+
if (!captureInitialTagOwnerRevision(db, sessionId, cached, rows.length)) {
|
|
2766
|
+
rows = null;
|
|
2767
|
+
}
|
|
2768
|
+
} else {
|
|
2769
|
+
try {
|
|
2770
|
+
globalLastRowId = db.prepare("SELECT COALESCE(MAX(id),0) AS id FROM tags").get().id;
|
|
2771
|
+
rows = appendedSinceForeignCommit(db, sessionId, cached);
|
|
2772
|
+
const adopted = rows ? adoptedSinceForeignCommit(db, cached) : [];
|
|
2773
|
+
if (!rows || rows.length > TAG_OWNER_PAGE_LIMIT || adopted.length > TAG_OWNER_PAGE_LIMIT) {
|
|
2774
|
+
rows = null;
|
|
2775
|
+
} else {
|
|
2776
|
+
tagOwnerRowsRead += rows.length + adopted.length;
|
|
2777
|
+
for (const row of rows)
|
|
2778
|
+
foldTagOwnerRow(cached, row);
|
|
2779
|
+
for (const row of adopted)
|
|
2780
|
+
foldTagOwnerRow(cached, row, false);
|
|
2781
|
+
cached.dataVersion = dataVersion;
|
|
2782
|
+
cached.generation = generation;
|
|
2783
|
+
cached.globalLastRowId = globalLastRowId;
|
|
2784
|
+
return cached;
|
|
2785
|
+
}
|
|
2786
|
+
} catch {
|
|
2787
|
+
rows = null;
|
|
2788
|
+
}
|
|
2789
|
+
}
|
|
2790
|
+
if (rows) {
|
|
2791
|
+
const adopted = cached.dataVersion === dataVersion ? [] : adoptedSinceForeignCommit(db, cached);
|
|
2792
|
+
tagOwnerRowsRead += rows.length + adopted.length;
|
|
2793
|
+
for (const row of rows)
|
|
2794
|
+
foldTagOwnerRow(cached, row);
|
|
2795
|
+
for (const row of adopted)
|
|
2796
|
+
foldTagOwnerRow(cached, row, false);
|
|
2797
|
+
cached.dataVersion = dataVersion;
|
|
2798
|
+
cached.generation = generation;
|
|
2799
|
+
cached.globalLastRowId = globalLastRowId;
|
|
2800
|
+
return cached;
|
|
2801
|
+
}
|
|
2802
|
+
}
|
|
2803
|
+
const ratios = [
|
|
2804
|
+
...new Set([...cached?.reasoningEstimatesByRatio.keys() ?? [], ...extraRatios])
|
|
2805
|
+
];
|
|
2806
|
+
if (proseRatio !== undefined && !ratios.includes(proseRatio))
|
|
2807
|
+
ratios.push(proseRatio);
|
|
2808
|
+
const summary = {
|
|
2809
|
+
dataVersion,
|
|
2810
|
+
schemaVersion,
|
|
2811
|
+
generation,
|
|
2812
|
+
lastRowId: 0,
|
|
2813
|
+
globalLastRowId: db.prepare("SELECT COALESCE(MAX(id), 0) AS id FROM tags").get().id,
|
|
2814
|
+
rowCount: 0,
|
|
2815
|
+
unownedToolRows: new Set,
|
|
2816
|
+
identityRevision: readTagIdentityRevision(db, sessionId),
|
|
2817
|
+
maxTagByOwner: new Map,
|
|
2818
|
+
reasoningEstimatesByRatio: new Map(ratios.slice(-MAX_CACHED_PROSE_RATIOS).map((ratio) => [ratio, new Map]))
|
|
2819
|
+
};
|
|
2820
|
+
const rows = db.prepare(`SELECT ${TAG_OWNER_COLUMNS} FROM tags WHERE session_id = ? ORDER BY id`).all(sessionId);
|
|
2821
|
+
tagOwnerRowsRead += rows.length;
|
|
2822
|
+
for (const row of rows)
|
|
2823
|
+
foldTagOwnerRow(summary, row);
|
|
2824
|
+
if (!inTransaction) {
|
|
2825
|
+
sessions.set(sessionId, summary);
|
|
2826
|
+
tagOwnerSummaries.set(db, sessions);
|
|
2827
|
+
}
|
|
2828
|
+
return summary;
|
|
2247
2829
|
}
|
|
2248
2830
|
function updateTagInputTokenCount(db, sessionId, tagNumber, newInputTokenCount) {
|
|
2249
2831
|
getUpdateTagInputTokenCountStatement(db).run(newInputTokenCount, sessionId, tagNumber);
|
|
@@ -2278,7 +2860,7 @@ function getTagNumberByMessageIdStatement(db) {
|
|
|
2278
2860
|
function getAssignableTagNumberByMessageIdStatement(db) {
|
|
2279
2861
|
let stmt = getAssignableTagNumberByMessageIdStatements.get(db);
|
|
2280
2862
|
if (!stmt) {
|
|
2281
|
-
stmt = db.prepare(`SELECT tag_number FROM tags
|
|
2863
|
+
stmt = db.prepare(`SELECT tag_number FROM tags INDEXED BY idx_tags_session_message_id
|
|
2282
2864
|
WHERE session_id = ?
|
|
2283
2865
|
AND message_id = ?
|
|
2284
2866
|
AND NOT (
|
|
@@ -2340,30 +2922,23 @@ function updateTagStatus(db, sessionId, tagId, status) {
|
|
|
2340
2922
|
getUpdateTagStatusStatement(db).run(status, sessionId, tagId);
|
|
2341
2923
|
}
|
|
2342
2924
|
var inertWhitespaceStatements = new WeakMap;
|
|
2343
|
-
|
|
2344
|
-
|
|
2345
|
-
|
|
2346
|
-
|
|
2347
|
-
|
|
2348
|
-
|
|
2349
|
-
|
|
2350
|
-
|
|
2351
|
-
|
|
2352
|
-
|
|
2353
|
-
|
|
2354
|
-
|
|
2355
|
-
|
|
2356
|
-
|
|
2357
|
-
|
|
2358
|
-
|
|
2359
|
-
|
|
2360
|
-
}
|
|
2361
|
-
const rows = db.prepare(`SELECT tag_number AS tagNumber, entry_fingerprint AS entryFingerprint
|
|
2362
|
-
FROM tags
|
|
2363
|
-
WHERE session_id = ?
|
|
2364
|
-
AND type = 'message'
|
|
2365
|
-
AND status = 'compacted'
|
|
2366
|
-
AND entry_fingerprint LIKE ?`).all(sessionId, `${WHITESPACE_ASSISTANT_INERT_FINGERPRINT_PREFIX}%`);
|
|
2925
|
+
var inertWhitespaceByNumberStatements = new WeakMap;
|
|
2926
|
+
function getInertWhitespaceAssistantTagsByNumbers(db, sessionId, tagNumbers) {
|
|
2927
|
+
if (tagNumbers.length === 0)
|
|
2928
|
+
return [];
|
|
2929
|
+
let statement = inertWhitespaceByNumberStatements.get(db);
|
|
2930
|
+
if (!statement) {
|
|
2931
|
+
statement = db.prepare(`SELECT tags.tag_number AS tagNumber, tags.entry_fingerprint AS entryFingerprint
|
|
2932
|
+
FROM json_each(?) AS wanted CROSS JOIN tags
|
|
2933
|
+
WHERE tags.session_id = ?
|
|
2934
|
+
AND tags.tag_number = wanted.value
|
|
2935
|
+
AND tags.type = 'message'
|
|
2936
|
+
AND tags.status = 'compacted'
|
|
2937
|
+
AND tags.entry_fingerprint LIKE ?
|
|
2938
|
+
ORDER BY tags.tag_number, tags.id`);
|
|
2939
|
+
inertWhitespaceByNumberStatements.set(db, statement);
|
|
2940
|
+
}
|
|
2941
|
+
const rows = statement.all(JSON.stringify([...new Set(tagNumbers)]), sessionId, `${WHITESPACE_ASSISTANT_INERT_FINGERPRINT_PREFIX}%`);
|
|
2367
2942
|
return rows.flatMap((row) => {
|
|
2368
2943
|
const contentId = row.entryFingerprint.slice(WHITESPACE_ASSISTANT_INERT_FINGERPRINT_PREFIX.length);
|
|
2369
2944
|
return contentId.length > 0 ? [{ tagNumber: row.tagNumber, contentId }] : [];
|
|
@@ -2562,13 +3137,54 @@ function findPiFallbackToolOwnerTags(db, sessionId) {
|
|
|
2562
3137
|
message_id AS callId,
|
|
2563
3138
|
tool_owner_message_id AS toolOwnerMessageId,
|
|
2564
3139
|
status
|
|
2565
|
-
FROM tags
|
|
3140
|
+
FROM json_each(?) prefix CROSS JOIN tags
|
|
2566
3141
|
WHERE session_id = ?
|
|
2567
|
-
AND type = 'tool'
|
|
2568
|
-
|
|
2569
|
-
|
|
3142
|
+
AND tags.type = 'tool'
|
|
3143
|
+
AND tool_owner_message_id >= prefix.value
|
|
3144
|
+
AND tool_owner_message_id < substr(prefix.value, 1, 6) || '.'
|
|
3145
|
+
AND tool_owner_message_id LIKE 'pi-msg-%'
|
|
3146
|
+
ORDER BY tag_number ASC`).all(PI_FALLBACK_PREFIXES_JSON, sessionId).filter(isPiFallbackToolOwnerTag);
|
|
2570
3147
|
}
|
|
2571
|
-
|
|
3148
|
+
|
|
3149
|
+
class PiTagIdentityConflictError extends Error {
|
|
3150
|
+
code = "PI_TAG_IDENTITY_CONFLICT";
|
|
3151
|
+
constructor(detail, kind = "tool") {
|
|
3152
|
+
super(`Magic Context ${kind}-tag identity conflict: ${detail}. This turn was refused without sending raw history; resending alone will not repair it`);
|
|
3153
|
+
this.name = "PiTagIdentityConflictError";
|
|
3154
|
+
}
|
|
3155
|
+
}
|
|
3156
|
+
function selectPiTagSurvivor(rows, servedTagNumbers, cachedSurvivor, kind, rebuildingSurvivor) {
|
|
3157
|
+
const served = rows.filter((row) => servedTagNumbers.has(row.tagNumber));
|
|
3158
|
+
const cached = cachedSurvivor === undefined ? undefined : rows.find((row) => row.tagNumber === cachedSurvivor);
|
|
3159
|
+
const rebuilding = rows.find((row) => row.tagNumber === rebuildingSurvivor);
|
|
3160
|
+
const noProofInputs = served.length === 0 && cachedSurvivor === undefined;
|
|
3161
|
+
if (cachedSurvivor !== undefined && !cached || served.length !== 1 && !cached && !rebuilding && !noProofInputs) {
|
|
3162
|
+
throw new PiTagIdentityConflictError(served.length > 1 ? `Conflicting served Pi ${kind} tag numbers; no byte-safe cached survivor is proven` : `duplicate ${kind} tag identities have no proven served-byte survivor`, kind);
|
|
3163
|
+
}
|
|
3164
|
+
const kept = cached ?? rebuilding ?? served[0] ?? rows[1] ?? rows[0];
|
|
3165
|
+
if (!kept)
|
|
3166
|
+
throw new PiTagIdentityConflictError("no served survivor is proven", kind);
|
|
3167
|
+
return kept;
|
|
3168
|
+
}
|
|
3169
|
+
function foldPiIdentityDuplicate(db, sessionId, kept, duplicate) {
|
|
3170
|
+
const owesDrop = kept.status !== "dropped" && duplicate.status === "dropped";
|
|
3171
|
+
foldDuplicateIntoSurvivor(db, sessionId, kept, duplicate, true, false);
|
|
3172
|
+
if (owesDrop)
|
|
3173
|
+
queuePendingOp(db, sessionId, kept.tagNumber, "drop");
|
|
3174
|
+
}
|
|
3175
|
+
function getPiMessageTagCandidates(db, sessionId, messageId) {
|
|
3176
|
+
return getPiFallbackMessageFoldTagRowsByMessageId(db, sessionId, messageId);
|
|
3177
|
+
}
|
|
3178
|
+
function findPiTagIdentityConflict(error) {
|
|
3179
|
+
const seen = new Set;
|
|
3180
|
+
while (error instanceof Error && !seen.has(error)) {
|
|
3181
|
+
if (error instanceof PiTagIdentityConflictError)
|
|
3182
|
+
return error;
|
|
3183
|
+
seen.add(error);
|
|
3184
|
+
error = error.cause;
|
|
3185
|
+
}
|
|
3186
|
+
}
|
|
3187
|
+
function adoptPiFallbackToolOwnerTag(db, sessionId, tagNumber, callId, oldOwnerMessageId, newOwnerMessageId, servedTagNumbers = new Set, cachedSurvivor, rebuildingSurvivor) {
|
|
2572
3188
|
const survivor = getPiFallbackFoldTagRowByNumber(db, sessionId, tagNumber);
|
|
2573
3189
|
if (survivor === null || survivor.type !== "tool" || survivor.messageId !== callId || survivor.toolOwnerMessageId !== oldOwnerMessageId) {
|
|
2574
3190
|
return { action: "skipped" };
|
|
@@ -2587,22 +3203,21 @@ function adoptPiFallbackToolOwnerTag(db, sessionId, tagNumber, callId, oldOwnerM
|
|
|
2587
3203
|
if (existing.tagNumber === tagNumber) {
|
|
2588
3204
|
return { action: "skipped" };
|
|
2589
3205
|
}
|
|
2590
|
-
|
|
2591
|
-
|
|
2592
|
-
|
|
2593
|
-
|
|
2594
|
-
foldDuplicateIntoSurvivor(db, sessionId, survivor, existing);
|
|
3206
|
+
const toolRows = [survivor, existing];
|
|
3207
|
+
const kept = selectPiTagSurvivor(toolRows, servedTagNumbers, cachedSurvivor, "tool", rebuildingSurvivor);
|
|
3208
|
+
if (kept.tagNumber === survivor.tagNumber) {
|
|
3209
|
+
foldPiIdentityDuplicate(db, sessionId, survivor, existing);
|
|
2595
3210
|
db.prepare("UPDATE tags SET tool_owner_message_id = ? WHERE session_id = ? AND tag_number = ?").run(newOwnerMessageId, sessionId, tagNumber);
|
|
2596
3211
|
return { action: "folded", tagNumber, deletedTagNumbers: [existing.tagNumber] };
|
|
2597
3212
|
}
|
|
2598
|
-
|
|
3213
|
+
foldPiIdentityDuplicate(db, sessionId, existing, survivor);
|
|
2599
3214
|
return {
|
|
2600
3215
|
action: "folded",
|
|
2601
3216
|
tagNumber: existing.tagNumber,
|
|
2602
3217
|
deletedTagNumbers: [tagNumber]
|
|
2603
3218
|
};
|
|
2604
3219
|
}
|
|
2605
|
-
function adoptPiFallbackMessageTag(db, sessionId, tagNumber, oldFallbackMessageId, newRealMessageId, servedTagNumbers = new Set) {
|
|
3220
|
+
function adoptPiFallbackMessageTag(db, sessionId, tagNumber, oldFallbackMessageId, newRealMessageId, servedTagNumbers = new Set, cachedSurvivor, rebuildingSurvivor) {
|
|
2606
3221
|
if (oldFallbackMessageId.startsWith(WHITESPACE_ASSISTANT_INERT_MESSAGE_PREFIX)) {
|
|
2607
3222
|
return { action: "skipped" };
|
|
2608
3223
|
}
|
|
@@ -2620,18 +3235,13 @@ function adoptPiFallbackMessageTag(db, sessionId, tagNumber, oldFallbackMessageI
|
|
|
2620
3235
|
AND message_id = ?`).run(newRealMessageId, sessionId, tagNumber, oldFallbackMessageId);
|
|
2621
3236
|
return (result.changes ?? 0) > 0 ? { action: "rekeyed", tagNumber } : { action: "skipped" };
|
|
2622
3237
|
}
|
|
2623
|
-
const
|
|
2624
|
-
|
|
2625
|
-
throw new Error("Conflicting served Pi message tag numbers; refusing identity adoption");
|
|
2626
|
-
}
|
|
2627
|
-
const realSurvivor = servedRows[0] ?? duplicates[0];
|
|
2628
|
-
if (!realSurvivor)
|
|
2629
|
-
return { action: "skipped" };
|
|
3238
|
+
const messageRows = [survivor, ...duplicates];
|
|
3239
|
+
const realSurvivor = selectPiTagSurvivor(messageRows, servedTagNumbers, cachedSurvivor, "message", rebuildingSurvivor);
|
|
2630
3240
|
const deletedTagNumbers = [];
|
|
2631
|
-
for (const duplicate of
|
|
3241
|
+
for (const duplicate of messageRows) {
|
|
2632
3242
|
if (duplicate.tagNumber === realSurvivor.tagNumber)
|
|
2633
3243
|
continue;
|
|
2634
|
-
|
|
3244
|
+
foldPiIdentityDuplicate(db, sessionId, realSurvivor, duplicate);
|
|
2635
3245
|
deletedTagNumbers.push(duplicate.tagNumber);
|
|
2636
3246
|
}
|
|
2637
3247
|
if (realSurvivor.tagNumber === tagNumber) {
|
|
@@ -3409,88 +4019,8 @@ function claimOpenCodeDbDiagnosticOnce(surface, resolution) {
|
|
|
3409
4019
|
return true;
|
|
3410
4020
|
}
|
|
3411
4021
|
|
|
3412
|
-
// ../plugin/src/hooks/magic-context/read-session-db.ts
|
|
3413
|
-
function openCodeDbExists() {
|
|
3414
|
-
return harnessOwnsOpenCodeStore() && openCodeDbPathExists(resolveOpenCodeDbPath());
|
|
3415
|
-
}
|
|
3416
|
-
var cachedReadOnlyDb = null;
|
|
3417
|
-
function closeCachedReadOnlyDb() {
|
|
3418
|
-
if (!cachedReadOnlyDb) {
|
|
3419
|
-
return;
|
|
3420
|
-
}
|
|
3421
|
-
try {
|
|
3422
|
-
closeQuietly(cachedReadOnlyDb.db);
|
|
3423
|
-
} catch (error) {
|
|
3424
|
-
log("[magic-context] failed to close cached OpenCode read-only DB:", error);
|
|
3425
|
-
} finally {
|
|
3426
|
-
cachedReadOnlyDb = null;
|
|
3427
|
-
}
|
|
3428
|
-
}
|
|
3429
|
-
function getReadOnlySessionDb() {
|
|
3430
|
-
if (!harnessOwnsOpenCodeStore()) {
|
|
3431
|
-
throw new Error("OpenCode session database is not readable from a Pi-compatible process; its history lives in Pi sessions");
|
|
3432
|
-
}
|
|
3433
|
-
const resolution = resolveOpenCodeDbPath();
|
|
3434
|
-
const dbPath = resolution.path;
|
|
3435
|
-
if (!openCodeDbPathExists(resolution)) {
|
|
3436
|
-
throw new Error(`OpenCode session database is unavailable at ${dbPath} (source=${resolution.source})`);
|
|
3437
|
-
}
|
|
3438
|
-
if (cachedReadOnlyDb?.path === dbPath) {
|
|
3439
|
-
return cachedReadOnlyDb.db;
|
|
3440
|
-
}
|
|
3441
|
-
closeCachedReadOnlyDb();
|
|
3442
|
-
const db = new Database(dbPath, { readonly: true });
|
|
3443
|
-
try {
|
|
3444
|
-
assertOpenCodeStoreGeneration(db, "v1", dbPath);
|
|
3445
|
-
} catch (error) {
|
|
3446
|
-
closeQuietly(db);
|
|
3447
|
-
throw error;
|
|
3448
|
-
}
|
|
3449
|
-
cachedReadOnlyDb = { path: dbPath, db };
|
|
3450
|
-
clearOpenCodeDbReadFailure();
|
|
3451
|
-
return db;
|
|
3452
|
-
}
|
|
3453
|
-
function withReadOnlySessionDb(fn) {
|
|
3454
|
-
return fn(getReadOnlySessionDb());
|
|
3455
|
-
}
|
|
3456
|
-
function getRawSessionMessageCountFromDb(db, sessionId) {
|
|
3457
|
-
const row = db.prepare(`SELECT COUNT(*) as count FROM message WHERE session_id = ?
|
|
3458
|
-
AND NOT (COALESCE(json_extract(data, '$.summary'), 0) = 1
|
|
3459
|
-
AND COALESCE(json_extract(data, '$.finish'), '') = 'stop')`).get(sessionId);
|
|
3460
|
-
return typeof row?.count === "number" ? row.count : 0;
|
|
3461
|
-
}
|
|
3462
|
-
var trackedSessions = new Map;
|
|
3463
|
-
var pendingParts = new Map;
|
|
3464
|
-
var probeLogObserverForTests;
|
|
3465
|
-
function logProbeFailureOnce(resolution, error) {
|
|
3466
|
-
const failure = recordOpenCodeDbReadFailure(resolution, error);
|
|
3467
|
-
if (!claimOpenCodeDbDiagnosticOnce("session-state-probe", resolution))
|
|
3468
|
-
return;
|
|
3469
|
-
const message = `[magic-context] OpenCode DB probe failed: path=${resolution.path} source=${resolution.source} cause=${failure.message}`;
|
|
3470
|
-
probeLogObserverForTests?.(message);
|
|
3471
|
-
log(message);
|
|
3472
|
-
}
|
|
3473
|
-
function getMessageTimesFromOpenCodeDb(sessionId, messageIds) {
|
|
3474
|
-
const result = new Map;
|
|
3475
|
-
if (messageIds.length === 0 || !harnessOwnsOpenCodeStore())
|
|
3476
|
-
return result;
|
|
3477
|
-
try {
|
|
3478
|
-
withReadOnlySessionDb((db) => {
|
|
3479
|
-
const placeholders = messageIds.map(() => "?").join(",");
|
|
3480
|
-
const rows = db.prepare(`SELECT id, time_created FROM message WHERE session_id = ? AND id IN (${placeholders})`).all(sessionId, ...messageIds);
|
|
3481
|
-
for (const row of rows) {
|
|
3482
|
-
if (typeof row.id === "string" && typeof row.time_created === "number") {
|
|
3483
|
-
result.set(row.id, row.time_created);
|
|
3484
|
-
}
|
|
3485
|
-
}
|
|
3486
|
-
});
|
|
3487
|
-
} catch (error) {
|
|
3488
|
-
logProbeFailureOnce(resolveOpenCodeDbPath(), error);
|
|
3489
|
-
}
|
|
3490
|
-
return result;
|
|
3491
|
-
}
|
|
3492
|
-
|
|
3493
4022
|
// ../plugin/src/hooks/magic-context/read-session-raw.ts
|
|
4023
|
+
import { isMainThread } from "node:worker_threads";
|
|
3494
4024
|
var RAW_MESSAGE_PARTS_BY_ID_SQL = "SELECT message_id, data, time_updated FROM part WHERE +session_id = ? AND likelihood(message_id = ?, 0.000001) ORDER BY time_created ASC, id ASC";
|
|
3495
4025
|
function isRawMessageRow(row) {
|
|
3496
4026
|
if (row === null || typeof row !== "object")
|
|
@@ -3521,6 +4051,25 @@ function isRawCompactionSummaryInfo(info) {
|
|
|
3521
4051
|
const candidate = info;
|
|
3522
4052
|
return candidate.summary === true && candidate.finish === "stop";
|
|
3523
4053
|
}
|
|
4054
|
+
var RAW_METADATA_KEEP_CHARS = 16384;
|
|
4055
|
+
var RAW_METADATA_KEPT_KEYS = [
|
|
4056
|
+
"description",
|
|
4057
|
+
"title",
|
|
4058
|
+
"userAnswer",
|
|
4059
|
+
"answers",
|
|
4060
|
+
"answer",
|
|
4061
|
+
"selectedOptions",
|
|
4062
|
+
"customInput",
|
|
4063
|
+
"results"
|
|
4064
|
+
];
|
|
4065
|
+
var RAW_PART_DATA_SQL = `CASE
|
|
4066
|
+
WHEN json_valid(data) = 1
|
|
4067
|
+
AND json_type(data, '$.state.metadata') = 'object'
|
|
4068
|
+
AND COALESCE(json_extract(data, '$.state.status'), '') <> 'error'
|
|
4069
|
+
AND length(json_extract(data, '$.state.metadata')) > ${RAW_METADATA_KEEP_CHARS}
|
|
4070
|
+
THEN json_set(data, '$.state.metadata', json_patch('{}', json_object(${RAW_METADATA_KEPT_KEYS.map((key) => `'${key}', json_extract(data, '$.state.metadata.${key}')`).join(", ")})))
|
|
4071
|
+
ELSE data
|
|
4072
|
+
END`;
|
|
3524
4073
|
function parseJsonUnknown(value) {
|
|
3525
4074
|
try {
|
|
3526
4075
|
return JSON.parse(value);
|
|
@@ -3551,7 +4100,7 @@ function readRawSessionMessagesFromDb(db, sessionId) {
|
|
|
3551
4100
|
if (messageIds.length === 0)
|
|
3552
4101
|
continue;
|
|
3553
4102
|
const placeholders = messageIds.map(() => "?").join(", ");
|
|
3554
|
-
const partRows = db.prepare(`SELECT message_id, data, time_updated
|
|
4103
|
+
const partRows = db.prepare(`SELECT message_id, ${RAW_PART_DATA_SQL} AS data, time_updated
|
|
3555
4104
|
FROM part
|
|
3556
4105
|
WHERE +session_id = ?
|
|
3557
4106
|
AND likelihood(message_id IN (${placeholders}), 0.000001)
|
|
@@ -3583,7 +4132,7 @@ function readRawSessionMessagePageFromDb(db, sessionId, afterOrdinal, limit, fin
|
|
|
3583
4132
|
if (messageRows.length === 0)
|
|
3584
4133
|
return [];
|
|
3585
4134
|
const placeholders = messageRows.map(() => "?").join(", ");
|
|
3586
|
-
const partRows = db.prepare(`SELECT message_id, data, time_updated
|
|
4135
|
+
const partRows = db.prepare(`SELECT message_id, ${RAW_PART_DATA_SQL} AS data, time_updated
|
|
3587
4136
|
FROM part
|
|
3588
4137
|
WHERE +session_id = ?
|
|
3589
4138
|
AND likelihood(message_id IN (${placeholders}), 0.000001)
|
|
@@ -3591,6 +4140,65 @@ function readRawSessionMessagePageFromDb(db, sessionId, afterOrdinal, limit, fin
|
|
|
3591
4140
|
return assembleRawMessagePage(messageRows, partRows);
|
|
3592
4141
|
}
|
|
3593
4142
|
function readRawMessagePageRows(db, sessionId, afterOrdinal, limit, finalWatermark, after) {
|
|
4143
|
+
try {
|
|
4144
|
+
return readRawMessagePageRowsBounded(db, sessionId, afterOrdinal, limit, finalWatermark, after);
|
|
4145
|
+
} catch (error) {
|
|
4146
|
+
if (error !== ORDINALS_COLD)
|
|
4147
|
+
throw error;
|
|
4148
|
+
if (warmRawSessionOrdinalsForDb(db, sessionId)) {
|
|
4149
|
+
try {
|
|
4150
|
+
return readRawMessagePageRowsBounded(db, sessionId, afterOrdinal, limit, finalWatermark, after);
|
|
4151
|
+
} catch (retry) {
|
|
4152
|
+
if (retry !== ORDINALS_COLD)
|
|
4153
|
+
throw retry;
|
|
4154
|
+
}
|
|
4155
|
+
}
|
|
4156
|
+
return readRawMessagePageRowsOffset(db, sessionId, afterOrdinal, limit, finalWatermark, after);
|
|
4157
|
+
}
|
|
4158
|
+
}
|
|
4159
|
+
var ORDINALS_COLD = Symbol("mmcCanonicalOrdinalsCold");
|
|
4160
|
+
function readRawMessagePageRowsBounded(db, sessionId, afterOrdinal, limit, finalWatermark, after) {
|
|
4161
|
+
const remaining = Math.max(0, Math.floor(finalWatermark) - Math.floor(afterOrdinal));
|
|
4162
|
+
const pageSize = Math.min(Math.max(1, Math.floor(limit)), remaining);
|
|
4163
|
+
if (pageSize === 0)
|
|
4164
|
+
return [];
|
|
4165
|
+
return withReadSnapshot(db, () => {
|
|
4166
|
+
const ordinal = Math.max(0, Math.floor(afterOrdinal));
|
|
4167
|
+
const points = currentOrdinalCheckpoints(db, sessionId)?.scan.checkpoints ?? [];
|
|
4168
|
+
let low = 0;
|
|
4169
|
+
let high = points.length;
|
|
4170
|
+
while (low < high) {
|
|
4171
|
+
const middle = Math.floor((low + high) / 2);
|
|
4172
|
+
if (points[middle].ordinal <= ordinal)
|
|
4173
|
+
low = middle + 1;
|
|
4174
|
+
else
|
|
4175
|
+
high = middle;
|
|
4176
|
+
}
|
|
4177
|
+
const floor = points[low - 1];
|
|
4178
|
+
const useFloor = floor && (!after || compareOrdinalKeys(floor, after) > 0);
|
|
4179
|
+
const anchor = useFloor ? floor : after;
|
|
4180
|
+
const skip = anchor === after && after ? 0 : ordinal - (floor?.ordinal ?? 0);
|
|
4181
|
+
if (skip > ORDINAL_LOOKUP_LIMIT)
|
|
4182
|
+
throw ORDINALS_COLD;
|
|
4183
|
+
const rawLimit = skip + pageSize + ORDINAL_LOOKUP_LIMIT;
|
|
4184
|
+
const rows = db.prepare(`SELECT id, data, time_created, time_updated, ${EXCLUDED_ROW_SQL} AS excluded
|
|
4185
|
+
FROM message WHERE id IN (
|
|
4186
|
+
SELECT id FROM message WHERE session_id = ?
|
|
4187
|
+
${anchor ? "AND (time_created, id) > (?, ?)" : ""}
|
|
4188
|
+
ORDER BY time_created, id LIMIT ?)
|
|
4189
|
+
ORDER BY time_created, id`).all(sessionId, ...anchor ? [anchor.timeCreated, anchor.id] : [], rawLimit);
|
|
4190
|
+
rawMessagePageRowsVisited += rows.length;
|
|
4191
|
+
const eligible = rows.filter((row) => row.excluded !== 1);
|
|
4192
|
+
if (eligible.length <= skip && rows.length === rawLimit)
|
|
4193
|
+
throw ORDINALS_COLD;
|
|
4194
|
+
return eligible.slice(skip, skip + pageSize).filter(isRawMessageRow).map((row, index) => ({
|
|
4195
|
+
...row,
|
|
4196
|
+
ordinal: ordinal + index + 1
|
|
4197
|
+
}));
|
|
4198
|
+
});
|
|
4199
|
+
}
|
|
4200
|
+
var rawMessagePageRowsVisited = 0;
|
|
4201
|
+
function readRawMessagePageRowsOffset(db, sessionId, afterOrdinal, limit, finalWatermark, after) {
|
|
3594
4202
|
const remaining = Math.max(0, Math.floor(finalWatermark) - Math.floor(afterOrdinal));
|
|
3595
4203
|
const pageSize = Math.min(Math.max(1, Math.floor(limit)), remaining);
|
|
3596
4204
|
if (pageSize === 0)
|
|
@@ -3682,6 +4290,9 @@ function readRawSessionMessageSummaryPageFromDb(db, sessionId, afterOrdinal, lim
|
|
|
3682
4290
|
return assembleRawMessagePage(messageRows, partRows);
|
|
3683
4291
|
}
|
|
3684
4292
|
function countRawSessionMessageOrdinalsFromDb(db, sessionId) {
|
|
4293
|
+
const indexed = countCanonicalOrdinalsIndexed(db, sessionId, null, null);
|
|
4294
|
+
if (indexed !== null)
|
|
4295
|
+
return indexed;
|
|
3685
4296
|
const row = db.prepare(`SELECT COUNT(*) AS count
|
|
3686
4297
|
FROM message
|
|
3687
4298
|
WHERE session_id = ?
|
|
@@ -3740,7 +4351,7 @@ function readRawSessionTailFromDb(db, sessionId, baseOrdinal, anchorMessageId) {
|
|
|
3740
4351
|
for (let i = 0;i < ids.length; i += CHUNK) {
|
|
3741
4352
|
const slice = ids.slice(i, i + CHUNK);
|
|
3742
4353
|
const placeholders = slice.map(() => "?").join(",");
|
|
3743
|
-
const partRows = db.prepare(`SELECT message_id, data, time_updated FROM part WHERE +session_id = ? AND likelihood(message_id IN (${placeholders}), 0.000001) ORDER BY time_created ASC, id ASC`).all(sessionId, ...slice).filter(isRawPartRow);
|
|
4354
|
+
const partRows = db.prepare(`SELECT message_id, ${RAW_PART_DATA_SQL} AS data, time_updated FROM part WHERE +session_id = ? AND likelihood(message_id IN (${placeholders}), 0.000001) ORDER BY time_created ASC, id ASC`).all(sessionId, ...slice).filter(isRawPartRow);
|
|
3744
4355
|
for (const part of partRows) {
|
|
3745
4356
|
const list = partsByMessageId.get(part.message_id) ?? [];
|
|
3746
4357
|
list.push(attachRawPartVersion(parseJsonUnknown(part.data), part.time_updated));
|
|
@@ -3768,49 +4379,491 @@ function readRawSessionTailFromDb(db, sessionId, baseOrdinal, anchorMessageId) {
|
|
|
3768
4379
|
}
|
|
3769
4380
|
return { messages, absoluteMessageCount: Math.max(0, ord - 1) };
|
|
3770
4381
|
}
|
|
3771
|
-
|
|
3772
|
-
|
|
3773
|
-
|
|
3774
|
-
|
|
3775
|
-
|
|
3776
|
-
|
|
3777
|
-
|
|
3778
|
-
|
|
3779
|
-
|
|
3780
|
-
|
|
3781
|
-
|
|
3782
|
-
|
|
3783
|
-
)
|
|
3784
|
-
AND (
|
|
3785
|
-
|
|
3786
|
-
|
|
3787
|
-
|
|
3788
|
-
|
|
3789
|
-
|
|
3790
|
-
|
|
3791
|
-
|
|
3792
|
-
|
|
3793
|
-
|
|
3794
|
-
|
|
3795
|
-
|
|
3796
|
-
|
|
3797
|
-
|
|
4382
|
+
var ORDINAL_SUMMARY_FLAG_SQL = `(CASE WHEN json_valid(data) = 1
|
|
4383
|
+
THEN COALESCE(json_extract(data, '$.summary'), 0)
|
|
4384
|
+
ELSE 0 END = 1)`;
|
|
4385
|
+
var ORDINAL_STOP_FLAG_SQL = `(CASE WHEN json_valid(data) = 1
|
|
4386
|
+
THEN COALESCE(json_extract(data, '$.finish'), '')
|
|
4387
|
+
ELSE '' END = 'stop')`;
|
|
4388
|
+
function ordinalRangeSql(lowerBound) {
|
|
4389
|
+
return `SELECT COUNT(*) AS visited,
|
|
4390
|
+
COALESCE(SUM(CASE WHEN ${ORDINAL_SUMMARY_FLAG_SQL} AND ${ORDINAL_STOP_FLAG_SQL}
|
|
4391
|
+
THEN 0 ELSE 1 END), 0) AS eligible
|
|
4392
|
+
FROM message
|
|
4393
|
+
WHERE session_id = ?
|
|
4394
|
+
${lowerBound ? "AND (time_created, id) > (?, ?)" : ""}
|
|
4395
|
+
AND (time_created, id) <= (?, ?)`;
|
|
4396
|
+
}
|
|
4397
|
+
var ORDINAL_RANGE_FROM_START_SQL = ordinalRangeSql(false);
|
|
4398
|
+
var ORDINAL_RANGE_BETWEEN_SQL = ordinalRangeSql(true);
|
|
4399
|
+
function readStoreStamp(db) {
|
|
4400
|
+
try {
|
|
4401
|
+
const row = db.prepare(`SELECT data_version AS dataVersion, total_changes() AS totalChanges,
|
|
4402
|
+
(SELECT schema_version FROM pragma_schema_version) AS schemaVersion
|
|
4403
|
+
FROM pragma_data_version`).get();
|
|
4404
|
+
if (typeof row?.dataVersion !== "number" || typeof row.totalChanges !== "number" || typeof row.schemaVersion !== "number")
|
|
4405
|
+
return null;
|
|
4406
|
+
return {
|
|
4407
|
+
dataVersion: row.dataVersion,
|
|
4408
|
+
totalChanges: row.totalChanges,
|
|
4409
|
+
schemaVersion: row.schemaVersion
|
|
4410
|
+
};
|
|
4411
|
+
} catch {
|
|
4412
|
+
return null;
|
|
4413
|
+
}
|
|
3798
4414
|
}
|
|
3799
|
-
function
|
|
3800
|
-
|
|
3801
|
-
|
|
4415
|
+
function sameStoreStamp(left, right) {
|
|
4416
|
+
return left !== null && right !== null && left.dataVersion === right.dataVersion && left.totalChanges === right.totalChanges && left.schemaVersion === right.schemaVersion;
|
|
4417
|
+
}
|
|
4418
|
+
var ordinalWatermarks = new WeakMap;
|
|
4419
|
+
var ordinalWatermarkEpochs = new Map;
|
|
4420
|
+
var ordinalRowsVisited = 0;
|
|
4421
|
+
var STABLE_STORE_ATTEMPTS = 3;
|
|
4422
|
+
function compareOrdinalKeys(left, right) {
|
|
4423
|
+
if (left.timeCreated !== right.timeCreated)
|
|
4424
|
+
return left.timeCreated < right.timeCreated ? -1 : 1;
|
|
4425
|
+
if (left.id === right.id)
|
|
4426
|
+
return 0;
|
|
4427
|
+
return Buffer.compare(Buffer.from(left.id, "utf8"), Buffer.from(right.id, "utf8"));
|
|
4428
|
+
}
|
|
4429
|
+
function countEligibleOrdinals(db, sessionId, after, through) {
|
|
4430
|
+
const indexed = countCanonicalOrdinalsIndexed(db, sessionId, after, through);
|
|
4431
|
+
if (indexed !== null)
|
|
4432
|
+
return indexed;
|
|
4433
|
+
const row = after ? db.prepare(ORDINAL_RANGE_BETWEEN_SQL).get(sessionId, after.timeCreated, after.id, through.timeCreated, through.id) : db.prepare(ORDINAL_RANGE_FROM_START_SQL).get(sessionId, through.timeCreated, through.id);
|
|
4434
|
+
const visited = typeof row?.visited === "number" ? row.visited : 0;
|
|
4435
|
+
ordinalRowsVisited += visited;
|
|
4436
|
+
ordinalJsonRowsRead += visited;
|
|
4437
|
+
return typeof row?.eligible === "number" ? row.eligible : 0;
|
|
4438
|
+
}
|
|
4439
|
+
var EXCLUDED_ROW_SQL = `${ORDINAL_SUMMARY_FLAG_SQL} AND ${ORDINAL_STOP_FLAG_SQL}`;
|
|
4440
|
+
var SUMMARY_ANCHOR_SPAN = 256;
|
|
4441
|
+
var SUMMARY_ANCHOR_LIMIT = 64;
|
|
4442
|
+
var summaryCandidateSets = new WeakMap;
|
|
4443
|
+
var summaryCandidateEpochs = new Map;
|
|
4444
|
+
var writtenSummaryIds = new Map;
|
|
4445
|
+
var indexedCountUnsupported = new WeakSet;
|
|
4446
|
+
var ordinalJsonRowsRead = 0;
|
|
4447
|
+
var summaryFullScans = 0;
|
|
4448
|
+
var ANCHOR_ROWS_SQL = `SELECT rowid AS rid, id, 1 AS kind FROM message
|
|
4449
|
+
WHERE rowid > (SELECT MAX(rowid) FROM message) - ${SUMMARY_ANCHOR_SPAN}`;
|
|
4450
|
+
var SUMMARY_FULL_SCAN_SQL = `${ANCHOR_ROWS_SQL}
|
|
4451
|
+
UNION ALL
|
|
4452
|
+
SELECT rowid, id, 2 FROM message WHERE session_id = ? AND ${ORDINAL_SUMMARY_FLAG_SQL}
|
|
4453
|
+
UNION ALL
|
|
4454
|
+
SELECT COUNT(*), NULL, 4 FROM message WHERE session_id = ?`;
|
|
4455
|
+
var SUMMARY_SCAN_AFTER_SQL = `${ANCHOR_ROWS_SQL}
|
|
4456
|
+
UNION ALL
|
|
4457
|
+
SELECT rowid, id, CASE WHEN ${ORDINAL_SUMMARY_FLAG_SQL} THEN 2 ELSE 3 END
|
|
4458
|
+
FROM message WHERE rowid > ? AND +session_id = ?`;
|
|
4459
|
+
var ANCHOR_CHECK_SQL = "SELECT rowid AS rid, id FROM message WHERE rowid IN (SELECT value FROM json_each(?))";
|
|
4460
|
+
function survivingAnchor(db, set) {
|
|
4461
|
+
if (set.anchors.length === 0)
|
|
3802
4462
|
return null;
|
|
4463
|
+
const current = new Map(db.prepare(ANCHOR_CHECK_SQL).all(JSON.stringify(set.anchors.map((anchor) => anchor.rowid))).map((row) => [row.rid, row.id]));
|
|
4464
|
+
return set.anchors.find((anchor) => current.get(anchor.rowid) === anchor.id);
|
|
4465
|
+
}
|
|
4466
|
+
var ordinalCheckpoints = new WeakMap;
|
|
4467
|
+
var ordinalSummarySeeds = new WeakMap;
|
|
4468
|
+
var ORDINAL_LOOKUP_LIMIT = 512;
|
|
4469
|
+
function currentOrdinalCheckpoints(db, sessionId) {
|
|
4470
|
+
const cached = ordinalCheckpoints.get(db)?.get(sessionId);
|
|
4471
|
+
if (!cached)
|
|
4472
|
+
return;
|
|
4473
|
+
return withReadSnapshot(db, () => {
|
|
4474
|
+
if (!extendOrdinalCheckpoints(db, sessionId, cached.scan, ORDINAL_LOOKUP_LIMIT))
|
|
4475
|
+
return;
|
|
4476
|
+
cached.epoch = getRawSessionSummaryEpoch(sessionId);
|
|
4477
|
+
return cached;
|
|
4478
|
+
});
|
|
4479
|
+
}
|
|
4480
|
+
function extendOrdinalCheckpoints(db, sessionId, scan, limit) {
|
|
4481
|
+
const points = scan.checkpoints ?? [];
|
|
4482
|
+
const exists = (point) => Boolean(db.prepare("SELECT 1 FROM message WHERE id=? AND session_id=? AND time_created=?").get(point.id, sessionId, point.timeCreated));
|
|
4483
|
+
if (points.length && !exists(points[points.length - 1])) {
|
|
4484
|
+
let low = 0;
|
|
4485
|
+
let high = points.length;
|
|
4486
|
+
while (low < high) {
|
|
4487
|
+
const middle = Math.floor((low + high) / 2);
|
|
4488
|
+
if (exists(points[middle]))
|
|
4489
|
+
low = middle + 1;
|
|
4490
|
+
else
|
|
4491
|
+
high = middle;
|
|
4492
|
+
}
|
|
4493
|
+
points.length = low;
|
|
4494
|
+
scan.total = points.at(-1)?.ordinal ?? 0;
|
|
4495
|
+
scan.sessionRows = points.at(-1)?.storedCount ?? 0;
|
|
4496
|
+
}
|
|
4497
|
+
const tail = points.at(-1);
|
|
4498
|
+
const rows = db.prepare(`SELECT id, time_created AS timeCreated, ${EXCLUDED_ROW_SQL} AS excluded FROM message
|
|
4499
|
+
WHERE session_id=?${tail ? " AND (time_created,id) > (?,?)" : ""}
|
|
4500
|
+
ORDER BY time_created,id${limit === undefined ? "" : ` LIMIT ${limit + 1}`}`).all(sessionId, ...tail ? [tail.timeCreated, tail.id] : []);
|
|
4501
|
+
ordinalJsonRowsRead += rows.length;
|
|
4502
|
+
if (limit !== undefined && rows.length > limit)
|
|
4503
|
+
return false;
|
|
4504
|
+
let total = scan.total ?? 0;
|
|
4505
|
+
if (rows.length && tail && (tail.storedCount ?? 0) % 256 !== 0)
|
|
4506
|
+
points.pop();
|
|
4507
|
+
for (let index = 0;index < rows.length; index++) {
|
|
4508
|
+
const row = rows[index];
|
|
4509
|
+
if (row.excluded !== 1)
|
|
4510
|
+
total++;
|
|
4511
|
+
const storedCount = scan.sessionRows + index + 1;
|
|
4512
|
+
if (storedCount % 256 === 0 || index === rows.length - 1)
|
|
4513
|
+
points.push({ id: row.id, timeCreated: row.timeCreated, ordinal: total, storedCount });
|
|
4514
|
+
}
|
|
4515
|
+
scan.checkpoints = points;
|
|
4516
|
+
scan.sessionRows += rows.length;
|
|
4517
|
+
scan.total = total;
|
|
4518
|
+
return true;
|
|
4519
|
+
}
|
|
4520
|
+
function warmRawSessionOrdinalsForDb(db, sessionId) {
|
|
4521
|
+
if (isRawSessionSummaryWarm(db, sessionId))
|
|
4522
|
+
return true;
|
|
4523
|
+
try {
|
|
4524
|
+
const epoch = getRawSessionSummaryEpoch(sessionId);
|
|
4525
|
+
const scan = scanRawSessionSummaryRows(db, sessionId);
|
|
4526
|
+
installRawSessionSummaryScan(db, sessionId, scan, epoch, true);
|
|
4527
|
+
} catch {
|
|
4528
|
+
return false;
|
|
3803
4529
|
}
|
|
3804
|
-
|
|
3805
|
-
|
|
4530
|
+
return isRawSessionSummaryWarm(db, sessionId);
|
|
4531
|
+
}
|
|
4532
|
+
function boundedOrdinalPrefix(db, sessionId, through) {
|
|
4533
|
+
try {
|
|
4534
|
+
return boundedOrdinalPrefixBounded(db, sessionId, through);
|
|
4535
|
+
} catch (error) {
|
|
4536
|
+
if (error !== ORDINALS_COLD)
|
|
4537
|
+
throw error;
|
|
4538
|
+
if (warmRawSessionOrdinalsForDb(db, sessionId)) {
|
|
4539
|
+
try {
|
|
4540
|
+
return boundedOrdinalPrefixBounded(db, sessionId, through);
|
|
4541
|
+
} catch (retry) {
|
|
4542
|
+
if (retry !== ORDINALS_COLD)
|
|
4543
|
+
throw retry;
|
|
4544
|
+
}
|
|
4545
|
+
}
|
|
4546
|
+
const bindings = [sessionId];
|
|
4547
|
+
if (through)
|
|
4548
|
+
bindings.push(through.timeCreated, through.id);
|
|
4549
|
+
const rows = db.prepare(`SELECT ${EXCLUDED_ROW_SQL} AS excluded FROM message WHERE id IN (
|
|
4550
|
+
SELECT id FROM message WHERE session_id = ?${through ? " AND (time_created, id) <= (?, ?)" : ""}
|
|
4551
|
+
ORDER BY time_created, id)`).all(...bindings);
|
|
4552
|
+
ordinalRowsVisited += rows.length;
|
|
4553
|
+
ordinalJsonRowsRead += rows.length;
|
|
4554
|
+
return rows.reduce((sum, row) => sum + (row.excluded === 1 ? 0 : 1), 0);
|
|
4555
|
+
}
|
|
4556
|
+
}
|
|
4557
|
+
function boundedOrdinalPrefixBounded(db, sessionId, through) {
|
|
4558
|
+
const cached = currentOrdinalCheckpoints(db, sessionId);
|
|
4559
|
+
if (cached && through === null)
|
|
4560
|
+
return cached.scan.total ?? 0;
|
|
4561
|
+
let floor;
|
|
4562
|
+
if (cached && through) {
|
|
4563
|
+
const points = cached.scan.checkpoints ?? [];
|
|
4564
|
+
let low = 0;
|
|
4565
|
+
let high = points.length;
|
|
4566
|
+
while (low < high) {
|
|
4567
|
+
const middle = Math.floor((low + high) / 2);
|
|
4568
|
+
if (compareOrdinalKeys(points[middle], through) <= 0)
|
|
4569
|
+
low = middle + 1;
|
|
4570
|
+
else
|
|
4571
|
+
high = middle;
|
|
4572
|
+
}
|
|
4573
|
+
floor = points[low - 1];
|
|
4574
|
+
if (floor && compareOrdinalKeys(floor, through) === 0)
|
|
4575
|
+
return floor.ordinal;
|
|
4576
|
+
}
|
|
4577
|
+
const bindings = [sessionId];
|
|
4578
|
+
if (floor)
|
|
4579
|
+
bindings.push(floor.timeCreated, floor.id);
|
|
4580
|
+
if (through)
|
|
4581
|
+
bindings.push(through.timeCreated, through.id);
|
|
4582
|
+
const rows = db.prepare(`SELECT ${EXCLUDED_ROW_SQL} AS excluded FROM message WHERE id IN (
|
|
4583
|
+
SELECT id FROM message WHERE session_id = ?${floor ? " AND (time_created, id) > (?, ?)" : ""}${through ? " AND (time_created, id) <= (?, ?)" : ""}
|
|
4584
|
+
ORDER BY time_created, id LIMIT ${ORDINAL_LOOKUP_LIMIT + 1})`).all(...bindings);
|
|
4585
|
+
ordinalRowsVisited += rows.length;
|
|
4586
|
+
ordinalJsonRowsRead += rows.length;
|
|
4587
|
+
if (rows.length > ORDINAL_LOOKUP_LIMIT)
|
|
4588
|
+
throw ORDINALS_COLD;
|
|
4589
|
+
return (floor?.ordinal ?? 0) + rows.reduce((sum, row) => sum + (row.excluded === 1 ? 0 : 1), 0);
|
|
4590
|
+
}
|
|
4591
|
+
function scanRawSessionSummaryRows(db, sessionId, previous) {
|
|
4592
|
+
return withReadSnapshot(db, () => {
|
|
4593
|
+
if (previous?.checkpoints) {
|
|
4594
|
+
extendOrdinalCheckpoints(db, sessionId, previous);
|
|
4595
|
+
return previous;
|
|
4596
|
+
}
|
|
4597
|
+
if (previous) {
|
|
4598
|
+
const sessions = summaryCandidateSets.get(db) ?? new Map;
|
|
4599
|
+
sessions.set(sessionId, {
|
|
4600
|
+
epoch: getRawSessionSummaryEpoch(sessionId),
|
|
4601
|
+
anchors: previous.anchors,
|
|
4602
|
+
ids: new Set(previous.ids)
|
|
4603
|
+
});
|
|
4604
|
+
summaryCandidateSets.set(db, sessions);
|
|
4605
|
+
}
|
|
4606
|
+
const incremental = previous && isRawSessionSummaryWarm(db, sessionId) ? summaryCandidatesOf(db, sessionId) : undefined;
|
|
4607
|
+
const rows = incremental ? [] : db.prepare(SUMMARY_FULL_SCAN_SQL).all(sessionId, sessionId);
|
|
4608
|
+
const anchors = [];
|
|
4609
|
+
const ids = [];
|
|
4610
|
+
let sessionRows = 0;
|
|
4611
|
+
for (const row of rows) {
|
|
4612
|
+
if (row.kind === 1 && typeof row.rid === "number" && typeof row.id === "string")
|
|
4613
|
+
anchors.push({ rowid: row.rid, id: row.id });
|
|
4614
|
+
else if (row.kind === 2 && typeof row.id === "string")
|
|
4615
|
+
ids.push(row.id);
|
|
4616
|
+
else if (row.kind === 4 && typeof row.rid === "number")
|
|
4617
|
+
sessionRows = row.rid;
|
|
4618
|
+
}
|
|
4619
|
+
if (incremental) {
|
|
4620
|
+
ids.push(...incremental.ids);
|
|
4621
|
+
anchors.push(...summaryCandidateSets.get(db)?.get(sessionId)?.anchors ?? []);
|
|
4622
|
+
sessionRows = countStoredRawSessionRowsIndexed(db, sessionId);
|
|
4623
|
+
}
|
|
4624
|
+
anchors.sort((left, right) => right.rowid - left.rowid);
|
|
4625
|
+
anchors.length = Math.min(anchors.length, SUMMARY_ANCHOR_LIMIT);
|
|
4626
|
+
const keys = db.prepare(`SELECT id, time_created AS timeCreated, ${EXCLUDED_ROW_SQL} AS excluded FROM message WHERE session_id = ?
|
|
4627
|
+
ORDER BY time_created, id`).all(sessionId);
|
|
4628
|
+
const checkpoints = [];
|
|
4629
|
+
let total = 0;
|
|
4630
|
+
for (let index = 0;index < keys.length; index++) {
|
|
4631
|
+
const key = keys[index];
|
|
4632
|
+
if (key.excluded !== 1)
|
|
4633
|
+
total++;
|
|
4634
|
+
if (index % 256 === 255 || index === keys.length - 1)
|
|
4635
|
+
checkpoints.push({
|
|
4636
|
+
id: key.id,
|
|
4637
|
+
timeCreated: key.timeCreated,
|
|
4638
|
+
ordinal: total,
|
|
4639
|
+
storedCount: index + 1
|
|
4640
|
+
});
|
|
4641
|
+
}
|
|
4642
|
+
return { anchors, ids, sessionRows, checkpoints, total };
|
|
4643
|
+
});
|
|
4644
|
+
}
|
|
4645
|
+
function getRawSessionSummaryEpoch(sessionId) {
|
|
4646
|
+
return summaryCandidateEpochs.get(sessionId) ?? 0;
|
|
4647
|
+
}
|
|
4648
|
+
function installRawSessionSummaryScan(db, sessionId, scan, epoch, workerPrepared = false) {
|
|
4649
|
+
if (epoch !== getRawSessionSummaryEpoch(sessionId))
|
|
4650
|
+
return false;
|
|
4651
|
+
const seeds = ordinalSummarySeeds.get(db) ?? new Map;
|
|
4652
|
+
seeds.set(sessionId, { epoch, scan });
|
|
4653
|
+
ordinalSummarySeeds.set(db, seeds);
|
|
4654
|
+
if (scan.checkpoints && workerPrepared) {
|
|
4655
|
+
const counts = ordinalCheckpoints.get(db) ?? new Map;
|
|
4656
|
+
counts.set(sessionId, { epoch, scan });
|
|
4657
|
+
ordinalCheckpoints.set(db, counts);
|
|
4658
|
+
} else if (getSqliteDatabasePath(db) && isMainThread)
|
|
4659
|
+
return false;
|
|
4660
|
+
if (isRawSessionSummaryWarm(db, sessionId))
|
|
4661
|
+
return true;
|
|
4662
|
+
let sessions = summaryCandidateSets.get(db);
|
|
4663
|
+
if (!sessions) {
|
|
4664
|
+
sessions = new Map;
|
|
4665
|
+
summaryCandidateSets.set(db, sessions);
|
|
4666
|
+
}
|
|
4667
|
+
sessions.set(sessionId, { epoch, anchors: [...scan.anchors], ids: new Set(scan.ids) });
|
|
4668
|
+
return isRawSessionSummaryWarm(db, sessionId);
|
|
4669
|
+
}
|
|
4670
|
+
function isRawSessionSummaryWarm(db, sessionId) {
|
|
4671
|
+
if (isMainThread && getSqliteDatabasePath(db)) {
|
|
4672
|
+
return Boolean(currentOrdinalCheckpoints(db, sessionId));
|
|
4673
|
+
}
|
|
4674
|
+
const set = summaryCandidateSets.get(db)?.get(sessionId);
|
|
4675
|
+
if (!set || set.epoch !== getRawSessionSummaryEpoch(sessionId))
|
|
4676
|
+
return false;
|
|
4677
|
+
try {
|
|
4678
|
+
return survivingAnchor(db, set) !== undefined;
|
|
4679
|
+
} catch {
|
|
4680
|
+
return false;
|
|
4681
|
+
}
|
|
4682
|
+
}
|
|
4683
|
+
function countStoredRawSessionRowsIndexed(db, sessionId) {
|
|
4684
|
+
const row = db.prepare("SELECT COUNT(*) AS count FROM message WHERE session_id = ?").get(sessionId);
|
|
4685
|
+
return typeof row?.count === "number" ? row.count : 0;
|
|
4686
|
+
}
|
|
4687
|
+
function summaryCandidatesOf(db, sessionId) {
|
|
4688
|
+
let sessions = summaryCandidateSets.get(db);
|
|
4689
|
+
if (!sessions) {
|
|
4690
|
+
sessions = new Map;
|
|
4691
|
+
summaryCandidateSets.set(db, sessions);
|
|
4692
|
+
}
|
|
4693
|
+
const epoch = getRawSessionSummaryEpoch(sessionId);
|
|
4694
|
+
const cached = sessions.get(sessionId);
|
|
4695
|
+
const floor = cached && cached.epoch === epoch ? survivingAnchor(db, cached) : undefined;
|
|
4696
|
+
let ids;
|
|
4697
|
+
let anchors;
|
|
4698
|
+
if (floor === undefined) {
|
|
4699
|
+
const scan = scanRawSessionSummaryRows(db, sessionId);
|
|
4700
|
+
summaryFullScans += 1;
|
|
4701
|
+
ordinalJsonRowsRead += scan.sessionRows;
|
|
4702
|
+
ids = new Set(scan.ids);
|
|
4703
|
+
anchors = scan.anchors;
|
|
4704
|
+
} else {
|
|
4705
|
+
const rows = db.prepare(SUMMARY_SCAN_AFTER_SQL).all(floor?.rowid ?? 0, sessionId);
|
|
4706
|
+
ids = cached?.ids ?? new Set;
|
|
4707
|
+
anchors = [];
|
|
4708
|
+
for (const row of rows) {
|
|
4709
|
+
if (row.kind === 1 && typeof row.rid === "number" && typeof row.id === "string")
|
|
4710
|
+
anchors.push({ rowid: row.rid, id: row.id });
|
|
4711
|
+
else if (row.kind === 2 && typeof row.id === "string")
|
|
4712
|
+
ids.add(row.id);
|
|
4713
|
+
if (row.kind === 2 || row.kind === 3)
|
|
4714
|
+
ordinalJsonRowsRead += 1;
|
|
4715
|
+
}
|
|
4716
|
+
anchors.sort((left, right) => right.rowid - left.rowid);
|
|
4717
|
+
anchors.length = Math.min(anchors.length, SUMMARY_ANCHOR_LIMIT);
|
|
4718
|
+
}
|
|
4719
|
+
for (const id of writtenSummaryIds.get(sessionId) ?? [])
|
|
4720
|
+
ids.add(id);
|
|
4721
|
+
sessions.set(sessionId, { epoch, anchors, ids });
|
|
4722
|
+
const top = anchors[0];
|
|
4723
|
+
return { ids: [...ids], floorRowid: top?.rowid ?? 0, floorId: top?.id ?? null };
|
|
4724
|
+
}
|
|
4725
|
+
function canonicalCountSql(lower, upper) {
|
|
4726
|
+
const range = `${lower ? " AND (time_created, id) > (?, ?)" : ""}${upper ? " AND (time_created, id) <= (?, ?)" : ""}`;
|
|
4727
|
+
return `SELECT
|
|
4728
|
+
(SELECT COUNT(*) FROM message WHERE session_id = ?${range}) AS stored,
|
|
4729
|
+
(SELECT COUNT(*) FROM message
|
|
4730
|
+
WHERE id IN (SELECT value FROM json_each(?)) AND +session_id = ?${range}
|
|
4731
|
+
AND ${EXCLUDED_ROW_SQL}) AS knownExcluded,
|
|
4732
|
+
(SELECT COUNT(*) FROM message
|
|
4733
|
+
WHERE rowid > ? AND +session_id = ? AND id NOT IN (SELECT value FROM json_each(?))${range}
|
|
4734
|
+
AND ${EXCLUDED_ROW_SQL}) AS newExcluded,
|
|
4735
|
+
(SELECT id FROM message WHERE rowid = ?) AS floorId`;
|
|
4736
|
+
}
|
|
4737
|
+
var CANONICAL_COUNT_SQL = new Map;
|
|
4738
|
+
function canonicalCountStatementText(lower, upper) {
|
|
4739
|
+
const key = `${lower}:${upper}`;
|
|
4740
|
+
let sql = CANONICAL_COUNT_SQL.get(key);
|
|
4741
|
+
if (!sql) {
|
|
4742
|
+
sql = canonicalCountSql(lower, upper);
|
|
4743
|
+
CANONICAL_COUNT_SQL.set(key, sql);
|
|
4744
|
+
}
|
|
4745
|
+
return sql;
|
|
4746
|
+
}
|
|
4747
|
+
function countCanonicalOrdinalsIndexed(db, sessionId, after, through) {
|
|
4748
|
+
if (isMainThread && getSqliteDatabasePath(db)) {
|
|
4749
|
+
return withReadSnapshot(db, () => boundedOrdinalPrefix(db, sessionId, through) - (after ? boundedOrdinalPrefix(db, sessionId, after) : 0));
|
|
4750
|
+
}
|
|
4751
|
+
if (indexedCountUnsupported.has(db))
|
|
3806
4752
|
return null;
|
|
4753
|
+
const range = [];
|
|
4754
|
+
if (after)
|
|
4755
|
+
range.push(after.timeCreated, after.id);
|
|
4756
|
+
if (through)
|
|
4757
|
+
range.push(through.timeCreated, through.id);
|
|
4758
|
+
try {
|
|
4759
|
+
return withReadSnapshot(db, () => {
|
|
4760
|
+
for (let attempt = 0;attempt < STABLE_STORE_ATTEMPTS; attempt += 1) {
|
|
4761
|
+
const view = summaryCandidatesOf(db, sessionId);
|
|
4762
|
+
const ids = JSON.stringify(view.ids);
|
|
4763
|
+
const row = db.prepare(canonicalCountStatementText(after !== null, through !== null)).get(sessionId, ...range, ids, sessionId, ...range, view.floorRowid, sessionId, ids, ...range, view.floorRowid);
|
|
4764
|
+
if (view.floorId !== null && row?.floorId !== view.floorId)
|
|
4765
|
+
continue;
|
|
4766
|
+
const stored = typeof row?.stored === "number" ? row.stored : 0;
|
|
4767
|
+
const excluded = (typeof row?.knownExcluded === "number" ? row.knownExcluded : 0) + (typeof row?.newExcluded === "number" ? row.newExcluded : 0);
|
|
4768
|
+
ordinalRowsVisited += stored;
|
|
4769
|
+
ordinalJsonRowsRead += view.ids.length;
|
|
4770
|
+
return stored - excluded;
|
|
4771
|
+
}
|
|
4772
|
+
return null;
|
|
4773
|
+
});
|
|
4774
|
+
} catch (error) {
|
|
4775
|
+
if (/no such (column|table|function)|json_each/i.test(String(error))) {
|
|
4776
|
+
indexedCountUnsupported.add(db);
|
|
4777
|
+
return null;
|
|
4778
|
+
}
|
|
4779
|
+
throw error;
|
|
3807
4780
|
}
|
|
3808
|
-
|
|
3809
|
-
|
|
3810
|
-
|
|
3811
|
-
|
|
3812
|
-
|
|
3813
|
-
|
|
4781
|
+
}
|
|
4782
|
+
function connectionHasOpenTransaction(db) {
|
|
4783
|
+
const state = db;
|
|
4784
|
+
return state.inTransaction === true || state.isTransaction === true;
|
|
4785
|
+
}
|
|
4786
|
+
function withReadSnapshot(db, read) {
|
|
4787
|
+
if (connectionHasOpenTransaction(db))
|
|
4788
|
+
return read();
|
|
4789
|
+
db.exec("BEGIN");
|
|
4790
|
+
try {
|
|
4791
|
+
return read();
|
|
4792
|
+
} finally {
|
|
4793
|
+
if (connectionHasOpenTransaction(db))
|
|
4794
|
+
db.exec("ROLLBACK");
|
|
4795
|
+
}
|
|
4796
|
+
}
|
|
4797
|
+
function reusableWatermark(db, sessionId, stamp) {
|
|
4798
|
+
const watermark = ordinalWatermarks.get(db)?.get(sessionId);
|
|
4799
|
+
if (!watermark || watermark.epoch !== (ordinalWatermarkEpochs.get(sessionId) ?? 0) || !sameStoreStamp(watermark.stamp, stamp))
|
|
4800
|
+
return null;
|
|
4801
|
+
return watermark;
|
|
4802
|
+
}
|
|
4803
|
+
function rememberOrdinalWatermark(db, sessionId, key, ordinal, stamp) {
|
|
4804
|
+
let sessions = ordinalWatermarks.get(db);
|
|
4805
|
+
if (!sessions) {
|
|
4806
|
+
sessions = new Map;
|
|
4807
|
+
ordinalWatermarks.set(db, sessions);
|
|
4808
|
+
}
|
|
4809
|
+
const current = sessions.get(sessionId);
|
|
4810
|
+
if (current && sameStoreStamp(current.stamp, stamp) && compareOrdinalKeys(current, key) >= 0)
|
|
4811
|
+
return;
|
|
4812
|
+
sessions.set(sessionId, {
|
|
4813
|
+
timeCreated: key.timeCreated,
|
|
4814
|
+
id: key.id,
|
|
4815
|
+
ordinal,
|
|
4816
|
+
stamp,
|
|
4817
|
+
epoch: ordinalWatermarkEpochs.get(sessionId) ?? 0
|
|
4818
|
+
});
|
|
4819
|
+
}
|
|
4820
|
+
function canonicalOrdinalOf(db, sessionId, readTarget) {
|
|
4821
|
+
for (let attempt = 1;; attempt += 1) {
|
|
4822
|
+
const before = readStoreStamp(db);
|
|
4823
|
+
const target = readTarget();
|
|
4824
|
+
if (!target)
|
|
4825
|
+
return null;
|
|
4826
|
+
const watermark = attempt < STABLE_STORE_ATTEMPTS ? reusableWatermark(db, sessionId, before) : null;
|
|
4827
|
+
if (!watermark) {
|
|
4828
|
+
const ordinal = countEligibleOrdinals(db, sessionId, null, target);
|
|
4829
|
+
if (before && sameStoreStamp(before, readStoreStamp(db)))
|
|
4830
|
+
rememberOrdinalWatermark(db, sessionId, target, ordinal, before);
|
|
4831
|
+
return { target, ordinal };
|
|
4832
|
+
}
|
|
4833
|
+
const order = compareOrdinalKeys(target, watermark);
|
|
4834
|
+
const ordinal = order === 0 ? watermark.ordinal : order < 0 ? watermark.ordinal - countEligibleOrdinals(db, sessionId, target, watermark) : watermark.ordinal + countEligibleOrdinals(db, sessionId, watermark, target);
|
|
4835
|
+
if (sameStoreStamp(before, readStoreStamp(db))) {
|
|
4836
|
+
rememberOrdinalWatermark(db, sessionId, target, ordinal, before);
|
|
4837
|
+
return { target, ordinal };
|
|
4838
|
+
}
|
|
4839
|
+
}
|
|
4840
|
+
}
|
|
4841
|
+
function readRawSessionMessageOrdinalByIdFromDb(db, sessionId, messageId) {
|
|
4842
|
+
const counted = canonicalOrdinalOf(db, sessionId, () => {
|
|
4843
|
+
const row = db.prepare(`SELECT id, time_created, ${ORDINAL_SUMMARY_FLAG_SQL} AND ${ORDINAL_STOP_FLAG_SQL} AS excluded
|
|
4844
|
+
FROM message WHERE session_id = ? AND id = ?`).get(sessionId, messageId);
|
|
4845
|
+
if (typeof row?.id !== "string" || typeof row.time_created !== "number")
|
|
4846
|
+
return null;
|
|
4847
|
+
return { id: row.id, timeCreated: row.time_created, excluded: row.excluded === 1 };
|
|
4848
|
+
});
|
|
4849
|
+
if (!counted || counted.target.excluded)
|
|
4850
|
+
return null;
|
|
4851
|
+
return counted.ordinal > 0 ? counted.ordinal : null;
|
|
4852
|
+
}
|
|
4853
|
+
function readRawSessionMessageByIdFromDb(db, sessionId, messageId) {
|
|
4854
|
+
const counted = canonicalOrdinalOf(db, sessionId, () => {
|
|
4855
|
+
const row = db.prepare("SELECT id, data, time_created, time_updated FROM message WHERE session_id = ? AND id = ?").get(sessionId, messageId);
|
|
4856
|
+
if (!row || !isRawMessageRow(row) || typeof row.time_created !== "number")
|
|
4857
|
+
return null;
|
|
4858
|
+
const parsed = parseJsonRecord(row.data);
|
|
4859
|
+
if (!parsed || isRawCompactionSummaryInfo(parsed))
|
|
4860
|
+
return null;
|
|
4861
|
+
return { ...row, timeCreated: row.time_created, info: parsed };
|
|
4862
|
+
});
|
|
4863
|
+
if (!counted)
|
|
4864
|
+
return null;
|
|
4865
|
+
const { target: row, ordinal } = counted;
|
|
4866
|
+
const info = row.info;
|
|
3814
4867
|
if (ordinal <= 0) {
|
|
3815
4868
|
return null;
|
|
3816
4869
|
}
|
|
@@ -3836,6 +4889,167 @@ function hasRawSessionMessageByIdFromDb(db, sessionId, messageId) {
|
|
|
3836
4889
|
return readRawSessionMessageByIdFromDb(db, sessionId, messageId) !== null;
|
|
3837
4890
|
return true;
|
|
3838
4891
|
}
|
|
4892
|
+
var boundaryEndpointCaches = new WeakMap;
|
|
4893
|
+
|
|
4894
|
+
// ../plugin/src/hooks/magic-context/read-session-db.ts
|
|
4895
|
+
function openCodeDbExists() {
|
|
4896
|
+
return harnessOwnsOpenCodeStore() && openCodeDbPathExists(resolveOpenCodeDbPath());
|
|
4897
|
+
}
|
|
4898
|
+
var cachedReadOnlyDb = null;
|
|
4899
|
+
function closeCachedReadOnlyDb() {
|
|
4900
|
+
if (!cachedReadOnlyDb) {
|
|
4901
|
+
return;
|
|
4902
|
+
}
|
|
4903
|
+
try {
|
|
4904
|
+
closeQuietly(cachedReadOnlyDb.db);
|
|
4905
|
+
} catch (error) {
|
|
4906
|
+
log("[magic-context] failed to close cached OpenCode read-only DB:", error);
|
|
4907
|
+
} finally {
|
|
4908
|
+
cachedReadOnlyDb = null;
|
|
4909
|
+
}
|
|
4910
|
+
}
|
|
4911
|
+
function getReadOnlySessionDb() {
|
|
4912
|
+
if (!harnessOwnsOpenCodeStore()) {
|
|
4913
|
+
throw new Error("OpenCode session database is not readable from a Pi-compatible process; its history lives in Pi sessions");
|
|
4914
|
+
}
|
|
4915
|
+
const resolution = resolveOpenCodeDbPath();
|
|
4916
|
+
const dbPath = resolution.path;
|
|
4917
|
+
if (!openCodeDbPathExists(resolution)) {
|
|
4918
|
+
throw new Error(`OpenCode session database is unavailable at ${dbPath} (source=${resolution.source})`);
|
|
4919
|
+
}
|
|
4920
|
+
if (cachedReadOnlyDb?.path === dbPath) {
|
|
4921
|
+
return cachedReadOnlyDb.db;
|
|
4922
|
+
}
|
|
4923
|
+
closeCachedReadOnlyDb();
|
|
4924
|
+
const db = new Database(dbPath, { readonly: true });
|
|
4925
|
+
try {
|
|
4926
|
+
assertOpenCodeStoreGeneration(db, "v1", dbPath);
|
|
4927
|
+
} catch (error) {
|
|
4928
|
+
closeQuietly(db);
|
|
4929
|
+
throw error;
|
|
4930
|
+
}
|
|
4931
|
+
cachedReadOnlyDb = { path: dbPath, db };
|
|
4932
|
+
clearOpenCodeDbReadFailure();
|
|
4933
|
+
return db;
|
|
4934
|
+
}
|
|
4935
|
+
function withReadOnlySessionDb(fn) {
|
|
4936
|
+
return fn(getReadOnlySessionDb());
|
|
4937
|
+
}
|
|
4938
|
+
function getRawSessionMessageCountFromDb(db, sessionId) {
|
|
4939
|
+
const row = db.prepare(`SELECT COUNT(*) as count FROM message WHERE session_id = ?
|
|
4940
|
+
AND NOT (COALESCE(json_extract(data, '$.summary'), 0) = 1
|
|
4941
|
+
AND COALESCE(json_extract(data, '$.finish'), '') = 'stop')`).get(sessionId);
|
|
4942
|
+
return typeof row?.count === "number" ? row.count : 0;
|
|
4943
|
+
}
|
|
4944
|
+
var trackedSessions = new Map;
|
|
4945
|
+
var pendingParts = new Map;
|
|
4946
|
+
var probeLogObserverForTests;
|
|
4947
|
+
function logProbeFailureOnce(resolution, error) {
|
|
4948
|
+
const failure = recordOpenCodeDbReadFailure(resolution, error);
|
|
4949
|
+
if (!claimOpenCodeDbDiagnosticOnce("session-state-probe", resolution))
|
|
4950
|
+
return;
|
|
4951
|
+
const message = `[magic-context] OpenCode DB probe failed: path=${resolution.path} source=${resolution.source} cause=${failure.message}`;
|
|
4952
|
+
probeLogObserverForTests?.(message);
|
|
4953
|
+
log(message);
|
|
4954
|
+
}
|
|
4955
|
+
function getMessageTimesFromOpenCodeDb(sessionId, messageIds) {
|
|
4956
|
+
const result = new Map;
|
|
4957
|
+
if (messageIds.length === 0 || !harnessOwnsOpenCodeStore())
|
|
4958
|
+
return result;
|
|
4959
|
+
try {
|
|
4960
|
+
withReadOnlySessionDb((db) => {
|
|
4961
|
+
const placeholders = messageIds.map(() => "?").join(",");
|
|
4962
|
+
const rows = db.prepare(`SELECT id, time_created FROM message WHERE +session_id = ? AND id IN (${placeholders})`).all(sessionId, ...messageIds);
|
|
4963
|
+
for (const row of rows) {
|
|
4964
|
+
if (typeof row.id === "string" && typeof row.time_created === "number") {
|
|
4965
|
+
result.set(row.id, row.time_created);
|
|
4966
|
+
}
|
|
4967
|
+
}
|
|
4968
|
+
});
|
|
4969
|
+
} catch (error) {
|
|
4970
|
+
logProbeFailureOnce(resolveOpenCodeDbPath(), error);
|
|
4971
|
+
}
|
|
4972
|
+
return result;
|
|
4973
|
+
}
|
|
4974
|
+
|
|
4975
|
+
// ../plugin/src/hooks/magic-context/provider-visible-parts.ts
|
|
4976
|
+
function isRecord2(value) {
|
|
4977
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
4978
|
+
}
|
|
4979
|
+
var KEPT_TOOL_STATE_FIELDS = new Set([
|
|
4980
|
+
"status",
|
|
4981
|
+
"input",
|
|
4982
|
+
"output",
|
|
4983
|
+
"error",
|
|
4984
|
+
"attachments",
|
|
4985
|
+
"content"
|
|
4986
|
+
]);
|
|
4987
|
+
var KEPT_TOOL_STATE_TIME_FIELD = "compacted";
|
|
4988
|
+
var KEPT_ERROR_METADATA_FIELDS = ["interrupted", "output"];
|
|
4989
|
+
function providerVisibleToolState(state) {
|
|
4990
|
+
const view = {};
|
|
4991
|
+
for (const key of Object.keys(state)) {
|
|
4992
|
+
const value = state[key];
|
|
4993
|
+
if (KEPT_TOOL_STATE_FIELDS.has(key)) {
|
|
4994
|
+
view[key] = value;
|
|
4995
|
+
} else if (key === "time" && isRecord2(value)) {
|
|
4996
|
+
view.time = value[KEPT_TOOL_STATE_TIME_FIELD] === undefined ? {} : { [KEPT_TOOL_STATE_TIME_FIELD]: value[KEPT_TOOL_STATE_TIME_FIELD] };
|
|
4997
|
+
} else if (key === "metadata" && state.status === "error" && isRecord2(value)) {
|
|
4998
|
+
const kept = {};
|
|
4999
|
+
for (const field of KEPT_ERROR_METADATA_FIELDS) {
|
|
5000
|
+
if (Object.hasOwn(value, field))
|
|
5001
|
+
kept[field] = value[field];
|
|
5002
|
+
}
|
|
5003
|
+
view.metadata = kept;
|
|
5004
|
+
}
|
|
5005
|
+
}
|
|
5006
|
+
return view;
|
|
5007
|
+
}
|
|
5008
|
+
function toolStateIsReduced(state) {
|
|
5009
|
+
for (const key of Object.keys(state)) {
|
|
5010
|
+
if (KEPT_TOOL_STATE_FIELDS.has(key))
|
|
5011
|
+
continue;
|
|
5012
|
+
const value = state[key];
|
|
5013
|
+
if (key === "time" && isRecord2(value)) {
|
|
5014
|
+
const keys = Object.keys(value);
|
|
5015
|
+
if (keys.length === 0)
|
|
5016
|
+
continue;
|
|
5017
|
+
if (keys.length === 1 && keys[0] === KEPT_TOOL_STATE_TIME_FIELD)
|
|
5018
|
+
continue;
|
|
5019
|
+
return false;
|
|
5020
|
+
}
|
|
5021
|
+
if (key === "metadata" && state.status === "error" && isRecord2(value)) {
|
|
5022
|
+
const extra = Object.keys(value).some((field) => !KEPT_ERROR_METADATA_FIELDS.includes(field));
|
|
5023
|
+
if (!extra)
|
|
5024
|
+
continue;
|
|
5025
|
+
}
|
|
5026
|
+
return false;
|
|
5027
|
+
}
|
|
5028
|
+
return true;
|
|
5029
|
+
}
|
|
5030
|
+
function providerVisiblePart(part) {
|
|
5031
|
+
if (!isRecord2(part) || part.type !== "tool" || !isRecord2(part.state))
|
|
5032
|
+
return part;
|
|
5033
|
+
if (toolStateIsReduced(part.state))
|
|
5034
|
+
return part;
|
|
5035
|
+
return { ...part, state: providerVisibleToolState(part.state) };
|
|
5036
|
+
}
|
|
5037
|
+
function providerVisibleMessage(message) {
|
|
5038
|
+
if (!isRecord2(message) || !Array.isArray(message.parts))
|
|
5039
|
+
return message;
|
|
5040
|
+
let parts;
|
|
5041
|
+
for (let index = 0;index < message.parts.length; index += 1) {
|
|
5042
|
+
const part = message.parts[index];
|
|
5043
|
+
const view = providerVisiblePart(part);
|
|
5044
|
+
if (view !== part) {
|
|
5045
|
+
parts ??= message.parts.slice(0, index);
|
|
5046
|
+
parts.push(view);
|
|
5047
|
+
} else if (parts) {
|
|
5048
|
+
parts.push(part);
|
|
5049
|
+
}
|
|
5050
|
+
}
|
|
5051
|
+
return parts ? { ...message, parts } : message;
|
|
5052
|
+
}
|
|
3839
5053
|
|
|
3840
5054
|
// ../plugin/src/hooks/magic-context/read-session-true-raw-tokens.ts
|
|
3841
5055
|
function completedToolArcCrossesBoundary(invOrdinal, resOrdinal, boundary) {
|
|
@@ -3974,7 +5188,7 @@ function callIdFromPart(part) {
|
|
|
3974
5188
|
const state = isRecord(part.state) ? part.state : null;
|
|
3975
5189
|
return state ? firstStringField(state, ["callID", "callId", "toolCallId", "tool_call_id", "id"]) ?? "" : "";
|
|
3976
5190
|
}
|
|
3977
|
-
function toolSignalFromPart(part) {
|
|
5191
|
+
function toolSignalFromPart(part, withText = true) {
|
|
3978
5192
|
if (!isRecord(part))
|
|
3979
5193
|
return null;
|
|
3980
5194
|
const type = partType(part);
|
|
@@ -3993,8 +5207,8 @@ function toolSignalFromPart(part) {
|
|
|
3993
5207
|
callId,
|
|
3994
5208
|
hasInput: hasInput || openInvocation,
|
|
3995
5209
|
hasOutput,
|
|
3996
|
-
inputText: hasInput && state ? stringValue(state.input) : "",
|
|
3997
|
-
outputText: hasOutput ? stringValue(outputValue) : ""
|
|
5210
|
+
inputText: withText && hasInput && state ? stringValue(state.input) : "",
|
|
5211
|
+
outputText: withText && hasOutput ? stringValue(outputValue) : ""
|
|
3998
5212
|
};
|
|
3999
5213
|
}
|
|
4000
5214
|
if (type === "tool-invocation") {
|
|
@@ -4003,7 +5217,7 @@ function toolSignalFromPart(part) {
|
|
|
4003
5217
|
callId,
|
|
4004
5218
|
hasInput: args !== undefined,
|
|
4005
5219
|
hasOutput: false,
|
|
4006
|
-
inputText: args !== undefined ? stringValue(args) : "",
|
|
5220
|
+
inputText: withText && args !== undefined ? stringValue(args) : "",
|
|
4007
5221
|
outputText: ""
|
|
4008
5222
|
};
|
|
4009
5223
|
}
|
|
@@ -4013,7 +5227,7 @@ function toolSignalFromPart(part) {
|
|
|
4013
5227
|
callId,
|
|
4014
5228
|
hasInput: input !== undefined,
|
|
4015
5229
|
hasOutput: false,
|
|
4016
|
-
inputText: input !== undefined ? stringValue(input) : "",
|
|
5230
|
+
inputText: withText && input !== undefined ? stringValue(input) : "",
|
|
4017
5231
|
outputText: ""
|
|
4018
5232
|
};
|
|
4019
5233
|
}
|
|
@@ -4024,7 +5238,7 @@ function toolSignalFromPart(part) {
|
|
|
4024
5238
|
hasInput: false,
|
|
4025
5239
|
hasOutput: content !== undefined,
|
|
4026
5240
|
inputText: "",
|
|
4027
|
-
outputText: content !== undefined ? textFromToolResultContent(content) : ""
|
|
5241
|
+
outputText: withText && content !== undefined ? textFromToolResultContent(content) : ""
|
|
4028
5242
|
};
|
|
4029
5243
|
}
|
|
4030
5244
|
return null;
|
|
@@ -4034,7 +5248,7 @@ function partCheapFingerprint(part) {
|
|
|
4034
5248
|
return `${typeof part}:${recursiveByteLength(part)}`;
|
|
4035
5249
|
const version = rawPartVersion(part);
|
|
4036
5250
|
const type = typeof part.type === "string" ? part.type : "";
|
|
4037
|
-
return `${type}:${String(version)}:${recursiveByteLength(part)}`;
|
|
5251
|
+
return `${type}:${String(version)}:${recursiveByteLength(providerVisiblePart(part))}`;
|
|
4038
5252
|
}
|
|
4039
5253
|
function messageCacheKey(message, options) {
|
|
4040
5254
|
const namespace = "cacheNamespace" in options ? options.cacheNamespace : "estimate";
|
|
@@ -4154,7 +5368,7 @@ function buildToolArcs(messages) {
|
|
|
4154
5368
|
const arcs = [];
|
|
4155
5369
|
for (const message of messages) {
|
|
4156
5370
|
for (const part of message.parts) {
|
|
4157
|
-
const signal = toolSignalFromPart(part);
|
|
5371
|
+
const signal = toolSignalFromPart(part, false);
|
|
4158
5372
|
if (!signal || signal.callId.length === 0)
|
|
4159
5373
|
continue;
|
|
4160
5374
|
if (signal.hasInput && signal.hasOutput) {
|
|
@@ -4514,11 +5728,11 @@ function containsDroppedInputPlaceholder(value) {
|
|
|
4514
5728
|
};
|
|
4515
5729
|
return visit(value);
|
|
4516
5730
|
}
|
|
4517
|
-
function
|
|
5731
|
+
function isRecord3(value) {
|
|
4518
5732
|
return value !== null && typeof value === "object";
|
|
4519
5733
|
}
|
|
4520
5734
|
function isOptionalZodField(field) {
|
|
4521
|
-
if (!
|
|
5735
|
+
if (!isRecord3(field))
|
|
4522
5736
|
return false;
|
|
4523
5737
|
const isOptional = field.isOptional;
|
|
4524
5738
|
if (typeof isOptional === "function") {
|
|
@@ -4530,10 +5744,10 @@ function isOptionalZodField(field) {
|
|
|
4530
5744
|
return internals?.optin === "optional";
|
|
4531
5745
|
}
|
|
4532
5746
|
function toolParameterNames(schema) {
|
|
4533
|
-
if (!
|
|
5747
|
+
if (!isRecord3(schema))
|
|
4534
5748
|
return;
|
|
4535
5749
|
const properties = schema.properties;
|
|
4536
|
-
if (
|
|
5750
|
+
if (isRecord3(properties) && !Array.isArray(properties)) {
|
|
4537
5751
|
const names = Object.keys(properties);
|
|
4538
5752
|
const requiredList = Array.isArray(schema.required) ? schema.required : [];
|
|
4539
5753
|
const required = new Set(requiredList.filter((name) => typeof name === "string"));
|
|
@@ -4548,7 +5762,7 @@ function toolParameterNames(schema) {
|
|
|
4548
5762
|
} catch {
|
|
4549
5763
|
shape = undefined;
|
|
4550
5764
|
}
|
|
4551
|
-
if (
|
|
5765
|
+
if (isRecord3(shape) && !Array.isArray(shape)) {
|
|
4552
5766
|
const result = { required: [], optional: [] };
|
|
4553
5767
|
for (const [name, field] of Object.entries(shape)) {
|
|
4554
5768
|
(isOptionalZodField(field) ? result.optional : result.required).push(name);
|
|
@@ -5308,6 +6522,14 @@ function readRawSessionMessageRange2(sessionId, fromOrdinal, toOrdinal) {
|
|
|
5308
6522
|
}
|
|
5309
6523
|
readRawSessionMessages2.readPage = readRawSessionMessagePage2;
|
|
5310
6524
|
readRawSessionMessages2.getCount = getRawSessionMessageOrdinalCount2;
|
|
6525
|
+
readRawSessionMessages2.prepare = prepareRawSessionOrdinals;
|
|
6526
|
+
async function prepareRawSessionOrdinals(sessionId) {
|
|
6527
|
+
if (sessionProviders.get(sessionId)?.provider)
|
|
6528
|
+
return true;
|
|
6529
|
+
if (!openCodeDbExists())
|
|
6530
|
+
return true;
|
|
6531
|
+
return withReadOnlySessionDb((db) => warmRawSessionOrdinalsForDb(db, sessionId));
|
|
6532
|
+
}
|
|
5311
6533
|
function primeTailRawMessageCache2(args) {
|
|
5312
6534
|
const { sessionId, lastCompartmentEnd, anchorMessageId } = args;
|
|
5313
6535
|
if (!activeRawMessageCache)
|
|
@@ -5459,6 +6681,7 @@ function readRawSessionMessageById2(sessionId, messageId) {
|
|
|
5459
6681
|
return null;
|
|
5460
6682
|
return withReadOnlySessionDb((db) => readRawSessionMessageByIdFromDb(db, sessionId, messageId));
|
|
5461
6683
|
}
|
|
6684
|
+
readRawSessionMessageById2.prepare = prepareRawSessionOrdinals;
|
|
5462
6685
|
function readRawSessionMessagesFromSource(sessionId) {
|
|
5463
6686
|
const provider = sessionProviders.get(sessionId)?.provider;
|
|
5464
6687
|
if (provider)
|
|
@@ -5839,6 +7062,14 @@ function readSessionChunk2(sessionId, tokenBudget, offset = 1, eligibleEndOrdina
|
|
|
5839
7062
|
pinComponentWhenFormattedBudgetCrosses(msg.ordinal, text);
|
|
5840
7063
|
}
|
|
5841
7064
|
if (flushCurrentBlock() && pendingNoiseMeta.length > 0) {
|
|
7065
|
+
const lastNoise = pendingNoiseMeta[pendingNoiseMeta.length - 1];
|
|
7066
|
+
if (messagesProcessed > 0 && lastNoise) {
|
|
7067
|
+
lineMeta.push(...pendingNoiseMeta);
|
|
7068
|
+
messagesProcessed += pendingNoiseMeta.length;
|
|
7069
|
+
lastOrdinal = lastNoise.ordinal;
|
|
7070
|
+
lastMessageId = lastNoise.messageId;
|
|
7071
|
+
pendingNoiseMeta = [];
|
|
7072
|
+
}
|
|
5842
7073
|
highestScannedOrdinal = Math.max(highestScannedOrdinal, pendingNoiseMeta[pendingNoiseMeta.length - 1]?.ordinal ?? highestScannedOrdinal);
|
|
5843
7074
|
}
|
|
5844
7075
|
const toolOnlyRanges = [];
|
|
@@ -5876,4 +7107,4 @@ function readSessionChunk2(sessionId, tokenBudget, offset = 1, eligibleEndOrdina
|
|
|
5876
7107
|
toolResultBoundaries
|
|
5877
7108
|
};
|
|
5878
7109
|
}
|
|
5879
|
-
export { canonicalModelIdentity, piModelRefToCanonical, resolveModelRefForPi, modelRefLookupOrder, toolTemplateError, DEFAULT_PROTECTED_TOOLS, normalizeProtectedToolName, mergeProtectedTools, detectThinkingBindingMismatch, detectLatestTurnThinkingMismatch, isPrefixBoundThinkingModel, detectOverflow, stableStringify, UNKNOWN_FIT_RATIO, resolveDecisionCalibration, hasMeasuredDecisionCalibration, providerMass, localBudget, calibrationForModelKey, historyLocalBudget, adoptedProtectedToolsPolicy, protectedToolTagNumbers, protectedToolTokenCount, removeSystemReminders, hasMeaningfulUserText2, extractTexts2, preloadTokenizer, hasTokenizerForFit, tokenCountUsesByteBound, estimateTokens, estimateFixedPromptTokens, stripWellFormedLeadingTagPrefix, stripPersistedAssistantText, byteSize, stripTagPrefix, peelLeadingMcTagNotation, prependTag, isRecord, TEXT_TAG_IDENTITY_MARKER, contentTagOwnerMessageId, estimateImageTokensFromDataUrl, TODO_STATUSES, TODO_PRIORITIES, TITLE_DONE_STATUSES, normalizeTodoStateJson, buildSyntheticTodoPart, stripChannel1ReminderSpans, compareMeasuredTailPrefix, formatTailHygienePrefixMismatch, freezeTailHygieneMeasurement, effectiveTailHygiene, CHANNEL1_SENTINEL, CHANNEL1_FLOOR_TOKENS, decideChannel1, evaluateChannel2, reclaimableToolOutputCount, buildChannel2Reminder, buildChannel1Reminder, planEmergencyDrop, measureEmergencyTag, closeQuietly, isKnownAutocommit, installV3PerfSchema, queuePendingOp, queuePendingDropForUnchangedTag, getPendingOps, getPendingOpsCount, clearPendingOps, removePendingOp, updateTagByteSize, getRecentTagOwnerMessageIds, AGE_RECLAIM_MIN_TOKENS, getOldestActiveUnprotectedToolTags, getActiveToolTagsForAgeReclaim, getTriggerTagTokenUpperBound, updateTagInputByteSize, updateTagTokenCount, getPersistedToolTagAccounting, getAllStatusTagTokenTotalsFlat, getReasoningTokenEstimatesByMessage, updateTagInputTokenCount, tagTokenCountIsNull, backfillTagTokenCounts, insertTag, updateTagStatus,
|
|
7110
|
+
export { canonicalModelIdentity, piModelRefToCanonical, resolveModelRefForPi, modelRefLookupOrder, toolTemplateError, DEFAULT_PROTECTED_TOOLS, normalizeProtectedToolName, mergeProtectedTools, detectThinkingBindingMismatch, detectLatestTurnThinkingMismatch, isPrefixBoundThinkingModel, detectOverflow, stableStringify, UNKNOWN_FIT_RATIO, resolveDecisionCalibration, hasMeasuredDecisionCalibration, providerMass, localBudget, calibrationForModelKey, historyLocalBudget, adoptedProtectedToolsPolicy, protectedToolTagNumbers, protectedToolTokenCount, removeSystemReminders, hasMeaningfulUserText2, extractTexts2, preloadTokenizer, hasTokenizerForFit, tokenCountUsesByteBound, estimateTokens, estimateFixedPromptTokens, stripWellFormedLeadingTagPrefix, stripPersistedAssistantText, byteSize, stripTagPrefix, peelLeadingMcTagNotation, prependTag, isRecord, TEXT_TAG_IDENTITY_MARKER, PI_FALLBACK_PREFIXES_JSON, contentTagOwnerMessageId, estimateImageTokensFromDataUrl, TODO_STATUSES, TODO_PRIORITIES, TITLE_DONE_STATUSES, normalizeTodoStateJson, buildSyntheticTodoPart, stripChannel1ReminderSpans, compareMeasuredTailPrefix, formatTailHygienePrefixMismatch, freezeTailHygieneMeasurement, effectiveTailHygiene, CHANNEL1_SENTINEL, CHANNEL1_FLOOR_TOKENS, decideChannel1, evaluateChannel2, reclaimableToolOutputCount, buildChannel2Reminder, buildChannel1Reminder, planEmergencyDrop, measureEmergencyTag, closeQuietly, isKnownAutocommit, installV3PerfSchema, queuePendingOp, queuePendingDropForUnchangedTag, getPendingOps, getPendingOpsCount, clearPendingOps, removePendingOp, tagRevisionIncarnationSql, tagRevisionCountSql, tagRevisionKey, tagIdentityRevisionKey, TAG_INSERT_CONFLICT_ROWS, installTagIdentityRevisionTrigger, updateTagByteSize, getRecentTagOwnerMessageIds, AGE_RECLAIM_MIN_TOKENS, getOldestActiveUnprotectedToolTags, getActiveToolTagsForAgeReclaim, getTriggerTagTokenUpperBound, updateTagInputByteSize, updateTagTokenCount, getPersistedToolTagAccounting, getAllStatusTagTokenTotalsFlat, getReasoningTokenEstimatesByMessage, updateTagInputTokenCount, tagTokenCountIsNull, backfillTagTokenCounts, insertTag, updateTagStatus, getInertWhitespaceAssistantTagsByNumbers, updateTagDropMode, updateCavemanDepth, hasPiFallbackMessageTags, findAdoptableFallbackTags, hasPiFallbackToolOwnerTags, findPiFallbackToolOwnerTags, PiTagIdentityConflictError, getPiMessageTagCandidates, findPiTagIdentityConflict, adoptPiFallbackToolOwnerTag, adoptPiFallbackMessageTag, getMaxTagNumberBySession, getTagNumberByMessageId, getAssignableTagNumberByMessageId, deriveTagLoadFloor, TAG_SELECT_COLUMNS, getTagsBySession, getActiveTagsBySession, getTagsForPendingOperations, getTagsByNumbers, getNewestToolTagNumbers, getDroppedTagsByNumbers, getMaxDroppedTagNumber, getTagById, getToolTagNumberByOwner, getNullOwnerToolTag, adoptNullOwnerToolTag, resolveOpenCodeDbPath, hasV1MessageTables, assertOpenCodeStoreGeneration, withReadOnlySessionDb, expandToolPart, retreatPastHostUnservedRows, providerVisibleMessage, completedToolArcCrossesBoundary, estimateTrueRawMessageTokens, buildToolArcs, fenceBoundaryForCompletedToolArcs, fenceBoundaryForToolArcs, buildTrueRawTokenIndex, computeRawRangeFingerprint, invalidateTrueRawTokenCache, stripSystemInjection, hasUserAnswerMetadata, droppedInputMarker, createDroppedInputGuard, isEditTool, applyEditMarkerToInput, setDatabase, loadToolDefinitionMeasurements, modelAcceptsEmptyContent, isAnthropicFamilyRoute, variantChangeBustsProviderCache, SKELETON_REAL_INPUT_MAX_BYTES, toolInputStringBytes, resolveHostServedBoundaryId2, hasRawMessageProvider2, setRawMessageProvider2, withRawMessageProvider2, cleanUserText2, withRawSessionMessageCache2, readRawSessionMessages2, readRawSessionMessagePage2, getRawSessionMessageOrdinalCount2, visitRawSessionMessages2, readRawSessionMessageRange2, primeTailRawMessageCache2, getCachedAbsoluteMessageCount2, primeInMemoryTailRawMessageCache2, readRawSessionMessageIdOrdinalsForRange2, hasRawSessionMessageById2, readRawSessionMessageOrdinalById2, compareRawSessionMessageOrder2, readRawSessionMessageById2, getRawSessionMessageCount2, getRawSessionTagKeysThrough2, getLegacyProtectedTailStartOrdinal2, readSessionChunk2 };
|