@retaw106/mmc-pi 2.2.1 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +14 -3
- package/dist/config-init.js +14 -3
- package/dist/{index-v6scmxvt.js → index-1p318k54.js} +1 -1
- package/dist/{index-dx9vhsy8.js → index-4ktngyya.js} +1 -1
- package/dist/{index-54cgx9gm.js → index-83p1tc98.js} +6 -2
- package/dist/{index-ny303arh.js → index-cr4hzd76.js} +293 -36
- package/dist/{index-j0jjtcwm.js → index-zmx631ey.js} +1711 -219
- package/dist/index.js +26063 -20928
- package/dist/{message-time-backfill-a3p9r5rh.js → message-time-backfill-fvkrfyp4.js} +3 -3
- package/dist/{read-session-chunk-ktp56z65.js → read-session-chunk-g164kwxz.js} +3 -3
- package/dist/{safe-notification-target-hn43s7bc.js → safe-notification-target-5z28m0n9.js} +2 -2
- package/dist/web/main.js +424 -27
- package/package.json +7 -9
|
@@ -1,12 +1,14 @@
|
|
|
1
|
+
import {
|
|
2
|
+
getTagIdentityWriteGeneration,
|
|
3
|
+
Database,
|
|
4
|
+
getSqliteDatabasePath
|
|
5
|
+
} from "./index-cr4hzd76.js";
|
|
1
6
|
import {
|
|
2
7
|
getHarness,
|
|
3
8
|
harnessOwnsOpenCodeStore,
|
|
4
9
|
log,
|
|
5
10
|
sessionLog
|
|
6
|
-
} from "./index-
|
|
7
|
-
import {
|
|
8
|
-
Database
|
|
9
|
-
} from "./index-ny303arh.js";
|
|
11
|
+
} from "./index-83p1tc98.js";
|
|
10
12
|
|
|
11
13
|
// ../plugin/src/hooks/magic-context/tokenizer-calibration-seeds.json
|
|
12
14
|
var tokenizer_calibration_seeds_default = [
|
|
@@ -571,7 +573,7 @@ import { existsSync, readFileSync, realpathSync, statSync } from "node:fs";
|
|
|
571
573
|
import { createRequire } from "node:module";
|
|
572
574
|
import { homedir } from "node:os";
|
|
573
575
|
import { dirname, join, resolve } from "node:path";
|
|
574
|
-
import { pathToFileURL } from "node:url";
|
|
576
|
+
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
575
577
|
|
|
576
578
|
// ../plugin/src/shared/commit-detection.ts
|
|
577
579
|
var HASH_HEX = "[0-9a-f]{7,12}";
|
|
@@ -593,6 +595,272 @@ function removeSystemReminders(text) {
|
|
|
593
595
|
return text.replace(/<system-reminder>[\s\S]*?<\/system-reminder>/gi, "").trim();
|
|
594
596
|
}
|
|
595
597
|
|
|
598
|
+
// ../plugin/src/hooks/magic-context/token-count-exact.ts
|
|
599
|
+
var NO_RANK = 4294967295;
|
|
600
|
+
var LONG_PIECE_BYTES = 256;
|
|
601
|
+
var MERGE_CACHE_GENERATION = 50000;
|
|
602
|
+
function hasInternals(tokenizer) {
|
|
603
|
+
const value = tokenizer;
|
|
604
|
+
return !!value && value.patternRegex instanceof RegExp && value.patternRegex.global && typeof value.stringRankEncoder === "object" && value.stringRankEncoder !== null && Array.isArray(value.binaryFirstByteIndex);
|
|
605
|
+
}
|
|
606
|
+
var textEncoder = new TextEncoder;
|
|
607
|
+
var textDecoder = new TextDecoder("utf-8");
|
|
608
|
+
function isValidUtf8(bytes) {
|
|
609
|
+
let index = 0;
|
|
610
|
+
while (index < bytes.length) {
|
|
611
|
+
const first = bytes[index];
|
|
612
|
+
let size = 0;
|
|
613
|
+
let codePoint = 0;
|
|
614
|
+
if (first <= 127) {
|
|
615
|
+
size = 1;
|
|
616
|
+
codePoint = first;
|
|
617
|
+
} else if ((first & 224) === 192) {
|
|
618
|
+
size = 2;
|
|
619
|
+
codePoint = first & 31;
|
|
620
|
+
if (first <= 193)
|
|
621
|
+
return false;
|
|
622
|
+
} else if ((first & 240) === 224) {
|
|
623
|
+
size = 3;
|
|
624
|
+
codePoint = first & 15;
|
|
625
|
+
} else if ((first & 248) === 240) {
|
|
626
|
+
size = 4;
|
|
627
|
+
codePoint = first & 7;
|
|
628
|
+
if (first > 244)
|
|
629
|
+
return false;
|
|
630
|
+
} else
|
|
631
|
+
return false;
|
|
632
|
+
if (index + size > bytes.length)
|
|
633
|
+
return false;
|
|
634
|
+
for (let offset = 1;offset < size; offset += 1) {
|
|
635
|
+
const byte = bytes[index + offset];
|
|
636
|
+
if ((byte & 192) !== 128)
|
|
637
|
+
return false;
|
|
638
|
+
codePoint = codePoint << 6 | byte & 63;
|
|
639
|
+
}
|
|
640
|
+
if (size === 2 && codePoint < 128)
|
|
641
|
+
return false;
|
|
642
|
+
if (size === 3 && codePoint < 2048)
|
|
643
|
+
return false;
|
|
644
|
+
if (size === 4 && codePoint < 65536)
|
|
645
|
+
return false;
|
|
646
|
+
if (codePoint >= 55296 && codePoint <= 57343)
|
|
647
|
+
return false;
|
|
648
|
+
if (codePoint > 1114111)
|
|
649
|
+
return false;
|
|
650
|
+
index += size;
|
|
651
|
+
}
|
|
652
|
+
return true;
|
|
653
|
+
}
|
|
654
|
+
function binarySearchBytes(sorted, key) {
|
|
655
|
+
let low = 0;
|
|
656
|
+
let high = sorted.length - 1;
|
|
657
|
+
while (low <= high) {
|
|
658
|
+
const mid = low + high >>> 1;
|
|
659
|
+
const midKey = sorted[mid][0];
|
|
660
|
+
let compare = 0;
|
|
661
|
+
const shared = Math.min(midKey.length, key.length);
|
|
662
|
+
for (let index = 0;index < shared; index += 1) {
|
|
663
|
+
compare = midKey[index] - key[index];
|
|
664
|
+
if (compare !== 0)
|
|
665
|
+
break;
|
|
666
|
+
}
|
|
667
|
+
if (compare === 0)
|
|
668
|
+
compare = midKey.length - key.length;
|
|
669
|
+
if (compare === 0)
|
|
670
|
+
return sorted[mid][1];
|
|
671
|
+
if (compare < 0)
|
|
672
|
+
low = mid + 1;
|
|
673
|
+
else
|
|
674
|
+
high = mid - 1;
|
|
675
|
+
}
|
|
676
|
+
return -1;
|
|
677
|
+
}
|
|
678
|
+
function rankOf(tokenizer, slice) {
|
|
679
|
+
if (isValidUtf8(slice)) {
|
|
680
|
+
const rank = tokenizer.stringRankEncoder[textDecoder.decode(slice)];
|
|
681
|
+
if (rank !== undefined)
|
|
682
|
+
return rank;
|
|
683
|
+
}
|
|
684
|
+
const bucket = tokenizer.binaryFirstByteIndex[slice[0]];
|
|
685
|
+
if (bucket !== null && bucket !== undefined) {
|
|
686
|
+
const rank = binarySearchBytes(bucket, slice);
|
|
687
|
+
if (rank !== -1)
|
|
688
|
+
return rank;
|
|
689
|
+
}
|
|
690
|
+
return NO_RANK;
|
|
691
|
+
}
|
|
692
|
+
function mergeByScanning(tokenizer, bytes) {
|
|
693
|
+
const starts = [];
|
|
694
|
+
const ranks = [];
|
|
695
|
+
const pairRank = (index) => {
|
|
696
|
+
const end = starts[index + 2];
|
|
697
|
+
return end === undefined ? NO_RANK : rankOf(tokenizer, bytes.subarray(starts[index], end));
|
|
698
|
+
};
|
|
699
|
+
for (let index = 0;index <= bytes.length; index += 1)
|
|
700
|
+
starts.push(index);
|
|
701
|
+
for (let index = 0;index < starts.length; index += 1)
|
|
702
|
+
ranks.push(pairRank(index));
|
|
703
|
+
while (starts.length > 1) {
|
|
704
|
+
let minRank = NO_RANK;
|
|
705
|
+
let minIndex = -1;
|
|
706
|
+
for (let index = 0;index < ranks.length - 1; index += 1) {
|
|
707
|
+
if (ranks[index] < minRank) {
|
|
708
|
+
minRank = ranks[index];
|
|
709
|
+
minIndex = index;
|
|
710
|
+
}
|
|
711
|
+
}
|
|
712
|
+
if (minIndex === -1)
|
|
713
|
+
break;
|
|
714
|
+
starts.splice(minIndex + 1, 1);
|
|
715
|
+
ranks.splice(minIndex, 1);
|
|
716
|
+
ranks[minIndex] = pairRank(minIndex);
|
|
717
|
+
if (minIndex > 0)
|
|
718
|
+
ranks[minIndex - 1] = pairRank(minIndex - 1);
|
|
719
|
+
}
|
|
720
|
+
const tokens = [];
|
|
721
|
+
for (let index = 0;index < starts.length - 1; index += 1) {
|
|
722
|
+
const rank = rankOf(tokenizer, bytes.subarray(starts[index], starts[index + 1]));
|
|
723
|
+
if (rank !== NO_RANK)
|
|
724
|
+
tokens.push(rank);
|
|
725
|
+
}
|
|
726
|
+
return tokens;
|
|
727
|
+
}
|
|
728
|
+
function mergeWithHeap(tokenizer, bytes) {
|
|
729
|
+
const length = bytes.length;
|
|
730
|
+
const next = new Int32Array(length + 1);
|
|
731
|
+
const prev = new Int32Array(length + 1);
|
|
732
|
+
const alive = new Uint8Array(length + 1).fill(1);
|
|
733
|
+
const rank = new Float64Array(length + 1).fill(NO_RANK);
|
|
734
|
+
for (let index = 0;index <= length; index += 1) {
|
|
735
|
+
next[index] = index + 1;
|
|
736
|
+
prev[index] = index - 1;
|
|
737
|
+
}
|
|
738
|
+
const heapRank = [];
|
|
739
|
+
const heapAt = [];
|
|
740
|
+
const less = (a, b) => heapRank[a] < heapRank[b] || heapRank[a] === heapRank[b] && heapAt[a] < heapAt[b];
|
|
741
|
+
const swap = (a, b) => {
|
|
742
|
+
[heapRank[a], heapRank[b]] = [heapRank[b], heapRank[a]];
|
|
743
|
+
[heapAt[a], heapAt[b]] = [heapAt[b], heapAt[a]];
|
|
744
|
+
};
|
|
745
|
+
const push = (value, at) => {
|
|
746
|
+
heapRank.push(value);
|
|
747
|
+
heapAt.push(at);
|
|
748
|
+
let child = heapRank.length - 1;
|
|
749
|
+
while (child > 0) {
|
|
750
|
+
const parent = child - 1 >> 1;
|
|
751
|
+
if (!less(child, parent))
|
|
752
|
+
break;
|
|
753
|
+
swap(child, parent);
|
|
754
|
+
child = parent;
|
|
755
|
+
}
|
|
756
|
+
};
|
|
757
|
+
const pop = () => {
|
|
758
|
+
const last = heapRank.length - 1;
|
|
759
|
+
swap(0, last);
|
|
760
|
+
heapRank.pop();
|
|
761
|
+
heapAt.pop();
|
|
762
|
+
let parent = 0;
|
|
763
|
+
for (;; ) {
|
|
764
|
+
const left = parent * 2 + 1;
|
|
765
|
+
const right = left + 1;
|
|
766
|
+
let smallest = parent;
|
|
767
|
+
if (left < heapRank.length && less(left, smallest))
|
|
768
|
+
smallest = left;
|
|
769
|
+
if (right < heapRank.length && less(right, smallest))
|
|
770
|
+
smallest = right;
|
|
771
|
+
if (smallest === parent)
|
|
772
|
+
break;
|
|
773
|
+
swap(parent, smallest);
|
|
774
|
+
parent = smallest;
|
|
775
|
+
}
|
|
776
|
+
};
|
|
777
|
+
const pairRank = (at) => {
|
|
778
|
+
const middle = next[at];
|
|
779
|
+
if (middle >= length)
|
|
780
|
+
return NO_RANK;
|
|
781
|
+
return rankOf(tokenizer, bytes.subarray(at, next[middle]));
|
|
782
|
+
};
|
|
783
|
+
const update = (at) => {
|
|
784
|
+
rank[at] = pairRank(at);
|
|
785
|
+
if (rank[at] !== NO_RANK)
|
|
786
|
+
push(rank[at], at);
|
|
787
|
+
};
|
|
788
|
+
for (let at = 0;at < length; at += 1)
|
|
789
|
+
update(at);
|
|
790
|
+
while (heapRank.length > 0) {
|
|
791
|
+
const value = heapRank[0];
|
|
792
|
+
const at = heapAt[0];
|
|
793
|
+
pop();
|
|
794
|
+
if (!alive[at] || rank[at] !== value)
|
|
795
|
+
continue;
|
|
796
|
+
const removed = next[at];
|
|
797
|
+
alive[removed] = 0;
|
|
798
|
+
next[at] = next[removed];
|
|
799
|
+
prev[next[removed]] = at;
|
|
800
|
+
update(at);
|
|
801
|
+
if (at > 0)
|
|
802
|
+
update(prev[at]);
|
|
803
|
+
}
|
|
804
|
+
const tokens = [];
|
|
805
|
+
for (let at = 0;at < length; at = next[at]) {
|
|
806
|
+
const value = rankOf(tokenizer, bytes.subarray(at, next[at]));
|
|
807
|
+
if (value !== NO_RANK)
|
|
808
|
+
tokens.push(value);
|
|
809
|
+
}
|
|
810
|
+
return tokens;
|
|
811
|
+
}
|
|
812
|
+
var mergeCaches = new WeakMap;
|
|
813
|
+
var mergedPieces = 0;
|
|
814
|
+
function mergePiece(tokenizer, piece) {
|
|
815
|
+
let cache = mergeCaches.get(tokenizer);
|
|
816
|
+
if (!cache) {
|
|
817
|
+
cache = { current: new Map, previous: new Map };
|
|
818
|
+
mergeCaches.set(tokenizer, cache);
|
|
819
|
+
}
|
|
820
|
+
let tokens = cache.current.get(piece);
|
|
821
|
+
if (tokens)
|
|
822
|
+
return tokens;
|
|
823
|
+
tokens = cache.previous.get(piece);
|
|
824
|
+
if (!tokens) {
|
|
825
|
+
const bytes = textEncoder.encode(piece);
|
|
826
|
+
mergedPieces += 1;
|
|
827
|
+
tokens = bytes.length > LONG_PIECE_BYTES ? mergeWithHeap(tokenizer, bytes) : mergeByScanning(tokenizer, bytes);
|
|
828
|
+
}
|
|
829
|
+
if (cache.current.size >= MERGE_CACHE_GENERATION) {
|
|
830
|
+
cache.previous = cache.current;
|
|
831
|
+
cache.current = new Map;
|
|
832
|
+
}
|
|
833
|
+
cache.current.set(piece, tokens);
|
|
834
|
+
return tokens;
|
|
835
|
+
}
|
|
836
|
+
function encodeTokensExactly(tokenizer, text) {
|
|
837
|
+
if (!hasInternals(tokenizer)) {
|
|
838
|
+
return tokenizer.encode(text, "all");
|
|
839
|
+
}
|
|
840
|
+
if (text.length < 10) {
|
|
841
|
+
const direct = tokenizer.stringRankEncoder[text];
|
|
842
|
+
if (direct !== undefined)
|
|
843
|
+
return [direct];
|
|
844
|
+
}
|
|
845
|
+
const tokens = [];
|
|
846
|
+
const regex = tokenizer.patternRegex;
|
|
847
|
+
regex.lastIndex = 0;
|
|
848
|
+
for (let match = regex.exec(text);match !== null; match = regex.exec(text)) {
|
|
849
|
+
const piece = match[0];
|
|
850
|
+
const direct = tokenizer.stringRankEncoder[piece];
|
|
851
|
+
if (direct !== undefined) {
|
|
852
|
+
tokens.push(direct);
|
|
853
|
+
continue;
|
|
854
|
+
}
|
|
855
|
+
for (const token of mergePiece(tokenizer, piece))
|
|
856
|
+
tokens.push(token);
|
|
857
|
+
}
|
|
858
|
+
return tokens;
|
|
859
|
+
}
|
|
860
|
+
function countTokensExactly(tokenizer, text) {
|
|
861
|
+
return encodeTokensExactly(tokenizer, text).length;
|
|
862
|
+
}
|
|
863
|
+
|
|
596
864
|
// ../plugin/src/hooks/magic-context/read-session-formatting.ts
|
|
597
865
|
var MAX_COMMITS_PER_BLOCK = 5;
|
|
598
866
|
function hasMeaningfulUserText2(parts) {
|
|
@@ -705,22 +973,30 @@ var tokenizerSerializedTableBytes;
|
|
|
705
973
|
function tokenizerPackageRoots() {
|
|
706
974
|
const cwd = process.cwd();
|
|
707
975
|
const openCodeCache = join(process.env.XDG_CACHE_HOME ?? join(homedir(), ".cache"), "opencode");
|
|
708
|
-
const roots = [cwd, openCodeCache];
|
|
709
976
|
const candidates = [];
|
|
710
|
-
|
|
977
|
+
const probeRoot = (root) => {
|
|
711
978
|
for (const packageDir of TOKENIZER_PACKAGE_DIRS) {
|
|
712
979
|
candidates.push(join(root, "node_modules", ...packageDir, "node_modules", "ai-tokenizer"));
|
|
713
980
|
}
|
|
714
981
|
candidates.push(join(root, "node_modules", "ai-tokenizer"));
|
|
982
|
+
};
|
|
983
|
+
for (const root of [cwd, openCodeCache]) {
|
|
984
|
+
probeRoot(root);
|
|
715
985
|
}
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
986
|
+
const pushAncestors = (startDir) => {
|
|
987
|
+
let ancestor = startDir;
|
|
988
|
+
while (true) {
|
|
989
|
+
candidates.push(join(ancestor, "node_modules", "ai-tokenizer"));
|
|
990
|
+
const parent = dirname(ancestor);
|
|
991
|
+
if (parent === ancestor)
|
|
992
|
+
break;
|
|
993
|
+
ancestor = parent;
|
|
994
|
+
}
|
|
995
|
+
};
|
|
996
|
+
pushAncestors(process.argv[1] ? dirname(resolve(process.argv[1])) : cwd);
|
|
997
|
+
const ownDir = dirname(fileURLToPath(new URL(import.meta.url)));
|
|
998
|
+
pushAncestors(ownDir);
|
|
999
|
+
probeRoot(join(homedir(), ".omp", "plugins"));
|
|
724
1000
|
return [...new Set(candidates)];
|
|
725
1001
|
}
|
|
726
1002
|
function packageImportTarget(value) {
|
|
@@ -838,7 +1114,7 @@ function estimateTokens(text) {
|
|
|
838
1114
|
if (!activeTokenizer)
|
|
839
1115
|
return estimateTokensHeuristically(text);
|
|
840
1116
|
try {
|
|
841
|
-
return activeTokenizer
|
|
1117
|
+
return countTokensExactly(activeTokenizer, text);
|
|
842
1118
|
} catch (error) {
|
|
843
1119
|
tokenizer = undefined;
|
|
844
1120
|
tokenizerLoadAttempted = true;
|
|
@@ -1013,6 +1289,16 @@ function prependTag(tagId, value) {
|
|
|
1013
1289
|
return `§${tagId}§ ${stripped}`;
|
|
1014
1290
|
}
|
|
1015
1291
|
|
|
1292
|
+
// ../plugin/src/hooks/magic-context/tail-hygiene-output-count.ts
|
|
1293
|
+
var CURRENT_OUTPUT_COUNT = Symbol.for("magic-context.current-reclaimable-tool-output-count");
|
|
1294
|
+
function countReclaimableToolOutputs(parts) {
|
|
1295
|
+
return parts.filter((part) => part.kind === "toolOutput" && part.uTokens > 0).length;
|
|
1296
|
+
}
|
|
1297
|
+
function rememberCurrentToolOutputCount(baseline, measurement) {
|
|
1298
|
+
return Object.assign(baseline, {
|
|
1299
|
+
[CURRENT_OUTPUT_COUNT]: countReclaimableToolOutputs(measurement.parts)
|
|
1300
|
+
});
|
|
1301
|
+
}
|
|
1016
1302
|
// ../plugin/src/shared/record-type-guard.ts
|
|
1017
1303
|
function isRecord(value) {
|
|
1018
1304
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
@@ -1020,15 +1306,34 @@ function isRecord(value) {
|
|
|
1020
1306
|
|
|
1021
1307
|
// ../plugin/src/shared/stable-json.ts
|
|
1022
1308
|
function stableStringify(value, seen = new WeakSet) {
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1309
|
+
const pieces = [];
|
|
1310
|
+
writeStable(value, seen, (piece) => pieces.push(piece));
|
|
1311
|
+
return pieces.join("");
|
|
1312
|
+
}
|
|
1313
|
+
function writeStable(value, seen, write) {
|
|
1314
|
+
if (value === undefined) {
|
|
1315
|
+
write("undefined");
|
|
1316
|
+
return;
|
|
1317
|
+
}
|
|
1318
|
+
if (value === null || typeof value !== "object") {
|
|
1319
|
+
write(JSON.stringify(value) ?? String(value));
|
|
1320
|
+
return;
|
|
1321
|
+
}
|
|
1322
|
+
if (seen.has(value)) {
|
|
1323
|
+
write('"[Circular]"');
|
|
1324
|
+
return;
|
|
1325
|
+
}
|
|
1029
1326
|
seen.add(value);
|
|
1030
1327
|
if (Array.isArray(value)) {
|
|
1031
|
-
|
|
1328
|
+
write("[");
|
|
1329
|
+
for (let index = 0;index < value.length; index += 1) {
|
|
1330
|
+
if (index > 0)
|
|
1331
|
+
write(",");
|
|
1332
|
+
if (index in value)
|
|
1333
|
+
writeStable(value[index], seen, write);
|
|
1334
|
+
}
|
|
1335
|
+
write("]");
|
|
1336
|
+
return;
|
|
1032
1337
|
}
|
|
1033
1338
|
const entries = Object.entries(value).sort(([a], [b]) => {
|
|
1034
1339
|
if (a < b)
|
|
@@ -1037,11 +1342,34 @@ function stableStringify(value, seen = new WeakSet) {
|
|
|
1037
1342
|
return 1;
|
|
1038
1343
|
return 0;
|
|
1039
1344
|
});
|
|
1040
|
-
|
|
1345
|
+
if (entries.length > 0 && entries.length <= 16 && entries.every(([key, child]) => key.length <= 4096 && !/^(0|[1-9]\d*)$/.test(key) && (child === null || typeof child === "number" || typeof child === "boolean" || typeof child === "string" && child.length <= 4096))) {
|
|
1346
|
+
write(JSON.stringify(Object.fromEntries(entries)));
|
|
1347
|
+
return;
|
|
1348
|
+
}
|
|
1349
|
+
write("{");
|
|
1350
|
+
entries.forEach(([key, child], index) => {
|
|
1351
|
+
if (index > 0)
|
|
1352
|
+
write(",");
|
|
1353
|
+
write(JSON.stringify(key));
|
|
1354
|
+
write(":");
|
|
1355
|
+
writeStable(child, seen, write);
|
|
1356
|
+
});
|
|
1357
|
+
write("}");
|
|
1358
|
+
}
|
|
1359
|
+
function jsonHashSink(prefix = "") {
|
|
1360
|
+
let hash = 2166136261;
|
|
1361
|
+
const write = (piece) => {
|
|
1362
|
+
for (let index = 0;index < piece.length; index++) {
|
|
1363
|
+
hash = Math.imul(hash ^ piece.charCodeAt(index), 16777619);
|
|
1364
|
+
}
|
|
1365
|
+
};
|
|
1366
|
+
write(prefix);
|
|
1367
|
+
return { write, digest: () => (hash >>> 0).toString(16).padStart(8, "0") };
|
|
1041
1368
|
}
|
|
1042
1369
|
|
|
1043
1370
|
// ../plugin/src/shared/tag-owner-id.ts
|
|
1044
1371
|
var TEXT_TAG_IDENTITY_MARKER = ":mc-text-v1:";
|
|
1372
|
+
var PI_FALLBACK_PREFIXES_JSON = JSON.stringify([..."pi-msg-"].reduce((prefixes, char) => prefixes.flatMap((prefix) => /[a-z]/.test(char) ? [prefix + char, prefix + char.toUpperCase()] : [prefix + char]), [""]));
|
|
1045
1373
|
var POSITIONAL_CONTENT_SUFFIX = /:(?:p|file)\d+$/;
|
|
1046
1374
|
var CONTENT_DERIVED_TEXT_SUFFIX = /:mc-text-v1:[0-9a-f]+:[0-9a-f]+:o\d+$/;
|
|
1047
1375
|
function contentTagOwnerMessageId(contentId) {
|
|
@@ -1054,6 +1382,197 @@ function contentTagOwnerMessageId(contentId) {
|
|
|
1054
1382
|
return contentId;
|
|
1055
1383
|
}
|
|
1056
1384
|
|
|
1385
|
+
// ../plugin/src/shared/tail-hygiene-json.ts
|
|
1386
|
+
var MAX_MEMO_BYTES = 64 * 1024 * 1024;
|
|
1387
|
+
var MAX_MEMO_ENTRIES = 1e4;
|
|
1388
|
+
|
|
1389
|
+
class TailHygieneJsonMemo {
|
|
1390
|
+
entries = new Map;
|
|
1391
|
+
bytes = 0;
|
|
1392
|
+
passText = new WeakMap;
|
|
1393
|
+
passHash = new WeakMap;
|
|
1394
|
+
beginPass() {
|
|
1395
|
+
this.passText = new WeakMap;
|
|
1396
|
+
this.passHash = new WeakMap;
|
|
1397
|
+
}
|
|
1398
|
+
get(key, value, compute, fingerprint = false) {
|
|
1399
|
+
const pass = fingerprint ? this.passHash : this.passText;
|
|
1400
|
+
const object = value !== null && typeof value === "object" ? value : undefined;
|
|
1401
|
+
const local = object ? pass.get(object) : undefined;
|
|
1402
|
+
if (local !== undefined)
|
|
1403
|
+
return local;
|
|
1404
|
+
const previous = this.entries.get(key);
|
|
1405
|
+
if (previous && sameJson(previous.snapshot, value)) {
|
|
1406
|
+
if (object)
|
|
1407
|
+
pass.set(object, previous.result);
|
|
1408
|
+
return previous.result;
|
|
1409
|
+
}
|
|
1410
|
+
const result = compute();
|
|
1411
|
+
if (previous) {
|
|
1412
|
+
this.bytes -= previous.bytes;
|
|
1413
|
+
this.entries.delete(key);
|
|
1414
|
+
}
|
|
1415
|
+
try {
|
|
1416
|
+
const copy = copyJson(value);
|
|
1417
|
+
const bytes = copy.bytes + (key.length + result.length) * 2;
|
|
1418
|
+
if (bytes <= MAX_MEMO_BYTES) {
|
|
1419
|
+
this.entries.set(key, { snapshot: copy.value, result, bytes });
|
|
1420
|
+
this.bytes += bytes;
|
|
1421
|
+
if (object)
|
|
1422
|
+
pass.set(object, result);
|
|
1423
|
+
}
|
|
1424
|
+
} catch {}
|
|
1425
|
+
while (this.bytes > MAX_MEMO_BYTES || this.entries.size > MAX_MEMO_ENTRIES) {
|
|
1426
|
+
const oldest = this.entries.entries().next().value;
|
|
1427
|
+
if (!oldest)
|
|
1428
|
+
break;
|
|
1429
|
+
this.bytes -= oldest[1].bytes;
|
|
1430
|
+
this.entries.delete(oldest[0]);
|
|
1431
|
+
}
|
|
1432
|
+
return result;
|
|
1433
|
+
}
|
|
1434
|
+
}
|
|
1435
|
+
function copyJson(value) {
|
|
1436
|
+
const seen = new Map;
|
|
1437
|
+
let bytes = 0;
|
|
1438
|
+
const copy = (current) => {
|
|
1439
|
+
bytes += 16;
|
|
1440
|
+
if (typeof current === "string")
|
|
1441
|
+
bytes += current.length * 2;
|
|
1442
|
+
if (bytes > MAX_MEMO_BYTES)
|
|
1443
|
+
throw new Error("memo capacity");
|
|
1444
|
+
if (current === null || typeof current !== "object") {
|
|
1445
|
+
if (["function", "symbol", "bigint"].includes(typeof current))
|
|
1446
|
+
throw new Error("exotic");
|
|
1447
|
+
return current;
|
|
1448
|
+
}
|
|
1449
|
+
if (seen.has(current))
|
|
1450
|
+
return seen.get(current);
|
|
1451
|
+
const prototype = Object.getPrototypeOf(current);
|
|
1452
|
+
if (!Array.isArray(current) && prototype !== Object.prototype && prototype !== null)
|
|
1453
|
+
throw new Error("exotic");
|
|
1454
|
+
const result = Array.isArray(current) ? new Array(current.length) : Object.create(prototype);
|
|
1455
|
+
seen.set(current, result);
|
|
1456
|
+
for (const key of Object.keys(current)) {
|
|
1457
|
+
const descriptor = Object.getOwnPropertyDescriptor(current, key);
|
|
1458
|
+
if (!descriptor || !("value" in descriptor))
|
|
1459
|
+
throw new Error("accessor");
|
|
1460
|
+
bytes += key.length * 2 + 16;
|
|
1461
|
+
if (key === "__proto__") {
|
|
1462
|
+
Object.defineProperty(result, key, {
|
|
1463
|
+
value: copy(descriptor.value),
|
|
1464
|
+
enumerable: true,
|
|
1465
|
+
writable: true,
|
|
1466
|
+
configurable: true
|
|
1467
|
+
});
|
|
1468
|
+
} else
|
|
1469
|
+
result[key] = copy(descriptor.value);
|
|
1470
|
+
}
|
|
1471
|
+
return result;
|
|
1472
|
+
};
|
|
1473
|
+
return { value: copy(value), bytes };
|
|
1474
|
+
}
|
|
1475
|
+
function sameJson(before, after) {
|
|
1476
|
+
const forward = new Map;
|
|
1477
|
+
const reverse = new Set;
|
|
1478
|
+
const same = (left, right) => {
|
|
1479
|
+
if (left === null || right === null || typeof left !== "object" || typeof right !== "object")
|
|
1480
|
+
return Object.is(left, right);
|
|
1481
|
+
if (forward.has(left))
|
|
1482
|
+
return forward.get(left) === right;
|
|
1483
|
+
if (reverse.has(right) || Object.getPrototypeOf(left) !== Object.getPrototypeOf(right))
|
|
1484
|
+
return false;
|
|
1485
|
+
forward.set(left, right);
|
|
1486
|
+
reverse.add(right);
|
|
1487
|
+
if (Array.isArray(left) && (!Array.isArray(right) || left.length !== right.length))
|
|
1488
|
+
return false;
|
|
1489
|
+
const keys = Object.keys(left);
|
|
1490
|
+
const otherKeys = Object.keys(right);
|
|
1491
|
+
if (keys.length !== otherKeys.length)
|
|
1492
|
+
return false;
|
|
1493
|
+
for (let index = 0;index < keys.length; index++) {
|
|
1494
|
+
const key = keys[index];
|
|
1495
|
+
if (key !== otherKeys[index])
|
|
1496
|
+
return false;
|
|
1497
|
+
const descriptor = Object.getOwnPropertyDescriptor(right, key);
|
|
1498
|
+
if (!descriptor || !("value" in descriptor) || !same(left[key], descriptor.value))
|
|
1499
|
+
return false;
|
|
1500
|
+
}
|
|
1501
|
+
return true;
|
|
1502
|
+
};
|
|
1503
|
+
return same(before, after);
|
|
1504
|
+
}
|
|
1505
|
+
function piJsonHash(value, fallback) {
|
|
1506
|
+
const sink = jsonHashSink("excluded\x00");
|
|
1507
|
+
try {
|
|
1508
|
+
const seen = new Set;
|
|
1509
|
+
const arrays = new Set;
|
|
1510
|
+
const write = (current) => {
|
|
1511
|
+
if (current === null || typeof current !== "object") {
|
|
1512
|
+
if (["function", "symbol", "bigint"].includes(typeof current))
|
|
1513
|
+
throw new Error("exotic");
|
|
1514
|
+
sink.write(JSON.stringify(current) ?? "");
|
|
1515
|
+
} else if (Array.isArray(current)) {
|
|
1516
|
+
if (Object.getPrototypeOf(current) !== Array.prototype || Object.hasOwn(current, "toJSON"))
|
|
1517
|
+
throw new Error("exotic array");
|
|
1518
|
+
if (arrays.has(current))
|
|
1519
|
+
throw new Error("array cycle");
|
|
1520
|
+
arrays.add(current);
|
|
1521
|
+
sink.write("[");
|
|
1522
|
+
for (let index = 0;index < current.length; index++) {
|
|
1523
|
+
if (index)
|
|
1524
|
+
sink.write(",");
|
|
1525
|
+
const descriptor = Object.getOwnPropertyDescriptor(current, index);
|
|
1526
|
+
if (descriptor && !("value" in descriptor))
|
|
1527
|
+
throw new Error("accessor");
|
|
1528
|
+
if (!descriptor && index in current)
|
|
1529
|
+
throw new Error("inherited array element");
|
|
1530
|
+
write(descriptor?.value === undefined ? null : descriptor.value);
|
|
1531
|
+
}
|
|
1532
|
+
sink.write("]");
|
|
1533
|
+
arrays.delete(current);
|
|
1534
|
+
} else if (seen.has(current)) {
|
|
1535
|
+
sink.write('"[Circular]"');
|
|
1536
|
+
} else {
|
|
1537
|
+
const prototype = Object.getPrototypeOf(current);
|
|
1538
|
+
if (prototype !== Object.prototype && prototype !== null)
|
|
1539
|
+
throw new Error("exotic");
|
|
1540
|
+
seen.add(current);
|
|
1541
|
+
const entries = Object.keys(current).map((key) => {
|
|
1542
|
+
const descriptor = Object.getOwnPropertyDescriptor(current, key);
|
|
1543
|
+
if (!descriptor || !("value" in descriptor))
|
|
1544
|
+
throw new Error("accessor");
|
|
1545
|
+
return [key, descriptor.value];
|
|
1546
|
+
});
|
|
1547
|
+
const sorted = Object.fromEntries(entries.sort(([a], [b]) => a.localeCompare(b)));
|
|
1548
|
+
if (entries.length <= 16 && entries.every(([key, child]) => key.length <= 4096 && (child === null || child === undefined || typeof child === "number" || typeof child === "boolean" || typeof child === "string" && child.length <= 4096))) {
|
|
1549
|
+
sink.write(JSON.stringify(sorted));
|
|
1550
|
+
return;
|
|
1551
|
+
}
|
|
1552
|
+
sink.write("{");
|
|
1553
|
+
let first = true;
|
|
1554
|
+
for (const [key, child] of Object.entries(sorted)) {
|
|
1555
|
+
if (child === undefined)
|
|
1556
|
+
continue;
|
|
1557
|
+
if (!first)
|
|
1558
|
+
sink.write(",");
|
|
1559
|
+
first = false;
|
|
1560
|
+
sink.write(JSON.stringify(key));
|
|
1561
|
+
sink.write(":");
|
|
1562
|
+
write(child);
|
|
1563
|
+
}
|
|
1564
|
+
sink.write("}");
|
|
1565
|
+
}
|
|
1566
|
+
};
|
|
1567
|
+
write(value);
|
|
1568
|
+
return sink.digest();
|
|
1569
|
+
} catch {
|
|
1570
|
+
const failed = jsonHashSink("excluded\x00");
|
|
1571
|
+
failed.write(fallback());
|
|
1572
|
+
return failed.digest();
|
|
1573
|
+
}
|
|
1574
|
+
}
|
|
1575
|
+
|
|
1057
1576
|
// ../plugin/src/hooks/magic-context/image-token-estimate.ts
|
|
1058
1577
|
var IMAGE_TOKEN_DIVISOR = 750;
|
|
1059
1578
|
var IMAGE_FALLBACK_TOKENS = 1200;
|
|
@@ -1091,6 +1610,23 @@ function estimateImageTokensFromDataUrl(url) {
|
|
|
1091
1610
|
}
|
|
1092
1611
|
return IMAGE_FALLBACK_TOKENS;
|
|
1093
1612
|
}
|
|
1613
|
+
function estimateToolAttachmentImageTokens(state) {
|
|
1614
|
+
if (!state || typeof state !== "object")
|
|
1615
|
+
return 0;
|
|
1616
|
+
const attachments = state.attachments;
|
|
1617
|
+
if (!Array.isArray(attachments))
|
|
1618
|
+
return 0;
|
|
1619
|
+
let tokens = 0;
|
|
1620
|
+
for (const attachment of attachments) {
|
|
1621
|
+
if (!attachment || typeof attachment !== "object")
|
|
1622
|
+
continue;
|
|
1623
|
+
const { mime, url } = attachment;
|
|
1624
|
+
if (typeof mime !== "string" || !mime.startsWith("image/"))
|
|
1625
|
+
continue;
|
|
1626
|
+
tokens += typeof url === "string" && url.startsWith("data:") ? estimateImageTokensFromDataUrl(url) : IMAGE_FALLBACK_TOKENS;
|
|
1627
|
+
}
|
|
1628
|
+
return tokens;
|
|
1629
|
+
}
|
|
1094
1630
|
function clampImageTokens(n) {
|
|
1095
1631
|
if (n < 1)
|
|
1096
1632
|
return 1;
|
|
@@ -1229,6 +1765,88 @@ function readUint32BE(b, offset) {
|
|
|
1229
1765
|
return (b[offset] << 24 | b[offset + 1] << 16 | b[offset + 2] << 8 | b[offset + 3]) >>> 0;
|
|
1230
1766
|
}
|
|
1231
1767
|
|
|
1768
|
+
// ../plugin/src/hooks/magic-context/provider-visible-parts.ts
|
|
1769
|
+
function isRecord2(value) {
|
|
1770
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
1771
|
+
}
|
|
1772
|
+
var KEPT_TOOL_STATE_FIELDS = new Set([
|
|
1773
|
+
"status",
|
|
1774
|
+
"input",
|
|
1775
|
+
"output",
|
|
1776
|
+
"error",
|
|
1777
|
+
"attachments",
|
|
1778
|
+
"content"
|
|
1779
|
+
]);
|
|
1780
|
+
var KEPT_TOOL_STATE_TIME_FIELD = "compacted";
|
|
1781
|
+
function providerVisiblePart(part) {
|
|
1782
|
+
if (!isRecord2(part) || part.type !== "tool" || !isRecord2(part.state))
|
|
1783
|
+
return part;
|
|
1784
|
+
if (toolStateIsReduced(part.state))
|
|
1785
|
+
return part;
|
|
1786
|
+
return { ...part, state: providerVisibleToolState(part.state) };
|
|
1787
|
+
}
|
|
1788
|
+
var KEPT_ERROR_METADATA_FIELDS = ["interrupted", "output"];
|
|
1789
|
+
function providerVisibleToolState(state) {
|
|
1790
|
+
const view = {};
|
|
1791
|
+
for (const key of Object.keys(state)) {
|
|
1792
|
+
const value = state[key];
|
|
1793
|
+
if (KEPT_TOOL_STATE_FIELDS.has(key)) {
|
|
1794
|
+
view[key] = value;
|
|
1795
|
+
} else if (key === "time" && isRecord2(value)) {
|
|
1796
|
+
view.time = value[KEPT_TOOL_STATE_TIME_FIELD] === undefined ? {} : { [KEPT_TOOL_STATE_TIME_FIELD]: value[KEPT_TOOL_STATE_TIME_FIELD] };
|
|
1797
|
+
} else if (key === "metadata" && state.status === "error" && isRecord2(value)) {
|
|
1798
|
+
const kept = {};
|
|
1799
|
+
for (const field of KEPT_ERROR_METADATA_FIELDS) {
|
|
1800
|
+
if (Object.hasOwn(value, field))
|
|
1801
|
+
kept[field] = value[field];
|
|
1802
|
+
}
|
|
1803
|
+
view.metadata = kept;
|
|
1804
|
+
}
|
|
1805
|
+
}
|
|
1806
|
+
return view;
|
|
1807
|
+
}
|
|
1808
|
+
function toolStateIsReduced(state) {
|
|
1809
|
+
for (const key of Object.keys(state)) {
|
|
1810
|
+
if (KEPT_TOOL_STATE_FIELDS.has(key))
|
|
1811
|
+
continue;
|
|
1812
|
+
const value = state[key];
|
|
1813
|
+
if (key === "time" && isRecord2(value)) {
|
|
1814
|
+
const keys = Object.keys(value);
|
|
1815
|
+
if (keys.length === 0)
|
|
1816
|
+
continue;
|
|
1817
|
+
if (keys.length === 1 && keys[0] === KEPT_TOOL_STATE_TIME_FIELD)
|
|
1818
|
+
continue;
|
|
1819
|
+
return false;
|
|
1820
|
+
}
|
|
1821
|
+
if (key === "metadata" && state.status === "error" && isRecord2(value)) {
|
|
1822
|
+
const extra = Object.keys(value).some((field) => !KEPT_ERROR_METADATA_FIELDS.includes(field));
|
|
1823
|
+
if (!extra)
|
|
1824
|
+
continue;
|
|
1825
|
+
}
|
|
1826
|
+
return false;
|
|
1827
|
+
}
|
|
1828
|
+
return true;
|
|
1829
|
+
}
|
|
1830
|
+
function providerVisibleMessage(message) {
|
|
1831
|
+
return projectMessage(message, providerVisiblePart);
|
|
1832
|
+
}
|
|
1833
|
+
function projectMessage(message, projectPart) {
|
|
1834
|
+
if (!isRecord2(message) || !Array.isArray(message.parts))
|
|
1835
|
+
return message;
|
|
1836
|
+
let parts;
|
|
1837
|
+
for (let index = 0;index < message.parts.length; index += 1) {
|
|
1838
|
+
const part = message.parts[index];
|
|
1839
|
+
const view = projectPart(part);
|
|
1840
|
+
if (view !== part) {
|
|
1841
|
+
parts ??= message.parts.slice(0, index);
|
|
1842
|
+
parts.push(view);
|
|
1843
|
+
} else if (parts) {
|
|
1844
|
+
parts.push(part);
|
|
1845
|
+
}
|
|
1846
|
+
}
|
|
1847
|
+
return parts ? { ...message, parts } : message;
|
|
1848
|
+
}
|
|
1849
|
+
|
|
1232
1850
|
// ../plugin/src/hooks/magic-context/todo-view.ts
|
|
1233
1851
|
import { createHash } from "node:crypto";
|
|
1234
1852
|
var TODO_STATUS_PENDING = "pending";
|
|
@@ -1337,7 +1955,7 @@ function isTodoItem(value) {
|
|
|
1337
1955
|
}
|
|
1338
1956
|
|
|
1339
1957
|
// ../plugin/src/hooks/magic-context/tail-hygiene-walk.ts
|
|
1340
|
-
var MAX_CONTENT_MEMO_BYTES =
|
|
1958
|
+
var MAX_CONTENT_MEMO_BYTES = 256 * 1024 * 1024;
|
|
1341
1959
|
var contentMemo = new Map;
|
|
1342
1960
|
var contentMemoOrder = new Set;
|
|
1343
1961
|
var CHANNEL1_REMINDER_OPEN = `
|
|
@@ -1356,6 +1974,7 @@ function stripChannel1ReminderSpans(output) {
|
|
|
1356
1974
|
}
|
|
1357
1975
|
return stripped;
|
|
1358
1976
|
}
|
|
1977
|
+
var partJsonMemo = new TailHygieneJsonMemo;
|
|
1359
1978
|
function messageIdFromPartKey(key) {
|
|
1360
1979
|
const separator = key.indexOf("\x00");
|
|
1361
1980
|
return separator > 0 ? key.slice(0, separator) : key;
|
|
@@ -1653,7 +2272,7 @@ function formatOldestReclaimableHint(hint) {
|
|
|
1653
2272
|
oldest reclaimable: ${rendered}.` : "";
|
|
1654
2273
|
}
|
|
1655
2274
|
function reclaimableToolOutputCount(parts) {
|
|
1656
|
-
return parts
|
|
2275
|
+
return countReclaimableToolOutputs(parts);
|
|
1657
2276
|
}
|
|
1658
2277
|
function formatReclaimableOutputSummary(count, tokens) {
|
|
1659
2278
|
const outputCount = Math.max(0, Math.floor(count));
|
|
@@ -1991,6 +2610,102 @@ function removePendingOp(db, sessionId, tagId) {
|
|
|
1991
2610
|
getRemovePendingOpStatement(db).run(sessionId, tagId);
|
|
1992
2611
|
}
|
|
1993
2612
|
|
|
2613
|
+
// ../plugin/src/features/magic-context/storage-tag-identity-revision.ts
|
|
2614
|
+
var TAG_IDENTITY_REVISION_PREFIX = "tag_identity_revision:";
|
|
2615
|
+
var TAG_REVISION_PREFIX = "tag_revision:";
|
|
2616
|
+
var TAG_REVISION_INCARNATION_KEY = "tag_revision_incarnation";
|
|
2617
|
+
var tagRevisionIncarnationSql = (value) => `CASE WHEN INSTR(${value}, ':') > 0 THEN SUBSTR(${value}, 1, INSTR(${value}, ':') - 1) ELSE '0' END`;
|
|
2618
|
+
var tagRevisionCountSql = (value) => `CAST(CASE WHEN INSTR(${value}, ':') > 0 THEN SUBSTR(${value}, INSTR(${value}, ':') + 1) ELSE ${value} END AS INTEGER)`;
|
|
2619
|
+
var ADVANCE = (delta = "1") => `CASE
|
|
2620
|
+
WHEN ${tagRevisionCountSql("value")} <= 9223372036854775807 - ${delta}
|
|
2621
|
+
THEN (${tagRevisionIncarnationSql("value")}) || ':' || CAST(${tagRevisionCountSql("value")} + ${delta} AS TEXT)
|
|
2622
|
+
ELSE NULL END`;
|
|
2623
|
+
function tagRevisionKey(sessionId) {
|
|
2624
|
+
return `${TAG_REVISION_PREFIX}${sessionId}`;
|
|
2625
|
+
}
|
|
2626
|
+
function tagIdentityRevisionKey(sessionId) {
|
|
2627
|
+
return `${TAG_IDENTITY_REVISION_PREFIX}${sessionId}`;
|
|
2628
|
+
}
|
|
2629
|
+
var BUMP = (session, prefix = TAG_IDENTITY_REVISION_PREFIX) => `INSERT INTO schema_migrations_meta (key, value)
|
|
2630
|
+
VALUES ('${prefix}' || ${session}, '1')
|
|
2631
|
+
ON CONFLICT(key) DO UPDATE SET value = ${ADVANCE()}`;
|
|
2632
|
+
var TAG_INSERT_CONFLICT_ROWS = `SELECT id, session_id, tag_number FROM tags WHERE id = NEW.id
|
|
2633
|
+
UNION SELECT id, session_id, tag_number FROM tags WHERE session_id = NEW.session_id AND tag_number = NEW.tag_number
|
|
2634
|
+
UNION SELECT id, session_id, tag_number FROM tags
|
|
2635
|
+
WHERE NEW.type = 'tool' AND NEW.tool_owner_message_id IS NOT NULL
|
|
2636
|
+
AND type = 'tool' AND tool_owner_message_id IS NOT NULL
|
|
2637
|
+
AND session_id = NEW.session_id AND message_id = NEW.message_id
|
|
2638
|
+
AND tool_owner_message_id = NEW.tool_owner_message_id`;
|
|
2639
|
+
var BUMP_INSERT_CONFLICTS = (prefix) => `INSERT INTO schema_migrations_meta (key, value)
|
|
2640
|
+
SELECT '${prefix}' || session_id, CAST(COUNT(*) AS TEXT)
|
|
2641
|
+
FROM (${TAG_INSERT_CONFLICT_ROWS}) GROUP BY session_id
|
|
2642
|
+
ON CONFLICT(key) DO UPDATE SET value = ${ADVANCE("CAST(excluded.value AS INTEGER)")}`;
|
|
2643
|
+
var TRIGGER_SQL = `CREATE TEMP TRIGGER IF NOT EXISTS mc_tag_identity_revision_au
|
|
2644
|
+
AFTER UPDATE OF id, session_id, message_id, tag_number, type, tool_owner_message_id, reasoning_token_count
|
|
2645
|
+
ON main.tags
|
|
2646
|
+
BEGIN
|
|
2647
|
+
${BUMP("OLD.session_id")};
|
|
2648
|
+
INSERT INTO schema_migrations_meta (key, value)
|
|
2649
|
+
SELECT '${TAG_IDENTITY_REVISION_PREFIX}' || NEW.session_id, '1'
|
|
2650
|
+
WHERE NEW.session_id IS NOT OLD.session_id
|
|
2651
|
+
ON CONFLICT(key) DO UPDATE SET value = ${ADVANCE()};
|
|
2652
|
+
END`;
|
|
2653
|
+
function installTagIdentityRevisionTrigger(db) {
|
|
2654
|
+
db.exec(`DROP TRIGGER IF EXISTS temp.mc_tag_identity_revision_au;
|
|
2655
|
+
DROP TRIGGER IF EXISTS temp.mc_tag_revision_bi;
|
|
2656
|
+
DROP TRIGGER IF EXISTS temp.mc_tag_revision_ai;
|
|
2657
|
+
DROP TRIGGER IF EXISTS temp.mc_tag_revision_au;
|
|
2658
|
+
DROP TRIGGER IF EXISTS temp.mc_tag_revision_ad;
|
|
2659
|
+
DROP TRIGGER IF EXISTS temp.mc_tag_revision_seed_ai;`);
|
|
2660
|
+
db.exec(`CREATE TEMP TRIGGER mc_tag_revision_seed_ai AFTER INSERT ON main.schema_migrations_meta
|
|
2661
|
+
WHEN INSTR(NEW.value, ':') = 0 AND (
|
|
2662
|
+
SUBSTR(NEW.key, 1, ${TAG_REVISION_PREFIX.length}) = '${TAG_REVISION_PREFIX}'
|
|
2663
|
+
OR SUBSTR(NEW.key, 1, ${TAG_IDENTITY_REVISION_PREFIX.length}) = '${TAG_IDENTITY_REVISION_PREFIX}')
|
|
2664
|
+
BEGIN
|
|
2665
|
+
INSERT INTO schema_migrations_meta (key, value) VALUES ('${TAG_REVISION_INCARNATION_KEY}', '1')
|
|
2666
|
+
ON CONFLICT(key) DO UPDATE SET value = CAST(CASE
|
|
2667
|
+
WHEN CAST(value AS INTEGER) < 9223372036854775807 THEN CAST(value AS INTEGER) + 1
|
|
2668
|
+
ELSE NULL END AS TEXT);
|
|
2669
|
+
UPDATE schema_migrations_meta SET value =
|
|
2670
|
+
(SELECT value FROM schema_migrations_meta WHERE key = '${TAG_REVISION_INCARNATION_KEY}')
|
|
2671
|
+
|| ':' || CAST(NEW.value AS TEXT) WHERE key = NEW.key;
|
|
2672
|
+
INSERT INTO schema_migrations_meta (key, value)
|
|
2673
|
+
SELECT '${TAG_REVISION_INCARNATION_KEY}', '1'
|
|
2674
|
+
WHERE SUBSTR(NEW.key, 1, ${TAG_REVISION_PREFIX.length}) = '${TAG_REVISION_PREFIX}'
|
|
2675
|
+
AND NOT EXISTS (SELECT 1 FROM schema_migrations_meta
|
|
2676
|
+
WHERE key = '${TAG_IDENTITY_REVISION_PREFIX}' || SUBSTR(NEW.key, ${TAG_REVISION_PREFIX.length + 1}))
|
|
2677
|
+
ON CONFLICT(key) DO UPDATE SET value = CAST(CASE
|
|
2678
|
+
WHEN CAST(value AS INTEGER) < 9223372036854775807 THEN CAST(value AS INTEGER) + 1
|
|
2679
|
+
ELSE NULL END AS TEXT);
|
|
2680
|
+
INSERT INTO schema_migrations_meta (key, value)
|
|
2681
|
+
SELECT '${TAG_IDENTITY_REVISION_PREFIX}' || SUBSTR(NEW.key, ${TAG_REVISION_PREFIX.length + 1}),
|
|
2682
|
+
(SELECT value FROM schema_migrations_meta WHERE key = '${TAG_REVISION_INCARNATION_KEY}') || ':0'
|
|
2683
|
+
WHERE SUBSTR(NEW.key, 1, ${TAG_REVISION_PREFIX.length}) = '${TAG_REVISION_PREFIX}'
|
|
2684
|
+
AND NOT EXISTS (SELECT 1 FROM schema_migrations_meta
|
|
2685
|
+
WHERE key = '${TAG_IDENTITY_REVISION_PREFIX}' || SUBSTR(NEW.key, ${TAG_REVISION_PREFIX.length + 1}));
|
|
2686
|
+
END;`);
|
|
2687
|
+
db.exec(TRIGGER_SQL);
|
|
2688
|
+
db.exec(`CREATE TEMP TRIGGER IF NOT EXISTS mc_tag_revision_bi BEFORE INSERT ON main.tags BEGIN
|
|
2689
|
+
${BUMP_INSERT_CONFLICTS(TAG_REVISION_PREFIX)};
|
|
2690
|
+
${BUMP_INSERT_CONFLICTS(TAG_IDENTITY_REVISION_PREFIX)};
|
|
2691
|
+
UPDATE session_meta SET tags_version = tags_version + 1
|
|
2692
|
+
WHERE session_id IN (SELECT session_id FROM (${TAG_INSERT_CONFLICT_ROWS}));
|
|
2693
|
+
END;`);
|
|
2694
|
+
db.exec(`CREATE TEMP TRIGGER IF NOT EXISTS mc_tag_revision_ai AFTER INSERT ON main.tags BEGIN
|
|
2695
|
+
${BUMP("NEW.session_id", TAG_REVISION_PREFIX)};
|
|
2696
|
+
END;
|
|
2697
|
+
CREATE TEMP TRIGGER IF NOT EXISTS mc_tag_revision_au AFTER UPDATE ON main.tags BEGIN
|
|
2698
|
+
${BUMP("OLD.session_id", TAG_REVISION_PREFIX)};
|
|
2699
|
+
INSERT INTO schema_migrations_meta (key, value)
|
|
2700
|
+
SELECT '${TAG_REVISION_PREFIX}' || NEW.session_id, '1'
|
|
2701
|
+
WHERE NEW.session_id IS NOT OLD.session_id
|
|
2702
|
+
ON CONFLICT(key) DO UPDATE SET value = ${ADVANCE()};
|
|
2703
|
+
END;
|
|
2704
|
+
CREATE TEMP TRIGGER IF NOT EXISTS mc_tag_revision_ad AFTER DELETE ON main.tags BEGIN
|
|
2705
|
+
${BUMP("OLD.session_id", TAG_REVISION_PREFIX)};
|
|
2706
|
+
END;`);
|
|
2707
|
+
}
|
|
2708
|
+
|
|
1994
2709
|
// ../plugin/src/features/magic-context/storage-tags.ts
|
|
1995
2710
|
var insertTagStatements = new WeakMap;
|
|
1996
2711
|
var updateTagStatusStatements = new WeakMap;
|
|
@@ -2207,8 +2922,41 @@ function getPersistedToolTagAccounting(db, sessionId, tagNumber) {
|
|
|
2207
2922
|
}
|
|
2208
2923
|
return row;
|
|
2209
2924
|
}
|
|
2210
|
-
function getAllStatusTagTokenTotalsFlat(db, sessionId, floor = 0, calibration = { proseRatio: 1, toolsRatio: 1 }, options) {
|
|
2211
|
-
const
|
|
2925
|
+
function getAllStatusTagTokenTotalsFlat(db, sessionId, floor = 0, calibration = { proseRatio: 1, toolsRatio: 1 }, options, ownerMessageIds) {
|
|
2926
|
+
const columns = "type, message_id, tool_owner_message_id, token_count, input_token_count, reasoning_token_count";
|
|
2927
|
+
const owners = ownerMessageIds ? new Set(ownerMessageIds) : undefined;
|
|
2928
|
+
let scanOrder = "id";
|
|
2929
|
+
if (owners) {
|
|
2930
|
+
const planStatement = db.prepare(`EXPLAIN QUERY PLAN SELECT ${columns} FROM tags WHERE session_id = ?${floor > 0 ? " AND tag_number >= ?" : ""}`);
|
|
2931
|
+
let plan;
|
|
2932
|
+
try {
|
|
2933
|
+
plan = planStatement.all(sessionId, ...floor > 0 ? [floor] : []);
|
|
2934
|
+
} finally {
|
|
2935
|
+
planStatement.finalize?.();
|
|
2936
|
+
}
|
|
2937
|
+
const index = plan.map((row) => /USING (?:COVERING )?INDEX (\S+)/.exec(row.detail)?.[1]).find(Boolean);
|
|
2938
|
+
if (index) {
|
|
2939
|
+
const keyStatement = db.prepare("SELECT name, coll, desc FROM pragma_index_xinfo(?) WHERE key = 1 ORDER BY seqno");
|
|
2940
|
+
let keys;
|
|
2941
|
+
try {
|
|
2942
|
+
keys = keyStatement.all(index);
|
|
2943
|
+
} finally {
|
|
2944
|
+
keyStatement.finalize?.();
|
|
2945
|
+
}
|
|
2946
|
+
scanOrder = [
|
|
2947
|
+
...keys.map((key) => `"${(key.name ?? "id").replaceAll('"', '""')}" COLLATE "${key.coll.replaceAll('"', '""')}"${key.desc ? " DESC" : ""}`),
|
|
2948
|
+
"id"
|
|
2949
|
+
].join(", ");
|
|
2950
|
+
}
|
|
2951
|
+
}
|
|
2952
|
+
const rows = owners ? db.prepare(`SELECT ${columns} FROM tags WHERE id IN (
|
|
2953
|
+
SELECT t.id FROM json_each(?) o CROSS JOIN tags t INDEXED BY idx_tags_session_message_id
|
|
2954
|
+
ON t.session_id = ? AND t.message_id >= o.value AND t.message_id < o.value || ';'
|
|
2955
|
+
UNION
|
|
2956
|
+
SELECT t.id FROM json_each(?) o CROSS JOIN tags t INDEXED BY idx_tags_pi_fallback_tool_owner
|
|
2957
|
+
ON t.session_id = ? AND t.tool_owner_message_id = o.value
|
|
2958
|
+
WHERE t.type = 'tool'
|
|
2959
|
+
) AND tag_number >= ? ORDER BY ${scanOrder}`).all(JSON.stringify([...owners]), sessionId, JSON.stringify([...owners]), sessionId, floor) : floor > 0 ? db.prepare(`SELECT type, message_id, tool_owner_message_id, token_count, input_token_count, reasoning_token_count
|
|
2212
2960
|
FROM tags
|
|
2213
2961
|
WHERE session_id = ? AND tag_number >= ?`).all(sessionId, floor) : db.prepare(`SELECT type, message_id, tool_owner_message_id, token_count, input_token_count, reasoning_token_count
|
|
2214
2962
|
FROM tags
|
|
@@ -2221,6 +2969,8 @@ function getAllStatusTagTokenTotalsFlat(db, sessionId, floor = 0, calibration =
|
|
|
2221
2969
|
if (options?.excludeToolTags === true && row.type === "tool")
|
|
2222
2970
|
continue;
|
|
2223
2971
|
const owner = ownerMessageIdForTagRow(row);
|
|
2972
|
+
if (owners && !owners.has(owner))
|
|
2973
|
+
continue;
|
|
2224
2974
|
if (row.token_count === null) {
|
|
2225
2975
|
nullMessageIds.add(owner);
|
|
2226
2976
|
totals.delete(owner);
|
|
@@ -2235,15 +2985,163 @@ function getAllStatusTagTokenTotalsFlat(db, sessionId, floor = 0, calibration =
|
|
|
2235
2985
|
return { totals, nullMessageIds };
|
|
2236
2986
|
}
|
|
2237
2987
|
function getReasoningTokenEstimatesByMessage(db, sessionId, proseRatio) {
|
|
2238
|
-
const
|
|
2239
|
-
|
|
2240
|
-
|
|
2241
|
-
|
|
2242
|
-
|
|
2243
|
-
|
|
2244
|
-
|
|
2988
|
+
const summary = readTagOwnerSummary(db, sessionId, proseRatio);
|
|
2989
|
+
return summary.reasoningEstimatesByRatio.get(proseRatio) ?? new Map;
|
|
2990
|
+
}
|
|
2991
|
+
var tagOwnerSummaries = new WeakMap;
|
|
2992
|
+
var MAX_CACHED_PROSE_RATIOS = 4;
|
|
2993
|
+
var TAG_OWNER_PAGE_LIMIT = 512;
|
|
2994
|
+
var tagOwnerRowsRead = 0;
|
|
2995
|
+
var tagOwnerShapeChecks = 0;
|
|
2996
|
+
function foldTagOwnerRow(summary, row, newRow = true) {
|
|
2997
|
+
summary.lastRowId = Math.max(summary.lastRowId, row.id);
|
|
2998
|
+
if (newRow)
|
|
2999
|
+
summary.rowCount += 1;
|
|
3000
|
+
if (row.type === "tool" && typeof row.tool_owner_message_id !== "string")
|
|
3001
|
+
summary.unownedToolRows.add(row.id);
|
|
3002
|
+
else
|
|
3003
|
+
summary.unownedToolRows.delete(row.id);
|
|
3004
|
+
const owner = row.type === "tool" ? typeof row.tool_owner_message_id === "string" ? row.tool_owner_message_id : null : typeof row.message_id === "string" ? contentTagOwnerMessageId(row.message_id) : null;
|
|
3005
|
+
if (owner) {
|
|
3006
|
+
summary.maxTagByOwner.set(owner, Math.max(summary.maxTagByOwner.get(owner) ?? 0, row.tag_number));
|
|
3007
|
+
}
|
|
3008
|
+
if (row.type !== "message" || row.reasoning_token_count === null)
|
|
3009
|
+
return;
|
|
3010
|
+
const reasoningOwner = ownerMessageIdForTagRow({
|
|
3011
|
+
type: row.type,
|
|
3012
|
+
message_id: row.message_id,
|
|
3013
|
+
tool_owner_message_id: row.tool_owner_message_id
|
|
3014
|
+
});
|
|
3015
|
+
for (const [ratio, estimates] of summary.reasoningEstimatesByRatio) {
|
|
3016
|
+
estimates.set(reasoningOwner, Math.max(estimates.get(reasoningOwner) ?? 0, row.reasoning_token_count * ratio));
|
|
3017
|
+
}
|
|
3018
|
+
}
|
|
3019
|
+
var TAG_OWNER_COLUMNS = "id, type, message_id, tool_owner_message_id, tag_number, reasoning_token_count";
|
|
3020
|
+
function connectionInTransaction(db) {
|
|
3021
|
+
try {
|
|
3022
|
+
const state = db;
|
|
3023
|
+
return state.inTransaction !== false && state.isTransaction !== false;
|
|
3024
|
+
} catch {
|
|
3025
|
+
return true;
|
|
3026
|
+
}
|
|
3027
|
+
}
|
|
3028
|
+
var TAG_OWNER_APPENDED_WITH_SHAPE_SQL = `SELECT ${TAG_OWNER_COLUMNS}, 0 AS shape, NULL AS tag_count, NULL AS max_id, NULL AS revision
|
|
3029
|
+
FROM tags WHERE id > ? AND +session_id = ?
|
|
3030
|
+
UNION ALL
|
|
3031
|
+
SELECT NULL, NULL, NULL, NULL, NULL, NULL, 1, COUNT(*), MAX(id), (SELECT value FROM schema_migrations_meta WHERE key = ?)
|
|
3032
|
+
FROM tags WHERE session_id = ?`;
|
|
3033
|
+
function readTagIdentityRevision(db, sessionId) {
|
|
3034
|
+
const row = db.prepare("SELECT value FROM schema_migrations_meta WHERE key = ?").get(tagIdentityRevisionKey(sessionId));
|
|
3035
|
+
return typeof row?.value === "string" ? row.value : null;
|
|
3036
|
+
}
|
|
3037
|
+
function captureInitialTagOwnerRevision(db, sessionId, cached, appendedRows) {
|
|
3038
|
+
if (cached.identityRevision !== null || appendedRows === 0)
|
|
3039
|
+
return true;
|
|
3040
|
+
const revision = readTagIdentityRevision(db, sessionId);
|
|
3041
|
+
if (revision !== null && !/^[1-9]\d*:0$/.test(revision))
|
|
3042
|
+
return false;
|
|
3043
|
+
cached.identityRevision = revision;
|
|
3044
|
+
return true;
|
|
3045
|
+
}
|
|
3046
|
+
function appendedSinceForeignCommit(db, sessionId, cached) {
|
|
3047
|
+
tagOwnerShapeChecks += 1;
|
|
3048
|
+
const rows = db.prepare(TAG_OWNER_APPENDED_WITH_SHAPE_SQL).all(cached.globalLastRowId, sessionId, tagIdentityRevisionKey(sessionId), sessionId);
|
|
3049
|
+
const shape = rows.find((row) => row.shape === 1);
|
|
3050
|
+
const appended = rows.filter((row) => row.shape === 0);
|
|
3051
|
+
const highest = appended.reduce((max, row) => Math.max(max, row.id), cached.lastRowId);
|
|
3052
|
+
if (!shape || (shape.revision ?? null) !== cached.identityRevision || shape.tag_count !== cached.rowCount + appended.length || (shape.max_id ?? 0) !== highest)
|
|
3053
|
+
return null;
|
|
3054
|
+
return appended;
|
|
3055
|
+
}
|
|
3056
|
+
function adoptedSinceForeignCommit(db, cached) {
|
|
3057
|
+
if (cached.unownedToolRows.size === 0)
|
|
3058
|
+
return [];
|
|
3059
|
+
return db.prepare(`SELECT ${TAG_OWNER_COLUMNS} FROM tags
|
|
3060
|
+
WHERE id IN (SELECT value FROM json_each(?)) AND tool_owner_message_id IS NOT NULL LIMIT ${TAG_OWNER_PAGE_LIMIT + 1}`).all(JSON.stringify([...cached.unownedToolRows]));
|
|
3061
|
+
}
|
|
3062
|
+
function readTagOwnerSummary(db, sessionId, proseRatio, extraRatios = []) {
|
|
3063
|
+
const inTransaction = connectionInTransaction(db);
|
|
3064
|
+
const versions = inTransaction ? null : db.prepare("SELECT data_version AS dataVersion, (SELECT schema_version FROM pragma_schema_version) AS schemaVersion FROM pragma_data_version").get();
|
|
3065
|
+
const dataVersion = versions?.dataVersion ?? -1;
|
|
3066
|
+
const schemaVersion = versions?.schemaVersion ?? -1;
|
|
3067
|
+
const generation = getTagIdentityWriteGeneration();
|
|
3068
|
+
const sessions = tagOwnerSummaries.get(db) ?? new Map;
|
|
3069
|
+
const cached = inTransaction ? undefined : sessions.get(sessionId);
|
|
3070
|
+
if (cached && (cached.generation === generation || cached.identityRevision === readTagIdentityRevision(db, sessionId)) && cached.schemaVersion === schemaVersion && (proseRatio === undefined || cached.reasoningEstimatesByRatio.has(proseRatio))) {
|
|
3071
|
+
let rows;
|
|
3072
|
+
let globalLastRowId = db.prepare("SELECT COALESCE(MAX(id), 0) AS id FROM tags").get().id;
|
|
3073
|
+
if (cached.dataVersion === dataVersion && cached.generation === generation) {
|
|
3074
|
+
const statement = db.prepare(`SELECT ${TAG_OWNER_COLUMNS} FROM tags WHERE id > ? AND +session_id = ? ORDER BY id LIMIT ${TAG_OWNER_PAGE_LIMIT + 1}`);
|
|
3075
|
+
rows = statement.all(cached.globalLastRowId, sessionId);
|
|
3076
|
+
let page = rows;
|
|
3077
|
+
while (page.length === TAG_OWNER_PAGE_LIMIT + 1) {
|
|
3078
|
+
page = statement.all(page[page.length - 1].id, sessionId);
|
|
3079
|
+
rows.push(...page);
|
|
3080
|
+
}
|
|
3081
|
+
if (!captureInitialTagOwnerRevision(db, sessionId, cached, rows.length)) {
|
|
3082
|
+
rows = null;
|
|
3083
|
+
}
|
|
3084
|
+
} else {
|
|
3085
|
+
try {
|
|
3086
|
+
globalLastRowId = db.prepare("SELECT COALESCE(MAX(id),0) AS id FROM tags").get().id;
|
|
3087
|
+
rows = appendedSinceForeignCommit(db, sessionId, cached);
|
|
3088
|
+
const adopted = rows ? adoptedSinceForeignCommit(db, cached) : [];
|
|
3089
|
+
if (!rows || rows.length > TAG_OWNER_PAGE_LIMIT || adopted.length > TAG_OWNER_PAGE_LIMIT) {
|
|
3090
|
+
rows = null;
|
|
3091
|
+
} else {
|
|
3092
|
+
tagOwnerRowsRead += rows.length + adopted.length;
|
|
3093
|
+
for (const row of rows)
|
|
3094
|
+
foldTagOwnerRow(cached, row);
|
|
3095
|
+
for (const row of adopted)
|
|
3096
|
+
foldTagOwnerRow(cached, row, false);
|
|
3097
|
+
cached.dataVersion = dataVersion;
|
|
3098
|
+
cached.generation = generation;
|
|
3099
|
+
cached.globalLastRowId = globalLastRowId;
|
|
3100
|
+
return cached;
|
|
3101
|
+
}
|
|
3102
|
+
} catch {
|
|
3103
|
+
rows = null;
|
|
3104
|
+
}
|
|
3105
|
+
}
|
|
3106
|
+
if (rows) {
|
|
3107
|
+
const adopted = cached.dataVersion === dataVersion ? [] : adoptedSinceForeignCommit(db, cached);
|
|
3108
|
+
tagOwnerRowsRead += rows.length + adopted.length;
|
|
3109
|
+
for (const row of rows)
|
|
3110
|
+
foldTagOwnerRow(cached, row);
|
|
3111
|
+
for (const row of adopted)
|
|
3112
|
+
foldTagOwnerRow(cached, row, false);
|
|
3113
|
+
cached.dataVersion = dataVersion;
|
|
3114
|
+
cached.generation = generation;
|
|
3115
|
+
cached.globalLastRowId = globalLastRowId;
|
|
3116
|
+
return cached;
|
|
3117
|
+
}
|
|
3118
|
+
}
|
|
3119
|
+
const ratios = [
|
|
3120
|
+
...new Set([...cached?.reasoningEstimatesByRatio.keys() ?? [], ...extraRatios])
|
|
3121
|
+
];
|
|
3122
|
+
if (proseRatio !== undefined && !ratios.includes(proseRatio))
|
|
3123
|
+
ratios.push(proseRatio);
|
|
3124
|
+
const summary = {
|
|
3125
|
+
dataVersion,
|
|
3126
|
+
schemaVersion,
|
|
3127
|
+
generation,
|
|
3128
|
+
lastRowId: 0,
|
|
3129
|
+
globalLastRowId: db.prepare("SELECT COALESCE(MAX(id), 0) AS id FROM tags").get().id,
|
|
3130
|
+
rowCount: 0,
|
|
3131
|
+
unownedToolRows: new Set,
|
|
3132
|
+
identityRevision: readTagIdentityRevision(db, sessionId),
|
|
3133
|
+
maxTagByOwner: new Map,
|
|
3134
|
+
reasoningEstimatesByRatio: new Map(ratios.slice(-MAX_CACHED_PROSE_RATIOS).map((ratio) => [ratio, new Map]))
|
|
3135
|
+
};
|
|
3136
|
+
const rows = db.prepare(`SELECT ${TAG_OWNER_COLUMNS} FROM tags WHERE session_id = ? ORDER BY id`).all(sessionId);
|
|
3137
|
+
tagOwnerRowsRead += rows.length;
|
|
3138
|
+
for (const row of rows)
|
|
3139
|
+
foldTagOwnerRow(summary, row);
|
|
3140
|
+
if (!inTransaction) {
|
|
3141
|
+
sessions.set(sessionId, summary);
|
|
3142
|
+
tagOwnerSummaries.set(db, sessions);
|
|
2245
3143
|
}
|
|
2246
|
-
return
|
|
3144
|
+
return summary;
|
|
2247
3145
|
}
|
|
2248
3146
|
function updateTagInputTokenCount(db, sessionId, tagNumber, newInputTokenCount) {
|
|
2249
3147
|
getUpdateTagInputTokenCountStatement(db).run(newInputTokenCount, sessionId, tagNumber);
|
|
@@ -2278,7 +3176,7 @@ function getTagNumberByMessageIdStatement(db) {
|
|
|
2278
3176
|
function getAssignableTagNumberByMessageIdStatement(db) {
|
|
2279
3177
|
let stmt = getAssignableTagNumberByMessageIdStatements.get(db);
|
|
2280
3178
|
if (!stmt) {
|
|
2281
|
-
stmt = db.prepare(`SELECT tag_number FROM tags
|
|
3179
|
+
stmt = db.prepare(`SELECT tag_number FROM tags INDEXED BY idx_tags_session_message_id
|
|
2282
3180
|
WHERE session_id = ?
|
|
2283
3181
|
AND message_id = ?
|
|
2284
3182
|
AND NOT (
|
|
@@ -2339,31 +3237,37 @@ function insertTag(db, sessionId, messageId, type, byteSize, tagNumber, reasonin
|
|
|
2339
3237
|
function updateTagStatus(db, sessionId, tagId, status) {
|
|
2340
3238
|
getUpdateTagStatusStatement(db).run(status, sessionId, tagId);
|
|
2341
3239
|
}
|
|
2342
|
-
var
|
|
2343
|
-
function
|
|
2344
|
-
if (
|
|
2345
|
-
|
|
2346
|
-
|
|
2347
|
-
statement = db.prepare(`SELECT tag_number AS tagNumber, entry_fingerprint AS entryFingerprint
|
|
2348
|
-
FROM json_each(?) AS owners CROSS JOIN tags
|
|
2349
|
-
WHERE tags.session_id = ? AND tags.type = 'message' AND tags.status = 'compacted'
|
|
2350
|
-
AND entry_fingerprint >= ? || owners.value || ':p'
|
|
2351
|
-
AND entry_fingerprint < ? || owners.value || ':q'
|
|
2352
|
-
ORDER BY CAST(owners.key AS INTEGER), entry_fingerprint, tags.id`);
|
|
2353
|
-
inertWhitespaceStatements.set(db, statement);
|
|
2354
|
-
}
|
|
2355
|
-
const rows = statement.all(JSON.stringify([...new Set(messageIds)]), sessionId, WHITESPACE_ASSISTANT_INERT_FINGERPRINT_PREFIX, WHITESPACE_ASSISTANT_INERT_FINGERPRINT_PREFIX);
|
|
2356
|
-
return rows.map((row) => ({
|
|
2357
|
-
tagNumber: row.tagNumber,
|
|
2358
|
-
contentId: row.entryFingerprint.slice(WHITESPACE_ASSISTANT_INERT_FINGERPRINT_PREFIX.length)
|
|
2359
|
-
}));
|
|
3240
|
+
var droppedTagStatements = new WeakMap;
|
|
3241
|
+
function markTagDropped(db, sessionId, tagId, mode) {
|
|
3242
|
+
if (!mode) {
|
|
3243
|
+
updateTagStatus(db, sessionId, tagId, "dropped");
|
|
3244
|
+
return;
|
|
2360
3245
|
}
|
|
2361
|
-
|
|
2362
|
-
|
|
2363
|
-
|
|
2364
|
-
|
|
2365
|
-
|
|
2366
|
-
|
|
3246
|
+
let statement = droppedTagStatements.get(db);
|
|
3247
|
+
if (!statement) {
|
|
3248
|
+
statement = db.prepare("UPDATE tags SET status = ?, drop_mode = ? WHERE session_id = ? AND tag_number = ?");
|
|
3249
|
+
droppedTagStatements.set(db, statement);
|
|
3250
|
+
}
|
|
3251
|
+
statement.run("dropped", mode, sessionId, tagId);
|
|
3252
|
+
}
|
|
3253
|
+
var inertWhitespaceStatements = new WeakMap;
|
|
3254
|
+
var inertWhitespaceByNumberStatements = new WeakMap;
|
|
3255
|
+
function getInertWhitespaceAssistantTagsByNumbers(db, sessionId, tagNumbers) {
|
|
3256
|
+
if (tagNumbers.length === 0)
|
|
3257
|
+
return [];
|
|
3258
|
+
let statement = inertWhitespaceByNumberStatements.get(db);
|
|
3259
|
+
if (!statement) {
|
|
3260
|
+
statement = db.prepare(`SELECT tags.tag_number AS tagNumber, tags.entry_fingerprint AS entryFingerprint
|
|
3261
|
+
FROM json_each(?) AS wanted CROSS JOIN tags
|
|
3262
|
+
WHERE tags.session_id = ?
|
|
3263
|
+
AND tags.tag_number = wanted.value
|
|
3264
|
+
AND tags.type = 'message'
|
|
3265
|
+
AND tags.status = 'compacted'
|
|
3266
|
+
AND tags.entry_fingerprint LIKE ?
|
|
3267
|
+
ORDER BY tags.tag_number, tags.id`);
|
|
3268
|
+
inertWhitespaceByNumberStatements.set(db, statement);
|
|
3269
|
+
}
|
|
3270
|
+
const rows = statement.all(JSON.stringify([...new Set(tagNumbers)]), sessionId, `${WHITESPACE_ASSISTANT_INERT_FINGERPRINT_PREFIX}%`);
|
|
2367
3271
|
return rows.flatMap((row) => {
|
|
2368
3272
|
const contentId = row.entryFingerprint.slice(WHITESPACE_ASSISTANT_INERT_FINGERPRINT_PREFIX.length);
|
|
2369
3273
|
return contentId.length > 0 ? [{ tagNumber: row.tagNumber, contentId }] : [];
|
|
@@ -2562,13 +3466,54 @@ function findPiFallbackToolOwnerTags(db, sessionId) {
|
|
|
2562
3466
|
message_id AS callId,
|
|
2563
3467
|
tool_owner_message_id AS toolOwnerMessageId,
|
|
2564
3468
|
status
|
|
2565
|
-
FROM tags
|
|
3469
|
+
FROM json_each(?) prefix CROSS JOIN tags
|
|
2566
3470
|
WHERE session_id = ?
|
|
2567
|
-
AND type = 'tool'
|
|
2568
|
-
|
|
2569
|
-
|
|
3471
|
+
AND tags.type = 'tool'
|
|
3472
|
+
AND tool_owner_message_id >= prefix.value
|
|
3473
|
+
AND tool_owner_message_id < substr(prefix.value, 1, 6) || '.'
|
|
3474
|
+
AND tool_owner_message_id LIKE 'pi-msg-%'
|
|
3475
|
+
ORDER BY tag_number ASC`).all(PI_FALLBACK_PREFIXES_JSON, sessionId).filter(isPiFallbackToolOwnerTag);
|
|
3476
|
+
}
|
|
3477
|
+
|
|
3478
|
+
class PiTagIdentityConflictError extends Error {
|
|
3479
|
+
code = "PI_TAG_IDENTITY_CONFLICT";
|
|
3480
|
+
constructor(detail, kind = "tool") {
|
|
3481
|
+
super(`Magic Context ${kind}-tag identity conflict: ${detail}. This turn was refused without sending raw history; resending alone will not repair it`);
|
|
3482
|
+
this.name = "PiTagIdentityConflictError";
|
|
3483
|
+
}
|
|
3484
|
+
}
|
|
3485
|
+
function selectPiTagSurvivor(rows, servedTagNumbers, cachedSurvivor, kind, rebuildingSurvivor) {
|
|
3486
|
+
const served = rows.filter((row) => servedTagNumbers.has(row.tagNumber));
|
|
3487
|
+
const cached = cachedSurvivor === undefined ? undefined : rows.find((row) => row.tagNumber === cachedSurvivor);
|
|
3488
|
+
const rebuilding = rows.find((row) => row.tagNumber === rebuildingSurvivor);
|
|
3489
|
+
const noProofInputs = served.length === 0 && cachedSurvivor === undefined;
|
|
3490
|
+
if (cachedSurvivor !== undefined && !cached || served.length !== 1 && !cached && !rebuilding && !noProofInputs) {
|
|
3491
|
+
throw new PiTagIdentityConflictError(served.length > 1 ? `Conflicting served Pi ${kind} tag numbers; no byte-safe cached survivor is proven` : `duplicate ${kind} tag identities have no proven served-byte survivor`, kind);
|
|
3492
|
+
}
|
|
3493
|
+
const kept = cached ?? rebuilding ?? served[0] ?? rows[1] ?? rows[0];
|
|
3494
|
+
if (!kept)
|
|
3495
|
+
throw new PiTagIdentityConflictError("no served survivor is proven", kind);
|
|
3496
|
+
return kept;
|
|
3497
|
+
}
|
|
3498
|
+
function foldPiIdentityDuplicate(db, sessionId, kept, duplicate) {
|
|
3499
|
+
const owesDrop = kept.status !== "dropped" && duplicate.status === "dropped";
|
|
3500
|
+
foldDuplicateIntoSurvivor(db, sessionId, kept, duplicate, true, false);
|
|
3501
|
+
if (owesDrop)
|
|
3502
|
+
queuePendingOp(db, sessionId, kept.tagNumber, "drop");
|
|
3503
|
+
}
|
|
3504
|
+
function getPiMessageTagCandidates(db, sessionId, messageId) {
|
|
3505
|
+
return getPiFallbackMessageFoldTagRowsByMessageId(db, sessionId, messageId);
|
|
3506
|
+
}
|
|
3507
|
+
function findPiTagIdentityConflict(error) {
|
|
3508
|
+
const seen = new Set;
|
|
3509
|
+
while (error instanceof Error && !seen.has(error)) {
|
|
3510
|
+
if (error instanceof PiTagIdentityConflictError)
|
|
3511
|
+
return error;
|
|
3512
|
+
seen.add(error);
|
|
3513
|
+
error = error.cause;
|
|
3514
|
+
}
|
|
2570
3515
|
}
|
|
2571
|
-
function adoptPiFallbackToolOwnerTag(db, sessionId, tagNumber, callId, oldOwnerMessageId, newOwnerMessageId, servedTagNumbers = new Set) {
|
|
3516
|
+
function adoptPiFallbackToolOwnerTag(db, sessionId, tagNumber, callId, oldOwnerMessageId, newOwnerMessageId, servedTagNumbers = new Set, cachedSurvivor, rebuildingSurvivor) {
|
|
2572
3517
|
const survivor = getPiFallbackFoldTagRowByNumber(db, sessionId, tagNumber);
|
|
2573
3518
|
if (survivor === null || survivor.type !== "tool" || survivor.messageId !== callId || survivor.toolOwnerMessageId !== oldOwnerMessageId) {
|
|
2574
3519
|
return { action: "skipped" };
|
|
@@ -2587,22 +3532,21 @@ function adoptPiFallbackToolOwnerTag(db, sessionId, tagNumber, callId, oldOwnerM
|
|
|
2587
3532
|
if (existing.tagNumber === tagNumber) {
|
|
2588
3533
|
return { action: "skipped" };
|
|
2589
3534
|
}
|
|
2590
|
-
|
|
2591
|
-
|
|
2592
|
-
|
|
2593
|
-
|
|
2594
|
-
foldDuplicateIntoSurvivor(db, sessionId, survivor, existing);
|
|
3535
|
+
const toolRows = [survivor, existing];
|
|
3536
|
+
const kept = selectPiTagSurvivor(toolRows, servedTagNumbers, cachedSurvivor, "tool", rebuildingSurvivor);
|
|
3537
|
+
if (kept.tagNumber === survivor.tagNumber) {
|
|
3538
|
+
foldPiIdentityDuplicate(db, sessionId, survivor, existing);
|
|
2595
3539
|
db.prepare("UPDATE tags SET tool_owner_message_id = ? WHERE session_id = ? AND tag_number = ?").run(newOwnerMessageId, sessionId, tagNumber);
|
|
2596
3540
|
return { action: "folded", tagNumber, deletedTagNumbers: [existing.tagNumber] };
|
|
2597
3541
|
}
|
|
2598
|
-
|
|
3542
|
+
foldPiIdentityDuplicate(db, sessionId, existing, survivor);
|
|
2599
3543
|
return {
|
|
2600
3544
|
action: "folded",
|
|
2601
3545
|
tagNumber: existing.tagNumber,
|
|
2602
3546
|
deletedTagNumbers: [tagNumber]
|
|
2603
3547
|
};
|
|
2604
3548
|
}
|
|
2605
|
-
function adoptPiFallbackMessageTag(db, sessionId, tagNumber, oldFallbackMessageId, newRealMessageId, servedTagNumbers = new Set) {
|
|
3549
|
+
function adoptPiFallbackMessageTag(db, sessionId, tagNumber, oldFallbackMessageId, newRealMessageId, servedTagNumbers = new Set, cachedSurvivor, rebuildingSurvivor) {
|
|
2606
3550
|
if (oldFallbackMessageId.startsWith(WHITESPACE_ASSISTANT_INERT_MESSAGE_PREFIX)) {
|
|
2607
3551
|
return { action: "skipped" };
|
|
2608
3552
|
}
|
|
@@ -2620,18 +3564,13 @@ function adoptPiFallbackMessageTag(db, sessionId, tagNumber, oldFallbackMessageI
|
|
|
2620
3564
|
AND message_id = ?`).run(newRealMessageId, sessionId, tagNumber, oldFallbackMessageId);
|
|
2621
3565
|
return (result.changes ?? 0) > 0 ? { action: "rekeyed", tagNumber } : { action: "skipped" };
|
|
2622
3566
|
}
|
|
2623
|
-
const
|
|
2624
|
-
|
|
2625
|
-
throw new Error("Conflicting served Pi message tag numbers; refusing identity adoption");
|
|
2626
|
-
}
|
|
2627
|
-
const realSurvivor = servedRows[0] ?? duplicates[0];
|
|
2628
|
-
if (!realSurvivor)
|
|
2629
|
-
return { action: "skipped" };
|
|
3567
|
+
const messageRows = [survivor, ...duplicates];
|
|
3568
|
+
const realSurvivor = selectPiTagSurvivor(messageRows, servedTagNumbers, cachedSurvivor, "message", rebuildingSurvivor);
|
|
2630
3569
|
const deletedTagNumbers = [];
|
|
2631
|
-
for (const duplicate of
|
|
3570
|
+
for (const duplicate of messageRows) {
|
|
2632
3571
|
if (duplicate.tagNumber === realSurvivor.tagNumber)
|
|
2633
3572
|
continue;
|
|
2634
|
-
|
|
3573
|
+
foldPiIdentityDuplicate(db, sessionId, realSurvivor, duplicate);
|
|
2635
3574
|
deletedTagNumbers.push(duplicate.tagNumber);
|
|
2636
3575
|
}
|
|
2637
3576
|
if (realSurvivor.tagNumber === tagNumber) {
|
|
@@ -2774,7 +3713,7 @@ function getNewestToolTagNumbers(db, sessionId, limit) {
|
|
|
2774
3713
|
let statement = newestToolTagNumberStatements.get(db);
|
|
2775
3714
|
if (!statement) {
|
|
2776
3715
|
statement = db.prepare(`SELECT tag_number FROM tags
|
|
2777
|
-
WHERE session_id = ? AND type = 'tool'
|
|
3716
|
+
WHERE session_id = ? AND +type = 'tool'
|
|
2778
3717
|
AND typeof(message_id) = 'text'
|
|
2779
3718
|
AND typeof(status) = 'text'
|
|
2780
3719
|
AND typeof(byte_size) IN ('integer', 'real')
|
|
@@ -3409,88 +4348,8 @@ function claimOpenCodeDbDiagnosticOnce(surface, resolution) {
|
|
|
3409
4348
|
return true;
|
|
3410
4349
|
}
|
|
3411
4350
|
|
|
3412
|
-
// ../plugin/src/hooks/magic-context/read-session-db.ts
|
|
3413
|
-
function openCodeDbExists() {
|
|
3414
|
-
return harnessOwnsOpenCodeStore() && openCodeDbPathExists(resolveOpenCodeDbPath());
|
|
3415
|
-
}
|
|
3416
|
-
var cachedReadOnlyDb = null;
|
|
3417
|
-
function closeCachedReadOnlyDb() {
|
|
3418
|
-
if (!cachedReadOnlyDb) {
|
|
3419
|
-
return;
|
|
3420
|
-
}
|
|
3421
|
-
try {
|
|
3422
|
-
closeQuietly(cachedReadOnlyDb.db);
|
|
3423
|
-
} catch (error) {
|
|
3424
|
-
log("[magic-context] failed to close cached OpenCode read-only DB:", error);
|
|
3425
|
-
} finally {
|
|
3426
|
-
cachedReadOnlyDb = null;
|
|
3427
|
-
}
|
|
3428
|
-
}
|
|
3429
|
-
function getReadOnlySessionDb() {
|
|
3430
|
-
if (!harnessOwnsOpenCodeStore()) {
|
|
3431
|
-
throw new Error("OpenCode session database is not readable from a Pi-compatible process; its history lives in Pi sessions");
|
|
3432
|
-
}
|
|
3433
|
-
const resolution = resolveOpenCodeDbPath();
|
|
3434
|
-
const dbPath = resolution.path;
|
|
3435
|
-
if (!openCodeDbPathExists(resolution)) {
|
|
3436
|
-
throw new Error(`OpenCode session database is unavailable at ${dbPath} (source=${resolution.source})`);
|
|
3437
|
-
}
|
|
3438
|
-
if (cachedReadOnlyDb?.path === dbPath) {
|
|
3439
|
-
return cachedReadOnlyDb.db;
|
|
3440
|
-
}
|
|
3441
|
-
closeCachedReadOnlyDb();
|
|
3442
|
-
const db = new Database(dbPath, { readonly: true });
|
|
3443
|
-
try {
|
|
3444
|
-
assertOpenCodeStoreGeneration(db, "v1", dbPath);
|
|
3445
|
-
} catch (error) {
|
|
3446
|
-
closeQuietly(db);
|
|
3447
|
-
throw error;
|
|
3448
|
-
}
|
|
3449
|
-
cachedReadOnlyDb = { path: dbPath, db };
|
|
3450
|
-
clearOpenCodeDbReadFailure();
|
|
3451
|
-
return db;
|
|
3452
|
-
}
|
|
3453
|
-
function withReadOnlySessionDb(fn) {
|
|
3454
|
-
return fn(getReadOnlySessionDb());
|
|
3455
|
-
}
|
|
3456
|
-
function getRawSessionMessageCountFromDb(db, sessionId) {
|
|
3457
|
-
const row = db.prepare(`SELECT COUNT(*) as count FROM message WHERE session_id = ?
|
|
3458
|
-
AND NOT (COALESCE(json_extract(data, '$.summary'), 0) = 1
|
|
3459
|
-
AND COALESCE(json_extract(data, '$.finish'), '') = 'stop')`).get(sessionId);
|
|
3460
|
-
return typeof row?.count === "number" ? row.count : 0;
|
|
3461
|
-
}
|
|
3462
|
-
var trackedSessions = new Map;
|
|
3463
|
-
var pendingParts = new Map;
|
|
3464
|
-
var probeLogObserverForTests;
|
|
3465
|
-
function logProbeFailureOnce(resolution, error) {
|
|
3466
|
-
const failure = recordOpenCodeDbReadFailure(resolution, error);
|
|
3467
|
-
if (!claimOpenCodeDbDiagnosticOnce("session-state-probe", resolution))
|
|
3468
|
-
return;
|
|
3469
|
-
const message = `[magic-context] OpenCode DB probe failed: path=${resolution.path} source=${resolution.source} cause=${failure.message}`;
|
|
3470
|
-
probeLogObserverForTests?.(message);
|
|
3471
|
-
log(message);
|
|
3472
|
-
}
|
|
3473
|
-
function getMessageTimesFromOpenCodeDb(sessionId, messageIds) {
|
|
3474
|
-
const result = new Map;
|
|
3475
|
-
if (messageIds.length === 0 || !harnessOwnsOpenCodeStore())
|
|
3476
|
-
return result;
|
|
3477
|
-
try {
|
|
3478
|
-
withReadOnlySessionDb((db) => {
|
|
3479
|
-
const placeholders = messageIds.map(() => "?").join(",");
|
|
3480
|
-
const rows = db.prepare(`SELECT id, time_created FROM message WHERE session_id = ? AND id IN (${placeholders})`).all(sessionId, ...messageIds);
|
|
3481
|
-
for (const row of rows) {
|
|
3482
|
-
if (typeof row.id === "string" && typeof row.time_created === "number") {
|
|
3483
|
-
result.set(row.id, row.time_created);
|
|
3484
|
-
}
|
|
3485
|
-
}
|
|
3486
|
-
});
|
|
3487
|
-
} catch (error) {
|
|
3488
|
-
logProbeFailureOnce(resolveOpenCodeDbPath(), error);
|
|
3489
|
-
}
|
|
3490
|
-
return result;
|
|
3491
|
-
}
|
|
3492
|
-
|
|
3493
4351
|
// ../plugin/src/hooks/magic-context/read-session-raw.ts
|
|
4352
|
+
import { isMainThread } from "node:worker_threads";
|
|
3494
4353
|
var RAW_MESSAGE_PARTS_BY_ID_SQL = "SELECT message_id, data, time_updated FROM part WHERE +session_id = ? AND likelihood(message_id = ?, 0.000001) ORDER BY time_created ASC, id ASC";
|
|
3495
4354
|
function isRawMessageRow(row) {
|
|
3496
4355
|
if (row === null || typeof row !== "object")
|
|
@@ -3521,6 +4380,25 @@ function isRawCompactionSummaryInfo(info) {
|
|
|
3521
4380
|
const candidate = info;
|
|
3522
4381
|
return candidate.summary === true && candidate.finish === "stop";
|
|
3523
4382
|
}
|
|
4383
|
+
var RAW_METADATA_KEEP_CHARS = 16384;
|
|
4384
|
+
var RAW_METADATA_KEPT_KEYS = [
|
|
4385
|
+
"description",
|
|
4386
|
+
"title",
|
|
4387
|
+
"userAnswer",
|
|
4388
|
+
"answers",
|
|
4389
|
+
"answer",
|
|
4390
|
+
"selectedOptions",
|
|
4391
|
+
"customInput",
|
|
4392
|
+
"results"
|
|
4393
|
+
];
|
|
4394
|
+
var RAW_PART_DATA_SQL = `CASE
|
|
4395
|
+
WHEN json_valid(data) = 1
|
|
4396
|
+
AND json_type(data, '$.state.metadata') = 'object'
|
|
4397
|
+
AND COALESCE(json_extract(data, '$.state.status'), '') <> 'error'
|
|
4398
|
+
AND length(json_extract(data, '$.state.metadata')) > ${RAW_METADATA_KEEP_CHARS}
|
|
4399
|
+
THEN json_set(data, '$.state.metadata', json_patch('{}', json_object(${RAW_METADATA_KEPT_KEYS.map((key) => `'${key}', json_extract(data, '$.state.metadata.${key}')`).join(", ")})))
|
|
4400
|
+
ELSE data
|
|
4401
|
+
END`;
|
|
3524
4402
|
function parseJsonUnknown(value) {
|
|
3525
4403
|
try {
|
|
3526
4404
|
return JSON.parse(value);
|
|
@@ -3551,7 +4429,7 @@ function readRawSessionMessagesFromDb(db, sessionId) {
|
|
|
3551
4429
|
if (messageIds.length === 0)
|
|
3552
4430
|
continue;
|
|
3553
4431
|
const placeholders = messageIds.map(() => "?").join(", ");
|
|
3554
|
-
const partRows = db.prepare(`SELECT message_id, data, time_updated
|
|
4432
|
+
const partRows = db.prepare(`SELECT message_id, ${RAW_PART_DATA_SQL} AS data, time_updated
|
|
3555
4433
|
FROM part
|
|
3556
4434
|
WHERE +session_id = ?
|
|
3557
4435
|
AND likelihood(message_id IN (${placeholders}), 0.000001)
|
|
@@ -3583,7 +4461,7 @@ function readRawSessionMessagePageFromDb(db, sessionId, afterOrdinal, limit, fin
|
|
|
3583
4461
|
if (messageRows.length === 0)
|
|
3584
4462
|
return [];
|
|
3585
4463
|
const placeholders = messageRows.map(() => "?").join(", ");
|
|
3586
|
-
const partRows = db.prepare(`SELECT message_id, data, time_updated
|
|
4464
|
+
const partRows = db.prepare(`SELECT message_id, ${RAW_PART_DATA_SQL} AS data, time_updated
|
|
3587
4465
|
FROM part
|
|
3588
4466
|
WHERE +session_id = ?
|
|
3589
4467
|
AND likelihood(message_id IN (${placeholders}), 0.000001)
|
|
@@ -3591,6 +4469,65 @@ function readRawSessionMessagePageFromDb(db, sessionId, afterOrdinal, limit, fin
|
|
|
3591
4469
|
return assembleRawMessagePage(messageRows, partRows);
|
|
3592
4470
|
}
|
|
3593
4471
|
function readRawMessagePageRows(db, sessionId, afterOrdinal, limit, finalWatermark, after) {
|
|
4472
|
+
try {
|
|
4473
|
+
return readRawMessagePageRowsBounded(db, sessionId, afterOrdinal, limit, finalWatermark, after);
|
|
4474
|
+
} catch (error) {
|
|
4475
|
+
if (error !== ORDINALS_COLD)
|
|
4476
|
+
throw error;
|
|
4477
|
+
if (warmRawSessionOrdinalsForDb(db, sessionId)) {
|
|
4478
|
+
try {
|
|
4479
|
+
return readRawMessagePageRowsBounded(db, sessionId, afterOrdinal, limit, finalWatermark, after);
|
|
4480
|
+
} catch (retry) {
|
|
4481
|
+
if (retry !== ORDINALS_COLD)
|
|
4482
|
+
throw retry;
|
|
4483
|
+
}
|
|
4484
|
+
}
|
|
4485
|
+
return readRawMessagePageRowsOffset(db, sessionId, afterOrdinal, limit, finalWatermark, after);
|
|
4486
|
+
}
|
|
4487
|
+
}
|
|
4488
|
+
var ORDINALS_COLD = Symbol("mmcCanonicalOrdinalsCold");
|
|
4489
|
+
function readRawMessagePageRowsBounded(db, sessionId, afterOrdinal, limit, finalWatermark, after) {
|
|
4490
|
+
const remaining = Math.max(0, Math.floor(finalWatermark) - Math.floor(afterOrdinal));
|
|
4491
|
+
const pageSize = Math.min(Math.max(1, Math.floor(limit)), remaining);
|
|
4492
|
+
if (pageSize === 0)
|
|
4493
|
+
return [];
|
|
4494
|
+
return withReadSnapshot(db, () => {
|
|
4495
|
+
const ordinal = Math.max(0, Math.floor(afterOrdinal));
|
|
4496
|
+
const points = currentOrdinalCheckpoints(db, sessionId)?.scan.checkpoints ?? [];
|
|
4497
|
+
let low = 0;
|
|
4498
|
+
let high = points.length;
|
|
4499
|
+
while (low < high) {
|
|
4500
|
+
const middle = Math.floor((low + high) / 2);
|
|
4501
|
+
if (points[middle].ordinal <= ordinal)
|
|
4502
|
+
low = middle + 1;
|
|
4503
|
+
else
|
|
4504
|
+
high = middle;
|
|
4505
|
+
}
|
|
4506
|
+
const floor = points[low - 1];
|
|
4507
|
+
const useFloor = floor && (!after || compareOrdinalKeys(floor, after) > 0);
|
|
4508
|
+
const anchor = useFloor ? floor : after;
|
|
4509
|
+
const skip = anchor === after && after ? 0 : ordinal - (floor?.ordinal ?? 0);
|
|
4510
|
+
if (skip > ORDINAL_LOOKUP_LIMIT)
|
|
4511
|
+
throw ORDINALS_COLD;
|
|
4512
|
+
const rawLimit = skip + pageSize + ORDINAL_LOOKUP_LIMIT;
|
|
4513
|
+
const rows = db.prepare(`SELECT id, data, time_created, time_updated, ${EXCLUDED_ROW_SQL} AS excluded
|
|
4514
|
+
FROM message WHERE id IN (
|
|
4515
|
+
SELECT id FROM message WHERE session_id = ?
|
|
4516
|
+
${anchor ? "AND (time_created, id) > (?, ?)" : ""}
|
|
4517
|
+
ORDER BY time_created, id LIMIT ?)
|
|
4518
|
+
ORDER BY time_created, id`).all(sessionId, ...anchor ? [anchor.timeCreated, anchor.id] : [], rawLimit);
|
|
4519
|
+
rawMessagePageRowsVisited += rows.length;
|
|
4520
|
+
const eligible = rows.filter((row) => row.excluded !== 1);
|
|
4521
|
+
if (eligible.length <= skip && rows.length === rawLimit)
|
|
4522
|
+
throw ORDINALS_COLD;
|
|
4523
|
+
return eligible.slice(skip, skip + pageSize).filter(isRawMessageRow).map((row, index) => ({
|
|
4524
|
+
...row,
|
|
4525
|
+
ordinal: ordinal + index + 1
|
|
4526
|
+
}));
|
|
4527
|
+
});
|
|
4528
|
+
}
|
|
4529
|
+
var rawMessagePageRowsVisited = 0;
|
|
4530
|
+
function readRawMessagePageRowsOffset(db, sessionId, afterOrdinal, limit, finalWatermark, after) {
|
|
3594
4531
|
const remaining = Math.max(0, Math.floor(finalWatermark) - Math.floor(afterOrdinal));
|
|
3595
4532
|
const pageSize = Math.min(Math.max(1, Math.floor(limit)), remaining);
|
|
3596
4533
|
if (pageSize === 0)
|
|
@@ -3682,6 +4619,9 @@ function readRawSessionMessageSummaryPageFromDb(db, sessionId, afterOrdinal, lim
|
|
|
3682
4619
|
return assembleRawMessagePage(messageRows, partRows);
|
|
3683
4620
|
}
|
|
3684
4621
|
function countRawSessionMessageOrdinalsFromDb(db, sessionId) {
|
|
4622
|
+
const indexed = countCanonicalOrdinalsIndexed(db, sessionId, null, null);
|
|
4623
|
+
if (indexed !== null)
|
|
4624
|
+
return indexed;
|
|
3685
4625
|
const row = db.prepare(`SELECT COUNT(*) AS count
|
|
3686
4626
|
FROM message
|
|
3687
4627
|
WHERE session_id = ?
|
|
@@ -3740,7 +4680,7 @@ function readRawSessionTailFromDb(db, sessionId, baseOrdinal, anchorMessageId) {
|
|
|
3740
4680
|
for (let i = 0;i < ids.length; i += CHUNK) {
|
|
3741
4681
|
const slice = ids.slice(i, i + CHUNK);
|
|
3742
4682
|
const placeholders = slice.map(() => "?").join(",");
|
|
3743
|
-
const partRows = db.prepare(`SELECT message_id, data, time_updated FROM part WHERE +session_id = ? AND likelihood(message_id IN (${placeholders}), 0.000001) ORDER BY time_created ASC, id ASC`).all(sessionId, ...slice).filter(isRawPartRow);
|
|
4683
|
+
const partRows = db.prepare(`SELECT message_id, ${RAW_PART_DATA_SQL} AS data, time_updated FROM part WHERE +session_id = ? AND likelihood(message_id IN (${placeholders}), 0.000001) ORDER BY time_created ASC, id ASC`).all(sessionId, ...slice).filter(isRawPartRow);
|
|
3744
4684
|
for (const part of partRows) {
|
|
3745
4685
|
const list = partsByMessageId.get(part.message_id) ?? [];
|
|
3746
4686
|
list.push(attachRawPartVersion(parseJsonUnknown(part.data), part.time_updated));
|
|
@@ -3768,49 +4708,491 @@ function readRawSessionTailFromDb(db, sessionId, baseOrdinal, anchorMessageId) {
|
|
|
3768
4708
|
}
|
|
3769
4709
|
return { messages, absoluteMessageCount: Math.max(0, ord - 1) };
|
|
3770
4710
|
}
|
|
3771
|
-
|
|
3772
|
-
|
|
3773
|
-
|
|
3774
|
-
|
|
3775
|
-
|
|
3776
|
-
|
|
3777
|
-
|
|
3778
|
-
|
|
3779
|
-
|
|
3780
|
-
|
|
3781
|
-
|
|
3782
|
-
|
|
3783
|
-
)
|
|
3784
|
-
AND (
|
|
3785
|
-
|
|
3786
|
-
|
|
3787
|
-
|
|
3788
|
-
|
|
3789
|
-
|
|
3790
|
-
|
|
3791
|
-
|
|
3792
|
-
|
|
3793
|
-
|
|
3794
|
-
|
|
3795
|
-
|
|
3796
|
-
|
|
3797
|
-
|
|
4711
|
+
var ORDINAL_SUMMARY_FLAG_SQL = `(CASE WHEN json_valid(data) = 1
|
|
4712
|
+
THEN COALESCE(json_extract(data, '$.summary'), 0)
|
|
4713
|
+
ELSE 0 END = 1)`;
|
|
4714
|
+
var ORDINAL_STOP_FLAG_SQL = `(CASE WHEN json_valid(data) = 1
|
|
4715
|
+
THEN COALESCE(json_extract(data, '$.finish'), '')
|
|
4716
|
+
ELSE '' END = 'stop')`;
|
|
4717
|
+
function ordinalRangeSql(lowerBound) {
|
|
4718
|
+
return `SELECT COUNT(*) AS visited,
|
|
4719
|
+
COALESCE(SUM(CASE WHEN ${ORDINAL_SUMMARY_FLAG_SQL} AND ${ORDINAL_STOP_FLAG_SQL}
|
|
4720
|
+
THEN 0 ELSE 1 END), 0) AS eligible
|
|
4721
|
+
FROM message
|
|
4722
|
+
WHERE session_id = ?
|
|
4723
|
+
${lowerBound ? "AND (time_created, id) > (?, ?)" : ""}
|
|
4724
|
+
AND (time_created, id) <= (?, ?)`;
|
|
4725
|
+
}
|
|
4726
|
+
var ORDINAL_RANGE_FROM_START_SQL = ordinalRangeSql(false);
|
|
4727
|
+
var ORDINAL_RANGE_BETWEEN_SQL = ordinalRangeSql(true);
|
|
4728
|
+
function readStoreStamp(db) {
|
|
4729
|
+
try {
|
|
4730
|
+
const row = db.prepare(`SELECT data_version AS dataVersion, total_changes() AS totalChanges,
|
|
4731
|
+
(SELECT schema_version FROM pragma_schema_version) AS schemaVersion
|
|
4732
|
+
FROM pragma_data_version`).get();
|
|
4733
|
+
if (typeof row?.dataVersion !== "number" || typeof row.totalChanges !== "number" || typeof row.schemaVersion !== "number")
|
|
4734
|
+
return null;
|
|
4735
|
+
return {
|
|
4736
|
+
dataVersion: row.dataVersion,
|
|
4737
|
+
totalChanges: row.totalChanges,
|
|
4738
|
+
schemaVersion: row.schemaVersion
|
|
4739
|
+
};
|
|
4740
|
+
} catch {
|
|
4741
|
+
return null;
|
|
4742
|
+
}
|
|
3798
4743
|
}
|
|
3799
|
-
function
|
|
3800
|
-
|
|
3801
|
-
|
|
4744
|
+
function sameStoreStamp(left, right) {
|
|
4745
|
+
return left !== null && right !== null && left.dataVersion === right.dataVersion && left.totalChanges === right.totalChanges && left.schemaVersion === right.schemaVersion;
|
|
4746
|
+
}
|
|
4747
|
+
var ordinalWatermarks = new WeakMap;
|
|
4748
|
+
var ordinalWatermarkEpochs = new Map;
|
|
4749
|
+
var ordinalRowsVisited = 0;
|
|
4750
|
+
var STABLE_STORE_ATTEMPTS = 3;
|
|
4751
|
+
function compareOrdinalKeys(left, right) {
|
|
4752
|
+
if (left.timeCreated !== right.timeCreated)
|
|
4753
|
+
return left.timeCreated < right.timeCreated ? -1 : 1;
|
|
4754
|
+
if (left.id === right.id)
|
|
4755
|
+
return 0;
|
|
4756
|
+
return Buffer.compare(Buffer.from(left.id, "utf8"), Buffer.from(right.id, "utf8"));
|
|
4757
|
+
}
|
|
4758
|
+
function countEligibleOrdinals(db, sessionId, after, through) {
|
|
4759
|
+
const indexed = countCanonicalOrdinalsIndexed(db, sessionId, after, through);
|
|
4760
|
+
if (indexed !== null)
|
|
4761
|
+
return indexed;
|
|
4762
|
+
const row = after ? db.prepare(ORDINAL_RANGE_BETWEEN_SQL).get(sessionId, after.timeCreated, after.id, through.timeCreated, through.id) : db.prepare(ORDINAL_RANGE_FROM_START_SQL).get(sessionId, through.timeCreated, through.id);
|
|
4763
|
+
const visited = typeof row?.visited === "number" ? row.visited : 0;
|
|
4764
|
+
ordinalRowsVisited += visited;
|
|
4765
|
+
ordinalJsonRowsRead += visited;
|
|
4766
|
+
return typeof row?.eligible === "number" ? row.eligible : 0;
|
|
4767
|
+
}
|
|
4768
|
+
var EXCLUDED_ROW_SQL = `${ORDINAL_SUMMARY_FLAG_SQL} AND ${ORDINAL_STOP_FLAG_SQL}`;
|
|
4769
|
+
var SUMMARY_ANCHOR_SPAN = 256;
|
|
4770
|
+
var SUMMARY_ANCHOR_LIMIT = 64;
|
|
4771
|
+
var summaryCandidateSets = new WeakMap;
|
|
4772
|
+
var summaryCandidateEpochs = new Map;
|
|
4773
|
+
var writtenSummaryIds = new Map;
|
|
4774
|
+
var indexedCountUnsupported = new WeakSet;
|
|
4775
|
+
var ordinalJsonRowsRead = 0;
|
|
4776
|
+
var summaryFullScans = 0;
|
|
4777
|
+
var ANCHOR_ROWS_SQL = `SELECT rowid AS rid, id, 1 AS kind FROM message
|
|
4778
|
+
WHERE rowid > (SELECT MAX(rowid) FROM message) - ${SUMMARY_ANCHOR_SPAN}`;
|
|
4779
|
+
var SUMMARY_FULL_SCAN_SQL = `${ANCHOR_ROWS_SQL}
|
|
4780
|
+
UNION ALL
|
|
4781
|
+
SELECT rowid, id, 2 FROM message WHERE session_id = ? AND ${ORDINAL_SUMMARY_FLAG_SQL}
|
|
4782
|
+
UNION ALL
|
|
4783
|
+
SELECT COUNT(*), NULL, 4 FROM message WHERE session_id = ?`;
|
|
4784
|
+
var SUMMARY_SCAN_AFTER_SQL = `${ANCHOR_ROWS_SQL}
|
|
4785
|
+
UNION ALL
|
|
4786
|
+
SELECT rowid, id, CASE WHEN ${ORDINAL_SUMMARY_FLAG_SQL} THEN 2 ELSE 3 END
|
|
4787
|
+
FROM message WHERE rowid > ? AND +session_id = ?`;
|
|
4788
|
+
var ANCHOR_CHECK_SQL = "SELECT rowid AS rid, id FROM message WHERE rowid IN (SELECT value FROM json_each(?))";
|
|
4789
|
+
function survivingAnchor(db, set) {
|
|
4790
|
+
if (set.anchors.length === 0)
|
|
3802
4791
|
return null;
|
|
4792
|
+
const current = new Map(db.prepare(ANCHOR_CHECK_SQL).all(JSON.stringify(set.anchors.map((anchor) => anchor.rowid))).map((row) => [row.rid, row.id]));
|
|
4793
|
+
return set.anchors.find((anchor) => current.get(anchor.rowid) === anchor.id);
|
|
4794
|
+
}
|
|
4795
|
+
var ordinalCheckpoints = new WeakMap;
|
|
4796
|
+
var ordinalSummarySeeds = new WeakMap;
|
|
4797
|
+
var ORDINAL_LOOKUP_LIMIT = 512;
|
|
4798
|
+
function currentOrdinalCheckpoints(db, sessionId) {
|
|
4799
|
+
const cached = ordinalCheckpoints.get(db)?.get(sessionId);
|
|
4800
|
+
if (!cached)
|
|
4801
|
+
return;
|
|
4802
|
+
return withReadSnapshot(db, () => {
|
|
4803
|
+
if (!extendOrdinalCheckpoints(db, sessionId, cached.scan, ORDINAL_LOOKUP_LIMIT))
|
|
4804
|
+
return;
|
|
4805
|
+
cached.epoch = getRawSessionSummaryEpoch(sessionId);
|
|
4806
|
+
return cached;
|
|
4807
|
+
});
|
|
4808
|
+
}
|
|
4809
|
+
function extendOrdinalCheckpoints(db, sessionId, scan, limit) {
|
|
4810
|
+
const points = scan.checkpoints ?? [];
|
|
4811
|
+
const exists = (point) => Boolean(db.prepare("SELECT 1 FROM message WHERE id=? AND session_id=? AND time_created=?").get(point.id, sessionId, point.timeCreated));
|
|
4812
|
+
if (points.length && !exists(points[points.length - 1])) {
|
|
4813
|
+
let low = 0;
|
|
4814
|
+
let high = points.length;
|
|
4815
|
+
while (low < high) {
|
|
4816
|
+
const middle = Math.floor((low + high) / 2);
|
|
4817
|
+
if (exists(points[middle]))
|
|
4818
|
+
low = middle + 1;
|
|
4819
|
+
else
|
|
4820
|
+
high = middle;
|
|
4821
|
+
}
|
|
4822
|
+
points.length = low;
|
|
4823
|
+
scan.total = points.at(-1)?.ordinal ?? 0;
|
|
4824
|
+
scan.sessionRows = points.at(-1)?.storedCount ?? 0;
|
|
4825
|
+
}
|
|
4826
|
+
const tail = points.at(-1);
|
|
4827
|
+
const rows = db.prepare(`SELECT id, time_created AS timeCreated, ${EXCLUDED_ROW_SQL} AS excluded FROM message
|
|
4828
|
+
WHERE session_id=?${tail ? " AND (time_created,id) > (?,?)" : ""}
|
|
4829
|
+
ORDER BY time_created,id${limit === undefined ? "" : ` LIMIT ${limit + 1}`}`).all(sessionId, ...tail ? [tail.timeCreated, tail.id] : []);
|
|
4830
|
+
ordinalJsonRowsRead += rows.length;
|
|
4831
|
+
if (limit !== undefined && rows.length > limit)
|
|
4832
|
+
return false;
|
|
4833
|
+
let total = scan.total ?? 0;
|
|
4834
|
+
if (rows.length && tail && (tail.storedCount ?? 0) % 256 !== 0)
|
|
4835
|
+
points.pop();
|
|
4836
|
+
for (let index = 0;index < rows.length; index++) {
|
|
4837
|
+
const row = rows[index];
|
|
4838
|
+
if (row.excluded !== 1)
|
|
4839
|
+
total++;
|
|
4840
|
+
const storedCount = scan.sessionRows + index + 1;
|
|
4841
|
+
if (storedCount % 256 === 0 || index === rows.length - 1)
|
|
4842
|
+
points.push({ id: row.id, timeCreated: row.timeCreated, ordinal: total, storedCount });
|
|
4843
|
+
}
|
|
4844
|
+
scan.checkpoints = points;
|
|
4845
|
+
scan.sessionRows += rows.length;
|
|
4846
|
+
scan.total = total;
|
|
4847
|
+
return true;
|
|
4848
|
+
}
|
|
4849
|
+
function warmRawSessionOrdinalsForDb(db, sessionId) {
|
|
4850
|
+
if (isRawSessionSummaryWarm(db, sessionId))
|
|
4851
|
+
return true;
|
|
4852
|
+
try {
|
|
4853
|
+
const epoch = getRawSessionSummaryEpoch(sessionId);
|
|
4854
|
+
const scan = scanRawSessionSummaryRows(db, sessionId);
|
|
4855
|
+
installRawSessionSummaryScan(db, sessionId, scan, epoch, true);
|
|
4856
|
+
} catch {
|
|
4857
|
+
return false;
|
|
3803
4858
|
}
|
|
3804
|
-
|
|
3805
|
-
|
|
4859
|
+
return isRawSessionSummaryWarm(db, sessionId);
|
|
4860
|
+
}
|
|
4861
|
+
function boundedOrdinalPrefix(db, sessionId, through) {
|
|
4862
|
+
try {
|
|
4863
|
+
return boundedOrdinalPrefixBounded(db, sessionId, through);
|
|
4864
|
+
} catch (error) {
|
|
4865
|
+
if (error !== ORDINALS_COLD)
|
|
4866
|
+
throw error;
|
|
4867
|
+
if (warmRawSessionOrdinalsForDb(db, sessionId)) {
|
|
4868
|
+
try {
|
|
4869
|
+
return boundedOrdinalPrefixBounded(db, sessionId, through);
|
|
4870
|
+
} catch (retry) {
|
|
4871
|
+
if (retry !== ORDINALS_COLD)
|
|
4872
|
+
throw retry;
|
|
4873
|
+
}
|
|
4874
|
+
}
|
|
4875
|
+
const bindings = [sessionId];
|
|
4876
|
+
if (through)
|
|
4877
|
+
bindings.push(through.timeCreated, through.id);
|
|
4878
|
+
const rows = db.prepare(`SELECT ${EXCLUDED_ROW_SQL} AS excluded FROM message WHERE id IN (
|
|
4879
|
+
SELECT id FROM message WHERE session_id = ?${through ? " AND (time_created, id) <= (?, ?)" : ""}
|
|
4880
|
+
ORDER BY time_created, id)`).all(...bindings);
|
|
4881
|
+
ordinalRowsVisited += rows.length;
|
|
4882
|
+
ordinalJsonRowsRead += rows.length;
|
|
4883
|
+
return rows.reduce((sum, row) => sum + (row.excluded === 1 ? 0 : 1), 0);
|
|
4884
|
+
}
|
|
4885
|
+
}
|
|
4886
|
+
function boundedOrdinalPrefixBounded(db, sessionId, through) {
|
|
4887
|
+
const cached = currentOrdinalCheckpoints(db, sessionId);
|
|
4888
|
+
if (cached && through === null)
|
|
4889
|
+
return cached.scan.total ?? 0;
|
|
4890
|
+
let floor;
|
|
4891
|
+
if (cached && through) {
|
|
4892
|
+
const points = cached.scan.checkpoints ?? [];
|
|
4893
|
+
let low = 0;
|
|
4894
|
+
let high = points.length;
|
|
4895
|
+
while (low < high) {
|
|
4896
|
+
const middle = Math.floor((low + high) / 2);
|
|
4897
|
+
if (compareOrdinalKeys(points[middle], through) <= 0)
|
|
4898
|
+
low = middle + 1;
|
|
4899
|
+
else
|
|
4900
|
+
high = middle;
|
|
4901
|
+
}
|
|
4902
|
+
floor = points[low - 1];
|
|
4903
|
+
if (floor && compareOrdinalKeys(floor, through) === 0)
|
|
4904
|
+
return floor.ordinal;
|
|
4905
|
+
}
|
|
4906
|
+
const bindings = [sessionId];
|
|
4907
|
+
if (floor)
|
|
4908
|
+
bindings.push(floor.timeCreated, floor.id);
|
|
4909
|
+
if (through)
|
|
4910
|
+
bindings.push(through.timeCreated, through.id);
|
|
4911
|
+
const rows = db.prepare(`SELECT ${EXCLUDED_ROW_SQL} AS excluded FROM message WHERE id IN (
|
|
4912
|
+
SELECT id FROM message WHERE session_id = ?${floor ? " AND (time_created, id) > (?, ?)" : ""}${through ? " AND (time_created, id) <= (?, ?)" : ""}
|
|
4913
|
+
ORDER BY time_created, id LIMIT ${ORDINAL_LOOKUP_LIMIT + 1})`).all(...bindings);
|
|
4914
|
+
ordinalRowsVisited += rows.length;
|
|
4915
|
+
ordinalJsonRowsRead += rows.length;
|
|
4916
|
+
if (rows.length > ORDINAL_LOOKUP_LIMIT)
|
|
4917
|
+
throw ORDINALS_COLD;
|
|
4918
|
+
return (floor?.ordinal ?? 0) + rows.reduce((sum, row) => sum + (row.excluded === 1 ? 0 : 1), 0);
|
|
4919
|
+
}
|
|
4920
|
+
function scanRawSessionSummaryRows(db, sessionId, previous) {
|
|
4921
|
+
return withReadSnapshot(db, () => {
|
|
4922
|
+
if (previous?.checkpoints) {
|
|
4923
|
+
extendOrdinalCheckpoints(db, sessionId, previous);
|
|
4924
|
+
return previous;
|
|
4925
|
+
}
|
|
4926
|
+
if (previous) {
|
|
4927
|
+
const sessions = summaryCandidateSets.get(db) ?? new Map;
|
|
4928
|
+
sessions.set(sessionId, {
|
|
4929
|
+
epoch: getRawSessionSummaryEpoch(sessionId),
|
|
4930
|
+
anchors: previous.anchors,
|
|
4931
|
+
ids: new Set(previous.ids)
|
|
4932
|
+
});
|
|
4933
|
+
summaryCandidateSets.set(db, sessions);
|
|
4934
|
+
}
|
|
4935
|
+
const incremental = previous && isRawSessionSummaryWarm(db, sessionId) ? summaryCandidatesOf(db, sessionId) : undefined;
|
|
4936
|
+
const rows = incremental ? [] : db.prepare(SUMMARY_FULL_SCAN_SQL).all(sessionId, sessionId);
|
|
4937
|
+
const anchors = [];
|
|
4938
|
+
const ids = [];
|
|
4939
|
+
let sessionRows = 0;
|
|
4940
|
+
for (const row of rows) {
|
|
4941
|
+
if (row.kind === 1 && typeof row.rid === "number" && typeof row.id === "string")
|
|
4942
|
+
anchors.push({ rowid: row.rid, id: row.id });
|
|
4943
|
+
else if (row.kind === 2 && typeof row.id === "string")
|
|
4944
|
+
ids.push(row.id);
|
|
4945
|
+
else if (row.kind === 4 && typeof row.rid === "number")
|
|
4946
|
+
sessionRows = row.rid;
|
|
4947
|
+
}
|
|
4948
|
+
if (incremental) {
|
|
4949
|
+
ids.push(...incremental.ids);
|
|
4950
|
+
anchors.push(...summaryCandidateSets.get(db)?.get(sessionId)?.anchors ?? []);
|
|
4951
|
+
sessionRows = countStoredRawSessionRowsIndexed(db, sessionId);
|
|
4952
|
+
}
|
|
4953
|
+
anchors.sort((left, right) => right.rowid - left.rowid);
|
|
4954
|
+
anchors.length = Math.min(anchors.length, SUMMARY_ANCHOR_LIMIT);
|
|
4955
|
+
const keys = db.prepare(`SELECT id, time_created AS timeCreated, ${EXCLUDED_ROW_SQL} AS excluded FROM message WHERE session_id = ?
|
|
4956
|
+
ORDER BY time_created, id`).all(sessionId);
|
|
4957
|
+
const checkpoints = [];
|
|
4958
|
+
let total = 0;
|
|
4959
|
+
for (let index = 0;index < keys.length; index++) {
|
|
4960
|
+
const key = keys[index];
|
|
4961
|
+
if (key.excluded !== 1)
|
|
4962
|
+
total++;
|
|
4963
|
+
if (index % 256 === 255 || index === keys.length - 1)
|
|
4964
|
+
checkpoints.push({
|
|
4965
|
+
id: key.id,
|
|
4966
|
+
timeCreated: key.timeCreated,
|
|
4967
|
+
ordinal: total,
|
|
4968
|
+
storedCount: index + 1
|
|
4969
|
+
});
|
|
4970
|
+
}
|
|
4971
|
+
return { anchors, ids, sessionRows, checkpoints, total };
|
|
4972
|
+
});
|
|
4973
|
+
}
|
|
4974
|
+
function getRawSessionSummaryEpoch(sessionId) {
|
|
4975
|
+
return summaryCandidateEpochs.get(sessionId) ?? 0;
|
|
4976
|
+
}
|
|
4977
|
+
function installRawSessionSummaryScan(db, sessionId, scan, epoch, workerPrepared = false) {
|
|
4978
|
+
if (epoch !== getRawSessionSummaryEpoch(sessionId))
|
|
4979
|
+
return false;
|
|
4980
|
+
const seeds = ordinalSummarySeeds.get(db) ?? new Map;
|
|
4981
|
+
seeds.set(sessionId, { epoch, scan });
|
|
4982
|
+
ordinalSummarySeeds.set(db, seeds);
|
|
4983
|
+
if (scan.checkpoints && workerPrepared) {
|
|
4984
|
+
const counts = ordinalCheckpoints.get(db) ?? new Map;
|
|
4985
|
+
counts.set(sessionId, { epoch, scan });
|
|
4986
|
+
ordinalCheckpoints.set(db, counts);
|
|
4987
|
+
} else if (getSqliteDatabasePath(db) && isMainThread)
|
|
4988
|
+
return false;
|
|
4989
|
+
if (isRawSessionSummaryWarm(db, sessionId))
|
|
4990
|
+
return true;
|
|
4991
|
+
let sessions = summaryCandidateSets.get(db);
|
|
4992
|
+
if (!sessions) {
|
|
4993
|
+
sessions = new Map;
|
|
4994
|
+
summaryCandidateSets.set(db, sessions);
|
|
4995
|
+
}
|
|
4996
|
+
sessions.set(sessionId, { epoch, anchors: [...scan.anchors], ids: new Set(scan.ids) });
|
|
4997
|
+
return isRawSessionSummaryWarm(db, sessionId);
|
|
4998
|
+
}
|
|
4999
|
+
function isRawSessionSummaryWarm(db, sessionId) {
|
|
5000
|
+
if (isMainThread && getSqliteDatabasePath(db)) {
|
|
5001
|
+
return Boolean(currentOrdinalCheckpoints(db, sessionId));
|
|
5002
|
+
}
|
|
5003
|
+
const set = summaryCandidateSets.get(db)?.get(sessionId);
|
|
5004
|
+
if (!set || set.epoch !== getRawSessionSummaryEpoch(sessionId))
|
|
5005
|
+
return false;
|
|
5006
|
+
try {
|
|
5007
|
+
return survivingAnchor(db, set) !== undefined;
|
|
5008
|
+
} catch {
|
|
5009
|
+
return false;
|
|
5010
|
+
}
|
|
5011
|
+
}
|
|
5012
|
+
function countStoredRawSessionRowsIndexed(db, sessionId) {
|
|
5013
|
+
const row = db.prepare("SELECT COUNT(*) AS count FROM message WHERE session_id = ?").get(sessionId);
|
|
5014
|
+
return typeof row?.count === "number" ? row.count : 0;
|
|
5015
|
+
}
|
|
5016
|
+
function summaryCandidatesOf(db, sessionId) {
|
|
5017
|
+
let sessions = summaryCandidateSets.get(db);
|
|
5018
|
+
if (!sessions) {
|
|
5019
|
+
sessions = new Map;
|
|
5020
|
+
summaryCandidateSets.set(db, sessions);
|
|
5021
|
+
}
|
|
5022
|
+
const epoch = getRawSessionSummaryEpoch(sessionId);
|
|
5023
|
+
const cached = sessions.get(sessionId);
|
|
5024
|
+
const floor = cached && cached.epoch === epoch ? survivingAnchor(db, cached) : undefined;
|
|
5025
|
+
let ids;
|
|
5026
|
+
let anchors;
|
|
5027
|
+
if (floor === undefined) {
|
|
5028
|
+
const scan = scanRawSessionSummaryRows(db, sessionId);
|
|
5029
|
+
summaryFullScans += 1;
|
|
5030
|
+
ordinalJsonRowsRead += scan.sessionRows;
|
|
5031
|
+
ids = new Set(scan.ids);
|
|
5032
|
+
anchors = scan.anchors;
|
|
5033
|
+
} else {
|
|
5034
|
+
const rows = db.prepare(SUMMARY_SCAN_AFTER_SQL).all(floor?.rowid ?? 0, sessionId);
|
|
5035
|
+
ids = cached?.ids ?? new Set;
|
|
5036
|
+
anchors = [];
|
|
5037
|
+
for (const row of rows) {
|
|
5038
|
+
if (row.kind === 1 && typeof row.rid === "number" && typeof row.id === "string")
|
|
5039
|
+
anchors.push({ rowid: row.rid, id: row.id });
|
|
5040
|
+
else if (row.kind === 2 && typeof row.id === "string")
|
|
5041
|
+
ids.add(row.id);
|
|
5042
|
+
if (row.kind === 2 || row.kind === 3)
|
|
5043
|
+
ordinalJsonRowsRead += 1;
|
|
5044
|
+
}
|
|
5045
|
+
anchors.sort((left, right) => right.rowid - left.rowid);
|
|
5046
|
+
anchors.length = Math.min(anchors.length, SUMMARY_ANCHOR_LIMIT);
|
|
5047
|
+
}
|
|
5048
|
+
for (const id of writtenSummaryIds.get(sessionId) ?? [])
|
|
5049
|
+
ids.add(id);
|
|
5050
|
+
sessions.set(sessionId, { epoch, anchors, ids });
|
|
5051
|
+
const top = anchors[0];
|
|
5052
|
+
return { ids: [...ids], floorRowid: top?.rowid ?? 0, floorId: top?.id ?? null };
|
|
5053
|
+
}
|
|
5054
|
+
function canonicalCountSql(lower, upper) {
|
|
5055
|
+
const range = `${lower ? " AND (time_created, id) > (?, ?)" : ""}${upper ? " AND (time_created, id) <= (?, ?)" : ""}`;
|
|
5056
|
+
return `SELECT
|
|
5057
|
+
(SELECT COUNT(*) FROM message WHERE session_id = ?${range}) AS stored,
|
|
5058
|
+
(SELECT COUNT(*) FROM message
|
|
5059
|
+
WHERE id IN (SELECT value FROM json_each(?)) AND +session_id = ?${range}
|
|
5060
|
+
AND ${EXCLUDED_ROW_SQL}) AS knownExcluded,
|
|
5061
|
+
(SELECT COUNT(*) FROM message
|
|
5062
|
+
WHERE rowid > ? AND +session_id = ? AND id NOT IN (SELECT value FROM json_each(?))${range}
|
|
5063
|
+
AND ${EXCLUDED_ROW_SQL}) AS newExcluded,
|
|
5064
|
+
(SELECT id FROM message WHERE rowid = ?) AS floorId`;
|
|
5065
|
+
}
|
|
5066
|
+
var CANONICAL_COUNT_SQL = new Map;
|
|
5067
|
+
function canonicalCountStatementText(lower, upper) {
|
|
5068
|
+
const key = `${lower}:${upper}`;
|
|
5069
|
+
let sql = CANONICAL_COUNT_SQL.get(key);
|
|
5070
|
+
if (!sql) {
|
|
5071
|
+
sql = canonicalCountSql(lower, upper);
|
|
5072
|
+
CANONICAL_COUNT_SQL.set(key, sql);
|
|
5073
|
+
}
|
|
5074
|
+
return sql;
|
|
5075
|
+
}
|
|
5076
|
+
function countCanonicalOrdinalsIndexed(db, sessionId, after, through) {
|
|
5077
|
+
if (isMainThread && getSqliteDatabasePath(db)) {
|
|
5078
|
+
return withReadSnapshot(db, () => boundedOrdinalPrefix(db, sessionId, through) - (after ? boundedOrdinalPrefix(db, sessionId, after) : 0));
|
|
5079
|
+
}
|
|
5080
|
+
if (indexedCountUnsupported.has(db))
|
|
3806
5081
|
return null;
|
|
5082
|
+
const range = [];
|
|
5083
|
+
if (after)
|
|
5084
|
+
range.push(after.timeCreated, after.id);
|
|
5085
|
+
if (through)
|
|
5086
|
+
range.push(through.timeCreated, through.id);
|
|
5087
|
+
try {
|
|
5088
|
+
return withReadSnapshot(db, () => {
|
|
5089
|
+
for (let attempt = 0;attempt < STABLE_STORE_ATTEMPTS; attempt += 1) {
|
|
5090
|
+
const view = summaryCandidatesOf(db, sessionId);
|
|
5091
|
+
const ids = JSON.stringify(view.ids);
|
|
5092
|
+
const row = db.prepare(canonicalCountStatementText(after !== null, through !== null)).get(sessionId, ...range, ids, sessionId, ...range, view.floorRowid, sessionId, ids, ...range, view.floorRowid);
|
|
5093
|
+
if (view.floorId !== null && row?.floorId !== view.floorId)
|
|
5094
|
+
continue;
|
|
5095
|
+
const stored = typeof row?.stored === "number" ? row.stored : 0;
|
|
5096
|
+
const excluded = (typeof row?.knownExcluded === "number" ? row.knownExcluded : 0) + (typeof row?.newExcluded === "number" ? row.newExcluded : 0);
|
|
5097
|
+
ordinalRowsVisited += stored;
|
|
5098
|
+
ordinalJsonRowsRead += view.ids.length;
|
|
5099
|
+
return stored - excluded;
|
|
5100
|
+
}
|
|
5101
|
+
return null;
|
|
5102
|
+
});
|
|
5103
|
+
} catch (error) {
|
|
5104
|
+
if (/no such (column|table|function)|json_each/i.test(String(error))) {
|
|
5105
|
+
indexedCountUnsupported.add(db);
|
|
5106
|
+
return null;
|
|
5107
|
+
}
|
|
5108
|
+
throw error;
|
|
3807
5109
|
}
|
|
3808
|
-
|
|
3809
|
-
|
|
3810
|
-
|
|
3811
|
-
|
|
3812
|
-
|
|
3813
|
-
|
|
5110
|
+
}
|
|
5111
|
+
function connectionHasOpenTransaction(db) {
|
|
5112
|
+
const state = db;
|
|
5113
|
+
return state.inTransaction === true || state.isTransaction === true;
|
|
5114
|
+
}
|
|
5115
|
+
function withReadSnapshot(db, read) {
|
|
5116
|
+
if (connectionHasOpenTransaction(db))
|
|
5117
|
+
return read();
|
|
5118
|
+
db.exec("BEGIN");
|
|
5119
|
+
try {
|
|
5120
|
+
return read();
|
|
5121
|
+
} finally {
|
|
5122
|
+
if (connectionHasOpenTransaction(db))
|
|
5123
|
+
db.exec("ROLLBACK");
|
|
5124
|
+
}
|
|
5125
|
+
}
|
|
5126
|
+
function reusableWatermark(db, sessionId, stamp) {
|
|
5127
|
+
const watermark = ordinalWatermarks.get(db)?.get(sessionId);
|
|
5128
|
+
if (!watermark || watermark.epoch !== (ordinalWatermarkEpochs.get(sessionId) ?? 0) || !sameStoreStamp(watermark.stamp, stamp))
|
|
5129
|
+
return null;
|
|
5130
|
+
return watermark;
|
|
5131
|
+
}
|
|
5132
|
+
function rememberOrdinalWatermark(db, sessionId, key, ordinal, stamp) {
|
|
5133
|
+
let sessions = ordinalWatermarks.get(db);
|
|
5134
|
+
if (!sessions) {
|
|
5135
|
+
sessions = new Map;
|
|
5136
|
+
ordinalWatermarks.set(db, sessions);
|
|
5137
|
+
}
|
|
5138
|
+
const current = sessions.get(sessionId);
|
|
5139
|
+
if (current && sameStoreStamp(current.stamp, stamp) && compareOrdinalKeys(current, key) >= 0)
|
|
5140
|
+
return;
|
|
5141
|
+
sessions.set(sessionId, {
|
|
5142
|
+
timeCreated: key.timeCreated,
|
|
5143
|
+
id: key.id,
|
|
5144
|
+
ordinal,
|
|
5145
|
+
stamp,
|
|
5146
|
+
epoch: ordinalWatermarkEpochs.get(sessionId) ?? 0
|
|
5147
|
+
});
|
|
5148
|
+
}
|
|
5149
|
+
function canonicalOrdinalOf(db, sessionId, readTarget) {
|
|
5150
|
+
for (let attempt = 1;; attempt += 1) {
|
|
5151
|
+
const before = readStoreStamp(db);
|
|
5152
|
+
const target = readTarget();
|
|
5153
|
+
if (!target)
|
|
5154
|
+
return null;
|
|
5155
|
+
const watermark = attempt < STABLE_STORE_ATTEMPTS ? reusableWatermark(db, sessionId, before) : null;
|
|
5156
|
+
if (!watermark) {
|
|
5157
|
+
const ordinal = countEligibleOrdinals(db, sessionId, null, target);
|
|
5158
|
+
if (before && sameStoreStamp(before, readStoreStamp(db)))
|
|
5159
|
+
rememberOrdinalWatermark(db, sessionId, target, ordinal, before);
|
|
5160
|
+
return { target, ordinal };
|
|
5161
|
+
}
|
|
5162
|
+
const order = compareOrdinalKeys(target, watermark);
|
|
5163
|
+
const ordinal = order === 0 ? watermark.ordinal : order < 0 ? watermark.ordinal - countEligibleOrdinals(db, sessionId, target, watermark) : watermark.ordinal + countEligibleOrdinals(db, sessionId, watermark, target);
|
|
5164
|
+
if (sameStoreStamp(before, readStoreStamp(db))) {
|
|
5165
|
+
rememberOrdinalWatermark(db, sessionId, target, ordinal, before);
|
|
5166
|
+
return { target, ordinal };
|
|
5167
|
+
}
|
|
5168
|
+
}
|
|
5169
|
+
}
|
|
5170
|
+
function readRawSessionMessageOrdinalByIdFromDb(db, sessionId, messageId) {
|
|
5171
|
+
const counted = canonicalOrdinalOf(db, sessionId, () => {
|
|
5172
|
+
const row = db.prepare(`SELECT id, time_created, ${ORDINAL_SUMMARY_FLAG_SQL} AND ${ORDINAL_STOP_FLAG_SQL} AS excluded
|
|
5173
|
+
FROM message WHERE session_id = ? AND id = ?`).get(sessionId, messageId);
|
|
5174
|
+
if (typeof row?.id !== "string" || typeof row.time_created !== "number")
|
|
5175
|
+
return null;
|
|
5176
|
+
return { id: row.id, timeCreated: row.time_created, excluded: row.excluded === 1 };
|
|
5177
|
+
});
|
|
5178
|
+
if (!counted || counted.target.excluded)
|
|
5179
|
+
return null;
|
|
5180
|
+
return counted.ordinal > 0 ? counted.ordinal : null;
|
|
5181
|
+
}
|
|
5182
|
+
function readRawSessionMessageByIdFromDb(db, sessionId, messageId) {
|
|
5183
|
+
const counted = canonicalOrdinalOf(db, sessionId, () => {
|
|
5184
|
+
const row = db.prepare("SELECT id, data, time_created, time_updated FROM message WHERE session_id = ? AND id = ?").get(sessionId, messageId);
|
|
5185
|
+
if (!row || !isRawMessageRow(row) || typeof row.time_created !== "number")
|
|
5186
|
+
return null;
|
|
5187
|
+
const parsed = parseJsonRecord(row.data);
|
|
5188
|
+
if (!parsed || isRawCompactionSummaryInfo(parsed))
|
|
5189
|
+
return null;
|
|
5190
|
+
return { ...row, timeCreated: row.time_created, info: parsed };
|
|
5191
|
+
});
|
|
5192
|
+
if (!counted)
|
|
5193
|
+
return null;
|
|
5194
|
+
const { target: row, ordinal } = counted;
|
|
5195
|
+
const info = row.info;
|
|
3814
5196
|
if (ordinal <= 0) {
|
|
3815
5197
|
return null;
|
|
3816
5198
|
}
|
|
@@ -3836,6 +5218,88 @@ function hasRawSessionMessageByIdFromDb(db, sessionId, messageId) {
|
|
|
3836
5218
|
return readRawSessionMessageByIdFromDb(db, sessionId, messageId) !== null;
|
|
3837
5219
|
return true;
|
|
3838
5220
|
}
|
|
5221
|
+
var boundaryEndpointCaches = new WeakMap;
|
|
5222
|
+
|
|
5223
|
+
// ../plugin/src/hooks/magic-context/read-session-db.ts
|
|
5224
|
+
function openCodeDbExists() {
|
|
5225
|
+
return harnessOwnsOpenCodeStore() && openCodeDbPathExists(resolveOpenCodeDbPath());
|
|
5226
|
+
}
|
|
5227
|
+
var cachedReadOnlyDb = null;
|
|
5228
|
+
function closeCachedReadOnlyDb() {
|
|
5229
|
+
if (!cachedReadOnlyDb) {
|
|
5230
|
+
return;
|
|
5231
|
+
}
|
|
5232
|
+
try {
|
|
5233
|
+
closeQuietly(cachedReadOnlyDb.db);
|
|
5234
|
+
} catch (error) {
|
|
5235
|
+
log("[magic-context] failed to close cached OpenCode read-only DB:", error);
|
|
5236
|
+
} finally {
|
|
5237
|
+
cachedReadOnlyDb = null;
|
|
5238
|
+
}
|
|
5239
|
+
}
|
|
5240
|
+
function getReadOnlySessionDb() {
|
|
5241
|
+
if (!harnessOwnsOpenCodeStore()) {
|
|
5242
|
+
throw new Error("OpenCode session database is not readable from a Pi-compatible process; its history lives in Pi sessions");
|
|
5243
|
+
}
|
|
5244
|
+
const resolution = resolveOpenCodeDbPath();
|
|
5245
|
+
const dbPath = resolution.path;
|
|
5246
|
+
if (!openCodeDbPathExists(resolution)) {
|
|
5247
|
+
throw new Error(`OpenCode session database is unavailable at ${dbPath} (source=${resolution.source})`);
|
|
5248
|
+
}
|
|
5249
|
+
if (cachedReadOnlyDb?.path === dbPath) {
|
|
5250
|
+
return cachedReadOnlyDb.db;
|
|
5251
|
+
}
|
|
5252
|
+
closeCachedReadOnlyDb();
|
|
5253
|
+
const db = new Database(dbPath, { readonly: true });
|
|
5254
|
+
try {
|
|
5255
|
+
assertOpenCodeStoreGeneration(db, "v1", dbPath);
|
|
5256
|
+
} catch (error) {
|
|
5257
|
+
closeQuietly(db);
|
|
5258
|
+
throw error;
|
|
5259
|
+
}
|
|
5260
|
+
cachedReadOnlyDb = { path: dbPath, db };
|
|
5261
|
+
clearOpenCodeDbReadFailure();
|
|
5262
|
+
return db;
|
|
5263
|
+
}
|
|
5264
|
+
function withReadOnlySessionDb(fn) {
|
|
5265
|
+
return fn(getReadOnlySessionDb());
|
|
5266
|
+
}
|
|
5267
|
+
function getRawSessionMessageCountFromDb(db, sessionId) {
|
|
5268
|
+
const row = db.prepare(`SELECT COUNT(*) as count FROM message WHERE session_id = ?
|
|
5269
|
+
AND NOT (COALESCE(json_extract(data, '$.summary'), 0) = 1
|
|
5270
|
+
AND COALESCE(json_extract(data, '$.finish'), '') = 'stop')`).get(sessionId);
|
|
5271
|
+
return typeof row?.count === "number" ? row.count : 0;
|
|
5272
|
+
}
|
|
5273
|
+
var trackedSessions = new Map;
|
|
5274
|
+
var pendingParts = new Map;
|
|
5275
|
+
var probeLogObserverForTests;
|
|
5276
|
+
function logProbeFailureOnce(resolution, error) {
|
|
5277
|
+
const failure = recordOpenCodeDbReadFailure(resolution, error);
|
|
5278
|
+
if (!claimOpenCodeDbDiagnosticOnce("session-state-probe", resolution))
|
|
5279
|
+
return;
|
|
5280
|
+
const message = `[magic-context] OpenCode DB probe failed: path=${resolution.path} source=${resolution.source} cause=${failure.message}`;
|
|
5281
|
+
probeLogObserverForTests?.(message);
|
|
5282
|
+
log(message);
|
|
5283
|
+
}
|
|
5284
|
+
function getMessageTimesFromOpenCodeDb(sessionId, messageIds) {
|
|
5285
|
+
const result = new Map;
|
|
5286
|
+
if (messageIds.length === 0 || !harnessOwnsOpenCodeStore())
|
|
5287
|
+
return result;
|
|
5288
|
+
try {
|
|
5289
|
+
withReadOnlySessionDb((db) => {
|
|
5290
|
+
const placeholders = messageIds.map(() => "?").join(",");
|
|
5291
|
+
const rows = db.prepare(`SELECT id, time_created FROM message WHERE +session_id = ? AND id IN (${placeholders})`).all(sessionId, ...messageIds);
|
|
5292
|
+
for (const row of rows) {
|
|
5293
|
+
if (typeof row.id === "string" && typeof row.time_created === "number") {
|
|
5294
|
+
result.set(row.id, row.time_created);
|
|
5295
|
+
}
|
|
5296
|
+
}
|
|
5297
|
+
});
|
|
5298
|
+
} catch (error) {
|
|
5299
|
+
logProbeFailureOnce(resolveOpenCodeDbPath(), error);
|
|
5300
|
+
}
|
|
5301
|
+
return result;
|
|
5302
|
+
}
|
|
3839
5303
|
|
|
3840
5304
|
// ../plugin/src/hooks/magic-context/read-session-true-raw-tokens.ts
|
|
3841
5305
|
function completedToolArcCrossesBoundary(invOrdinal, resOrdinal, boundary) {
|
|
@@ -3974,7 +5438,7 @@ function callIdFromPart(part) {
|
|
|
3974
5438
|
const state = isRecord(part.state) ? part.state : null;
|
|
3975
5439
|
return state ? firstStringField(state, ["callID", "callId", "toolCallId", "tool_call_id", "id"]) ?? "" : "";
|
|
3976
5440
|
}
|
|
3977
|
-
function toolSignalFromPart(part) {
|
|
5441
|
+
function toolSignalFromPart(part, withText = true) {
|
|
3978
5442
|
if (!isRecord(part))
|
|
3979
5443
|
return null;
|
|
3980
5444
|
const type = partType(part);
|
|
@@ -3993,8 +5457,8 @@ function toolSignalFromPart(part) {
|
|
|
3993
5457
|
callId,
|
|
3994
5458
|
hasInput: hasInput || openInvocation,
|
|
3995
5459
|
hasOutput,
|
|
3996
|
-
inputText: hasInput && state ? stringValue(state.input) : "",
|
|
3997
|
-
outputText: hasOutput ? stringValue(outputValue) : ""
|
|
5460
|
+
inputText: withText && hasInput && state ? stringValue(state.input) : "",
|
|
5461
|
+
outputText: withText && hasOutput ? stringValue(outputValue) : ""
|
|
3998
5462
|
};
|
|
3999
5463
|
}
|
|
4000
5464
|
if (type === "tool-invocation") {
|
|
@@ -4003,7 +5467,7 @@ function toolSignalFromPart(part) {
|
|
|
4003
5467
|
callId,
|
|
4004
5468
|
hasInput: args !== undefined,
|
|
4005
5469
|
hasOutput: false,
|
|
4006
|
-
inputText: args !== undefined ? stringValue(args) : "",
|
|
5470
|
+
inputText: withText && args !== undefined ? stringValue(args) : "",
|
|
4007
5471
|
outputText: ""
|
|
4008
5472
|
};
|
|
4009
5473
|
}
|
|
@@ -4013,7 +5477,7 @@ function toolSignalFromPart(part) {
|
|
|
4013
5477
|
callId,
|
|
4014
5478
|
hasInput: input !== undefined,
|
|
4015
5479
|
hasOutput: false,
|
|
4016
|
-
inputText: input !== undefined ? stringValue(input) : "",
|
|
5480
|
+
inputText: withText && input !== undefined ? stringValue(input) : "",
|
|
4017
5481
|
outputText: ""
|
|
4018
5482
|
};
|
|
4019
5483
|
}
|
|
@@ -4024,7 +5488,7 @@ function toolSignalFromPart(part) {
|
|
|
4024
5488
|
hasInput: false,
|
|
4025
5489
|
hasOutput: content !== undefined,
|
|
4026
5490
|
inputText: "",
|
|
4027
|
-
outputText: content !== undefined ? textFromToolResultContent(content) : ""
|
|
5491
|
+
outputText: withText && content !== undefined ? textFromToolResultContent(content) : ""
|
|
4028
5492
|
};
|
|
4029
5493
|
}
|
|
4030
5494
|
return null;
|
|
@@ -4034,7 +5498,7 @@ function partCheapFingerprint(part) {
|
|
|
4034
5498
|
return `${typeof part}:${recursiveByteLength(part)}`;
|
|
4035
5499
|
const version = rawPartVersion(part);
|
|
4036
5500
|
const type = typeof part.type === "string" ? part.type : "";
|
|
4037
|
-
return `${type}:${String(version)}:${recursiveByteLength(part)}`;
|
|
5501
|
+
return `${type}:${String(version)}:${recursiveByteLength(providerVisiblePart(part))}`;
|
|
4038
5502
|
}
|
|
4039
5503
|
function messageCacheKey(message, options) {
|
|
4040
5504
|
const namespace = "cacheNamespace" in options ? options.cacheNamespace : "estimate";
|
|
@@ -4154,7 +5618,7 @@ function buildToolArcs(messages) {
|
|
|
4154
5618
|
const arcs = [];
|
|
4155
5619
|
for (const message of messages) {
|
|
4156
5620
|
for (const part of message.parts) {
|
|
4157
|
-
const signal = toolSignalFromPart(part);
|
|
5621
|
+
const signal = toolSignalFromPart(part, false);
|
|
4158
5622
|
if (!signal || signal.callId.length === 0)
|
|
4159
5623
|
continue;
|
|
4160
5624
|
if (signal.hasInput && signal.hasOutput) {
|
|
@@ -4514,11 +5978,11 @@ function containsDroppedInputPlaceholder(value) {
|
|
|
4514
5978
|
};
|
|
4515
5979
|
return visit(value);
|
|
4516
5980
|
}
|
|
4517
|
-
function
|
|
5981
|
+
function isRecord3(value) {
|
|
4518
5982
|
return value !== null && typeof value === "object";
|
|
4519
5983
|
}
|
|
4520
5984
|
function isOptionalZodField(field) {
|
|
4521
|
-
if (!
|
|
5985
|
+
if (!isRecord3(field))
|
|
4522
5986
|
return false;
|
|
4523
5987
|
const isOptional = field.isOptional;
|
|
4524
5988
|
if (typeof isOptional === "function") {
|
|
@@ -4530,10 +5994,10 @@ function isOptionalZodField(field) {
|
|
|
4530
5994
|
return internals?.optin === "optional";
|
|
4531
5995
|
}
|
|
4532
5996
|
function toolParameterNames(schema) {
|
|
4533
|
-
if (!
|
|
5997
|
+
if (!isRecord3(schema))
|
|
4534
5998
|
return;
|
|
4535
5999
|
const properties = schema.properties;
|
|
4536
|
-
if (
|
|
6000
|
+
if (isRecord3(properties) && !Array.isArray(properties)) {
|
|
4537
6001
|
const names = Object.keys(properties);
|
|
4538
6002
|
const requiredList = Array.isArray(schema.required) ? schema.required : [];
|
|
4539
6003
|
const required = new Set(requiredList.filter((name) => typeof name === "string"));
|
|
@@ -4548,7 +6012,7 @@ function toolParameterNames(schema) {
|
|
|
4548
6012
|
} catch {
|
|
4549
6013
|
shape = undefined;
|
|
4550
6014
|
}
|
|
4551
|
-
if (
|
|
6015
|
+
if (isRecord3(shape) && !Array.isArray(shape)) {
|
|
4552
6016
|
const result = { required: [], optional: [] };
|
|
4553
6017
|
for (const [name, field] of Object.entries(shape)) {
|
|
4554
6018
|
(isOptionalZodField(field) ? result.optional : result.required).push(name);
|
|
@@ -4700,6 +6164,17 @@ function loadToolDefinitionMeasurements(db) {
|
|
|
4700
6164
|
|
|
4701
6165
|
// ../plugin/src/hooks/magic-context/final-wire-token-estimate.ts
|
|
4702
6166
|
var wireTextTokens = createTokenCountMemo(1e5, 128 * 1024 * 1024);
|
|
6167
|
+
function estimatePartImageTokens(raw) {
|
|
6168
|
+
if (!raw || typeof raw !== "object")
|
|
6169
|
+
return 0;
|
|
6170
|
+
const part = raw;
|
|
6171
|
+
if (part.ignored)
|
|
6172
|
+
return 0;
|
|
6173
|
+
if (part.type === "file" && part.mime?.startsWith("image/")) {
|
|
6174
|
+
return typeof part.url === "string" && part.url.startsWith("data:") ? estimateImageTokensFromDataUrl(part.url) : 1200;
|
|
6175
|
+
}
|
|
6176
|
+
return part.type === "tool" ? estimateToolAttachmentImageTokens(part.state) : 0;
|
|
6177
|
+
}
|
|
4703
6178
|
|
|
4704
6179
|
// ../plugin/src/features/magic-context/overflow-detection.ts
|
|
4705
6180
|
var OVERFLOW_PATTERNS = [
|
|
@@ -5308,6 +6783,14 @@ function readRawSessionMessageRange2(sessionId, fromOrdinal, toOrdinal) {
|
|
|
5308
6783
|
}
|
|
5309
6784
|
readRawSessionMessages2.readPage = readRawSessionMessagePage2;
|
|
5310
6785
|
readRawSessionMessages2.getCount = getRawSessionMessageOrdinalCount2;
|
|
6786
|
+
readRawSessionMessages2.prepare = prepareRawSessionOrdinals;
|
|
6787
|
+
async function prepareRawSessionOrdinals(sessionId) {
|
|
6788
|
+
if (sessionProviders.get(sessionId)?.provider)
|
|
6789
|
+
return true;
|
|
6790
|
+
if (!openCodeDbExists())
|
|
6791
|
+
return true;
|
|
6792
|
+
return withReadOnlySessionDb((db) => warmRawSessionOrdinalsForDb(db, sessionId));
|
|
6793
|
+
}
|
|
5311
6794
|
function primeTailRawMessageCache2(args) {
|
|
5312
6795
|
const { sessionId, lastCompartmentEnd, anchorMessageId } = args;
|
|
5313
6796
|
if (!activeRawMessageCache)
|
|
@@ -5459,6 +6942,7 @@ function readRawSessionMessageById2(sessionId, messageId) {
|
|
|
5459
6942
|
return null;
|
|
5460
6943
|
return withReadOnlySessionDb((db) => readRawSessionMessageByIdFromDb(db, sessionId, messageId));
|
|
5461
6944
|
}
|
|
6945
|
+
readRawSessionMessageById2.prepare = prepareRawSessionOrdinals;
|
|
5462
6946
|
function readRawSessionMessagesFromSource(sessionId) {
|
|
5463
6947
|
const provider = sessionProviders.get(sessionId)?.provider;
|
|
5464
6948
|
if (provider)
|
|
@@ -5839,6 +7323,14 @@ function readSessionChunk2(sessionId, tokenBudget, offset = 1, eligibleEndOrdina
|
|
|
5839
7323
|
pinComponentWhenFormattedBudgetCrosses(msg.ordinal, text);
|
|
5840
7324
|
}
|
|
5841
7325
|
if (flushCurrentBlock() && pendingNoiseMeta.length > 0) {
|
|
7326
|
+
const lastNoise = pendingNoiseMeta[pendingNoiseMeta.length - 1];
|
|
7327
|
+
if (messagesProcessed > 0 && lastNoise) {
|
|
7328
|
+
lineMeta.push(...pendingNoiseMeta);
|
|
7329
|
+
messagesProcessed += pendingNoiseMeta.length;
|
|
7330
|
+
lastOrdinal = lastNoise.ordinal;
|
|
7331
|
+
lastMessageId = lastNoise.messageId;
|
|
7332
|
+
pendingNoiseMeta = [];
|
|
7333
|
+
}
|
|
5842
7334
|
highestScannedOrdinal = Math.max(highestScannedOrdinal, pendingNoiseMeta[pendingNoiseMeta.length - 1]?.ordinal ?? highestScannedOrdinal);
|
|
5843
7335
|
}
|
|
5844
7336
|
const toolOnlyRanges = [];
|
|
@@ -5876,4 +7368,4 @@ function readSessionChunk2(sessionId, tokenBudget, offset = 1, eligibleEndOrdina
|
|
|
5876
7368
|
toolResultBoundaries
|
|
5877
7369
|
};
|
|
5878
7370
|
}
|
|
5879
|
-
export { canonicalModelIdentity, piModelRefToCanonical, resolveModelRefForPi, modelRefLookupOrder, toolTemplateError, DEFAULT_PROTECTED_TOOLS, normalizeProtectedToolName, mergeProtectedTools, detectThinkingBindingMismatch, detectLatestTurnThinkingMismatch, isPrefixBoundThinkingModel, detectOverflow, stableStringify, UNKNOWN_FIT_RATIO, resolveDecisionCalibration, hasMeasuredDecisionCalibration, providerMass, localBudget, calibrationForModelKey, historyLocalBudget, adoptedProtectedToolsPolicy, protectedToolTagNumbers, protectedToolTokenCount, removeSystemReminders, hasMeaningfulUserText2, extractTexts2, preloadTokenizer, hasTokenizerForFit, tokenCountUsesByteBound, estimateTokens, estimateFixedPromptTokens, stripWellFormedLeadingTagPrefix, stripPersistedAssistantText, byteSize, stripTagPrefix, peelLeadingMcTagNotation, prependTag, isRecord, TEXT_TAG_IDENTITY_MARKER, contentTagOwnerMessageId, estimateImageTokensFromDataUrl, TODO_STATUSES, TODO_PRIORITIES, TITLE_DONE_STATUSES, normalizeTodoStateJson, buildSyntheticTodoPart, stripChannel1ReminderSpans, compareMeasuredTailPrefix, formatTailHygienePrefixMismatch, freezeTailHygieneMeasurement, effectiveTailHygiene, CHANNEL1_SENTINEL, CHANNEL1_FLOOR_TOKENS, decideChannel1, evaluateChannel2, reclaimableToolOutputCount, buildChannel2Reminder, buildChannel1Reminder, planEmergencyDrop, measureEmergencyTag, closeQuietly, isKnownAutocommit, installV3PerfSchema, queuePendingOp, queuePendingDropForUnchangedTag, getPendingOps, getPendingOpsCount, clearPendingOps, removePendingOp, updateTagByteSize, getRecentTagOwnerMessageIds, AGE_RECLAIM_MIN_TOKENS, getOldestActiveUnprotectedToolTags, getActiveToolTagsForAgeReclaim, getTriggerTagTokenUpperBound, updateTagInputByteSize, updateTagTokenCount, getPersistedToolTagAccounting, getAllStatusTagTokenTotalsFlat, getReasoningTokenEstimatesByMessage, updateTagInputTokenCount, tagTokenCountIsNull, backfillTagTokenCounts, insertTag, updateTagStatus,
|
|
7371
|
+
export { canonicalModelIdentity, piModelRefToCanonical, resolveModelRefForPi, modelRefLookupOrder, toolTemplateError, DEFAULT_PROTECTED_TOOLS, normalizeProtectedToolName, mergeProtectedTools, detectThinkingBindingMismatch, detectLatestTurnThinkingMismatch, isPrefixBoundThinkingModel, detectOverflow, stableStringify, UNKNOWN_FIT_RATIO, resolveDecisionCalibration, hasMeasuredDecisionCalibration, providerMass, localBudget, calibrationForModelKey, historyLocalBudget, adoptedProtectedToolsPolicy, protectedToolTagNumbers, protectedToolTokenCount, removeSystemReminders, hasMeaningfulUserText2, extractTexts2, preloadTokenizer, hasTokenizerForFit, tokenCountUsesByteBound, estimateTokens, estimateFixedPromptTokens, stripWellFormedLeadingTagPrefix, stripPersistedAssistantText, byteSize, stripTagPrefix, peelLeadingMcTagNotation, prependTag, rememberCurrentToolOutputCount, isRecord, TEXT_TAG_IDENTITY_MARKER, PI_FALLBACK_PREFIXES_JSON, contentTagOwnerMessageId, TailHygieneJsonMemo, piJsonHash, estimateImageTokensFromDataUrl, providerVisibleMessage, TODO_STATUSES, TODO_PRIORITIES, TITLE_DONE_STATUSES, normalizeTodoStateJson, buildSyntheticTodoPart, stripChannel1ReminderSpans, compareMeasuredTailPrefix, formatTailHygienePrefixMismatch, freezeTailHygieneMeasurement, effectiveTailHygiene, CHANNEL1_SENTINEL, CHANNEL1_FLOOR_TOKENS, decideChannel1, evaluateChannel2, reclaimableToolOutputCount, buildChannel2Reminder, buildChannel1Reminder, planEmergencyDrop, measureEmergencyTag, closeQuietly, isKnownAutocommit, installV3PerfSchema, queuePendingOp, queuePendingDropForUnchangedTag, getPendingOps, getPendingOpsCount, clearPendingOps, removePendingOp, tagRevisionIncarnationSql, tagRevisionCountSql, tagRevisionKey, tagIdentityRevisionKey, TAG_INSERT_CONFLICT_ROWS, installTagIdentityRevisionTrigger, updateTagByteSize, getRecentTagOwnerMessageIds, AGE_RECLAIM_MIN_TOKENS, getOldestActiveUnprotectedToolTags, getActiveToolTagsForAgeReclaim, getTriggerTagTokenUpperBound, updateTagInputByteSize, updateTagTokenCount, getPersistedToolTagAccounting, getAllStatusTagTokenTotalsFlat, getReasoningTokenEstimatesByMessage, updateTagInputTokenCount, tagTokenCountIsNull, backfillTagTokenCounts, insertTag, updateTagStatus, markTagDropped, getInertWhitespaceAssistantTagsByNumbers, updateTagDropMode, updateCavemanDepth, hasPiFallbackMessageTags, findAdoptableFallbackTags, hasPiFallbackToolOwnerTags, findPiFallbackToolOwnerTags, PiTagIdentityConflictError, getPiMessageTagCandidates, findPiTagIdentityConflict, adoptPiFallbackToolOwnerTag, adoptPiFallbackMessageTag, getMaxTagNumberBySession, getTagNumberByMessageId, getAssignableTagNumberByMessageId, deriveTagLoadFloor, TAG_SELECT_COLUMNS, getTagsBySession, getActiveTagsBySession, getTagsForPendingOperations, getTagsByNumbers, getNewestToolTagNumbers, getDroppedTagsByNumbers, getMaxDroppedTagNumber, getTagById, getToolTagNumberByOwner, getNullOwnerToolTag, adoptNullOwnerToolTag, resolveOpenCodeDbPath, hasV1MessageTables, assertOpenCodeStoreGeneration, withReadOnlySessionDb, expandToolPart, retreatPastHostUnservedRows, completedToolArcCrossesBoundary, estimateTrueRawMessageTokens, buildToolArcs, fenceBoundaryForCompletedToolArcs, fenceBoundaryForToolArcs, buildTrueRawTokenIndex, computeRawRangeFingerprint, invalidateTrueRawTokenCache, stripSystemInjection, hasUserAnswerMetadata, droppedInputMarker, createDroppedInputGuard, isEditTool, applyEditMarkerToInput, setDatabase, loadToolDefinitionMeasurements, estimatePartImageTokens, modelAcceptsEmptyContent, isAnthropicFamilyRoute, variantChangeBustsProviderCache, SKELETON_REAL_INPUT_MAX_BYTES, toolInputStringBytes, resolveHostServedBoundaryId2, hasRawMessageProvider2, setRawMessageProvider2, withRawMessageProvider2, cleanUserText2, withRawSessionMessageCache2, readRawSessionMessages2, readRawSessionMessagePage2, getRawSessionMessageOrdinalCount2, visitRawSessionMessages2, readRawSessionMessageRange2, primeTailRawMessageCache2, getCachedAbsoluteMessageCount2, primeInMemoryTailRawMessageCache2, readRawSessionMessageIdOrdinalsForRange2, hasRawSessionMessageById2, readRawSessionMessageOrdinalById2, compareRawSessionMessageOrder2, readRawSessionMessageById2, getRawSessionMessageCount2, getRawSessionTagKeysThrough2, getLegacyProtectedTailStartOrdinal2, readSessionChunk2 };
|