@c3-oss/prosa 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -5
- package/dist/bin/prosa.js +3282 -2227
- package/dist/bin/prosa.js.map +1 -1
- package/dist/cli/main.js +3282 -2227
- package/dist/cli/main.js.map +1 -1
- package/dist/index.js +1055 -1350
- package/dist/index.js.map +1 -1
- package/package.json +5 -1
package/dist/index.js
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { createRequire as __prosaCreateRequire } from 'module'; const require = __prosaCreateRequire(import.meta.url);
|
|
2
|
+
var __defProp = Object.defineProperty;
|
|
3
|
+
var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
|
|
2
4
|
|
|
3
5
|
// src/core/bundle.ts
|
|
4
6
|
import { constants as fsConstants } from "fs";
|
|
@@ -10,15 +12,22 @@ import path from "path";
|
|
|
10
12
|
import Database from "better-sqlite3";
|
|
11
13
|
function openDb(path15) {
|
|
12
14
|
const db = new Database(path15);
|
|
15
|
+
db.pragma("page_size = 16384");
|
|
13
16
|
db.pragma("journal_mode = WAL");
|
|
14
17
|
db.pragma("foreign_keys = ON");
|
|
15
18
|
db.pragma("synchronous = NORMAL");
|
|
16
19
|
db.pragma("busy_timeout = 5000");
|
|
20
|
+
db.pragma("cache_size = -262144");
|
|
21
|
+
db.pragma("mmap_size = 268435456");
|
|
22
|
+
db.pragma("temp_store = MEMORY");
|
|
23
|
+
db.pragma("wal_autocheckpoint = 20000");
|
|
17
24
|
return db;
|
|
18
25
|
}
|
|
26
|
+
__name(openDb, "openDb");
|
|
19
27
|
function closeDb(db) {
|
|
20
28
|
db.close();
|
|
21
29
|
}
|
|
30
|
+
__name(closeDb, "closeDb");
|
|
22
31
|
var stmtCache = /* @__PURE__ */ new WeakMap();
|
|
23
32
|
function prepare(db, sql) {
|
|
24
33
|
let cache = stmtCache.get(db);
|
|
@@ -33,10 +42,12 @@ function prepare(db, sql) {
|
|
|
33
42
|
}
|
|
34
43
|
return stmt;
|
|
35
44
|
}
|
|
45
|
+
__name(prepare, "prepare");
|
|
36
46
|
function transactional(db, fn) {
|
|
37
47
|
const wrapped = db.transaction(fn);
|
|
38
48
|
return wrapped();
|
|
39
49
|
}
|
|
50
|
+
__name(transactional, "transactional");
|
|
40
51
|
|
|
41
52
|
// src/core/schema/sql/001_init.ts
|
|
42
53
|
var SQL_001_INIT = String.raw`
|
|
@@ -683,10 +694,26 @@ ALTER TABLE search_index_status ADD COLUMN schema_fingerprint TEXT;
|
|
|
683
694
|
|
|
684
695
|
// src/core/schema/migrate.ts
|
|
685
696
|
var MIGRATIONS = [
|
|
686
|
-
{
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
697
|
+
{
|
|
698
|
+
version: 1,
|
|
699
|
+
name: "init",
|
|
700
|
+
sql: SQL_001_INIT
|
|
701
|
+
},
|
|
702
|
+
{
|
|
703
|
+
version: 2,
|
|
704
|
+
name: "search_index_status",
|
|
705
|
+
sql: SQL_002_SEARCH_INDEX_STATUS
|
|
706
|
+
},
|
|
707
|
+
{
|
|
708
|
+
version: 3,
|
|
709
|
+
name: "analytics_views",
|
|
710
|
+
sql: SQL_003_ANALYTICS_VIEWS
|
|
711
|
+
},
|
|
712
|
+
{
|
|
713
|
+
version: 4,
|
|
714
|
+
name: "tantivy_checkpoint",
|
|
715
|
+
sql: SQL_004_TANTIVY_CHECKPOINT
|
|
716
|
+
}
|
|
690
717
|
];
|
|
691
718
|
function runMigrations(db) {
|
|
692
719
|
db.exec(`
|
|
@@ -696,35 +723,31 @@ function runMigrations(db) {
|
|
|
696
723
|
applied_at TEXT NOT NULL
|
|
697
724
|
);
|
|
698
725
|
`);
|
|
699
|
-
const applied = new Set(
|
|
700
|
-
db.prepare(`SELECT version FROM _prosa_migrations`).all().map((row) => row.version)
|
|
701
|
-
);
|
|
726
|
+
const applied = new Set(db.prepare(`SELECT version FROM _prosa_migrations`).all().map((row) => row.version));
|
|
702
727
|
const newlyApplied = [];
|
|
703
728
|
for (const migration of MIGRATIONS) {
|
|
704
729
|
if (applied.has(migration.version)) continue;
|
|
705
730
|
const tx = db.transaction(() => {
|
|
706
731
|
db.exec(migration.sql);
|
|
707
|
-
db.prepare(`INSERT INTO _prosa_migrations(version, name, applied_at) VALUES (?, ?, ?)`).run(
|
|
708
|
-
migration.version,
|
|
709
|
-
migration.name,
|
|
710
|
-
(/* @__PURE__ */ new Date()).toISOString()
|
|
711
|
-
);
|
|
732
|
+
db.prepare(`INSERT INTO _prosa_migrations(version, name, applied_at) VALUES (?, ?, ?)`).run(migration.version, migration.name, (/* @__PURE__ */ new Date()).toISOString());
|
|
712
733
|
});
|
|
713
734
|
tx();
|
|
714
735
|
newlyApplied.push(migration.version);
|
|
715
736
|
}
|
|
716
|
-
return {
|
|
737
|
+
return {
|
|
738
|
+
applied: newlyApplied
|
|
739
|
+
};
|
|
717
740
|
}
|
|
741
|
+
__name(runMigrations, "runMigrations");
|
|
718
742
|
function currentSchemaVersion(db) {
|
|
719
743
|
try {
|
|
720
|
-
const row = db.prepare(
|
|
721
|
-
`SELECT MAX(version) AS version FROM _prosa_migrations`
|
|
722
|
-
).get();
|
|
744
|
+
const row = db.prepare(`SELECT MAX(version) AS version FROM _prosa_migrations`).get();
|
|
723
745
|
return row?.version ?? 0;
|
|
724
746
|
} catch {
|
|
725
747
|
return 0;
|
|
726
748
|
}
|
|
727
749
|
}
|
|
750
|
+
__name(currentSchemaVersion, "currentSchemaVersion");
|
|
728
751
|
|
|
729
752
|
// src/core/version.ts
|
|
730
753
|
var PROSA_PARSER_VERSION = "0.1.0";
|
|
@@ -736,6 +759,7 @@ function defaultBundlePath() {
|
|
|
736
759
|
if (env && env.length > 0) return path.resolve(env);
|
|
737
760
|
return path.join(os.homedir(), ".prosa");
|
|
738
761
|
}
|
|
762
|
+
__name(defaultBundlePath, "defaultBundlePath");
|
|
739
763
|
function bundlePaths(rootPath) {
|
|
740
764
|
return {
|
|
741
765
|
db: path.join(rootPath, "prosa.sqlite"),
|
|
@@ -749,6 +773,7 @@ function bundlePaths(rootPath) {
|
|
|
749
773
|
lock: path.join(rootPath, "prosa.lock")
|
|
750
774
|
};
|
|
751
775
|
}
|
|
776
|
+
__name(bundlePaths, "bundlePaths");
|
|
752
777
|
async function exists(p) {
|
|
753
778
|
try {
|
|
754
779
|
await access(p, fsConstants.F_OK);
|
|
@@ -757,21 +782,34 @@ async function exists(p) {
|
|
|
757
782
|
return false;
|
|
758
783
|
}
|
|
759
784
|
}
|
|
785
|
+
__name(exists, "exists");
|
|
760
786
|
async function initBundle(rootPath) {
|
|
761
787
|
const resolved = path.resolve(rootPath);
|
|
762
788
|
const paths = bundlePaths(resolved);
|
|
763
|
-
await mkdir(resolved, {
|
|
789
|
+
await mkdir(resolved, {
|
|
790
|
+
recursive: true
|
|
791
|
+
});
|
|
764
792
|
if (await exists(paths.manifest)) {
|
|
765
|
-
throw new Error(
|
|
766
|
-
`bundle already exists at ${resolved} (found manifest.json) \u2014 use openBundle instead`
|
|
767
|
-
);
|
|
793
|
+
throw new Error(`bundle already exists at ${resolved} (found manifest.json) \u2014 use openBundle instead`);
|
|
768
794
|
}
|
|
769
|
-
await mkdir(paths.objects, {
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
await mkdir(paths.
|
|
773
|
-
|
|
774
|
-
|
|
795
|
+
await mkdir(paths.objects, {
|
|
796
|
+
recursive: true
|
|
797
|
+
});
|
|
798
|
+
await mkdir(paths.rawSources, {
|
|
799
|
+
recursive: true
|
|
800
|
+
});
|
|
801
|
+
await mkdir(paths.search, {
|
|
802
|
+
recursive: true
|
|
803
|
+
});
|
|
804
|
+
await mkdir(paths.tantivy, {
|
|
805
|
+
recursive: true
|
|
806
|
+
});
|
|
807
|
+
await mkdir(paths.exports, {
|
|
808
|
+
recursive: true
|
|
809
|
+
});
|
|
810
|
+
await mkdir(paths.parquet, {
|
|
811
|
+
recursive: true
|
|
812
|
+
});
|
|
775
813
|
const manifest = {
|
|
776
814
|
version: 1,
|
|
777
815
|
parser_version: PROSA_PARSER_VERSION,
|
|
@@ -784,8 +822,14 @@ async function initBundle(rootPath) {
|
|
|
784
822
|
`, "utf8");
|
|
785
823
|
const db = openDb(paths.db);
|
|
786
824
|
runMigrations(db);
|
|
787
|
-
return {
|
|
825
|
+
return {
|
|
826
|
+
path: resolved,
|
|
827
|
+
db,
|
|
828
|
+
manifest,
|
|
829
|
+
paths
|
|
830
|
+
};
|
|
788
831
|
}
|
|
832
|
+
__name(initBundle, "initBundle");
|
|
789
833
|
async function openBundle(rootPath) {
|
|
790
834
|
const resolved = path.resolve(rootPath);
|
|
791
835
|
const paths = bundlePaths(resolved);
|
|
@@ -794,13 +838,15 @@ async function openBundle(rootPath) {
|
|
|
794
838
|
throw new Error(`bundle path not found or not a directory: ${resolved}`);
|
|
795
839
|
}
|
|
796
840
|
if (!await exists(paths.manifest)) {
|
|
797
|
-
throw new Error(
|
|
798
|
-
`no manifest.json in ${resolved} \u2014 initialize first with \`prosa init --store ${resolved}\``
|
|
799
|
-
);
|
|
841
|
+
throw new Error(`no manifest.json in ${resolved} \u2014 initialize first with \`prosa init --store ${resolved}\``);
|
|
800
842
|
}
|
|
801
843
|
const manifest = JSON.parse(await readFile(paths.manifest, "utf8"));
|
|
802
|
-
await mkdir(paths.search, {
|
|
803
|
-
|
|
844
|
+
await mkdir(paths.search, {
|
|
845
|
+
recursive: true
|
|
846
|
+
});
|
|
847
|
+
await mkdir(paths.tantivy, {
|
|
848
|
+
recursive: true
|
|
849
|
+
});
|
|
804
850
|
const db = openDb(paths.db);
|
|
805
851
|
runMigrations(db);
|
|
806
852
|
const currentVersion = currentSchemaVersion(db);
|
|
@@ -808,13 +854,27 @@ async function openBundle(rootPath) {
|
|
|
808
854
|
closeDb(db);
|
|
809
855
|
throw new Error(`schema version mismatch (db=${currentVersion}, code=${PROSA_SCHEMA_VERSION})`);
|
|
810
856
|
}
|
|
857
|
+
let manifestDirty = false;
|
|
811
858
|
if (manifest.parser_version !== PROSA_PARSER_VERSION) {
|
|
812
859
|
manifest.parser_version = PROSA_PARSER_VERSION;
|
|
860
|
+
manifestDirty = true;
|
|
861
|
+
}
|
|
862
|
+
if (manifest.schema_version !== currentVersion) {
|
|
863
|
+
manifest.schema_version = currentVersion;
|
|
864
|
+
manifestDirty = true;
|
|
865
|
+
}
|
|
866
|
+
if (manifestDirty) {
|
|
813
867
|
await writeFile(paths.manifest, `${JSON.stringify(manifest, null, 2)}
|
|
814
868
|
`, "utf8");
|
|
815
869
|
}
|
|
816
|
-
return {
|
|
870
|
+
return {
|
|
871
|
+
path: resolved,
|
|
872
|
+
db,
|
|
873
|
+
manifest,
|
|
874
|
+
paths
|
|
875
|
+
};
|
|
817
876
|
}
|
|
877
|
+
__name(openBundle, "openBundle");
|
|
818
878
|
async function openOrInitBundle(rootPath) {
|
|
819
879
|
const resolved = path.resolve(rootPath);
|
|
820
880
|
const paths = bundlePaths(resolved);
|
|
@@ -827,9 +887,11 @@ async function openOrInitBundle(rootPath) {
|
|
|
827
887
|
}
|
|
828
888
|
return await openBundle(resolved);
|
|
829
889
|
}
|
|
890
|
+
__name(openOrInitBundle, "openOrInitBundle");
|
|
830
891
|
function closeBundle(bundle) {
|
|
831
892
|
closeDb(bundle.db);
|
|
832
893
|
}
|
|
894
|
+
__name(closeBundle, "closeBundle");
|
|
833
895
|
|
|
834
896
|
// src/core/cas/index.ts
|
|
835
897
|
import { mkdir as mkdir2, readFile as readFile2, writeFile as writeFile2 } from "fs/promises";
|
|
@@ -841,15 +903,25 @@ var COMPRESS_THRESHOLD_BYTES = 256;
|
|
|
841
903
|
var ZSTD_LEVEL = 3;
|
|
842
904
|
function compressBytes(input) {
|
|
843
905
|
if (input.byteLength < COMPRESS_THRESHOLD_BYTES) {
|
|
844
|
-
return {
|
|
906
|
+
return {
|
|
907
|
+
bytes: Buffer.from(input),
|
|
908
|
+
compression: "none"
|
|
909
|
+
};
|
|
845
910
|
}
|
|
846
|
-
const out = zstdCompress(Buffer.from(input), {
|
|
847
|
-
|
|
911
|
+
const out = zstdCompress(Buffer.from(input), {
|
|
912
|
+
compressionLevel: ZSTD_LEVEL
|
|
913
|
+
});
|
|
914
|
+
return {
|
|
915
|
+
bytes: out,
|
|
916
|
+
compression: "zstd"
|
|
917
|
+
};
|
|
848
918
|
}
|
|
919
|
+
__name(compressBytes, "compressBytes");
|
|
849
920
|
function decompressBytes(input, compression) {
|
|
850
921
|
if (compression === "none") return input;
|
|
851
922
|
return zstdDecompress(input);
|
|
852
923
|
}
|
|
924
|
+
__name(decompressBytes, "decompressBytes");
|
|
853
925
|
|
|
854
926
|
// src/core/cas/hash.ts
|
|
855
927
|
import { createHash } from "crypto";
|
|
@@ -858,60 +930,52 @@ import { bytesToHex } from "@noble/hashes/utils";
|
|
|
858
930
|
function blake3Hex(bytes) {
|
|
859
931
|
return bytesToHex(blake3(bytes));
|
|
860
932
|
}
|
|
933
|
+
__name(blake3Hex, "blake3Hex");
|
|
861
934
|
function sha256Hex(bytes) {
|
|
862
935
|
return createHash("sha256").update(bytes).digest("hex");
|
|
863
936
|
}
|
|
937
|
+
__name(sha256Hex, "sha256Hex");
|
|
864
938
|
function objectIdFromHash(hashHex) {
|
|
865
939
|
return `blake3:${hashHex}`;
|
|
866
940
|
}
|
|
941
|
+
__name(objectIdFromHash, "objectIdFromHash");
|
|
867
942
|
function objectStoragePath(hashHex, compression) {
|
|
868
943
|
const ext = compression === "zstd" ? ".zst" : ".bin";
|
|
869
944
|
const a = hashHex.slice(0, 2);
|
|
870
945
|
const b = hashHex.slice(2, 4);
|
|
871
946
|
return `objects/blake3/${a}/${b}/${hashHex}${ext}`;
|
|
872
947
|
}
|
|
948
|
+
__name(objectStoragePath, "objectStoragePath");
|
|
873
949
|
|
|
874
950
|
// src/core/cas/index.ts
|
|
875
951
|
var ensuredDirs = /* @__PURE__ */ new Set();
|
|
876
952
|
async function ensureDir(absoluteDir) {
|
|
877
953
|
if (ensuredDirs.has(absoluteDir)) return;
|
|
878
|
-
await mkdir2(absoluteDir, {
|
|
954
|
+
await mkdir2(absoluteDir, {
|
|
955
|
+
recursive: true
|
|
956
|
+
});
|
|
879
957
|
ensuredDirs.add(absoluteDir);
|
|
880
958
|
}
|
|
959
|
+
__name(ensureDir, "ensureDir");
|
|
881
960
|
async function putBytes(bundle, bytes, options = {}) {
|
|
882
961
|
const hash = blake3Hex(bytes);
|
|
883
962
|
const objectId = objectIdFromHash(hash);
|
|
884
|
-
const existing = prepare(
|
|
885
|
-
bundle.db,
|
|
886
|
-
`SELECT object_id, hash_alg, hash, size_bytes, compressed_size_bytes,
|
|
963
|
+
const existing = prepare(bundle.db, `SELECT object_id, hash_alg, hash, size_bytes, compressed_size_bytes,
|
|
887
964
|
compression, mime_type, encoding, storage_path, created_at
|
|
888
|
-
FROM objects WHERE object_id = ?`
|
|
889
|
-
).get(objectId);
|
|
965
|
+
FROM objects WHERE object_id = ?`).get(objectId);
|
|
890
966
|
if (existing) return objectId;
|
|
891
967
|
const { bytes: stored, compression } = compressBytes(bytes);
|
|
892
968
|
const storagePath = objectStoragePath(hash, compression);
|
|
893
969
|
const absolutePath = path2.join(bundle.path, storagePath);
|
|
894
970
|
await ensureDir(path2.dirname(absolutePath));
|
|
895
971
|
await writeFile2(absolutePath, stored);
|
|
896
|
-
prepare(
|
|
897
|
-
bundle.db,
|
|
898
|
-
`INSERT INTO objects (
|
|
972
|
+
prepare(bundle.db, `INSERT INTO objects (
|
|
899
973
|
object_id, hash_alg, hash, size_bytes, compressed_size_bytes,
|
|
900
974
|
compression, mime_type, encoding, storage_path, created_at
|
|
901
|
-
) VALUES (?, 'blake3', ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
902
|
-
).run(
|
|
903
|
-
objectId,
|
|
904
|
-
hash,
|
|
905
|
-
bytes.byteLength,
|
|
906
|
-
compression === "zstd" ? stored.byteLength : null,
|
|
907
|
-
compression,
|
|
908
|
-
options.mimeType ?? null,
|
|
909
|
-
options.encoding ?? null,
|
|
910
|
-
storagePath,
|
|
911
|
-
(/* @__PURE__ */ new Date()).toISOString()
|
|
912
|
-
);
|
|
975
|
+
) VALUES (?, 'blake3', ?, ?, ?, ?, ?, ?, ?, ?)`).run(objectId, hash, bytes.byteLength, compression === "zstd" ? stored.byteLength : null, compression, options.mimeType ?? null, options.encoding ?? null, storagePath, (/* @__PURE__ */ new Date()).toISOString());
|
|
913
976
|
return objectId;
|
|
914
977
|
}
|
|
978
|
+
__name(putBytes, "putBytes");
|
|
915
979
|
async function putText(bundle, text, options = {}) {
|
|
916
980
|
const buf = Buffer.from(text, "utf8");
|
|
917
981
|
return putBytes(bundle, buf, {
|
|
@@ -919,6 +983,7 @@ async function putText(bundle, text, options = {}) {
|
|
|
919
983
|
encoding: "utf-8"
|
|
920
984
|
});
|
|
921
985
|
}
|
|
986
|
+
__name(putText, "putText");
|
|
922
987
|
async function putJson(bundle, value) {
|
|
923
988
|
const text = JSON.stringify(value);
|
|
924
989
|
return putBytes(bundle, Buffer.from(text, "utf8"), {
|
|
@@ -926,38 +991,40 @@ async function putJson(bundle, value) {
|
|
|
926
991
|
encoding: "utf-8"
|
|
927
992
|
});
|
|
928
993
|
}
|
|
994
|
+
__name(putJson, "putJson");
|
|
929
995
|
async function getBytes(bundle, objectId) {
|
|
930
|
-
const meta = prepare(
|
|
931
|
-
bundle.db,
|
|
932
|
-
`SELECT object_id, hash_alg, hash, size_bytes, compressed_size_bytes,
|
|
996
|
+
const meta = prepare(bundle.db, `SELECT object_id, hash_alg, hash, size_bytes, compressed_size_bytes,
|
|
933
997
|
compression, mime_type, encoding, storage_path, created_at
|
|
934
|
-
FROM objects WHERE object_id = ?`
|
|
935
|
-
).get(objectId);
|
|
998
|
+
FROM objects WHERE object_id = ?`).get(objectId);
|
|
936
999
|
if (!meta) {
|
|
937
1000
|
throw new Error(`object not found: ${objectId}`);
|
|
938
1001
|
}
|
|
939
1002
|
const buf = await readFile2(path2.join(bundle.path, meta.storage_path));
|
|
940
1003
|
return decompressBytes(buf, meta.compression);
|
|
941
1004
|
}
|
|
1005
|
+
__name(getBytes, "getBytes");
|
|
942
1006
|
async function getText(bundle, objectId) {
|
|
943
1007
|
const buf = await getBytes(bundle, objectId);
|
|
944
1008
|
return buf.toString("utf8");
|
|
945
1009
|
}
|
|
1010
|
+
__name(getText, "getText");
|
|
946
1011
|
async function getJson(bundle, objectId) {
|
|
947
1012
|
const text = await getText(bundle, objectId);
|
|
948
1013
|
return JSON.parse(text);
|
|
949
1014
|
}
|
|
1015
|
+
__name(getJson, "getJson");
|
|
950
1016
|
function getObjectMeta(bundle, objectId) {
|
|
951
|
-
return prepare(
|
|
952
|
-
bundle.db,
|
|
953
|
-
`SELECT object_id, hash_alg, hash, size_bytes, compressed_size_bytes,
|
|
1017
|
+
return prepare(bundle.db, `SELECT object_id, hash_alg, hash, size_bytes, compressed_size_bytes,
|
|
954
1018
|
compression, mime_type, encoding, storage_path, created_at
|
|
955
|
-
FROM objects WHERE object_id = ?`
|
|
956
|
-
).get(objectId) ?? null;
|
|
1019
|
+
FROM objects WHERE object_id = ?`).get(objectId) ?? null;
|
|
957
1020
|
}
|
|
1021
|
+
__name(getObjectMeta, "getObjectMeta");
|
|
958
1022
|
function createPendingObjects() {
|
|
959
|
-
return {
|
|
1023
|
+
return {
|
|
1024
|
+
byId: /* @__PURE__ */ new Map()
|
|
1025
|
+
};
|
|
960
1026
|
}
|
|
1027
|
+
__name(createPendingObjects, "createPendingObjects");
|
|
961
1028
|
function stageBytes(pending, bytes, options = {}) {
|
|
962
1029
|
const buf = Buffer.isBuffer(bytes) ? bytes : Buffer.from(bytes);
|
|
963
1030
|
const hash = blake3Hex(buf);
|
|
@@ -973,21 +1040,26 @@ function stageBytes(pending, bytes, options = {}) {
|
|
|
973
1040
|
}
|
|
974
1041
|
return objectId;
|
|
975
1042
|
}
|
|
1043
|
+
__name(stageBytes, "stageBytes");
|
|
976
1044
|
function stageText(pending, text, options = {}) {
|
|
977
1045
|
return stageBytes(pending, Buffer.from(text, "utf8"), {
|
|
978
1046
|
mimeType: options.mimeType ?? "text/plain; charset=utf-8",
|
|
979
1047
|
encoding: "utf-8"
|
|
980
1048
|
});
|
|
981
1049
|
}
|
|
1050
|
+
__name(stageText, "stageText");
|
|
982
1051
|
function stageJson(pending, value) {
|
|
983
1052
|
return stageBytes(pending, Buffer.from(JSON.stringify(value), "utf8"), {
|
|
984
1053
|
mimeType: "application/json",
|
|
985
1054
|
encoding: "utf-8"
|
|
986
1055
|
});
|
|
987
1056
|
}
|
|
1057
|
+
__name(stageJson, "stageJson");
|
|
988
1058
|
async function flushPendingObjects(bundle, pending) {
|
|
989
1059
|
if (pending.byId.size === 0) return;
|
|
990
|
-
const ids = [
|
|
1060
|
+
const ids = [
|
|
1061
|
+
...pending.byId.keys()
|
|
1062
|
+
];
|
|
991
1063
|
const existingIds = queryExistingObjectIds(bundle, ids);
|
|
992
1064
|
const toWrite = [];
|
|
993
1065
|
for (const obj of pending.byId.values()) {
|
|
@@ -1005,28 +1077,16 @@ async function flushPendingObjects(bundle, pending) {
|
|
|
1005
1077
|
if (toWrite.length > 0) {
|
|
1006
1078
|
await writeFilesParallel(toWrite);
|
|
1007
1079
|
}
|
|
1008
|
-
const insertObject = prepare(
|
|
1009
|
-
bundle.db,
|
|
1010
|
-
`INSERT OR IGNORE INTO objects (
|
|
1080
|
+
const insertObject = prepare(bundle.db, `INSERT OR IGNORE INTO objects (
|
|
1011
1081
|
object_id, hash_alg, hash, size_bytes, compressed_size_bytes,
|
|
1012
1082
|
compression, mime_type, encoding, storage_path, created_at
|
|
1013
|
-
) VALUES (?, 'blake3', ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
1014
|
-
);
|
|
1083
|
+
) VALUES (?, 'blake3', ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
1015
1084
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1016
1085
|
for (const p of toWrite) {
|
|
1017
|
-
insertObject.run(
|
|
1018
|
-
p.staged.objectId,
|
|
1019
|
-
p.staged.hash,
|
|
1020
|
-
p.staged.bytes.byteLength,
|
|
1021
|
-
p.compression === "zstd" ? p.compressedBytes.byteLength : null,
|
|
1022
|
-
p.compression,
|
|
1023
|
-
p.staged.mimeType,
|
|
1024
|
-
p.staged.encoding,
|
|
1025
|
-
p.storagePath,
|
|
1026
|
-
now
|
|
1027
|
-
);
|
|
1086
|
+
insertObject.run(p.staged.objectId, p.staged.hash, p.staged.bytes.byteLength, p.compression === "zstd" ? p.compressedBytes.byteLength : null, p.compression, p.staged.mimeType, p.staged.encoding, p.storagePath, now);
|
|
1028
1087
|
}
|
|
1029
1088
|
}
|
|
1089
|
+
__name(flushPendingObjects, "flushPendingObjects");
|
|
1030
1090
|
function queryExistingObjectIds(bundle, ids) {
|
|
1031
1091
|
const found = /* @__PURE__ */ new Set();
|
|
1032
1092
|
if (ids.length === 0) return found;
|
|
@@ -1034,75 +1094,141 @@ function queryExistingObjectIds(bundle, ids) {
|
|
|
1034
1094
|
for (let start = 0; start < ids.length; start += CHUNK) {
|
|
1035
1095
|
const slice = ids.slice(start, start + CHUNK);
|
|
1036
1096
|
const placeholders = slice.map(() => "?").join(",");
|
|
1037
|
-
const rows = bundle.db.prepare(
|
|
1038
|
-
`SELECT object_id FROM objects WHERE object_id IN (${placeholders})`
|
|
1039
|
-
).all(...slice);
|
|
1097
|
+
const rows = bundle.db.prepare(`SELECT object_id FROM objects WHERE object_id IN (${placeholders})`).all(...slice);
|
|
1040
1098
|
for (const row of rows) found.add(row.object_id);
|
|
1041
1099
|
}
|
|
1042
1100
|
return found;
|
|
1043
1101
|
}
|
|
1102
|
+
__name(queryExistingObjectIds, "queryExistingObjectIds");
|
|
1044
1103
|
var FS_WRITE_CONCURRENCY = 16;
|
|
1045
1104
|
async function writeFilesParallel(tasks) {
|
|
1046
1105
|
let cursor = 0;
|
|
1047
1106
|
const workers = [];
|
|
1048
1107
|
const limit2 = Math.min(FS_WRITE_CONCURRENCY, tasks.length);
|
|
1049
1108
|
for (let w = 0; w < limit2; w++) {
|
|
1050
|
-
workers.push(
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
})()
|
|
1060
|
-
);
|
|
1109
|
+
workers.push((async () => {
|
|
1110
|
+
while (true) {
|
|
1111
|
+
const i = cursor++;
|
|
1112
|
+
if (i >= tasks.length) return;
|
|
1113
|
+
const task = tasks[i];
|
|
1114
|
+
await ensureDir(path2.dirname(task.absolutePath));
|
|
1115
|
+
await writeFile2(task.absolutePath, task.compressedBytes);
|
|
1116
|
+
}
|
|
1117
|
+
})());
|
|
1061
1118
|
}
|
|
1062
1119
|
await Promise.all(workers);
|
|
1063
1120
|
}
|
|
1121
|
+
__name(writeFilesParallel, "writeFilesParallel");
|
|
1064
1122
|
|
|
1065
1123
|
// src/core/domain/ids.ts
|
|
1066
1124
|
var ID_PREFIX_BYTES = 16;
|
|
1067
1125
|
function tupleId(parts) {
|
|
1068
1126
|
return sha256Hex(parts.join("\0")).slice(0, ID_PREFIX_BYTES * 2);
|
|
1069
1127
|
}
|
|
1128
|
+
__name(tupleId, "tupleId");
|
|
1070
1129
|
function sourceFileId(sourceTool, absolutePath, contentHash) {
|
|
1071
|
-
return tupleId([
|
|
1130
|
+
return tupleId([
|
|
1131
|
+
"source_file",
|
|
1132
|
+
sourceTool,
|
|
1133
|
+
absolutePath,
|
|
1134
|
+
contentHash
|
|
1135
|
+
]);
|
|
1072
1136
|
}
|
|
1137
|
+
__name(sourceFileId, "sourceFileId");
|
|
1073
1138
|
function rawRecordId(sourceFileId2, ordinal, rawObjectId) {
|
|
1074
|
-
return tupleId([
|
|
1139
|
+
return tupleId([
|
|
1140
|
+
"raw_record",
|
|
1141
|
+
sourceFileId2,
|
|
1142
|
+
String(ordinal ?? -1),
|
|
1143
|
+
rawObjectId
|
|
1144
|
+
]);
|
|
1075
1145
|
}
|
|
1146
|
+
__name(rawRecordId, "rawRecordId");
|
|
1076
1147
|
function sessionId(sourceTool, sourceSessionId) {
|
|
1077
|
-
return tupleId([
|
|
1148
|
+
return tupleId([
|
|
1149
|
+
"session",
|
|
1150
|
+
sourceTool,
|
|
1151
|
+
sourceSessionId
|
|
1152
|
+
]);
|
|
1078
1153
|
}
|
|
1154
|
+
__name(sessionId, "sessionId");
|
|
1079
1155
|
function turnId(sessionId2, ordinal, sourceTurnId) {
|
|
1080
|
-
return tupleId([
|
|
1156
|
+
return tupleId([
|
|
1157
|
+
"turn",
|
|
1158
|
+
sessionId2,
|
|
1159
|
+
String(ordinal),
|
|
1160
|
+
sourceTurnId ?? ""
|
|
1161
|
+
]);
|
|
1081
1162
|
}
|
|
1163
|
+
__name(turnId, "turnId");
|
|
1082
1164
|
function eventId(sessionId2, ordinal, kind) {
|
|
1083
|
-
return tupleId([
|
|
1165
|
+
return tupleId([
|
|
1166
|
+
"event",
|
|
1167
|
+
sessionId2,
|
|
1168
|
+
String(ordinal),
|
|
1169
|
+
kind
|
|
1170
|
+
]);
|
|
1084
1171
|
}
|
|
1172
|
+
__name(eventId, "eventId");
|
|
1085
1173
|
function messageId(sessionId2, ordinal, sourceMsgId) {
|
|
1086
|
-
return tupleId([
|
|
1174
|
+
return tupleId([
|
|
1175
|
+
"message",
|
|
1176
|
+
sessionId2,
|
|
1177
|
+
String(ordinal),
|
|
1178
|
+
sourceMsgId ?? ""
|
|
1179
|
+
]);
|
|
1087
1180
|
}
|
|
1181
|
+
__name(messageId, "messageId");
|
|
1088
1182
|
function blockId(messageOrEventId, ordinal) {
|
|
1089
|
-
return tupleId([
|
|
1183
|
+
return tupleId([
|
|
1184
|
+
"block",
|
|
1185
|
+
messageOrEventId,
|
|
1186
|
+
String(ordinal)
|
|
1187
|
+
]);
|
|
1090
1188
|
}
|
|
1189
|
+
__name(blockId, "blockId");
|
|
1091
1190
|
function toolCallId(sessionId2, sourceCallId) {
|
|
1092
|
-
return tupleId([
|
|
1191
|
+
return tupleId([
|
|
1192
|
+
"tool_call",
|
|
1193
|
+
sessionId2,
|
|
1194
|
+
sourceCallId
|
|
1195
|
+
]);
|
|
1093
1196
|
}
|
|
1197
|
+
__name(toolCallId, "toolCallId");
|
|
1094
1198
|
function toolResultId(sessionId2, sourceCallId) {
|
|
1095
|
-
return tupleId([
|
|
1199
|
+
return tupleId([
|
|
1200
|
+
"tool_result",
|
|
1201
|
+
sessionId2,
|
|
1202
|
+
sourceCallId
|
|
1203
|
+
]);
|
|
1096
1204
|
}
|
|
1205
|
+
__name(toolResultId, "toolResultId");
|
|
1097
1206
|
function artifactId(sessionId2, sourceTool, key) {
|
|
1098
|
-
return tupleId([
|
|
1207
|
+
return tupleId([
|
|
1208
|
+
"artifact",
|
|
1209
|
+
sessionId2 ?? "",
|
|
1210
|
+
sourceTool,
|
|
1211
|
+
key
|
|
1212
|
+
]);
|
|
1099
1213
|
}
|
|
1214
|
+
__name(artifactId, "artifactId");
|
|
1100
1215
|
function projectId(sourceTool, sourceProjectId) {
|
|
1101
|
-
return tupleId([
|
|
1216
|
+
return tupleId([
|
|
1217
|
+
"project",
|
|
1218
|
+
sourceTool,
|
|
1219
|
+
sourceProjectId
|
|
1220
|
+
]);
|
|
1102
1221
|
}
|
|
1222
|
+
__name(projectId, "projectId");
|
|
1103
1223
|
function importBatchId(sourceTool, startedAtIso) {
|
|
1104
|
-
return tupleId([
|
|
1224
|
+
return tupleId([
|
|
1225
|
+
"import_batch",
|
|
1226
|
+
sourceTool,
|
|
1227
|
+
startedAtIso,
|
|
1228
|
+
String(Math.random())
|
|
1229
|
+
]);
|
|
1105
1230
|
}
|
|
1231
|
+
__name(importBatchId, "importBatchId");
|
|
1106
1232
|
|
|
1107
1233
|
// src/core/ingest/batch.ts
|
|
1108
1234
|
function emptyCounts() {
|
|
@@ -1123,15 +1249,13 @@ function emptyCounts() {
|
|
|
1123
1249
|
errors: 0
|
|
1124
1250
|
};
|
|
1125
1251
|
}
|
|
1252
|
+
__name(emptyCounts, "emptyCounts");
|
|
1126
1253
|
function startBatch(bundle, sourceTool, paths) {
|
|
1127
1254
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
1128
1255
|
const id = importBatchId(sourceTool ?? "all", startedAt);
|
|
1129
|
-
prepare(
|
|
1130
|
-
bundle.db,
|
|
1131
|
-
`INSERT INTO import_batches (
|
|
1256
|
+
prepare(bundle.db, `INSERT INTO import_batches (
|
|
1132
1257
|
batch_id, parser_version, source_tool, paths, started_at, status
|
|
1133
|
-
) VALUES (?, ?, ?, ?, ?, 'running')`
|
|
1134
|
-
).run(id, PROSA_PARSER_VERSION, sourceTool, JSON.stringify(paths), startedAt);
|
|
1258
|
+
) VALUES (?, ?, ?, ?, ?, 'running')`).run(id, PROSA_PARSER_VERSION, sourceTool, JSON.stringify(paths), startedAt);
|
|
1135
1259
|
return {
|
|
1136
1260
|
batch_id: id,
|
|
1137
1261
|
source_tool: sourceTool,
|
|
@@ -1140,35 +1264,24 @@ function startBatch(bundle, sourceTool, paths) {
|
|
|
1140
1264
|
started_at: startedAt
|
|
1141
1265
|
};
|
|
1142
1266
|
}
|
|
1267
|
+
__name(startBatch, "startBatch");
|
|
1143
1268
|
function finishBatch(bundle, batch, counts, status) {
|
|
1144
|
-
prepare(
|
|
1145
|
-
bundle.db,
|
|
1146
|
-
`UPDATE import_batches
|
|
1269
|
+
prepare(bundle.db, `UPDATE import_batches
|
|
1147
1270
|
SET finished_at = ?, status = ?, counts_json = ?
|
|
1148
|
-
WHERE batch_id = ?`
|
|
1149
|
-
).run((/* @__PURE__ */ new Date()).toISOString(), status, JSON.stringify(counts), batch.batch_id);
|
|
1271
|
+
WHERE batch_id = ?`).run((/* @__PURE__ */ new Date()).toISOString(), status, JSON.stringify(counts), batch.batch_id);
|
|
1150
1272
|
}
|
|
1273
|
+
__name(finishBatch, "finishBatch");
|
|
1151
1274
|
async function recordError(bundle, batchId, args) {
|
|
1152
1275
|
let payloadObjectId = null;
|
|
1153
1276
|
if (args.payload !== void 0) {
|
|
1154
1277
|
payloadObjectId = await putJson(bundle, args.payload);
|
|
1155
1278
|
}
|
|
1156
|
-
prepare(
|
|
1157
|
-
bundle.db,
|
|
1158
|
-
`INSERT INTO import_errors (
|
|
1279
|
+
prepare(bundle.db, `INSERT INTO import_errors (
|
|
1159
1280
|
batch_id, source_file_id, raw_record_id, kind, message,
|
|
1160
1281
|
payload_object_id, occurred_at
|
|
1161
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?)`
|
|
1162
|
-
).run(
|
|
1163
|
-
batchId,
|
|
1164
|
-
args.sourceFileId ?? null,
|
|
1165
|
-
args.rawRecordId ?? null,
|
|
1166
|
-
args.kind,
|
|
1167
|
-
args.message,
|
|
1168
|
-
payloadObjectId,
|
|
1169
|
-
(/* @__PURE__ */ new Date()).toISOString()
|
|
1170
|
-
);
|
|
1282
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?)`).run(batchId, args.sourceFileId ?? null, args.rawRecordId ?? null, args.kind, args.message, payloadObjectId, (/* @__PURE__ */ new Date()).toISOString());
|
|
1171
1283
|
}
|
|
1284
|
+
__name(recordError, "recordError");
|
|
1172
1285
|
|
|
1173
1286
|
// src/core/ingest/idempotency.ts
|
|
1174
1287
|
import { access as access2, readFile as readFile3, stat as stat2, writeFile as writeFile3 } from "fs/promises";
|
|
@@ -1177,14 +1290,11 @@ async function registerSourceFile(bundle, args) {
|
|
|
1177
1290
|
const st = await stat2(args.absolutePath);
|
|
1178
1291
|
const size = st.size;
|
|
1179
1292
|
const mtime = st.mtime.toISOString();
|
|
1180
|
-
const cheap = prepare(
|
|
1181
|
-
bundle.db,
|
|
1182
|
-
`SELECT source_file_id, source_tool, path, file_kind, size_bytes, mtime,
|
|
1293
|
+
const cheap = prepare(bundle.db, `SELECT source_file_id, source_tool, path, file_kind, size_bytes, mtime,
|
|
1183
1294
|
content_hash, object_id, discovered_at, workspace_hint
|
|
1184
1295
|
FROM source_files
|
|
1185
1296
|
WHERE source_tool = ? AND path = ? AND size_bytes = ? AND mtime = ?
|
|
1186
|
-
LIMIT 1`
|
|
1187
|
-
).get(args.sourceTool, args.absolutePath, size, mtime);
|
|
1297
|
+
LIMIT 1`).get(args.sourceTool, args.absolutePath, size, mtime);
|
|
1188
1298
|
if (cheap) {
|
|
1189
1299
|
return {
|
|
1190
1300
|
row: await ensureSourceFilePreserved(bundle, cheap, args.absolutePath),
|
|
@@ -1193,14 +1303,11 @@ async function registerSourceFile(bundle, args) {
|
|
|
1193
1303
|
}
|
|
1194
1304
|
const buf = await readFile3(args.absolutePath);
|
|
1195
1305
|
const contentHash = sha256Hex(buf);
|
|
1196
|
-
const exact = prepare(
|
|
1197
|
-
bundle.db,
|
|
1198
|
-
`SELECT source_file_id, source_tool, path, file_kind, size_bytes, mtime,
|
|
1306
|
+
const exact = prepare(bundle.db, `SELECT source_file_id, source_tool, path, file_kind, size_bytes, mtime,
|
|
1199
1307
|
content_hash, object_id, discovered_at, workspace_hint
|
|
1200
1308
|
FROM source_files
|
|
1201
1309
|
WHERE source_tool = ? AND path = ? AND content_hash = ?
|
|
1202
|
-
LIMIT 1`
|
|
1203
|
-
).get(args.sourceTool, args.absolutePath, contentHash);
|
|
1310
|
+
LIMIT 1`).get(args.sourceTool, args.absolutePath, contentHash);
|
|
1204
1311
|
if (exact) {
|
|
1205
1312
|
return {
|
|
1206
1313
|
row: await ensureSourceFilePreserved(bundle, exact, args.absolutePath, buf),
|
|
@@ -1221,36 +1328,27 @@ async function registerSourceFile(bundle, args) {
|
|
|
1221
1328
|
discovered_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1222
1329
|
workspace_hint: args.workspaceHint ?? null
|
|
1223
1330
|
};
|
|
1224
|
-
prepare(
|
|
1225
|
-
bundle.db,
|
|
1226
|
-
`INSERT INTO source_files (
|
|
1331
|
+
prepare(bundle.db, `INSERT INTO source_files (
|
|
1227
1332
|
source_file_id, source_tool, path, file_kind, size_bytes, mtime,
|
|
1228
1333
|
content_hash, object_id, discovered_at, workspace_hint
|
|
1229
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
1230
|
-
|
|
1231
|
-
row
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
row.file_kind,
|
|
1235
|
-
row.size_bytes,
|
|
1236
|
-
row.mtime,
|
|
1237
|
-
row.content_hash,
|
|
1238
|
-
row.object_id,
|
|
1239
|
-
row.discovered_at,
|
|
1240
|
-
row.workspace_hint
|
|
1241
|
-
);
|
|
1242
|
-
return { row, alreadyKnown: false };
|
|
1334
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(row.source_file_id, row.source_tool, row.path, row.file_kind, row.size_bytes, row.mtime, row.content_hash, row.object_id, row.discovered_at, row.workspace_hint);
|
|
1335
|
+
return {
|
|
1336
|
+
row,
|
|
1337
|
+
alreadyKnown: false
|
|
1338
|
+
};
|
|
1243
1339
|
}
|
|
1340
|
+
__name(registerSourceFile, "registerSourceFile");
|
|
1244
1341
|
async function ensureSourceFilePreserved(bundle, row, absolutePath, bytes) {
|
|
1245
1342
|
if (row.object_id) return row;
|
|
1246
1343
|
const sourceBytes = bytes ?? await readFile3(absolutePath);
|
|
1247
1344
|
const objectId = await preserveRawSourceBytes(bundle, sourceBytes);
|
|
1248
|
-
prepare(
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1345
|
+
prepare(bundle.db, `UPDATE source_files SET object_id = ? WHERE source_file_id = ?`).run(objectId, row.source_file_id);
|
|
1346
|
+
return {
|
|
1347
|
+
...row,
|
|
1348
|
+
object_id: objectId
|
|
1349
|
+
};
|
|
1253
1350
|
}
|
|
1351
|
+
__name(ensureSourceFilePreserved, "ensureSourceFilePreserved");
|
|
1254
1352
|
async function preserveRawSourceBytes(bundle, bytes) {
|
|
1255
1353
|
const hash = blake3Hex(bytes);
|
|
1256
1354
|
const objectId = objectIdFromHash(hash);
|
|
@@ -1261,35 +1359,21 @@ async function preserveRawSourceBytes(bundle, bytes) {
|
|
|
1261
1359
|
if (!await fileExists(absolutePath)) {
|
|
1262
1360
|
await writeFile3(absolutePath, stored);
|
|
1263
1361
|
}
|
|
1264
|
-
const existing = prepare(
|
|
1265
|
-
bundle.db,
|
|
1266
|
-
`SELECT object_id FROM objects WHERE object_id = ?`
|
|
1267
|
-
).get(objectId);
|
|
1362
|
+
const existing = prepare(bundle.db, `SELECT object_id FROM objects WHERE object_id = ?`).get(objectId);
|
|
1268
1363
|
if (!existing) {
|
|
1269
|
-
prepare(
|
|
1270
|
-
bundle.db,
|
|
1271
|
-
`INSERT INTO objects (
|
|
1364
|
+
prepare(bundle.db, `INSERT INTO objects (
|
|
1272
1365
|
object_id, hash_alg, hash, size_bytes, compressed_size_bytes,
|
|
1273
1366
|
compression, mime_type, encoding, storage_path, created_at
|
|
1274
|
-
) VALUES (?, 'blake3', ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
1275
|
-
).run(
|
|
1276
|
-
objectId,
|
|
1277
|
-
hash,
|
|
1278
|
-
bytes.byteLength,
|
|
1279
|
-
compression === "zstd" ? stored.byteLength : null,
|
|
1280
|
-
compression,
|
|
1281
|
-
"application/octet-stream",
|
|
1282
|
-
null,
|
|
1283
|
-
storagePath,
|
|
1284
|
-
(/* @__PURE__ */ new Date()).toISOString()
|
|
1285
|
-
);
|
|
1367
|
+
) VALUES (?, 'blake3', ?, ?, ?, ?, ?, ?, ?, ?)`).run(objectId, hash, bytes.byteLength, compression === "zstd" ? stored.byteLength : null, compression, "application/octet-stream", null, storagePath, (/* @__PURE__ */ new Date()).toISOString());
|
|
1286
1368
|
}
|
|
1287
1369
|
return objectId;
|
|
1288
1370
|
}
|
|
1371
|
+
__name(preserveRawSourceBytes, "preserveRawSourceBytes");
|
|
1289
1372
|
function rawSourceStoragePath(hashHex, compression) {
|
|
1290
1373
|
const ext = compression === "zstd" ? ".zst" : ".bin";
|
|
1291
1374
|
return `raw/sources/${hashHex}${ext}`;
|
|
1292
1375
|
}
|
|
1376
|
+
__name(rawSourceStoragePath, "rawSourceStoragePath");
|
|
1293
1377
|
async function fileExists(filePath) {
|
|
1294
1378
|
try {
|
|
1295
1379
|
await access2(filePath);
|
|
@@ -1298,11 +1382,13 @@ async function fileExists(filePath) {
|
|
|
1298
1382
|
return false;
|
|
1299
1383
|
}
|
|
1300
1384
|
}
|
|
1385
|
+
__name(fileExists, "fileExists");
|
|
1301
1386
|
|
|
1302
1387
|
// src/core/limits.ts
|
|
1303
1388
|
function clampLimit(value, opts) {
|
|
1304
1389
|
return Math.max(opts.min ?? 1, Math.min(opts.max, value ?? opts.fallback));
|
|
1305
1390
|
}
|
|
1391
|
+
__name(clampLimit, "clampLimit");
|
|
1306
1392
|
|
|
1307
1393
|
// src/services/sessions.ts
|
|
1308
1394
|
function sessionFilterWhere(filters) {
|
|
@@ -1325,9 +1411,13 @@ function sessionFilterWhere(filters) {
|
|
|
1325
1411
|
params
|
|
1326
1412
|
};
|
|
1327
1413
|
}
|
|
1414
|
+
__name(sessionFilterWhere, "sessionFilterWhere");
|
|
1328
1415
|
function listSessions(bundle, filters = {}) {
|
|
1329
1416
|
const { where, params } = sessionFilterWhere(filters);
|
|
1330
|
-
const limit2 = clampLimit(filters.limit, {
|
|
1417
|
+
const limit2 = clampLimit(filters.limit, {
|
|
1418
|
+
max: 1e3,
|
|
1419
|
+
fallback: 50
|
|
1420
|
+
});
|
|
1331
1421
|
const sql = `
|
|
1332
1422
|
SELECT s.session_id,
|
|
1333
1423
|
s.source_tool,
|
|
@@ -1352,33 +1442,30 @@ function listSessions(bundle, filters = {}) {
|
|
|
1352
1442
|
`;
|
|
1353
1443
|
return bundle.db.prepare(sql).all(...params);
|
|
1354
1444
|
}
|
|
1445
|
+
__name(listSessions, "listSessions");
|
|
1355
1446
|
function countSessions(bundle, filters = {}) {
|
|
1356
1447
|
const { where, params } = sessionFilterWhere(filters);
|
|
1357
|
-
const row = bundle.db.prepare(
|
|
1358
|
-
`
|
|
1448
|
+
const row = bundle.db.prepare(`
|
|
1359
1449
|
SELECT count(*) AS count
|
|
1360
1450
|
FROM sessions s
|
|
1361
1451
|
${where}
|
|
1362
|
-
`
|
|
1363
|
-
).get(...params);
|
|
1452
|
+
`).get(...params);
|
|
1364
1453
|
return row?.count ?? 0;
|
|
1365
1454
|
}
|
|
1455
|
+
__name(countSessions, "countSessions");
|
|
1366
1456
|
function getSession(bundle, sessionId2) {
|
|
1367
1457
|
const rows = listSessions(bundle);
|
|
1368
|
-
const row = bundle.db.prepare(
|
|
1369
|
-
`SELECT s.session_id, s.source_tool, s.source_session_id, s.parent_session_id,
|
|
1458
|
+
const row = bundle.db.prepare(`SELECT s.session_id, s.source_tool, s.source_session_id, s.parent_session_id,
|
|
1370
1459
|
s.is_subagent, s.title, s.start_ts, s.end_ts, s.cwd_initial,
|
|
1371
1460
|
s.git_branch_initial, s.model_first, s.model_last, s.status,
|
|
1372
1461
|
s.timeline_confidence,
|
|
1373
1462
|
(SELECT count(*) FROM messages m WHERE m.session_id = s.session_id) AS message_count,
|
|
1374
1463
|
(SELECT count(*) FROM tool_calls tc WHERE tc.session_id = s.session_id) AS tool_call_count
|
|
1375
1464
|
FROM sessions s
|
|
1376
|
-
WHERE s.session_id = ?`
|
|
1377
|
-
).get(sessionId2);
|
|
1465
|
+
WHERE s.session_id = ?`).get(sessionId2);
|
|
1378
1466
|
void rows;
|
|
1379
1467
|
if (!row) return null;
|
|
1380
|
-
const events = bundle.db.prepare(
|
|
1381
|
-
`SELECT e.ordinal,
|
|
1468
|
+
const events = bundle.db.prepare(`SELECT e.ordinal,
|
|
1382
1469
|
e.timestamp,
|
|
1383
1470
|
e.event_type,
|
|
1384
1471
|
e.source_type,
|
|
@@ -1394,17 +1481,20 @@ function getSession(bundle, sessionId2) {
|
|
|
1394
1481
|
LEFT JOIN tool_calls tc ON tc.event_id = e.event_id
|
|
1395
1482
|
LEFT JOIN tool_results tr ON tr.event_id = e.event_id
|
|
1396
1483
|
WHERE e.session_id = ?
|
|
1397
|
-
ORDER BY e.ordinal`
|
|
1398
|
-
|
|
1399
|
-
|
|
1484
|
+
ORDER BY e.ordinal`).all(sessionId2);
|
|
1485
|
+
return {
|
|
1486
|
+
session: row,
|
|
1487
|
+
events
|
|
1488
|
+
};
|
|
1400
1489
|
}
|
|
1490
|
+
__name(getSession, "getSession");
|
|
1401
1491
|
|
|
1402
1492
|
// src/services/search.ts
|
|
1403
1493
|
import { existsSync as existsSync2 } from "fs";
|
|
1404
1494
|
import { createRequire } from "module";
|
|
1405
1495
|
|
|
1406
1496
|
// src/core/errors.ts
|
|
1407
|
-
var getErrorMessage = (err) => err instanceof Error ? err.message : String(err);
|
|
1497
|
+
var getErrorMessage = /* @__PURE__ */ __name((err) => err instanceof Error ? err.message : String(err), "getErrorMessage");
|
|
1408
1498
|
|
|
1409
1499
|
// src/services/indexing.ts
|
|
1410
1500
|
import { createHash as createHash2 } from "crypto";
|
|
@@ -1436,6 +1526,7 @@ END;
|
|
|
1436
1526
|
function enableFts5Triggers(bundle) {
|
|
1437
1527
|
bundle.db.exec(FTS5_TRIGGER_SQL);
|
|
1438
1528
|
}
|
|
1529
|
+
__name(enableFts5Triggers, "enableFts5Triggers");
|
|
1439
1530
|
function disableFts5Triggers(bundle) {
|
|
1440
1531
|
bundle.db.exec(`
|
|
1441
1532
|
DROP TRIGGER IF EXISTS search_docs_ai;
|
|
@@ -1443,22 +1534,21 @@ function disableFts5Triggers(bundle) {
|
|
|
1443
1534
|
DROP TRIGGER IF EXISTS search_docs_au;
|
|
1444
1535
|
`);
|
|
1445
1536
|
}
|
|
1537
|
+
__name(disableFts5Triggers, "disableFts5Triggers");
|
|
1446
1538
|
function getSearchIndexStatuses(bundle) {
|
|
1447
1539
|
ensureSearchIndexStatusRows(bundle);
|
|
1448
|
-
return bundle.db.prepare(
|
|
1449
|
-
`SELECT ${SEARCH_INDEX_STATUS_COLUMNS}
|
|
1540
|
+
return bundle.db.prepare(`SELECT ${SEARCH_INDEX_STATUS_COLUMNS}
|
|
1450
1541
|
FROM search_index_status
|
|
1451
|
-
ORDER BY engine`
|
|
1452
|
-
).all();
|
|
1542
|
+
ORDER BY engine`).all();
|
|
1453
1543
|
}
|
|
1544
|
+
__name(getSearchIndexStatuses, "getSearchIndexStatuses");
|
|
1454
1545
|
function getSearchIndexStatus(bundle, engine) {
|
|
1455
1546
|
ensureSearchIndexStatusRows(bundle);
|
|
1456
|
-
return bundle.db.prepare(
|
|
1457
|
-
`SELECT ${SEARCH_INDEX_STATUS_COLUMNS}
|
|
1547
|
+
return bundle.db.prepare(`SELECT ${SEARCH_INDEX_STATUS_COLUMNS}
|
|
1458
1548
|
FROM search_index_status
|
|
1459
|
-
WHERE engine = ?`
|
|
1460
|
-
).get(engine) ?? null;
|
|
1549
|
+
WHERE engine = ?`).get(engine) ?? null;
|
|
1461
1550
|
}
|
|
1551
|
+
__name(getSearchIndexStatus, "getSearchIndexStatus");
|
|
1462
1552
|
function markIndexesAfterImport(bundle, options) {
|
|
1463
1553
|
if (!options.changed) return;
|
|
1464
1554
|
const tantivy = getSearchIndexStatus(bundle, "tantivy");
|
|
@@ -1471,6 +1561,7 @@ function markIndexesAfterImport(bundle, options) {
|
|
|
1471
1561
|
});
|
|
1472
1562
|
}
|
|
1473
1563
|
}
|
|
1564
|
+
__name(markIndexesAfterImport, "markIndexesAfterImport");
|
|
1474
1565
|
function rebuildFts5Index(bundle) {
|
|
1475
1566
|
ensureSearchIndexStatusRows(bundle);
|
|
1476
1567
|
updateSearchIndexStatus(bundle, "fts5", {
|
|
@@ -1501,38 +1592,80 @@ function rebuildFts5Index(bundle) {
|
|
|
1501
1592
|
}
|
|
1502
1593
|
return getSearchIndexStatus(bundle, "fts5");
|
|
1503
1594
|
}
|
|
1595
|
+
__name(rebuildFts5Index, "rebuildFts5Index");
|
|
1504
1596
|
var TANTIVY_SCHEMA_FIELDS = [
|
|
1505
|
-
{
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
{
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
{
|
|
1514
|
-
|
|
1597
|
+
{
|
|
1598
|
+
name: "doc_id",
|
|
1599
|
+
tokenizer: "raw"
|
|
1600
|
+
},
|
|
1601
|
+
{
|
|
1602
|
+
name: "entity_type",
|
|
1603
|
+
tokenizer: "raw"
|
|
1604
|
+
},
|
|
1605
|
+
{
|
|
1606
|
+
name: "entity_id",
|
|
1607
|
+
tokenizer: "raw"
|
|
1608
|
+
},
|
|
1609
|
+
{
|
|
1610
|
+
name: "session_id",
|
|
1611
|
+
tokenizer: "raw"
|
|
1612
|
+
},
|
|
1613
|
+
{
|
|
1614
|
+
name: "project_id",
|
|
1615
|
+
tokenizer: "raw"
|
|
1616
|
+
},
|
|
1617
|
+
{
|
|
1618
|
+
name: "timestamp",
|
|
1619
|
+
tokenizer: "raw"
|
|
1620
|
+
},
|
|
1621
|
+
{
|
|
1622
|
+
name: "role",
|
|
1623
|
+
tokenizer: "raw"
|
|
1624
|
+
},
|
|
1625
|
+
{
|
|
1626
|
+
name: "tool_name",
|
|
1627
|
+
tokenizer: "raw"
|
|
1628
|
+
},
|
|
1629
|
+
{
|
|
1630
|
+
name: "canonical_tool_type",
|
|
1631
|
+
tokenizer: "raw"
|
|
1632
|
+
},
|
|
1633
|
+
{
|
|
1634
|
+
name: "field_kind",
|
|
1635
|
+
tokenizer: "raw"
|
|
1636
|
+
},
|
|
1515
1637
|
// The text field uses tantivy's default tokenizer (en_stem in the binding).
|
|
1516
|
-
{
|
|
1638
|
+
{
|
|
1639
|
+
name: "text",
|
|
1640
|
+
tokenizer: "default"
|
|
1641
|
+
}
|
|
1517
1642
|
];
|
|
1518
1643
|
function buildTantivySchema(tantivy) {
|
|
1519
1644
|
const builder = new tantivy.SchemaBuilder();
|
|
1520
1645
|
for (const field of TANTIVY_SCHEMA_FIELDS) {
|
|
1521
1646
|
if (field.tokenizer === "default") {
|
|
1522
|
-
builder.addTextField(field.name, {
|
|
1647
|
+
builder.addTextField(field.name, {
|
|
1648
|
+
stored: true
|
|
1649
|
+
});
|
|
1523
1650
|
} else {
|
|
1524
|
-
builder.addTextField(field.name, {
|
|
1651
|
+
builder.addTextField(field.name, {
|
|
1652
|
+
stored: true,
|
|
1653
|
+
tokenizerName: field.tokenizer
|
|
1654
|
+
});
|
|
1525
1655
|
}
|
|
1526
1656
|
}
|
|
1527
1657
|
return builder.build();
|
|
1528
1658
|
}
|
|
1529
|
-
|
|
1659
|
+
__name(buildTantivySchema, "buildTantivySchema");
|
|
1660
|
+
function getCurrentTantivySchemaFingerprint() {
|
|
1530
1661
|
const canonical = TANTIVY_SCHEMA_FIELDS.map((f) => `${f.name}:${f.tokenizer}:stored`).join("|");
|
|
1531
1662
|
return createHash2("sha256").update(canonical).digest("hex");
|
|
1532
1663
|
}
|
|
1533
|
-
|
|
1664
|
+
__name(getCurrentTantivySchemaFingerprint, "getCurrentTantivySchemaFingerprint");
|
|
1665
|
+
function tantivyIndexDirIsValid(dir) {
|
|
1534
1666
|
return existsSync(path4.join(dir, "meta.json"));
|
|
1535
1667
|
}
|
|
1668
|
+
__name(tantivyIndexDirIsValid, "tantivyIndexDirIsValid");
|
|
1536
1669
|
function makeTantivyDoc(tantivy, row) {
|
|
1537
1670
|
const doc = new tantivy.Document();
|
|
1538
1671
|
doc.addText("doc_id", row.doc_id);
|
|
@@ -1548,6 +1681,7 @@ function makeTantivyDoc(tantivy, row) {
|
|
|
1548
1681
|
doc.addText("text", row.text);
|
|
1549
1682
|
return doc;
|
|
1550
1683
|
}
|
|
1684
|
+
__name(makeTantivyDoc, "makeTantivyDoc");
|
|
1551
1685
|
var SEARCH_DOCS_SELECT = `
|
|
1552
1686
|
SELECT rowid, doc_id, entity_type, entity_id, session_id, project_id, timestamp,
|
|
1553
1687
|
role, tool_name, canonical_tool_type, field_kind, text
|
|
@@ -1557,8 +1691,8 @@ async function rebuildTantivyIndex(bundle, options = {}) {
|
|
|
1557
1691
|
ensureSearchIndexStatusRows(bundle);
|
|
1558
1692
|
const sourceDocCount = countSearchDocs(bundle);
|
|
1559
1693
|
const prev = getSearchIndexStatus(bundle, "tantivy");
|
|
1560
|
-
const fingerprint =
|
|
1561
|
-
const indexDirValid =
|
|
1694
|
+
const fingerprint = getCurrentTantivySchemaFingerprint();
|
|
1695
|
+
const indexDirValid = tantivyIndexDirIsValid(bundle.paths.tantivy);
|
|
1562
1696
|
const fingerprintMatches = prev?.schema_fingerprint === fingerprint;
|
|
1563
1697
|
const lastIndexedRowid = typeof prev?.last_indexed_rowid === "number" ? prev.last_indexed_rowid : 0;
|
|
1564
1698
|
const wantFullRebuild = options.overwrite === true || !indexDirValid || !fingerprintMatches || lastIndexedRowid <= 0;
|
|
@@ -1573,8 +1707,13 @@ async function rebuildTantivyIndex(bundle, options = {}) {
|
|
|
1573
1707
|
const schema = buildTantivySchema(tantivy);
|
|
1574
1708
|
let index;
|
|
1575
1709
|
if (wantFullRebuild) {
|
|
1576
|
-
await rm(bundle.paths.tantivy, {
|
|
1577
|
-
|
|
1710
|
+
await rm(bundle.paths.tantivy, {
|
|
1711
|
+
recursive: true,
|
|
1712
|
+
force: true
|
|
1713
|
+
});
|
|
1714
|
+
await mkdir3(bundle.paths.tantivy, {
|
|
1715
|
+
recursive: true
|
|
1716
|
+
});
|
|
1578
1717
|
index = new tantivy.Index(schema, bundle.paths.tantivy, false);
|
|
1579
1718
|
} else {
|
|
1580
1719
|
index = tantivy.Index.open(bundle.paths.tantivy);
|
|
@@ -1595,25 +1734,17 @@ async function rebuildTantivyIndex(bundle, options = {}) {
|
|
|
1595
1734
|
index.reload();
|
|
1596
1735
|
writer.waitMergingThreads();
|
|
1597
1736
|
const indexedDocCount = wantFullRebuild ? addedDocCount : countTantivyDocsBest(prev, addedDocCount);
|
|
1598
|
-
await writeFile4(
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
schema_fingerprint: fingerprint
|
|
1610
|
-
},
|
|
1611
|
-
null,
|
|
1612
|
-
2
|
|
1613
|
-
)}
|
|
1614
|
-
`,
|
|
1615
|
-
"utf8"
|
|
1616
|
-
);
|
|
1737
|
+
await writeFile4(path4.join(bundle.paths.tantivy, "prosa-index.json"), `${JSON.stringify({
|
|
1738
|
+
engine: "tantivy",
|
|
1739
|
+
source: "search_docs",
|
|
1740
|
+
built_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1741
|
+
mode: wantFullRebuild ? "full" : "incremental",
|
|
1742
|
+
source_doc_count: sourceDocCount,
|
|
1743
|
+
indexed_doc_count: indexedDocCount,
|
|
1744
|
+
last_indexed_rowid: maxRowid,
|
|
1745
|
+
schema_fingerprint: fingerprint
|
|
1746
|
+
}, null, 2)}
|
|
1747
|
+
`, "utf8");
|
|
1617
1748
|
updateSearchIndexStatus(bundle, "tantivy", {
|
|
1618
1749
|
status: "ready",
|
|
1619
1750
|
sourceDocCount,
|
|
@@ -1633,24 +1764,24 @@ async function rebuildTantivyIndex(bundle, options = {}) {
|
|
|
1633
1764
|
}
|
|
1634
1765
|
return getSearchIndexStatus(bundle, "tantivy");
|
|
1635
1766
|
}
|
|
1767
|
+
__name(rebuildTantivyIndex, "rebuildTantivyIndex");
|
|
1636
1768
|
function countTantivyDocsBest(prev, added) {
|
|
1637
1769
|
if (prev && typeof prev.indexed_doc_count === "number") {
|
|
1638
1770
|
return prev.indexed_doc_count + added;
|
|
1639
1771
|
}
|
|
1640
1772
|
return added;
|
|
1641
1773
|
}
|
|
1774
|
+
__name(countTantivyDocsBest, "countTantivyDocsBest");
|
|
1642
1775
|
function ensureSearchIndexStatusRows(bundle) {
|
|
1643
1776
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1644
|
-
const stmt = prepare(
|
|
1645
|
-
bundle.db,
|
|
1646
|
-
`INSERT OR IGNORE INTO search_index_status (
|
|
1777
|
+
const stmt = prepare(bundle.db, `INSERT OR IGNORE INTO search_index_status (
|
|
1647
1778
|
engine, status, source_doc_count, indexed_doc_count, updated_at,
|
|
1648
1779
|
error_message, last_indexed_rowid, schema_fingerprint
|
|
1649
|
-
) VALUES (?, ?, 0, 0, ?, NULL, NULL, NULL)`
|
|
1650
|
-
);
|
|
1780
|
+
) VALUES (?, ?, 0, 0, ?, NULL, NULL, NULL)`);
|
|
1651
1781
|
stmt.run("fts5", "ready", now);
|
|
1652
1782
|
stmt.run("tantivy", "missing", now);
|
|
1653
1783
|
}
|
|
1784
|
+
__name(ensureSearchIndexStatusRows, "ensureSearchIndexStatusRows");
|
|
1654
1785
|
function updateSearchIndexStatus(bundle, engine, values) {
|
|
1655
1786
|
ensureSearchIndexStatusRows(bundle);
|
|
1656
1787
|
const setClauses = [
|
|
@@ -1676,28 +1807,32 @@ function updateSearchIndexStatus(bundle, engine, values) {
|
|
|
1676
1807
|
params.push(values.schemaFingerprint);
|
|
1677
1808
|
}
|
|
1678
1809
|
params.push(engine);
|
|
1679
|
-
prepare(
|
|
1680
|
-
bundle.db,
|
|
1681
|
-
`UPDATE search_index_status SET ${setClauses.join(", ")} WHERE engine = ?`
|
|
1682
|
-
).run(...params);
|
|
1810
|
+
prepare(bundle.db, `UPDATE search_index_status SET ${setClauses.join(", ")} WHERE engine = ?`).run(...params);
|
|
1683
1811
|
}
|
|
1812
|
+
__name(updateSearchIndexStatus, "updateSearchIndexStatus");
|
|
1684
1813
|
function countSearchDocs(bundle) {
|
|
1685
1814
|
return bundle.db.prepare(`SELECT count(*) AS n FROM search_docs`).get()?.n ?? 0;
|
|
1686
1815
|
}
|
|
1816
|
+
__name(countSearchDocs, "countSearchDocs");
|
|
1687
1817
|
function countFts5Docs(bundle) {
|
|
1688
1818
|
return bundle.db.prepare(`SELECT count(*) AS n FROM search_docs_fts`).get()?.n ?? 0;
|
|
1689
1819
|
}
|
|
1820
|
+
__name(countFts5Docs, "countFts5Docs");
|
|
1690
1821
|
|
|
1691
1822
|
// src/services/search.ts
|
|
1692
1823
|
var require2 = createRequire(import.meta.url);
|
|
1693
1824
|
function escapeFtsQuery(q) {
|
|
1694
1825
|
return q.split(/\s+/).filter((t) => t.length > 0).map((t) => `"${t.replace(/"/g, '""')}"`).join(" ");
|
|
1695
1826
|
}
|
|
1827
|
+
__name(escapeFtsQuery, "escapeFtsQuery");
|
|
1696
1828
|
function searchFullText(bundle, options) {
|
|
1697
1829
|
if (options.engine === "tantivy") {
|
|
1698
1830
|
return searchTantivy(bundle, options);
|
|
1699
1831
|
}
|
|
1700
|
-
const limit2 = clampLimit(options.limit, {
|
|
1832
|
+
const limit2 = clampLimit(options.limit, {
|
|
1833
|
+
max: 500,
|
|
1834
|
+
fallback: 50
|
|
1835
|
+
});
|
|
1701
1836
|
const sql = `
|
|
1702
1837
|
SELECT d.doc_id,
|
|
1703
1838
|
d.entity_type,
|
|
@@ -1718,24 +1853,36 @@ function searchFullText(bundle, options) {
|
|
|
1718
1853
|
if (!ftsQuery) return [];
|
|
1719
1854
|
return bundle.db.prepare(sql).all(ftsQuery);
|
|
1720
1855
|
}
|
|
1856
|
+
__name(searchFullText, "searchFullText");
|
|
1721
1857
|
function searchTantivy(bundle, options) {
|
|
1722
1858
|
if (!existsSync2(bundle.paths.tantivy)) {
|
|
1723
1859
|
throw new Error("tantivy index not found; run `prosa index tantivy` first");
|
|
1724
1860
|
}
|
|
1725
1861
|
const status = getSearchIndexStatus(bundle, "tantivy");
|
|
1726
1862
|
if (status?.status !== "ready") {
|
|
1727
|
-
throw new Error(
|
|
1728
|
-
`tantivy index is ${status?.status ?? "missing"}; run \`prosa index tantivy\` first`
|
|
1729
|
-
);
|
|
1863
|
+
throw new Error(`tantivy index is ${status?.status ?? "missing"}; run \`prosa index tantivy\` first`);
|
|
1730
1864
|
}
|
|
1731
|
-
const limit2 = clampLimit(options.limit, {
|
|
1865
|
+
const limit2 = clampLimit(options.limit, {
|
|
1866
|
+
max: 500,
|
|
1867
|
+
fallback: 50
|
|
1868
|
+
});
|
|
1732
1869
|
const queryText = options.query.trim();
|
|
1733
1870
|
if (!queryText) return [];
|
|
1734
1871
|
const tantivy = requireTantivy();
|
|
1735
1872
|
const index = tantivy.Index.open(bundle.paths.tantivy);
|
|
1736
1873
|
const searcher = index.searcher();
|
|
1737
|
-
const [query] = options.raw ? [
|
|
1738
|
-
|
|
1874
|
+
const [query] = options.raw ? [
|
|
1875
|
+
index.parseQuery(queryText, [
|
|
1876
|
+
"text"
|
|
1877
|
+
])
|
|
1878
|
+
] : index.parseQueryLenient(queryText, [
|
|
1879
|
+
"text"
|
|
1880
|
+
], void 0, {
|
|
1881
|
+
text: [
|
|
1882
|
+
true,
|
|
1883
|
+
2,
|
|
1884
|
+
true
|
|
1885
|
+
]
|
|
1739
1886
|
});
|
|
1740
1887
|
const result = searcher.search(query, limit2, true);
|
|
1741
1888
|
const snippets = tantivy.SnippetGenerator.create(searcher, query, index.schema, "text");
|
|
@@ -1758,6 +1905,7 @@ function searchTantivy(bundle, options) {
|
|
|
1758
1905
|
};
|
|
1759
1906
|
});
|
|
1760
1907
|
}
|
|
1908
|
+
__name(searchTantivy, "searchTantivy");
|
|
1761
1909
|
function requireTantivy() {
|
|
1762
1910
|
try {
|
|
1763
1911
|
return require2("@oxdev03/node-tantivy-binding");
|
|
@@ -1765,6 +1913,7 @@ function requireTantivy() {
|
|
|
1765
1913
|
throw new Error(`tantivy engine is unavailable: ${getErrorMessage(error)}`);
|
|
1766
1914
|
}
|
|
1767
1915
|
}
|
|
1916
|
+
__name(requireTantivy, "requireTantivy");
|
|
1768
1917
|
function getStoredText(doc, field) {
|
|
1769
1918
|
const value = doc.getFirst(field);
|
|
1770
1919
|
if (typeof value === "string") return value;
|
|
@@ -1772,9 +1921,11 @@ function getStoredText(doc, field) {
|
|
|
1772
1921
|
if (value == null) return "";
|
|
1773
1922
|
return String(value);
|
|
1774
1923
|
}
|
|
1924
|
+
__name(getStoredText, "getStoredText");
|
|
1775
1925
|
function nullIfEmpty(value) {
|
|
1776
1926
|
return value.length > 0 ? value : null;
|
|
1777
1927
|
}
|
|
1928
|
+
__name(nullIfEmpty, "nullIfEmpty");
|
|
1778
1929
|
function highlightSnippet(fragment, ranges) {
|
|
1779
1930
|
if (ranges.length === 0) return fragment;
|
|
1780
1931
|
let out = "";
|
|
@@ -1787,6 +1938,7 @@ function highlightSnippet(fragment, ranges) {
|
|
|
1787
1938
|
out += fragment.slice(cursor);
|
|
1788
1939
|
return out;
|
|
1789
1940
|
}
|
|
1941
|
+
__name(highlightSnippet, "highlightSnippet");
|
|
1790
1942
|
|
|
1791
1943
|
// src/services/compile.ts
|
|
1792
1944
|
import os2 from "os";
|
|
@@ -1807,6 +1959,7 @@ async function* discoverClaudeFiles(root) {
|
|
|
1807
1959
|
yield* walkProject(projectRoot, project.name);
|
|
1808
1960
|
}
|
|
1809
1961
|
}
|
|
1962
|
+
__name(discoverClaudeFiles, "discoverClaudeFiles");
|
|
1810
1963
|
async function* walkProject(projectRoot, projectSlug) {
|
|
1811
1964
|
const entries = await readdirSafe(projectRoot);
|
|
1812
1965
|
for (const entry of entries) {
|
|
@@ -1829,9 +1982,7 @@ async function* walkProject(projectRoot, projectSlug) {
|
|
|
1829
1982
|
if (!sub.name.startsWith("agent-")) continue;
|
|
1830
1983
|
const agentId = sub.name.slice("agent-".length, -".jsonl".length);
|
|
1831
1984
|
const metaCandidate = path5.join(subagentsDir, `agent-${agentId}.meta.json`);
|
|
1832
|
-
const metaExists = subagentEntries.some(
|
|
1833
|
-
(e) => e.isFile() && e.name === `agent-${agentId}.meta.json`
|
|
1834
|
-
);
|
|
1985
|
+
const metaExists = subagentEntries.some((e) => e.isFile() && e.name === `agent-${agentId}.meta.json`);
|
|
1835
1986
|
yield {
|
|
1836
1987
|
filePath: path5.join(subagentsDir, sub.name),
|
|
1837
1988
|
projectSlug,
|
|
@@ -1844,62 +1995,80 @@ async function* walkProject(projectRoot, projectSlug) {
|
|
|
1844
1995
|
}
|
|
1845
1996
|
}
|
|
1846
1997
|
}
|
|
1998
|
+
__name(walkProject, "walkProject");
|
|
1847
1999
|
async function readdirSafe(dir) {
|
|
1848
2000
|
try {
|
|
1849
|
-
return await readdir(dir, {
|
|
2001
|
+
return await readdir(dir, {
|
|
2002
|
+
withFileTypes: true
|
|
2003
|
+
});
|
|
1850
2004
|
} catch {
|
|
1851
2005
|
return [];
|
|
1852
2006
|
}
|
|
1853
2007
|
}
|
|
2008
|
+
__name(readdirSafe, "readdirSafe");
|
|
1854
2009
|
|
|
1855
2010
|
// src/importers/claude/index.ts
|
|
1856
2011
|
var PREVIEW_MAX = 4e3;
|
|
1857
2012
|
async function compileClaude(bundle, root, options = {}) {
|
|
1858
2013
|
const logger = options.logger;
|
|
1859
|
-
const batch = startBatch(bundle, "claude", [
|
|
2014
|
+
const batch = startBatch(bundle, "claude", [
|
|
2015
|
+
root
|
|
2016
|
+
]);
|
|
1860
2017
|
const counts = emptyCounts();
|
|
1861
|
-
logger?.info({
|
|
2018
|
+
logger?.info({
|
|
2019
|
+
batch_id: batch.batch_id,
|
|
2020
|
+
root
|
|
2021
|
+
}, "claude batch started");
|
|
1862
2022
|
try {
|
|
1863
2023
|
for await (const file of discoverClaudeFiles(root)) {
|
|
1864
2024
|
counts.source_files_seen++;
|
|
1865
|
-
logger?.debug(
|
|
1866
|
-
|
|
1867
|
-
|
|
1868
|
-
|
|
1869
|
-
|
|
1870
|
-
},
|
|
1871
|
-
"claude source file discovered"
|
|
1872
|
-
);
|
|
2025
|
+
logger?.debug({
|
|
2026
|
+
path: file.filePath,
|
|
2027
|
+
project_slug: file.projectSlug,
|
|
2028
|
+
is_subagent: file.isSubagent
|
|
2029
|
+
}, "claude source file discovered");
|
|
1873
2030
|
try {
|
|
1874
2031
|
const fc = await compileClaudeFile(bundle, batch, file, logger);
|
|
1875
2032
|
addCounts(counts, fc);
|
|
1876
2033
|
} catch (error) {
|
|
1877
2034
|
counts.errors++;
|
|
1878
|
-
logger?.warn(
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
|
|
1882
|
-
},
|
|
1883
|
-
"claude source file failed"
|
|
1884
|
-
);
|
|
2035
|
+
logger?.warn({
|
|
2036
|
+
err: error,
|
|
2037
|
+
path: file.filePath
|
|
2038
|
+
}, "claude source file failed");
|
|
1885
2039
|
await recordError(bundle, batch.batch_id, {
|
|
1886
2040
|
kind: "claude_file_failed",
|
|
1887
2041
|
message: getErrorMessage(error),
|
|
1888
|
-
payload: {
|
|
2042
|
+
payload: {
|
|
2043
|
+
path: file.filePath
|
|
2044
|
+
}
|
|
1889
2045
|
});
|
|
1890
2046
|
}
|
|
1891
2047
|
}
|
|
1892
2048
|
linkSubagentParents(bundle);
|
|
1893
|
-
logger?.debug({
|
|
2049
|
+
logger?.debug({
|
|
2050
|
+
batch_id: batch.batch_id
|
|
2051
|
+
}, "claude subagent parent links refreshed");
|
|
1894
2052
|
finishBatch(bundle, batch, counts, "completed");
|
|
1895
|
-
logger?.info({
|
|
2053
|
+
logger?.info({
|
|
2054
|
+
batch_id: batch.batch_id,
|
|
2055
|
+
counts
|
|
2056
|
+
}, "claude batch completed");
|
|
1896
2057
|
} catch (error) {
|
|
1897
2058
|
finishBatch(bundle, batch, counts, "failed");
|
|
1898
|
-
logger?.error({
|
|
2059
|
+
logger?.error({
|
|
2060
|
+
err: error,
|
|
2061
|
+
batch_id: batch.batch_id,
|
|
2062
|
+
counts
|
|
2063
|
+
}, "claude batch failed");
|
|
1899
2064
|
throw error;
|
|
1900
2065
|
}
|
|
1901
|
-
return {
|
|
2066
|
+
return {
|
|
2067
|
+
batch,
|
|
2068
|
+
counts
|
|
2069
|
+
};
|
|
1902
2070
|
}
|
|
2071
|
+
__name(compileClaude, "compileClaude");
|
|
1903
2072
|
function linkSubagentParents(bundle) {
|
|
1904
2073
|
bundle.db.exec(`
|
|
1905
2074
|
UPDATE sessions
|
|
@@ -1917,6 +2086,7 @@ function linkSubagentParents(bundle) {
|
|
|
1917
2086
|
AND source_tool = 'claude'
|
|
1918
2087
|
`);
|
|
1919
2088
|
}
|
|
2089
|
+
__name(linkSubagentParents, "linkSubagentParents");
|
|
1920
2090
|
function emptyFileCounts() {
|
|
1921
2091
|
return {
|
|
1922
2092
|
source_files_imported: 0,
|
|
@@ -1934,6 +2104,7 @@ function emptyFileCounts() {
|
|
|
1934
2104
|
errors: 0
|
|
1935
2105
|
};
|
|
1936
2106
|
}
|
|
2107
|
+
__name(emptyFileCounts, "emptyFileCounts");
|
|
1937
2108
|
function addCounts(target, source) {
|
|
1938
2109
|
target.source_files_imported += source.source_files_imported;
|
|
1939
2110
|
target.source_files_skipped += source.source_files_skipped;
|
|
@@ -1949,6 +2120,7 @@ function addCounts(target, source) {
|
|
|
1949
2120
|
target.edges += source.edges;
|
|
1950
2121
|
target.errors += source.errors;
|
|
1951
2122
|
}
|
|
2123
|
+
__name(addCounts, "addCounts");
|
|
1952
2124
|
async function compileClaudeFile(bundle, batch, file, logger) {
|
|
1953
2125
|
const counts = emptyFileCounts();
|
|
1954
2126
|
const { row: sourceFile, alreadyKnown } = await registerSourceFile(bundle, {
|
|
@@ -1959,17 +2131,17 @@ async function compileClaudeFile(bundle, batch, file, logger) {
|
|
|
1959
2131
|
});
|
|
1960
2132
|
if (alreadyKnown) {
|
|
1961
2133
|
counts.source_files_skipped = 1;
|
|
1962
|
-
logger?.debug(
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
);
|
|
2134
|
+
logger?.debug({
|
|
2135
|
+
path: file.filePath,
|
|
2136
|
+
source_file_id: sourceFile.source_file_id
|
|
2137
|
+
}, "claude source file skipped");
|
|
1966
2138
|
return counts;
|
|
1967
2139
|
}
|
|
1968
2140
|
counts.source_files_imported = 1;
|
|
1969
|
-
logger?.debug(
|
|
1970
|
-
|
|
1971
|
-
|
|
1972
|
-
);
|
|
2141
|
+
logger?.debug({
|
|
2142
|
+
path: file.filePath,
|
|
2143
|
+
source_file_id: sourceFile.source_file_id
|
|
2144
|
+
}, "claude source file registered");
|
|
1973
2145
|
const text = await readFile4(file.filePath, "utf8");
|
|
1974
2146
|
const rawLines = text.split("\n");
|
|
1975
2147
|
const lines = rawLines[rawLines.length - 1] === "" ? rawLines.slice(0, -1) : rawLines;
|
|
@@ -2059,11 +2231,7 @@ async function compileClaudeFile(bundle, batch, file, logger) {
|
|
|
2059
2231
|
const msgRole = type === "user" ? "user" : "assistant";
|
|
2060
2232
|
const role = inferRoleFromContent(parsed, msgRole);
|
|
2061
2233
|
const msgOrdinal = messageOrdinal++;
|
|
2062
|
-
const messageId2 = messageId(
|
|
2063
|
-
sessionId2,
|
|
2064
|
-
msgOrdinal,
|
|
2065
|
-
parsed.message?.id ?? parsed.uuid ?? null
|
|
2066
|
-
);
|
|
2234
|
+
const messageId2 = messageId(sessionId2, msgOrdinal, parsed.message?.id ?? parsed.uuid ?? null);
|
|
2067
2235
|
const eventId2 = eventId(sessionId2, ordinal, "message");
|
|
2068
2236
|
pending.events.push({
|
|
2069
2237
|
event_id: eventId2,
|
|
@@ -2119,17 +2287,7 @@ async function compileClaudeFile(bundle, batch, file, logger) {
|
|
|
2119
2287
|
for (let bi = 0; bi < content.length; bi++) {
|
|
2120
2288
|
const block = content[bi];
|
|
2121
2289
|
if (!block) continue;
|
|
2122
|
-
await processContentBlock(
|
|
2123
|
-
bundle,
|
|
2124
|
-
sessionId2,
|
|
2125
|
-
messageId2,
|
|
2126
|
-
eventId2,
|
|
2127
|
-
bi,
|
|
2128
|
-
block,
|
|
2129
|
-
ts,
|
|
2130
|
-
rawRecordId2,
|
|
2131
|
-
pending
|
|
2132
|
-
);
|
|
2290
|
+
await processContentBlock(bundle, sessionId2, messageId2, eventId2, bi, block, ts, rawRecordId2, pending);
|
|
2133
2291
|
}
|
|
2134
2292
|
}
|
|
2135
2293
|
continue;
|
|
@@ -2198,11 +2356,7 @@ async function compileClaudeFile(bundle, batch, file, logger) {
|
|
|
2198
2356
|
confidence: "high"
|
|
2199
2357
|
});
|
|
2200
2358
|
pending.artifacts.push({
|
|
2201
|
-
artifact_id: artifactId(
|
|
2202
|
-
sessionId2,
|
|
2203
|
-
"claude",
|
|
2204
|
-
`snapshot:${parsed.snapshot?.messageId ?? ordinal}`
|
|
2205
|
-
),
|
|
2359
|
+
artifact_id: artifactId(sessionId2, "claude", `snapshot:${parsed.snapshot?.messageId ?? ordinal}`),
|
|
2206
2360
|
kind: "snapshot",
|
|
2207
2361
|
path: null,
|
|
2208
2362
|
logical_path: null,
|
|
@@ -2253,7 +2407,10 @@ async function compileClaudeFile(bundle, batch, file, logger) {
|
|
|
2253
2407
|
buildSearchDocs(pending);
|
|
2254
2408
|
await flushPendingObjects(bundle, pending.objects);
|
|
2255
2409
|
transactional(bundle.db, () => {
|
|
2256
|
-
flushPending(bundle, pending, {
|
|
2410
|
+
flushPending(bundle, pending, {
|
|
2411
|
+
modelFirst,
|
|
2412
|
+
modelLast
|
|
2413
|
+
});
|
|
2257
2414
|
});
|
|
2258
2415
|
counts.raw_records = pending.rawRecords.length;
|
|
2259
2416
|
counts.sessions = pending.session ? 1 : 0;
|
|
@@ -2264,12 +2421,14 @@ async function compileClaudeFile(bundle, batch, file, logger) {
|
|
|
2264
2421
|
counts.tool_results = pending.toolResults.length;
|
|
2265
2422
|
counts.artifacts = pending.artifacts.length;
|
|
2266
2423
|
counts.edges = pending.edges.length;
|
|
2267
|
-
logger?.debug(
|
|
2268
|
-
|
|
2269
|
-
|
|
2270
|
-
|
|
2424
|
+
logger?.debug({
|
|
2425
|
+
path: file.filePath,
|
|
2426
|
+
source_file_id: sourceFile.source_file_id,
|
|
2427
|
+
counts
|
|
2428
|
+
}, "claude source file imported");
|
|
2271
2429
|
return counts;
|
|
2272
2430
|
}
|
|
2431
|
+
__name(compileClaudeFile, "compileClaudeFile");
|
|
2273
2432
|
function createSessionFromFirstRecord(file, parsed, meta, ts, rawRecordId2) {
|
|
2274
2433
|
const sourceSid = parsed.sessionId;
|
|
2275
2434
|
const composite = file.isSubagent && file.agentId ? `${sourceSid}:${file.agentId}` : sourceSid;
|
|
@@ -2288,6 +2447,7 @@ function createSessionFromFirstRecord(file, parsed, meta, ts, rawRecordId2) {
|
|
|
2288
2447
|
parent_session_id_pending: null
|
|
2289
2448
|
};
|
|
2290
2449
|
}
|
|
2450
|
+
__name(createSessionFromFirstRecord, "createSessionFromFirstRecord");
|
|
2291
2451
|
async function readMeta(metaPath) {
|
|
2292
2452
|
try {
|
|
2293
2453
|
const text = await readFile4(metaPath, "utf8");
|
|
@@ -2296,15 +2456,15 @@ async function readMeta(metaPath) {
|
|
|
2296
2456
|
return null;
|
|
2297
2457
|
}
|
|
2298
2458
|
}
|
|
2459
|
+
__name(readMeta, "readMeta");
|
|
2299
2460
|
function inferRoleFromContent(parsed, fallback) {
|
|
2300
2461
|
if (fallback !== "user") return "assistant";
|
|
2301
2462
|
const c = parsed.message?.content;
|
|
2302
2463
|
if (!Array.isArray(c) || c.length === 0) return "user";
|
|
2303
|
-
const allToolResult = c.every(
|
|
2304
|
-
(b) => b && typeof b === "object" && b.type === "tool_result"
|
|
2305
|
-
);
|
|
2464
|
+
const allToolResult = c.every((b) => b && typeof b === "object" && b.type === "tool_result");
|
|
2306
2465
|
return allToolResult ? "tool" : "user";
|
|
2307
2466
|
}
|
|
2467
|
+
__name(inferRoleFromContent, "inferRoleFromContent");
|
|
2308
2468
|
async function processContentBlock(bundle, sessionId2, messageId2, eventId2, blockOrdinal, block, ts, rawRecordId2, pending) {
|
|
2309
2469
|
const blkId = blockId(messageId2, blockOrdinal);
|
|
2310
2470
|
if (block.type === "text") {
|
|
@@ -2447,6 +2607,7 @@ async function processContentBlock(bundle, sessionId2, messageId2, eventId2, blo
|
|
|
2447
2607
|
raw_record_id: rawRecordId2
|
|
2448
2608
|
});
|
|
2449
2609
|
}
|
|
2610
|
+
__name(processContentBlock, "processContentBlock");
|
|
2450
2611
|
function canonicalToolType(toolName) {
|
|
2451
2612
|
const lower = toolName.toLowerCase();
|
|
2452
2613
|
if (lower.startsWith("mcp__")) return "mcp";
|
|
@@ -2463,6 +2624,7 @@ function canonicalToolType(toolName) {
|
|
|
2463
2624
|
if (lower === "applypatch" || lower === "apply_patch") return "patch";
|
|
2464
2625
|
return "other";
|
|
2465
2626
|
}
|
|
2627
|
+
__name(canonicalToolType, "canonicalToolType");
|
|
2466
2628
|
function inferCommandFromArgs(toolName, args) {
|
|
2467
2629
|
if (!args || typeof args !== "object") return null;
|
|
2468
2630
|
const obj = args;
|
|
@@ -2470,6 +2632,7 @@ function inferCommandFromArgs(toolName, args) {
|
|
|
2470
2632
|
if (toolName.toLowerCase() === "bash" && typeof obj.cmd === "string") return obj.cmd;
|
|
2471
2633
|
return null;
|
|
2472
2634
|
}
|
|
2635
|
+
__name(inferCommandFromArgs, "inferCommandFromArgs");
|
|
2473
2636
|
function inferPathFromArgs(args) {
|
|
2474
2637
|
if (!args || typeof args !== "object") return null;
|
|
2475
2638
|
const obj = args;
|
|
@@ -2478,6 +2641,7 @@ function inferPathFromArgs(args) {
|
|
|
2478
2641
|
if (typeof obj.absolute_path === "string") return obj.absolute_path;
|
|
2479
2642
|
return null;
|
|
2480
2643
|
}
|
|
2644
|
+
__name(inferPathFromArgs, "inferPathFromArgs");
|
|
2481
2645
|
function stringifyOrNull(value) {
|
|
2482
2646
|
if (value == null) return null;
|
|
2483
2647
|
if (typeof value === "string") return value;
|
|
@@ -2487,6 +2651,7 @@ function stringifyOrNull(value) {
|
|
|
2487
2651
|
return null;
|
|
2488
2652
|
}
|
|
2489
2653
|
}
|
|
2654
|
+
__name(stringifyOrNull, "stringifyOrNull");
|
|
2490
2655
|
function buildSearchDocs(pending) {
|
|
2491
2656
|
const sessionId2 = pending.session?.session_id ?? null;
|
|
2492
2657
|
if (!sessionId2) return;
|
|
@@ -2494,8 +2659,7 @@ function buildSearchDocs(pending) {
|
|
|
2494
2659
|
for (const b of pending.blocks) {
|
|
2495
2660
|
if (!b.message_id) continue;
|
|
2496
2661
|
if (!b.text_inline) continue;
|
|
2497
|
-
if (b.block_type !== "text" && b.block_type !== "thinking" && b.block_type !== "tool_result")
|
|
2498
|
-
continue;
|
|
2662
|
+
if (b.block_type !== "text" && b.block_type !== "thinking" && b.block_type !== "tool_result") continue;
|
|
2499
2663
|
const list = blocksByMsg.get(b.message_id) ?? [];
|
|
2500
2664
|
list.push(b);
|
|
2501
2665
|
blocksByMsg.set(b.message_id, list);
|
|
@@ -2559,240 +2723,88 @@ function buildSearchDocs(pending) {
|
|
|
2559
2723
|
});
|
|
2560
2724
|
}
|
|
2561
2725
|
}
|
|
2726
|
+
__name(buildSearchDocs, "buildSearchDocs");
|
|
2562
2727
|
function flushPending(bundle, pending, meta) {
|
|
2563
2728
|
if (!pending.session) return;
|
|
2564
|
-
const insertRaw = prepare(
|
|
2565
|
-
bundle.db,
|
|
2566
|
-
`INSERT OR IGNORE INTO raw_records (
|
|
2729
|
+
const insertRaw = prepare(bundle.db, `INSERT OR IGNORE INTO raw_records (
|
|
2567
2730
|
raw_record_id, source_file_id, source_tool, record_kind, ordinal,
|
|
2568
2731
|
line_no, json_pointer, native_id, raw_object_id, decoded_json_object_id,
|
|
2569
2732
|
parser_status, confidence, import_batch_id
|
|
2570
|
-
) VALUES (?, ?, 'claude', 'jsonl_line', ?, ?, NULL, ?, ?, ?, ?, ?, ?)`
|
|
2571
|
-
);
|
|
2733
|
+
) VALUES (?, ?, 'claude', 'jsonl_line', ?, ?, NULL, ?, ?, ?, ?, ?, ?)`);
|
|
2572
2734
|
for (const r of pending.rawRecords) {
|
|
2573
|
-
insertRaw.run(
|
|
2574
|
-
r.raw_record_id,
|
|
2575
|
-
r.source_file_id,
|
|
2576
|
-
r.ordinal,
|
|
2577
|
-
r.line_no,
|
|
2578
|
-
r.native_id,
|
|
2579
|
-
r.raw_object_id,
|
|
2580
|
-
r.decoded_json_object_id,
|
|
2581
|
-
r.parser_status,
|
|
2582
|
-
r.confidence,
|
|
2583
|
-
r.import_batch_id
|
|
2584
|
-
);
|
|
2735
|
+
insertRaw.run(r.raw_record_id, r.source_file_id, r.ordinal, r.line_no, r.native_id, r.raw_object_id, r.decoded_json_object_id, r.parser_status, r.confidence, r.import_batch_id);
|
|
2585
2736
|
}
|
|
2586
|
-
prepare(
|
|
2587
|
-
bundle.db,
|
|
2588
|
-
`INSERT OR REPLACE INTO sessions (
|
|
2737
|
+
prepare(bundle.db, `INSERT OR REPLACE INTO sessions (
|
|
2589
2738
|
session_id, source_tool, source_session_id, project_id, parent_session_id,
|
|
2590
2739
|
is_subagent, agent_role, agent_nickname, title, summary,
|
|
2591
2740
|
start_ts, end_ts, cwd_initial, git_branch_initial,
|
|
2592
2741
|
model_first, model_last, status, timeline_confidence, raw_record_id
|
|
2593
|
-
) VALUES (?, 'claude', ?, NULL, NULL, ?, ?, ?, ?, NULL, ?, ?, ?, ?, ?, ?, 'completed', 'high', ?)`
|
|
2594
|
-
|
|
2595
|
-
pending.session.session_id,
|
|
2596
|
-
pending.session.source_session_id,
|
|
2597
|
-
pending.session.is_subagent,
|
|
2598
|
-
pending.session.agent_role,
|
|
2599
|
-
pending.session.agent_nickname,
|
|
2600
|
-
pending.session.title,
|
|
2601
|
-
pending.session.start_ts,
|
|
2602
|
-
pending.session.end_ts,
|
|
2603
|
-
pending.session.cwd_initial,
|
|
2604
|
-
pending.session.git_branch_initial,
|
|
2605
|
-
meta.modelFirst,
|
|
2606
|
-
meta.modelLast,
|
|
2607
|
-
pending.session.raw_record_id
|
|
2608
|
-
);
|
|
2609
|
-
const insertEvent = prepare(
|
|
2610
|
-
bundle.db,
|
|
2611
|
-
`INSERT OR REPLACE INTO events (
|
|
2742
|
+
) VALUES (?, 'claude', ?, NULL, NULL, ?, ?, ?, ?, NULL, ?, ?, ?, ?, ?, ?, 'completed', 'high', ?)`).run(pending.session.session_id, pending.session.source_session_id, pending.session.is_subagent, pending.session.agent_role, pending.session.agent_nickname, pending.session.title, pending.session.start_ts, pending.session.end_ts, pending.session.cwd_initial, pending.session.git_branch_initial, meta.modelFirst, meta.modelLast, pending.session.raw_record_id);
|
|
2743
|
+
const insertEvent = prepare(bundle.db, `INSERT OR REPLACE INTO events (
|
|
2612
2744
|
event_id, session_id, turn_id, source_event_id, event_type, source_type,
|
|
2613
2745
|
subtype, timestamp, ordinal, actor, payload_object_id, raw_record_id,
|
|
2614
2746
|
confidence, is_derived
|
|
2615
|
-
) VALUES (?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0)`
|
|
2616
|
-
);
|
|
2747
|
+
) VALUES (?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0)`);
|
|
2617
2748
|
for (const e of pending.events) {
|
|
2618
|
-
insertEvent.run(
|
|
2619
|
-
e.event_id,
|
|
2620
|
-
pending.session.session_id,
|
|
2621
|
-
e.source_event_id,
|
|
2622
|
-
e.event_type,
|
|
2623
|
-
e.source_type,
|
|
2624
|
-
e.subtype,
|
|
2625
|
-
e.timestamp,
|
|
2626
|
-
e.ordinal,
|
|
2627
|
-
e.actor,
|
|
2628
|
-
e.payload_object_id,
|
|
2629
|
-
e.raw_record_id,
|
|
2630
|
-
e.confidence
|
|
2631
|
-
);
|
|
2749
|
+
insertEvent.run(e.event_id, pending.session.session_id, e.source_event_id, e.event_type, e.source_type, e.subtype, e.timestamp, e.ordinal, e.actor, e.payload_object_id, e.raw_record_id, e.confidence);
|
|
2632
2750
|
}
|
|
2633
|
-
const insertMessage = prepare(
|
|
2634
|
-
bundle.db,
|
|
2635
|
-
`INSERT OR REPLACE INTO messages (
|
|
2751
|
+
const insertMessage = prepare(bundle.db, `INSERT OR REPLACE INTO messages (
|
|
2636
2752
|
message_id, session_id, turn_id, event_id, source_message_id, role,
|
|
2637
2753
|
author_name, model, timestamp, ordinal, parent_message_id, request_id,
|
|
2638
2754
|
status, raw_record_id
|
|
2639
|
-
) VALUES (?, ?, NULL, ?, ?, ?, NULL, ?, ?, ?, NULL, NULL, NULL, ?)`
|
|
2640
|
-
);
|
|
2755
|
+
) VALUES (?, ?, NULL, ?, ?, ?, NULL, ?, ?, ?, NULL, NULL, NULL, ?)`);
|
|
2641
2756
|
for (const m of pending.messages) {
|
|
2642
|
-
insertMessage.run(
|
|
2643
|
-
m.message_id,
|
|
2644
|
-
pending.session.session_id,
|
|
2645
|
-
m.event_id,
|
|
2646
|
-
m.source_message_id,
|
|
2647
|
-
m.role,
|
|
2648
|
-
m.model,
|
|
2649
|
-
m.timestamp,
|
|
2650
|
-
m.ordinal,
|
|
2651
|
-
m.raw_record_id
|
|
2652
|
-
);
|
|
2757
|
+
insertMessage.run(m.message_id, pending.session.session_id, m.event_id, m.source_message_id, m.role, m.model, m.timestamp, m.ordinal, m.raw_record_id);
|
|
2653
2758
|
}
|
|
2654
|
-
const insertBlock = prepare(
|
|
2655
|
-
bundle.db,
|
|
2656
|
-
`INSERT OR REPLACE INTO content_blocks (
|
|
2759
|
+
const insertBlock = prepare(bundle.db, `INSERT OR REPLACE INTO content_blocks (
|
|
2657
2760
|
block_id, message_id, event_id, session_id, ordinal, block_type,
|
|
2658
2761
|
text_object_id, text_inline, mime_type, token_count, is_error,
|
|
2659
2762
|
is_redacted, visibility, raw_record_id
|
|
2660
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, ?, 0, ?, ?)`
|
|
2661
|
-
);
|
|
2763
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, ?, 0, ?, ?)`);
|
|
2662
2764
|
for (const b of pending.blocks) {
|
|
2663
|
-
insertBlock.run(
|
|
2664
|
-
b.block_id,
|
|
2665
|
-
b.message_id,
|
|
2666
|
-
b.event_id,
|
|
2667
|
-
pending.session.session_id,
|
|
2668
|
-
b.ordinal,
|
|
2669
|
-
b.block_type,
|
|
2670
|
-
b.text_object_id,
|
|
2671
|
-
b.text_inline,
|
|
2672
|
-
b.is_error,
|
|
2673
|
-
b.visibility,
|
|
2674
|
-
b.raw_record_id
|
|
2675
|
-
);
|
|
2765
|
+
insertBlock.run(b.block_id, b.message_id, b.event_id, pending.session.session_id, b.ordinal, b.block_type, b.text_object_id, b.text_inline, b.is_error, b.visibility, b.raw_record_id);
|
|
2676
2766
|
}
|
|
2677
|
-
const insertToolCall = prepare(
|
|
2678
|
-
bundle.db,
|
|
2679
|
-
`INSERT OR REPLACE INTO tool_calls (
|
|
2767
|
+
const insertToolCall = prepare(bundle.db, `INSERT OR REPLACE INTO tool_calls (
|
|
2680
2768
|
tool_call_id, session_id, turn_id, message_id, event_id,
|
|
2681
2769
|
source_call_id, tool_name, canonical_tool_type, args_object_id,
|
|
2682
2770
|
command, cwd, path, query, timestamp_start, timestamp_end, status,
|
|
2683
2771
|
raw_record_id
|
|
2684
|
-
) VALUES (?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, ?)`
|
|
2685
|
-
);
|
|
2772
|
+
) VALUES (?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, ?)`);
|
|
2686
2773
|
for (const c of pending.toolCallsList) {
|
|
2687
|
-
insertToolCall.run(
|
|
2688
|
-
c.tool_call_id,
|
|
2689
|
-
pending.session.session_id,
|
|
2690
|
-
c.message_id,
|
|
2691
|
-
c.event_id,
|
|
2692
|
-
c.source_call_id,
|
|
2693
|
-
c.tool_name,
|
|
2694
|
-
c.canonical_tool_type,
|
|
2695
|
-
c.args_object_id,
|
|
2696
|
-
c.command,
|
|
2697
|
-
c.cwd,
|
|
2698
|
-
c.path,
|
|
2699
|
-
c.query,
|
|
2700
|
-
c.timestamp_start,
|
|
2701
|
-
c.status,
|
|
2702
|
-
c.raw_record_id
|
|
2703
|
-
);
|
|
2774
|
+
insertToolCall.run(c.tool_call_id, pending.session.session_id, c.message_id, c.event_id, c.source_call_id, c.tool_name, c.canonical_tool_type, c.args_object_id, c.command, c.cwd, c.path, c.query, c.timestamp_start, c.status, c.raw_record_id);
|
|
2704
2775
|
}
|
|
2705
|
-
const insertToolResult = prepare(
|
|
2706
|
-
bundle.db,
|
|
2707
|
-
`INSERT OR REPLACE INTO tool_results (
|
|
2776
|
+
const insertToolResult = prepare(bundle.db, `INSERT OR REPLACE INTO tool_results (
|
|
2708
2777
|
tool_result_id, tool_call_id, session_id, message_id, event_id,
|
|
2709
2778
|
source_call_id, status, is_error, exit_code, duration_ms,
|
|
2710
2779
|
stdout_object_id, stderr_object_id, output_object_id, preview, raw_record_id
|
|
2711
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
2712
|
-
);
|
|
2780
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
2713
2781
|
for (const r of pending.toolResults) {
|
|
2714
|
-
insertToolResult.run(
|
|
2715
|
-
r.tool_result_id,
|
|
2716
|
-
r.tool_call_id,
|
|
2717
|
-
pending.session.session_id,
|
|
2718
|
-
r.message_id,
|
|
2719
|
-
r.event_id,
|
|
2720
|
-
r.source_call_id,
|
|
2721
|
-
r.status,
|
|
2722
|
-
r.is_error,
|
|
2723
|
-
r.exit_code,
|
|
2724
|
-
r.duration_ms,
|
|
2725
|
-
r.stdout_object_id,
|
|
2726
|
-
r.stderr_object_id,
|
|
2727
|
-
r.output_object_id,
|
|
2728
|
-
r.preview,
|
|
2729
|
-
r.raw_record_id
|
|
2730
|
-
);
|
|
2782
|
+
insertToolResult.run(r.tool_result_id, r.tool_call_id, pending.session.session_id, r.message_id, r.event_id, r.source_call_id, r.status, r.is_error, r.exit_code, r.duration_ms, r.stdout_object_id, r.stderr_object_id, r.output_object_id, r.preview, r.raw_record_id);
|
|
2731
2783
|
}
|
|
2732
|
-
const insertArtifact = prepare(
|
|
2733
|
-
bundle.db,
|
|
2734
|
-
`INSERT OR REPLACE INTO artifacts (
|
|
2784
|
+
const insertArtifact = prepare(bundle.db, `INSERT OR REPLACE INTO artifacts (
|
|
2735
2785
|
artifact_id, session_id, project_id, source_tool, kind, path,
|
|
2736
2786
|
logical_path, object_id, text_object_id, mime_type, size_bytes,
|
|
2737
2787
|
created_ts, raw_record_id
|
|
2738
|
-
) VALUES (?, ?, NULL, 'claude', ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
2739
|
-
);
|
|
2788
|
+
) VALUES (?, ?, NULL, 'claude', ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
2740
2789
|
for (const a of pending.artifacts) {
|
|
2741
|
-
insertArtifact.run(
|
|
2742
|
-
a.artifact_id,
|
|
2743
|
-
pending.session.session_id,
|
|
2744
|
-
a.kind,
|
|
2745
|
-
a.path,
|
|
2746
|
-
a.logical_path,
|
|
2747
|
-
a.object_id,
|
|
2748
|
-
a.text_object_id,
|
|
2749
|
-
a.mime_type,
|
|
2750
|
-
a.size_bytes,
|
|
2751
|
-
a.created_ts,
|
|
2752
|
-
a.raw_record_id
|
|
2753
|
-
);
|
|
2790
|
+
insertArtifact.run(a.artifact_id, pending.session.session_id, a.kind, a.path, a.logical_path, a.object_id, a.text_object_id, a.mime_type, a.size_bytes, a.created_ts, a.raw_record_id);
|
|
2754
2791
|
}
|
|
2755
|
-
const insertEdge = prepare(
|
|
2756
|
-
bundle.db,
|
|
2757
|
-
`INSERT OR IGNORE INTO edges (
|
|
2792
|
+
const insertEdge = prepare(bundle.db, `INSERT OR IGNORE INTO edges (
|
|
2758
2793
|
src_type, src_id, dst_type, dst_id, edge_type, confidence, source,
|
|
2759
2794
|
raw_record_id, metadata_object_id
|
|
2760
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULL)`
|
|
2761
|
-
);
|
|
2795
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULL)`);
|
|
2762
2796
|
for (const e of pending.edges) {
|
|
2763
|
-
insertEdge.run(
|
|
2764
|
-
e.src_type,
|
|
2765
|
-
e.src_id,
|
|
2766
|
-
e.dst_type,
|
|
2767
|
-
e.dst_id,
|
|
2768
|
-
e.edge_type,
|
|
2769
|
-
e.confidence,
|
|
2770
|
-
e.source,
|
|
2771
|
-
e.raw_record_id
|
|
2772
|
-
);
|
|
2797
|
+
insertEdge.run(e.src_type, e.src_id, e.dst_type, e.dst_id, e.edge_type, e.confidence, e.source, e.raw_record_id);
|
|
2773
2798
|
}
|
|
2774
|
-
const insertSearch = prepare(
|
|
2775
|
-
bundle.db,
|
|
2776
|
-
`INSERT OR REPLACE INTO search_docs (
|
|
2799
|
+
const insertSearch = prepare(bundle.db, `INSERT OR REPLACE INTO search_docs (
|
|
2777
2800
|
doc_id, entity_type, entity_id, session_id, project_id, timestamp,
|
|
2778
2801
|
role, tool_name, canonical_tool_type, field_kind, text
|
|
2779
|
-
) VALUES (?, ?, ?, ?, NULL, ?, ?, ?, ?, ?, ?)`
|
|
2780
|
-
);
|
|
2802
|
+
) VALUES (?, ?, ?, ?, NULL, ?, ?, ?, ?, ?, ?)`);
|
|
2781
2803
|
for (const d of pending.searchDocs) {
|
|
2782
|
-
insertSearch.run(
|
|
2783
|
-
d.doc_id,
|
|
2784
|
-
d.entity_type,
|
|
2785
|
-
d.entity_id,
|
|
2786
|
-
pending.session.session_id,
|
|
2787
|
-
d.timestamp,
|
|
2788
|
-
d.role,
|
|
2789
|
-
d.tool_name,
|
|
2790
|
-
d.canonical_tool_type,
|
|
2791
|
-
d.field_kind,
|
|
2792
|
-
d.text
|
|
2793
|
-
);
|
|
2804
|
+
insertSearch.run(d.doc_id, d.entity_type, d.entity_id, pending.session.session_id, d.timestamp, d.role, d.tool_name, d.canonical_tool_type, d.field_kind, d.text);
|
|
2794
2805
|
}
|
|
2795
2806
|
}
|
|
2807
|
+
__name(flushPending, "flushPending");
|
|
2796
2808
|
|
|
2797
2809
|
// src/importers/codex/index.ts
|
|
2798
2810
|
import { readFile as readFile5 } from "fs/promises";
|
|
@@ -2804,10 +2816,13 @@ import path7 from "path";
|
|
|
2804
2816
|
async function* discoverCodexSessions(root) {
|
|
2805
2817
|
yield* walk(root);
|
|
2806
2818
|
}
|
|
2819
|
+
__name(discoverCodexSessions, "discoverCodexSessions");
|
|
2807
2820
|
async function* walk(dir) {
|
|
2808
2821
|
let entries;
|
|
2809
2822
|
try {
|
|
2810
|
-
entries = await readdir2(dir, {
|
|
2823
|
+
entries = await readdir2(dir, {
|
|
2824
|
+
withFileTypes: true
|
|
2825
|
+
});
|
|
2811
2826
|
} catch {
|
|
2812
2827
|
return;
|
|
2813
2828
|
}
|
|
@@ -2820,48 +2835,108 @@ async function* walk(dir) {
|
|
|
2820
2835
|
}
|
|
2821
2836
|
}
|
|
2822
2837
|
}
|
|
2838
|
+
__name(walk, "walk");
|
|
2823
2839
|
|
|
2824
2840
|
// src/importers/codex/index.ts
|
|
2825
2841
|
var PREVIEW_MAX2 = 4e3;
|
|
2842
|
+
var CODEX_PREPARE_CONCURRENCY = 8;
|
|
2826
2843
|
async function compileCodex(bundle, root, options = {}) {
|
|
2827
2844
|
const logger = options.logger;
|
|
2828
|
-
const batch = startBatch(bundle, "codex", [
|
|
2845
|
+
const batch = startBatch(bundle, "codex", [
|
|
2846
|
+
root
|
|
2847
|
+
]);
|
|
2829
2848
|
const counts = emptyCounts();
|
|
2830
|
-
logger?.info({
|
|
2849
|
+
logger?.info({
|
|
2850
|
+
batch_id: batch.batch_id,
|
|
2851
|
+
root
|
|
2852
|
+
}, "codex batch started");
|
|
2831
2853
|
try {
|
|
2854
|
+
const files = [];
|
|
2832
2855
|
for await (const filePath of discoverCodexSessions(root)) {
|
|
2833
|
-
|
|
2834
|
-
logger?.debug({
|
|
2835
|
-
|
|
2836
|
-
|
|
2837
|
-
|
|
2838
|
-
|
|
2839
|
-
|
|
2840
|
-
|
|
2841
|
-
|
|
2842
|
-
err: error,
|
|
2843
|
-
path: filePath
|
|
2844
|
-
},
|
|
2845
|
-
"codex source file failed"
|
|
2846
|
-
);
|
|
2847
|
-
await recordError(bundle, batch.batch_id, {
|
|
2848
|
-
kind: "codex_file_failed",
|
|
2849
|
-
message: getErrorMessage(error),
|
|
2850
|
-
payload: { path: filePath }
|
|
2851
|
-
});
|
|
2852
|
-
}
|
|
2856
|
+
files.push(filePath);
|
|
2857
|
+
logger?.debug({
|
|
2858
|
+
path: filePath
|
|
2859
|
+
}, "codex source file discovered");
|
|
2860
|
+
}
|
|
2861
|
+
counts.source_files_seen = files.length;
|
|
2862
|
+
for (let i = 0; i < files.length; i += CODEX_PREPARE_CONCURRENCY) {
|
|
2863
|
+
const slice = files.slice(i, i + CODEX_PREPARE_CONCURRENCY);
|
|
2864
|
+
await processCodexBatch(bundle, batch, slice, counts, logger);
|
|
2853
2865
|
}
|
|
2854
2866
|
linkSubagentParents2(bundle);
|
|
2855
|
-
logger?.debug({
|
|
2867
|
+
logger?.debug({
|
|
2868
|
+
batch_id: batch.batch_id
|
|
2869
|
+
}, "codex subagent parent links refreshed");
|
|
2856
2870
|
finishBatch(bundle, batch, counts, "completed");
|
|
2857
|
-
logger?.info({
|
|
2871
|
+
logger?.info({
|
|
2872
|
+
batch_id: batch.batch_id,
|
|
2873
|
+
counts
|
|
2874
|
+
}, "codex batch completed");
|
|
2858
2875
|
} catch (error) {
|
|
2859
2876
|
finishBatch(bundle, batch, counts, "failed");
|
|
2860
|
-
logger?.error({
|
|
2877
|
+
logger?.error({
|
|
2878
|
+
err: error,
|
|
2879
|
+
batch_id: batch.batch_id,
|
|
2880
|
+
counts
|
|
2881
|
+
}, "codex batch failed");
|
|
2861
2882
|
throw error;
|
|
2862
2883
|
}
|
|
2863
|
-
return {
|
|
2884
|
+
return {
|
|
2885
|
+
batch,
|
|
2886
|
+
counts
|
|
2887
|
+
};
|
|
2864
2888
|
}
|
|
2889
|
+
__name(compileCodex, "compileCodex");
|
|
2890
|
+
async function processCodexBatch(bundle, batch, slice, counts, logger) {
|
|
2891
|
+
const items = await Promise.all(slice.map(async (filePath) => {
|
|
2892
|
+
try {
|
|
2893
|
+
const result = await prepareCodexFile(bundle, batch, filePath, logger);
|
|
2894
|
+
return {
|
|
2895
|
+
filePath,
|
|
2896
|
+
prepared: result.prepared,
|
|
2897
|
+
fileCounts: result.counts,
|
|
2898
|
+
prepareError: null,
|
|
2899
|
+
applyError: null
|
|
2900
|
+
};
|
|
2901
|
+
} catch (err) {
|
|
2902
|
+
return {
|
|
2903
|
+
filePath,
|
|
2904
|
+
prepared: null,
|
|
2905
|
+
fileCounts: emptyFileCounts2(),
|
|
2906
|
+
prepareError: err,
|
|
2907
|
+
applyError: null
|
|
2908
|
+
};
|
|
2909
|
+
}
|
|
2910
|
+
}));
|
|
2911
|
+
for (const item of items) {
|
|
2912
|
+
if (item.prepareError || !item.prepared) continue;
|
|
2913
|
+
try {
|
|
2914
|
+
transactional(bundle.db, () => applyCodexFile(bundle, item.prepared));
|
|
2915
|
+
} catch (err) {
|
|
2916
|
+
item.applyError = err;
|
|
2917
|
+
}
|
|
2918
|
+
}
|
|
2919
|
+
for (const item of items) {
|
|
2920
|
+
const err = item.prepareError ?? item.applyError;
|
|
2921
|
+
if (err) {
|
|
2922
|
+
counts.errors++;
|
|
2923
|
+
logger?.warn({
|
|
2924
|
+
err,
|
|
2925
|
+
path: item.filePath
|
|
2926
|
+
}, "codex source file failed");
|
|
2927
|
+
await recordError(bundle, batch.batch_id, {
|
|
2928
|
+
kind: "codex_file_failed",
|
|
2929
|
+
message: getErrorMessage(err),
|
|
2930
|
+
payload: {
|
|
2931
|
+
path: item.filePath
|
|
2932
|
+
}
|
|
2933
|
+
});
|
|
2934
|
+
} else {
|
|
2935
|
+
addCounts2(counts, item.fileCounts);
|
|
2936
|
+
}
|
|
2937
|
+
}
|
|
2938
|
+
}
|
|
2939
|
+
__name(processCodexBatch, "processCodexBatch");
|
|
2865
2940
|
function linkSubagentParents2(bundle) {
|
|
2866
2941
|
bundle.db.exec(`
|
|
2867
2942
|
UPDATE sessions
|
|
@@ -2878,6 +2953,7 @@ function linkSubagentParents2(bundle) {
|
|
|
2878
2953
|
AND is_subagent = 1
|
|
2879
2954
|
`);
|
|
2880
2955
|
}
|
|
2956
|
+
__name(linkSubagentParents2, "linkSubagentParents");
|
|
2881
2957
|
function emptyFileCounts2() {
|
|
2882
2958
|
return {
|
|
2883
2959
|
source_files_imported: 0,
|
|
@@ -2895,6 +2971,7 @@ function emptyFileCounts2() {
|
|
|
2895
2971
|
errors: 0
|
|
2896
2972
|
};
|
|
2897
2973
|
}
|
|
2974
|
+
__name(emptyFileCounts2, "emptyFileCounts");
|
|
2898
2975
|
function addCounts2(target, source) {
|
|
2899
2976
|
target.source_files_imported += source.source_files_imported;
|
|
2900
2977
|
target.source_files_skipped += source.source_files_skipped;
|
|
@@ -2910,7 +2987,8 @@ function addCounts2(target, source) {
|
|
|
2910
2987
|
target.edges += source.edges;
|
|
2911
2988
|
target.errors += source.errors;
|
|
2912
2989
|
}
|
|
2913
|
-
|
|
2990
|
+
__name(addCounts2, "addCounts");
|
|
2991
|
+
async function prepareCodexFile(bundle, batch, filePath, logger) {
|
|
2914
2992
|
const counts = emptyFileCounts2();
|
|
2915
2993
|
const { row: sourceFileRow, alreadyKnown } = await registerSourceFile(bundle, {
|
|
2916
2994
|
sourceTool: "codex",
|
|
@@ -2919,17 +2997,20 @@ async function compileCodexFile(bundle, batch, filePath, logger) {
|
|
|
2919
2997
|
});
|
|
2920
2998
|
if (alreadyKnown) {
|
|
2921
2999
|
counts.source_files_skipped = 1;
|
|
2922
|
-
logger?.debug(
|
|
2923
|
-
|
|
2924
|
-
|
|
2925
|
-
);
|
|
2926
|
-
return
|
|
3000
|
+
logger?.debug({
|
|
3001
|
+
path: filePath,
|
|
3002
|
+
source_file_id: sourceFileRow.source_file_id
|
|
3003
|
+
}, "codex source file skipped");
|
|
3004
|
+
return {
|
|
3005
|
+
prepared: null,
|
|
3006
|
+
counts
|
|
3007
|
+
};
|
|
2927
3008
|
}
|
|
2928
3009
|
counts.source_files_imported = 1;
|
|
2929
|
-
logger?.debug(
|
|
2930
|
-
|
|
2931
|
-
|
|
2932
|
-
);
|
|
3010
|
+
logger?.debug({
|
|
3011
|
+
path: filePath,
|
|
3012
|
+
source_file_id: sourceFileRow.source_file_id
|
|
3013
|
+
}, "codex source file registered");
|
|
2933
3014
|
const text = await readFile5(filePath, "utf8");
|
|
2934
3015
|
const rawLines = text.split("\n");
|
|
2935
3016
|
const lines = rawLines[rawLines.length - 1] === "" ? rawLines.slice(0, -1) : rawLines;
|
|
@@ -2941,7 +3022,6 @@ async function compileCodexFile(bundle, batch, filePath, logger) {
|
|
|
2941
3022
|
messages: [],
|
|
2942
3023
|
blocks: [],
|
|
2943
3024
|
toolCalls: /* @__PURE__ */ new Map(),
|
|
2944
|
-
// by source_call_id
|
|
2945
3025
|
toolCallsList: [],
|
|
2946
3026
|
toolResults: [],
|
|
2947
3027
|
artifacts: [],
|
|
@@ -3073,34 +3153,12 @@ async function compileCodexFile(bundle, batch, filePath, logger) {
|
|
|
3073
3153
|
const currentTurnId = pending.turns.length > 0 ? pending.turns[pending.turns.length - 1].turn_id : null;
|
|
3074
3154
|
if (type === "response_item") {
|
|
3075
3155
|
const ri = payload;
|
|
3076
|
-
handleResponseItem(
|
|
3077
|
-
bundle,
|
|
3078
|
-
sessionId2,
|
|
3079
|
-
currentTurnId,
|
|
3080
|
-
rawRecordId2,
|
|
3081
|
-
ordinal,
|
|
3082
|
-
ts,
|
|
3083
|
-
ri,
|
|
3084
|
-
decodedObjectId,
|
|
3085
|
-
() => messageOrdinal++,
|
|
3086
|
-
modelLast,
|
|
3087
|
-
pending
|
|
3088
|
-
);
|
|
3156
|
+
handleResponseItem(bundle, sessionId2, currentTurnId, rawRecordId2, ordinal, ts, ri, decodedObjectId, () => messageOrdinal++, modelLast, pending);
|
|
3089
3157
|
continue;
|
|
3090
3158
|
}
|
|
3091
3159
|
if (type === "event_msg") {
|
|
3092
3160
|
const em = payload;
|
|
3093
|
-
await handleEventMsg(
|
|
3094
|
-
bundle,
|
|
3095
|
-
sessionId2,
|
|
3096
|
-
currentTurnId,
|
|
3097
|
-
rawRecordId2,
|
|
3098
|
-
ordinal,
|
|
3099
|
-
ts,
|
|
3100
|
-
em,
|
|
3101
|
-
decodedObjectId,
|
|
3102
|
-
pending
|
|
3103
|
-
);
|
|
3161
|
+
await handleEventMsg(bundle, sessionId2, currentTurnId, rawRecordId2, ordinal, ts, em, decodedObjectId, pending);
|
|
3104
3162
|
continue;
|
|
3105
3163
|
}
|
|
3106
3164
|
if (type === "compacted") {
|
|
@@ -3122,19 +3180,10 @@ async function compileCodexFile(bundle, batch, filePath, logger) {
|
|
|
3122
3180
|
}
|
|
3123
3181
|
if (type === "message") {
|
|
3124
3182
|
const ri = payload;
|
|
3125
|
-
handleResponseItem(
|
|
3126
|
-
|
|
3127
|
-
|
|
3128
|
-
|
|
3129
|
-
rawRecordId2,
|
|
3130
|
-
ordinal,
|
|
3131
|
-
ts,
|
|
3132
|
-
{ ...ri, type: "message" },
|
|
3133
|
-
decodedObjectId,
|
|
3134
|
-
() => messageOrdinal++,
|
|
3135
|
-
modelLast,
|
|
3136
|
-
pending
|
|
3137
|
-
);
|
|
3183
|
+
handleResponseItem(bundle, sessionId2, currentTurnId, rawRecordId2, ordinal, ts, {
|
|
3184
|
+
...ri,
|
|
3185
|
+
type: "message"
|
|
3186
|
+
}, decodedObjectId, () => messageOrdinal++, modelLast, pending);
|
|
3138
3187
|
}
|
|
3139
3188
|
}
|
|
3140
3189
|
if (pending.session) {
|
|
@@ -3142,14 +3191,6 @@ async function compileCodexFile(bundle, batch, filePath, logger) {
|
|
|
3142
3191
|
}
|
|
3143
3192
|
buildSearchDocs2(pending);
|
|
3144
3193
|
await flushPendingObjects(bundle, pending.objects);
|
|
3145
|
-
transactional(bundle.db, () => {
|
|
3146
|
-
flushPending2(bundle, pending, {
|
|
3147
|
-
sessionEndTs,
|
|
3148
|
-
modelFirst,
|
|
3149
|
-
modelLast,
|
|
3150
|
-
sourceTool: "codex"
|
|
3151
|
-
});
|
|
3152
|
-
});
|
|
3153
3194
|
counts.raw_records = pending.rawRecords.length;
|
|
3154
3195
|
counts.sessions = pending.session ? 1 : 0;
|
|
3155
3196
|
counts.turns = pending.turns.length;
|
|
@@ -3160,12 +3201,34 @@ async function compileCodexFile(bundle, batch, filePath, logger) {
|
|
|
3160
3201
|
counts.tool_results = pending.toolResults.length;
|
|
3161
3202
|
counts.artifacts = pending.artifacts.length;
|
|
3162
3203
|
counts.edges = pending.edges.length;
|
|
3163
|
-
logger?.debug(
|
|
3164
|
-
|
|
3165
|
-
|
|
3166
|
-
|
|
3167
|
-
|
|
3204
|
+
logger?.debug({
|
|
3205
|
+
path: filePath,
|
|
3206
|
+
source_file_id: sourceFileRow.source_file_id,
|
|
3207
|
+
counts
|
|
3208
|
+
}, "codex source file prepared");
|
|
3209
|
+
return {
|
|
3210
|
+
prepared: {
|
|
3211
|
+
filePath,
|
|
3212
|
+
pending,
|
|
3213
|
+
meta: {
|
|
3214
|
+
sessionEndTs,
|
|
3215
|
+
modelFirst,
|
|
3216
|
+
modelLast
|
|
3217
|
+
}
|
|
3218
|
+
},
|
|
3219
|
+
counts
|
|
3220
|
+
};
|
|
3221
|
+
}
|
|
3222
|
+
__name(prepareCodexFile, "prepareCodexFile");
|
|
3223
|
+
function applyCodexFile(bundle, prep) {
|
|
3224
|
+
flushPending2(bundle, prep.pending, {
|
|
3225
|
+
sessionEndTs: prep.meta.sessionEndTs,
|
|
3226
|
+
modelFirst: prep.meta.modelFirst,
|
|
3227
|
+
modelLast: prep.meta.modelLast,
|
|
3228
|
+
sourceTool: "codex"
|
|
3229
|
+
});
|
|
3168
3230
|
}
|
|
3231
|
+
__name(applyCodexFile, "applyCodexFile");
|
|
3169
3232
|
function handleResponseItem(_bundle, sessionId2, currentTurnId, rawRecordId2, ordinal, ts, ri, payloadObjectId, nextMsgOrdinal, currentModel, pending) {
|
|
3170
3233
|
const subtype = ri.type ?? null;
|
|
3171
3234
|
if (subtype === "message") {
|
|
@@ -3294,7 +3357,6 @@ function handleResponseItem(_bundle, sessionId2, currentTurnId, rawRecordId2, or
|
|
|
3294
3357
|
stdout_object_id: null,
|
|
3295
3358
|
stderr_object_id: null,
|
|
3296
3359
|
output_object_id: null,
|
|
3297
|
-
// small previews only at this stage
|
|
3298
3360
|
preview: outputText.slice(0, PREVIEW_MAX2),
|
|
3299
3361
|
raw_record_id: rawRecordId2
|
|
3300
3362
|
});
|
|
@@ -3319,16 +3381,18 @@ function handleResponseItem(_bundle, sessionId2, currentTurnId, rawRecordId2, or
|
|
|
3319
3381
|
confidence: "high"
|
|
3320
3382
|
});
|
|
3321
3383
|
}
|
|
3384
|
+
__name(handleResponseItem, "handleResponseItem");
|
|
3322
3385
|
async function handleEventMsg(bundle, sessionId2, currentTurnId, rawRecordId2, ordinal, ts, em, payloadObjectId, pending) {
|
|
3323
3386
|
const subtype = em.type ?? "unknown";
|
|
3324
3387
|
if (subtype === "exec_command_end") {
|
|
3325
3388
|
const sourceCallId = em.call_id ?? null;
|
|
3326
|
-
const stdoutId = em.stdout ? stageText(pending.objects, em.stdout, {
|
|
3327
|
-
|
|
3328
|
-
|
|
3329
|
-
|
|
3330
|
-
|
|
3331
|
-
);
|
|
3389
|
+
const stdoutId = em.stdout ? stageText(pending.objects, em.stdout, {
|
|
3390
|
+
mimeType: "text/plain"
|
|
3391
|
+
}) : null;
|
|
3392
|
+
const stderrId = em.stderr ? stageText(pending.objects, em.stderr, {
|
|
3393
|
+
mimeType: "text/plain"
|
|
3394
|
+
}) : null;
|
|
3395
|
+
const preview = (em.formatted_output ?? em.aggregated_output ?? em.stdout ?? "").slice(0, PREVIEW_MAX2);
|
|
3332
3396
|
const exitCode = typeof em.exit_code === "number" ? em.exit_code : null;
|
|
3333
3397
|
const isError = exitCode != null && exitCode !== 0 ? 1 : 0;
|
|
3334
3398
|
const eventId2 = eventId(sessionId2, ordinal, "exec_command_end");
|
|
@@ -3472,6 +3536,7 @@ async function handleEventMsg(bundle, sessionId2, currentTurnId, rawRecordId2, o
|
|
|
3472
3536
|
confidence: "high"
|
|
3473
3537
|
});
|
|
3474
3538
|
}
|
|
3539
|
+
__name(handleEventMsg, "handleEventMsg");
|
|
3475
3540
|
function extractNativeId(env) {
|
|
3476
3541
|
const p = env.payload;
|
|
3477
3542
|
if (!p) return null;
|
|
@@ -3480,6 +3545,7 @@ function extractNativeId(env) {
|
|
|
3480
3545
|
if (typeof p.turn_id === "string") return p.turn_id;
|
|
3481
3546
|
return null;
|
|
3482
3547
|
}
|
|
3548
|
+
__name(extractNativeId, "extractNativeId");
|
|
3483
3549
|
function mapMessageRole(role) {
|
|
3484
3550
|
switch (role) {
|
|
3485
3551
|
case "user":
|
|
@@ -3496,6 +3562,7 @@ function mapMessageRole(role) {
|
|
|
3496
3562
|
return "operational";
|
|
3497
3563
|
}
|
|
3498
3564
|
}
|
|
3565
|
+
__name(mapMessageRole, "mapMessageRole");
|
|
3499
3566
|
function canonicalToolType2(toolName) {
|
|
3500
3567
|
const lower = toolName.toLowerCase();
|
|
3501
3568
|
if (lower.startsWith("mcp__")) return "mcp";
|
|
@@ -3511,6 +3578,7 @@ function canonicalToolType2(toolName) {
|
|
|
3511
3578
|
if (lower === "agent" || lower === "subagent" || lower === "collab_spawn") return "subagent";
|
|
3512
3579
|
return "other";
|
|
3513
3580
|
}
|
|
3581
|
+
__name(canonicalToolType2, "canonicalToolType");
|
|
3514
3582
|
function inferCommandFromArgs2(toolName, args) {
|
|
3515
3583
|
if (!args || typeof args !== "object") {
|
|
3516
3584
|
if (typeof args === "string") {
|
|
@@ -3530,6 +3598,7 @@ function inferCommandFromArgs2(toolName, args) {
|
|
|
3530
3598
|
}
|
|
3531
3599
|
return null;
|
|
3532
3600
|
}
|
|
3601
|
+
__name(inferCommandFromArgs2, "inferCommandFromArgs");
|
|
3533
3602
|
function inferPathFromArgs2(args) {
|
|
3534
3603
|
if (!args || typeof args !== "object") {
|
|
3535
3604
|
if (typeof args === "string") {
|
|
@@ -3548,15 +3617,18 @@ function inferPathFromArgs2(args) {
|
|
|
3548
3617
|
if (typeof obj.absolute_path === "string") return obj.absolute_path;
|
|
3549
3618
|
return null;
|
|
3550
3619
|
}
|
|
3620
|
+
__name(inferPathFromArgs2, "inferPathFromArgs");
|
|
3551
3621
|
function looksLikeError(text) {
|
|
3552
3622
|
return /\b(error|exception|failed|stack trace)\b/i.test(text);
|
|
3553
3623
|
}
|
|
3624
|
+
__name(looksLikeError, "looksLikeError");
|
|
3554
3625
|
function durationMs(d) {
|
|
3555
3626
|
if (!d || typeof d !== "object") return null;
|
|
3556
3627
|
const secs = typeof d.secs === "number" ? d.secs : 0;
|
|
3557
3628
|
const nanos = typeof d.nanos === "number" ? d.nanos : 0;
|
|
3558
3629
|
return secs * 1e3 + Math.floor(nanos / 1e6);
|
|
3559
3630
|
}
|
|
3631
|
+
__name(durationMs, "durationMs");
|
|
3560
3632
|
function stringifyOrNull2(value) {
|
|
3561
3633
|
if (value == null) return null;
|
|
3562
3634
|
if (typeof value === "string") return value;
|
|
@@ -3566,6 +3638,7 @@ function stringifyOrNull2(value) {
|
|
|
3566
3638
|
return null;
|
|
3567
3639
|
}
|
|
3568
3640
|
}
|
|
3641
|
+
__name(stringifyOrNull2, "stringifyOrNull");
|
|
3569
3642
|
function parseSubagent(source) {
|
|
3570
3643
|
if (!source || typeof source !== "object") return null;
|
|
3571
3644
|
const obj = source;
|
|
@@ -3582,6 +3655,7 @@ function parseSubagent(source) {
|
|
|
3582
3655
|
agent_nickname: typeof tso.agent_nickname === "string" ? tso.agent_nickname : void 0
|
|
3583
3656
|
};
|
|
3584
3657
|
}
|
|
3658
|
+
__name(parseSubagent, "parseSubagent");
|
|
3585
3659
|
function buildSearchDocs2(pending) {
|
|
3586
3660
|
const sessionId2 = pending.session?.session_id ?? null;
|
|
3587
3661
|
if (!sessionId2) return;
|
|
@@ -3593,9 +3667,7 @@ function buildSearchDocs2(pending) {
|
|
|
3593
3667
|
blocksByMsg.set(b.message_id, list);
|
|
3594
3668
|
}
|
|
3595
3669
|
for (const m of pending.messages) {
|
|
3596
|
-
const text = (blocksByMsg.get(m.message_id) ?? []).filter(
|
|
3597
|
-
(b) => b.text_inline && (b.block_type === "input_text" || b.block_type === "output_text" || b.block_type === "text")
|
|
3598
|
-
).map((b) => b.text_inline).join("\n");
|
|
3670
|
+
const text = (blocksByMsg.get(m.message_id) ?? []).filter((b) => b.text_inline && (b.block_type === "input_text" || b.block_type === "output_text" || b.block_type === "text")).map((b) => b.text_inline).join("\n");
|
|
3599
3671
|
if (!text || text.length === 0) continue;
|
|
3600
3672
|
pending.searchDocs.push({
|
|
3601
3673
|
doc_id: `msg:${m.message_id}`,
|
|
@@ -3653,284 +3725,96 @@ function buildSearchDocs2(pending) {
|
|
|
3653
3725
|
}
|
|
3654
3726
|
}
|
|
3655
3727
|
}
|
|
3728
|
+
__name(buildSearchDocs2, "buildSearchDocs");
|
|
3656
3729
|
function flushPending2(bundle, pending, meta) {
|
|
3657
3730
|
if (!pending.session) return;
|
|
3658
|
-
const insertRaw = prepare(
|
|
3659
|
-
bundle.db,
|
|
3660
|
-
`INSERT OR IGNORE INTO raw_records (
|
|
3731
|
+
const insertRaw = prepare(bundle.db, `INSERT OR IGNORE INTO raw_records (
|
|
3661
3732
|
raw_record_id, source_file_id, source_tool, record_kind, ordinal,
|
|
3662
3733
|
line_no, json_pointer, native_id, raw_object_id, decoded_json_object_id,
|
|
3663
3734
|
parser_status, confidence, import_batch_id
|
|
3664
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
3665
|
-
);
|
|
3735
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
3666
3736
|
for (const r of pending.rawRecords) {
|
|
3667
|
-
insertRaw.run(
|
|
3668
|
-
r.raw_record_id,
|
|
3669
|
-
r.source_file_id,
|
|
3670
|
-
r.source_tool,
|
|
3671
|
-
r.record_kind,
|
|
3672
|
-
r.ordinal,
|
|
3673
|
-
r.line_no,
|
|
3674
|
-
r.json_pointer,
|
|
3675
|
-
r.native_id,
|
|
3676
|
-
r.raw_object_id,
|
|
3677
|
-
r.decoded_json_object_id,
|
|
3678
|
-
r.parser_status,
|
|
3679
|
-
r.confidence,
|
|
3680
|
-
r.import_batch_id
|
|
3681
|
-
);
|
|
3737
|
+
insertRaw.run(r.raw_record_id, r.source_file_id, r.source_tool, r.record_kind, r.ordinal, r.line_no, r.json_pointer, r.native_id, r.raw_object_id, r.decoded_json_object_id, r.parser_status, r.confidence, r.import_batch_id);
|
|
3682
3738
|
}
|
|
3683
|
-
const insertSession = prepare(
|
|
3684
|
-
bundle.db,
|
|
3685
|
-
`INSERT OR REPLACE INTO sessions (
|
|
3739
|
+
const insertSession = prepare(bundle.db, `INSERT OR REPLACE INTO sessions (
|
|
3686
3740
|
session_id, source_tool, source_session_id, project_id, parent_session_id,
|
|
3687
3741
|
is_subagent, agent_role, agent_nickname, title, summary,
|
|
3688
3742
|
start_ts, end_ts, cwd_initial, git_branch_initial,
|
|
3689
3743
|
model_first, model_last, status, timeline_confidence, raw_record_id
|
|
3690
|
-
) VALUES (?, ?, ?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'high', ?)`
|
|
3691
|
-
);
|
|
3692
|
-
|
|
3693
|
-
pending.session.session_id,
|
|
3694
|
-
meta.sourceTool,
|
|
3695
|
-
pending.session.source_session_id,
|
|
3696
|
-
null,
|
|
3697
|
-
pending.session.is_subagent,
|
|
3698
|
-
pending.session.agent_role,
|
|
3699
|
-
pending.session.agent_nickname,
|
|
3700
|
-
pending.session.title,
|
|
3701
|
-
null,
|
|
3702
|
-
pending.session.start_ts,
|
|
3703
|
-
meta.sessionEndTs,
|
|
3704
|
-
pending.session.cwd_initial,
|
|
3705
|
-
pending.session.git_branch_initial,
|
|
3706
|
-
meta.modelFirst,
|
|
3707
|
-
meta.modelLast,
|
|
3708
|
-
"completed",
|
|
3709
|
-
pending.session.raw_record_id
|
|
3710
|
-
);
|
|
3711
|
-
const insertTurn = prepare(
|
|
3712
|
-
bundle.db,
|
|
3713
|
-
`INSERT OR REPLACE INTO turns (
|
|
3744
|
+
) VALUES (?, ?, ?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'high', ?)`);
|
|
3745
|
+
insertSession.run(pending.session.session_id, meta.sourceTool, pending.session.source_session_id, null, pending.session.is_subagent, pending.session.agent_role, pending.session.agent_nickname, pending.session.title, null, pending.session.start_ts, meta.sessionEndTs, pending.session.cwd_initial, pending.session.git_branch_initial, meta.modelFirst, meta.modelLast, "completed", pending.session.raw_record_id);
|
|
3746
|
+
const insertTurn = prepare(bundle.db, `INSERT OR REPLACE INTO turns (
|
|
3714
3747
|
turn_id, session_id, source_turn_id, ordinal, start_ts, end_ts,
|
|
3715
3748
|
model, cwd, git_branch, approval_policy, sandbox_policy, effort, raw_record_id
|
|
3716
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
3717
|
-
);
|
|
3749
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
3718
3750
|
for (const t of pending.turns) {
|
|
3719
|
-
insertTurn.run(
|
|
3720
|
-
t.turn_id,
|
|
3721
|
-
pending.session.session_id,
|
|
3722
|
-
t.source_turn_id,
|
|
3723
|
-
t.ordinal,
|
|
3724
|
-
t.start_ts,
|
|
3725
|
-
null,
|
|
3726
|
-
t.model,
|
|
3727
|
-
t.cwd,
|
|
3728
|
-
null,
|
|
3729
|
-
t.approval_policy,
|
|
3730
|
-
t.sandbox_policy,
|
|
3731
|
-
t.effort,
|
|
3732
|
-
t.raw_record_id
|
|
3733
|
-
);
|
|
3751
|
+
insertTurn.run(t.turn_id, pending.session.session_id, t.source_turn_id, t.ordinal, t.start_ts, null, t.model, t.cwd, null, t.approval_policy, t.sandbox_policy, t.effort, t.raw_record_id);
|
|
3734
3752
|
}
|
|
3735
|
-
const insertEvent = prepare(
|
|
3736
|
-
bundle.db,
|
|
3737
|
-
`INSERT OR REPLACE INTO events (
|
|
3753
|
+
const insertEvent = prepare(bundle.db, `INSERT OR REPLACE INTO events (
|
|
3738
3754
|
event_id, session_id, turn_id, source_event_id, event_type, source_type,
|
|
3739
3755
|
subtype, timestamp, ordinal, actor, payload_object_id, raw_record_id,
|
|
3740
3756
|
confidence, is_derived
|
|
3741
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0)`
|
|
3742
|
-
);
|
|
3757
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0)`);
|
|
3743
3758
|
for (const e of pending.events) {
|
|
3744
|
-
insertEvent.run(
|
|
3745
|
-
e.event_id,
|
|
3746
|
-
pending.session.session_id,
|
|
3747
|
-
e.turn_id,
|
|
3748
|
-
e.source_event_id,
|
|
3749
|
-
e.event_type,
|
|
3750
|
-
e.source_type,
|
|
3751
|
-
e.subtype,
|
|
3752
|
-
e.timestamp,
|
|
3753
|
-
e.ordinal,
|
|
3754
|
-
e.actor,
|
|
3755
|
-
e.payload_object_id,
|
|
3756
|
-
e.raw_record_id,
|
|
3757
|
-
e.confidence
|
|
3758
|
-
);
|
|
3759
|
+
insertEvent.run(e.event_id, pending.session.session_id, e.turn_id, e.source_event_id, e.event_type, e.source_type, e.subtype, e.timestamp, e.ordinal, e.actor, e.payload_object_id, e.raw_record_id, e.confidence);
|
|
3759
3760
|
}
|
|
3760
|
-
const insertMessage = prepare(
|
|
3761
|
-
bundle.db,
|
|
3762
|
-
`INSERT OR REPLACE INTO messages (
|
|
3761
|
+
const insertMessage = prepare(bundle.db, `INSERT OR REPLACE INTO messages (
|
|
3763
3762
|
message_id, session_id, turn_id, event_id, source_message_id, role,
|
|
3764
3763
|
author_name, model, timestamp, ordinal, parent_message_id, request_id,
|
|
3765
3764
|
status, raw_record_id
|
|
3766
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
3767
|
-
);
|
|
3765
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
3768
3766
|
for (const m of pending.messages) {
|
|
3769
|
-
insertMessage.run(
|
|
3770
|
-
m.message_id,
|
|
3771
|
-
pending.session.session_id,
|
|
3772
|
-
m.turn_id,
|
|
3773
|
-
m.event_id,
|
|
3774
|
-
m.source_message_id,
|
|
3775
|
-
m.role,
|
|
3776
|
-
null,
|
|
3777
|
-
m.model,
|
|
3778
|
-
m.timestamp,
|
|
3779
|
-
m.ordinal,
|
|
3780
|
-
null,
|
|
3781
|
-
null,
|
|
3782
|
-
null,
|
|
3783
|
-
m.raw_record_id
|
|
3784
|
-
);
|
|
3767
|
+
insertMessage.run(m.message_id, pending.session.session_id, m.turn_id, m.event_id, m.source_message_id, m.role, null, m.model, m.timestamp, m.ordinal, null, null, null, m.raw_record_id);
|
|
3785
3768
|
}
|
|
3786
|
-
const insertBlock = prepare(
|
|
3787
|
-
bundle.db,
|
|
3788
|
-
`INSERT OR REPLACE INTO content_blocks (
|
|
3769
|
+
const insertBlock = prepare(bundle.db, `INSERT OR REPLACE INTO content_blocks (
|
|
3789
3770
|
block_id, message_id, event_id, session_id, ordinal, block_type,
|
|
3790
3771
|
text_object_id, text_inline, mime_type, token_count, is_error,
|
|
3791
3772
|
is_redacted, visibility, raw_record_id
|
|
3792
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'default', ?)`
|
|
3793
|
-
);
|
|
3773
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'default', ?)`);
|
|
3794
3774
|
for (const b of pending.blocks) {
|
|
3795
|
-
insertBlock.run(
|
|
3796
|
-
b.block_id,
|
|
3797
|
-
b.message_id,
|
|
3798
|
-
b.event_id,
|
|
3799
|
-
pending.session.session_id,
|
|
3800
|
-
b.ordinal,
|
|
3801
|
-
b.block_type,
|
|
3802
|
-
b.text_object_id,
|
|
3803
|
-
b.text_inline,
|
|
3804
|
-
null,
|
|
3805
|
-
null,
|
|
3806
|
-
0,
|
|
3807
|
-
0,
|
|
3808
|
-
b.raw_record_id
|
|
3809
|
-
);
|
|
3775
|
+
insertBlock.run(b.block_id, b.message_id, b.event_id, pending.session.session_id, b.ordinal, b.block_type, b.text_object_id, b.text_inline, null, null, 0, 0, b.raw_record_id);
|
|
3810
3776
|
}
|
|
3811
|
-
const insertToolCall = prepare(
|
|
3812
|
-
bundle.db,
|
|
3813
|
-
`INSERT OR REPLACE INTO tool_calls (
|
|
3777
|
+
const insertToolCall = prepare(bundle.db, `INSERT OR REPLACE INTO tool_calls (
|
|
3814
3778
|
tool_call_id, session_id, turn_id, message_id, event_id,
|
|
3815
3779
|
source_call_id, tool_name, canonical_tool_type, args_object_id,
|
|
3816
3780
|
command, cwd, path, query, timestamp_start, timestamp_end, status,
|
|
3817
3781
|
raw_record_id
|
|
3818
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
3819
|
-
);
|
|
3782
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
3820
3783
|
for (const c of pending.toolCallsList) {
|
|
3821
|
-
insertToolCall.run(
|
|
3822
|
-
c.tool_call_id,
|
|
3823
|
-
pending.session.session_id,
|
|
3824
|
-
c.turn_id,
|
|
3825
|
-
c.message_id,
|
|
3826
|
-
c.event_id,
|
|
3827
|
-
c.source_call_id,
|
|
3828
|
-
c.tool_name,
|
|
3829
|
-
c.canonical_tool_type,
|
|
3830
|
-
c.args_object_id,
|
|
3831
|
-
c.command,
|
|
3832
|
-
c.cwd,
|
|
3833
|
-
c.path,
|
|
3834
|
-
c.query,
|
|
3835
|
-
c.timestamp_start,
|
|
3836
|
-
null,
|
|
3837
|
-
c.status,
|
|
3838
|
-
c.raw_record_id
|
|
3839
|
-
);
|
|
3784
|
+
insertToolCall.run(c.tool_call_id, pending.session.session_id, c.turn_id, c.message_id, c.event_id, c.source_call_id, c.tool_name, c.canonical_tool_type, c.args_object_id, c.command, c.cwd, c.path, c.query, c.timestamp_start, null, c.status, c.raw_record_id);
|
|
3840
3785
|
}
|
|
3841
|
-
const insertToolResult = prepare(
|
|
3842
|
-
bundle.db,
|
|
3843
|
-
`INSERT OR REPLACE INTO tool_results (
|
|
3786
|
+
const insertToolResult = prepare(bundle.db, `INSERT OR REPLACE INTO tool_results (
|
|
3844
3787
|
tool_result_id, tool_call_id, session_id, message_id, event_id,
|
|
3845
3788
|
source_call_id, status, is_error, exit_code, duration_ms,
|
|
3846
3789
|
stdout_object_id, stderr_object_id, output_object_id, preview, raw_record_id
|
|
3847
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
3848
|
-
);
|
|
3790
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
3849
3791
|
for (const r of pending.toolResults) {
|
|
3850
|
-
insertToolResult.run(
|
|
3851
|
-
r.tool_result_id,
|
|
3852
|
-
r.tool_call_id,
|
|
3853
|
-
pending.session.session_id,
|
|
3854
|
-
r.message_id,
|
|
3855
|
-
r.event_id,
|
|
3856
|
-
r.source_call_id,
|
|
3857
|
-
r.status,
|
|
3858
|
-
r.is_error,
|
|
3859
|
-
r.exit_code,
|
|
3860
|
-
r.duration_ms,
|
|
3861
|
-
r.stdout_object_id,
|
|
3862
|
-
r.stderr_object_id,
|
|
3863
|
-
r.output_object_id,
|
|
3864
|
-
r.preview,
|
|
3865
|
-
r.raw_record_id
|
|
3866
|
-
);
|
|
3792
|
+
insertToolResult.run(r.tool_result_id, r.tool_call_id, pending.session.session_id, r.message_id, r.event_id, r.source_call_id, r.status, r.is_error, r.exit_code, r.duration_ms, r.stdout_object_id, r.stderr_object_id, r.output_object_id, r.preview, r.raw_record_id);
|
|
3867
3793
|
}
|
|
3868
|
-
const insertArtifact = prepare(
|
|
3869
|
-
bundle.db,
|
|
3870
|
-
`INSERT OR REPLACE INTO artifacts (
|
|
3794
|
+
const insertArtifact = prepare(bundle.db, `INSERT OR REPLACE INTO artifacts (
|
|
3871
3795
|
artifact_id, session_id, project_id, source_tool, kind, path,
|
|
3872
3796
|
logical_path, object_id, text_object_id, mime_type, size_bytes,
|
|
3873
3797
|
created_ts, raw_record_id
|
|
3874
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
3875
|
-
);
|
|
3798
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
3876
3799
|
for (const a of pending.artifacts) {
|
|
3877
|
-
insertArtifact.run(
|
|
3878
|
-
a.artifact_id,
|
|
3879
|
-
pending.session.session_id,
|
|
3880
|
-
null,
|
|
3881
|
-
"codex",
|
|
3882
|
-
a.kind,
|
|
3883
|
-
a.path,
|
|
3884
|
-
a.logical_path,
|
|
3885
|
-
a.object_id,
|
|
3886
|
-
a.text_object_id,
|
|
3887
|
-
a.mime_type,
|
|
3888
|
-
a.size_bytes,
|
|
3889
|
-
a.created_ts,
|
|
3890
|
-
a.raw_record_id
|
|
3891
|
-
);
|
|
3800
|
+
insertArtifact.run(a.artifact_id, pending.session.session_id, null, "codex", a.kind, a.path, a.logical_path, a.object_id, a.text_object_id, a.mime_type, a.size_bytes, a.created_ts, a.raw_record_id);
|
|
3892
3801
|
}
|
|
3893
|
-
const insertEdge = prepare(
|
|
3894
|
-
bundle.db,
|
|
3895
|
-
`INSERT OR IGNORE INTO edges (
|
|
3802
|
+
const insertEdge = prepare(bundle.db, `INSERT OR IGNORE INTO edges (
|
|
3896
3803
|
src_type, src_id, dst_type, dst_id, edge_type, confidence, source,
|
|
3897
3804
|
raw_record_id, metadata_object_id
|
|
3898
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULL)`
|
|
3899
|
-
);
|
|
3805
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULL)`);
|
|
3900
3806
|
for (const e of pending.edges) {
|
|
3901
|
-
insertEdge.run(
|
|
3902
|
-
e.src_type,
|
|
3903
|
-
e.src_id,
|
|
3904
|
-
e.dst_type,
|
|
3905
|
-
e.dst_id,
|
|
3906
|
-
e.edge_type,
|
|
3907
|
-
e.confidence,
|
|
3908
|
-
e.source,
|
|
3909
|
-
e.raw_record_id
|
|
3910
|
-
);
|
|
3807
|
+
insertEdge.run(e.src_type, e.src_id, e.dst_type, e.dst_id, e.edge_type, e.confidence, e.source, e.raw_record_id);
|
|
3911
3808
|
}
|
|
3912
|
-
const insertSearch = prepare(
|
|
3913
|
-
bundle.db,
|
|
3914
|
-
`INSERT OR REPLACE INTO search_docs (
|
|
3809
|
+
const insertSearch = prepare(bundle.db, `INSERT OR REPLACE INTO search_docs (
|
|
3915
3810
|
doc_id, entity_type, entity_id, session_id, project_id, timestamp,
|
|
3916
3811
|
role, tool_name, canonical_tool_type, field_kind, text
|
|
3917
|
-
) VALUES (?, ?, ?, ?, NULL, ?, ?, ?, ?, ?, ?)`
|
|
3918
|
-
);
|
|
3812
|
+
) VALUES (?, ?, ?, ?, NULL, ?, ?, ?, ?, ?, ?)`);
|
|
3919
3813
|
for (const d of pending.searchDocs) {
|
|
3920
|
-
insertSearch.run(
|
|
3921
|
-
d.doc_id,
|
|
3922
|
-
d.entity_type,
|
|
3923
|
-
d.entity_id,
|
|
3924
|
-
pending.session.session_id,
|
|
3925
|
-
d.timestamp,
|
|
3926
|
-
d.role,
|
|
3927
|
-
d.tool_name,
|
|
3928
|
-
d.canonical_tool_type,
|
|
3929
|
-
d.field_kind,
|
|
3930
|
-
d.text
|
|
3931
|
-
);
|
|
3814
|
+
insertSearch.run(d.doc_id, d.entity_type, d.entity_id, pending.session.session_id, d.timestamp, d.role, d.tool_name, d.canonical_tool_type, d.field_kind, d.text);
|
|
3932
3815
|
}
|
|
3933
3816
|
}
|
|
3817
|
+
__name(flushPending2, "flushPending");
|
|
3934
3818
|
|
|
3935
3819
|
// src/importers/cursor/index.ts
|
|
3936
3820
|
import path10 from "path";
|
|
@@ -3959,60 +3843,76 @@ async function* discoverCursorStores(root) {
|
|
|
3959
3843
|
}
|
|
3960
3844
|
}
|
|
3961
3845
|
}
|
|
3846
|
+
__name(discoverCursorStores, "discoverCursorStores");
|
|
3962
3847
|
async function readdirSafe2(dir) {
|
|
3963
3848
|
try {
|
|
3964
|
-
return await readdir3(dir, {
|
|
3849
|
+
return await readdir3(dir, {
|
|
3850
|
+
withFileTypes: true
|
|
3851
|
+
});
|
|
3965
3852
|
} catch {
|
|
3966
3853
|
return [];
|
|
3967
3854
|
}
|
|
3968
3855
|
}
|
|
3856
|
+
__name(readdirSafe2, "readdirSafe");
|
|
3969
3857
|
|
|
3970
3858
|
// src/importers/cursor/index.ts
|
|
3971
3859
|
var PREVIEW_MAX3 = 4e3;
|
|
3972
3860
|
async function compileCursor(bundle, root, options = {}) {
|
|
3973
3861
|
const logger = options.logger;
|
|
3974
|
-
const batch = startBatch(bundle, "cursor", [
|
|
3862
|
+
const batch = startBatch(bundle, "cursor", [
|
|
3863
|
+
root
|
|
3864
|
+
]);
|
|
3975
3865
|
const counts = emptyCounts();
|
|
3976
|
-
logger?.info({
|
|
3866
|
+
logger?.info({
|
|
3867
|
+
batch_id: batch.batch_id,
|
|
3868
|
+
root
|
|
3869
|
+
}, "cursor batch started");
|
|
3977
3870
|
try {
|
|
3978
3871
|
for await (const store of discoverCursorStores(root)) {
|
|
3979
3872
|
counts.source_files_seen++;
|
|
3980
|
-
logger?.debug(
|
|
3981
|
-
|
|
3982
|
-
|
|
3983
|
-
|
|
3984
|
-
|
|
3985
|
-
},
|
|
3986
|
-
"cursor store discovered"
|
|
3987
|
-
);
|
|
3873
|
+
logger?.debug({
|
|
3874
|
+
path: store.filePath,
|
|
3875
|
+
workspace_id: store.workspaceId,
|
|
3876
|
+
agent_id: store.agentId
|
|
3877
|
+
}, "cursor store discovered");
|
|
3988
3878
|
try {
|
|
3989
3879
|
const fc = await compileCursorStore(bundle, batch, store, logger);
|
|
3990
3880
|
addCounts3(counts, fc);
|
|
3991
3881
|
} catch (error) {
|
|
3992
3882
|
counts.errors++;
|
|
3993
|
-
logger?.warn(
|
|
3994
|
-
|
|
3995
|
-
|
|
3996
|
-
|
|
3997
|
-
},
|
|
3998
|
-
"cursor store failed"
|
|
3999
|
-
);
|
|
3883
|
+
logger?.warn({
|
|
3884
|
+
err: error,
|
|
3885
|
+
path: store.filePath
|
|
3886
|
+
}, "cursor store failed");
|
|
4000
3887
|
await recordError(bundle, batch.batch_id, {
|
|
4001
3888
|
kind: "cursor_store_failed",
|
|
4002
3889
|
message: getErrorMessage(error),
|
|
4003
|
-
payload: {
|
|
3890
|
+
payload: {
|
|
3891
|
+
path: store.filePath
|
|
3892
|
+
}
|
|
4004
3893
|
});
|
|
4005
3894
|
}
|
|
4006
3895
|
}
|
|
4007
3896
|
finishBatch(bundle, batch, counts, "completed");
|
|
4008
|
-
logger?.info({
|
|
3897
|
+
logger?.info({
|
|
3898
|
+
batch_id: batch.batch_id,
|
|
3899
|
+
counts
|
|
3900
|
+
}, "cursor batch completed");
|
|
4009
3901
|
} catch (error) {
|
|
4010
3902
|
finishBatch(bundle, batch, counts, "failed");
|
|
4011
|
-
logger?.error({
|
|
3903
|
+
logger?.error({
|
|
3904
|
+
err: error,
|
|
3905
|
+
batch_id: batch.batch_id,
|
|
3906
|
+
counts
|
|
3907
|
+
}, "cursor batch failed");
|
|
4012
3908
|
throw error;
|
|
4013
3909
|
}
|
|
4014
|
-
return {
|
|
3910
|
+
return {
|
|
3911
|
+
batch,
|
|
3912
|
+
counts
|
|
3913
|
+
};
|
|
4015
3914
|
}
|
|
3915
|
+
__name(compileCursor, "compileCursor");
|
|
4016
3916
|
function emptyFileCounts3() {
|
|
4017
3917
|
return {
|
|
4018
3918
|
source_files_imported: 0,
|
|
@@ -4029,6 +3929,7 @@ function emptyFileCounts3() {
|
|
|
4029
3929
|
errors: 0
|
|
4030
3930
|
};
|
|
4031
3931
|
}
|
|
3932
|
+
__name(emptyFileCounts3, "emptyFileCounts");
|
|
4032
3933
|
function addCounts3(target, source) {
|
|
4033
3934
|
target.source_files_imported += source.source_files_imported;
|
|
4034
3935
|
target.source_files_skipped += source.source_files_skipped;
|
|
@@ -4043,6 +3944,7 @@ function addCounts3(target, source) {
|
|
|
4043
3944
|
target.edges += source.edges;
|
|
4044
3945
|
target.errors += source.errors;
|
|
4045
3946
|
}
|
|
3947
|
+
__name(addCounts3, "addCounts");
|
|
4046
3948
|
async function compileCursorStore(bundle, batch, store, logger) {
|
|
4047
3949
|
const counts = emptyFileCounts3();
|
|
4048
3950
|
const { row: sourceFile, alreadyKnown } = await registerSourceFile(bundle, {
|
|
@@ -4053,18 +3955,21 @@ async function compileCursorStore(bundle, batch, store, logger) {
|
|
|
4053
3955
|
});
|
|
4054
3956
|
if (alreadyKnown) {
|
|
4055
3957
|
counts.source_files_skipped = 1;
|
|
4056
|
-
logger?.debug(
|
|
4057
|
-
|
|
4058
|
-
|
|
4059
|
-
);
|
|
3958
|
+
logger?.debug({
|
|
3959
|
+
path: store.filePath,
|
|
3960
|
+
source_file_id: sourceFile.source_file_id
|
|
3961
|
+
}, "cursor store skipped");
|
|
4060
3962
|
return counts;
|
|
4061
3963
|
}
|
|
4062
3964
|
counts.source_files_imported = 1;
|
|
4063
|
-
logger?.debug(
|
|
4064
|
-
|
|
4065
|
-
|
|
4066
|
-
);
|
|
4067
|
-
const cdb = new Database2(store.filePath, {
|
|
3965
|
+
logger?.debug({
|
|
3966
|
+
path: store.filePath,
|
|
3967
|
+
source_file_id: sourceFile.source_file_id
|
|
3968
|
+
}, "cursor store registered");
|
|
3969
|
+
const cdb = new Database2(store.filePath, {
|
|
3970
|
+
readonly: true,
|
|
3971
|
+
fileMustExist: true
|
|
3972
|
+
});
|
|
4068
3973
|
try {
|
|
4069
3974
|
const pending = {
|
|
4070
3975
|
rawRecords: [],
|
|
@@ -4151,7 +4056,6 @@ async function compileCursorStore(bundle, batch, store, logger) {
|
|
|
4151
4056
|
decoded_json_object_id: parsed != null ? stageJson(pending.objects, parsed) : null,
|
|
4152
4057
|
parser_status: parsed != null ? "ok" : looksJson ? "failed" : "partial",
|
|
4153
4058
|
confidence: "low",
|
|
4154
|
-
// timeline order from blob list isn't canonical
|
|
4155
4059
|
import_batch_id: batch.batch_id,
|
|
4156
4060
|
record_kind: "sqlite_blob"
|
|
4157
4061
|
});
|
|
@@ -4189,16 +4093,7 @@ async function compileCursorStore(bundle, batch, store, logger) {
|
|
|
4189
4093
|
for (let bi = 0; bi < content.length; bi++) {
|
|
4190
4094
|
const item = content[bi];
|
|
4191
4095
|
if (!item) continue;
|
|
4192
|
-
await processContentItem(
|
|
4193
|
-
bundle,
|
|
4194
|
-
sessionPk,
|
|
4195
|
-
messageId2,
|
|
4196
|
-
eventId2,
|
|
4197
|
-
bi,
|
|
4198
|
-
item,
|
|
4199
|
-
blobRawId,
|
|
4200
|
-
pending
|
|
4201
|
-
);
|
|
4096
|
+
await processContentItem(bundle, sessionPk, messageId2, eventId2, bi, item, blobRawId, pending);
|
|
4202
4097
|
}
|
|
4203
4098
|
}
|
|
4204
4099
|
}
|
|
@@ -4215,18 +4110,21 @@ async function compileCursorStore(bundle, batch, store, logger) {
|
|
|
4215
4110
|
counts.tool_calls = pending.toolCallsList.length;
|
|
4216
4111
|
counts.tool_results = pending.toolResults.length;
|
|
4217
4112
|
counts.artifacts = pending.artifacts.length;
|
|
4218
|
-
logger?.debug(
|
|
4219
|
-
|
|
4220
|
-
|
|
4221
|
-
|
|
4113
|
+
logger?.debug({
|
|
4114
|
+
path: store.filePath,
|
|
4115
|
+
source_file_id: sourceFile.source_file_id,
|
|
4116
|
+
counts
|
|
4117
|
+
}, "cursor store imported");
|
|
4222
4118
|
return counts;
|
|
4223
4119
|
} finally {
|
|
4224
4120
|
cdb.close();
|
|
4225
4121
|
}
|
|
4226
4122
|
}
|
|
4123
|
+
__name(compileCursorStore, "compileCursorStore");
|
|
4227
4124
|
function hexToUtf8(hex) {
|
|
4228
4125
|
return Buffer.from(hex, "hex").toString("utf8");
|
|
4229
4126
|
}
|
|
4127
|
+
__name(hexToUtf8, "hexToUtf8");
|
|
4230
4128
|
function mapRole(role) {
|
|
4231
4129
|
switch (role) {
|
|
4232
4130
|
case "user":
|
|
@@ -4241,6 +4139,7 @@ function mapRole(role) {
|
|
|
4241
4139
|
return "operational";
|
|
4242
4140
|
}
|
|
4243
4141
|
}
|
|
4142
|
+
__name(mapRole, "mapRole");
|
|
4244
4143
|
async function pushTextBlock(bundle, pending, messageId2, ordinal, blockType, text, rawRecordId2, visibility = "default") {
|
|
4245
4144
|
if (!text) return;
|
|
4246
4145
|
const overflow = text.length > PREVIEW_MAX3 ? stageText(pending.objects, text) : null;
|
|
@@ -4257,6 +4156,7 @@ async function pushTextBlock(bundle, pending, messageId2, ordinal, blockType, te
|
|
|
4257
4156
|
raw_record_id: rawRecordId2
|
|
4258
4157
|
});
|
|
4259
4158
|
}
|
|
4159
|
+
__name(pushTextBlock, "pushTextBlock");
|
|
4260
4160
|
async function processContentItem(bundle, sessionId2, messageId2, eventId2, ordinal, item, rawRecordId2, pending) {
|
|
4261
4161
|
const t = item.type;
|
|
4262
4162
|
if (t === "text") {
|
|
@@ -4264,16 +4164,7 @@ async function processContentItem(bundle, sessionId2, messageId2, eventId2, ordi
|
|
|
4264
4164
|
return;
|
|
4265
4165
|
}
|
|
4266
4166
|
if (t === "reasoning") {
|
|
4267
|
-
await pushTextBlock(
|
|
4268
|
-
bundle,
|
|
4269
|
-
pending,
|
|
4270
|
-
messageId2,
|
|
4271
|
-
ordinal,
|
|
4272
|
-
"thinking",
|
|
4273
|
-
item.text ?? "",
|
|
4274
|
-
rawRecordId2,
|
|
4275
|
-
"hidden_by_default"
|
|
4276
|
-
);
|
|
4167
|
+
await pushTextBlock(bundle, pending, messageId2, ordinal, "thinking", item.text ?? "", rawRecordId2, "hidden_by_default");
|
|
4277
4168
|
return;
|
|
4278
4169
|
}
|
|
4279
4170
|
if (t === "redacted-reasoning") {
|
|
@@ -4374,11 +4265,13 @@ async function processContentItem(bundle, sessionId2, messageId2, eventId2, ordi
|
|
|
4374
4265
|
raw_record_id: rawRecordId2
|
|
4375
4266
|
});
|
|
4376
4267
|
}
|
|
4268
|
+
__name(processContentItem, "processContentItem");
|
|
4377
4269
|
function readIsError(item) {
|
|
4378
4270
|
const exp = item.experimental_content;
|
|
4379
4271
|
if (exp && typeof exp.isError === "boolean") return exp.isError;
|
|
4380
4272
|
return false;
|
|
4381
4273
|
}
|
|
4274
|
+
__name(readIsError, "readIsError");
|
|
4382
4275
|
function canonicalToolType3(toolName) {
|
|
4383
4276
|
const lower = toolName.toLowerCase();
|
|
4384
4277
|
if (lower.startsWith("mcp__")) return "mcp";
|
|
@@ -4395,6 +4288,7 @@ function canonicalToolType3(toolName) {
|
|
|
4395
4288
|
if (lower === "applypatch" || lower === "apply_patch") return "patch";
|
|
4396
4289
|
return "other";
|
|
4397
4290
|
}
|
|
4291
|
+
__name(canonicalToolType3, "canonicalToolType");
|
|
4398
4292
|
function stringifyOrNull3(value) {
|
|
4399
4293
|
if (value == null) return null;
|
|
4400
4294
|
if (typeof value === "string") return value;
|
|
@@ -4404,6 +4298,7 @@ function stringifyOrNull3(value) {
|
|
|
4404
4298
|
return null;
|
|
4405
4299
|
}
|
|
4406
4300
|
}
|
|
4301
|
+
__name(stringifyOrNull3, "stringifyOrNull");
|
|
4407
4302
|
function buildSearchDocs3(pending) {
|
|
4408
4303
|
const sessionId2 = pending.session?.session_id ?? null;
|
|
4409
4304
|
if (!sessionId2) return;
|
|
@@ -4460,193 +4355,74 @@ function buildSearchDocs3(pending) {
|
|
|
4460
4355
|
}
|
|
4461
4356
|
}
|
|
4462
4357
|
}
|
|
4358
|
+
__name(buildSearchDocs3, "buildSearchDocs");
|
|
4463
4359
|
function flushPending3(bundle, pending) {
|
|
4464
4360
|
if (!pending.session) return;
|
|
4465
|
-
const insertRaw = prepare(
|
|
4466
|
-
bundle.db,
|
|
4467
|
-
`INSERT OR IGNORE INTO raw_records (
|
|
4361
|
+
const insertRaw = prepare(bundle.db, `INSERT OR IGNORE INTO raw_records (
|
|
4468
4362
|
raw_record_id, source_file_id, source_tool, record_kind, ordinal,
|
|
4469
4363
|
line_no, json_pointer, native_id, raw_object_id, decoded_json_object_id,
|
|
4470
4364
|
parser_status, confidence, import_batch_id
|
|
4471
|
-
) VALUES (?, ?, 'cursor', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
4472
|
-
);
|
|
4365
|
+
) VALUES (?, ?, 'cursor', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
4473
4366
|
for (const r of pending.rawRecords) {
|
|
4474
|
-
insertRaw.run(
|
|
4475
|
-
r.raw_record_id,
|
|
4476
|
-
r.source_file_id,
|
|
4477
|
-
r.record_kind,
|
|
4478
|
-
r.ordinal,
|
|
4479
|
-
r.line_no,
|
|
4480
|
-
r.json_pointer,
|
|
4481
|
-
r.native_id,
|
|
4482
|
-
r.raw_object_id,
|
|
4483
|
-
r.decoded_json_object_id,
|
|
4484
|
-
r.parser_status,
|
|
4485
|
-
r.confidence,
|
|
4486
|
-
r.import_batch_id
|
|
4487
|
-
);
|
|
4367
|
+
insertRaw.run(r.raw_record_id, r.source_file_id, r.record_kind, r.ordinal, r.line_no, r.json_pointer, r.native_id, r.raw_object_id, r.decoded_json_object_id, r.parser_status, r.confidence, r.import_batch_id);
|
|
4488
4368
|
}
|
|
4489
|
-
prepare(
|
|
4490
|
-
bundle.db,
|
|
4491
|
-
`INSERT OR REPLACE INTO sessions (
|
|
4369
|
+
prepare(bundle.db, `INSERT OR REPLACE INTO sessions (
|
|
4492
4370
|
session_id, source_tool, source_session_id, project_id, parent_session_id,
|
|
4493
4371
|
is_subagent, agent_role, agent_nickname, title, summary,
|
|
4494
4372
|
start_ts, end_ts, cwd_initial, git_branch_initial,
|
|
4495
4373
|
model_first, model_last, status, timeline_confidence, raw_record_id
|
|
4496
|
-
) VALUES (?, 'cursor', ?, NULL, NULL, 0, ?, ?, ?, NULL, ?, NULL, NULL, NULL, ?, ?, NULL, 'low', ?)`
|
|
4497
|
-
|
|
4498
|
-
pending.session.session_id,
|
|
4499
|
-
pending.session.source_session_id,
|
|
4500
|
-
pending.session.agent_role,
|
|
4501
|
-
pending.session.agent_nickname,
|
|
4502
|
-
pending.session.title,
|
|
4503
|
-
pending.session.start_ts,
|
|
4504
|
-
pending.session.model,
|
|
4505
|
-
pending.session.model,
|
|
4506
|
-
pending.session.raw_record_id
|
|
4507
|
-
);
|
|
4508
|
-
const insertEvent = prepare(
|
|
4509
|
-
bundle.db,
|
|
4510
|
-
`INSERT OR REPLACE INTO events (
|
|
4374
|
+
) VALUES (?, 'cursor', ?, NULL, NULL, 0, ?, ?, ?, NULL, ?, NULL, NULL, NULL, ?, ?, NULL, 'low', ?)`).run(pending.session.session_id, pending.session.source_session_id, pending.session.agent_role, pending.session.agent_nickname, pending.session.title, pending.session.start_ts, pending.session.model, pending.session.model, pending.session.raw_record_id);
|
|
4375
|
+
const insertEvent = prepare(bundle.db, `INSERT OR REPLACE INTO events (
|
|
4511
4376
|
event_id, session_id, turn_id, source_event_id, event_type, source_type,
|
|
4512
4377
|
subtype, timestamp, ordinal, actor, payload_object_id, raw_record_id,
|
|
4513
4378
|
confidence, is_derived
|
|
4514
|
-
) VALUES (?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0)`
|
|
4515
|
-
);
|
|
4379
|
+
) VALUES (?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0)`);
|
|
4516
4380
|
for (const e of pending.events) {
|
|
4517
|
-
insertEvent.run(
|
|
4518
|
-
e.event_id,
|
|
4519
|
-
pending.session.session_id,
|
|
4520
|
-
e.source_event_id,
|
|
4521
|
-
e.event_type,
|
|
4522
|
-
e.source_type,
|
|
4523
|
-
e.subtype,
|
|
4524
|
-
e.timestamp,
|
|
4525
|
-
e.ordinal,
|
|
4526
|
-
e.actor,
|
|
4527
|
-
e.payload_object_id,
|
|
4528
|
-
e.raw_record_id,
|
|
4529
|
-
e.confidence
|
|
4530
|
-
);
|
|
4381
|
+
insertEvent.run(e.event_id, pending.session.session_id, e.source_event_id, e.event_type, e.source_type, e.subtype, e.timestamp, e.ordinal, e.actor, e.payload_object_id, e.raw_record_id, e.confidence);
|
|
4531
4382
|
}
|
|
4532
|
-
const insertMsg = prepare(
|
|
4533
|
-
bundle.db,
|
|
4534
|
-
`INSERT OR REPLACE INTO messages (
|
|
4383
|
+
const insertMsg = prepare(bundle.db, `INSERT OR REPLACE INTO messages (
|
|
4535
4384
|
message_id, session_id, turn_id, event_id, source_message_id, role,
|
|
4536
4385
|
author_name, model, timestamp, ordinal, parent_message_id, request_id,
|
|
4537
4386
|
status, raw_record_id
|
|
4538
|
-
) VALUES (?, ?, NULL, ?, ?, ?, NULL, ?, ?, ?, NULL, NULL, NULL, ?)`
|
|
4539
|
-
);
|
|
4387
|
+
) VALUES (?, ?, NULL, ?, ?, ?, NULL, ?, ?, ?, NULL, NULL, NULL, ?)`);
|
|
4540
4388
|
for (const m of pending.messages) {
|
|
4541
|
-
insertMsg.run(
|
|
4542
|
-
m.message_id,
|
|
4543
|
-
pending.session.session_id,
|
|
4544
|
-
m.event_id,
|
|
4545
|
-
m.source_message_id,
|
|
4546
|
-
m.role,
|
|
4547
|
-
m.model,
|
|
4548
|
-
m.timestamp,
|
|
4549
|
-
m.ordinal,
|
|
4550
|
-
m.raw_record_id
|
|
4551
|
-
);
|
|
4389
|
+
insertMsg.run(m.message_id, pending.session.session_id, m.event_id, m.source_message_id, m.role, m.model, m.timestamp, m.ordinal, m.raw_record_id);
|
|
4552
4390
|
}
|
|
4553
|
-
const insertBlock = prepare(
|
|
4554
|
-
bundle.db,
|
|
4555
|
-
`INSERT OR REPLACE INTO content_blocks (
|
|
4391
|
+
const insertBlock = prepare(bundle.db, `INSERT OR REPLACE INTO content_blocks (
|
|
4556
4392
|
block_id, message_id, event_id, session_id, ordinal, block_type,
|
|
4557
4393
|
text_object_id, text_inline, mime_type, token_count, is_error,
|
|
4558
4394
|
is_redacted, visibility, raw_record_id
|
|
4559
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, ?, 0, ?, ?)`
|
|
4560
|
-
);
|
|
4395
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, ?, 0, ?, ?)`);
|
|
4561
4396
|
for (const b of pending.blocks) {
|
|
4562
|
-
insertBlock.run(
|
|
4563
|
-
b.block_id,
|
|
4564
|
-
b.message_id,
|
|
4565
|
-
b.event_id,
|
|
4566
|
-
pending.session.session_id,
|
|
4567
|
-
b.ordinal,
|
|
4568
|
-
b.block_type,
|
|
4569
|
-
b.text_object_id,
|
|
4570
|
-
b.text_inline,
|
|
4571
|
-
b.is_error,
|
|
4572
|
-
b.visibility,
|
|
4573
|
-
b.raw_record_id
|
|
4574
|
-
);
|
|
4397
|
+
insertBlock.run(b.block_id, b.message_id, b.event_id, pending.session.session_id, b.ordinal, b.block_type, b.text_object_id, b.text_inline, b.is_error, b.visibility, b.raw_record_id);
|
|
4575
4398
|
}
|
|
4576
|
-
const insertCall = prepare(
|
|
4577
|
-
bundle.db,
|
|
4578
|
-
`INSERT OR REPLACE INTO tool_calls (
|
|
4399
|
+
const insertCall = prepare(bundle.db, `INSERT OR REPLACE INTO tool_calls (
|
|
4579
4400
|
tool_call_id, session_id, turn_id, message_id, event_id,
|
|
4580
4401
|
source_call_id, tool_name, canonical_tool_type, args_object_id,
|
|
4581
4402
|
command, cwd, path, query, timestamp_start, timestamp_end, status,
|
|
4582
4403
|
raw_record_id
|
|
4583
|
-
) VALUES (?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, ?)`
|
|
4584
|
-
);
|
|
4404
|
+
) VALUES (?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, ?)`);
|
|
4585
4405
|
for (const c of pending.toolCallsList) {
|
|
4586
|
-
insertCall.run(
|
|
4587
|
-
c.tool_call_id,
|
|
4588
|
-
pending.session.session_id,
|
|
4589
|
-
c.message_id,
|
|
4590
|
-
c.event_id,
|
|
4591
|
-
c.source_call_id,
|
|
4592
|
-
c.tool_name,
|
|
4593
|
-
c.canonical_tool_type,
|
|
4594
|
-
c.args_object_id,
|
|
4595
|
-
c.command,
|
|
4596
|
-
c.cwd,
|
|
4597
|
-
c.path,
|
|
4598
|
-
c.query,
|
|
4599
|
-
c.timestamp_start,
|
|
4600
|
-
c.status,
|
|
4601
|
-
c.raw_record_id
|
|
4602
|
-
);
|
|
4406
|
+
insertCall.run(c.tool_call_id, pending.session.session_id, c.message_id, c.event_id, c.source_call_id, c.tool_name, c.canonical_tool_type, c.args_object_id, c.command, c.cwd, c.path, c.query, c.timestamp_start, c.status, c.raw_record_id);
|
|
4603
4407
|
}
|
|
4604
|
-
const insertResult = prepare(
|
|
4605
|
-
bundle.db,
|
|
4606
|
-
`INSERT OR REPLACE INTO tool_results (
|
|
4408
|
+
const insertResult = prepare(bundle.db, `INSERT OR REPLACE INTO tool_results (
|
|
4607
4409
|
tool_result_id, tool_call_id, session_id, message_id, event_id,
|
|
4608
4410
|
source_call_id, status, is_error, exit_code, duration_ms,
|
|
4609
4411
|
stdout_object_id, stderr_object_id, output_object_id, preview, raw_record_id
|
|
4610
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, NULL, NULL, ?, ?, ?)`
|
|
4611
|
-
);
|
|
4412
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, NULL, NULL, ?, ?, ?)`);
|
|
4612
4413
|
for (const r of pending.toolResults) {
|
|
4613
|
-
insertResult.run(
|
|
4614
|
-
r.tool_result_id,
|
|
4615
|
-
r.tool_call_id,
|
|
4616
|
-
pending.session.session_id,
|
|
4617
|
-
r.message_id,
|
|
4618
|
-
r.event_id,
|
|
4619
|
-
r.source_call_id,
|
|
4620
|
-
r.status,
|
|
4621
|
-
r.is_error,
|
|
4622
|
-
r.output_object_id,
|
|
4623
|
-
r.preview,
|
|
4624
|
-
r.raw_record_id
|
|
4625
|
-
);
|
|
4414
|
+
insertResult.run(r.tool_result_id, r.tool_call_id, pending.session.session_id, r.message_id, r.event_id, r.source_call_id, r.status, r.is_error, r.output_object_id, r.preview, r.raw_record_id);
|
|
4626
4415
|
}
|
|
4627
|
-
const insertSearch = prepare(
|
|
4628
|
-
bundle.db,
|
|
4629
|
-
`INSERT OR REPLACE INTO search_docs (
|
|
4416
|
+
const insertSearch = prepare(bundle.db, `INSERT OR REPLACE INTO search_docs (
|
|
4630
4417
|
doc_id, entity_type, entity_id, session_id, project_id, timestamp,
|
|
4631
4418
|
role, tool_name, canonical_tool_type, field_kind, text
|
|
4632
|
-
) VALUES (?, ?, ?, ?, NULL, ?, ?, ?, ?, ?, ?)`
|
|
4633
|
-
);
|
|
4419
|
+
) VALUES (?, ?, ?, ?, NULL, ?, ?, ?, ?, ?, ?)`);
|
|
4634
4420
|
for (const d of pending.searchDocs) {
|
|
4635
|
-
insertSearch.run(
|
|
4636
|
-
d.doc_id,
|
|
4637
|
-
d.entity_type,
|
|
4638
|
-
d.entity_id,
|
|
4639
|
-
pending.session.session_id,
|
|
4640
|
-
d.timestamp,
|
|
4641
|
-
d.role,
|
|
4642
|
-
d.tool_name,
|
|
4643
|
-
d.canonical_tool_type,
|
|
4644
|
-
d.field_kind,
|
|
4645
|
-
d.text
|
|
4646
|
-
);
|
|
4421
|
+
insertSearch.run(d.doc_id, d.entity_type, d.entity_id, pending.session.session_id, d.timestamp, d.role, d.tool_name, d.canonical_tool_type, d.field_kind, d.text);
|
|
4647
4422
|
}
|
|
4648
4423
|
void artifactId;
|
|
4649
4424
|
}
|
|
4425
|
+
__name(flushPending3, "flushPending");
|
|
4650
4426
|
|
|
4651
4427
|
// src/importers/gemini/index.ts
|
|
4652
4428
|
import { readFile as readFile7 } from "fs/promises";
|
|
@@ -4674,6 +4450,7 @@ async function* discoverGeminiChats(root) {
|
|
|
4674
4450
|
}
|
|
4675
4451
|
}
|
|
4676
4452
|
}
|
|
4453
|
+
__name(discoverGeminiChats, "discoverGeminiChats");
|
|
4677
4454
|
async function readProjectRoot(dir) {
|
|
4678
4455
|
try {
|
|
4679
4456
|
const text = await readFile6(path11.join(dir, ".project_root"), "utf8");
|
|
@@ -4682,60 +4459,76 @@ async function readProjectRoot(dir) {
|
|
|
4682
4459
|
return null;
|
|
4683
4460
|
}
|
|
4684
4461
|
}
|
|
4462
|
+
__name(readProjectRoot, "readProjectRoot");
|
|
4685
4463
|
async function readdirSafe3(dir) {
|
|
4686
4464
|
try {
|
|
4687
|
-
return await readdir4(dir, {
|
|
4465
|
+
return await readdir4(dir, {
|
|
4466
|
+
withFileTypes: true
|
|
4467
|
+
});
|
|
4688
4468
|
} catch {
|
|
4689
4469
|
return [];
|
|
4690
4470
|
}
|
|
4691
4471
|
}
|
|
4472
|
+
__name(readdirSafe3, "readdirSafe");
|
|
4692
4473
|
|
|
4693
4474
|
// src/importers/gemini/index.ts
|
|
4694
4475
|
var PREVIEW_MAX4 = 4e3;
|
|
4695
4476
|
async function compileGemini(bundle, root, options = {}) {
|
|
4696
4477
|
const logger = options.logger;
|
|
4697
|
-
const batch = startBatch(bundle, "gemini", [
|
|
4478
|
+
const batch = startBatch(bundle, "gemini", [
|
|
4479
|
+
root
|
|
4480
|
+
]);
|
|
4698
4481
|
const counts = emptyCounts();
|
|
4699
|
-
logger?.info({
|
|
4482
|
+
logger?.info({
|
|
4483
|
+
batch_id: batch.batch_id,
|
|
4484
|
+
root
|
|
4485
|
+
}, "gemini batch started");
|
|
4700
4486
|
try {
|
|
4701
4487
|
for await (const file of discoverGeminiChats(root)) {
|
|
4702
4488
|
counts.source_files_seen++;
|
|
4703
|
-
logger?.debug(
|
|
4704
|
-
|
|
4705
|
-
|
|
4706
|
-
|
|
4707
|
-
|
|
4708
|
-
},
|
|
4709
|
-
"gemini source file discovered"
|
|
4710
|
-
);
|
|
4489
|
+
logger?.debug({
|
|
4490
|
+
path: file.filePath,
|
|
4491
|
+
project_dir: file.projectDir,
|
|
4492
|
+
project_root: file.projectRoot
|
|
4493
|
+
}, "gemini source file discovered");
|
|
4711
4494
|
try {
|
|
4712
4495
|
const fc = await compileGeminiFile(bundle, batch, file, logger);
|
|
4713
4496
|
addCounts4(counts, fc);
|
|
4714
4497
|
} catch (error) {
|
|
4715
4498
|
counts.errors++;
|
|
4716
|
-
logger?.warn(
|
|
4717
|
-
|
|
4718
|
-
|
|
4719
|
-
|
|
4720
|
-
},
|
|
4721
|
-
"gemini source file failed"
|
|
4722
|
-
);
|
|
4499
|
+
logger?.warn({
|
|
4500
|
+
err: error,
|
|
4501
|
+
path: file.filePath
|
|
4502
|
+
}, "gemini source file failed");
|
|
4723
4503
|
await recordError(bundle, batch.batch_id, {
|
|
4724
4504
|
kind: "gemini_file_failed",
|
|
4725
4505
|
message: getErrorMessage(error),
|
|
4726
|
-
payload: {
|
|
4506
|
+
payload: {
|
|
4507
|
+
path: file.filePath
|
|
4508
|
+
}
|
|
4727
4509
|
});
|
|
4728
4510
|
}
|
|
4729
4511
|
}
|
|
4730
4512
|
finishBatch(bundle, batch, counts, "completed");
|
|
4731
|
-
logger?.info({
|
|
4513
|
+
logger?.info({
|
|
4514
|
+
batch_id: batch.batch_id,
|
|
4515
|
+
counts
|
|
4516
|
+
}, "gemini batch completed");
|
|
4732
4517
|
} catch (error) {
|
|
4733
4518
|
finishBatch(bundle, batch, counts, "failed");
|
|
4734
|
-
logger?.error({
|
|
4519
|
+
logger?.error({
|
|
4520
|
+
err: error,
|
|
4521
|
+
batch_id: batch.batch_id,
|
|
4522
|
+
counts
|
|
4523
|
+
}, "gemini batch failed");
|
|
4735
4524
|
throw error;
|
|
4736
4525
|
}
|
|
4737
|
-
return {
|
|
4526
|
+
return {
|
|
4527
|
+
batch,
|
|
4528
|
+
counts
|
|
4529
|
+
};
|
|
4738
4530
|
}
|
|
4531
|
+
__name(compileGemini, "compileGemini");
|
|
4739
4532
|
function emptyFileCounts4() {
|
|
4740
4533
|
return {
|
|
4741
4534
|
source_files_imported: 0,
|
|
@@ -4752,6 +4545,7 @@ function emptyFileCounts4() {
|
|
|
4752
4545
|
errors: 0
|
|
4753
4546
|
};
|
|
4754
4547
|
}
|
|
4548
|
+
__name(emptyFileCounts4, "emptyFileCounts");
|
|
4755
4549
|
function addCounts4(target, source) {
|
|
4756
4550
|
target.source_files_imported += source.source_files_imported;
|
|
4757
4551
|
target.source_files_skipped += source.source_files_skipped;
|
|
@@ -4766,6 +4560,7 @@ function addCounts4(target, source) {
|
|
|
4766
4560
|
target.edges += source.edges;
|
|
4767
4561
|
target.errors += source.errors;
|
|
4768
4562
|
}
|
|
4563
|
+
__name(addCounts4, "addCounts");
|
|
4769
4564
|
async function compileGeminiFile(bundle, batch, file, logger) {
|
|
4770
4565
|
const counts = emptyFileCounts4();
|
|
4771
4566
|
const { row: sourceFile, alreadyKnown } = await registerSourceFile(bundle, {
|
|
@@ -4776,17 +4571,17 @@ async function compileGeminiFile(bundle, batch, file, logger) {
|
|
|
4776
4571
|
});
|
|
4777
4572
|
if (alreadyKnown) {
|
|
4778
4573
|
counts.source_files_skipped = 1;
|
|
4779
|
-
logger?.debug(
|
|
4780
|
-
|
|
4781
|
-
|
|
4782
|
-
);
|
|
4574
|
+
logger?.debug({
|
|
4575
|
+
path: file.filePath,
|
|
4576
|
+
source_file_id: sourceFile.source_file_id
|
|
4577
|
+
}, "gemini source file skipped");
|
|
4783
4578
|
return counts;
|
|
4784
4579
|
}
|
|
4785
4580
|
counts.source_files_imported = 1;
|
|
4786
|
-
logger?.debug(
|
|
4787
|
-
|
|
4788
|
-
|
|
4789
|
-
);
|
|
4581
|
+
logger?.debug({
|
|
4582
|
+
path: file.filePath,
|
|
4583
|
+
source_file_id: sourceFile.source_file_id
|
|
4584
|
+
}, "gemini source file registered");
|
|
4790
4585
|
const text = await readFile7(file.filePath, "utf8");
|
|
4791
4586
|
const parsed = JSON.parse(text);
|
|
4792
4587
|
const objects = createPendingObjects();
|
|
@@ -4847,15 +4642,7 @@ async function compileGeminiFile(bundle, batch, file, logger) {
|
|
|
4847
4642
|
for (let i = 0; i < messages.length; i++) {
|
|
4848
4643
|
const msg = messages[i];
|
|
4849
4644
|
if (!msg) continue;
|
|
4850
|
-
await processMessage(
|
|
4851
|
-
bundle,
|
|
4852
|
-
sessionPk,
|
|
4853
|
-
sourceFile.source_file_id,
|
|
4854
|
-
i,
|
|
4855
|
-
msg,
|
|
4856
|
-
batch.batch_id,
|
|
4857
|
-
pending
|
|
4858
|
-
);
|
|
4645
|
+
await processMessage(bundle, sessionPk, sourceFile.source_file_id, i, msg, batch.batch_id, pending);
|
|
4859
4646
|
}
|
|
4860
4647
|
buildSearchDocs4(pending);
|
|
4861
4648
|
await flushPendingObjects(bundle, pending.objects);
|
|
@@ -4870,12 +4657,14 @@ async function compileGeminiFile(bundle, batch, file, logger) {
|
|
|
4870
4657
|
counts.tool_calls = pending.toolCallsList.length;
|
|
4871
4658
|
counts.tool_results = pending.toolResults.length;
|
|
4872
4659
|
counts.artifacts = pending.artifacts.length;
|
|
4873
|
-
logger?.debug(
|
|
4874
|
-
|
|
4875
|
-
|
|
4876
|
-
|
|
4660
|
+
logger?.debug({
|
|
4661
|
+
path: file.filePath,
|
|
4662
|
+
source_file_id: sourceFile.source_file_id,
|
|
4663
|
+
counts
|
|
4664
|
+
}, "gemini source file imported");
|
|
4877
4665
|
return counts;
|
|
4878
4666
|
}
|
|
4667
|
+
__name(compileGeminiFile, "compileGeminiFile");
|
|
4879
4668
|
async function processMessage(bundle, sessionId2, sourceFileId2, index, msg, batchId, pending) {
|
|
4880
4669
|
const ordinal = index + 1;
|
|
4881
4670
|
const ts = msg.timestamp ?? null;
|
|
@@ -4943,17 +4732,11 @@ ${JSON.stringify(msg)}`);
|
|
|
4943
4732
|
for (let i = 0; i < thoughts.length; i++) {
|
|
4944
4733
|
const th = thoughts[i];
|
|
4945
4734
|
if (!th) continue;
|
|
4946
|
-
const text = [
|
|
4947
|
-
|
|
4948
|
-
|
|
4949
|
-
|
|
4950
|
-
|
|
4951
|
-
100 + i,
|
|
4952
|
-
"thinking",
|
|
4953
|
-
text,
|
|
4954
|
-
rawRecordId2,
|
|
4955
|
-
"hidden_by_default"
|
|
4956
|
-
);
|
|
4735
|
+
const text = [
|
|
4736
|
+
th.subject,
|
|
4737
|
+
th.description
|
|
4738
|
+
].filter(Boolean).join("\n\n");
|
|
4739
|
+
await pushTextBlock2(bundle, pending, messageId2, 100 + i, "thinking", text, rawRecordId2, "hidden_by_default");
|
|
4957
4740
|
}
|
|
4958
4741
|
const toolCalls = Array.isArray(msg.toolCalls) ? msg.toolCalls : [];
|
|
4959
4742
|
for (let i = 0; i < toolCalls.length; i++) {
|
|
@@ -4994,6 +4777,7 @@ ${JSON.stringify(msg)}`);
|
|
|
4994
4777
|
confidence: "high"
|
|
4995
4778
|
});
|
|
4996
4779
|
}
|
|
4780
|
+
__name(processMessage, "processMessage");
|
|
4997
4781
|
async function pushTextBlock2(bundle, pending, messageId2, blockOrdinal, blockType, text, rawRecordId2, visibility = "default") {
|
|
4998
4782
|
if (!text) return;
|
|
4999
4783
|
const overflowId = text.length > PREVIEW_MAX4 ? stageText(pending.objects, text) : null;
|
|
@@ -5009,6 +4793,7 @@ async function pushTextBlock2(bundle, pending, messageId2, blockOrdinal, blockTy
|
|
|
5009
4793
|
raw_record_id: rawRecordId2
|
|
5010
4794
|
});
|
|
5011
4795
|
}
|
|
4796
|
+
__name(pushTextBlock2, "pushTextBlock");
|
|
5012
4797
|
async function processToolCall(bundle, sessionId2, messageId2, eventId2, index, tc, rawRecordId2, pending) {
|
|
5013
4798
|
const sourceCallId = tc.id ?? `${messageId2}:${index}`;
|
|
5014
4799
|
const toolName = tc.name ?? "unknown";
|
|
@@ -5049,7 +4834,9 @@ async function processToolCall(bundle, sessionId2, messageId2, eventId2, index,
|
|
|
5049
4834
|
const rd = tc.resultDisplay;
|
|
5050
4835
|
if (rd.fileDiff || rd.filePath) {
|
|
5051
4836
|
const diffText = rd.fileDiff ?? "";
|
|
5052
|
-
const diffId = diffText ? stageText(pending.objects, diffText, {
|
|
4837
|
+
const diffId = diffText ? stageText(pending.objects, diffText, {
|
|
4838
|
+
mimeType: "text/x-diff"
|
|
4839
|
+
}) : null;
|
|
5053
4840
|
pending.artifacts.push({
|
|
5054
4841
|
artifact_id: artifactId(sessionId2, "gemini", `${toolCallId2}:diff`),
|
|
5055
4842
|
kind: "diff",
|
|
@@ -5065,6 +4852,7 @@ async function processToolCall(bundle, sessionId2, messageId2, eventId2, index,
|
|
|
5065
4852
|
}
|
|
5066
4853
|
}
|
|
5067
4854
|
}
|
|
4855
|
+
__name(processToolCall, "processToolCall");
|
|
5068
4856
|
function renderToolResultText(result) {
|
|
5069
4857
|
if (!Array.isArray(result)) return "";
|
|
5070
4858
|
const parts = [];
|
|
@@ -5075,14 +4863,13 @@ function renderToolResultText(result) {
|
|
|
5075
4863
|
}
|
|
5076
4864
|
if (r.functionResponse?.response) {
|
|
5077
4865
|
const rr = r.functionResponse.response;
|
|
5078
|
-
if (rr.error != null)
|
|
5079
|
-
|
|
5080
|
-
else if (rr.output != null)
|
|
5081
|
-
parts.push(typeof rr.output === "string" ? rr.output : JSON.stringify(rr.output));
|
|
4866
|
+
if (rr.error != null) parts.push(typeof rr.error === "string" ? rr.error : JSON.stringify(rr.error));
|
|
4867
|
+
else if (rr.output != null) parts.push(typeof rr.output === "string" ? rr.output : JSON.stringify(rr.output));
|
|
5082
4868
|
}
|
|
5083
4869
|
}
|
|
5084
4870
|
return parts.join("\n");
|
|
5085
4871
|
}
|
|
4872
|
+
__name(renderToolResultText, "renderToolResultText");
|
|
5086
4873
|
function canonicalToolType4(toolName) {
|
|
5087
4874
|
switch (toolName) {
|
|
5088
4875
|
case "run_shell_command":
|
|
@@ -5110,6 +4897,7 @@ function canonicalToolType4(toolName) {
|
|
|
5110
4897
|
return toolName.startsWith("mcp__") ? "mcp" : "other";
|
|
5111
4898
|
}
|
|
5112
4899
|
}
|
|
4900
|
+
__name(canonicalToolType4, "canonicalToolType");
|
|
5113
4901
|
function buildSearchDocs4(pending) {
|
|
5114
4902
|
const sessionId2 = pending.session?.session_id ?? null;
|
|
5115
4903
|
if (!sessionId2) return;
|
|
@@ -5179,230 +4967,87 @@ function buildSearchDocs4(pending) {
|
|
|
5179
4967
|
});
|
|
5180
4968
|
}
|
|
5181
4969
|
}
|
|
4970
|
+
__name(buildSearchDocs4, "buildSearchDocs");
|
|
5182
4971
|
function flushPending4(bundle, pending) {
|
|
5183
4972
|
if (!pending.session) return;
|
|
5184
|
-
const insertRaw = prepare(
|
|
5185
|
-
bundle.db,
|
|
5186
|
-
`INSERT OR IGNORE INTO raw_records (
|
|
4973
|
+
const insertRaw = prepare(bundle.db, `INSERT OR IGNORE INTO raw_records (
|
|
5187
4974
|
raw_record_id, source_file_id, source_tool, record_kind, ordinal,
|
|
5188
4975
|
line_no, json_pointer, native_id, raw_object_id, decoded_json_object_id,
|
|
5189
4976
|
parser_status, confidence, import_batch_id
|
|
5190
|
-
) VALUES (?, ?, 'gemini', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
5191
|
-
);
|
|
4977
|
+
) VALUES (?, ?, 'gemini', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
5192
4978
|
for (const r of pending.rawRecords) {
|
|
5193
|
-
insertRaw.run(
|
|
5194
|
-
r.raw_record_id,
|
|
5195
|
-
r.source_file_id,
|
|
5196
|
-
r.record_kind,
|
|
5197
|
-
r.ordinal,
|
|
5198
|
-
r.line_no,
|
|
5199
|
-
r.json_pointer,
|
|
5200
|
-
r.native_id,
|
|
5201
|
-
r.raw_object_id,
|
|
5202
|
-
r.decoded_json_object_id,
|
|
5203
|
-
r.parser_status,
|
|
5204
|
-
r.confidence,
|
|
5205
|
-
r.import_batch_id
|
|
5206
|
-
);
|
|
4979
|
+
insertRaw.run(r.raw_record_id, r.source_file_id, r.record_kind, r.ordinal, r.line_no, r.json_pointer, r.native_id, r.raw_object_id, r.decoded_json_object_id, r.parser_status, r.confidence, r.import_batch_id);
|
|
5207
4980
|
}
|
|
5208
4981
|
if (pending.project) {
|
|
5209
|
-
prepare(
|
|
5210
|
-
bundle.db,
|
|
5211
|
-
`INSERT OR IGNORE INTO projects (
|
|
4982
|
+
prepare(bundle.db, `INSERT OR IGNORE INTO projects (
|
|
5212
4983
|
project_id, canonical_path, path_hash, source_tool, source_project_id,
|
|
5213
4984
|
display_name, created_at
|
|
5214
|
-
) VALUES (?, ?, ?, 'gemini', ?, NULL, ?)`
|
|
5215
|
-
).run(
|
|
5216
|
-
pending.project.project_id,
|
|
5217
|
-
pending.project.canonical_path,
|
|
5218
|
-
pending.project.canonical_path ? sha256Hex(pending.project.canonical_path).slice(0, 32) : null,
|
|
5219
|
-
pending.project.source_project_id,
|
|
5220
|
-
(/* @__PURE__ */ new Date()).toISOString()
|
|
5221
|
-
);
|
|
4985
|
+
) VALUES (?, ?, ?, 'gemini', ?, NULL, ?)`).run(pending.project.project_id, pending.project.canonical_path, pending.project.canonical_path ? sha256Hex(pending.project.canonical_path).slice(0, 32) : null, pending.project.source_project_id, (/* @__PURE__ */ new Date()).toISOString());
|
|
5222
4986
|
}
|
|
5223
|
-
prepare(
|
|
5224
|
-
bundle.db,
|
|
5225
|
-
`INSERT OR REPLACE INTO sessions (
|
|
4987
|
+
prepare(bundle.db, `INSERT OR REPLACE INTO sessions (
|
|
5226
4988
|
session_id, source_tool, source_session_id, project_id, parent_session_id,
|
|
5227
4989
|
is_subagent, agent_role, agent_nickname, title, summary,
|
|
5228
4990
|
start_ts, end_ts, cwd_initial, git_branch_initial,
|
|
5229
4991
|
model_first, model_last, status, timeline_confidence, raw_record_id
|
|
5230
|
-
) VALUES (?, 'gemini', ?, ?, NULL, 0, NULL, NULL, ?, NULL, ?, ?, ?, NULL, NULL, NULL, 'completed', 'high', ?)`
|
|
5231
|
-
|
|
5232
|
-
pending.session.session_id,
|
|
5233
|
-
pending.session.source_session_id,
|
|
5234
|
-
pending.project?.project_id ?? null,
|
|
5235
|
-
pending.session.title,
|
|
5236
|
-
pending.session.start_ts,
|
|
5237
|
-
pending.session.end_ts,
|
|
5238
|
-
pending.session.cwd_initial,
|
|
5239
|
-
pending.session.raw_record_id
|
|
5240
|
-
);
|
|
5241
|
-
const insertEvent = prepare(
|
|
5242
|
-
bundle.db,
|
|
5243
|
-
`INSERT OR REPLACE INTO events (
|
|
4992
|
+
) VALUES (?, 'gemini', ?, ?, NULL, 0, NULL, NULL, ?, NULL, ?, ?, ?, NULL, NULL, NULL, 'completed', 'high', ?)`).run(pending.session.session_id, pending.session.source_session_id, pending.project?.project_id ?? null, pending.session.title, pending.session.start_ts, pending.session.end_ts, pending.session.cwd_initial, pending.session.raw_record_id);
|
|
4993
|
+
const insertEvent = prepare(bundle.db, `INSERT OR REPLACE INTO events (
|
|
5244
4994
|
event_id, session_id, turn_id, source_event_id, event_type, source_type,
|
|
5245
4995
|
subtype, timestamp, ordinal, actor, payload_object_id, raw_record_id,
|
|
5246
4996
|
confidence, is_derived
|
|
5247
|
-
) VALUES (?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0)`
|
|
5248
|
-
);
|
|
4997
|
+
) VALUES (?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0)`);
|
|
5249
4998
|
for (const e of pending.events) {
|
|
5250
|
-
insertEvent.run(
|
|
5251
|
-
e.event_id,
|
|
5252
|
-
pending.session.session_id,
|
|
5253
|
-
e.source_event_id,
|
|
5254
|
-
e.event_type,
|
|
5255
|
-
e.source_type,
|
|
5256
|
-
e.subtype,
|
|
5257
|
-
e.timestamp,
|
|
5258
|
-
e.ordinal,
|
|
5259
|
-
e.actor,
|
|
5260
|
-
e.payload_object_id,
|
|
5261
|
-
e.raw_record_id,
|
|
5262
|
-
e.confidence
|
|
5263
|
-
);
|
|
4999
|
+
insertEvent.run(e.event_id, pending.session.session_id, e.source_event_id, e.event_type, e.source_type, e.subtype, e.timestamp, e.ordinal, e.actor, e.payload_object_id, e.raw_record_id, e.confidence);
|
|
5264
5000
|
}
|
|
5265
|
-
const insertMsg = prepare(
|
|
5266
|
-
bundle.db,
|
|
5267
|
-
`INSERT OR REPLACE INTO messages (
|
|
5001
|
+
const insertMsg = prepare(bundle.db, `INSERT OR REPLACE INTO messages (
|
|
5268
5002
|
message_id, session_id, turn_id, event_id, source_message_id, role,
|
|
5269
5003
|
author_name, model, timestamp, ordinal, parent_message_id, request_id,
|
|
5270
5004
|
status, raw_record_id
|
|
5271
|
-
) VALUES (?, ?, NULL, ?, ?, ?, NULL, ?, ?, ?, NULL, NULL, NULL, ?)`
|
|
5272
|
-
);
|
|
5005
|
+
) VALUES (?, ?, NULL, ?, ?, ?, NULL, ?, ?, ?, NULL, NULL, NULL, ?)`);
|
|
5273
5006
|
for (const m of pending.messages) {
|
|
5274
|
-
insertMsg.run(
|
|
5275
|
-
m.message_id,
|
|
5276
|
-
pending.session.session_id,
|
|
5277
|
-
m.event_id,
|
|
5278
|
-
m.source_message_id,
|
|
5279
|
-
m.role,
|
|
5280
|
-
m.model,
|
|
5281
|
-
m.timestamp,
|
|
5282
|
-
m.ordinal,
|
|
5283
|
-
m.raw_record_id
|
|
5284
|
-
);
|
|
5007
|
+
insertMsg.run(m.message_id, pending.session.session_id, m.event_id, m.source_message_id, m.role, m.model, m.timestamp, m.ordinal, m.raw_record_id);
|
|
5285
5008
|
}
|
|
5286
|
-
const insertBlock = prepare(
|
|
5287
|
-
bundle.db,
|
|
5288
|
-
`INSERT OR REPLACE INTO content_blocks (
|
|
5009
|
+
const insertBlock = prepare(bundle.db, `INSERT OR REPLACE INTO content_blocks (
|
|
5289
5010
|
block_id, message_id, event_id, session_id, ordinal, block_type,
|
|
5290
5011
|
text_object_id, text_inline, mime_type, token_count, is_error,
|
|
5291
5012
|
is_redacted, visibility, raw_record_id
|
|
5292
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, 0, 0, ?, ?)`
|
|
5293
|
-
);
|
|
5013
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, 0, 0, ?, ?)`);
|
|
5294
5014
|
for (const b of pending.blocks) {
|
|
5295
|
-
insertBlock.run(
|
|
5296
|
-
b.block_id,
|
|
5297
|
-
b.message_id,
|
|
5298
|
-
b.event_id,
|
|
5299
|
-
pending.session.session_id,
|
|
5300
|
-
b.ordinal,
|
|
5301
|
-
b.block_type,
|
|
5302
|
-
b.text_object_id,
|
|
5303
|
-
b.text_inline,
|
|
5304
|
-
b.visibility,
|
|
5305
|
-
b.raw_record_id
|
|
5306
|
-
);
|
|
5015
|
+
insertBlock.run(b.block_id, b.message_id, b.event_id, pending.session.session_id, b.ordinal, b.block_type, b.text_object_id, b.text_inline, b.visibility, b.raw_record_id);
|
|
5307
5016
|
}
|
|
5308
|
-
const insertCall = prepare(
|
|
5309
|
-
bundle.db,
|
|
5310
|
-
`INSERT OR REPLACE INTO tool_calls (
|
|
5017
|
+
const insertCall = prepare(bundle.db, `INSERT OR REPLACE INTO tool_calls (
|
|
5311
5018
|
tool_call_id, session_id, turn_id, message_id, event_id,
|
|
5312
5019
|
source_call_id, tool_name, canonical_tool_type, args_object_id,
|
|
5313
5020
|
command, cwd, path, query, timestamp_start, timestamp_end, status,
|
|
5314
5021
|
raw_record_id
|
|
5315
|
-
) VALUES (?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, ?)`
|
|
5316
|
-
);
|
|
5022
|
+
) VALUES (?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, ?)`);
|
|
5317
5023
|
for (const c of pending.toolCallsList) {
|
|
5318
|
-
insertCall.run(
|
|
5319
|
-
c.tool_call_id,
|
|
5320
|
-
pending.session.session_id,
|
|
5321
|
-
c.message_id,
|
|
5322
|
-
c.event_id,
|
|
5323
|
-
c.source_call_id,
|
|
5324
|
-
c.tool_name,
|
|
5325
|
-
c.canonical_tool_type,
|
|
5326
|
-
c.args_object_id,
|
|
5327
|
-
c.command,
|
|
5328
|
-
c.cwd,
|
|
5329
|
-
c.path,
|
|
5330
|
-
c.query,
|
|
5331
|
-
c.timestamp_start,
|
|
5332
|
-
c.status,
|
|
5333
|
-
c.raw_record_id
|
|
5334
|
-
);
|
|
5024
|
+
insertCall.run(c.tool_call_id, pending.session.session_id, c.message_id, c.event_id, c.source_call_id, c.tool_name, c.canonical_tool_type, c.args_object_id, c.command, c.cwd, c.path, c.query, c.timestamp_start, c.status, c.raw_record_id);
|
|
5335
5025
|
}
|
|
5336
|
-
const insertResult = prepare(
|
|
5337
|
-
bundle.db,
|
|
5338
|
-
`INSERT OR REPLACE INTO tool_results (
|
|
5026
|
+
const insertResult = prepare(bundle.db, `INSERT OR REPLACE INTO tool_results (
|
|
5339
5027
|
tool_result_id, tool_call_id, session_id, message_id, event_id,
|
|
5340
5028
|
source_call_id, status, is_error, exit_code, duration_ms,
|
|
5341
5029
|
stdout_object_id, stderr_object_id, output_object_id, preview, raw_record_id
|
|
5342
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, NULL, NULL, ?, ?, ?)`
|
|
5343
|
-
);
|
|
5030
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, NULL, NULL, ?, ?, ?)`);
|
|
5344
5031
|
for (const r of pending.toolResults) {
|
|
5345
|
-
insertResult.run(
|
|
5346
|
-
r.tool_result_id,
|
|
5347
|
-
r.tool_call_id,
|
|
5348
|
-
pending.session.session_id,
|
|
5349
|
-
r.message_id,
|
|
5350
|
-
r.event_id,
|
|
5351
|
-
r.source_call_id,
|
|
5352
|
-
r.status,
|
|
5353
|
-
r.is_error,
|
|
5354
|
-
r.output_object_id,
|
|
5355
|
-
r.preview,
|
|
5356
|
-
r.raw_record_id
|
|
5357
|
-
);
|
|
5032
|
+
insertResult.run(r.tool_result_id, r.tool_call_id, pending.session.session_id, r.message_id, r.event_id, r.source_call_id, r.status, r.is_error, r.output_object_id, r.preview, r.raw_record_id);
|
|
5358
5033
|
}
|
|
5359
|
-
const insertArtifact = prepare(
|
|
5360
|
-
bundle.db,
|
|
5361
|
-
`INSERT OR REPLACE INTO artifacts (
|
|
5034
|
+
const insertArtifact = prepare(bundle.db, `INSERT OR REPLACE INTO artifacts (
|
|
5362
5035
|
artifact_id, session_id, project_id, source_tool, kind, path,
|
|
5363
5036
|
logical_path, object_id, text_object_id, mime_type, size_bytes,
|
|
5364
5037
|
created_ts, raw_record_id
|
|
5365
|
-
) VALUES (?, ?, ?, 'gemini', ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
5366
|
-
);
|
|
5038
|
+
) VALUES (?, ?, ?, 'gemini', ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
5367
5039
|
for (const a of pending.artifacts) {
|
|
5368
|
-
insertArtifact.run(
|
|
5369
|
-
a.artifact_id,
|
|
5370
|
-
pending.session.session_id,
|
|
5371
|
-
pending.project?.project_id ?? null,
|
|
5372
|
-
a.kind,
|
|
5373
|
-
a.path,
|
|
5374
|
-
a.logical_path,
|
|
5375
|
-
a.object_id,
|
|
5376
|
-
a.text_object_id,
|
|
5377
|
-
a.mime_type,
|
|
5378
|
-
a.size_bytes,
|
|
5379
|
-
a.created_ts,
|
|
5380
|
-
a.raw_record_id
|
|
5381
|
-
);
|
|
5040
|
+
insertArtifact.run(a.artifact_id, pending.session.session_id, pending.project?.project_id ?? null, a.kind, a.path, a.logical_path, a.object_id, a.text_object_id, a.mime_type, a.size_bytes, a.created_ts, a.raw_record_id);
|
|
5382
5041
|
}
|
|
5383
|
-
const insertSearch = prepare(
|
|
5384
|
-
bundle.db,
|
|
5385
|
-
`INSERT OR REPLACE INTO search_docs (
|
|
5042
|
+
const insertSearch = prepare(bundle.db, `INSERT OR REPLACE INTO search_docs (
|
|
5386
5043
|
doc_id, entity_type, entity_id, session_id, project_id, timestamp,
|
|
5387
5044
|
role, tool_name, canonical_tool_type, field_kind, text
|
|
5388
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
5389
|
-
);
|
|
5045
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
5390
5046
|
for (const d of pending.searchDocs) {
|
|
5391
|
-
insertSearch.run(
|
|
5392
|
-
d.doc_id,
|
|
5393
|
-
d.entity_type,
|
|
5394
|
-
d.entity_id,
|
|
5395
|
-
pending.session.session_id,
|
|
5396
|
-
pending.project?.project_id ?? null,
|
|
5397
|
-
d.timestamp,
|
|
5398
|
-
d.role,
|
|
5399
|
-
d.tool_name,
|
|
5400
|
-
d.canonical_tool_type,
|
|
5401
|
-
d.field_kind,
|
|
5402
|
-
d.text
|
|
5403
|
-
);
|
|
5047
|
+
insertSearch.run(d.doc_id, d.entity_type, d.entity_id, pending.session.session_id, pending.project?.project_id ?? null, d.timestamp, d.role, d.tool_name, d.canonical_tool_type, d.field_kind, d.text);
|
|
5404
5048
|
}
|
|
5405
5049
|
}
|
|
5050
|
+
__name(flushPending4, "flushPending");
|
|
5406
5051
|
|
|
5407
5052
|
// src/services/export/parquet.ts
|
|
5408
5053
|
import { mkdir as mkdir4, rm as rm2, writeFile as writeFile5 } from "fs/promises";
|
|
@@ -5437,21 +5082,27 @@ var ANALYTICS_VIEWS = [
|
|
|
5437
5082
|
async function exportBundleParquet(options) {
|
|
5438
5083
|
const snapshot = await openBundleSnapshot(options.bundlePath);
|
|
5439
5084
|
const outDir = path13.resolve(options.outDir ?? snapshot.defaultOutDir);
|
|
5440
|
-
await mkdir4(outDir, {
|
|
5441
|
-
|
|
5442
|
-
|
|
5443
|
-
)
|
|
5085
|
+
await mkdir4(outDir, {
|
|
5086
|
+
recursive: true
|
|
5087
|
+
});
|
|
5088
|
+
const files = Object.fromEntries(PARQUET_TABLES.map((table) => [
|
|
5089
|
+
table,
|
|
5090
|
+
path13.join(outDir, `${table}.parquet`)
|
|
5091
|
+
]));
|
|
5444
5092
|
const manifestPath = path13.join(outDir, "manifest.json");
|
|
5445
|
-
for (const file of [
|
|
5446
|
-
|
|
5093
|
+
for (const file of [
|
|
5094
|
+
...Object.values(files),
|
|
5095
|
+
manifestPath
|
|
5096
|
+
]) {
|
|
5097
|
+
await rm2(file, {
|
|
5098
|
+
force: true
|
|
5099
|
+
});
|
|
5447
5100
|
}
|
|
5448
5101
|
const connection = await createDuckDbConnection();
|
|
5449
5102
|
try {
|
|
5450
5103
|
await attachSqlite(connection, snapshot.dbPath);
|
|
5451
5104
|
for (const table of PARQUET_TABLES) {
|
|
5452
|
-
await connection.run(
|
|
5453
|
-
`COPY (SELECT * FROM prosa.${quoteIdentifier(table)}) TO ${sqlString(files[table])} (FORMAT parquet, COMPRESSION zstd, COMPRESSION_LEVEL 1, ROW_GROUP_SIZE 100000)`
|
|
5454
|
-
);
|
|
5105
|
+
await connection.run(`COPY (SELECT * FROM prosa.${quoteIdentifier(table)}) TO ${sqlString(files[table])} (FORMAT parquet, COMPRESSION zstd, COMPRESSION_LEVEL 1, ROW_GROUP_SIZE 100000)`);
|
|
5455
5106
|
}
|
|
5456
5107
|
} finally {
|
|
5457
5108
|
connection.closeSync();
|
|
@@ -5461,30 +5112,30 @@ async function exportBundleParquet(options) {
|
|
|
5461
5112
|
source_db: snapshot.dbPath,
|
|
5462
5113
|
schema_version: snapshot.schemaVersion,
|
|
5463
5114
|
parser_version: snapshot.parserVersion,
|
|
5464
|
-
tables: Object.fromEntries(
|
|
5465
|
-
|
|
5466
|
-
|
|
5467
|
-
|
|
5468
|
-
|
|
5469
|
-
|
|
5470
|
-
|
|
5471
|
-
])
|
|
5472
|
-
)
|
|
5115
|
+
tables: Object.fromEntries(PARQUET_TABLES.map((table) => [
|
|
5116
|
+
table,
|
|
5117
|
+
{
|
|
5118
|
+
file: path13.basename(files[table]),
|
|
5119
|
+
rows: snapshot.counts[table]
|
|
5120
|
+
}
|
|
5121
|
+
]))
|
|
5473
5122
|
};
|
|
5474
5123
|
await writeFile5(manifestPath, `${JSON.stringify(manifest, null, 2)}
|
|
5475
5124
|
`, "utf8");
|
|
5476
|
-
return {
|
|
5125
|
+
return {
|
|
5126
|
+
outDir,
|
|
5127
|
+
manifestPath,
|
|
5128
|
+
files,
|
|
5129
|
+
counts: snapshot.counts
|
|
5130
|
+
};
|
|
5477
5131
|
}
|
|
5132
|
+
__name(exportBundleParquet, "exportBundleParquet");
|
|
5478
5133
|
async function queryDuckDbParquet(options) {
|
|
5479
5134
|
const parquetDir = path13.resolve(options.parquetDir);
|
|
5480
5135
|
const connection = await createDuckDbConnection();
|
|
5481
5136
|
try {
|
|
5482
5137
|
for (const table of PARQUET_TABLES) {
|
|
5483
|
-
await connection.run(
|
|
5484
|
-
`CREATE OR REPLACE VIEW ${quoteIdentifier(table)} AS SELECT * FROM read_parquet(${sqlString(
|
|
5485
|
-
path13.join(parquetDir, `${table}.parquet`)
|
|
5486
|
-
)})`
|
|
5487
|
-
);
|
|
5138
|
+
await connection.run(`CREATE OR REPLACE VIEW ${quoteIdentifier(table)} AS SELECT * FROM read_parquet(${sqlString(path13.join(parquetDir, `${table}.parquet`))})`);
|
|
5488
5139
|
}
|
|
5489
5140
|
await createAnalyticsViews(connection);
|
|
5490
5141
|
const reader = await connection.runAndReadAll(options.sql);
|
|
@@ -5494,29 +5145,28 @@ async function queryDuckDbParquet(options) {
|
|
|
5494
5145
|
};
|
|
5495
5146
|
} catch (error) {
|
|
5496
5147
|
if (isMissingParquetError(error)) {
|
|
5497
|
-
throw new Error(
|
|
5498
|
-
`Parquet export not found in ${parquetDir}; run \`prosa export parquet --store <path>\` first`
|
|
5499
|
-
);
|
|
5148
|
+
throw new Error(`Parquet export not found in ${parquetDir}; run \`prosa export parquet --store <path>\` first`);
|
|
5500
5149
|
}
|
|
5501
5150
|
throw error;
|
|
5502
5151
|
} finally {
|
|
5503
5152
|
connection.closeSync();
|
|
5504
5153
|
}
|
|
5505
5154
|
}
|
|
5155
|
+
__name(queryDuckDbParquet, "queryDuckDbParquet");
|
|
5506
5156
|
async function createDuckDbConnection() {
|
|
5507
5157
|
return DuckDBConnection.create();
|
|
5508
5158
|
}
|
|
5159
|
+
__name(createDuckDbConnection, "createDuckDbConnection");
|
|
5509
5160
|
async function attachSqlite(connection, dbPath) {
|
|
5510
5161
|
try {
|
|
5511
5162
|
await connection.run("INSTALL sqlite");
|
|
5512
5163
|
await connection.run("LOAD sqlite");
|
|
5513
5164
|
await connection.run(`ATTACH ${sqlString(dbPath)} AS prosa (TYPE sqlite)`);
|
|
5514
5165
|
} catch (error) {
|
|
5515
|
-
throw new Error(
|
|
5516
|
-
`DuckDB could not attach prosa.sqlite via the sqlite extension: ${getErrorMessage(error)}`
|
|
5517
|
-
);
|
|
5166
|
+
throw new Error(`DuckDB could not attach prosa.sqlite via the sqlite extension: ${getErrorMessage(error)}`);
|
|
5518
5167
|
}
|
|
5519
5168
|
}
|
|
5169
|
+
__name(attachSqlite, "attachSqlite");
|
|
5520
5170
|
async function createAnalyticsViews(connection) {
|
|
5521
5171
|
await connection.run(`
|
|
5522
5172
|
CREATE OR REPLACE VIEW session_facts AS
|
|
@@ -5796,15 +5446,17 @@ async function createAnalyticsViews(connection) {
|
|
|
5796
5446
|
GROUP BY s.source_tool, s.project_id, p.display_name, s.cwd_initial, p.canonical_path
|
|
5797
5447
|
`);
|
|
5798
5448
|
}
|
|
5449
|
+
__name(createAnalyticsViews, "createAnalyticsViews");
|
|
5799
5450
|
async function openBundleSnapshot(bundlePath) {
|
|
5800
5451
|
const bundle = await openBundle(bundlePath);
|
|
5801
5452
|
try {
|
|
5802
|
-
const counts = Object.fromEntries(
|
|
5803
|
-
|
|
5804
|
-
|
|
5805
|
-
|
|
5806
|
-
|
|
5807
|
-
|
|
5453
|
+
const counts = Object.fromEntries(PARQUET_TABLES.map((table) => {
|
|
5454
|
+
const row = bundle.db.prepare(`SELECT count(*) AS n FROM ${quoteIdentifier(table)}`).get();
|
|
5455
|
+
return [
|
|
5456
|
+
table,
|
|
5457
|
+
row?.n ?? 0
|
|
5458
|
+
];
|
|
5459
|
+
}));
|
|
5808
5460
|
return {
|
|
5809
5461
|
dbPath: bundle.paths.db,
|
|
5810
5462
|
schemaVersion: bundle.manifest.schema_version,
|
|
@@ -5816,16 +5468,20 @@ async function openBundleSnapshot(bundlePath) {
|
|
|
5816
5468
|
closeBundle(bundle);
|
|
5817
5469
|
}
|
|
5818
5470
|
}
|
|
5471
|
+
__name(openBundleSnapshot, "openBundleSnapshot");
|
|
5819
5472
|
function quoteIdentifier(value) {
|
|
5820
5473
|
return `"${value.replace(/"/g, '""')}"`;
|
|
5821
5474
|
}
|
|
5475
|
+
__name(quoteIdentifier, "quoteIdentifier");
|
|
5822
5476
|
function sqlString(value) {
|
|
5823
5477
|
return `'${value.replace(/'/g, "''")}'`;
|
|
5824
5478
|
}
|
|
5479
|
+
__name(sqlString, "sqlString");
|
|
5825
5480
|
function isMissingParquetError(error) {
|
|
5826
5481
|
const message = getErrorMessage(error);
|
|
5827
5482
|
return /No files found|does not exist|not found/i.test(message) && /\.parquet/i.test(message);
|
|
5828
5483
|
}
|
|
5484
|
+
__name(isMissingParquetError, "isMissingParquetError");
|
|
5829
5485
|
|
|
5830
5486
|
// src/services/compile.ts
|
|
5831
5487
|
var COMPILE_PROVIDERS = [
|
|
@@ -5833,28 +5489,28 @@ var COMPILE_PROVIDERS = [
|
|
|
5833
5489
|
name: "codex",
|
|
5834
5490
|
description: "Import Codex CLI session histories into the bundle.",
|
|
5835
5491
|
pathHelp: "root of Codex CLI sessions",
|
|
5836
|
-
defaultSessionsPath: () => path14.join(os2.homedir(), ".codex", "sessions"),
|
|
5492
|
+
defaultSessionsPath: /* @__PURE__ */ __name(() => path14.join(os2.homedir(), ".codex", "sessions"), "defaultSessionsPath"),
|
|
5837
5493
|
compile: compileCodex
|
|
5838
5494
|
},
|
|
5839
5495
|
{
|
|
5840
5496
|
name: "claude",
|
|
5841
5497
|
description: "Import Claude Code project histories into the bundle.",
|
|
5842
5498
|
pathHelp: "root of Claude Code projects",
|
|
5843
|
-
defaultSessionsPath: () => path14.join(os2.homedir(), ".claude", "projects"),
|
|
5499
|
+
defaultSessionsPath: /* @__PURE__ */ __name(() => path14.join(os2.homedir(), ".claude", "projects"), "defaultSessionsPath"),
|
|
5844
5500
|
compile: compileClaude
|
|
5845
5501
|
},
|
|
5846
5502
|
{
|
|
5847
5503
|
name: "gemini",
|
|
5848
5504
|
description: "Import Gemini CLI session histories into the bundle.",
|
|
5849
5505
|
pathHelp: "root of Gemini CLI tmp dir",
|
|
5850
|
-
defaultSessionsPath: () => path14.join(os2.homedir(), ".gemini", "tmp"),
|
|
5506
|
+
defaultSessionsPath: /* @__PURE__ */ __name(() => path14.join(os2.homedir(), ".gemini", "tmp"), "defaultSessionsPath"),
|
|
5851
5507
|
compile: compileGemini
|
|
5852
5508
|
},
|
|
5853
5509
|
{
|
|
5854
5510
|
name: "cursor",
|
|
5855
5511
|
description: "Import Cursor agent stores into the bundle.",
|
|
5856
5512
|
pathHelp: "root of Cursor agent stores",
|
|
5857
|
-
defaultSessionsPath: () => path14.join(os2.homedir(), ".cursor", "chats"),
|
|
5513
|
+
defaultSessionsPath: /* @__PURE__ */ __name(() => path14.join(os2.homedir(), ".cursor", "chats"), "defaultSessionsPath"),
|
|
5858
5514
|
compile: compileCursor
|
|
5859
5515
|
}
|
|
5860
5516
|
];
|
|
@@ -5865,11 +5521,13 @@ function getCompileProvider(source) {
|
|
|
5865
5521
|
}
|
|
5866
5522
|
return provider;
|
|
5867
5523
|
}
|
|
5524
|
+
__name(getCompileProvider, "getCompileProvider");
|
|
5868
5525
|
function resolveCompilePath(p) {
|
|
5869
5526
|
if (p === "~") return os2.homedir();
|
|
5870
5527
|
if (p.startsWith("~/")) return path14.join(os2.homedir(), p.slice(2));
|
|
5871
5528
|
return path14.resolve(p);
|
|
5872
5529
|
}
|
|
5530
|
+
__name(resolveCompilePath, "resolveCompilePath");
|
|
5873
5531
|
async function runCompileImports(options) {
|
|
5874
5532
|
const { bundle, providers, logger } = options;
|
|
5875
5533
|
const overwrite = options.overwrite === true;
|
|
@@ -5879,6 +5537,13 @@ async function runCompileImports(options) {
|
|
|
5879
5537
|
let tantivyError = null;
|
|
5880
5538
|
let fts5Error = null;
|
|
5881
5539
|
try {
|
|
5540
|
+
const sweep = bundle.db.prepare(`UPDATE import_batches SET status = 'failed', finished_at = datetime('now')
|
|
5541
|
+
WHERE finished_at IS NULL`).run();
|
|
5542
|
+
if (sweep.changes > 0) {
|
|
5543
|
+
logger?.warn({
|
|
5544
|
+
batches_reaped: sweep.changes
|
|
5545
|
+
}, "reaped unfinished import_batches from a prior crash");
|
|
5546
|
+
}
|
|
5882
5547
|
logger?.info("disabling FTS5 triggers for bulk rebuild");
|
|
5883
5548
|
disableFts5Triggers(bundle);
|
|
5884
5549
|
for (const provider of providers) {
|
|
@@ -5888,15 +5553,14 @@ async function runCompileImports(options) {
|
|
|
5888
5553
|
source_path: sourcePath
|
|
5889
5554
|
});
|
|
5890
5555
|
providerLogger?.info("starting compile");
|
|
5891
|
-
const r = await provider.compile(bundle, sourcePath, {
|
|
5556
|
+
const r = await provider.compile(bundle, sourcePath, {
|
|
5557
|
+
logger: providerLogger
|
|
5558
|
+
});
|
|
5892
5559
|
importedAny ||= r.counts.source_files_imported > 0;
|
|
5893
|
-
providerLogger?.info(
|
|
5894
|
-
|
|
5895
|
-
|
|
5896
|
-
|
|
5897
|
-
},
|
|
5898
|
-
"compile finished"
|
|
5899
|
-
);
|
|
5560
|
+
providerLogger?.info({
|
|
5561
|
+
batch_id: r.batch.batch_id,
|
|
5562
|
+
counts: r.counts
|
|
5563
|
+
}, "compile finished");
|
|
5900
5564
|
const summary = {
|
|
5901
5565
|
source: provider.name,
|
|
5902
5566
|
sourcePath,
|
|
@@ -5909,26 +5573,38 @@ async function runCompileImports(options) {
|
|
|
5909
5573
|
}
|
|
5910
5574
|
const shouldRebuildIndexes = importedAny || overwrite;
|
|
5911
5575
|
if (shouldRebuildIndexes) {
|
|
5912
|
-
logger?.info(
|
|
5913
|
-
|
|
5914
|
-
|
|
5915
|
-
);
|
|
5916
|
-
markIndexesAfterImport(bundle, {
|
|
5576
|
+
logger?.info({
|
|
5577
|
+
changed: importedAny,
|
|
5578
|
+
overwrite
|
|
5579
|
+
}, importedAny ? "marking indexes" : "overwrite forces rebuild despite no new imports");
|
|
5580
|
+
markIndexesAfterImport(bundle, {
|
|
5581
|
+
changed: true
|
|
5582
|
+
});
|
|
5917
5583
|
try {
|
|
5918
5584
|
logger?.info("rebuilding fts5 index");
|
|
5919
5585
|
rebuildFts5Index(bundle);
|
|
5920
5586
|
} catch (error) {
|
|
5921
5587
|
fts5Error = getErrorMessage(error);
|
|
5922
|
-
logger?.error({
|
|
5588
|
+
logger?.error({
|
|
5589
|
+
err: error
|
|
5590
|
+
}, "fts5 rebuild failed; SQLite data is intact");
|
|
5923
5591
|
}
|
|
5924
5592
|
try {
|
|
5925
|
-
logger?.info({
|
|
5926
|
-
|
|
5927
|
-
|
|
5593
|
+
logger?.info({
|
|
5594
|
+
overwrite
|
|
5595
|
+
}, "rebuilding tantivy index");
|
|
5596
|
+
const status = await rebuildTantivyIndex(bundle, {
|
|
5597
|
+
overwrite
|
|
5598
|
+
});
|
|
5599
|
+
tantivy = {
|
|
5600
|
+
indexedDocCount: status.indexed_doc_count
|
|
5601
|
+
};
|
|
5928
5602
|
options.onTantivyComplete?.(tantivy);
|
|
5929
5603
|
} catch (error) {
|
|
5930
5604
|
tantivyError = getErrorMessage(error);
|
|
5931
|
-
logger?.error({
|
|
5605
|
+
logger?.error({
|
|
5606
|
+
err: error
|
|
5607
|
+
}, "tantivy rebuild failed; SQLite data is intact");
|
|
5932
5608
|
}
|
|
5933
5609
|
}
|
|
5934
5610
|
} finally {
|
|
@@ -5942,10 +5618,15 @@ async function runCompileImports(options) {
|
|
|
5942
5618
|
fts5Error
|
|
5943
5619
|
};
|
|
5944
5620
|
}
|
|
5621
|
+
__name(runCompileImports, "runCompileImports");
|
|
5945
5622
|
async function exportCompileParquet(options) {
|
|
5946
5623
|
const storePath = resolveCompilePath(options.storePath);
|
|
5947
|
-
options.logger?.info({
|
|
5948
|
-
|
|
5624
|
+
options.logger?.info({
|
|
5625
|
+
store_path: storePath
|
|
5626
|
+
}, "exporting parquet");
|
|
5627
|
+
const result = await exportBundleParquet({
|
|
5628
|
+
bundlePath: storePath
|
|
5629
|
+
});
|
|
5949
5630
|
return {
|
|
5950
5631
|
outDir: result.outDir,
|
|
5951
5632
|
manifestPath: result.manifestPath,
|
|
@@ -5954,22 +5635,34 @@ async function exportCompileParquet(options) {
|
|
|
5954
5635
|
counts: result.counts
|
|
5955
5636
|
};
|
|
5956
5637
|
}
|
|
5638
|
+
__name(exportCompileParquet, "exportCompileParquet");
|
|
5957
5639
|
|
|
5958
5640
|
// src/services/analytics.ts
|
|
5959
|
-
var ANALYTICS_REPORTS = [
|
|
5641
|
+
var ANALYTICS_REPORTS = [
|
|
5642
|
+
"sessions",
|
|
5643
|
+
"tools",
|
|
5644
|
+
"errors",
|
|
5645
|
+
"models",
|
|
5646
|
+
"projects"
|
|
5647
|
+
];
|
|
5960
5648
|
async function runAnalyticsReport(options) {
|
|
5961
5649
|
return queryDuckDbParquet({
|
|
5962
5650
|
parquetDir: options.parquetDir,
|
|
5963
5651
|
sql: buildAnalyticsSql(options.report, options.filters ?? {}, "duckdb")
|
|
5964
5652
|
});
|
|
5965
5653
|
}
|
|
5654
|
+
__name(runAnalyticsReport, "runAnalyticsReport");
|
|
5966
5655
|
function runAnalyticsReportFromBundle(options) {
|
|
5967
5656
|
const sql = buildAnalyticsSql(options.report, options.filters ?? {}, "sqlite");
|
|
5968
5657
|
const stmt = options.bundle.db.prepare(sql);
|
|
5969
5658
|
const rows = stmt.all();
|
|
5970
5659
|
const columns = stmt.columns().map((column) => column.name);
|
|
5971
|
-
return {
|
|
5660
|
+
return {
|
|
5661
|
+
columns,
|
|
5662
|
+
rows
|
|
5663
|
+
};
|
|
5972
5664
|
}
|
|
5665
|
+
__name(runAnalyticsReportFromBundle, "runAnalyticsReportFromBundle");
|
|
5973
5666
|
function buildAnalyticsSql(report, filters, dialect) {
|
|
5974
5667
|
switch (report) {
|
|
5975
5668
|
case "sessions":
|
|
@@ -5984,6 +5677,7 @@ function buildAnalyticsSql(report, filters, dialect) {
|
|
|
5984
5677
|
return buildProjectsSql(filters, dialect);
|
|
5985
5678
|
}
|
|
5986
5679
|
}
|
|
5680
|
+
__name(buildAnalyticsSql, "buildAnalyticsSql");
|
|
5987
5681
|
function buildSessionsSql(filters, dialect) {
|
|
5988
5682
|
const where = buildWhere([
|
|
5989
5683
|
sourceFilter(filters),
|
|
@@ -6003,6 +5697,7 @@ function buildSessionsSql(filters, dialect) {
|
|
|
6003
5697
|
LIMIT ${limit(filters)}
|
|
6004
5698
|
`;
|
|
6005
5699
|
}
|
|
5700
|
+
__name(buildSessionsSql, "buildSessionsSql");
|
|
6006
5701
|
function buildToolsSql(filters, dialect) {
|
|
6007
5702
|
const where = buildWhere([
|
|
6008
5703
|
sourceFilter(filters),
|
|
@@ -6025,6 +5720,7 @@ function buildToolsSql(filters, dialect) {
|
|
|
6025
5720
|
LIMIT ${limit(filters)}
|
|
6026
5721
|
`;
|
|
6027
5722
|
}
|
|
5723
|
+
__name(buildToolsSql, "buildToolsSql");
|
|
6028
5724
|
function buildErrorsSql(filters, dialect) {
|
|
6029
5725
|
const where = buildWhere([
|
|
6030
5726
|
sourceFilter(filters),
|
|
@@ -6042,6 +5738,7 @@ function buildErrorsSql(filters, dialect) {
|
|
|
6042
5738
|
LIMIT ${limit(filters)}
|
|
6043
5739
|
`;
|
|
6044
5740
|
}
|
|
5741
|
+
__name(buildErrorsSql, "buildErrorsSql");
|
|
6045
5742
|
function buildModelsSql(filters, dialect) {
|
|
6046
5743
|
const where = buildWhere([
|
|
6047
5744
|
sourceFilter(filters),
|
|
@@ -6058,6 +5755,7 @@ function buildModelsSql(filters, dialect) {
|
|
|
6058
5755
|
LIMIT ${limit(filters)}
|
|
6059
5756
|
`;
|
|
6060
5757
|
}
|
|
5758
|
+
__name(buildModelsSql, "buildModelsSql");
|
|
6061
5759
|
function buildProjectsSql(filters, dialect) {
|
|
6062
5760
|
const where = buildWhere([
|
|
6063
5761
|
sourceFilter(filters),
|
|
@@ -6074,17 +5772,18 @@ function buildProjectsSql(filters, dialect) {
|
|
|
6074
5772
|
LIMIT ${limit(filters)}
|
|
6075
5773
|
`;
|
|
6076
5774
|
}
|
|
5775
|
+
__name(buildProjectsSql, "buildProjectsSql");
|
|
6077
5776
|
function sourceFilter(filters) {
|
|
6078
5777
|
return filters.source ? `source_tool = ${sqlString2(filters.source)}` : null;
|
|
6079
5778
|
}
|
|
5779
|
+
__name(sourceFilter, "sourceFilter");
|
|
6080
5780
|
function timeFilter(column, filters) {
|
|
6081
5781
|
const filtersSql = [];
|
|
6082
|
-
if (filters.since)
|
|
6083
|
-
|
|
6084
|
-
if (filters.until)
|
|
6085
|
-
filtersSql.push(`(${column} IS NULL OR ${column} < ${sqlString2(filters.until)})`);
|
|
5782
|
+
if (filters.since) filtersSql.push(`(${column} IS NULL OR ${column} >= ${sqlString2(filters.since)})`);
|
|
5783
|
+
if (filters.until) filtersSql.push(`(${column} IS NULL OR ${column} < ${sqlString2(filters.until)})`);
|
|
6086
5784
|
return filtersSql.length ? filtersSql.join(" AND ") : null;
|
|
6087
5785
|
}
|
|
5786
|
+
__name(timeFilter, "timeFilter");
|
|
6088
5787
|
function rangeOverlapFilter(firstColumn, lastColumn, filters) {
|
|
6089
5788
|
const filtersSql = [];
|
|
6090
5789
|
if (filters.since) {
|
|
@@ -6095,6 +5794,7 @@ function rangeOverlapFilter(firstColumn, lastColumn, filters) {
|
|
|
6095
5794
|
}
|
|
6096
5795
|
return filtersSql.length ? filtersSql.join(" AND ") : null;
|
|
6097
5796
|
}
|
|
5797
|
+
__name(rangeOverlapFilter, "rangeOverlapFilter");
|
|
6098
5798
|
function projectFilter(filters, dialect) {
|
|
6099
5799
|
if (!filters.project) return null;
|
|
6100
5800
|
const exact = sqlString2(filters.project);
|
|
@@ -6102,20 +5802,28 @@ function projectFilter(filters, dialect) {
|
|
|
6102
5802
|
const op = dialect === "duckdb" ? "ILIKE" : "LIKE";
|
|
6103
5803
|
return `(project_id = ${exact} OR project_name ${op} ${like} ESCAPE '\\' OR project_path ${op} ${like} ESCAPE '\\')`;
|
|
6104
5804
|
}
|
|
5805
|
+
__name(projectFilter, "projectFilter");
|
|
6105
5806
|
function buildWhere(filters) {
|
|
6106
5807
|
const active = filters.filter((filter) => Boolean(filter));
|
|
6107
5808
|
return active.length ? `WHERE ${active.join(" AND ")}` : "";
|
|
6108
5809
|
}
|
|
5810
|
+
__name(buildWhere, "buildWhere");
|
|
6109
5811
|
function limit(filters) {
|
|
6110
5812
|
const value = Number.isFinite(filters.limit) ? filters.limit : void 0;
|
|
6111
|
-
return clampLimit(value, {
|
|
5813
|
+
return clampLimit(value, {
|
|
5814
|
+
max: 500,
|
|
5815
|
+
fallback: 50
|
|
5816
|
+
});
|
|
6112
5817
|
}
|
|
5818
|
+
__name(limit, "limit");
|
|
6113
5819
|
function sqlString2(value) {
|
|
6114
5820
|
return `'${value.replace(/'/g, "''")}'`;
|
|
6115
5821
|
}
|
|
5822
|
+
__name(sqlString2, "sqlString");
|
|
6116
5823
|
function escapeLike(value) {
|
|
6117
5824
|
return value.replace(/[\\%_]/g, (match) => `\\${match}`);
|
|
6118
5825
|
}
|
|
5826
|
+
__name(escapeLike, "escapeLike");
|
|
6119
5827
|
|
|
6120
5828
|
// src/services/tool_calls.ts
|
|
6121
5829
|
function listToolCalls(bundle, filters = {}) {
|
|
@@ -6150,7 +5858,10 @@ function listToolCalls(bundle, filters = {}) {
|
|
|
6150
5858
|
params.push(filters.untilIso);
|
|
6151
5859
|
}
|
|
6152
5860
|
const where = conds.length ? `WHERE ${conds.join(" AND ")}` : "";
|
|
6153
|
-
const limit2 = clampLimit(filters.limit, {
|
|
5861
|
+
const limit2 = clampLimit(filters.limit, {
|
|
5862
|
+
max: 500,
|
|
5863
|
+
fallback: 100
|
|
5864
|
+
});
|
|
6154
5865
|
const toolCallSql = `
|
|
6155
5866
|
SELECT 'tool_call' AS entity_type,
|
|
6156
5867
|
tc.session_id,
|
|
@@ -6199,33 +5910,26 @@ function listToolCalls(bundle, filters = {}) {
|
|
|
6199
5910
|
`;
|
|
6200
5911
|
return bundle.db.prepare(sql).all(...params, `%${filters.pathSubstring}%`);
|
|
6201
5912
|
}
|
|
5913
|
+
__name(listToolCalls, "listToolCalls");
|
|
6202
5914
|
|
|
6203
5915
|
// src/services/export/markdown.ts
|
|
6204
5916
|
async function exportSessionMarkdown(bundle, sessionId2) {
|
|
6205
|
-
const session = bundle.db.prepare(
|
|
6206
|
-
`SELECT session_id, source_tool, source_session_id, title, start_ts, end_ts,
|
|
5917
|
+
const session = bundle.db.prepare(`SELECT session_id, source_tool, source_session_id, title, start_ts, end_ts,
|
|
6207
5918
|
cwd_initial, git_branch_initial, model_first, model_last, timeline_confidence
|
|
6208
|
-
FROM sessions WHERE session_id = ?`
|
|
6209
|
-
).get(sessionId2);
|
|
5919
|
+
FROM sessions WHERE session_id = ?`).get(sessionId2);
|
|
6210
5920
|
if (!session) {
|
|
6211
5921
|
throw new Error(`session not found: ${sessionId2}`);
|
|
6212
5922
|
}
|
|
6213
|
-
const messages = bundle.db.prepare(
|
|
6214
|
-
|
|
6215
|
-
|
|
6216
|
-
|
|
6217
|
-
const
|
|
6218
|
-
`SELECT message_id, block_type, text_object_id, text_inline, ordinal
|
|
6219
|
-
FROM content_blocks WHERE session_id = ? ORDER BY ordinal`
|
|
6220
|
-
).all(sessionId2);
|
|
6221
|
-
const toolCalls = bundle.db.prepare(
|
|
6222
|
-
`SELECT tc.tool_call_id, tc.message_id, tc.tool_name, tc.command, tc.path,
|
|
5923
|
+
const messages = bundle.db.prepare(`SELECT message_id, role, timestamp, ordinal, model
|
|
5924
|
+
FROM messages WHERE session_id = ? ORDER BY ordinal`).all(sessionId2);
|
|
5925
|
+
const blocks = bundle.db.prepare(`SELECT message_id, block_type, text_object_id, text_inline, ordinal
|
|
5926
|
+
FROM content_blocks WHERE session_id = ? ORDER BY ordinal`).all(sessionId2);
|
|
5927
|
+
const toolCalls = bundle.db.prepare(`SELECT tc.tool_call_id, tc.message_id, tc.tool_name, tc.command, tc.path,
|
|
6223
5928
|
tc.status, tc.timestamp_start,
|
|
6224
5929
|
tr.is_error, tr.preview
|
|
6225
5930
|
FROM tool_calls tc
|
|
6226
5931
|
LEFT JOIN tool_results tr ON tr.tool_call_id = tc.tool_call_id
|
|
6227
|
-
WHERE tc.session_id = ? ORDER BY tc.timestamp_start, tc.tool_call_id`
|
|
6228
|
-
).all(sessionId2);
|
|
5932
|
+
WHERE tc.session_id = ? ORDER BY tc.timestamp_start, tc.tool_call_id`).all(sessionId2);
|
|
6229
5933
|
const blocksByMessage = /* @__PURE__ */ new Map();
|
|
6230
5934
|
for (const b of blocks) {
|
|
6231
5935
|
if (!b.message_id) continue;
|
|
@@ -6251,9 +5955,7 @@ async function exportSessionMarkdown(bundle, sessionId2) {
|
|
|
6251
5955
|
if (session.cwd_initial) lines.push(`- **cwd**: \`${session.cwd_initial}\``);
|
|
6252
5956
|
if (session.git_branch_initial) lines.push(`- **git branch**: ${session.git_branch_initial}`);
|
|
6253
5957
|
if (session.model_first || session.model_last) {
|
|
6254
|
-
lines.push(
|
|
6255
|
-
`- **models**: ${session.model_first ?? "?"} \u2192 ${session.model_last ?? session.model_first ?? "?"}`
|
|
6256
|
-
);
|
|
5958
|
+
lines.push(`- **models**: ${session.model_first ?? "?"} \u2192 ${session.model_last ?? session.model_first ?? "?"}`);
|
|
6257
5959
|
}
|
|
6258
5960
|
lines.push(`- **timeline confidence**: ${session.timeline_confidence}`);
|
|
6259
5961
|
lines.push("");
|
|
@@ -6282,6 +5984,7 @@ async function exportSessionMarkdown(bundle, sessionId2) {
|
|
|
6282
5984
|
return `${lines.join("\n")}
|
|
6283
5985
|
`;
|
|
6284
5986
|
}
|
|
5987
|
+
__name(exportSessionMarkdown, "exportSessionMarkdown");
|
|
6285
5988
|
async function renderBlockText(bundle, block) {
|
|
6286
5989
|
if (block.text_inline) return block.text_inline;
|
|
6287
5990
|
if (block.text_object_id) {
|
|
@@ -6293,6 +5996,7 @@ async function renderBlockText(bundle, block) {
|
|
|
6293
5996
|
}
|
|
6294
5997
|
return null;
|
|
6295
5998
|
}
|
|
5999
|
+
__name(renderBlockText, "renderBlockText");
|
|
6296
6000
|
function renderToolCall(c) {
|
|
6297
6001
|
const status = c.status ? ` \xB7 ${c.status}` : "";
|
|
6298
6002
|
const errFlag = c.is_error === 1 ? " \xB7 ERROR" : "";
|
|
@@ -6309,6 +6013,7 @@ function renderToolCall(c) {
|
|
|
6309
6013
|
}
|
|
6310
6014
|
return lines.join("\n");
|
|
6311
6015
|
}
|
|
6016
|
+
__name(renderToolCall, "renderToolCall");
|
|
6312
6017
|
export {
|
|
6313
6018
|
ANALYTICS_REPORTS,
|
|
6314
6019
|
ANALYTICS_VIEWS,
|