@nocoo/pika 0.2.2 → 0.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin.js +336 -268
- package/package.json +1 -1
package/dist/bin.js
CHANGED
|
@@ -1,71 +1,22 @@
|
|
|
1
1
|
#!/usr/bin/env bun
|
|
2
2
|
// @bun
|
|
3
3
|
var __defProp = Object.defineProperty;
|
|
4
|
+
var __returnValue = (v) => v;
|
|
5
|
+
function __exportSetter(name, newValue) {
|
|
6
|
+
this[name] = __returnValue.bind(null, newValue);
|
|
7
|
+
}
|
|
4
8
|
var __export = (target, all) => {
|
|
5
9
|
for (var name in all)
|
|
6
10
|
__defProp(target, name, {
|
|
7
11
|
get: all[name],
|
|
8
12
|
enumerable: true,
|
|
9
13
|
configurable: true,
|
|
10
|
-
set: (
|
|
14
|
+
set: __exportSetter.bind(all, name)
|
|
11
15
|
});
|
|
12
16
|
};
|
|
13
17
|
var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
|
|
14
18
|
var __require = import.meta.require;
|
|
15
19
|
|
|
16
|
-
// ../core/src/types.ts
|
|
17
|
-
var SOURCES;
|
|
18
|
-
var init_types = __esm(() => {
|
|
19
|
-
SOURCES = [
|
|
20
|
-
"claude-code",
|
|
21
|
-
"codex",
|
|
22
|
-
"gemini-cli",
|
|
23
|
-
"opencode",
|
|
24
|
-
"vscode-copilot"
|
|
25
|
-
];
|
|
26
|
-
});
|
|
27
|
-
|
|
28
|
-
// ../../package.json
|
|
29
|
-
var package_default;
|
|
30
|
-
var init_package = __esm(() => {
|
|
31
|
-
package_default = {
|
|
32
|
-
name: "pika",
|
|
33
|
-
version: "0.2.2",
|
|
34
|
-
private: true,
|
|
35
|
-
workspaces: [
|
|
36
|
-
"packages/*"
|
|
37
|
-
],
|
|
38
|
-
scripts: {
|
|
39
|
-
dev: "bun run --cwd packages/web dev",
|
|
40
|
-
build: "bun run --filter '*' build",
|
|
41
|
-
test: "vitest run",
|
|
42
|
-
"test:watch": "vitest",
|
|
43
|
-
"test:coverage": "vitest run --coverage",
|
|
44
|
-
lint: "tsc --noEmit && tsc --noEmit -p packages/web/tsconfig.json",
|
|
45
|
-
"lint:typecheck": "tsc --noEmit && tsc --noEmit -p packages/web/tsconfig.json",
|
|
46
|
-
"sync-versions": "bun scripts/sync-versions.ts --write",
|
|
47
|
-
prepare: "husky"
|
|
48
|
-
},
|
|
49
|
-
devDependencies: {
|
|
50
|
-
"@vitest/coverage-v8": "^3.1.1",
|
|
51
|
-
husky: "^9.1.7",
|
|
52
|
-
typescript: "^5.8.2",
|
|
53
|
-
vitest: "^3.1.1"
|
|
54
|
-
},
|
|
55
|
-
overrides: {
|
|
56
|
-
undici: ">=7.24.0",
|
|
57
|
-
cookie: ">=0.7.0"
|
|
58
|
-
}
|
|
59
|
-
};
|
|
60
|
-
});
|
|
61
|
-
|
|
62
|
-
// ../core/src/version.ts
|
|
63
|
-
var PIKA_VERSION;
|
|
64
|
-
var init_version = __esm(() => {
|
|
65
|
-
init_package();
|
|
66
|
-
PIKA_VERSION = package_default.version;
|
|
67
|
-
});
|
|
68
|
-
|
|
69
20
|
// ../core/src/constants.ts
|
|
70
21
|
var PARSER_REVISION = 2, SCHEMA_VERSION = 1, METADATA_BATCH_SIZE = 50, LOGIN_TIMEOUT_MS = 120000, CONFIG_DIR = "pika", CONFIG_FILE = "config.json", DEV_CONFIG_FILE = "config.dev.json", PARSE_ERRORS_FILE = "parse-errors.jsonl", MAX_UPLOAD_RETRIES = 2, INITIAL_BACKOFF_MS = 1000, CONTENT_UPLOAD_CONCURRENCY = 2, MAX_CONTENT_UPLOAD_BYTES, MAX_METADATA_BODY_BYTES, MAX_DECOMPRESSED_CONTENT_BYTES;
|
|
71
22
|
var init_constants = __esm(() => {
|
|
@@ -74,12 +25,6 @@ var init_constants = __esm(() => {
|
|
|
74
25
|
MAX_DECOMPRESSED_CONTENT_BYTES = 256 * 1024 * 1024;
|
|
75
26
|
});
|
|
76
27
|
|
|
77
|
-
// ../core/src/validation.ts
|
|
78
|
-
var init_validation = __esm(() => {
|
|
79
|
-
init_types();
|
|
80
|
-
init_constants();
|
|
81
|
-
});
|
|
82
|
-
|
|
83
28
|
// ../core/src/chunking.ts
|
|
84
29
|
var init_chunking = __esm(() => {
|
|
85
30
|
init_constants();
|
|
@@ -121,25 +66,95 @@ function truncateAtWord(text, maxLen) {
|
|
|
121
66
|
const cutoff = maxLen - 1;
|
|
122
67
|
const lastSpace = text.lastIndexOf(" ", cutoff);
|
|
123
68
|
if (lastSpace > 0) {
|
|
124
|
-
return text.slice(0, lastSpace)
|
|
69
|
+
return `${text.slice(0, lastSpace)}\u2026`;
|
|
125
70
|
}
|
|
126
|
-
return text.slice(0, cutoff)
|
|
71
|
+
return `${text.slice(0, cutoff)}\u2026`;
|
|
127
72
|
}
|
|
128
73
|
var MAX_TITLE_LENGTH = 80;
|
|
129
74
|
|
|
75
|
+
// ../core/src/types.ts
|
|
76
|
+
var SOURCES;
|
|
77
|
+
var init_types = __esm(() => {
|
|
78
|
+
SOURCES = [
|
|
79
|
+
"claude-code",
|
|
80
|
+
"codex",
|
|
81
|
+
"gemini-cli",
|
|
82
|
+
"opencode",
|
|
83
|
+
"vscode-copilot"
|
|
84
|
+
];
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
// ../core/src/validation.ts
|
|
88
|
+
var init_validation = __esm(() => {
|
|
89
|
+
init_constants();
|
|
90
|
+
init_types();
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
// ../../package.json
|
|
94
|
+
var package_default;
|
|
95
|
+
var init_package = __esm(() => {
|
|
96
|
+
package_default = {
|
|
97
|
+
name: "pika",
|
|
98
|
+
version: "0.5.4",
|
|
99
|
+
private: true,
|
|
100
|
+
workspaces: [
|
|
101
|
+
"packages/*"
|
|
102
|
+
],
|
|
103
|
+
scripts: {
|
|
104
|
+
dev: "bun run --cwd packages/web dev",
|
|
105
|
+
build: "bun run --filter '*' build",
|
|
106
|
+
test: "vitest run",
|
|
107
|
+
"test:watch": "vitest",
|
|
108
|
+
"test:coverage": "vitest run --coverage",
|
|
109
|
+
"test:e2e": "vitest run --config packages/web/vitest.e2e.config.ts",
|
|
110
|
+
"test:all": "bun run test:coverage && bun run test:e2e",
|
|
111
|
+
lint: "tsc --noEmit && tsc --noEmit -p packages/web/tsconfig.json",
|
|
112
|
+
"lint:typecheck": "tsc --noEmit && tsc --noEmit -p packages/web/tsconfig.json",
|
|
113
|
+
"lint:biome": "bunx biome check packages/",
|
|
114
|
+
"lint:secrets": "gitleaks detect --source . --no-git --config .gitleaks.toml",
|
|
115
|
+
"lint:deps": "osv-scanner scan --lockfile=bun.lock",
|
|
116
|
+
"lint:all": "bun run lint && bunx biome check packages/ && bun run lint:secrets && bun run lint:deps",
|
|
117
|
+
"sync-versions": "bun scripts/sync-versions.ts --write",
|
|
118
|
+
prepare: "husky"
|
|
119
|
+
},
|
|
120
|
+
devDependencies: {
|
|
121
|
+
"@biomejs/biome": "^2.4.9",
|
|
122
|
+
"@vitest/coverage-v8": "^3.1.1",
|
|
123
|
+
husky: "^9.1.7",
|
|
124
|
+
typescript: "^5.8.2",
|
|
125
|
+
vitest: "^3.1.1"
|
|
126
|
+
},
|
|
127
|
+
overrides: {
|
|
128
|
+
undici: ">=7.24.0",
|
|
129
|
+
cookie: ">=0.7.0",
|
|
130
|
+
picomatch: ">=4.0.4",
|
|
131
|
+
"brace-expansion": ">=5.0.5",
|
|
132
|
+
"fast-xml-parser": ">=5.5.9",
|
|
133
|
+
next: ">=16.2.1"
|
|
134
|
+
}
|
|
135
|
+
};
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
// ../core/src/version.ts
|
|
139
|
+
var PIKA_VERSION;
|
|
140
|
+
var init_version = __esm(() => {
|
|
141
|
+
init_package();
|
|
142
|
+
PIKA_VERSION = package_default.version;
|
|
143
|
+
});
|
|
144
|
+
|
|
130
145
|
// ../core/src/index.ts
|
|
131
146
|
var init_src = __esm(() => {
|
|
132
|
-
|
|
133
|
-
init_version();
|
|
147
|
+
init_chunking();
|
|
134
148
|
init_constants();
|
|
149
|
+
init_types();
|
|
135
150
|
init_validation();
|
|
136
|
-
|
|
151
|
+
init_version();
|
|
137
152
|
});
|
|
138
153
|
|
|
139
154
|
// src/config/manager.ts
|
|
140
|
-
import { readFileSync, writeFileSync, mkdirSync, existsSync } from "fs";
|
|
141
|
-
import { join } from "path";
|
|
142
155
|
import { randomUUID } from "crypto";
|
|
156
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
|
|
157
|
+
import { join } from "path";
|
|
143
158
|
|
|
144
159
|
class ConfigManager {
|
|
145
160
|
configDir;
|
|
@@ -166,7 +181,7 @@ class ConfigManager {
|
|
|
166
181
|
}
|
|
167
182
|
const existing = this.read();
|
|
168
183
|
const merged = { ...existing, ...partial };
|
|
169
|
-
writeFileSync(this.configPath, JSON.stringify(merged, null, 2)
|
|
184
|
+
writeFileSync(this.configPath, `${JSON.stringify(merged, null, 2)}
|
|
170
185
|
`, {
|
|
171
186
|
mode: 384
|
|
172
187
|
});
|
|
@@ -194,45 +209,6 @@ var init_manager = __esm(() => {
|
|
|
194
209
|
init_src();
|
|
195
210
|
});
|
|
196
211
|
|
|
197
|
-
// src/storage/cursor-store.ts
|
|
198
|
-
import { readFile, writeFile, mkdir } from "fs/promises";
|
|
199
|
-
import { join as join2, dirname } from "path";
|
|
200
|
-
function emptyState() {
|
|
201
|
-
return { version: 1, files: {}, updatedAt: null };
|
|
202
|
-
}
|
|
203
|
-
|
|
204
|
-
class CursorStore {
|
|
205
|
-
filePath;
|
|
206
|
-
constructor(storeDir) {
|
|
207
|
-
this.filePath = join2(storeDir, CURSORS_FILE2);
|
|
208
|
-
}
|
|
209
|
-
async load() {
|
|
210
|
-
try {
|
|
211
|
-
const raw = await readFile(this.filePath, "utf-8");
|
|
212
|
-
return JSON.parse(raw);
|
|
213
|
-
} catch {
|
|
214
|
-
return emptyState();
|
|
215
|
-
}
|
|
216
|
-
}
|
|
217
|
-
async save(state) {
|
|
218
|
-
const dir = dirname(this.filePath);
|
|
219
|
-
await mkdir(dir, { recursive: true });
|
|
220
|
-
await writeFile(this.filePath, JSON.stringify(state, null, 2) + `
|
|
221
|
-
`);
|
|
222
|
-
}
|
|
223
|
-
}
|
|
224
|
-
var CURSORS_FILE2 = "cursors.json";
|
|
225
|
-
var init_cursor_store = () => {};
|
|
226
|
-
|
|
227
|
-
// src/utils/file-changed.ts
|
|
228
|
-
function fileUnchanged(prev, curr) {
|
|
229
|
-
if (!prev)
|
|
230
|
-
return false;
|
|
231
|
-
if (prev.mtimeMs === undefined || prev.size === undefined)
|
|
232
|
-
return false;
|
|
233
|
-
return prev.inode === curr.inode && prev.mtimeMs === curr.mtimeMs && prev.size === curr.size;
|
|
234
|
-
}
|
|
235
|
-
|
|
236
212
|
// src/utils/hash-project-ref.ts
|
|
237
213
|
import { createHash } from "crypto";
|
|
238
214
|
function hashProjectRef(raw) {
|
|
@@ -461,7 +437,7 @@ function extractProjectName(filePath) {
|
|
|
461
437
|
}
|
|
462
438
|
async function parseClaudeFileMulti(filePath, startOffset = 0) {
|
|
463
439
|
const st = await stat(filePath).catch(() => null);
|
|
464
|
-
if (!st
|
|
440
|
+
if (!st?.isFile() || st.size === 0)
|
|
465
441
|
return [];
|
|
466
442
|
if (startOffset >= st.size)
|
|
467
443
|
return [];
|
|
@@ -494,11 +470,20 @@ var init_claude = __esm(() => {
|
|
|
494
470
|
init_hash_project_ref();
|
|
495
471
|
});
|
|
496
472
|
|
|
473
|
+
// src/utils/file-changed.ts
|
|
474
|
+
function fileUnchanged(prev, curr) {
|
|
475
|
+
if (!prev)
|
|
476
|
+
return false;
|
|
477
|
+
if (prev.mtimeMs === undefined || prev.size === undefined)
|
|
478
|
+
return false;
|
|
479
|
+
return prev.inode === curr.inode && prev.mtimeMs === curr.mtimeMs && prev.size === curr.size;
|
|
480
|
+
}
|
|
481
|
+
|
|
497
482
|
// src/drivers/session/claude.ts
|
|
498
483
|
import { readdir, stat as stat2 } from "fs/promises";
|
|
499
|
-
import { join as
|
|
484
|
+
import { join as join2 } from "path";
|
|
500
485
|
async function discoverClaudeFiles(claudeDir) {
|
|
501
|
-
const projectsDir =
|
|
486
|
+
const projectsDir = join2(claudeDir, "projects");
|
|
502
487
|
try {
|
|
503
488
|
await stat2(projectsDir);
|
|
504
489
|
} catch {
|
|
@@ -516,7 +501,9 @@ async function walkJsonl(dir, results) {
|
|
|
516
501
|
return;
|
|
517
502
|
}
|
|
518
503
|
for (const entry of entries) {
|
|
519
|
-
|
|
504
|
+
if (entry.isDirectory() && SKIP_DIRS.has(entry.name))
|
|
505
|
+
continue;
|
|
506
|
+
const fullPath = join2(dir, entry.name);
|
|
520
507
|
if (entry.isDirectory()) {
|
|
521
508
|
await walkJsonl(fullPath, results);
|
|
522
509
|
} else if (entry.isFile() && entry.name.endsWith(".jsonl")) {
|
|
@@ -524,9 +511,10 @@ async function walkJsonl(dir, results) {
|
|
|
524
511
|
}
|
|
525
512
|
}
|
|
526
513
|
}
|
|
527
|
-
var claudeSessionDriver;
|
|
514
|
+
var SKIP_DIRS, claudeSessionDriver;
|
|
528
515
|
var init_claude2 = __esm(() => {
|
|
529
516
|
init_claude();
|
|
517
|
+
SKIP_DIRS = new Set(["subagents"]);
|
|
530
518
|
claudeSessionDriver = {
|
|
531
519
|
source: "claude-code",
|
|
532
520
|
async discover(opts) {
|
|
@@ -822,7 +810,7 @@ function buildEmptyResult(filePath) {
|
|
|
822
810
|
}
|
|
823
811
|
async function parseCodexFile(filePath, startOffset = 0) {
|
|
824
812
|
const st = await stat3(filePath).catch(() => null);
|
|
825
|
-
if (!st
|
|
813
|
+
if (!st?.isFile() || st.size === 0)
|
|
826
814
|
return buildEmptyResult(filePath);
|
|
827
815
|
if (startOffset >= st.size)
|
|
828
816
|
return buildEmptyResult(filePath);
|
|
@@ -865,7 +853,7 @@ var init_codex = __esm(() => {
|
|
|
865
853
|
|
|
866
854
|
// src/drivers/session/codex.ts
|
|
867
855
|
import { readdir as readdir2, stat as stat4 } from "fs/promises";
|
|
868
|
-
import { join as
|
|
856
|
+
import { join as join3 } from "path";
|
|
869
857
|
async function walkFiltered(dir, results, filter) {
|
|
870
858
|
let entries;
|
|
871
859
|
try {
|
|
@@ -874,7 +862,7 @@ async function walkFiltered(dir, results, filter) {
|
|
|
874
862
|
return;
|
|
875
863
|
}
|
|
876
864
|
for (const entry of entries) {
|
|
877
|
-
const fullPath =
|
|
865
|
+
const fullPath = join3(dir, entry.name);
|
|
878
866
|
if (entry.isDirectory()) {
|
|
879
867
|
await walkFiltered(fullPath, results, filter);
|
|
880
868
|
} else if (entry.isFile() && filter(entry.name)) {
|
|
@@ -907,10 +895,20 @@ var init_codex2 = __esm(() => {
|
|
|
907
895
|
},
|
|
908
896
|
resumeState(cursor, fingerprint) {
|
|
909
897
|
if (!cursor || cursor.inode !== fingerprint.inode) {
|
|
910
|
-
return {
|
|
898
|
+
return {
|
|
899
|
+
kind: "codex",
|
|
900
|
+
startOffset: 0,
|
|
901
|
+
lastTotalTokens: 0,
|
|
902
|
+
lastModel: null
|
|
903
|
+
};
|
|
911
904
|
}
|
|
912
905
|
if (cursor.offset > fingerprint.size) {
|
|
913
|
-
return {
|
|
906
|
+
return {
|
|
907
|
+
kind: "codex",
|
|
908
|
+
startOffset: 0,
|
|
909
|
+
lastTotalTokens: 0,
|
|
910
|
+
lastModel: null
|
|
911
|
+
};
|
|
914
912
|
}
|
|
915
913
|
return {
|
|
916
914
|
kind: "codex",
|
|
@@ -921,6 +919,8 @@ var init_codex2 = __esm(() => {
|
|
|
921
919
|
},
|
|
922
920
|
async parse(filePath, resume) {
|
|
923
921
|
const result = await parseCodexFile(filePath, resume.startOffset);
|
|
922
|
+
if (result.canonical.messages.length === 0)
|
|
923
|
+
return [];
|
|
924
924
|
return [result];
|
|
925
925
|
},
|
|
926
926
|
buildCursor(fingerprint, results) {
|
|
@@ -945,7 +945,7 @@ var init_codex2 = __esm(() => {
|
|
|
945
945
|
});
|
|
946
946
|
|
|
947
947
|
// src/parsers/gemini.ts
|
|
948
|
-
import { readFile
|
|
948
|
+
import { readFile, stat as stat5 } from "fs/promises";
|
|
949
949
|
function toNonNegInt3(value) {
|
|
950
950
|
if (typeof value !== "number" || !Number.isFinite(value) || value < 0)
|
|
951
951
|
return 0;
|
|
@@ -1103,11 +1103,11 @@ function buildEmptyResult2(filePath) {
|
|
|
1103
1103
|
}
|
|
1104
1104
|
async function parseGeminiFile(filePath, startIndex = 0) {
|
|
1105
1105
|
const st = await stat5(filePath).catch(() => null);
|
|
1106
|
-
if (!st
|
|
1106
|
+
if (!st?.isFile() || st.size === 0)
|
|
1107
1107
|
return buildEmptyResult2(filePath);
|
|
1108
1108
|
let rawContent;
|
|
1109
1109
|
try {
|
|
1110
|
-
rawContent = await
|
|
1110
|
+
rawContent = await readFile(filePath, "utf8");
|
|
1111
1111
|
} catch {
|
|
1112
1112
|
return buildEmptyResult2(filePath);
|
|
1113
1113
|
}
|
|
@@ -1157,9 +1157,9 @@ var init_gemini = __esm(() => {
|
|
|
1157
1157
|
|
|
1158
1158
|
// src/drivers/session/gemini.ts
|
|
1159
1159
|
import { readdir as readdir3, stat as stat6 } from "fs/promises";
|
|
1160
|
-
import { join as
|
|
1160
|
+
import { join as join4 } from "path";
|
|
1161
1161
|
async function discoverGeminiFiles(geminiDir) {
|
|
1162
|
-
const tmpDir =
|
|
1162
|
+
const tmpDir = join4(geminiDir, "tmp");
|
|
1163
1163
|
try {
|
|
1164
1164
|
await stat6(tmpDir);
|
|
1165
1165
|
} catch {
|
|
@@ -1175,7 +1175,7 @@ async function discoverGeminiFiles(geminiDir) {
|
|
|
1175
1175
|
for (const projEntry of projectDirs) {
|
|
1176
1176
|
if (!projEntry.isDirectory())
|
|
1177
1177
|
continue;
|
|
1178
|
-
const chatsDir =
|
|
1178
|
+
const chatsDir = join4(tmpDir, projEntry.name, "chats");
|
|
1179
1179
|
let chatFiles;
|
|
1180
1180
|
try {
|
|
1181
1181
|
chatFiles = await readdir3(chatsDir, { withFileTypes: true });
|
|
@@ -1184,7 +1184,7 @@ async function discoverGeminiFiles(geminiDir) {
|
|
|
1184
1184
|
}
|
|
1185
1185
|
for (const chatEntry of chatFiles) {
|
|
1186
1186
|
if (chatEntry.isFile() && chatEntry.name.startsWith("session-") && chatEntry.name.endsWith(".json")) {
|
|
1187
|
-
results.push(
|
|
1187
|
+
results.push(join4(chatsDir, chatEntry.name));
|
|
1188
1188
|
}
|
|
1189
1189
|
}
|
|
1190
1190
|
}
|
|
@@ -1229,6 +1229,8 @@ var init_gemini2 = __esm(() => {
|
|
|
1229
1229
|
},
|
|
1230
1230
|
async parse(filePath, resume) {
|
|
1231
1231
|
const result = await parseGeminiFile(filePath, resume.startIndex);
|
|
1232
|
+
if (result.canonical.messages.length === 0)
|
|
1233
|
+
return [];
|
|
1232
1234
|
return [result];
|
|
1233
1235
|
},
|
|
1234
1236
|
buildCursor(fingerprint, results) {
|
|
@@ -1256,8 +1258,8 @@ var init_gemini2 = __esm(() => {
|
|
|
1256
1258
|
});
|
|
1257
1259
|
|
|
1258
1260
|
// src/parsers/opencode.ts
|
|
1259
|
-
import {
|
|
1260
|
-
import { join as
|
|
1261
|
+
import { readdir as readdir4, readFile as readFile2 } from "fs/promises";
|
|
1262
|
+
import { join as join5 } from "path";
|
|
1261
1263
|
function toNonNegInt4(value) {
|
|
1262
1264
|
if (typeof value !== "number" || !Number.isFinite(value) || value < 0)
|
|
1263
1265
|
return 0;
|
|
@@ -1421,14 +1423,14 @@ function buildEmptyResult3(sessionId, rawPath) {
|
|
|
1421
1423
|
}
|
|
1422
1424
|
async function loadJsonFileWithRaw(filePath) {
|
|
1423
1425
|
try {
|
|
1424
|
-
const raw = await
|
|
1426
|
+
const raw = await readFile2(filePath, "utf8");
|
|
1425
1427
|
return { parsed: JSON.parse(raw), raw };
|
|
1426
1428
|
} catch {
|
|
1427
1429
|
return null;
|
|
1428
1430
|
}
|
|
1429
1431
|
}
|
|
1430
1432
|
async function loadPartsForMessageWithRaw(partDir, messageId) {
|
|
1431
|
-
const msgPartDir =
|
|
1433
|
+
const msgPartDir = join5(partDir, messageId);
|
|
1432
1434
|
let entries;
|
|
1433
1435
|
try {
|
|
1434
1436
|
entries = await readdir4(msgPartDir);
|
|
@@ -1440,7 +1442,7 @@ async function loadPartsForMessageWithRaw(partDir, messageId) {
|
|
|
1440
1442
|
for (const entry of entries.sort()) {
|
|
1441
1443
|
if (!entry.endsWith(".json"))
|
|
1442
1444
|
continue;
|
|
1443
|
-
const filePath =
|
|
1445
|
+
const filePath = join5(msgPartDir, entry);
|
|
1444
1446
|
const loaded = await loadJsonFileWithRaw(filePath);
|
|
1445
1447
|
if (loaded && typeof loaded.parsed.type === "string") {
|
|
1446
1448
|
parts.push(loaded.parsed);
|
|
@@ -1481,7 +1483,7 @@ async function parseOpenCodeJsonSession(sessionJsonPath, messageDir, partDir) {
|
|
|
1481
1483
|
const rawSourceFiles = [
|
|
1482
1484
|
{ path: sessionJsonPath, format: "json", content: sessionLoaded.raw }
|
|
1483
1485
|
];
|
|
1484
|
-
const sessionMsgDir =
|
|
1486
|
+
const sessionMsgDir = join5(messageDir, session.id);
|
|
1485
1487
|
let msgEntries;
|
|
1486
1488
|
try {
|
|
1487
1489
|
msgEntries = await readdir4(sessionMsgDir);
|
|
@@ -1492,12 +1494,16 @@ async function parseOpenCodeJsonSession(sessionJsonPath, messageDir, partDir) {
|
|
|
1492
1494
|
for (const entry of msgEntries.sort()) {
|
|
1493
1495
|
if (!entry.endsWith(".json"))
|
|
1494
1496
|
continue;
|
|
1495
|
-
const msgPath =
|
|
1497
|
+
const msgPath = join5(sessionMsgDir, entry);
|
|
1496
1498
|
const msgLoaded = await loadJsonFileWithRaw(msgPath);
|
|
1497
1499
|
if (!msgLoaded || typeof msgLoaded.parsed.role !== "string")
|
|
1498
1500
|
continue;
|
|
1499
1501
|
const msg = msgLoaded.parsed;
|
|
1500
|
-
rawSourceFiles.push({
|
|
1502
|
+
rawSourceFiles.push({
|
|
1503
|
+
path: msgPath,
|
|
1504
|
+
format: "json",
|
|
1505
|
+
content: msgLoaded.raw
|
|
1506
|
+
});
|
|
1501
1507
|
const partsResult = await loadPartsForMessageWithRaw(partDir, msg.id);
|
|
1502
1508
|
msg.parts = partsResult.parts;
|
|
1503
1509
|
rawSourceFiles.push(...partsResult.sourceFiles);
|
|
@@ -1521,13 +1527,13 @@ var init_opencode = __esm(() => {
|
|
|
1521
1527
|
});
|
|
1522
1528
|
|
|
1523
1529
|
// src/drivers/session/opencode.ts
|
|
1524
|
-
import { readdir as readdir5, stat as
|
|
1525
|
-
import {
|
|
1530
|
+
import { readdir as readdir5, stat as stat7 } from "fs/promises";
|
|
1531
|
+
import { basename as basename2, dirname, join as join6 } from "path";
|
|
1526
1532
|
async function discoverOpenCodeJsonFiles(messageDir, ctx, inodeToFilePath) {
|
|
1527
|
-
const storageDir =
|
|
1528
|
-
const sessionDir =
|
|
1533
|
+
const storageDir = dirname(messageDir);
|
|
1534
|
+
const sessionDir = join6(storageDir, "session");
|
|
1529
1535
|
try {
|
|
1530
|
-
await
|
|
1536
|
+
await stat7(sessionDir);
|
|
1531
1537
|
} catch {
|
|
1532
1538
|
return [];
|
|
1533
1539
|
}
|
|
@@ -1542,7 +1548,7 @@ async function discoverOpenCodeJsonFiles(messageDir, ctx, inodeToFilePath) {
|
|
|
1542
1548
|
for (const projEntry of projectDirs) {
|
|
1543
1549
|
if (!projEntry.isDirectory())
|
|
1544
1550
|
continue;
|
|
1545
|
-
const projDir =
|
|
1551
|
+
const projDir = join6(sessionDir, projEntry.name);
|
|
1546
1552
|
let sessionFiles;
|
|
1547
1553
|
try {
|
|
1548
1554
|
sessionFiles = await readdir5(projDir, { withFileTypes: true });
|
|
@@ -1551,18 +1557,18 @@ async function discoverOpenCodeJsonFiles(messageDir, ctx, inodeToFilePath) {
|
|
|
1551
1557
|
}
|
|
1552
1558
|
for (const fileEntry of sessionFiles) {
|
|
1553
1559
|
if (fileEntry.isFile() && fileEntry.name.startsWith("ses_") && fileEntry.name.endsWith(".json")) {
|
|
1554
|
-
const filePath =
|
|
1560
|
+
const filePath = join6(projDir, fileEntry.name);
|
|
1555
1561
|
results.push(filePath);
|
|
1556
1562
|
if (inodeToFilePath) {
|
|
1557
1563
|
try {
|
|
1558
|
-
const fileStat = await
|
|
1564
|
+
const fileStat = await stat7(filePath);
|
|
1559
1565
|
inodeToFilePath.set(fileStat.ino, filePath);
|
|
1560
1566
|
} catch {}
|
|
1561
1567
|
}
|
|
1562
1568
|
const sessionId = basename2(fileEntry.name, ".json");
|
|
1563
|
-
const sessionMsgDir =
|
|
1569
|
+
const sessionMsgDir = join6(messageDir, sessionId);
|
|
1564
1570
|
try {
|
|
1565
|
-
const msgStat = await
|
|
1571
|
+
const msgStat = await stat7(sessionMsgDir);
|
|
1566
1572
|
msgDirMtimes[filePath] = msgStat.mtimeMs;
|
|
1567
1573
|
} catch {}
|
|
1568
1574
|
}
|
|
@@ -1606,11 +1612,11 @@ function createOpenCodeJsonDriver(syncCtx) {
|
|
|
1606
1612
|
return { kind: "opencode-json" };
|
|
1607
1613
|
},
|
|
1608
1614
|
async parse(filePath, _resume) {
|
|
1609
|
-
const sessionDir =
|
|
1610
|
-
const projectSessionDir =
|
|
1611
|
-
const storageDir =
|
|
1612
|
-
const messageDir =
|
|
1613
|
-
const partDir =
|
|
1615
|
+
const sessionDir = dirname(filePath);
|
|
1616
|
+
const projectSessionDir = dirname(sessionDir);
|
|
1617
|
+
const storageDir = dirname(projectSessionDir);
|
|
1618
|
+
const messageDir = join6(storageDir, "message");
|
|
1619
|
+
const partDir = join6(storageDir, "part");
|
|
1614
1620
|
const result = await parseOpenCodeJsonSession(filePath, messageDir, partDir);
|
|
1615
1621
|
if (syncCtx) {
|
|
1616
1622
|
if (!syncCtx.openCodeSessionState) {
|
|
@@ -1621,6 +1627,8 @@ function createOpenCodeJsonDriver(syncCtx) {
|
|
|
1621
1627
|
totalMessages: result.canonical.messages.length
|
|
1622
1628
|
});
|
|
1623
1629
|
}
|
|
1630
|
+
if (result.canonical.messages.length === 0)
|
|
1631
|
+
return [];
|
|
1624
1632
|
return [result];
|
|
1625
1633
|
},
|
|
1626
1634
|
buildCursor(fingerprint, _results) {
|
|
@@ -1641,8 +1649,8 @@ var init_opencode2 = __esm(() => {
|
|
|
1641
1649
|
});
|
|
1642
1650
|
|
|
1643
1651
|
// src/parsers/vscode-copilot.ts
|
|
1644
|
-
import { readFile as
|
|
1645
|
-
import { dirname as
|
|
1652
|
+
import { readFile as readFile3, stat as stat8 } from "fs/promises";
|
|
1653
|
+
import { dirname as dirname2, join as join7 } from "path";
|
|
1646
1654
|
function toNonNegInt5(value) {
|
|
1647
1655
|
if (typeof value !== "number" || !Number.isFinite(value) || value < 0)
|
|
1648
1656
|
return 0;
|
|
@@ -1838,10 +1846,10 @@ function extractRequestMessages(req, fallbackModel, accum) {
|
|
|
1838
1846
|
}
|
|
1839
1847
|
async function extractWorkspaceFolder(sessionFilePath) {
|
|
1840
1848
|
try {
|
|
1841
|
-
const chatSessionsDir =
|
|
1842
|
-
const workspaceDir =
|
|
1843
|
-
const workspaceJsonPath =
|
|
1844
|
-
const content = await
|
|
1849
|
+
const chatSessionsDir = dirname2(sessionFilePath);
|
|
1850
|
+
const workspaceDir = dirname2(chatSessionsDir);
|
|
1851
|
+
const workspaceJsonPath = join7(workspaceDir, "workspace.json");
|
|
1852
|
+
const content = await readFile3(workspaceJsonPath, "utf8");
|
|
1845
1853
|
const data = JSON.parse(content);
|
|
1846
1854
|
if (typeof data?.folder === "string") {
|
|
1847
1855
|
const folder = data.folder.replace(/^file:\/\//, "");
|
|
@@ -1983,14 +1991,14 @@ function extractMessages(state, processedIds = new Set) {
|
|
|
1983
1991
|
return { accum, newRequestIds };
|
|
1984
1992
|
}
|
|
1985
1993
|
async function parseVscodeCopilotFile(filePath, startOffset = 0, processedRequestIds = [], workspaceFolder = null) {
|
|
1986
|
-
const st = await
|
|
1987
|
-
if (!st
|
|
1994
|
+
const st = await stat8(filePath).catch(() => null);
|
|
1995
|
+
if (!st?.isFile() || st.size === 0)
|
|
1988
1996
|
return { ...buildEmptyResult4(filePath), newRequestIds: [] };
|
|
1989
1997
|
if (startOffset >= st.size)
|
|
1990
1998
|
return { ...buildEmptyResult4(filePath), newRequestIds: [] };
|
|
1991
1999
|
let rawContent;
|
|
1992
2000
|
try {
|
|
1993
|
-
rawContent = await
|
|
2001
|
+
rawContent = await readFile3(filePath, "utf8");
|
|
1994
2002
|
} catch {
|
|
1995
2003
|
return { ...buildEmptyResult4(filePath), newRequestIds: [] };
|
|
1996
2004
|
}
|
|
@@ -2005,7 +2013,10 @@ async function parseVscodeCopilotFile(filePath, startOffset = 0, processedReques
|
|
|
2005
2013
|
if (accum.messages.length === 0)
|
|
2006
2014
|
return { ...buildEmptyResult4(filePath), newRequestIds: [] };
|
|
2007
2015
|
const folder = workspaceFolder ?? await extractWorkspaceFolder(filePath);
|
|
2008
|
-
return {
|
|
2016
|
+
return {
|
|
2017
|
+
...buildParseResult5(state, accum, filePath, rawContent, folder),
|
|
2018
|
+
newRequestIds
|
|
2019
|
+
};
|
|
2009
2020
|
}
|
|
2010
2021
|
var init_vscode_copilot = __esm(() => {
|
|
2011
2022
|
init_src();
|
|
@@ -2014,7 +2025,7 @@ var init_vscode_copilot = __esm(() => {
|
|
|
2014
2025
|
|
|
2015
2026
|
// src/drivers/session/vscode-copilot.ts
|
|
2016
2027
|
import { readdir as readdir6 } from "fs/promises";
|
|
2017
|
-
import { join as
|
|
2028
|
+
import { join as join8 } from "path";
|
|
2018
2029
|
async function collectJsonl(dir) {
|
|
2019
2030
|
let entries;
|
|
2020
2031
|
try {
|
|
@@ -2025,25 +2036,25 @@ async function collectJsonl(dir) {
|
|
|
2025
2036
|
const results = [];
|
|
2026
2037
|
for (const entry of entries) {
|
|
2027
2038
|
if (entry.isFile() && entry.name.endsWith(".jsonl")) {
|
|
2028
|
-
results.push(
|
|
2039
|
+
results.push(join8(dir, entry.name));
|
|
2029
2040
|
}
|
|
2030
2041
|
}
|
|
2031
2042
|
return results;
|
|
2032
2043
|
}
|
|
2033
2044
|
async function discoverInBaseDir(baseDir) {
|
|
2034
2045
|
const results = [];
|
|
2035
|
-
const wsDir =
|
|
2046
|
+
const wsDir = join8(baseDir, "workspaceStorage");
|
|
2036
2047
|
try {
|
|
2037
2048
|
const wsDirEntries = await readdir6(wsDir, { withFileTypes: true });
|
|
2038
2049
|
for (const entry of wsDirEntries) {
|
|
2039
2050
|
if (!entry.isDirectory())
|
|
2040
2051
|
continue;
|
|
2041
|
-
const chatDir =
|
|
2052
|
+
const chatDir = join8(wsDir, entry.name, "chatSessions");
|
|
2042
2053
|
const files = await collectJsonl(chatDir);
|
|
2043
2054
|
results.push(...files);
|
|
2044
2055
|
}
|
|
2045
2056
|
} catch {}
|
|
2046
|
-
const globalDir =
|
|
2057
|
+
const globalDir = join8(baseDir, "globalStorage", "emptyWindowChatSessions");
|
|
2047
2058
|
const globalFiles = await collectJsonl(globalDir);
|
|
2048
2059
|
results.push(...globalFiles);
|
|
2049
2060
|
return results;
|
|
@@ -2071,10 +2082,18 @@ var init_vscode_copilot2 = __esm(() => {
|
|
|
2071
2082
|
},
|
|
2072
2083
|
resumeState(cursor, fingerprint) {
|
|
2073
2084
|
if (!cursor || cursor.inode !== fingerprint.inode) {
|
|
2074
|
-
return {
|
|
2085
|
+
return {
|
|
2086
|
+
kind: "vscode-copilot",
|
|
2087
|
+
startOffset: 0,
|
|
2088
|
+
processedRequestIds: []
|
|
2089
|
+
};
|
|
2075
2090
|
}
|
|
2076
2091
|
if (cursor.offset > fingerprint.size) {
|
|
2077
|
-
return {
|
|
2092
|
+
return {
|
|
2093
|
+
kind: "vscode-copilot",
|
|
2094
|
+
startOffset: 0,
|
|
2095
|
+
processedRequestIds: []
|
|
2096
|
+
};
|
|
2078
2097
|
}
|
|
2079
2098
|
return {
|
|
2080
2099
|
kind: "vscode-copilot",
|
|
@@ -2106,25 +2125,25 @@ var init_vscode_copilot2 = __esm(() => {
|
|
|
2106
2125
|
});
|
|
2107
2126
|
|
|
2108
2127
|
// src/drivers/registry.ts
|
|
2109
|
-
import { stat as
|
|
2128
|
+
import { stat as stat9 } from "fs/promises";
|
|
2110
2129
|
import { homedir } from "os";
|
|
2111
|
-
import { join as
|
|
2130
|
+
import { join as join9 } from "path";
|
|
2112
2131
|
function resolveDefaultPaths(home) {
|
|
2113
2132
|
const h = home ?? homedir();
|
|
2114
2133
|
return {
|
|
2115
|
-
claudeDir:
|
|
2116
|
-
codexSessionsDir:
|
|
2117
|
-
geminiDir:
|
|
2118
|
-
openCodeDir:
|
|
2134
|
+
claudeDir: join9(h, ".claude"),
|
|
2135
|
+
codexSessionsDir: join9(h, ".codex", "sessions"),
|
|
2136
|
+
geminiDir: join9(h, ".gemini"),
|
|
2137
|
+
openCodeDir: join9(h, ".local", "share", "opencode"),
|
|
2119
2138
|
vscodeCopilotDirs: [
|
|
2120
|
-
|
|
2121
|
-
|
|
2139
|
+
join9(h, "Library", "Application Support", "Code", "User"),
|
|
2140
|
+
join9(h, "Library", "Application Support", "Code - Insiders", "User")
|
|
2122
2141
|
]
|
|
2123
2142
|
};
|
|
2124
2143
|
}
|
|
2125
2144
|
async function exists(path) {
|
|
2126
2145
|
try {
|
|
2127
|
-
await
|
|
2146
|
+
await stat9(path);
|
|
2128
2147
|
return true;
|
|
2129
2148
|
} catch {
|
|
2130
2149
|
return false;
|
|
@@ -2144,7 +2163,7 @@ async function buildDriverSet(overrides, syncCtx, sourceFilter) {
|
|
|
2144
2163
|
exists(paths.codexSessionsDir),
|
|
2145
2164
|
exists(paths.geminiDir),
|
|
2146
2165
|
exists(paths.openCodeDir),
|
|
2147
|
-
exists(
|
|
2166
|
+
exists(join9(paths.openCodeDir, "opencode.db")),
|
|
2148
2167
|
...paths.vscodeCopilotDirs.map((d) => exists(d))
|
|
2149
2168
|
]);
|
|
2150
2169
|
const discoverOpts = {};
|
|
@@ -2155,10 +2174,10 @@ async function buildDriverSet(overrides, syncCtx, sourceFilter) {
|
|
|
2155
2174
|
if (geminiExists)
|
|
2156
2175
|
discoverOpts.geminiDir = paths.geminiDir;
|
|
2157
2176
|
if (openCodeExists) {
|
|
2158
|
-
discoverOpts.openCodeMessageDir =
|
|
2177
|
+
discoverOpts.openCodeMessageDir = join9(paths.openCodeDir, "storage", "message");
|
|
2159
2178
|
}
|
|
2160
2179
|
if (openCodeDbExists) {
|
|
2161
|
-
discoverOpts.openCodeDbPath =
|
|
2180
|
+
discoverOpts.openCodeDbPath = join9(paths.openCodeDir, "opencode.db");
|
|
2162
2181
|
}
|
|
2163
2182
|
const activeVscodeDirs = paths.vscodeCopilotDirs.filter((_, i) => vscodeDirExists[i]);
|
|
2164
2183
|
if (activeVscodeDirs.length > 0) {
|
|
@@ -2193,7 +2212,7 @@ var init_registry = __esm(() => {
|
|
|
2193
2212
|
});
|
|
2194
2213
|
|
|
2195
2214
|
// src/drivers/session/opencode-sqlite.ts
|
|
2196
|
-
import { stat as
|
|
2215
|
+
import { stat as stat10 } from "fs/promises";
|
|
2197
2216
|
function shouldSkipForJson(sessionKey, sqliteLastMessageAt, sqliteTotalMessages, jsonState) {
|
|
2198
2217
|
if (!jsonState)
|
|
2199
2218
|
return false;
|
|
@@ -2242,9 +2261,7 @@ function queryMessagesForSession(db, sessionId, watermark, lastMessageIds) {
|
|
|
2242
2261
|
if (!data.id)
|
|
2243
2262
|
data.id = row.id;
|
|
2244
2263
|
messages.push(data);
|
|
2245
|
-
} catch {
|
|
2246
|
-
continue;
|
|
2247
|
-
}
|
|
2264
|
+
} catch {}
|
|
2248
2265
|
}
|
|
2249
2266
|
return messages;
|
|
2250
2267
|
}
|
|
@@ -2261,9 +2278,7 @@ function queryAllMessagesForSession(db, sessionId) {
|
|
|
2261
2278
|
data.id = row.id;
|
|
2262
2279
|
messages.push(data);
|
|
2263
2280
|
rawDataStrings.push(row.data);
|
|
2264
|
-
} catch {
|
|
2265
|
-
continue;
|
|
2266
|
-
}
|
|
2281
|
+
} catch {}
|
|
2267
2282
|
}
|
|
2268
2283
|
return { messages, rawDataStrings };
|
|
2269
2284
|
}
|
|
@@ -2278,9 +2293,7 @@ function queryPartsForMessageWithRaw(db, messageId) {
|
|
|
2278
2293
|
continue;
|
|
2279
2294
|
parts.push(data);
|
|
2280
2295
|
rawDataStrings.push(row.data);
|
|
2281
|
-
} catch {
|
|
2282
|
-
continue;
|
|
2283
|
-
}
|
|
2296
|
+
} catch {}
|
|
2284
2297
|
}
|
|
2285
2298
|
return { parts, rawDataStrings };
|
|
2286
2299
|
}
|
|
@@ -2317,7 +2330,7 @@ function createOpenCodeSqliteDriver(openDb, dbPath) {
|
|
|
2317
2330
|
async run(prevCursor, ctx) {
|
|
2318
2331
|
let dbStat;
|
|
2319
2332
|
try {
|
|
2320
|
-
dbStat = await
|
|
2333
|
+
dbStat = await stat10(dbPath);
|
|
2321
2334
|
} catch {
|
|
2322
2335
|
return {
|
|
2323
2336
|
results: [],
|
|
@@ -2424,6 +2437,36 @@ var init_opencode_sqlite = __esm(() => {
|
|
|
2424
2437
|
init_opencode();
|
|
2425
2438
|
});
|
|
2426
2439
|
|
|
2440
|
+
// src/storage/cursor-store.ts
|
|
2441
|
+
import { mkdir, readFile as readFile4, writeFile } from "fs/promises";
|
|
2442
|
+
import { dirname as dirname3, join as join10 } from "path";
|
|
2443
|
+
function emptyState() {
|
|
2444
|
+
return { version: 1, files: {}, updatedAt: null };
|
|
2445
|
+
}
|
|
2446
|
+
|
|
2447
|
+
class CursorStore {
|
|
2448
|
+
filePath;
|
|
2449
|
+
constructor(storeDir) {
|
|
2450
|
+
this.filePath = join10(storeDir, CURSORS_FILE2);
|
|
2451
|
+
}
|
|
2452
|
+
async load() {
|
|
2453
|
+
try {
|
|
2454
|
+
const raw = await readFile4(this.filePath, "utf-8");
|
|
2455
|
+
return JSON.parse(raw);
|
|
2456
|
+
} catch {
|
|
2457
|
+
return emptyState();
|
|
2458
|
+
}
|
|
2459
|
+
}
|
|
2460
|
+
async save(state) {
|
|
2461
|
+
const dir = dirname3(this.filePath);
|
|
2462
|
+
await mkdir(dir, { recursive: true });
|
|
2463
|
+
await writeFile(this.filePath, `${JSON.stringify(state, null, 2)}
|
|
2464
|
+
`);
|
|
2465
|
+
}
|
|
2466
|
+
}
|
|
2467
|
+
var CURSORS_FILE2 = "cursors.json";
|
|
2468
|
+
var init_cursor_store = () => {};
|
|
2469
|
+
|
|
2427
2470
|
// src/upload/engine.ts
|
|
2428
2471
|
import { createHash as createHash2 } from "crypto";
|
|
2429
2472
|
function sha256(input) {
|
|
@@ -2483,11 +2526,11 @@ function parseRetryAfter(value) {
|
|
|
2483
2526
|
if (!value)
|
|
2484
2527
|
return INITIAL_BACKOFF_MS;
|
|
2485
2528
|
const seconds = parseInt(value, 10);
|
|
2486
|
-
if (!isNaN(seconds) && seconds >= 0) {
|
|
2529
|
+
if (!Number.isNaN(seconds) && seconds >= 0) {
|
|
2487
2530
|
return seconds * 1000;
|
|
2488
2531
|
}
|
|
2489
2532
|
const date = new Date(value);
|
|
2490
|
-
if (!isNaN(date.getTime())) {
|
|
2533
|
+
if (!Number.isNaN(date.getTime())) {
|
|
2491
2534
|
const ms = date.getTime() - Date.now();
|
|
2492
2535
|
return Math.max(0, ms);
|
|
2493
2536
|
}
|
|
@@ -2534,9 +2577,8 @@ async function uploadBatch(snapshots, opts) {
|
|
|
2534
2577
|
}
|
|
2535
2578
|
if (response.status >= 500) {
|
|
2536
2579
|
if (attempt < MAX_UPLOAD_RETRIES) {
|
|
2537
|
-
const backoff = INITIAL_BACKOFF_MS *
|
|
2580
|
+
const backoff = INITIAL_BACKOFF_MS * 2 ** attempt;
|
|
2538
2581
|
await sleepFn(backoff);
|
|
2539
|
-
continue;
|
|
2540
2582
|
}
|
|
2541
2583
|
}
|
|
2542
2584
|
}
|
|
@@ -2601,8 +2643,8 @@ var init_engine = __esm(() => {
|
|
|
2601
2643
|
});
|
|
2602
2644
|
|
|
2603
2645
|
// src/upload/content.ts
|
|
2604
|
-
import { gzip } from "zlib";
|
|
2605
2646
|
import { promisify } from "util";
|
|
2647
|
+
import { gzip } from "zlib";
|
|
2606
2648
|
async function gzipCompress(input) {
|
|
2607
2649
|
return gzipAsync(Buffer.from(input, "utf-8"));
|
|
2608
2650
|
}
|
|
@@ -2649,9 +2691,8 @@ async function putWithRetry(url, body, headers, opts) {
|
|
|
2649
2691
|
}
|
|
2650
2692
|
if (response.status >= 500) {
|
|
2651
2693
|
if (attempt < MAX_UPLOAD_RETRIES) {
|
|
2652
|
-
const backoff = INITIAL_BACKOFF_MS *
|
|
2694
|
+
const backoff = INITIAL_BACKOFF_MS * 2 ** attempt;
|
|
2653
2695
|
await sleepFn(backoff);
|
|
2654
|
-
continue;
|
|
2655
2696
|
}
|
|
2656
2697
|
}
|
|
2657
2698
|
}
|
|
@@ -2697,7 +2738,7 @@ async function uploadToPresignedUrl(presignedUrl, body, opts) {
|
|
|
2697
2738
|
}
|
|
2698
2739
|
if (response.status >= 500) {
|
|
2699
2740
|
if (attempt < MAX_UPLOAD_RETRIES) {
|
|
2700
|
-
const backoff = INITIAL_BACKOFF_MS *
|
|
2741
|
+
const backoff = INITIAL_BACKOFF_MS * 2 ** attempt;
|
|
2701
2742
|
await sleepFn(backoff);
|
|
2702
2743
|
continue;
|
|
2703
2744
|
}
|
|
@@ -2817,9 +2858,9 @@ var init_content = __esm(() => {
|
|
|
2817
2858
|
});
|
|
2818
2859
|
|
|
2819
2860
|
// src/commands/sync-pipeline.ts
|
|
2820
|
-
import { stat as
|
|
2861
|
+
import { stat as stat11 } from "fs/promises";
|
|
2821
2862
|
async function getFingerprint(filePath) {
|
|
2822
|
-
const s = await
|
|
2863
|
+
const s = await stat11(filePath);
|
|
2823
2864
|
return {
|
|
2824
2865
|
inode: s.ino,
|
|
2825
2866
|
mtimeMs: s.mtimeMs,
|
|
@@ -2827,14 +2868,12 @@ async function getFingerprint(filePath) {
|
|
|
2827
2868
|
};
|
|
2828
2869
|
}
|
|
2829
2870
|
async function runSyncPipeline(input, opts) {
|
|
2830
|
-
const {
|
|
2831
|
-
fileDrivers,
|
|
2832
|
-
dbDriver,
|
|
2833
|
-
discoverOpts,
|
|
2834
|
-
syncCtx
|
|
2835
|
-
} = input;
|
|
2871
|
+
const { fileDrivers, dbDriver, discoverOpts, syncCtx } = input;
|
|
2836
2872
|
const log = opts.logger;
|
|
2837
|
-
|
|
2873
|
+
const cursorState = {
|
|
2874
|
+
...input.cursorState,
|
|
2875
|
+
files: { ...input.cursorState.files }
|
|
2876
|
+
};
|
|
2838
2877
|
const allResults = [];
|
|
2839
2878
|
const parseErrors = [];
|
|
2840
2879
|
let totalFiles = 0;
|
|
@@ -2904,15 +2943,11 @@ async function runSyncPipeline(input, opts) {
|
|
|
2904
2943
|
}
|
|
2905
2944
|
}
|
|
2906
2945
|
cursorState.updatedAt = new Date().toISOString();
|
|
2946
|
+
const emptyCount = allResults.filter((r) => r.canonical.messages.length === 0).length;
|
|
2947
|
+
const uploadableResults = allResults.filter((r) => r.canonical.messages.length > 0);
|
|
2907
2948
|
let uploadResult;
|
|
2908
2949
|
let contentResult;
|
|
2909
|
-
if (opts.upload &&
|
|
2910
|
-
const transformed = allResults.map((r) => toSessionSnapshot(r.canonical, r.raw));
|
|
2911
|
-
const snapshots = transformed.map((t) => t.snapshot);
|
|
2912
|
-
const precomputedMap = new Map;
|
|
2913
|
-
for (const t of transformed) {
|
|
2914
|
-
precomputedMap.set(t.snapshot.sessionKey, t.precomputed);
|
|
2915
|
-
}
|
|
2950
|
+
if (opts.upload && uploadableResults.length > 0) {
|
|
2916
2951
|
const uploadOpts = {
|
|
2917
2952
|
apiUrl: opts.apiUrl,
|
|
2918
2953
|
apiKey: opts.apiKey,
|
|
@@ -2920,70 +2955,99 @@ async function runSyncPipeline(input, opts) {
|
|
|
2920
2955
|
fetch: opts.fetch,
|
|
2921
2956
|
sleep: opts.sleep
|
|
2922
2957
|
};
|
|
2923
|
-
log?.uploadMetadataStart(snapshots.length);
|
|
2924
|
-
uploadResult = await uploadMetadataBatches(snapshots, uploadOpts);
|
|
2925
|
-
log?.uploadMetadataDone(uploadResult.totalIngested, uploadResult.totalConflicts);
|
|
2926
2958
|
const contentOpts = {
|
|
2927
2959
|
apiUrl: opts.apiUrl,
|
|
2928
2960
|
apiKey: opts.apiKey,
|
|
2929
2961
|
fetch: opts.fetch,
|
|
2930
2962
|
sleep: opts.sleep
|
|
2931
2963
|
};
|
|
2932
|
-
const totalSessions =
|
|
2933
|
-
|
|
2934
|
-
|
|
2935
|
-
|
|
2936
|
-
|
|
2964
|
+
const totalSessions = uploadableResults.length;
|
|
2965
|
+
const pipelineBatches = splitBatches(uploadableResults, METADATA_BATCH_SIZE);
|
|
2966
|
+
uploadResult = {
|
|
2967
|
+
totalIngested: 0,
|
|
2968
|
+
totalConflicts: 0,
|
|
2969
|
+
totalBatches: 0,
|
|
2970
|
+
errors: []
|
|
2937
2971
|
};
|
|
2938
|
-
|
|
2939
|
-
|
|
2940
|
-
|
|
2941
|
-
|
|
2942
|
-
|
|
2943
|
-
|
|
2944
|
-
|
|
2945
|
-
|
|
2946
|
-
|
|
2947
|
-
|
|
2948
|
-
|
|
2949
|
-
|
|
2950
|
-
|
|
2951
|
-
|
|
2952
|
-
}
|
|
2953
|
-
return response;
|
|
2954
|
-
};
|
|
2972
|
+
contentResult = {
|
|
2973
|
+
uploaded: 0,
|
|
2974
|
+
skipped: 0,
|
|
2975
|
+
errors: []
|
|
2976
|
+
};
|
|
2977
|
+
log?.uploadMetadataStart(totalSessions);
|
|
2978
|
+
for (const batch of pipelineBatches) {
|
|
2979
|
+
const transformed = batch.map((r) => toSessionSnapshot(r.canonical, r.raw));
|
|
2980
|
+
const batchSnapshots = transformed.map((t) => t.snapshot);
|
|
2981
|
+
const batchUploadResult = await uploadMetadataBatches(batchSnapshots, uploadOpts);
|
|
2982
|
+
uploadResult.totalIngested += batchUploadResult.totalIngested;
|
|
2983
|
+
uploadResult.totalConflicts += batchUploadResult.totalConflicts;
|
|
2984
|
+
uploadResult.totalBatches += batchUploadResult.totalBatches;
|
|
2985
|
+
uploadResult.errors.push(...batchUploadResult.errors);
|
|
2955
2986
|
}
|
|
2956
|
-
|
|
2957
|
-
|
|
2958
|
-
|
|
2959
|
-
|
|
2960
|
-
|
|
2961
|
-
|
|
2962
|
-
|
|
2963
|
-
|
|
2964
|
-
|
|
2965
|
-
|
|
2966
|
-
|
|
2967
|
-
|
|
2968
|
-
|
|
2969
|
-
|
|
2970
|
-
|
|
2971
|
-
|
|
2972
|
-
|
|
2973
|
-
|
|
2987
|
+
log?.uploadMetadataDone(uploadResult.totalIngested, uploadResult.totalConflicts);
|
|
2988
|
+
let contentDone = 0;
|
|
2989
|
+
log?.uploadContentStart(totalSessions);
|
|
2990
|
+
for (const batch of pipelineBatches) {
|
|
2991
|
+
const batchPrecomputed = new Map;
|
|
2992
|
+
for (const r of batch) {
|
|
2993
|
+
const t = toSessionSnapshot(r.canonical, r.raw);
|
|
2994
|
+
batchPrecomputed.set(t.snapshot.sessionKey, t.precomputed);
|
|
2995
|
+
}
|
|
2996
|
+
const wrappedContentOpts = { ...contentOpts };
|
|
2997
|
+
const originalFetch = contentOpts.fetch ?? globalThis.fetch;
|
|
2998
|
+
if (log) {
|
|
2999
|
+
const completedSessions = new Set;
|
|
3000
|
+
wrappedContentOpts.fetch = async (input2, init) => {
|
|
3001
|
+
const response = await originalFetch(input2, init);
|
|
3002
|
+
const url = typeof input2 === "string" ? input2 : input2.url;
|
|
3003
|
+
if (url.includes("/api/ingest/content/") && url.endsWith("/canonical")) {
|
|
3004
|
+
const parts = url.split("/");
|
|
3005
|
+
const sessionKey = decodeURIComponent(parts[parts.length - 2]);
|
|
3006
|
+
if (!completedSessions.has(sessionKey)) {
|
|
3007
|
+
completedSessions.add(sessionKey);
|
|
3008
|
+
contentDone++;
|
|
3009
|
+
log.uploadContentProgress(contentDone, totalSessions);
|
|
3010
|
+
}
|
|
3011
|
+
}
|
|
3012
|
+
return response;
|
|
3013
|
+
};
|
|
3014
|
+
}
|
|
3015
|
+
const batchContentResult = await uploadContentBatch(batch.map((r) => ({
|
|
3016
|
+
canonical: r.canonical,
|
|
3017
|
+
raw: r.raw,
|
|
3018
|
+
precomputed: batchPrecomputed.get(r.canonical.sessionKey)
|
|
3019
|
+
})), log ? wrappedContentOpts : contentOpts, opts.contentConcurrency);
|
|
3020
|
+
contentResult.uploaded += batchContentResult.uploaded;
|
|
3021
|
+
contentResult.skipped += batchContentResult.skipped;
|
|
3022
|
+
contentResult.errors.push(...batchContentResult.errors);
|
|
3023
|
+
if (batchContentResult.errors.length > 0) {
|
|
3024
|
+
const rolledBackFiles = new Set;
|
|
3025
|
+
let rollbackDbCursor = false;
|
|
3026
|
+
for (const { sessionKey } of batchContentResult.errors) {
|
|
3027
|
+
const filePath = sessionKeyToFile.get(sessionKey);
|
|
3028
|
+
if (filePath && !rolledBackFiles.has(filePath)) {
|
|
3029
|
+
rolledBackFiles.add(filePath);
|
|
3030
|
+
const prev = prevCursors.get(filePath);
|
|
3031
|
+
if (prev === undefined) {
|
|
3032
|
+
delete cursorState.files[filePath];
|
|
3033
|
+
} else {
|
|
3034
|
+
cursorState.files[filePath] = prev;
|
|
3035
|
+
}
|
|
3036
|
+
}
|
|
3037
|
+
if (dbSourcedSessionKeys.has(sessionKey)) {
|
|
3038
|
+
rollbackDbCursor = true;
|
|
2974
3039
|
}
|
|
2975
3040
|
}
|
|
2976
|
-
if (
|
|
2977
|
-
|
|
3041
|
+
if (rollbackDbCursor) {
|
|
3042
|
+
cursorState.openCodeSqlite = prevDbCursor;
|
|
2978
3043
|
}
|
|
2979
3044
|
}
|
|
2980
|
-
if (rollbackDbCursor) {
|
|
2981
|
-
cursorState.openCodeSqlite = prevDbCursor;
|
|
2982
|
-
}
|
|
2983
3045
|
}
|
|
3046
|
+
log?.uploadContentDone(contentResult.uploaded, contentResult.skipped, contentResult.errors.length);
|
|
2984
3047
|
}
|
|
2985
3048
|
return {
|
|
2986
|
-
totalParsed:
|
|
3049
|
+
totalParsed: uploadableResults.length,
|
|
3050
|
+
totalEmpty: emptyCount,
|
|
2987
3051
|
totalFiles,
|
|
2988
3052
|
totalSkipped,
|
|
2989
3053
|
parseErrors,
|
|
@@ -2993,8 +3057,9 @@ async function runSyncPipeline(input, opts) {
|
|
|
2993
3057
|
};
|
|
2994
3058
|
}
|
|
2995
3059
|
var init_sync_pipeline = __esm(() => {
|
|
2996
|
-
|
|
3060
|
+
init_src();
|
|
2997
3061
|
init_content();
|
|
3062
|
+
init_engine();
|
|
2998
3063
|
});
|
|
2999
3064
|
|
|
3000
3065
|
// src/commands/sync.ts
|
|
@@ -3003,8 +3068,8 @@ __export(exports_sync, {
|
|
|
3003
3068
|
default: () => sync_default,
|
|
3004
3069
|
buildDbDriver: () => buildDbDriver
|
|
3005
3070
|
});
|
|
3071
|
+
import { basename as basename3, join as join11 } from "path";
|
|
3006
3072
|
import { defineCommand } from "citty";
|
|
3007
|
-
import { join as join11, basename as basename3 } from "path";
|
|
3008
3073
|
import consola from "consola";
|
|
3009
3074
|
async function buildDbDriver(driverSet, openDbOverride) {
|
|
3010
3075
|
if (!driverSet.dbDriversAvailable || !driverSet.discoverOpts.openCodeDbPath) {
|
|
@@ -3024,9 +3089,9 @@ var sync_default;
|
|
|
3024
3089
|
var init_sync = __esm(() => {
|
|
3025
3090
|
init_src();
|
|
3026
3091
|
init_manager();
|
|
3027
|
-
init_cursor_store();
|
|
3028
3092
|
init_registry();
|
|
3029
3093
|
init_opencode_sqlite();
|
|
3094
|
+
init_cursor_store();
|
|
3030
3095
|
init_sync_pipeline();
|
|
3031
3096
|
sync_default = defineCommand({
|
|
3032
3097
|
meta: {
|
|
@@ -3134,7 +3199,7 @@ var init_sync = __esm(() => {
|
|
|
3134
3199
|
logger
|
|
3135
3200
|
});
|
|
3136
3201
|
await cursorStore.save(result.cursorState);
|
|
3137
|
-
consola.success(`Parsed ${result.totalParsed} session(s) from ${result.totalFiles} file(s) (${result.totalSkipped} unchanged)`);
|
|
3202
|
+
consola.success(`Parsed ${result.totalParsed} session(s) from ${result.totalFiles} file(s) (${result.totalSkipped} unchanged${result.totalEmpty > 0 ? `, ${result.totalEmpty} empty skipped` : ""})`);
|
|
3138
3203
|
if (result.uploadResult) {
|
|
3139
3204
|
consola.success(`Uploaded ${result.uploadResult.totalIngested} session(s) in ${result.uploadResult.totalBatches} batch(es)`);
|
|
3140
3205
|
if (result.uploadResult.totalConflicts > 0) {
|
|
@@ -3203,7 +3268,10 @@ function performLogin(deps) {
|
|
|
3203
3268
|
});
|
|
3204
3269
|
const timer = setTimeout(() => {
|
|
3205
3270
|
cleanup();
|
|
3206
|
-
resolve({
|
|
3271
|
+
resolve({
|
|
3272
|
+
success: false,
|
|
3273
|
+
error: "Login timeout \u2014 no response received"
|
|
3274
|
+
});
|
|
3207
3275
|
}, timeoutMs);
|
|
3208
3276
|
function cleanup() {
|
|
3209
3277
|
clearTimeout(timer);
|
|
@@ -3218,10 +3286,10 @@ var exports_login = {};
|
|
|
3218
3286
|
__export(exports_login, {
|
|
3219
3287
|
default: () => login_default
|
|
3220
3288
|
});
|
|
3289
|
+
import { homedir as homedir2, platform } from "os";
|
|
3290
|
+
import { join as join12 } from "path";
|
|
3221
3291
|
import { defineCommand as defineCommand2 } from "citty";
|
|
3222
3292
|
import consola2 from "consola";
|
|
3223
|
-
import { join as join12 } from "path";
|
|
3224
|
-
import { homedir as homedir2, platform } from "os";
|
|
3225
3293
|
function getBrowserCommand() {
|
|
3226
3294
|
switch (platform()) {
|
|
3227
3295
|
case "darwin":
|
|
@@ -3429,9 +3497,9 @@ var exports_status = {};
|
|
|
3429
3497
|
__export(exports_status, {
|
|
3430
3498
|
default: () => status_default
|
|
3431
3499
|
});
|
|
3432
|
-
import { defineCommand as defineCommand3 } from "citty";
|
|
3433
3500
|
import { readFile as readFile5 } from "fs/promises";
|
|
3434
3501
|
import { join as join13 } from "path";
|
|
3502
|
+
import { defineCommand as defineCommand3 } from "citty";
|
|
3435
3503
|
import consola3 from "consola";
|
|
3436
3504
|
var status_default;
|
|
3437
3505
|
var init_status = __esm(() => {
|