crispy-recall 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +146 -39
- package/dist/SKILL.md.template +28 -237
- package/dist/embed-pending.js +936 -269
- package/dist/push-pending.js +7958 -0
- package/dist/recall.js +7124 -1201
- package/dist/stop-hook.js +825 -134
- package/package.json +11 -5
package/dist/embed-pending.js
CHANGED
|
@@ -42,6 +42,12 @@ function recallRoot() {
|
|
|
42
42
|
return envRoot;
|
|
43
43
|
return (0, import_node_path.join)((0, import_node_os.homedir)(), ".recall");
|
|
44
44
|
}
|
|
45
|
+
function remoteRoot() {
|
|
46
|
+
const env = process.env["RECALL_REMOTE_ROOT"];
|
|
47
|
+
if (env && env.length > 0)
|
|
48
|
+
return env;
|
|
49
|
+
return (0, import_node_path.join)(recallRoot(), "remote");
|
|
50
|
+
}
|
|
45
51
|
function dbPath() {
|
|
46
52
|
return (0, import_node_path.join)(recallRoot(), "recall.db");
|
|
47
53
|
}
|
|
@@ -54,6 +60,9 @@ function binDir() {
|
|
|
54
60
|
function runDir() {
|
|
55
61
|
return (0, import_node_path.join)(recallRoot(), "run");
|
|
56
62
|
}
|
|
63
|
+
function logsDir() {
|
|
64
|
+
return (0, import_node_path.join)(recallRoot(), "logs");
|
|
65
|
+
}
|
|
57
66
|
function ensureDir() {
|
|
58
67
|
(0, import_node_fs.mkdirSync)(recallRoot(), { recursive: true });
|
|
59
68
|
}
|
|
@@ -149,7 +158,7 @@ var require_sqlite_error = __commonJS({
|
|
|
149
158
|
// node_modules/file-uri-to-path/index.js
|
|
150
159
|
var require_file_uri_to_path = __commonJS({
|
|
151
160
|
"node_modules/file-uri-to-path/index.js"(exports2, module2) {
|
|
152
|
-
var
|
|
161
|
+
var sep3 = require("path").sep || "/";
|
|
153
162
|
module2.exports = fileUriToPath;
|
|
154
163
|
function fileUriToPath(uri) {
|
|
155
164
|
if ("string" != typeof uri || uri.length <= 7 || "file://" != uri.substring(0, 7)) {
|
|
@@ -162,15 +171,15 @@ var require_file_uri_to_path = __commonJS({
|
|
|
162
171
|
if ("localhost" == host)
|
|
163
172
|
host = "";
|
|
164
173
|
if (host) {
|
|
165
|
-
host =
|
|
174
|
+
host = sep3 + sep3 + host;
|
|
166
175
|
}
|
|
167
176
|
path2 = path2.replace(/^(.+)\|/, "$1:");
|
|
168
|
-
if (
|
|
177
|
+
if (sep3 == "\\") {
|
|
169
178
|
path2 = path2.replace(/\//g, "\\");
|
|
170
179
|
}
|
|
171
180
|
if (/^.+\:/.test(path2)) {
|
|
172
181
|
} else {
|
|
173
|
-
path2 =
|
|
182
|
+
path2 = sep3 + path2;
|
|
174
183
|
}
|
|
175
184
|
return host + path2;
|
|
176
185
|
}
|
|
@@ -183,8 +192,8 @@ var require_bindings = __commonJS({
|
|
|
183
192
|
var fs3 = require("fs");
|
|
184
193
|
var path2 = require("path");
|
|
185
194
|
var fileURLToPath3 = require_file_uri_to_path();
|
|
186
|
-
var
|
|
187
|
-
var
|
|
195
|
+
var join9 = path2.join;
|
|
196
|
+
var dirname4 = path2.dirname;
|
|
188
197
|
var exists = fs3.accessSync && function(path3) {
|
|
189
198
|
try {
|
|
190
199
|
fs3.accessSync(path3);
|
|
@@ -244,7 +253,7 @@ var require_bindings = __commonJS({
|
|
|
244
253
|
var requireFunc = typeof __webpack_require__ === "function" ? __non_webpack_require__ : require;
|
|
245
254
|
var tries = [], i = 0, l = opts.try.length, n, b, err;
|
|
246
255
|
for (; i < l; i++) {
|
|
247
|
-
n =
|
|
256
|
+
n = join9.apply(
|
|
248
257
|
null,
|
|
249
258
|
opts.try[i].map(function(p) {
|
|
250
259
|
return opts[p] || p;
|
|
@@ -300,12 +309,12 @@ var require_bindings = __commonJS({
|
|
|
300
309
|
return fileName;
|
|
301
310
|
};
|
|
302
311
|
exports2.getRoot = function getRoot(file) {
|
|
303
|
-
var dir =
|
|
312
|
+
var dir = dirname4(file), prev;
|
|
304
313
|
while (true) {
|
|
305
314
|
if (dir === ".") {
|
|
306
315
|
dir = process.cwd();
|
|
307
316
|
}
|
|
308
|
-
if (exists(
|
|
317
|
+
if (exists(join9(dir, "package.json")) || exists(join9(dir, "node_modules"))) {
|
|
309
318
|
return dir;
|
|
310
319
|
}
|
|
311
320
|
if (prev === dir) {
|
|
@@ -314,7 +323,7 @@ var require_bindings = __commonJS({
|
|
|
314
323
|
);
|
|
315
324
|
}
|
|
316
325
|
prev = dir;
|
|
317
|
-
dir =
|
|
326
|
+
dir = join9(dir, "..");
|
|
318
327
|
}
|
|
319
328
|
};
|
|
320
329
|
}
|
|
@@ -515,13 +524,13 @@ var require_backup = __commonJS({
|
|
|
515
524
|
var runBackup = (backup, handler) => {
|
|
516
525
|
let rate = 0;
|
|
517
526
|
let useDefault = true;
|
|
518
|
-
return new Promise((
|
|
527
|
+
return new Promise((resolve3, reject) => {
|
|
519
528
|
setImmediate(function step() {
|
|
520
529
|
try {
|
|
521
530
|
const progress = backup.transfer(rate);
|
|
522
531
|
if (!progress.remainingPages) {
|
|
523
532
|
backup.close();
|
|
524
|
-
|
|
533
|
+
resolve3(progress);
|
|
525
534
|
return;
|
|
526
535
|
}
|
|
527
536
|
if (useDefault) {
|
|
@@ -1292,10 +1301,10 @@ var require_browser = __commonJS({
|
|
|
1292
1301
|
exports2.useColors = useColors;
|
|
1293
1302
|
exports2.storage = localstorage();
|
|
1294
1303
|
exports2.destroy = /* @__PURE__ */ (() => {
|
|
1295
|
-
let
|
|
1304
|
+
let warned3 = false;
|
|
1296
1305
|
return () => {
|
|
1297
|
-
if (!
|
|
1298
|
-
|
|
1306
|
+
if (!warned3) {
|
|
1307
|
+
warned3 = true;
|
|
1299
1308
|
console.warn("Instance method `debug.destroy()` is deprecated and no longer does anything. It will be removed in the next major version of `debug`.");
|
|
1300
1309
|
}
|
|
1301
1310
|
};
|
|
@@ -1966,7 +1975,7 @@ var require_get_stream = __commonJS({
|
|
|
1966
1975
|
};
|
|
1967
1976
|
const { maxBuffer } = options;
|
|
1968
1977
|
let stream2;
|
|
1969
|
-
await new Promise((
|
|
1978
|
+
await new Promise((resolve3, reject) => {
|
|
1970
1979
|
const rejectPromise = (error) => {
|
|
1971
1980
|
if (error && stream2.getBufferedLength() <= BufferConstants.MAX_LENGTH) {
|
|
1972
1981
|
error.bufferedData = stream2.getBufferedValue();
|
|
@@ -1978,7 +1987,7 @@ var require_get_stream = __commonJS({
|
|
|
1978
1987
|
rejectPromise(error);
|
|
1979
1988
|
return;
|
|
1980
1989
|
}
|
|
1981
|
-
|
|
1990
|
+
resolve3();
|
|
1982
1991
|
});
|
|
1983
1992
|
stream2.on("data", () => {
|
|
1984
1993
|
if (stream2.getBufferedLength() > maxBuffer) {
|
|
@@ -3349,7 +3358,7 @@ var require_extract_zip = __commonJS({
|
|
|
3349
3358
|
debug("opening", this.zipPath, "with opts", this.opts);
|
|
3350
3359
|
this.zipfile = await openZip(this.zipPath, { lazyEntries: true });
|
|
3351
3360
|
this.canceled = false;
|
|
3352
|
-
return new Promise((
|
|
3361
|
+
return new Promise((resolve3, reject) => {
|
|
3353
3362
|
this.zipfile.on("error", (err) => {
|
|
3354
3363
|
this.canceled = true;
|
|
3355
3364
|
reject(err);
|
|
@@ -3358,7 +3367,7 @@ var require_extract_zip = __commonJS({
|
|
|
3358
3367
|
this.zipfile.on("close", () => {
|
|
3359
3368
|
if (!this.canceled) {
|
|
3360
3369
|
debug("zip extraction complete");
|
|
3361
|
-
|
|
3370
|
+
resolve3();
|
|
3362
3371
|
}
|
|
3363
3372
|
});
|
|
3364
3373
|
this.zipfile.on("entry", async (entry) => {
|
|
@@ -3485,14 +3494,14 @@ var init_settings_merge = __esm({
|
|
|
3485
3494
|
|
|
3486
3495
|
// src/installer/config.ts
|
|
3487
3496
|
function configPath() {
|
|
3488
|
-
return (0,
|
|
3497
|
+
return (0, import_node_path7.join)(recallRoot(), "config.json");
|
|
3489
3498
|
}
|
|
3490
3499
|
function readConfig() {
|
|
3491
3500
|
const p = configPath();
|
|
3492
|
-
if (!(0,
|
|
3501
|
+
if (!(0, import_node_fs7.existsSync)(p))
|
|
3493
3502
|
return null;
|
|
3494
3503
|
try {
|
|
3495
|
-
return JSON.parse((0,
|
|
3504
|
+
return JSON.parse((0, import_node_fs7.readFileSync)(p, "utf-8"));
|
|
3496
3505
|
} catch (err) {
|
|
3497
3506
|
log({
|
|
3498
3507
|
source: "installer/config",
|
|
@@ -3508,12 +3517,12 @@ function readEmbedderConfig() {
|
|
|
3508
3517
|
return cfg.embedder;
|
|
3509
3518
|
return { mode: "cpu", ngl: 0, libDir: null, detectedAt: "" };
|
|
3510
3519
|
}
|
|
3511
|
-
var
|
|
3520
|
+
var import_node_fs7, import_node_path7;
|
|
3512
3521
|
var init_config = __esm({
|
|
3513
3522
|
"src/installer/config.ts"() {
|
|
3514
3523
|
"use strict";
|
|
3515
|
-
|
|
3516
|
-
|
|
3524
|
+
import_node_fs7 = require("node:fs");
|
|
3525
|
+
import_node_path7 = require("node:path");
|
|
3517
3526
|
init_paths();
|
|
3518
3527
|
init_log();
|
|
3519
3528
|
init_settings_merge();
|
|
@@ -3547,13 +3556,13 @@ function gpuRuntime() {
|
|
|
3547
3556
|
}
|
|
3548
3557
|
return _gpuRuntime;
|
|
3549
3558
|
}
|
|
3550
|
-
function withGpuLibDir(libDir,
|
|
3559
|
+
function withGpuLibDir(libDir, sep3) {
|
|
3551
3560
|
const gpu = gpuRuntime();
|
|
3552
|
-
return gpu?.libDir ? `${gpu.libDir}${
|
|
3561
|
+
return gpu?.libDir ? `${gpu.libDir}${sep3}${libDir}` : libDir;
|
|
3553
3562
|
}
|
|
3554
3563
|
async function withEmbedMutex(fn) {
|
|
3555
|
-
return new Promise((
|
|
3556
|
-
embedQueue.push({ resolve, reject, fn });
|
|
3564
|
+
return new Promise((resolve3, reject) => {
|
|
3565
|
+
embedQueue.push({ resolve: resolve3, reject, fn });
|
|
3557
3566
|
drainEmbedQueue();
|
|
3558
3567
|
});
|
|
3559
3568
|
}
|
|
@@ -3576,10 +3585,10 @@ function initEmbedder(binPath) {
|
|
|
3576
3585
|
binaryPath = binPath;
|
|
3577
3586
|
}
|
|
3578
3587
|
function getBinaryPath() {
|
|
3579
|
-
return (0,
|
|
3588
|
+
return (0, import_node_path8.join)(binDir(), BIN_NAME);
|
|
3580
3589
|
}
|
|
3581
3590
|
function getServerBinaryPath() {
|
|
3582
|
-
return (0,
|
|
3591
|
+
return (0, import_node_path8.join)(binDir(), SERVER_BIN_NAME);
|
|
3583
3592
|
}
|
|
3584
3593
|
async function hasNvidiaGpu() {
|
|
3585
3594
|
try {
|
|
@@ -3590,8 +3599,8 @@ async function hasNvidiaGpu() {
|
|
|
3590
3599
|
}
|
|
3591
3600
|
}
|
|
3592
3601
|
async function getBinaryAssetCandidates() {
|
|
3593
|
-
const p = (0,
|
|
3594
|
-
const a = (0,
|
|
3602
|
+
const p = (0, import_node_os4.platform)();
|
|
3603
|
+
const a = (0, import_node_os4.arch)();
|
|
3595
3604
|
const tag = LLAMA_RELEASE_TAG;
|
|
3596
3605
|
if (p === "linux" && a === "x64")
|
|
3597
3606
|
return [`llama-${tag}-bin-ubuntu-x64.zip`];
|
|
@@ -3615,8 +3624,8 @@ async function getBinaryAssetCandidates() {
|
|
|
3615
3624
|
async function ensureBinary() {
|
|
3616
3625
|
const binPath = getBinaryPath();
|
|
3617
3626
|
const serverBinPath = getServerBinaryPath();
|
|
3618
|
-
const embeddingExists = (0,
|
|
3619
|
-
const serverNeeded = SERVER_SUPPORTED && !(0,
|
|
3627
|
+
const embeddingExists = (0, import_node_fs8.existsSync)(binPath);
|
|
3628
|
+
const serverNeeded = SERVER_SUPPORTED && !(0, import_node_fs8.existsSync)(serverBinPath);
|
|
3620
3629
|
if (embeddingExists && !serverNeeded) {
|
|
3621
3630
|
binaryPath = binPath;
|
|
3622
3631
|
return binPath;
|
|
@@ -3639,11 +3648,11 @@ function isWantedFile(name) {
|
|
|
3639
3648
|
}
|
|
3640
3649
|
async function validateBinary(binPath) {
|
|
3641
3650
|
try {
|
|
3642
|
-
const libDir = (0,
|
|
3651
|
+
const libDir = (0, import_node_path8.join)(binPath, "..");
|
|
3643
3652
|
const env = { ...process.env };
|
|
3644
|
-
if ((0,
|
|
3653
|
+
if ((0, import_node_os4.platform)() === "win32") {
|
|
3645
3654
|
env.PATH = `${libDir};${process.env.PATH || ""}`;
|
|
3646
|
-
} else if ((0,
|
|
3655
|
+
} else if ((0, import_node_os4.platform)() === "darwin") {
|
|
3647
3656
|
env.DYLD_LIBRARY_PATH = libDir;
|
|
3648
3657
|
} else {
|
|
3649
3658
|
env.LD_LIBRARY_PATH = libDir;
|
|
@@ -3659,11 +3668,11 @@ async function validateBinary(binPath) {
|
|
|
3659
3668
|
}
|
|
3660
3669
|
}
|
|
3661
3670
|
async function clearBinDir() {
|
|
3662
|
-
if (!(0,
|
|
3671
|
+
if (!(0, import_node_fs8.existsSync)(binDir()))
|
|
3663
3672
|
return;
|
|
3664
3673
|
for (const file of await (0, import_promises.readdir)(binDir())) {
|
|
3665
3674
|
try {
|
|
3666
|
-
(0,
|
|
3675
|
+
(0, import_node_fs8.unlinkSync)((0, import_node_path8.join)(binDir(), file));
|
|
3667
3676
|
} catch {
|
|
3668
3677
|
}
|
|
3669
3678
|
}
|
|
@@ -3746,8 +3755,8 @@ function buildUnzipInvocations(plat, archivePath, dir) {
|
|
|
3746
3755
|
return plat === "darwin" ? [unzip, { cmd: "ditto", args: ["-x", "-k", archivePath, dir] }] : [unzip];
|
|
3747
3756
|
}
|
|
3748
3757
|
async function osUnzip(archivePath, dir) {
|
|
3749
|
-
(0,
|
|
3750
|
-
const invocations = buildUnzipInvocations((0,
|
|
3758
|
+
(0, import_node_fs8.mkdirSync)(dir, { recursive: true });
|
|
3759
|
+
const invocations = buildUnzipInvocations((0, import_node_os4.platform)(), archivePath, dir);
|
|
3751
3760
|
let lastErr;
|
|
3752
3761
|
for (const { cmd, args } of invocations) {
|
|
3753
3762
|
try {
|
|
@@ -3770,20 +3779,20 @@ async function extractArchive(archivePath, baseDir, assetName, timeoutMs = EXTRA
|
|
|
3770
3779
|
level: "warn",
|
|
3771
3780
|
summary: `Bundled unzip failed for ${assetName} (${msg}); falling back to system unzip`
|
|
3772
3781
|
});
|
|
3773
|
-
const fallbackDir = (0,
|
|
3782
|
+
const fallbackDir = (0, import_node_path8.join)(baseDir, "__os_unzip__");
|
|
3774
3783
|
await osUnzip(archivePath, fallbackDir);
|
|
3775
3784
|
return fallbackDir;
|
|
3776
3785
|
}
|
|
3777
3786
|
}
|
|
3778
3787
|
async function installExtractedBinaries(extractedDir, destDir = binDir()) {
|
|
3779
|
-
const buildBinDir = (0,
|
|
3780
|
-
const sourceDir = (0,
|
|
3781
|
-
(0,
|
|
3788
|
+
const buildBinDir = (0, import_node_path8.join)(extractedDir, "build", "bin");
|
|
3789
|
+
const sourceDir = (0, import_node_fs8.existsSync)(buildBinDir) ? buildBinDir : extractedDir;
|
|
3790
|
+
(0, import_node_fs8.mkdirSync)(destDir, { recursive: true });
|
|
3782
3791
|
let copiedCount = 0;
|
|
3783
3792
|
for (const file of await (0, import_promises.readdir)(sourceDir)) {
|
|
3784
3793
|
if (!isWantedFile(file))
|
|
3785
3794
|
continue;
|
|
3786
|
-
await (0, import_promises.cp)((0,
|
|
3795
|
+
await (0, import_promises.cp)((0, import_node_path8.join)(sourceDir, file), (0, import_node_path8.join)(destDir, file), { force: true });
|
|
3787
3796
|
copiedCount++;
|
|
3788
3797
|
}
|
|
3789
3798
|
return copiedCount;
|
|
@@ -3791,8 +3800,8 @@ async function installExtractedBinaries(extractedDir, destDir = binDir()) {
|
|
|
3791
3800
|
async function downloadAndExtract(assetName, binPath) {
|
|
3792
3801
|
const url = `https://github.com/ggml-org/llama.cpp/releases/download/${LLAMA_RELEASE_TAG}/${assetName}`;
|
|
3793
3802
|
const isGpu = assetName.includes("cuda") || assetName.includes("vulkan");
|
|
3794
|
-
(0,
|
|
3795
|
-
const archivePath = (0,
|
|
3803
|
+
(0, import_node_fs8.mkdirSync)(binDir(), { recursive: true });
|
|
3804
|
+
const archivePath = (0, import_node_path8.join)(binDir(), assetName);
|
|
3796
3805
|
const tmpPath = archivePath + ".tmp";
|
|
3797
3806
|
log({
|
|
3798
3807
|
source: "recall-catchup",
|
|
@@ -3801,12 +3810,12 @@ async function downloadAndExtract(assetName, binPath) {
|
|
|
3801
3810
|
data: { url, dest: binPath }
|
|
3802
3811
|
});
|
|
3803
3812
|
try {
|
|
3804
|
-
if ((0,
|
|
3805
|
-
(0,
|
|
3813
|
+
if ((0, import_node_fs8.existsSync)(tmpPath))
|
|
3814
|
+
(0, import_node_fs8.unlinkSync)(tmpPath);
|
|
3806
3815
|
await downloadFile(url, tmpPath);
|
|
3807
|
-
(0,
|
|
3808
|
-
const tmpExtractDir = (0,
|
|
3809
|
-
(0,
|
|
3816
|
+
(0, import_node_fs8.renameSync)(tmpPath, archivePath);
|
|
3817
|
+
const tmpExtractDir = (0, import_node_path8.join)((0, import_node_os4.tmpdir)(), `llama-extract-${Date.now()}`);
|
|
3818
|
+
(0, import_node_fs8.mkdirSync)(tmpExtractDir, { recursive: true });
|
|
3810
3819
|
try {
|
|
3811
3820
|
const extractedDir = await extractArchive(archivePath, tmpExtractDir, assetName);
|
|
3812
3821
|
const copiedCount = await installExtractedBinaries(extractedDir);
|
|
@@ -3815,32 +3824,32 @@ async function downloadAndExtract(assetName, binPath) {
|
|
|
3815
3824
|
}
|
|
3816
3825
|
} finally {
|
|
3817
3826
|
try {
|
|
3818
|
-
(0,
|
|
3827
|
+
(0, import_node_fs8.rmSync)(tmpExtractDir, { recursive: true, force: true });
|
|
3819
3828
|
} catch {
|
|
3820
3829
|
}
|
|
3821
3830
|
}
|
|
3822
|
-
if ((0,
|
|
3831
|
+
if ((0, import_node_fs8.existsSync)(archivePath)) {
|
|
3823
3832
|
try {
|
|
3824
|
-
(0,
|
|
3833
|
+
(0, import_node_fs8.unlinkSync)(archivePath);
|
|
3825
3834
|
} catch {
|
|
3826
3835
|
}
|
|
3827
3836
|
}
|
|
3828
|
-
if ((0,
|
|
3829
|
-
if ((0,
|
|
3830
|
-
(0,
|
|
3837
|
+
if ((0, import_node_os4.platform)() !== "win32") {
|
|
3838
|
+
if ((0, import_node_fs8.existsSync)(binPath))
|
|
3839
|
+
(0, import_node_fs8.chmodSync)(binPath, 493);
|
|
3831
3840
|
const serverBin = getServerBinaryPath();
|
|
3832
|
-
if ((0,
|
|
3833
|
-
(0,
|
|
3841
|
+
if ((0, import_node_fs8.existsSync)(serverBin)) {
|
|
3842
|
+
(0, import_node_fs8.chmodSync)(serverBin, 493);
|
|
3834
3843
|
}
|
|
3835
3844
|
}
|
|
3836
|
-
if (!(0,
|
|
3845
|
+
if (!(0, import_node_fs8.existsSync)(binPath)) {
|
|
3837
3846
|
throw new Error(`${BIN_NAME} not found after extracting ${assetName}`);
|
|
3838
3847
|
}
|
|
3839
3848
|
} catch (err) {
|
|
3840
3849
|
for (const p of [tmpPath, archivePath]) {
|
|
3841
|
-
if ((0,
|
|
3850
|
+
if ((0, import_node_fs8.existsSync)(p)) {
|
|
3842
3851
|
try {
|
|
3843
|
-
(0,
|
|
3852
|
+
(0, import_node_fs8.unlinkSync)(p);
|
|
3844
3853
|
} catch {
|
|
3845
3854
|
}
|
|
3846
3855
|
}
|
|
@@ -3849,12 +3858,12 @@ async function downloadAndExtract(assetName, binPath) {
|
|
|
3849
3858
|
}
|
|
3850
3859
|
}
|
|
3851
3860
|
function getModelPath() {
|
|
3852
|
-
return (0,
|
|
3861
|
+
return (0, import_node_path8.join)(modelsDir(), MODEL_FILENAME);
|
|
3853
3862
|
}
|
|
3854
3863
|
async function ensureModel() {
|
|
3855
3864
|
const modelPath = getModelPath();
|
|
3856
|
-
if ((0,
|
|
3857
|
-
const stat = (0,
|
|
3865
|
+
if ((0, import_node_fs8.existsSync)(modelPath)) {
|
|
3866
|
+
const stat = (0, import_node_fs8.statSync)(modelPath);
|
|
3858
3867
|
if (stat.size > 1e8)
|
|
3859
3868
|
return modelPath;
|
|
3860
3869
|
}
|
|
@@ -3870,9 +3879,9 @@ async function ensureModel() {
|
|
|
3870
3879
|
async function performModelDownload(modelPath) {
|
|
3871
3880
|
const tmpPath = modelPath + ".tmp";
|
|
3872
3881
|
try {
|
|
3873
|
-
(0,
|
|
3874
|
-
if ((0,
|
|
3875
|
-
(0,
|
|
3882
|
+
(0, import_node_fs8.mkdirSync)(modelsDir(), { recursive: true });
|
|
3883
|
+
if ((0, import_node_fs8.existsSync)(tmpPath)) {
|
|
3884
|
+
(0, import_node_fs8.unlinkSync)(tmpPath);
|
|
3876
3885
|
}
|
|
3877
3886
|
log({
|
|
3878
3887
|
source: "recall-catchup",
|
|
@@ -3881,7 +3890,7 @@ async function performModelDownload(modelPath) {
|
|
|
3881
3890
|
data: { url: MODEL_URL, dest: modelPath }
|
|
3882
3891
|
});
|
|
3883
3892
|
await downloadFile(MODEL_URL, tmpPath);
|
|
3884
|
-
(0,
|
|
3893
|
+
(0, import_node_fs8.renameSync)(tmpPath, modelPath);
|
|
3885
3894
|
log({
|
|
3886
3895
|
source: "recall-catchup",
|
|
3887
3896
|
level: "info",
|
|
@@ -3889,9 +3898,9 @@ async function performModelDownload(modelPath) {
|
|
|
3889
3898
|
});
|
|
3890
3899
|
return modelPath;
|
|
3891
3900
|
} catch (err) {
|
|
3892
|
-
if ((0,
|
|
3901
|
+
if ((0, import_node_fs8.existsSync)(tmpPath)) {
|
|
3893
3902
|
try {
|
|
3894
|
-
(0,
|
|
3903
|
+
(0, import_node_fs8.unlinkSync)(tmpPath);
|
|
3895
3904
|
} catch {
|
|
3896
3905
|
}
|
|
3897
3906
|
}
|
|
@@ -3900,7 +3909,7 @@ async function performModelDownload(modelPath) {
|
|
|
3900
3909
|
}
|
|
3901
3910
|
async function downloadFile(url, destPath) {
|
|
3902
3911
|
const DOWNLOAD_TIMEOUT_MS = 5 * 60 * 1e3;
|
|
3903
|
-
return new Promise((
|
|
3912
|
+
return new Promise((resolve3, reject) => {
|
|
3904
3913
|
let timeoutHandle = null;
|
|
3905
3914
|
const cleanup = () => {
|
|
3906
3915
|
if (timeoutHandle) {
|
|
@@ -3909,7 +3918,7 @@ async function downloadFile(url, destPath) {
|
|
|
3909
3918
|
}
|
|
3910
3919
|
};
|
|
3911
3920
|
import("node:https").then(({ default: https }) => {
|
|
3912
|
-
const file = (0,
|
|
3921
|
+
const file = (0, import_node_fs8.createWriteStream)(destPath);
|
|
3913
3922
|
const pipeResponse = (response) => {
|
|
3914
3923
|
if (response.statusCode !== 200) {
|
|
3915
3924
|
cleanup();
|
|
@@ -3929,7 +3938,7 @@ async function downloadFile(url, destPath) {
|
|
|
3929
3938
|
file.on("finish", () => {
|
|
3930
3939
|
cleanup();
|
|
3931
3940
|
file.close();
|
|
3932
|
-
|
|
3941
|
+
resolve3();
|
|
3933
3942
|
});
|
|
3934
3943
|
file.on("error", (err) => {
|
|
3935
3944
|
cleanup();
|
|
@@ -3973,13 +3982,13 @@ function isProcessAlive(pid) {
|
|
|
3973
3982
|
}
|
|
3974
3983
|
}
|
|
3975
3984
|
function getSocketPath() {
|
|
3976
|
-
return (0,
|
|
3985
|
+
return (0, import_node_path8.join)(runDir(), `llama-embed-${process.pid}.sock`);
|
|
3977
3986
|
}
|
|
3978
3987
|
function getPidFilePath() {
|
|
3979
|
-
return (0,
|
|
3988
|
+
return (0, import_node_path8.join)(runDir(), `llama-embed-${process.pid}.pid`);
|
|
3980
3989
|
}
|
|
3981
3990
|
function writePidFile(pid, socketPath) {
|
|
3982
|
-
(0,
|
|
3991
|
+
(0, import_node_fs8.writeFileSync)(getPidFilePath(), JSON.stringify({
|
|
3983
3992
|
pid,
|
|
3984
3993
|
socketPath,
|
|
3985
3994
|
startedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
@@ -3989,20 +3998,20 @@ function writePidFile(pid, socketPath) {
|
|
|
3989
3998
|
function cleanupPidAndSocket() {
|
|
3990
3999
|
for (const f of [getPidFilePath(), getSocketPath()]) {
|
|
3991
4000
|
try {
|
|
3992
|
-
(0,
|
|
4001
|
+
(0, import_node_fs8.unlinkSync)(f);
|
|
3993
4002
|
} catch {
|
|
3994
4003
|
}
|
|
3995
4004
|
}
|
|
3996
4005
|
}
|
|
3997
4006
|
function cleanupStalePidFiles() {
|
|
3998
|
-
if (!(0,
|
|
4007
|
+
if (!(0, import_node_fs8.existsSync)(runDir()))
|
|
3999
4008
|
return;
|
|
4000
4009
|
try {
|
|
4001
|
-
const files = (0,
|
|
4010
|
+
const files = (0, import_node_fs8.readdirSync)(runDir()).filter((f) => f.startsWith("llama-embed-") && f.endsWith(".pid"));
|
|
4002
4011
|
for (const f of files) {
|
|
4003
|
-
const pidFile = (0,
|
|
4012
|
+
const pidFile = (0, import_node_path8.join)(runDir(), f);
|
|
4004
4013
|
try {
|
|
4005
|
-
const data = JSON.parse((0,
|
|
4014
|
+
const data = JSON.parse((0, import_node_fs8.readFileSync)(pidFile, "utf-8"));
|
|
4006
4015
|
const ownerPid = data.ownerPid ?? data.pid;
|
|
4007
4016
|
if (!isProcessAlive(ownerPid)) {
|
|
4008
4017
|
if (data.pid && isProcessAlive(data.pid)) {
|
|
@@ -4012,19 +4021,19 @@ function cleanupStalePidFiles() {
|
|
|
4012
4021
|
}
|
|
4013
4022
|
}
|
|
4014
4023
|
try {
|
|
4015
|
-
(0,
|
|
4024
|
+
(0, import_node_fs8.unlinkSync)(pidFile);
|
|
4016
4025
|
} catch {
|
|
4017
4026
|
}
|
|
4018
4027
|
if (data.socketPath) {
|
|
4019
4028
|
try {
|
|
4020
|
-
(0,
|
|
4029
|
+
(0, import_node_fs8.unlinkSync)(data.socketPath);
|
|
4021
4030
|
} catch {
|
|
4022
4031
|
}
|
|
4023
4032
|
}
|
|
4024
4033
|
}
|
|
4025
4034
|
} catch {
|
|
4026
4035
|
try {
|
|
4027
|
-
(0,
|
|
4036
|
+
(0, import_node_fs8.unlinkSync)(pidFile);
|
|
4028
4037
|
} catch {
|
|
4029
4038
|
}
|
|
4030
4039
|
}
|
|
@@ -4033,17 +4042,17 @@ function cleanupStalePidFiles() {
|
|
|
4033
4042
|
}
|
|
4034
4043
|
}
|
|
4035
4044
|
function healthCheck(socketPath) {
|
|
4036
|
-
return new Promise((
|
|
4045
|
+
return new Promise((resolve3) => {
|
|
4037
4046
|
const req = (0, import_node_http.request)(
|
|
4038
4047
|
{ socketPath, path: "/health", method: "GET", timeout: 2e3 },
|
|
4039
4048
|
(res) => {
|
|
4040
|
-
|
|
4049
|
+
resolve3(res.statusCode === 200);
|
|
4041
4050
|
}
|
|
4042
4051
|
);
|
|
4043
|
-
req.on("error", () =>
|
|
4052
|
+
req.on("error", () => resolve3(false));
|
|
4044
4053
|
req.on("timeout", () => {
|
|
4045
4054
|
req.destroy();
|
|
4046
|
-
|
|
4055
|
+
resolve3(false);
|
|
4047
4056
|
});
|
|
4048
4057
|
req.end();
|
|
4049
4058
|
});
|
|
@@ -4063,22 +4072,22 @@ async function waitForHealth(socketPath, child) {
|
|
|
4063
4072
|
async function startServer() {
|
|
4064
4073
|
const modelPath = await ensureModel();
|
|
4065
4074
|
const serverBin = getServerBinaryPath();
|
|
4066
|
-
if (!(0,
|
|
4075
|
+
if (!(0, import_node_fs8.existsSync)(serverBin)) {
|
|
4067
4076
|
throw new Error("llama-server binary not available");
|
|
4068
4077
|
}
|
|
4069
|
-
(0,
|
|
4078
|
+
(0, import_node_fs8.mkdirSync)(runDir(), { recursive: true });
|
|
4070
4079
|
const socket = getSocketPath();
|
|
4071
|
-
if ((0,
|
|
4080
|
+
if ((0, import_node_fs8.existsSync)(socket)) {
|
|
4072
4081
|
try {
|
|
4073
|
-
(0,
|
|
4082
|
+
(0, import_node_fs8.unlinkSync)(socket);
|
|
4074
4083
|
} catch {
|
|
4075
4084
|
}
|
|
4076
4085
|
}
|
|
4077
|
-
const libDir = (0,
|
|
4078
|
-
const envKey = (0,
|
|
4086
|
+
const libDir = (0, import_node_path8.join)(serverBin, "..");
|
|
4087
|
+
const envKey = (0, import_node_os4.platform)() === "darwin" ? "DYLD_LIBRARY_PATH" : "LD_LIBRARY_PATH";
|
|
4079
4088
|
const gpu = gpuRuntime();
|
|
4080
4089
|
const nglArgs = gpu ? ["-ngl", String(gpu.ngl)] : [];
|
|
4081
|
-
const child = (0,
|
|
4090
|
+
const child = (0, import_node_child_process2.spawn)(serverBin, [
|
|
4082
4091
|
"-m",
|
|
4083
4092
|
modelPath,
|
|
4084
4093
|
"--embeddings",
|
|
@@ -4159,20 +4168,20 @@ async function killServer() {
|
|
|
4159
4168
|
cleanupPidAndSocket();
|
|
4160
4169
|
return;
|
|
4161
4170
|
}
|
|
4162
|
-
return new Promise((
|
|
4171
|
+
return new Promise((resolve3) => {
|
|
4163
4172
|
const forceKillTimer = setTimeout(() => {
|
|
4164
4173
|
try {
|
|
4165
4174
|
child.kill("SIGKILL");
|
|
4166
4175
|
} catch {
|
|
4167
4176
|
}
|
|
4168
4177
|
cleanupPidAndSocket();
|
|
4169
|
-
|
|
4178
|
+
resolve3();
|
|
4170
4179
|
}, SERVER_KILL_TIMEOUT_MS);
|
|
4171
4180
|
forceKillTimer.unref();
|
|
4172
4181
|
child.once("exit", () => {
|
|
4173
4182
|
clearTimeout(forceKillTimer);
|
|
4174
4183
|
cleanupPidAndSocket();
|
|
4175
|
-
|
|
4184
|
+
resolve3();
|
|
4176
4185
|
});
|
|
4177
4186
|
try {
|
|
4178
4187
|
child.kill("SIGTERM");
|
|
@@ -4231,7 +4240,7 @@ async function retryServer() {
|
|
|
4231
4240
|
}
|
|
4232
4241
|
}
|
|
4233
4242
|
function httpPost(socketPath, path2, body, timeoutMs) {
|
|
4234
|
-
return new Promise((
|
|
4243
|
+
return new Promise((resolve3, reject) => {
|
|
4235
4244
|
const data = JSON.stringify(body);
|
|
4236
4245
|
const req = (0, import_node_http.request)(
|
|
4237
4246
|
{
|
|
@@ -4248,7 +4257,7 @@ function httpPost(socketPath, path2, body, timeoutMs) {
|
|
|
4248
4257
|
const chunks = [];
|
|
4249
4258
|
res.on("data", (chunk) => chunks.push(chunk));
|
|
4250
4259
|
res.on("end", () => {
|
|
4251
|
-
|
|
4260
|
+
resolve3({ status: res.statusCode ?? 0, body: Buffer.concat(chunks).toString() });
|
|
4252
4261
|
});
|
|
4253
4262
|
res.on("error", reject);
|
|
4254
4263
|
}
|
|
@@ -4322,16 +4331,16 @@ async function embedViaProcess(texts, modelPath, opts) {
|
|
|
4322
4331
|
args.push("-ngl", String(gpu.ngl));
|
|
4323
4332
|
args.push("--embd-separator", BATCH_SEPARATOR);
|
|
4324
4333
|
if (useFile) {
|
|
4325
|
-
tmpFile = (0,
|
|
4334
|
+
tmpFile = (0, import_node_path8.join)((0, import_node_os4.tmpdir)(), `recall-embed-${Date.now()}-${Math.random().toString(36).slice(2)}.txt`);
|
|
4326
4335
|
await (0, import_promises.writeFile)(tmpFile, joined, "utf-8");
|
|
4327
4336
|
args.push("-f", tmpFile);
|
|
4328
4337
|
} else {
|
|
4329
4338
|
args.push("-p", joined);
|
|
4330
4339
|
}
|
|
4331
|
-
const libDir = (0,
|
|
4332
|
-
const envKey = (0,
|
|
4340
|
+
const libDir = (0, import_node_path8.join)(binaryPath, "..");
|
|
4341
|
+
const envKey = (0, import_node_os4.platform)() === "darwin" ? "DYLD_LIBRARY_PATH" : (0, import_node_os4.platform)() === "win32" ? "PATH" : "LD_LIBRARY_PATH";
|
|
4333
4342
|
const env = { ...process.env };
|
|
4334
|
-
if ((0,
|
|
4343
|
+
if ((0, import_node_os4.platform)() === "win32") {
|
|
4335
4344
|
env.PATH = withGpuLibDir(`${libDir};${process.env.PATH || ""}`, ";");
|
|
4336
4345
|
} else {
|
|
4337
4346
|
env[envKey] = withGpuLibDir(libDir, ":");
|
|
@@ -4339,6 +4348,13 @@ async function embedViaProcess(texts, modelPath, opts) {
|
|
|
4339
4348
|
const { stdout, stderr } = await execFileAsync(binaryPath, args, {
|
|
4340
4349
|
maxBuffer: 2 * 1024 * 1024,
|
|
4341
4350
|
env,
|
|
4351
|
+
// llama-embedding.exe is a console program. The Stop hook spawns
|
|
4352
|
+
// embed-pending detached (DETACHED_PROCESS), so that parent holds no
|
|
4353
|
+
// console for this child to inherit — without CREATE_NO_WINDOW, Windows
|
|
4354
|
+
// allocates a fresh console and a terminal window flashes on screen once
|
|
4355
|
+
// per batch, at the end of every turn. Server mode is off on Windows, so
|
|
4356
|
+
// this one-shot path runs for every embed there.
|
|
4357
|
+
windowsHide: true,
|
|
4342
4358
|
...opts?.timeoutMs ? { timeout: opts.timeoutMs, killSignal: "SIGKILL" } : {}
|
|
4343
4359
|
});
|
|
4344
4360
|
if (stderr) {
|
|
@@ -4447,32 +4463,32 @@ async function embedBatchInner(texts) {
|
|
|
4447
4463
|
async function disposeEmbedder() {
|
|
4448
4464
|
await killServer();
|
|
4449
4465
|
}
|
|
4450
|
-
var
|
|
4466
|
+
var import_node_child_process2, import_node_fs8, import_promises, import_node_http, import_node_path8, import_node_os4, import_node_util, import_extract_zip, execFileAsync, _gpuRuntime, MODEL_FILENAME, MODEL_URL, EXPECTED_DIMS, BATCH_SEPARATOR, MAX_ARG_BYTES, LLAMA_RELEASE_TAG, MACOS_MIN_VERSION, BIN_NAME, SERVER_BIN_NAME, SERVER_SUPPORTED, SERVER_THRESHOLD, IDLE_TIMEOUT_MS, HEALTH_POLL_INTERVAL_MS, HEALTH_POLL_TIMEOUT_MS, HTTP_REQUEST_TIMEOUT_MS, SERVER_KILL_TIMEOUT_MS, SERVER_COOLDOWN_MS, EXTRACT_TIMEOUT_MS, embedQueue, embedRunning, binaryPath, downloadPromise, binaryDownloadPromise, serverProcess, activeSocketPath, idleTimer, serverStartPromise, serverCooldownUntil, serverRetryPromise, activeServerRequests, WANTED_FILES, WANTED_LIB_PATTERNS;
|
|
4451
4467
|
var init_embedder = __esm({
|
|
4452
4468
|
"src/recall/embedder.ts"() {
|
|
4453
4469
|
"use strict";
|
|
4454
|
-
|
|
4455
|
-
|
|
4470
|
+
import_node_child_process2 = require("node:child_process");
|
|
4471
|
+
import_node_fs8 = require("node:fs");
|
|
4456
4472
|
import_promises = require("node:fs/promises");
|
|
4457
4473
|
import_node_http = require("node:http");
|
|
4458
|
-
|
|
4459
|
-
|
|
4474
|
+
import_node_path8 = require("node:path");
|
|
4475
|
+
import_node_os4 = require("node:os");
|
|
4460
4476
|
import_node_util = require("node:util");
|
|
4461
4477
|
import_extract_zip = __toESM(require_extract_zip());
|
|
4462
4478
|
init_log();
|
|
4463
4479
|
init_paths();
|
|
4464
4480
|
init_config();
|
|
4465
|
-
execFileAsync = (0, import_node_util.promisify)(
|
|
4481
|
+
execFileAsync = (0, import_node_util.promisify)(import_node_child_process2.execFile);
|
|
4466
4482
|
MODEL_FILENAME = "nomic-embed-text-v1.5.Q8_0.gguf";
|
|
4467
4483
|
MODEL_URL = "https://huggingface.co/nomic-ai/nomic-embed-text-v1.5-GGUF/resolve/main/nomic-embed-text-v1.5.Q8_0.gguf";
|
|
4468
4484
|
EXPECTED_DIMS = 768;
|
|
4469
4485
|
BATCH_SEPARATOR = "<#sep#>";
|
|
4470
|
-
MAX_ARG_BYTES = (0,
|
|
4486
|
+
MAX_ARG_BYTES = (0, import_node_os4.platform)() === "win32" ? 0 : 1e5;
|
|
4471
4487
|
LLAMA_RELEASE_TAG = "b5300";
|
|
4472
4488
|
MACOS_MIN_VERSION = { arm64: "14.0", x64: "13.7" };
|
|
4473
|
-
BIN_NAME = (0,
|
|
4474
|
-
SERVER_BIN_NAME = (0,
|
|
4475
|
-
SERVER_SUPPORTED = (0,
|
|
4489
|
+
BIN_NAME = (0, import_node_os4.platform)() === "win32" ? "llama-embedding.exe" : "llama-embedding";
|
|
4490
|
+
SERVER_BIN_NAME = (0, import_node_os4.platform)() === "win32" ? "llama-server.exe" : "llama-server";
|
|
4491
|
+
SERVER_SUPPORTED = (0, import_node_os4.platform)() !== "win32";
|
|
4476
4492
|
SERVER_THRESHOLD = 5;
|
|
4477
4493
|
IDLE_TIMEOUT_MS = 3e4;
|
|
4478
4494
|
HEALTH_POLL_INTERVAL_MS = 200;
|
|
@@ -4738,10 +4754,127 @@ var require_brace_expansion = __commonJS({
|
|
|
4738
4754
|
});
|
|
4739
4755
|
|
|
4740
4756
|
// src/cli/embed-pending.ts
|
|
4741
|
-
var
|
|
4757
|
+
var import_node_fs11 = require("node:fs");
|
|
4742
4758
|
init_paths();
|
|
4743
4759
|
|
|
4760
|
+
// src/recall/embed-failures.ts
|
|
4761
|
+
var import_node_fs2 = require("node:fs");
|
|
4762
|
+
var import_node_path2 = require("node:path");
|
|
4763
|
+
init_paths();
|
|
4764
|
+
var attempts = /* @__PURE__ */ new Map();
|
|
4765
|
+
var MAX_MESSAGE_ATTEMPTS = 3;
|
|
4766
|
+
function exhaustedEmbedMessageIds() {
|
|
4767
|
+
const durable = Object.entries(readEmbedFailure()?.retryAfter ?? {}).filter(([, until]) => until > Date.now()).map(([id]) => id);
|
|
4768
|
+
return [.../* @__PURE__ */ new Set([...durable, ...[...attempts].filter(([, n]) => n >= MAX_MESSAGE_ATTEMPTS).map(([id]) => id)])];
|
|
4769
|
+
}
|
|
4770
|
+
function readEmbedFailure() {
|
|
4771
|
+
try {
|
|
4772
|
+
const value = JSON.parse((0, import_node_fs2.readFileSync)((0, import_node_path2.join)(logsDir(), "embed-failure.json"), "utf8"));
|
|
4773
|
+
if (!value || typeof value !== "object" || typeof value.updatedAt !== "string" || typeof value.reason !== "string" || typeof value.attempts !== "number" || !Array.isArray(value.failedMessageIds) || !value.failedMessageIds.every((id) => typeof id === "string"))
|
|
4774
|
+
return null;
|
|
4775
|
+
if (value.retryAfter && (typeof value.retryAfter !== "object" || Array.isArray(value.retryAfter) || !Object.values(value.retryAfter).every((n) => typeof n === "number" && Number.isFinite(n))))
|
|
4776
|
+
return null;
|
|
4777
|
+
return value;
|
|
4778
|
+
} catch {
|
|
4779
|
+
return null;
|
|
4780
|
+
}
|
|
4781
|
+
}
|
|
4782
|
+
function recordEmbedFailure(messageId, reason) {
|
|
4783
|
+
attempts.set(messageId, (attempts.get(messageId) ?? 0) + 1);
|
|
4784
|
+
try {
|
|
4785
|
+
(0, import_node_fs2.mkdirSync)(logsDir(), { recursive: true });
|
|
4786
|
+
const previous = readEmbedFailure();
|
|
4787
|
+
const retryAfter = previous?.retryAfter ?? {};
|
|
4788
|
+
if (attempts.get(messageId) >= MAX_MESSAGE_ATTEMPTS)
|
|
4789
|
+
retryAfter[messageId] = Date.now() + 15 * 6e4;
|
|
4790
|
+
(0, import_node_fs2.writeFileSync)((0, import_node_path2.join)(logsDir(), "embed-failure.json"), JSON.stringify({
|
|
4791
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
4792
|
+
attempts: attempts.get(messageId),
|
|
4793
|
+
reason,
|
|
4794
|
+
failedMessageIds: [.../* @__PURE__ */ new Set([...previous?.failedMessageIds ?? [], messageId])],
|
|
4795
|
+
retryAfter
|
|
4796
|
+
}));
|
|
4797
|
+
} catch {
|
|
4798
|
+
}
|
|
4799
|
+
}
|
|
4800
|
+
function recordEmbedSuccess(messageId) {
|
|
4801
|
+
attempts.delete(messageId);
|
|
4802
|
+
try {
|
|
4803
|
+
const previous = readEmbedFailure();
|
|
4804
|
+
if (!previous)
|
|
4805
|
+
return;
|
|
4806
|
+
if (!previous.failedMessageIds.includes(messageId) && !(previous.retryAfter && messageId in previous.retryAfter))
|
|
4807
|
+
return;
|
|
4808
|
+
previous.failedMessageIds = previous.failedMessageIds.filter((id) => id !== messageId);
|
|
4809
|
+
if (previous.retryAfter)
|
|
4810
|
+
delete previous.retryAfter[messageId];
|
|
4811
|
+
if (previous.failedMessageIds.length)
|
|
4812
|
+
(0, import_node_fs2.writeFileSync)((0, import_node_path2.join)(logsDir(), "embed-failure.json"), JSON.stringify(previous));
|
|
4813
|
+
else
|
|
4814
|
+
(0, import_node_fs2.rmSync)((0, import_node_path2.join)(logsDir(), "embed-failure.json"), { force: true });
|
|
4815
|
+
} catch {
|
|
4816
|
+
}
|
|
4817
|
+
}
|
|
4818
|
+
|
|
4819
|
+
// src/adapters/system-context.ts
|
|
4820
|
+
function firstTextContent(message) {
|
|
4821
|
+
if (!message)
|
|
4822
|
+
return void 0;
|
|
4823
|
+
const content = message.content;
|
|
4824
|
+
if (typeof content === "string")
|
|
4825
|
+
return content;
|
|
4826
|
+
if (Array.isArray(content)) {
|
|
4827
|
+
const firstText = content.find(
|
|
4828
|
+
(b) => typeof b === "object" && b !== null && b.type === "text"
|
|
4829
|
+
);
|
|
4830
|
+
if (firstText && "text" in firstText) {
|
|
4831
|
+
return firstText.text;
|
|
4832
|
+
}
|
|
4833
|
+
}
|
|
4834
|
+
return void 0;
|
|
4835
|
+
}
|
|
4836
|
+
function isSystemContextContent(message) {
|
|
4837
|
+
if (!message || message.role !== "user")
|
|
4838
|
+
return false;
|
|
4839
|
+
const text = firstTextContent(message);
|
|
4840
|
+
if (!text)
|
|
4841
|
+
return false;
|
|
4842
|
+
if (text.startsWith("<system-reminder>"))
|
|
4843
|
+
return true;
|
|
4844
|
+
if (text.startsWith("<environment_context>"))
|
|
4845
|
+
return true;
|
|
4846
|
+
if (text.startsWith("<INSTRUCTIONS>"))
|
|
4847
|
+
return true;
|
|
4848
|
+
if (text.startsWith("# AGENTS.md instructions for"))
|
|
4849
|
+
return true;
|
|
4850
|
+
if (text.startsWith("<context>"))
|
|
4851
|
+
return true;
|
|
4852
|
+
if (text.startsWith("<task-notification>"))
|
|
4853
|
+
return true;
|
|
4854
|
+
if (text.startsWith("<command-name>"))
|
|
4855
|
+
return true;
|
|
4856
|
+
if (text.startsWith("<local-command-stdout>"))
|
|
4857
|
+
return true;
|
|
4858
|
+
if (text.startsWith("<local-command-caveat>"))
|
|
4859
|
+
return true;
|
|
4860
|
+
return false;
|
|
4861
|
+
}
|
|
4862
|
+
|
|
4744
4863
|
// src/recall/transcript-utils.ts
|
|
4864
|
+
var META_KEEP_PREFIXES = [
|
|
4865
|
+
"<task-notification>",
|
|
4866
|
+
"[SYSTEM NOTIFICATION",
|
|
4867
|
+
"Another Claude session sent a message"
|
|
4868
|
+
];
|
|
4869
|
+
function shouldDropAsMeta(entry) {
|
|
4870
|
+
if (entry.isMeta !== true)
|
|
4871
|
+
return false;
|
|
4872
|
+
const text = firstTextContent(entry.message);
|
|
4873
|
+
if (text && META_KEEP_PREFIXES.some((prefix) => text.startsWith(prefix))) {
|
|
4874
|
+
return false;
|
|
4875
|
+
}
|
|
4876
|
+
return true;
|
|
4877
|
+
}
|
|
4745
4878
|
function stripToolContent(entries) {
|
|
4746
4879
|
const out = [];
|
|
4747
4880
|
for (const entry of entries) {
|
|
@@ -4770,8 +4903,8 @@ function stripToolContent(entries) {
|
|
|
4770
4903
|
}
|
|
4771
4904
|
|
|
4772
4905
|
// src/db.ts
|
|
4773
|
-
var
|
|
4774
|
-
var
|
|
4906
|
+
var import_node_fs3 = require("node:fs");
|
|
4907
|
+
var import_node_path3 = require("node:path");
|
|
4775
4908
|
var import_node_module = require("node:module");
|
|
4776
4909
|
init_log();
|
|
4777
4910
|
init_paths();
|
|
@@ -4791,11 +4924,12 @@ Run \`recall doctor\` for details.`
|
|
|
4791
4924
|
}
|
|
4792
4925
|
};
|
|
4793
4926
|
var MigrationPendingError = class extends Error {
|
|
4794
|
-
constructor(dbPath2) {
|
|
4927
|
+
constructor(dbPath2, kind = "retrieval-class") {
|
|
4795
4928
|
super(
|
|
4796
|
-
"recall: this database needs a one-time schema migration \u2014 run `recall install` to finish it. (Normal commands refuse to rewrite the index unattended.)"
|
|
4929
|
+
kind === "codex-rekey" ? "recall: this database needs a one-time Codex message-id migration \u2014 run `recall install` (or `recall repair --rekey-codex`) to finish it. (Normal commands refuse to rewrite the index unattended.)" : "recall: this database needs a one-time schema migration \u2014 run `recall install` to finish it. (Normal commands refuse to rewrite the index unattended.)"
|
|
4797
4930
|
);
|
|
4798
4931
|
this.dbPath = dbPath2;
|
|
4932
|
+
this.kind = kind;
|
|
4799
4933
|
this.name = "MigrationPendingError";
|
|
4800
4934
|
}
|
|
4801
4935
|
};
|
|
@@ -4819,7 +4953,7 @@ function getDb(dbPath2, opts) {
|
|
|
4819
4953
|
if (db) {
|
|
4820
4954
|
closeDb();
|
|
4821
4955
|
}
|
|
4822
|
-
(0,
|
|
4956
|
+
(0, import_node_fs3.mkdirSync)((0, import_node_path3.dirname)(dbPath2), { recursive: true });
|
|
4823
4957
|
cleanupWasmArtifacts(dbPath2);
|
|
4824
4958
|
const raw = openDatabase(dbPath2);
|
|
4825
4959
|
try {
|
|
@@ -4836,16 +4970,43 @@ function getDb(dbPath2, opts) {
|
|
|
4836
4970
|
}
|
|
4837
4971
|
db = adapter;
|
|
4838
4972
|
currentDbPath = dbPath2;
|
|
4973
|
+
ensureStemScratch(db);
|
|
4839
4974
|
log({ source: "db", level: "info", summary: `DB: opened pending-migration DB at ${dbPath2} (installer mode)` });
|
|
4840
4975
|
return db;
|
|
4841
4976
|
}
|
|
4977
|
+
if (isCodexRekeyPending(adapter) && !opts?.allowPendingMigration) {
|
|
4978
|
+
raw.close();
|
|
4979
|
+
throw new MigrationPendingError(dbPath2, "codex-rekey");
|
|
4980
|
+
}
|
|
4842
4981
|
db = adapter;
|
|
4843
4982
|
currentDbPath = dbPath2;
|
|
4844
|
-
|
|
4983
|
+
try {
|
|
4984
|
+
ensureSchema(db);
|
|
4985
|
+
ensureStemScratch(db);
|
|
4986
|
+
} catch (error) {
|
|
4987
|
+
closeDb();
|
|
4988
|
+
throw error;
|
|
4989
|
+
}
|
|
4845
4990
|
log({ source: "db", level: "info", summary: `DB: initialized at ${dbPath2}` });
|
|
4846
4991
|
return db;
|
|
4847
4992
|
}
|
|
4993
|
+
function ensureStemScratch(db3) {
|
|
4994
|
+
db3.exec(`
|
|
4995
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS temp._stem USING fts5(
|
|
4996
|
+
t, tokenize='porter unicode61'
|
|
4997
|
+
);
|
|
4998
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS temp._stem_vocab
|
|
4999
|
+
USING fts5vocab(temp, _stem, 'row');
|
|
5000
|
+
`);
|
|
5001
|
+
}
|
|
4848
5002
|
var RETRIEVAL_MIGRATION_KEY = "retrieval_class_migration";
|
|
5003
|
+
function rethrowIfBusy(err) {
|
|
5004
|
+
const code = err?.code;
|
|
5005
|
+
const message = err?.message;
|
|
5006
|
+
if (typeof code === "string" && code.startsWith("SQLITE_BUSY") || typeof message === "string" && /database is locked|SQLITE_BUSY/i.test(message))
|
|
5007
|
+
throw err;
|
|
5008
|
+
}
|
|
5009
|
+
var PROJECT_KEY_BACKFILL_KEY = "project_key_backfill";
|
|
4849
5010
|
function isRetrievalMigrationPending(d) {
|
|
4850
5011
|
try {
|
|
4851
5012
|
const hasMessages = d.get(
|
|
@@ -4863,13 +5024,40 @@ function isRetrievalMigrationPending(d) {
|
|
|
4863
5024
|
[RETRIEVAL_MIGRATION_KEY]
|
|
4864
5025
|
);
|
|
4865
5026
|
return row?.value !== "complete";
|
|
4866
|
-
} catch {
|
|
5027
|
+
} catch (err) {
|
|
5028
|
+
rethrowIfBusy(err);
|
|
5029
|
+
return true;
|
|
5030
|
+
}
|
|
5031
|
+
}
|
|
5032
|
+
var CODEX_REKEY_MIGRATION_KEY = "codex_message_id_v2";
|
|
5033
|
+
function isCodexRekeyPending(d) {
|
|
5034
|
+
try {
|
|
5035
|
+
const hasMessages = d.get(
|
|
5036
|
+
`SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'messages'`
|
|
5037
|
+
);
|
|
5038
|
+
if (!hasMessages)
|
|
5039
|
+
return false;
|
|
5040
|
+
const hasMeta = d.get(
|
|
5041
|
+
`SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'schema_meta'`
|
|
5042
|
+
);
|
|
5043
|
+
if (!hasMeta)
|
|
5044
|
+
return true;
|
|
5045
|
+
const row = d.get(
|
|
5046
|
+
`SELECT value FROM schema_meta WHERE key = ?`,
|
|
5047
|
+
[CODEX_REKEY_MIGRATION_KEY]
|
|
5048
|
+
);
|
|
5049
|
+
return row?.value !== "complete";
|
|
5050
|
+
} catch (err) {
|
|
5051
|
+
rethrowIfBusy(err);
|
|
4867
5052
|
return true;
|
|
4868
5053
|
}
|
|
4869
5054
|
}
|
|
4870
5055
|
function closeDb() {
|
|
4871
|
-
if (db)
|
|
5056
|
+
if (!db)
|
|
5057
|
+
return;
|
|
5058
|
+
try {
|
|
4872
5059
|
db.close();
|
|
5060
|
+
} finally {
|
|
4873
5061
|
db = null;
|
|
4874
5062
|
currentDbPath = null;
|
|
4875
5063
|
}
|
|
@@ -4885,32 +5073,32 @@ function openDatabase(dbPath2) {
|
|
|
4885
5073
|
}
|
|
4886
5074
|
}
|
|
4887
5075
|
function resolveNativeBinding() {
|
|
4888
|
-
const sibling = (0,
|
|
4889
|
-
if ((0,
|
|
5076
|
+
const sibling = (0, import_node_path3.join)(__dirname, "better_sqlite3.node");
|
|
5077
|
+
if ((0, import_node_fs3.existsSync)(sibling))
|
|
4890
5078
|
return sibling;
|
|
4891
5079
|
const resolved = resolveInstalledBinding();
|
|
4892
5080
|
if (resolved)
|
|
4893
5081
|
return resolved;
|
|
4894
|
-
const staged = (0,
|
|
4895
|
-
if ((0,
|
|
5082
|
+
const staged = (0, import_node_path3.join)(binDir(), "better_sqlite3.node");
|
|
5083
|
+
if ((0, import_node_fs3.existsSync)(staged))
|
|
4896
5084
|
return staged;
|
|
4897
5085
|
return null;
|
|
4898
5086
|
}
|
|
4899
5087
|
function resolveInstalledBinding() {
|
|
4900
5088
|
try {
|
|
4901
5089
|
const pkgJson = (0, import_node_module.createRequire)(__filename).resolve("better-sqlite3/package.json");
|
|
4902
|
-
return findNativeBinding((0,
|
|
5090
|
+
return findNativeBinding((0, import_node_path3.dirname)(pkgJson));
|
|
4903
5091
|
} catch {
|
|
4904
5092
|
return null;
|
|
4905
5093
|
}
|
|
4906
5094
|
}
|
|
4907
5095
|
function findNativeBinding(baseDir) {
|
|
4908
5096
|
for (const c of [
|
|
4909
|
-
(0,
|
|
4910
|
-
(0,
|
|
5097
|
+
(0, import_node_path3.join)(baseDir, "build", "Release", "better_sqlite3.node"),
|
|
5098
|
+
(0, import_node_path3.join)(baseDir, "build", "Debug", "better_sqlite3.node")
|
|
4911
5099
|
]) {
|
|
4912
5100
|
try {
|
|
4913
|
-
if ((0,
|
|
5101
|
+
if ((0, import_node_fs3.statSync)(c).isFile())
|
|
4914
5102
|
return c;
|
|
4915
5103
|
} catch {
|
|
4916
5104
|
}
|
|
@@ -4920,12 +5108,12 @@ function findNativeBinding(baseDir) {
|
|
|
4920
5108
|
const dir = stack.pop();
|
|
4921
5109
|
let entries;
|
|
4922
5110
|
try {
|
|
4923
|
-
entries = (0,
|
|
5111
|
+
entries = (0, import_node_fs3.readdirSync)(dir, { withFileTypes: true });
|
|
4924
5112
|
} catch {
|
|
4925
5113
|
continue;
|
|
4926
5114
|
}
|
|
4927
5115
|
for (const e of entries) {
|
|
4928
|
-
const p = (0,
|
|
5116
|
+
const p = (0, import_node_path3.join)(dir, e.name);
|
|
4929
5117
|
if (e.isDirectory())
|
|
4930
5118
|
stack.push(p);
|
|
4931
5119
|
else if (e.name.endsWith(".node"))
|
|
@@ -4968,12 +5156,12 @@ function configurePragmas(raw, dbPath2) {
|
|
|
4968
5156
|
}
|
|
4969
5157
|
function cleanupWasmArtifacts(dbPath2) {
|
|
4970
5158
|
try {
|
|
4971
|
-
(0,
|
|
5159
|
+
(0, import_node_fs3.rmSync)(`${dbPath2}.lock`, { recursive: true, force: true });
|
|
4972
5160
|
} catch {
|
|
4973
5161
|
}
|
|
4974
5162
|
try {
|
|
4975
|
-
if ((0,
|
|
4976
|
-
(0,
|
|
5163
|
+
if ((0, import_node_fs3.existsSync)(`${dbPath2}.owner`))
|
|
5164
|
+
(0, import_node_fs3.unlinkSync)(`${dbPath2}.owner`);
|
|
4977
5165
|
} catch {
|
|
4978
5166
|
}
|
|
4979
5167
|
}
|
|
@@ -4989,6 +5177,7 @@ var RETRIEVAL_SCHEMA_DDL = {
|
|
|
4989
5177
|
created_at INTEGER NOT NULL,
|
|
4990
5178
|
message_role TEXT,
|
|
4991
5179
|
retrieval_class TEXT NOT NULL DEFAULT 'hot',
|
|
5180
|
+
project_key TEXT,
|
|
4992
5181
|
UNIQUE(session_id, message_id)
|
|
4993
5182
|
);
|
|
4994
5183
|
|
|
@@ -5078,15 +5267,18 @@ var RETRIEVAL_SCHEMA_DDL = {
|
|
|
5078
5267
|
`
|
|
5079
5268
|
};
|
|
5080
5269
|
function ensureSchema(db3) {
|
|
5270
|
+
const fresh = !db3.get(
|
|
5271
|
+
`SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'messages'`
|
|
5272
|
+
);
|
|
5081
5273
|
db3.exec("BEGIN IMMEDIATE");
|
|
5082
5274
|
try {
|
|
5083
5275
|
db3.exec(RETRIEVAL_SCHEMA_DDL.tables);
|
|
5084
5276
|
db3.exec(RETRIEVAL_SCHEMA_DDL.fts);
|
|
5085
5277
|
db3.exec(`
|
|
5086
|
-
CREATE VIRTUAL TABLE IF NOT EXISTS _stem USING fts5(
|
|
5278
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS main._stem USING fts5(
|
|
5087
5279
|
t, tokenize='porter unicode61'
|
|
5088
5280
|
);
|
|
5089
|
-
CREATE VIRTUAL TABLE IF NOT EXISTS _stem_vocab
|
|
5281
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS main._stem_vocab
|
|
5090
5282
|
USING fts5vocab(_stem, 'row');
|
|
5091
5283
|
`);
|
|
5092
5284
|
db3.exec(`
|
|
@@ -5107,6 +5299,16 @@ function ensureSchema(db3) {
|
|
|
5107
5299
|
throw e;
|
|
5108
5300
|
}
|
|
5109
5301
|
}
|
|
5302
|
+
const hasProjectKey = () => db3.all(`PRAGMA table_info(messages)`).some((c) => c.name === "project_key");
|
|
5303
|
+
if (!hasProjectKey()) {
|
|
5304
|
+
try {
|
|
5305
|
+
db3.exec(`ALTER TABLE messages ADD COLUMN project_key TEXT`);
|
|
5306
|
+
} catch (e) {
|
|
5307
|
+
if (!hasProjectKey())
|
|
5308
|
+
throw e;
|
|
5309
|
+
}
|
|
5310
|
+
}
|
|
5311
|
+
db3.exec(`CREATE INDEX IF NOT EXISTS idx_messages_project_key ON messages(project_key)`);
|
|
5110
5312
|
db3.exec(`
|
|
5111
5313
|
CREATE TABLE IF NOT EXISTS ingest_watermark (
|
|
5112
5314
|
transcript_path TEXT PRIMARY KEY,
|
|
@@ -5120,6 +5322,18 @@ function ensureSchema(db3) {
|
|
|
5120
5322
|
INSERT OR IGNORE INTO schema_meta(key, value)
|
|
5121
5323
|
VALUES ('${RETRIEVAL_MIGRATION_KEY}', 'complete');
|
|
5122
5324
|
`);
|
|
5325
|
+
if (fresh) {
|
|
5326
|
+
db3.exec(`
|
|
5327
|
+
INSERT OR IGNORE INTO schema_meta(key, value)
|
|
5328
|
+
VALUES ('${CODEX_REKEY_MIGRATION_KEY}', 'complete');
|
|
5329
|
+
`);
|
|
5330
|
+
}
|
|
5331
|
+
if (fresh) {
|
|
5332
|
+
db3.exec(`
|
|
5333
|
+
INSERT OR IGNORE INTO schema_meta(key, value)
|
|
5334
|
+
VALUES ('${PROJECT_KEY_BACKFILL_KEY}', 'complete');
|
|
5335
|
+
`);
|
|
5336
|
+
}
|
|
5123
5337
|
db3.exec("COMMIT");
|
|
5124
5338
|
} catch (e) {
|
|
5125
5339
|
try {
|
|
@@ -5147,6 +5361,8 @@ var ENRICH_MAX_CHARS = 200;
|
|
|
5147
5361
|
var ENRICH_PREV_CHARS = 512;
|
|
5148
5362
|
var ENRICH_SEP = "\n";
|
|
5149
5363
|
function buildEmbedText(messageText, prevText) {
|
|
5364
|
+
messageText = messageText.replaceAll("\0", "");
|
|
5365
|
+
prevText = prevText?.replaceAll("\0", "") ?? null;
|
|
5150
5366
|
if (messageText.length >= ENRICH_MAX_CHARS || !prevText)
|
|
5151
5367
|
return messageText;
|
|
5152
5368
|
return prevText.slice(-ENRICH_PREV_CHARS) + ENRICH_SEP + messageText;
|
|
@@ -5165,10 +5381,11 @@ function ensureDir2() {
|
|
|
5165
5381
|
}
|
|
5166
5382
|
function insertMessages(messages, opts) {
|
|
5167
5383
|
if (messages.length === 0 && !opts?.replaceSessionId && !opts?.provenance)
|
|
5168
|
-
return;
|
|
5384
|
+
return 0;
|
|
5169
5385
|
ensureDir2();
|
|
5170
5386
|
const d = db2();
|
|
5171
5387
|
d.exec("BEGIN IMMEDIATE");
|
|
5388
|
+
let inserted = 0;
|
|
5172
5389
|
try {
|
|
5173
5390
|
if (opts?.replaceSessionId) {
|
|
5174
5391
|
d.run(
|
|
@@ -5180,20 +5397,65 @@ function insertMessages(messages, opts) {
|
|
|
5180
5397
|
}
|
|
5181
5398
|
const stmt = d.prepare(
|
|
5182
5399
|
`INSERT OR IGNORE INTO messages
|
|
5183
|
-
(message_id, session_id, message_seq, message_text, project_id, created_at, message_role, retrieval_class)
|
|
5184
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
|
|
5400
|
+
(message_id, session_id, message_seq, message_text, project_id, created_at, message_role, retrieval_class, project_key)
|
|
5401
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
5185
5402
|
);
|
|
5403
|
+
const invalidatedSessions = /* @__PURE__ */ new Set();
|
|
5404
|
+
const sourceIds = /* @__PURE__ */ new Map();
|
|
5186
5405
|
for (const m of messages) {
|
|
5187
|
-
|
|
5188
|
-
|
|
5406
|
+
const ids = sourceIds.get(m.session_id) ?? /* @__PURE__ */ new Set();
|
|
5407
|
+
ids.add(m.message_id);
|
|
5408
|
+
ids.add(`session:${m.session_id}:${m.message_id}`);
|
|
5409
|
+
sourceIds.set(m.session_id, ids);
|
|
5410
|
+
}
|
|
5411
|
+
if (opts?.provenance && opts.sourceMessageIds) {
|
|
5412
|
+
const sid = opts.provenance.sessionId;
|
|
5413
|
+
const ids = sourceIds.get(sid);
|
|
5414
|
+
for (const id of opts.sourceMessageIds) {
|
|
5415
|
+
ids?.add(id);
|
|
5416
|
+
ids?.add(`session:${sid}:${id}`);
|
|
5417
|
+
}
|
|
5418
|
+
}
|
|
5419
|
+
const partialSessions = /* @__PURE__ */ new Map();
|
|
5420
|
+
const maxStoredSeq = /* @__PURE__ */ new Map();
|
|
5421
|
+
for (const [sid, ids] of sourceIds) {
|
|
5422
|
+
const stored = d.all("SELECT message_id, message_seq FROM messages WHERE session_id = ?", [sid]);
|
|
5423
|
+
const max = stored.reduce((acc, r) => Math.max(acc, r.message_seq), -1);
|
|
5424
|
+
maxStoredSeq.set(sid, max);
|
|
5425
|
+
if (stored.some((r) => !ids.has(r.message_id)))
|
|
5426
|
+
partialSessions.set(sid, max);
|
|
5427
|
+
}
|
|
5428
|
+
for (const m of messages) {
|
|
5429
|
+
const scopedId = `session:${m.session_id}:${m.message_id}`;
|
|
5430
|
+
const scoped = d.get("SELECT message_id FROM messages WHERE message_id = ? AND session_id = ?", [scopedId, m.session_id]);
|
|
5431
|
+
const owner = d.get("SELECT session_id FROM messages WHERE message_id = ?", [m.message_id]);
|
|
5432
|
+
const id = scoped || owner && owner.session_id !== m.session_id ? scopedId : m.message_id;
|
|
5433
|
+
const existing = d.get("SELECT message_seq FROM messages WHERE message_id = ? AND session_id = ?", [id, m.session_id]);
|
|
5434
|
+
let sequence = m.message_seq;
|
|
5435
|
+
if (partialSessions.has(m.session_id)) {
|
|
5436
|
+
sequence = existing?.message_seq ?? partialSessions.get(m.session_id) + 1;
|
|
5437
|
+
partialSessions.set(m.session_id, Math.max(partialSessions.get(m.session_id), sequence));
|
|
5438
|
+
}
|
|
5439
|
+
const insertsPredecessor = !existing && sequence <= (maxStoredSeq.get(m.session_id) ?? -1);
|
|
5440
|
+
if (existing && existing.message_seq !== sequence || insertsPredecessor) {
|
|
5441
|
+
if (!invalidatedSessions.has(m.session_id)) {
|
|
5442
|
+
d.run("DELETE FROM message_vectors WHERE message_id IN (SELECT message_id FROM messages WHERE session_id = ?)", [m.session_id]);
|
|
5443
|
+
invalidatedSessions.add(m.session_id);
|
|
5444
|
+
}
|
|
5445
|
+
d.run("UPDATE messages SET message_seq = ? WHERE message_id = ? AND session_id = ?", [sequence, id, m.session_id]);
|
|
5446
|
+
}
|
|
5447
|
+
maxStoredSeq.set(m.session_id, Math.max(maxStoredSeq.get(m.session_id) ?? -1, sequence));
|
|
5448
|
+
inserted += stmt.run([
|
|
5449
|
+
id,
|
|
5189
5450
|
m.session_id,
|
|
5190
|
-
|
|
5451
|
+
sequence,
|
|
5191
5452
|
m.message_text,
|
|
5192
5453
|
m.project_id,
|
|
5193
5454
|
m.created_at,
|
|
5194
5455
|
m.message_role,
|
|
5195
|
-
m.retrieval_class ?? "hot"
|
|
5196
|
-
|
|
5456
|
+
m.retrieval_class ?? "hot",
|
|
5457
|
+
m.project_key ?? null
|
|
5458
|
+
]).changes;
|
|
5197
5459
|
}
|
|
5198
5460
|
if (opts?.provenance) {
|
|
5199
5461
|
const p = opts.provenance;
|
|
@@ -5233,6 +5495,15 @@ function insertMessages(messages, opts) {
|
|
|
5233
5495
|
);
|
|
5234
5496
|
}
|
|
5235
5497
|
}
|
|
5498
|
+
if (opts?.adopt) {
|
|
5499
|
+
const { sessionId, projectKey, projectId } = opts.adopt;
|
|
5500
|
+
if (projectKey !== null) {
|
|
5501
|
+
d.run("UPDATE messages SET project_key = ? WHERE session_id = ? AND project_key IS NULL", [projectKey, sessionId]);
|
|
5502
|
+
}
|
|
5503
|
+
if (projectId !== null && projectId !== "") {
|
|
5504
|
+
d.run("UPDATE messages SET project_id = ? WHERE session_id = ? AND (project_id IS NULL OR project_id = '')", [projectId, sessionId]);
|
|
5505
|
+
}
|
|
5506
|
+
}
|
|
5236
5507
|
for (const a of opts?.aliases ?? []) {
|
|
5237
5508
|
d.run(
|
|
5238
5509
|
`INSERT OR REPLACE INTO session_aliases (alias_id, session_id, source, created_at)
|
|
@@ -5241,11 +5512,23 @@ function insertMessages(messages, opts) {
|
|
|
5241
5512
|
);
|
|
5242
5513
|
}
|
|
5243
5514
|
d.exec("COMMIT");
|
|
5515
|
+
return inserted;
|
|
5244
5516
|
} catch (e) {
|
|
5245
5517
|
d.exec("ROLLBACK");
|
|
5246
5518
|
throw e;
|
|
5247
5519
|
}
|
|
5248
5520
|
}
|
|
5521
|
+
function hasSessionMessages(sessionId) {
|
|
5522
|
+
try {
|
|
5523
|
+
const row = db2().get(
|
|
5524
|
+
"SELECT 1 FROM messages WHERE session_id = ? LIMIT 1",
|
|
5525
|
+
[sessionId]
|
|
5526
|
+
);
|
|
5527
|
+
return row != null;
|
|
5528
|
+
} catch {
|
|
5529
|
+
return false;
|
|
5530
|
+
}
|
|
5531
|
+
}
|
|
5249
5532
|
function insertMessageVectors(records) {
|
|
5250
5533
|
if (records.length === 0)
|
|
5251
5534
|
return;
|
|
@@ -5285,13 +5568,14 @@ function getUnembeddedMessages(limit) {
|
|
|
5285
5568
|
ORDER BY p.message_seq DESC LIMIT 1) AS prev_text
|
|
5286
5569
|
FROM messages m
|
|
5287
5570
|
WHERE m.message_text != ''
|
|
5571
|
+
AND m.message_id NOT IN (SELECT value FROM json_each(?))
|
|
5288
5572
|
AND m.retrieval_class = 'hot'
|
|
5289
|
-
AND (LENGTH(m.message_text) >= ${MIN_EMBED_CHARS}
|
|
5573
|
+
AND (LENGTH(m.message_text) >= ${MIN_EMBED_CHARS} OR INSTR(m.message_text, char(0)) > 0
|
|
5290
5574
|
OR EXISTS (SELECT 1 FROM messages p2 WHERE p2.session_id = m.session_id AND p2.message_seq < m.message_seq AND p2.retrieval_class = 'hot'))
|
|
5291
5575
|
AND NOT EXISTS (SELECT 1 FROM message_vectors mv WHERE mv.message_id = m.message_id AND mv.embed_version = ?)
|
|
5292
5576
|
ORDER BY m.created_at DESC
|
|
5293
5577
|
LIMIT ?`,
|
|
5294
|
-
[EMBED_VERSION, limit]
|
|
5578
|
+
[JSON.stringify(exhaustedEmbedMessageIds()), EMBED_VERSION, limit]
|
|
5295
5579
|
);
|
|
5296
5580
|
return rows.map((r) => {
|
|
5297
5581
|
const message_text = r.message_text;
|
|
@@ -5315,6 +5599,89 @@ init_log();
|
|
|
5315
5599
|
// src/adapters/codex/codex-jsonl-reader.ts
|
|
5316
5600
|
var fs = __toESM(require("fs"));
|
|
5317
5601
|
init_log();
|
|
5602
|
+
|
|
5603
|
+
// src/recall/transcript-roots.ts
|
|
5604
|
+
var import_node_os2 = require("node:os");
|
|
5605
|
+
var import_node_path5 = require("node:path");
|
|
5606
|
+
|
|
5607
|
+
// src/hub/runtime.ts
|
|
5608
|
+
var import_node_fs4 = require("node:fs");
|
|
5609
|
+
var import_node_path4 = require("node:path");
|
|
5610
|
+
init_paths();
|
|
5611
|
+
var STALE_RECORD_MS = 60 * 60 * 1e3;
|
|
5612
|
+
var REFUSED_RECENT_MAX = 5;
|
|
5613
|
+
function hubHostsPath() {
|
|
5614
|
+
return (0, import_node_path4.join)(runDir(), "hub-hosts.json");
|
|
5615
|
+
}
|
|
5616
|
+
function readHostRecords() {
|
|
5617
|
+
try {
|
|
5618
|
+
const parsed = JSON.parse((0, import_node_fs4.readFileSync)(hubHostsPath(), "utf-8"));
|
|
5619
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed))
|
|
5620
|
+
return {};
|
|
5621
|
+
const out = {};
|
|
5622
|
+
for (const [host, rec] of Object.entries(parsed)) {
|
|
5623
|
+
if (!rec || typeof rec !== "object")
|
|
5624
|
+
continue;
|
|
5625
|
+
const r = rec;
|
|
5626
|
+
out[host] = {
|
|
5627
|
+
...typeof r["lastPushAt"] === "string" ? { lastPushAt: r["lastPushAt"] } : {},
|
|
5628
|
+
...typeof r["lastQueryAt"] === "string" ? { lastQueryAt: r["lastQueryAt"] } : {},
|
|
5629
|
+
...typeof r["lastFullManifestAt"] === "string" ? { lastFullManifestAt: r["lastFullManifestAt"] } : {},
|
|
5630
|
+
refusedCollisions: typeof r["refusedCollisions"] === "number" ? r["refusedCollisions"] : 0,
|
|
5631
|
+
refusedRecent: Array.isArray(r["refusedRecent"]) ? r["refusedRecent"].filter((v) => typeof v === "string").slice(0, REFUSED_RECENT_MAX) : []
|
|
5632
|
+
};
|
|
5633
|
+
}
|
|
5634
|
+
return out;
|
|
5635
|
+
} catch {
|
|
5636
|
+
return {};
|
|
5637
|
+
}
|
|
5638
|
+
}
|
|
5639
|
+
|
|
5640
|
+
// src/recall/transcript-roots.ts
|
|
5641
|
+
init_log();
|
|
5642
|
+
var DRVFS = /^\/mnt\/[a-zA-Z]\//;
|
|
5643
|
+
var warned = /* @__PURE__ */ new Set();
|
|
5644
|
+
function defaultClaudeRoot() {
|
|
5645
|
+
return (0, import_node_path5.join)((0, import_node_os2.homedir)(), ".claude");
|
|
5646
|
+
}
|
|
5647
|
+
function defaultCodexRoot() {
|
|
5648
|
+
return (0, import_node_path5.join)((0, import_node_os2.homedir)(), ".codex");
|
|
5649
|
+
}
|
|
5650
|
+
function guardedHosts(override) {
|
|
5651
|
+
if (process.platform !== "linux")
|
|
5652
|
+
return [];
|
|
5653
|
+
const abs = (0, import_node_path5.resolve)(override).replace(/\\/g, "/");
|
|
5654
|
+
if (!DRVFS.test(abs.endsWith("/") ? abs : `${abs}/`))
|
|
5655
|
+
return [];
|
|
5656
|
+
return Object.keys(readHostRecords());
|
|
5657
|
+
}
|
|
5658
|
+
function resolveRoot(envVar, fallback) {
|
|
5659
|
+
const override = process.env[envVar];
|
|
5660
|
+
if (!override || override.length === 0)
|
|
5661
|
+
return fallback;
|
|
5662
|
+
const hosts = guardedHosts(override);
|
|
5663
|
+
if (hosts.length === 0)
|
|
5664
|
+
return override;
|
|
5665
|
+
if (!warned.has(envVar)) {
|
|
5666
|
+
warned.add(envVar);
|
|
5667
|
+
log({
|
|
5668
|
+
level: "warn",
|
|
5669
|
+
source: "transcript-roots",
|
|
5670
|
+
summary: `ignoring ${envVar}=${override}: this hub serves satellite ${hosts.join(", ")} and that tree is pushed by a satellite; set the variable elsewhere or unset it`
|
|
5671
|
+
});
|
|
5672
|
+
}
|
|
5673
|
+
return fallback;
|
|
5674
|
+
}
|
|
5675
|
+
function claudeRoot() {
|
|
5676
|
+
return resolveRoot("CLAUDE_CONFIG_DIR", defaultClaudeRoot());
|
|
5677
|
+
}
|
|
5678
|
+
function codexRoot() {
|
|
5679
|
+
return resolveRoot("CODEX_HOME", defaultCodexRoot());
|
|
5680
|
+
}
|
|
5681
|
+
|
|
5682
|
+
// src/adapters/codex/codex-jsonl-reader.ts
|
|
5683
|
+
var META_READ_LIMIT = 256 * 1024;
|
|
5684
|
+
var META_CHUNK_SIZE = 8192;
|
|
5318
5685
|
function parseCodexJsonlFile(filepath) {
|
|
5319
5686
|
try {
|
|
5320
5687
|
const content = fs.readFileSync(filepath, "utf-8");
|
|
@@ -5334,20 +5701,39 @@ function parseCodexJsonlFile(filepath) {
|
|
|
5334
5701
|
return records;
|
|
5335
5702
|
} catch (error) {
|
|
5336
5703
|
log({ level: "error", source: "codex-jsonl-reader", summary: `Failed to read ${filepath}: ${error instanceof Error ? error.message : String(error)}`, data: { filepath, error: String(error) } });
|
|
5337
|
-
|
|
5704
|
+
throw error;
|
|
5338
5705
|
}
|
|
5339
5706
|
}
|
|
5340
5707
|
function extractCodexSessionMeta(filepath) {
|
|
5341
5708
|
let fd = null;
|
|
5342
5709
|
try {
|
|
5343
5710
|
fd = fs.openSync(filepath, "r");
|
|
5344
|
-
const
|
|
5345
|
-
|
|
5346
|
-
|
|
5711
|
+
const chunks = [];
|
|
5712
|
+
let position = 0;
|
|
5713
|
+
let newlineIdx = -1;
|
|
5714
|
+
let atEof = false;
|
|
5715
|
+
while (position < META_READ_LIMIT) {
|
|
5716
|
+
const chunk = Buffer.alloc(Math.min(META_CHUNK_SIZE, META_READ_LIMIT - position));
|
|
5717
|
+
const bytesRead = fs.readSync(fd, chunk, 0, chunk.length, position);
|
|
5718
|
+
if (bytesRead === 0) {
|
|
5719
|
+
atEof = true;
|
|
5720
|
+
break;
|
|
5721
|
+
}
|
|
5722
|
+
const slice = chunk.subarray(0, bytesRead);
|
|
5723
|
+
const idx = slice.indexOf(10);
|
|
5724
|
+
chunks.push(slice);
|
|
5725
|
+
if (idx >= 0) {
|
|
5726
|
+
newlineIdx = position + idx;
|
|
5727
|
+
break;
|
|
5728
|
+
}
|
|
5729
|
+
position += bytesRead;
|
|
5730
|
+
}
|
|
5731
|
+
if (chunks.length === 0)
|
|
5347
5732
|
return null;
|
|
5348
|
-
|
|
5349
|
-
|
|
5350
|
-
const
|
|
5733
|
+
if (newlineIdx < 0 && !atEof)
|
|
5734
|
+
return null;
|
|
5735
|
+
const joined = Buffer.concat(chunks);
|
|
5736
|
+
const firstLine = (newlineIdx >= 0 ? joined.subarray(0, newlineIdx) : joined).toString("utf-8");
|
|
5351
5737
|
const record = JSON.parse(firstLine.trim());
|
|
5352
5738
|
if (record.type !== "session_meta")
|
|
5353
5739
|
return null;
|
|
@@ -5539,7 +5925,10 @@ function parseSubagentSource(source, _transcriptPath) {
|
|
|
5539
5925
|
const looksSubagent = subRaw !== void 0 || typeTag === "subagent";
|
|
5540
5926
|
if (!looksSubagent)
|
|
5541
5927
|
return none;
|
|
5542
|
-
if (subRaw
|
|
5928
|
+
if (typeof subRaw === "string" && ["review", "compact", "memory_consolidation"].includes(subRaw)) {
|
|
5929
|
+
return { ...none, isSubagent: true, meta: { type: subRaw } };
|
|
5930
|
+
}
|
|
5931
|
+
if (subRaw !== void 0 && (typeof subRaw !== "object" || subRaw === null || Array.isArray(subRaw))) {
|
|
5543
5932
|
return { ...none, malformed: true };
|
|
5544
5933
|
}
|
|
5545
5934
|
const sub = subRaw ?? {};
|
|
@@ -5548,7 +5937,7 @@ function parseSubagentSource(source, _transcriptPath) {
|
|
|
5548
5937
|
const parentThreadId = typeof spawn2.parent_thread_id === "string" && spawn2.parent_thread_id ? spawn2.parent_thread_id : null;
|
|
5549
5938
|
const depth = typeof spawn2.depth === "number" && Number.isFinite(spawn2.depth) ? spawn2.depth : null;
|
|
5550
5939
|
const meta = {};
|
|
5551
|
-
for (const k of ["agent_path", "agent_type", "agent_name", "path", "type", "name"]) {
|
|
5940
|
+
for (const k of ["agent_path", "agent_type", "agent_name", "agent_nickname", "agent_role", "path", "type", "name"]) {
|
|
5552
5941
|
const v = spawn2[k] ?? sub[k];
|
|
5553
5942
|
if (typeof v === "string" || typeof v === "number")
|
|
5554
5943
|
meta[k] = v;
|
|
@@ -5576,6 +5965,255 @@ function normalizePath(p) {
|
|
|
5576
5965
|
return normalized;
|
|
5577
5966
|
}
|
|
5578
5967
|
|
|
5968
|
+
// src/recall/project-key.ts
|
|
5969
|
+
var import_node_fs5 = require("node:fs");
|
|
5970
|
+
var import_node_os3 = require("node:os");
|
|
5971
|
+
var import_node_child_process = require("node:child_process");
|
|
5972
|
+
var cache = /* @__PURE__ */ new Map();
|
|
5973
|
+
var GIT_TIMEOUT_MS = 3e3;
|
|
5974
|
+
function runGit(cwd, args, timeout) {
|
|
5975
|
+
return (0, import_node_child_process.spawnSync)("git", args, {
|
|
5976
|
+
cwd,
|
|
5977
|
+
encoding: "utf8",
|
|
5978
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
5979
|
+
...timeout === void 0 ? {} : { timeout },
|
|
5980
|
+
windowsHide: true
|
|
5981
|
+
});
|
|
5982
|
+
}
|
|
5983
|
+
function isTransient(r) {
|
|
5984
|
+
const code = r.error?.code;
|
|
5985
|
+
if (code !== void 0 && code !== "ENOENT")
|
|
5986
|
+
return true;
|
|
5987
|
+
return r.signal != null;
|
|
5988
|
+
}
|
|
5989
|
+
function foldKeyPath(p, platform2 = process.platform) {
|
|
5990
|
+
if (/^[A-Za-z]:\//.test(p))
|
|
5991
|
+
return p.toLowerCase();
|
|
5992
|
+
if (/^\/(?!\/)/.test(p))
|
|
5993
|
+
return p;
|
|
5994
|
+
return platform2 === "win32" ? p.toLowerCase() : p;
|
|
5995
|
+
}
|
|
5996
|
+
function wslUncToPosix(p) {
|
|
5997
|
+
let s = p.trim();
|
|
5998
|
+
if (/^[\\/]{2}\?[\\/]UNC[\\/]/i.test(s))
|
|
5999
|
+
s = "//" + s.slice(8);
|
|
6000
|
+
else if (/^[\\/]{2}\?[\\/]/.test(s))
|
|
6001
|
+
s = s.slice(4);
|
|
6002
|
+
const m = /^[\\/]{2}(?:wsl\$|wsl\.localhost)[\\/]+([^\\/]+)(?:[\\/]+(.*))?$/i.exec(s);
|
|
6003
|
+
if (!m)
|
|
6004
|
+
return void 0;
|
|
6005
|
+
const rest = (m[2] ?? "").replace(/[\\/]+/g, "/").replace(/^\/+/, "").replace(/\/+$/, "");
|
|
6006
|
+
return { distro: m[1], posix: "/" + rest };
|
|
6007
|
+
}
|
|
6008
|
+
var MAX_RESOLVE_DEPTH = 64;
|
|
6009
|
+
function resolveCaseInsensitive(p) {
|
|
6010
|
+
if ((0, import_node_fs5.existsSync)(p))
|
|
6011
|
+
return p;
|
|
6012
|
+
const home = (0, import_node_os3.homedir)();
|
|
6013
|
+
if (!home || home === "/")
|
|
6014
|
+
return void 0;
|
|
6015
|
+
const prefix = home.replace(/\/+$/, "") + "/";
|
|
6016
|
+
if (!p.toLowerCase().startsWith(prefix.toLowerCase()))
|
|
6017
|
+
return void 0;
|
|
6018
|
+
let current = prefix.slice(0, -1);
|
|
6019
|
+
if (!(0, import_node_fs5.existsSync)(current))
|
|
6020
|
+
return void 0;
|
|
6021
|
+
const segments = p.slice(prefix.length).split("/").filter((seg) => seg.length > 0);
|
|
6022
|
+
if (segments.length > MAX_RESOLVE_DEPTH)
|
|
6023
|
+
return void 0;
|
|
6024
|
+
for (const seg of segments) {
|
|
6025
|
+
const exact = current + "/" + seg;
|
|
6026
|
+
if ((0, import_node_fs5.existsSync)(exact)) {
|
|
6027
|
+
current = exact;
|
|
6028
|
+
continue;
|
|
6029
|
+
}
|
|
6030
|
+
let entries;
|
|
6031
|
+
try {
|
|
6032
|
+
entries = (0, import_node_fs5.readdirSync)(current);
|
|
6033
|
+
} catch {
|
|
6034
|
+
return void 0;
|
|
6035
|
+
}
|
|
6036
|
+
const lower = seg.toLowerCase();
|
|
6037
|
+
const hits = entries.filter((e) => e.toLowerCase() === lower);
|
|
6038
|
+
if (hits.length !== 1)
|
|
6039
|
+
return void 0;
|
|
6040
|
+
current = current + "/" + hits[0];
|
|
6041
|
+
}
|
|
6042
|
+
return (0, import_node_fs5.existsSync)(current) ? current : void 0;
|
|
6043
|
+
}
|
|
6044
|
+
function pathKey(p) {
|
|
6045
|
+
const unc = wslUncToPosix(p);
|
|
6046
|
+
if (unc)
|
|
6047
|
+
return "path:" + unc.posix;
|
|
6048
|
+
return "path:" + foldKeyPath(normalizePath(p));
|
|
6049
|
+
}
|
|
6050
|
+
var upgradeCache = /* @__PURE__ */ new Map();
|
|
6051
|
+
function upgradeLocalPathKey(key) {
|
|
6052
|
+
if (!key.startsWith("path:"))
|
|
6053
|
+
return key;
|
|
6054
|
+
const hit = upgradeCache.get(key);
|
|
6055
|
+
if (hit !== void 0)
|
|
6056
|
+
return hit;
|
|
6057
|
+
const answer = (() => {
|
|
6058
|
+
const raw = key.slice("path:".length);
|
|
6059
|
+
const unc = wslUncToPosix(raw);
|
|
6060
|
+
const p = unc ? unc.posix : raw;
|
|
6061
|
+
if (!/^\/(?!\/)/.test(p))
|
|
6062
|
+
return key;
|
|
6063
|
+
const dir = resolveCaseInsensitive(p);
|
|
6064
|
+
if (dir === void 0)
|
|
6065
|
+
return key;
|
|
6066
|
+
const r = deriveProjectKey(dir);
|
|
6067
|
+
if (r.key && (r.kind === "git" || r.kind === "origin"))
|
|
6068
|
+
return r.key;
|
|
6069
|
+
return key;
|
|
6070
|
+
})();
|
|
6071
|
+
upgradeCache.set(key, answer);
|
|
6072
|
+
return answer;
|
|
6073
|
+
}
|
|
6074
|
+
function normalizeOrigin(url) {
|
|
6075
|
+
let s = url.trim();
|
|
6076
|
+
if (!s)
|
|
6077
|
+
return void 0;
|
|
6078
|
+
if (/^file:\/\//i.test(s))
|
|
6079
|
+
return void 0;
|
|
6080
|
+
if (/^[A-Za-z]:/.test(s))
|
|
6081
|
+
return void 0;
|
|
6082
|
+
const scheme = /^([A-Za-z][A-Za-z0-9+.-]*:)?\/\//.exec(s);
|
|
6083
|
+
if (scheme) {
|
|
6084
|
+
s = s.slice(scheme[0].length);
|
|
6085
|
+
} else if (s.includes("://")) {
|
|
6086
|
+
return void 0;
|
|
6087
|
+
}
|
|
6088
|
+
const firstSlash = s.indexOf("/");
|
|
6089
|
+
const authEnd = firstSlash < 0 ? s.length : firstSlash;
|
|
6090
|
+
const at = s.lastIndexOf("@", authEnd - 1);
|
|
6091
|
+
if (at >= 0)
|
|
6092
|
+
s = s.slice(at + 1);
|
|
6093
|
+
if (!scheme) {
|
|
6094
|
+
if (s.startsWith("/") || s.startsWith("."))
|
|
6095
|
+
return void 0;
|
|
6096
|
+
const colon = s.indexOf(":");
|
|
6097
|
+
if (colon < 0)
|
|
6098
|
+
return void 0;
|
|
6099
|
+
s = s.slice(0, colon) + "/" + s.slice(colon + 1).replace(/^\/+/, "");
|
|
6100
|
+
}
|
|
6101
|
+
const slash = s.indexOf("/");
|
|
6102
|
+
let host = slash < 0 ? s : s.slice(0, slash);
|
|
6103
|
+
const rest = slash < 0 ? "" : s.slice(slash);
|
|
6104
|
+
const port = /^(.*):(\d+)$/.exec(host);
|
|
6105
|
+
if (port)
|
|
6106
|
+
host = port[1];
|
|
6107
|
+
if (!host)
|
|
6108
|
+
return void 0;
|
|
6109
|
+
s = host + rest;
|
|
6110
|
+
s = s.replace(/\/+$/, "");
|
|
6111
|
+
s = s.replace(/\.git$/i, "");
|
|
6112
|
+
s = s.replace(/\/+$/, "");
|
|
6113
|
+
if (!s.includes("/"))
|
|
6114
|
+
return void 0;
|
|
6115
|
+
return s.toLowerCase();
|
|
6116
|
+
}
|
|
6117
|
+
function transient() {
|
|
6118
|
+
return { key: void 0, kind: "transient", transientFailure: true };
|
|
6119
|
+
}
|
|
6120
|
+
function deriveProjectKey(cwd) {
|
|
6121
|
+
const cacheKey = normalizePath(cwd);
|
|
6122
|
+
const hit = cache.get(cacheKey);
|
|
6123
|
+
if (hit)
|
|
6124
|
+
return hit;
|
|
6125
|
+
const result = derive(cwd);
|
|
6126
|
+
if (!result.transientFailure)
|
|
6127
|
+
cache.set(cacheKey, result);
|
|
6128
|
+
return result;
|
|
6129
|
+
}
|
|
6130
|
+
function derive(cwd) {
|
|
6131
|
+
if (!(0, import_node_fs5.existsSync)(cwd))
|
|
6132
|
+
return { kind: "path", key: pathKey(cwd) };
|
|
6133
|
+
let rev = runGit(cwd, ["rev-parse", "--is-shallow-repository", "--show-toplevel"], GIT_TIMEOUT_MS);
|
|
6134
|
+
if (rev.error?.code === "ENOENT") {
|
|
6135
|
+
return { kind: "path", key: pathKey(cwd), gitMissing: true };
|
|
6136
|
+
}
|
|
6137
|
+
if (isTransient(rev)) {
|
|
6138
|
+
rev = runGit(cwd, ["rev-parse", "--is-shallow-repository", "--show-toplevel"]);
|
|
6139
|
+
if (rev.error?.code === "ENOENT") {
|
|
6140
|
+
return { kind: "path", key: pathKey(cwd), gitMissing: true };
|
|
6141
|
+
}
|
|
6142
|
+
if (isTransient(rev))
|
|
6143
|
+
return transient();
|
|
6144
|
+
}
|
|
6145
|
+
if (rev.status === 128)
|
|
6146
|
+
return { kind: "path", key: pathKey(cwd) };
|
|
6147
|
+
if (rev.status !== 0)
|
|
6148
|
+
return { kind: "path", key: pathKey(cwd) };
|
|
6149
|
+
const lines = (rev.stdout ?? "").split("\n").map((l) => l.trim()).filter((l) => l.length > 0);
|
|
6150
|
+
const shallow = lines[0] === "true";
|
|
6151
|
+
const toplevel = lines[1];
|
|
6152
|
+
if (!shallow) {
|
|
6153
|
+
const args = ["--no-replace-objects", "rev-list", "--max-parents=0", "--format=%ct %H", "HEAD"];
|
|
6154
|
+
let roots = runGit(cwd, args, GIT_TIMEOUT_MS);
|
|
6155
|
+
if (isTransient(roots)) {
|
|
6156
|
+
roots = runGit(cwd, args);
|
|
6157
|
+
if (isTransient(roots))
|
|
6158
|
+
return transient();
|
|
6159
|
+
}
|
|
6160
|
+
if (roots.status === 0) {
|
|
6161
|
+
const parsed = [];
|
|
6162
|
+
for (const line of (roots.stdout ?? "").split("\n")) {
|
|
6163
|
+
const t = line.trim();
|
|
6164
|
+
if (!t || t.startsWith("commit "))
|
|
6165
|
+
continue;
|
|
6166
|
+
const m = /^(\d+)\s+([0-9a-f]{40})$/.exec(t);
|
|
6167
|
+
if (m)
|
|
6168
|
+
parsed.push({ ct: Number(m[1]), hash: m[2] });
|
|
6169
|
+
}
|
|
6170
|
+
parsed.sort((a, b) => a.ct - b.ct || (a.hash < b.hash ? -1 : a.hash > b.hash ? 1 : 0));
|
|
6171
|
+
const first = parsed[0];
|
|
6172
|
+
if (first) {
|
|
6173
|
+
return { kind: "git", key: "git:" + first.hash, ...toplevel ? { toplevel } : {} };
|
|
6174
|
+
}
|
|
6175
|
+
}
|
|
6176
|
+
}
|
|
6177
|
+
let origin = runGit(cwd, ["config", "--get", "remote.origin.url"], GIT_TIMEOUT_MS);
|
|
6178
|
+
if (isTransient(origin)) {
|
|
6179
|
+
origin = runGit(cwd, ["config", "--get", "remote.origin.url"]);
|
|
6180
|
+
if (isTransient(origin))
|
|
6181
|
+
return transient();
|
|
6182
|
+
}
|
|
6183
|
+
if (origin.status === 0) {
|
|
6184
|
+
const value = normalizeOrigin(origin.stdout ?? "");
|
|
6185
|
+
if (value)
|
|
6186
|
+
return { kind: "origin", key: "origin:" + value, ...toplevel ? { toplevel } : {} };
|
|
6187
|
+
}
|
|
6188
|
+
return { kind: "path", key: pathKey(toplevel ?? cwd), ...toplevel ? { toplevel } : {} };
|
|
6189
|
+
}
|
|
6190
|
+
|
|
6191
|
+
// src/recall/mirror-meta.ts
|
|
6192
|
+
var import_node_fs6 = require("node:fs");
|
|
6193
|
+
var import_node_path6 = require("node:path");
|
|
6194
|
+
init_paths();
|
|
6195
|
+
function isUnderRemoteRoot(p) {
|
|
6196
|
+
return (0, import_node_path6.resolve)(p).startsWith((0, import_node_path6.resolve)(remoteRoot()) + import_node_path6.sep);
|
|
6197
|
+
}
|
|
6198
|
+
function readMirrorMeta(transcriptPath) {
|
|
6199
|
+
let parsed;
|
|
6200
|
+
try {
|
|
6201
|
+
parsed = JSON.parse((0, import_node_fs6.readFileSync)(`${transcriptPath}.meta.json`, "utf-8"));
|
|
6202
|
+
} catch {
|
|
6203
|
+
return null;
|
|
6204
|
+
}
|
|
6205
|
+
if (!parsed || typeof parsed !== "object")
|
|
6206
|
+
return null;
|
|
6207
|
+
const m = parsed;
|
|
6208
|
+
if (m["v"] !== 1)
|
|
6209
|
+
return null;
|
|
6210
|
+
if (m["key"] !== void 0 && typeof m["key"] !== "string")
|
|
6211
|
+
return null;
|
|
6212
|
+
if (m["cwd"] !== void 0 && typeof m["cwd"] !== "string")
|
|
6213
|
+
return null;
|
|
6214
|
+
return parsed;
|
|
6215
|
+
}
|
|
6216
|
+
|
|
5579
6217
|
// src/recall/message-ingest.ts
|
|
5580
6218
|
init_log();
|
|
5581
6219
|
|
|
@@ -5605,48 +6243,11 @@ function parseJsonlFile(filepath) {
|
|
|
5605
6243
|
return entries;
|
|
5606
6244
|
} catch (error) {
|
|
5607
6245
|
log({ level: "error", source: "jsonl-reader", summary: `Failed to read JSONL file ${filepath}: ${error instanceof Error ? error.message : String(error)}`, data: { filepath, error: String(error) } });
|
|
5608
|
-
|
|
6246
|
+
throw error;
|
|
5609
6247
|
}
|
|
5610
6248
|
}
|
|
5611
6249
|
|
|
5612
6250
|
// src/adapters/claude/claude-entry-adapter.ts
|
|
5613
|
-
function isSystemContextContent(message) {
|
|
5614
|
-
if (!message || message.role !== "user")
|
|
5615
|
-
return false;
|
|
5616
|
-
const content = message.content;
|
|
5617
|
-
let text;
|
|
5618
|
-
if (typeof content === "string") {
|
|
5619
|
-
text = content;
|
|
5620
|
-
} else if (Array.isArray(content)) {
|
|
5621
|
-
const firstText = content.find(
|
|
5622
|
-
(b) => typeof b === "object" && b !== null && b.type === "text"
|
|
5623
|
-
);
|
|
5624
|
-
if (firstText && "text" in firstText) {
|
|
5625
|
-
text = firstText.text;
|
|
5626
|
-
}
|
|
5627
|
-
}
|
|
5628
|
-
if (!text)
|
|
5629
|
-
return false;
|
|
5630
|
-
if (text.startsWith("<system-reminder>"))
|
|
5631
|
-
return true;
|
|
5632
|
-
if (text.startsWith("<environment_context>"))
|
|
5633
|
-
return true;
|
|
5634
|
-
if (text.startsWith("<INSTRUCTIONS>"))
|
|
5635
|
-
return true;
|
|
5636
|
-
if (text.startsWith("# AGENTS.md instructions for"))
|
|
5637
|
-
return true;
|
|
5638
|
-
if (text.startsWith("<context>"))
|
|
5639
|
-
return true;
|
|
5640
|
-
if (text.startsWith("<task-notification>"))
|
|
5641
|
-
return true;
|
|
5642
|
-
if (text.startsWith("<command-name>"))
|
|
5643
|
-
return true;
|
|
5644
|
-
if (text.startsWith("<local-command-stdout>"))
|
|
5645
|
-
return true;
|
|
5646
|
-
if (text.startsWith("<local-command-caveat>"))
|
|
5647
|
-
return true;
|
|
5648
|
-
return false;
|
|
5649
|
-
}
|
|
5650
6251
|
function sanitizeContentBlocks(content) {
|
|
5651
6252
|
if (typeof content === "string")
|
|
5652
6253
|
return content;
|
|
@@ -5866,15 +6467,17 @@ function emitMessage(payload, base, model, counter) {
|
|
|
5866
6467
|
if (role === "developer")
|
|
5867
6468
|
return [];
|
|
5868
6469
|
if (role === "user") {
|
|
6470
|
+
const message = {
|
|
6471
|
+
role: "user",
|
|
6472
|
+
content: adaptContentItems(contentItems)
|
|
6473
|
+
};
|
|
5869
6474
|
return [
|
|
5870
6475
|
{
|
|
5871
6476
|
type: "user",
|
|
5872
6477
|
uuid: generateId(base.sessionId, counter),
|
|
5873
6478
|
...base,
|
|
5874
|
-
message
|
|
5875
|
-
|
|
5876
|
-
content: adaptContentItems(contentItems)
|
|
5877
|
-
}
|
|
6479
|
+
message,
|
|
6480
|
+
...isSystemContextContent(message) && { isMeta: true }
|
|
5878
6481
|
}
|
|
5879
6482
|
];
|
|
5880
6483
|
}
|
|
@@ -6351,6 +6954,11 @@ function parseCodexPatch(input) {
|
|
|
6351
6954
|
currentKind = "update";
|
|
6352
6955
|
continue;
|
|
6353
6956
|
}
|
|
6957
|
+
const moveMatch = line.match(/^\*\*\* Move to:\s*(.+)$/);
|
|
6958
|
+
if (moveMatch && currentKind === "update") {
|
|
6959
|
+
currentPath = moveMatch[1].trim();
|
|
6960
|
+
continue;
|
|
6961
|
+
}
|
|
6354
6962
|
const addMatch = line.match(/^\*\*\* Add File:\s*(.+)$/);
|
|
6355
6963
|
if (addMatch) {
|
|
6356
6964
|
flushCurrent();
|
|
@@ -6386,7 +6994,7 @@ function parseCodexPatch(input) {
|
|
|
6386
6994
|
return changes;
|
|
6387
6995
|
}
|
|
6388
6996
|
function generateId(sessionId, counter) {
|
|
6389
|
-
return `codex-jsonl-${sessionId
|
|
6997
|
+
return `codex-jsonl-${sessionId}-${counter}`;
|
|
6390
6998
|
}
|
|
6391
6999
|
|
|
6392
7000
|
// src/recall/message-ingest.ts
|
|
@@ -6395,12 +7003,19 @@ function extractEntryText(entry) {
|
|
|
6395
7003
|
if (!msg)
|
|
6396
7004
|
return "";
|
|
6397
7005
|
if (typeof msg.content === "string")
|
|
6398
|
-
return msg.content.trim();
|
|
7006
|
+
return msg.content.replaceAll("\0", "").trim();
|
|
6399
7007
|
if (Array.isArray(msg.content)) {
|
|
6400
|
-
return msg.content.filter((b) => b.type === "text").map((b) => b.text?.trim()).filter(Boolean).join("\n\n");
|
|
7008
|
+
return msg.content.filter((b) => b.type === "text").map((b) => b.text?.replaceAll("\0", "").trim()).filter(Boolean).join("\n\n");
|
|
6401
7009
|
}
|
|
6402
7010
|
return "";
|
|
6403
7011
|
}
|
|
7012
|
+
function loadTranscriptEntries(transcriptPath, vendor, canonicalSessionId) {
|
|
7013
|
+
if (vendor === "claude") {
|
|
7014
|
+
const raw = parseJsonlFile(transcriptPath);
|
|
7015
|
+
return adaptClaudeEntries(raw);
|
|
7016
|
+
}
|
|
7017
|
+
return adaptCodexJsonlRecords(parseCodexJsonlFile(transcriptPath), canonicalSessionId);
|
|
7018
|
+
}
|
|
6404
7019
|
function entriesCwd(entries) {
|
|
6405
7020
|
for (const e of entries) {
|
|
6406
7021
|
if (e.cwd)
|
|
@@ -6441,13 +7056,7 @@ async function ingestSessionMessages(sessionId, transcriptPath, vendor, options)
|
|
|
6441
7056
|
}));
|
|
6442
7057
|
let rawEntries;
|
|
6443
7058
|
try {
|
|
6444
|
-
|
|
6445
|
-
const raw = parseJsonlFile(transcriptPath);
|
|
6446
|
-
rawEntries = adaptClaudeEntries(raw);
|
|
6447
|
-
} else {
|
|
6448
|
-
const envelopes = parseCodexJsonlFile(transcriptPath);
|
|
6449
|
-
rawEntries = adaptCodexJsonlRecords(envelopes, canonicalId);
|
|
6450
|
-
}
|
|
7059
|
+
rawEntries = loadTranscriptEntries(transcriptPath, vendor, canonicalId);
|
|
6451
7060
|
} catch (err) {
|
|
6452
7061
|
return {
|
|
6453
7062
|
sessionId: canonicalId,
|
|
@@ -6468,11 +7077,22 @@ async function ingestSessionMessages(sessionId, transcriptPath, vendor, options)
|
|
|
6468
7077
|
}
|
|
6469
7078
|
const rawProjectId = options?.projectId ?? entriesCwd(rawEntries);
|
|
6470
7079
|
const projectId = rawProjectId ? normalizePath(rawProjectId) : null;
|
|
7080
|
+
const localKey = () => {
|
|
7081
|
+
const k = deriveProjectKey(rawProjectId).key ?? null;
|
|
7082
|
+
return k !== null && wslUncToPosix(rawProjectId) ? upgradeLocalPathKey(k) : k;
|
|
7083
|
+
};
|
|
7084
|
+
const sidecarKey = () => {
|
|
7085
|
+
const k = readMirrorMeta(transcriptPath)?.key;
|
|
7086
|
+
return k === void 0 ? null : upgradeLocalPathKey(k);
|
|
7087
|
+
};
|
|
7088
|
+
const projectKey = rawProjectId ? options?.projectKey !== void 0 ? options.projectKey : isUnderRemoteRoot(transcriptPath) ? sidecarKey() : localKey() : null;
|
|
6471
7089
|
const filtered = stripToolContent(rawEntries);
|
|
6472
7090
|
const topLevel = filtered.filter((e) => !e.parentToolUseID);
|
|
6473
7091
|
const records = [];
|
|
6474
7092
|
for (let i = 0; i < topLevel.length; i++) {
|
|
6475
7093
|
const entry = topLevel[i];
|
|
7094
|
+
if (shouldDropAsMeta(entry))
|
|
7095
|
+
continue;
|
|
6476
7096
|
if (!entry.uuid)
|
|
6477
7097
|
continue;
|
|
6478
7098
|
const text = extractEntryText(entry);
|
|
@@ -6487,7 +7107,8 @@ async function ingestSessionMessages(sessionId, transcriptPath, vendor, options)
|
|
|
6487
7107
|
project_id: projectId,
|
|
6488
7108
|
created_at: createdAt,
|
|
6489
7109
|
message_role: entry.message?.role ?? entry.type ?? null,
|
|
6490
|
-
retrieval_class: retrievalClass
|
|
7110
|
+
retrieval_class: retrievalClass,
|
|
7111
|
+
project_key: projectKey
|
|
6491
7112
|
});
|
|
6492
7113
|
}
|
|
6493
7114
|
if (records.length === 0) {
|
|
@@ -6499,11 +7120,19 @@ async function ingestSessionMessages(sessionId, transcriptPath, vendor, options)
|
|
|
6499
7120
|
}
|
|
6500
7121
|
return { sessionId: canonicalId, chunksCreated: 0, skipped: true, retrievalClass };
|
|
6501
7122
|
}
|
|
7123
|
+
let inserted = 0;
|
|
7124
|
+
const previouslyIndexed = hasSessionMessages(canonicalId);
|
|
6502
7125
|
try {
|
|
6503
|
-
insertMessages(records, {
|
|
7126
|
+
inserted = insertMessages(records, {
|
|
7127
|
+
sourceMessageIds: topLevel.flatMap((e) => e.uuid ? [e.uuid] : []),
|
|
6504
7128
|
...options?.force ? { replaceSessionId: canonicalId } : {},
|
|
6505
7129
|
provenance,
|
|
6506
|
-
aliases
|
|
7130
|
+
aliases,
|
|
7131
|
+
// Same identity this chunk's own rows get; back-fills the session's
|
|
7132
|
+
// NULL-identity rows atomically with the insert (M2). Runs on every
|
|
7133
|
+
// ingest path — push, sweep, backfill — so a sweep of a mirror whose
|
|
7134
|
+
// sidecar now carries the key repairs rows without a new push.
|
|
7135
|
+
adopt: { sessionId: canonicalId, projectKey, projectId }
|
|
6507
7136
|
});
|
|
6508
7137
|
} catch (err) {
|
|
6509
7138
|
return {
|
|
@@ -6514,9 +7143,13 @@ async function ingestSessionMessages(sessionId, transcriptPath, vendor, options)
|
|
|
6514
7143
|
retrievalClass
|
|
6515
7144
|
};
|
|
6516
7145
|
}
|
|
7146
|
+
if (!previouslyIndexed && inserted < records.length) {
|
|
7147
|
+
log({ source: "recall:ingest", level: "warn", summary: `Session ${canonicalId}: ${records.length} records offered, ${inserted} inserted (duplicate message IDs)` });
|
|
7148
|
+
}
|
|
6517
7149
|
return {
|
|
6518
7150
|
sessionId: canonicalId,
|
|
6519
|
-
chunksCreated:
|
|
7151
|
+
chunksCreated: inserted,
|
|
7152
|
+
recordsOffered: records.length,
|
|
6520
7153
|
skipped: false,
|
|
6521
7154
|
retrievalClass
|
|
6522
7155
|
};
|
|
@@ -6527,9 +7160,10 @@ var SAFE_EMBED_CHARS = 6e3;
|
|
|
6527
7160
|
async function embedRowsResilient(rows) {
|
|
6528
7161
|
const { embedBatch: embedBatch2 } = await Promise.resolve().then(() => (init_embedder(), embedder_exports));
|
|
6529
7162
|
const { quantizeToQ8: quantizeToQ82, computeNorm: computeNorm2 } = await Promise.resolve().then(() => (init_quantize(), quantize_exports));
|
|
6530
|
-
rows = rows.map((r) => ({ messageId: r.messageId, text: DOC_PREFIX + r.text }));
|
|
7163
|
+
rows = rows.map((r) => ({ messageId: r.messageId, text: DOC_PREFIX + r.text.replaceAll("\0", "") }));
|
|
6531
7164
|
const toRecord = (messageId, f32) => {
|
|
6532
7165
|
const { q8, scale } = quantizeToQ82(f32);
|
|
7166
|
+
recordEmbedSuccess(messageId);
|
|
6533
7167
|
return { messageId, embeddingQ8: q8, norm: computeNorm2(f32), quantScale: scale };
|
|
6534
7168
|
};
|
|
6535
7169
|
try {
|
|
@@ -6546,6 +7180,7 @@ async function embedRowsResilient(rows) {
|
|
|
6546
7180
|
const [v] = await embedBatch2([r.text.slice(0, SAFE_EMBED_CHARS)]);
|
|
6547
7181
|
records.push(toRecord(r.messageId, v));
|
|
6548
7182
|
} catch (err) {
|
|
7183
|
+
recordEmbedFailure(r.messageId, String(err));
|
|
6549
7184
|
log({
|
|
6550
7185
|
source: "recall:embed",
|
|
6551
7186
|
level: "warn",
|
|
@@ -6574,8 +7209,11 @@ async function embedSessionMessages(sessionId, force) {
|
|
|
6574
7209
|
ORDER BY m.message_seq ASC`,
|
|
6575
7210
|
force ? [sessionId] : [sessionId, EMBED_VERSION]
|
|
6576
7211
|
);
|
|
7212
|
+
const exhausted = new Set(force ? [] : exhaustedEmbedMessageIds());
|
|
6577
7213
|
const validRows = [];
|
|
6578
7214
|
for (const r of rows) {
|
|
7215
|
+
if (exhausted.has(r.message_id))
|
|
7216
|
+
continue;
|
|
6579
7217
|
const messageText = r.message_text.trim();
|
|
6580
7218
|
if (!messageText)
|
|
6581
7219
|
continue;
|
|
@@ -6608,8 +7246,11 @@ async function embedMessageBatch(messages) {
|
|
|
6608
7246
|
text: text.length > MAX_EMBED_CHARS ? text.slice(0, MAX_EMBED_CHARS) : text
|
|
6609
7247
|
});
|
|
6610
7248
|
}
|
|
6611
|
-
if (truncated.length === 0)
|
|
7249
|
+
if (truncated.length === 0) {
|
|
7250
|
+
for (const m of messages)
|
|
7251
|
+
recordEmbedFailure(m.message_id, "No non-whitespace embedding input");
|
|
6612
7252
|
return 0;
|
|
7253
|
+
}
|
|
6613
7254
|
const records = await embedRowsResilient(truncated);
|
|
6614
7255
|
insertMessageVectors(records);
|
|
6615
7256
|
return records.length;
|
|
@@ -6619,11 +7260,11 @@ async function embedMessageBatch(messages) {
|
|
|
6619
7260
|
init_embedder();
|
|
6620
7261
|
|
|
6621
7262
|
// src/recall/embed-lock.ts
|
|
6622
|
-
var
|
|
6623
|
-
var
|
|
7263
|
+
var import_node_fs9 = require("node:fs");
|
|
7264
|
+
var import_node_path9 = require("node:path");
|
|
6624
7265
|
init_paths();
|
|
6625
7266
|
function embedLockPath() {
|
|
6626
|
-
return (0,
|
|
7267
|
+
return (0, import_node_path9.join)(runDir(), "embed.lock");
|
|
6627
7268
|
}
|
|
6628
7269
|
var STALE_MS = 30 * 60 * 1e3;
|
|
6629
7270
|
function isAlive(pid) {
|
|
@@ -6635,23 +7276,27 @@ function isAlive(pid) {
|
|
|
6635
7276
|
}
|
|
6636
7277
|
}
|
|
6637
7278
|
function tryAcquireEmbedLock() {
|
|
7279
|
+
try {
|
|
7280
|
+
(0, import_node_fs9.mkdirSync)(runDir(), { recursive: true });
|
|
7281
|
+
} catch {
|
|
7282
|
+
}
|
|
6638
7283
|
for (let attempt = 0; attempt < 2; attempt++) {
|
|
6639
7284
|
try {
|
|
6640
|
-
(0,
|
|
7285
|
+
(0, import_node_fs9.writeFileSync)(embedLockPath(), String(process.pid), { flag: "wx" });
|
|
6641
7286
|
return true;
|
|
6642
7287
|
} catch {
|
|
6643
7288
|
let heldPid;
|
|
6644
7289
|
let ageMs;
|
|
6645
7290
|
try {
|
|
6646
|
-
heldPid = parseInt((0,
|
|
6647
|
-
ageMs = Date.now() - (0,
|
|
7291
|
+
heldPid = parseInt((0, import_node_fs9.readFileSync)(embedLockPath(), "utf8"), 10);
|
|
7292
|
+
ageMs = Date.now() - (0, import_node_fs9.statSync)(embedLockPath()).mtimeMs;
|
|
6648
7293
|
} catch {
|
|
6649
7294
|
continue;
|
|
6650
7295
|
}
|
|
6651
7296
|
if (isAlive(heldPid) && ageMs < STALE_MS)
|
|
6652
7297
|
return false;
|
|
6653
7298
|
try {
|
|
6654
|
-
(0,
|
|
7299
|
+
(0, import_node_fs9.unlinkSync)(embedLockPath());
|
|
6655
7300
|
} catch {
|
|
6656
7301
|
}
|
|
6657
7302
|
}
|
|
@@ -6660,9 +7305,9 @@ function tryAcquireEmbedLock() {
|
|
|
6660
7305
|
}
|
|
6661
7306
|
function releaseEmbedLock() {
|
|
6662
7307
|
try {
|
|
6663
|
-
const held = parseInt((0,
|
|
7308
|
+
const held = parseInt((0, import_node_fs9.readFileSync)(embedLockPath(), "utf8"), 10);
|
|
6664
7309
|
if (held === process.pid)
|
|
6665
|
-
(0,
|
|
7310
|
+
(0, import_node_fs9.unlinkSync)(embedLockPath());
|
|
6666
7311
|
} catch {
|
|
6667
7312
|
}
|
|
6668
7313
|
}
|
|
@@ -7476,8 +8121,8 @@ var path = {
|
|
|
7476
8121
|
win32: { sep: "\\" },
|
|
7477
8122
|
posix: { sep: "/" }
|
|
7478
8123
|
};
|
|
7479
|
-
var
|
|
7480
|
-
minimatch.sep =
|
|
8124
|
+
var sep2 = defaultPlatform === "win32" ? path.win32.sep : path.posix.sep;
|
|
8125
|
+
minimatch.sep = sep2;
|
|
7481
8126
|
var GLOBSTAR = Symbol("globstar **");
|
|
7482
8127
|
minimatch.GLOBSTAR = GLOBSTAR;
|
|
7483
8128
|
var qmark2 = "[^/]";
|
|
@@ -8206,7 +8851,7 @@ var import_node_url2 = require("node:url");
|
|
|
8206
8851
|
|
|
8207
8852
|
// node_modules/lru-cache/dist/esm/index.js
|
|
8208
8853
|
var perf = typeof performance === "object" && performance && typeof performance.now === "function" ? performance : Date;
|
|
8209
|
-
var
|
|
8854
|
+
var warned2 = /* @__PURE__ */ new Set();
|
|
8210
8855
|
var PROCESS = typeof process === "object" && !!process ? process : {};
|
|
8211
8856
|
var emitWarning = (msg, type, code, fn) => {
|
|
8212
8857
|
typeof PROCESS.emitWarning === "function" ? PROCESS.emitWarning(msg, type, code, fn) : console.error(`[${code}] ${type}: ${msg}`);
|
|
@@ -8247,7 +8892,7 @@ if (typeof AC === "undefined") {
|
|
|
8247
8892
|
emitWarning("AbortController is not defined. If using lru-cache in node 14, load an AbortController polyfill from the `node-abort-controller` package. A minimal polyfill is provided for use by LRUCache.fetch(), but it should not be relied upon in other contexts (eg, passing it to other APIs that use AbortController/AbortSignal might have undesirable effects). You may disable this with LRU_CACHE_IGNORE_AC_WARNING=1 in the env.", "NO_ABORT_CONTROLLER", "ENOTSUP", warnACPolyfill);
|
|
8248
8893
|
};
|
|
8249
8894
|
}
|
|
8250
|
-
var shouldWarn = (code) => !
|
|
8895
|
+
var shouldWarn = (code) => !warned2.has(code);
|
|
8251
8896
|
var TYPE = Symbol("type");
|
|
8252
8897
|
var isPosInt = (n) => n && n === Math.floor(n) && n > 0 && isFinite(n);
|
|
8253
8898
|
var getUintArray = (max) => !isPosInt(max) ? null : max <= Math.pow(2, 8) ? Uint8Array : max <= Math.pow(2, 16) ? Uint16Array : max <= Math.pow(2, 32) ? Uint32Array : max <= Number.MAX_SAFE_INTEGER ? ZeroArray : null;
|
|
@@ -8541,7 +9186,7 @@ var LRUCache = class _LRUCache {
|
|
|
8541
9186
|
if (!this.ttlAutopurge && !this.#max && !this.#maxSize) {
|
|
8542
9187
|
const code = "LRU_CACHE_UNBOUNDED";
|
|
8543
9188
|
if (shouldWarn(code)) {
|
|
8544
|
-
|
|
9189
|
+
warned2.add(code);
|
|
8545
9190
|
const msg = "TTL caching without ttlAutopurge, max, or maxSize can result in unbounded memory consumption.";
|
|
8546
9191
|
emitWarning(msg, "UnboundedCacheWarning", code, _LRUCache);
|
|
8547
9192
|
}
|
|
@@ -9574,7 +10219,7 @@ var LRUCache = class _LRUCache {
|
|
|
9574
10219
|
};
|
|
9575
10220
|
|
|
9576
10221
|
// node_modules/path-scurry/dist/esm/index.js
|
|
9577
|
-
var
|
|
10222
|
+
var import_node_path10 = require("node:path");
|
|
9578
10223
|
var import_node_url = require("node:url");
|
|
9579
10224
|
var import_fs = require("fs");
|
|
9580
10225
|
var actualFS = __toESM(require("node:fs"), 1);
|
|
@@ -10309,10 +10954,10 @@ var Minipass = class extends import_node_events.EventEmitter {
|
|
|
10309
10954
|
* Return a void Promise that resolves once the stream ends.
|
|
10310
10955
|
*/
|
|
10311
10956
|
async promise() {
|
|
10312
|
-
return new Promise((
|
|
10957
|
+
return new Promise((resolve3, reject) => {
|
|
10313
10958
|
this.on(DESTROYED, () => reject(new Error("stream destroyed")));
|
|
10314
10959
|
this.on("error", (er) => reject(er));
|
|
10315
|
-
this.on("end", () =>
|
|
10960
|
+
this.on("end", () => resolve3());
|
|
10316
10961
|
});
|
|
10317
10962
|
}
|
|
10318
10963
|
/**
|
|
@@ -10336,7 +10981,7 @@ var Minipass = class extends import_node_events.EventEmitter {
|
|
|
10336
10981
|
return Promise.resolve({ done: false, value: res });
|
|
10337
10982
|
if (this[EOF])
|
|
10338
10983
|
return stop();
|
|
10339
|
-
let
|
|
10984
|
+
let resolve3;
|
|
10340
10985
|
let reject;
|
|
10341
10986
|
const onerr = (er) => {
|
|
10342
10987
|
this.off("data", ondata);
|
|
@@ -10350,19 +10995,19 @@ var Minipass = class extends import_node_events.EventEmitter {
|
|
|
10350
10995
|
this.off("end", onend);
|
|
10351
10996
|
this.off(DESTROYED, ondestroy);
|
|
10352
10997
|
this.pause();
|
|
10353
|
-
|
|
10998
|
+
resolve3({ value, done: !!this[EOF] });
|
|
10354
10999
|
};
|
|
10355
11000
|
const onend = () => {
|
|
10356
11001
|
this.off("error", onerr);
|
|
10357
11002
|
this.off("data", ondata);
|
|
10358
11003
|
this.off(DESTROYED, ondestroy);
|
|
10359
11004
|
stop();
|
|
10360
|
-
|
|
11005
|
+
resolve3({ done: true, value: void 0 });
|
|
10361
11006
|
};
|
|
10362
11007
|
const ondestroy = () => onerr(new Error("stream destroyed"));
|
|
10363
11008
|
return new Promise((res2, rej) => {
|
|
10364
11009
|
reject = rej;
|
|
10365
|
-
|
|
11010
|
+
resolve3 = res2;
|
|
10366
11011
|
this.once(DESTROYED, ondestroy);
|
|
10367
11012
|
this.once("error", onerr);
|
|
10368
11013
|
this.once("end", onend);
|
|
@@ -11334,9 +11979,9 @@ var PathBase = class {
|
|
|
11334
11979
|
if (this.#asyncReaddirInFlight) {
|
|
11335
11980
|
await this.#asyncReaddirInFlight;
|
|
11336
11981
|
} else {
|
|
11337
|
-
let
|
|
11982
|
+
let resolve3 = () => {
|
|
11338
11983
|
};
|
|
11339
|
-
this.#asyncReaddirInFlight = new Promise((res) =>
|
|
11984
|
+
this.#asyncReaddirInFlight = new Promise((res) => resolve3 = res);
|
|
11340
11985
|
try {
|
|
11341
11986
|
for (const e of await this.#fs.promises.readdir(fullpath, {
|
|
11342
11987
|
withFileTypes: true
|
|
@@ -11349,7 +11994,7 @@ var PathBase = class {
|
|
|
11349
11994
|
children.provisional = 0;
|
|
11350
11995
|
}
|
|
11351
11996
|
this.#asyncReaddirInFlight = void 0;
|
|
11352
|
-
|
|
11997
|
+
resolve3();
|
|
11353
11998
|
}
|
|
11354
11999
|
return children.slice(0, children.provisional);
|
|
11355
12000
|
}
|
|
@@ -11483,7 +12128,7 @@ var PathWin32 = class _PathWin32 extends PathBase {
|
|
|
11483
12128
|
* @internal
|
|
11484
12129
|
*/
|
|
11485
12130
|
getRootString(path2) {
|
|
11486
|
-
return
|
|
12131
|
+
return import_node_path10.win32.parse(path2).root;
|
|
11487
12132
|
}
|
|
11488
12133
|
/**
|
|
11489
12134
|
* @internal
|
|
@@ -11579,7 +12224,7 @@ var PathScurryBase = class {
|
|
|
11579
12224
|
*
|
|
11580
12225
|
* @internal
|
|
11581
12226
|
*/
|
|
11582
|
-
constructor(cwd = process.cwd(), pathImpl,
|
|
12227
|
+
constructor(cwd = process.cwd(), pathImpl, sep3, { nocase, childrenCacheSize = 16 * 1024, fs: fs3 = defaultFS } = {}) {
|
|
11583
12228
|
this.#fs = fsFromOption(fs3);
|
|
11584
12229
|
if (cwd instanceof URL || cwd.startsWith("file://")) {
|
|
11585
12230
|
cwd = (0, import_node_url.fileURLToPath)(cwd);
|
|
@@ -11590,7 +12235,7 @@ var PathScurryBase = class {
|
|
|
11590
12235
|
this.#resolveCache = new ResolveCache();
|
|
11591
12236
|
this.#resolvePosixCache = new ResolveCache();
|
|
11592
12237
|
this.#children = new ChildrenCache(childrenCacheSize);
|
|
11593
|
-
const split = cwdPath.substring(this.rootPath.length).split(
|
|
12238
|
+
const split = cwdPath.substring(this.rootPath.length).split(sep3);
|
|
11594
12239
|
if (split.length === 1 && !split[0]) {
|
|
11595
12240
|
split.pop();
|
|
11596
12241
|
}
|
|
@@ -12123,7 +12768,7 @@ var PathScurryWin32 = class extends PathScurryBase {
|
|
|
12123
12768
|
sep = "\\";
|
|
12124
12769
|
constructor(cwd = process.cwd(), opts = {}) {
|
|
12125
12770
|
const { nocase = true } = opts;
|
|
12126
|
-
super(cwd,
|
|
12771
|
+
super(cwd, import_node_path10.win32, "\\", { ...opts, nocase });
|
|
12127
12772
|
this.nocase = nocase;
|
|
12128
12773
|
for (let p = this.cwd; p; p = p.parent) {
|
|
12129
12774
|
p.nocase = this.nocase;
|
|
@@ -12133,7 +12778,7 @@ var PathScurryWin32 = class extends PathScurryBase {
|
|
|
12133
12778
|
* @internal
|
|
12134
12779
|
*/
|
|
12135
12780
|
parseRootPath(dir) {
|
|
12136
|
-
return
|
|
12781
|
+
return import_node_path10.win32.parse(dir).root.toUpperCase();
|
|
12137
12782
|
}
|
|
12138
12783
|
/**
|
|
12139
12784
|
* @internal
|
|
@@ -12155,7 +12800,7 @@ var PathScurryPosix = class extends PathScurryBase {
|
|
|
12155
12800
|
sep = "/";
|
|
12156
12801
|
constructor(cwd = process.cwd(), opts = {}) {
|
|
12157
12802
|
const { nocase = false } = opts;
|
|
12158
|
-
super(cwd,
|
|
12803
|
+
super(cwd, import_node_path10.posix, "/", { ...opts, nocase });
|
|
12159
12804
|
this.nocase = nocase;
|
|
12160
12805
|
}
|
|
12161
12806
|
/**
|
|
@@ -13247,34 +13892,36 @@ var glob = Object.assign(glob_, {
|
|
|
13247
13892
|
glob.glob = glob;
|
|
13248
13893
|
|
|
13249
13894
|
// src/recall/mtime-scan.ts
|
|
13250
|
-
var
|
|
13251
|
-
var import_node_os3 = require("node:os");
|
|
13252
|
-
var import_node_path7 = require("node:path");
|
|
13895
|
+
var import_node_fs10 = require("node:fs");
|
|
13253
13896
|
init_paths();
|
|
13254
13897
|
init_log();
|
|
13255
13898
|
async function mtimeScan(opts) {
|
|
13256
|
-
const vendors = opts?.vendors ?? ["claude", "codex"];
|
|
13257
|
-
const claudeRoot = process.env["CLAUDE_CONFIG_DIR"] ?? (0, import_node_path7.join)((0, import_node_os3.homedir)(), ".claude");
|
|
13258
|
-
const codexRoot = process.env["CODEX_HOME"] ?? (0, import_node_path7.join)((0, import_node_os3.homedir)(), ".codex");
|
|
13259
13899
|
const patterns = [];
|
|
13260
|
-
if (
|
|
13261
|
-
|
|
13262
|
-
|
|
13263
|
-
|
|
13900
|
+
if (opts?.roots) {
|
|
13901
|
+
for (const r of opts.roots) {
|
|
13902
|
+
patterns.push([transcriptGlob(r.root, r.vendor === "claude" ? "projects" : "sessions", "**", "*.jsonl"), r.vendor]);
|
|
13903
|
+
}
|
|
13904
|
+
} else {
|
|
13905
|
+
const vendors = opts?.vendors ?? ["claude", "codex"];
|
|
13906
|
+
if (vendors.includes("claude"))
|
|
13907
|
+
patterns.push([transcriptGlob(claudeRoot(), "projects", "**", "*.jsonl"), "claude"]);
|
|
13908
|
+
if (vendors.includes("codex"))
|
|
13909
|
+
patterns.push([transcriptGlob(codexRoot(), "sessions", "**", "*.jsonl"), "codex"]);
|
|
13910
|
+
}
|
|
13264
13911
|
const db3 = getDb(dbPath());
|
|
13265
13912
|
const watermarks = /* @__PURE__ */ new Map();
|
|
13266
13913
|
const rows = db3.all("SELECT * FROM ingest_watermark");
|
|
13267
13914
|
for (const row of rows) {
|
|
13268
13915
|
watermarks.set(row.transcript_path, row);
|
|
13269
13916
|
}
|
|
13270
|
-
const result = { scanned: 0, unchanged: 0, ingested: 0, failed: 0 };
|
|
13917
|
+
const result = { scanned: 0, unchanged: 0, ingested: 0, refused: 0, failed: 0 };
|
|
13271
13918
|
for (const [pattern, vendor] of patterns) {
|
|
13272
13919
|
const files = await glob(pattern, { nodir: true });
|
|
13273
13920
|
for (const file of files) {
|
|
13274
13921
|
result.scanned++;
|
|
13275
13922
|
let stat;
|
|
13276
13923
|
try {
|
|
13277
|
-
stat = (0,
|
|
13924
|
+
stat = (0, import_node_fs10.statSync)(file);
|
|
13278
13925
|
} catch {
|
|
13279
13926
|
continue;
|
|
13280
13927
|
}
|
|
@@ -13285,6 +13932,24 @@ async function mtimeScan(opts) {
|
|
|
13285
13932
|
continue;
|
|
13286
13933
|
}
|
|
13287
13934
|
const sessionId = sessionIdFromPath(file, vendor);
|
|
13935
|
+
if (opts?.guard) {
|
|
13936
|
+
let refusal = null;
|
|
13937
|
+
try {
|
|
13938
|
+
refusal = opts.guard(file, vendor);
|
|
13939
|
+
} catch (e) {
|
|
13940
|
+
refusal = e.message;
|
|
13941
|
+
}
|
|
13942
|
+
if (refusal !== null) {
|
|
13943
|
+
result.refused++;
|
|
13944
|
+
log({
|
|
13945
|
+
level: "debug",
|
|
13946
|
+
source: "recall:mtime-scan",
|
|
13947
|
+
summary: `Ingest refused, watermark not advanced: ${file}`,
|
|
13948
|
+
data: { path: file, vendor, reason: refusal }
|
|
13949
|
+
});
|
|
13950
|
+
continue;
|
|
13951
|
+
}
|
|
13952
|
+
}
|
|
13288
13953
|
try {
|
|
13289
13954
|
const ingestResult = await ingestSessionMessages(sessionId, file, vendor);
|
|
13290
13955
|
if (ingestResult.error) {
|
|
@@ -13324,7 +13989,7 @@ var CATCHUP_BATCH_SIZE = 80;
|
|
|
13324
13989
|
var MAX_CONSECUTIVE_FAILURES = 3;
|
|
13325
13990
|
var HEARTBEAT_INTERVAL_MS = 5 * 60 * 1e3;
|
|
13326
13991
|
(async () => {
|
|
13327
|
-
(0,
|
|
13992
|
+
(0, import_node_fs11.mkdirSync)(runDir(), { recursive: true });
|
|
13328
13993
|
const sessionId = process.argv[2];
|
|
13329
13994
|
if (!tryAcquireEmbedLock())
|
|
13330
13995
|
process.exit(0);
|
|
@@ -13341,7 +14006,7 @@ var HEARTBEAT_INTERVAL_MS = 5 * 60 * 1e3;
|
|
|
13341
14006
|
process.on("SIGTERM", onSignal);
|
|
13342
14007
|
const heartbeat = setInterval(() => {
|
|
13343
14008
|
try {
|
|
13344
|
-
(0,
|
|
14009
|
+
(0, import_node_fs11.utimesSync)(embedLockPath(), /* @__PURE__ */ new Date(), /* @__PURE__ */ new Date());
|
|
13345
14010
|
} catch {
|
|
13346
14011
|
}
|
|
13347
14012
|
}, HEARTBEAT_INTERVAL_MS);
|
|
@@ -13352,13 +14017,14 @@ var HEARTBEAT_INTERVAL_MS = 5 * 60 * 1e3;
|
|
|
13352
14017
|
await embedSessionMessages(sessionId);
|
|
13353
14018
|
}
|
|
13354
14019
|
let consecutiveFailures = 0;
|
|
13355
|
-
|
|
14020
|
+
let rounds = 0;
|
|
14021
|
+
while (consecutiveFailures < MAX_CONSECUTIVE_FAILURES && rounds++ < 1e4) {
|
|
13356
14022
|
const batch = getUnembeddedMessages(CATCHUP_BATCH_SIZE);
|
|
13357
14023
|
if (batch.length === 0)
|
|
13358
14024
|
break;
|
|
13359
14025
|
try {
|
|
13360
|
-
await embedMessageBatch(batch);
|
|
13361
|
-
consecutiveFailures = 0;
|
|
14026
|
+
const embedded = await embedMessageBatch(batch);
|
|
14027
|
+
consecutiveFailures = embedded === 0 ? consecutiveFailures + 1 : 0;
|
|
13362
14028
|
} catch {
|
|
13363
14029
|
consecutiveFailures++;
|
|
13364
14030
|
}
|
|
@@ -13368,13 +14034,14 @@ var HEARTBEAT_INTERVAL_MS = 5 * 60 * 1e3;
|
|
|
13368
14034
|
const scan = await mtimeScan();
|
|
13369
14035
|
if (scan.ingested > 0) {
|
|
13370
14036
|
let consecutiveFailures = 0;
|
|
13371
|
-
|
|
14037
|
+
let rounds = 0;
|
|
14038
|
+
while (consecutiveFailures < MAX_CONSECUTIVE_FAILURES && rounds++ < 1e4) {
|
|
13372
14039
|
const batch = getUnembeddedMessages(CATCHUP_BATCH_SIZE);
|
|
13373
14040
|
if (batch.length === 0)
|
|
13374
14041
|
break;
|
|
13375
14042
|
try {
|
|
13376
|
-
await embedMessageBatch(batch);
|
|
13377
|
-
consecutiveFailures = 0;
|
|
14043
|
+
const embedded = await embedMessageBatch(batch);
|
|
14044
|
+
consecutiveFailures = embedded === 0 ? consecutiveFailures + 1 : 0;
|
|
13378
14045
|
} catch {
|
|
13379
14046
|
consecutiveFailures++;
|
|
13380
14047
|
}
|