om-memory-system 3.2.0-next.9 → 3.3.0-next.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -5
- package/dist/adapters/opencode/auto-capture-summary.js +50 -8
- package/dist/adapters/opencode/backfill-models.d.ts +25 -0
- package/dist/adapters/opencode/backfill-models.js +39 -0
- package/dist/adapters/opencode/backfill-startup.d.ts +7 -0
- package/dist/adapters/opencode/backfill-startup.js +12 -0
- package/dist/adapters/opencode/import-command.d.ts +3 -7
- package/dist/adapters/opencode/import-command.js +6 -54
- package/dist/adapters/pi/backfill-models.d.ts +9 -0
- package/dist/adapters/pi/backfill-models.js +30 -0
- package/dist/adapters/pi/extension.js +53 -0
- package/dist/adapters/pi/import-command.js +3 -0
- package/dist/adapters/pi/profile.js +2 -1
- package/dist/adapters/pi/provider.js +41 -14
- package/dist/cli/index.js +13 -1
- package/dist/cli/web-command.d.ts +3 -0
- package/dist/cli/web-command.js +72 -0
- package/dist/config.d.ts +30 -1
- package/dist/config.js +131 -9
- package/dist/core/capture-context.js +1 -1
- package/dist/core/capture.d.ts +4 -0
- package/dist/core/capture.js +30 -0
- package/dist/core/extraction.d.ts +23 -1
- package/dist/core/extraction.js +41 -0
- package/dist/core/host.d.ts +23 -0
- package/dist/core/host.js +9 -1
- package/dist/core/profile-analysis.js +9 -1
- package/dist/importer/auto-backfill.d.ts +20 -0
- package/dist/importer/auto-backfill.js +165 -0
- package/dist/importer/backfill-lock.d.ts +3 -0
- package/dist/importer/backfill-lock.js +44 -0
- package/dist/importer/backfill-model.d.ts +10 -0
- package/dist/importer/backfill-model.js +13 -0
- package/dist/importer/discovery.d.ts +32 -5
- package/dist/importer/discovery.js +88 -36
- package/dist/importer/import-args.d.ts +23 -1
- package/dist/importer/import-args.js +29 -1
- package/dist/importer/import-project.d.ts +16 -0
- package/dist/importer/import-project.js +35 -0
- package/dist/importer/import-readiness.d.ts +47 -0
- package/dist/importer/import-readiness.js +54 -0
- package/dist/importer/import-sessions.d.ts +98 -0
- package/dist/importer/import-sessions.js +222 -0
- package/dist/importer/import-sources.d.ts +47 -0
- package/dist/importer/import-sources.js +147 -0
- package/dist/importer/importer.d.ts +31 -1
- package/dist/importer/importer.js +96 -24
- package/dist/importer/manual-import-guard.d.ts +4 -0
- package/dist/importer/manual-import-guard.js +9 -0
- package/dist/importer/opencode-import.d.ts +11 -1
- package/dist/importer/opencode-import.js +133 -69
- package/dist/importer/opencode-project.d.ts +1 -4
- package/dist/importer/opencode-project.js +3 -36
- package/dist/importer/opencode-reader.d.ts +25 -5
- package/dist/importer/opencode-reader.js +92 -106
- package/dist/importer/opencode-snapshot.d.ts +82 -0
- package/dist/importer/opencode-snapshot.js +276 -0
- package/dist/importer/profile-import.d.ts +1 -0
- package/dist/importer/profile-import.js +7 -1
- package/dist/importer/run-import.d.ts +10 -0
- package/dist/importer/run-import.js +21 -0
- package/dist/importer/settings-health.d.ts +24 -0
- package/dist/importer/settings-health.js +127 -0
- package/dist/importer/web-import-api.d.ts +8 -0
- package/dist/importer/web-import-api.js +8 -0
- package/dist/importer/web-import-jobs.d.ts +53 -0
- package/dist/importer/web-import-jobs.js +227 -0
- package/dist/index.js +34 -0
- package/dist/services/ai/live-model-choice.js +6 -3
- package/dist/services/ai/opencode-import-models.d.ts +11 -0
- package/dist/services/ai/opencode-import-models.js +55 -0
- package/dist/services/ai/opencode-provider.d.ts +9 -0
- package/dist/services/ai/opencode-provider.js +31 -4
- package/dist/services/ai/providers/anthropic-messages.js +14 -6
- package/dist/services/ai/providers/base-provider.d.ts +8 -0
- package/dist/services/ai/providers/base-provider.js +12 -0
- package/dist/services/ai/providers/google-gemini.js +21 -7
- package/dist/services/ai/providers/openai-chat-completion.js +20 -7
- package/dist/services/ai/providers/openai-responses.js +11 -0
- package/dist/services/backfill-state.d.ts +25 -0
- package/dist/services/backfill-state.js +68 -0
- package/dist/services/capture-attempt-store.d.ts +21 -0
- package/dist/services/capture-attempt-store.js +108 -0
- package/dist/services/capture-diagnostics.d.ts +66 -0
- package/dist/services/capture-diagnostics.js +175 -0
- package/dist/services/cleanup-service.js +9 -0
- package/dist/services/global-config-writer.d.ts +10 -0
- package/dist/services/global-config-writer.js +110 -0
- package/dist/services/live-captured-entries.d.ts +3 -0
- package/dist/services/live-captured-entries.js +34 -0
- package/dist/services/log-path.d.ts +2 -0
- package/dist/services/log-path.js +15 -0
- package/dist/services/logger.js +1 -12
- package/dist/services/safe-health-error.d.ts +2 -0
- package/dist/services/safe-health-error.js +11 -0
- package/dist/services/settings-log.d.ts +5 -0
- package/dist/services/settings-log.js +36 -0
- package/dist/services/settings-models.d.ts +35 -0
- package/dist/services/settings-models.js +39 -0
- package/dist/services/settings-snapshot.d.ts +24 -0
- package/dist/services/settings-snapshot.js +83 -0
- package/dist/services/settings-traces.d.ts +7 -0
- package/dist/services/settings-traces.js +39 -0
- package/dist/services/user-memory-learning.js +2 -1
- package/dist/services/web-autostart.d.ts +29 -0
- package/dist/services/web-autostart.js +208 -0
- package/dist/services/web-server.d.ts +5 -0
- package/dist/services/web-server.js +187 -3
- package/dist/v2/legacy-client.js +19 -2
- package/dist/web/assets/index-B-_gMSn9.js +124 -0
- package/dist/web/assets/index-D5ak5KsI.css +2 -0
- package/dist/web/index.html +2 -2
- package/package.json +2 -1
- package/dist/web/assets/index-DPjEpD1h.css +0 -2
- package/dist/web/assets/index-do26c4JJ.js +0 -122
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { BackfillHost } from "./backfill-model.js";
|
|
2
|
+
import { runHistoryImport, type HistoryImportModels } from "./run-import.js";
|
|
3
|
+
type Run = typeof runHistoryImport;
|
|
4
|
+
export interface AutoBackfillOptions {
|
|
5
|
+
host: BackfillHost;
|
|
6
|
+
cwd: string;
|
|
7
|
+
resolveModels: () => Promise<{
|
|
8
|
+
model: string;
|
|
9
|
+
models: HistoryImportModels;
|
|
10
|
+
}>;
|
|
11
|
+
notify: (message: string) => void;
|
|
12
|
+
run?: Run;
|
|
13
|
+
wait?: () => Promise<void>;
|
|
14
|
+
now?: () => number;
|
|
15
|
+
enabled?: () => boolean;
|
|
16
|
+
signal?: AbortSignal;
|
|
17
|
+
}
|
|
18
|
+
/** Schedule one host's history import after startup; callers do not await it. */
|
|
19
|
+
export declare function scheduleAutoBackfill(options: AutoBackfillOptions): Promise<void>;
|
|
20
|
+
export {};
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
import { CONFIG, refreshConfigIfChanged } from "../config.js";
|
|
2
|
+
import { log } from "../services/logger.js";
|
|
3
|
+
import { tryAcquireBackfillLock } from "./backfill-lock.js";
|
|
4
|
+
import { getBackfillCutoff, updateBackfillStatus, } from "../services/backfill-state.js";
|
|
5
|
+
import { runHistoryImport, } from "./run-import.js";
|
|
6
|
+
import { isManualImportRunning } from "./manual-import-guard.js";
|
|
7
|
+
const zero = () => ({ imported: 0, skipped: 0, failed: 0, pending: 0, unresolved: 0 });
|
|
8
|
+
function counts(report) {
|
|
9
|
+
return {
|
|
10
|
+
imported: report.unitsImported,
|
|
11
|
+
skipped: report.unitsSkipped,
|
|
12
|
+
failed: report.unitsFailed,
|
|
13
|
+
pending: report.unitsWouldImport,
|
|
14
|
+
unresolved: (report.unresolvedProjects?.length ?? 0) + report.unresolvableSessions.length,
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
const delay = (signal) => new Promise((resolve) => {
|
|
18
|
+
if (signal?.aborted)
|
|
19
|
+
return resolve();
|
|
20
|
+
const done = () => {
|
|
21
|
+
clearTimeout(timer);
|
|
22
|
+
signal?.removeEventListener("abort", done);
|
|
23
|
+
resolve();
|
|
24
|
+
};
|
|
25
|
+
const timer = setTimeout(done, 30_000);
|
|
26
|
+
signal?.addEventListener("abort", done, { once: true });
|
|
27
|
+
});
|
|
28
|
+
/** Schedule one host's history import after startup; callers do not await it. */
|
|
29
|
+
export async function scheduleAutoBackfill(options) {
|
|
30
|
+
await (options.wait ?? (() => delay(options.signal)))();
|
|
31
|
+
if (options.signal?.aborted)
|
|
32
|
+
return;
|
|
33
|
+
if (!(options.enabled ??
|
|
34
|
+
(() => CONFIG.autoBackfill && process.env.OMMS_DISABLE_AUTO_BACKFILL !== "1"))())
|
|
35
|
+
return;
|
|
36
|
+
if (isManualImportRunning(options.host))
|
|
37
|
+
return;
|
|
38
|
+
const release = await tryAcquireBackfillLock(options.host);
|
|
39
|
+
if (!release)
|
|
40
|
+
return;
|
|
41
|
+
const clock = options.now ?? Date.now;
|
|
42
|
+
const run = options.run ?? runHistoryImport;
|
|
43
|
+
const controller = new AbortController();
|
|
44
|
+
const onAbort = () => controller.abort();
|
|
45
|
+
options.signal?.addEventListener("abort", onAbort, { once: true });
|
|
46
|
+
if (options.signal?.aborted)
|
|
47
|
+
controller.abort();
|
|
48
|
+
let state = "failed";
|
|
49
|
+
let model = null;
|
|
50
|
+
let current = zero();
|
|
51
|
+
let lastError;
|
|
52
|
+
try {
|
|
53
|
+
if (controller.signal.aborted) {
|
|
54
|
+
state = "stopped";
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
57
|
+
const cutoff = await getBackfillCutoff(options.host, clock());
|
|
58
|
+
const args = {
|
|
59
|
+
help: false,
|
|
60
|
+
dryRun: true,
|
|
61
|
+
force: false,
|
|
62
|
+
skipMemories: false,
|
|
63
|
+
skipProfile: false,
|
|
64
|
+
scope: "all-projects",
|
|
65
|
+
pathMaps: [],
|
|
66
|
+
until: cutoff,
|
|
67
|
+
errors: [],
|
|
68
|
+
};
|
|
69
|
+
const dry = await run(options.host, args, {
|
|
70
|
+
cwd: options.cwd,
|
|
71
|
+
models: {},
|
|
72
|
+
signal: controller.signal,
|
|
73
|
+
});
|
|
74
|
+
current = counts(dry);
|
|
75
|
+
if (controller.signal.aborted) {
|
|
76
|
+
state = "stopped";
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
79
|
+
if (!current.pending) {
|
|
80
|
+
state = "done";
|
|
81
|
+
return;
|
|
82
|
+
}
|
|
83
|
+
const resolved = await options.resolveModels();
|
|
84
|
+
if (controller.signal.aborted) {
|
|
85
|
+
state = "stopped";
|
|
86
|
+
return;
|
|
87
|
+
}
|
|
88
|
+
model = resolved.model;
|
|
89
|
+
options.notify(`${options.host} automatic import: ${current.pending} exchanges with ${model}. Set autoBackfill to false to stop it.`);
|
|
90
|
+
state = "running";
|
|
91
|
+
await updateBackfillStatus(options.host, { state, model, counts: current });
|
|
92
|
+
if (controller.signal.aborted) {
|
|
93
|
+
state = "stopped";
|
|
94
|
+
return;
|
|
95
|
+
}
|
|
96
|
+
let failures = 0;
|
|
97
|
+
const capture = resolved.models.capture;
|
|
98
|
+
const models = capture
|
|
99
|
+
? {
|
|
100
|
+
...resolved.models,
|
|
101
|
+
capture: {
|
|
102
|
+
summarize: async (request) => {
|
|
103
|
+
try {
|
|
104
|
+
const result = await capture.summarize(request);
|
|
105
|
+
failures = 0;
|
|
106
|
+
return result;
|
|
107
|
+
}
|
|
108
|
+
catch (error) {
|
|
109
|
+
lastError = error;
|
|
110
|
+
if (++failures >= 5)
|
|
111
|
+
controller.abort();
|
|
112
|
+
throw error;
|
|
113
|
+
}
|
|
114
|
+
},
|
|
115
|
+
},
|
|
116
|
+
}
|
|
117
|
+
: resolved.models;
|
|
118
|
+
let checkedAt = clock();
|
|
119
|
+
let processed = 0;
|
|
120
|
+
let pendingWrite = Promise.resolve();
|
|
121
|
+
const real = await run(options.host, { ...args, dryRun: false }, {
|
|
122
|
+
cwd: options.cwd,
|
|
123
|
+
models,
|
|
124
|
+
signal: controller.signal,
|
|
125
|
+
onProgress: (done, total) => {
|
|
126
|
+
processed = done;
|
|
127
|
+
const now = clock();
|
|
128
|
+
if (now - checkedAt < 5_000)
|
|
129
|
+
return;
|
|
130
|
+
checkedAt = now;
|
|
131
|
+
refreshConfigIfChanged(options.cwd);
|
|
132
|
+
if (!CONFIG.autoBackfill)
|
|
133
|
+
controller.abort();
|
|
134
|
+
current = { ...current, pending: Math.max(0, total - done) };
|
|
135
|
+
pendingWrite = pendingWrite
|
|
136
|
+
.then(() => updateBackfillStatus(options.host, { state: "running", model, counts: current }))
|
|
137
|
+
.catch((error) => {
|
|
138
|
+
log("Backfill status write failed", {
|
|
139
|
+
host: options.host,
|
|
140
|
+
error: error instanceof Error ? error.message : String(error),
|
|
141
|
+
});
|
|
142
|
+
});
|
|
143
|
+
},
|
|
144
|
+
});
|
|
145
|
+
await pendingWrite;
|
|
146
|
+
current = { ...counts(real), pending: Math.max(0, dry.unitsWouldImport - processed) };
|
|
147
|
+
state = controller.signal.aborted ? "stopped" : real.unitsFailed ? "failed" : "done";
|
|
148
|
+
if (!options.signal?.aborted) {
|
|
149
|
+
options.notify(`${options.host} automatic import ${state}: ${current.imported} imported, ${current.failed} failed. autoBackfill controls the next run.`);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
catch (error) {
|
|
153
|
+
lastError = error;
|
|
154
|
+
state = controller.signal.aborted ? "stopped" : "failed";
|
|
155
|
+
}
|
|
156
|
+
finally {
|
|
157
|
+
try {
|
|
158
|
+
await updateBackfillStatus(options.host, { state, model, counts: current, error: lastError });
|
|
159
|
+
}
|
|
160
|
+
finally {
|
|
161
|
+
options.signal?.removeEventListener("abort", onAbort);
|
|
162
|
+
await release();
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { CONFIG } from "../config.js";
|
|
4
|
+
import { tursoConnectionManager } from "../services/turso/connection-manager.js";
|
|
5
|
+
function live(pid) {
|
|
6
|
+
if (!Number.isSafeInteger(pid) || pid <= 0)
|
|
7
|
+
return false;
|
|
8
|
+
try {
|
|
9
|
+
process.kill(pid, 0);
|
|
10
|
+
return true;
|
|
11
|
+
}
|
|
12
|
+
catch (error) {
|
|
13
|
+
return error.code === "EPERM";
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
/** Claim one host's import without holding a database transaction during the run. */
|
|
17
|
+
export async function tryAcquireBackfillLock(host, storagePath = CONFIG.storagePath) {
|
|
18
|
+
const db = await tursoConnectionManager.getConnection(join(storagePath, "import-ledger.db"));
|
|
19
|
+
await db.run(`CREATE TABLE IF NOT EXISTS backfill_locks (
|
|
20
|
+
host TEXT PRIMARY KEY, pid INTEGER NOT NULL, token TEXT NOT NULL
|
|
21
|
+
)`);
|
|
22
|
+
const state = { pid: process.pid, token: randomUUID() };
|
|
23
|
+
for (let attempt = 0; attempt < 3; attempt++) {
|
|
24
|
+
const inserted = await db.run("INSERT OR IGNORE INTO backfill_locks (host, pid, token) VALUES (?, ?, ?)", [host, state.pid, state.token]);
|
|
25
|
+
if (!inserted) {
|
|
26
|
+
const holder = await db.get("SELECT pid, token FROM backfill_locks WHERE host = ?", [host]);
|
|
27
|
+
if (!holder)
|
|
28
|
+
continue;
|
|
29
|
+
if (live(holder.pid))
|
|
30
|
+
return null;
|
|
31
|
+
const replaced = await db.run("UPDATE backfill_locks SET pid = ?, token = ? WHERE host = ? AND pid = ? AND token = ?", [state.pid, state.token, host, holder.pid, holder.token]);
|
|
32
|
+
if (!replaced)
|
|
33
|
+
continue;
|
|
34
|
+
}
|
|
35
|
+
let released = false;
|
|
36
|
+
return async () => {
|
|
37
|
+
if (released)
|
|
38
|
+
return;
|
|
39
|
+
released = true;
|
|
40
|
+
await db.run("DELETE FROM backfill_locks WHERE host = ? AND token = ?", [host, state.token]);
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
return null;
|
|
44
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export type BackfillHost = "pi" | "opencode";
|
|
2
|
+
export type BackfillModel = "inherit" | {
|
|
3
|
+
provider: string;
|
|
4
|
+
model: string;
|
|
5
|
+
};
|
|
6
|
+
/** Parse the global backfill model without importing the runtime config. */
|
|
7
|
+
export declare function parseBackfillModel(config: {
|
|
8
|
+
piBackfillModel?: unknown;
|
|
9
|
+
opencodeBackfillModel?: unknown;
|
|
10
|
+
}, host: BackfillHost): BackfillModel;
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/** Parse the global backfill model without importing the runtime config. */
|
|
2
|
+
export function parseBackfillModel(config, host) {
|
|
3
|
+
const value = host === "pi" ? config.piBackfillModel : config.opencodeBackfillModel;
|
|
4
|
+
if (value === undefined || value === "inherit")
|
|
5
|
+
return "inherit";
|
|
6
|
+
if (typeof value !== "string")
|
|
7
|
+
throw new Error(`Invalid ${host}BackfillModel config`);
|
|
8
|
+
const slash = value.indexOf("/");
|
|
9
|
+
if (slash <= 0 || slash === value.length - 1) {
|
|
10
|
+
throw new Error(`Invalid ${host}BackfillModel config`);
|
|
11
|
+
}
|
|
12
|
+
return { provider: value.slice(0, slash), model: value.slice(slash + 1) };
|
|
13
|
+
}
|
|
@@ -1,15 +1,21 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Discovery of Pi session files for the historical importer. Read-only: only
|
|
3
|
-
* the first line (the session header) is parsed here,
|
|
4
|
-
*
|
|
3
|
+
* the first line (the session header) is parsed here, from at most the first
|
|
4
|
+
* 64 KB of each file, so discovery stays cheap on large histories and full
|
|
5
|
+
* parsing happens later per candidate session.
|
|
5
6
|
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
7
|
+
* The root may be a sessions folder or one `.jsonl` session file. Files that
|
|
8
|
+
* are not Pi session format (for example subagent artifacts, which use a
|
|
9
|
+
* different record format without a `type: "session"` header) are reported
|
|
10
|
+
* as unrecognized and skipped rather than treated as errors. Symlinked entries
|
|
11
|
+
* inside a folder are never followed.
|
|
9
12
|
*/
|
|
10
13
|
export declare const DEFAULT_PI_SESSION_ROOT: string;
|
|
14
|
+
export declare const PI_HEADER_READ_LIMIT: number;
|
|
11
15
|
export interface DiscoveredPiSession {
|
|
12
16
|
file: string;
|
|
17
|
+
/** Path relative to the root, with `/` separators; the file name for a file root. */
|
|
18
|
+
key: string;
|
|
13
19
|
sessionId: string | null;
|
|
14
20
|
cwd: string | null;
|
|
15
21
|
version: number | null;
|
|
@@ -24,8 +30,29 @@ export interface DiscoveryResult {
|
|
|
24
30
|
sessions: DiscoveredPiSession[];
|
|
25
31
|
unrecognized: UnrecognizedFile[];
|
|
26
32
|
}
|
|
33
|
+
/**
|
|
34
|
+
* Bounds on a folder walk. A Pi sessions folder holds a few hundred folders,
|
|
35
|
+
* so a walk past the folder or file limit is almost certainly a home folder,
|
|
36
|
+
* `/`, or a whole backup volume, and would block the process for minutes.
|
|
37
|
+
* Depth is only a backstop: subagent runs nest session folders seven or more
|
|
38
|
+
* levels deep (`<project>/<run>/<id>/run-0/session/<id>/run-0`).
|
|
39
|
+
*/
|
|
40
|
+
export interface DiscoveryLimits {
|
|
41
|
+
maxDepth: number;
|
|
42
|
+
maxFolders: number;
|
|
43
|
+
maxFiles: number;
|
|
44
|
+
}
|
|
45
|
+
export declare const DEFAULT_DISCOVERY_LIMITS: DiscoveryLimits;
|
|
46
|
+
export declare class DiscoveryLimitError extends Error {
|
|
47
|
+
constructor();
|
|
48
|
+
}
|
|
27
49
|
export interface DiscoveryOptions {
|
|
28
50
|
root?: string;
|
|
29
51
|
maxSessions?: number;
|
|
52
|
+
limits?: DiscoveryLimits;
|
|
30
53
|
}
|
|
54
|
+
/** Read the first line of a file without reading past `PI_HEADER_READ_LIMIT` bytes. */
|
|
55
|
+
export declare function readFirstLine(file: string, limit?: number): string;
|
|
56
|
+
/** Parse one file's header, or say why the file is not a Pi session. */
|
|
57
|
+
export declare function readPiSessionHeader(file: string, key: string): DiscoveredPiSession | UnrecognizedFile;
|
|
31
58
|
export declare function discoverPiSessions(options?: DiscoveryOptions): DiscoveryResult;
|
|
@@ -1,22 +1,39 @@
|
|
|
1
|
-
import { readdirSync,
|
|
2
|
-
import {
|
|
3
|
-
import { join } from "node:path";
|
|
1
|
+
import { closeSync, existsSync, openSync, readSync, readdirSync, statSync } from "node:fs";
|
|
2
|
+
import { basename, join, relative, sep } from "node:path";
|
|
4
3
|
import { homedir } from "node:os";
|
|
5
4
|
/**
|
|
6
5
|
* Discovery of Pi session files for the historical importer. Read-only: only
|
|
7
|
-
* the first line (the session header) is parsed here,
|
|
8
|
-
*
|
|
6
|
+
* the first line (the session header) is parsed here, from at most the first
|
|
7
|
+
* 64 KB of each file, so discovery stays cheap on large histories and full
|
|
8
|
+
* parsing happens later per candidate session.
|
|
9
9
|
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
10
|
+
* The root may be a sessions folder or one `.jsonl` session file. Files that
|
|
11
|
+
* are not Pi session format (for example subagent artifacts, which use a
|
|
12
|
+
* different record format without a `type: "session"` header) are reported
|
|
13
|
+
* as unrecognized and skipped rather than treated as errors. Symlinked entries
|
|
14
|
+
* inside a folder are never followed.
|
|
13
15
|
*/
|
|
14
16
|
export const DEFAULT_PI_SESSION_ROOT = join(homedir(), ".pi", "agent", "sessions");
|
|
15
|
-
|
|
17
|
+
export const PI_HEADER_READ_LIMIT = 64 * 1024;
|
|
18
|
+
export const DEFAULT_DISCOVERY_LIMITS = {
|
|
19
|
+
maxDepth: 32,
|
|
20
|
+
maxFolders: 5_000,
|
|
21
|
+
maxFiles: 20_000,
|
|
22
|
+
};
|
|
23
|
+
export class DiscoveryLimitError extends Error {
|
|
24
|
+
constructor() {
|
|
25
|
+
super("This folder is too large to scan for Pi sessions. Choose the Pi sessions folder itself, or one .jsonl file.");
|
|
26
|
+
this.name = "DiscoveryLimitError";
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
function listJsonlFiles(root, limits) {
|
|
16
30
|
const files = [];
|
|
17
|
-
const stack = [root];
|
|
31
|
+
const stack = [{ dir: root, depth: 0 }];
|
|
32
|
+
let folders = 0;
|
|
18
33
|
while (stack.length > 0) {
|
|
19
|
-
const dir = stack.pop();
|
|
34
|
+
const { dir, depth } = stack.pop();
|
|
35
|
+
if (++folders > limits.maxFolders)
|
|
36
|
+
throw new DiscoveryLimitError();
|
|
20
37
|
let entries;
|
|
21
38
|
try {
|
|
22
39
|
entries = readdirSync(dir, { withFileTypes: true });
|
|
@@ -27,29 +44,71 @@ function listJsonlFiles(root) {
|
|
|
27
44
|
for (const entry of entries) {
|
|
28
45
|
const full = join(dir, entry.name);
|
|
29
46
|
if (entry.isDirectory()) {
|
|
30
|
-
|
|
47
|
+
if (depth + 1 > limits.maxDepth)
|
|
48
|
+
throw new DiscoveryLimitError();
|
|
49
|
+
stack.push({ dir: full, depth: depth + 1 });
|
|
31
50
|
}
|
|
32
51
|
else if (entry.isFile() && entry.name.endsWith(".jsonl")) {
|
|
52
|
+
if (files.length >= limits.maxFiles)
|
|
53
|
+
throw new DiscoveryLimitError();
|
|
33
54
|
files.push(full);
|
|
34
55
|
}
|
|
35
56
|
}
|
|
36
57
|
}
|
|
37
58
|
return files.sort();
|
|
38
59
|
}
|
|
60
|
+
/** Read the first line of a file without reading past `PI_HEADER_READ_LIMIT` bytes. */
|
|
61
|
+
export function readFirstLine(file, limit = PI_HEADER_READ_LIMIT) {
|
|
62
|
+
const fd = openSync(file, "r");
|
|
63
|
+
try {
|
|
64
|
+
const buffer = Buffer.alloc(limit);
|
|
65
|
+
const read = readSync(fd, buffer, 0, limit, 0);
|
|
66
|
+
const text = buffer.subarray(0, read).toString("utf8");
|
|
67
|
+
const newline = text.indexOf("\n");
|
|
68
|
+
return newline === -1 ? text : text.slice(0, newline);
|
|
69
|
+
}
|
|
70
|
+
finally {
|
|
71
|
+
closeSync(fd);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
39
74
|
function readHeaderLine(file) {
|
|
40
|
-
let firstLine;
|
|
41
75
|
try {
|
|
42
|
-
const
|
|
43
|
-
const newline = stream.indexOf("\n");
|
|
44
|
-
firstLine = (newline === -1 ? stream : stream.slice(0, newline)).trim();
|
|
76
|
+
const firstLine = readFirstLine(file).trim();
|
|
45
77
|
if (!firstLine)
|
|
46
78
|
return { error: "empty file" };
|
|
47
79
|
return { header: JSON.parse(firstLine) };
|
|
48
80
|
}
|
|
49
81
|
catch (error) {
|
|
50
82
|
if (error instanceof SyntaxError)
|
|
51
|
-
return { error:
|
|
52
|
-
return { error: `unreadable: ${
|
|
83
|
+
return { error: "malformed header" };
|
|
84
|
+
return { error: `unreadable: ${error.code ?? "error"}` };
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
/** Parse one file's header, or say why the file is not a Pi session. */
|
|
88
|
+
export function readPiSessionHeader(file, key) {
|
|
89
|
+
const parsed = readHeaderLine(file);
|
|
90
|
+
if ("error" in parsed)
|
|
91
|
+
return { file, reason: parsed.error };
|
|
92
|
+
const header = parsed.header;
|
|
93
|
+
if (header?.type !== "session") {
|
|
94
|
+
return { file, reason: "not a Pi session file (no session header)" };
|
|
95
|
+
}
|
|
96
|
+
const timestamp = typeof header.timestamp === "string" ? Date.parse(header.timestamp) : NaN;
|
|
97
|
+
return {
|
|
98
|
+
file,
|
|
99
|
+
key,
|
|
100
|
+
sessionId: typeof header.id === "string" ? header.id : null,
|
|
101
|
+
cwd: typeof header.cwd === "string" ? header.cwd : null,
|
|
102
|
+
version: typeof header.version === "number" ? header.version : null,
|
|
103
|
+
timestamp: Number.isNaN(timestamp) ? null : timestamp,
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
function isFile(path) {
|
|
107
|
+
try {
|
|
108
|
+
return statSync(path).isFile();
|
|
109
|
+
}
|
|
110
|
+
catch {
|
|
111
|
+
return false;
|
|
53
112
|
}
|
|
54
113
|
}
|
|
55
114
|
export function discoverPiSessions(options = {}) {
|
|
@@ -59,25 +118,18 @@ export function discoverPiSessions(options = {}) {
|
|
|
59
118
|
if (!existsSync(root)) {
|
|
60
119
|
return { sessions, unrecognized };
|
|
61
120
|
}
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
unrecognized.push({ file, reason: parsed.error });
|
|
66
|
-
continue;
|
|
67
|
-
}
|
|
68
|
-
const header = parsed.header;
|
|
69
|
-
if (header?.type !== "session") {
|
|
70
|
-
unrecognized.push({ file, reason: "not a Pi session file (no session header)" });
|
|
71
|
-
continue;
|
|
72
|
-
}
|
|
73
|
-
const timestamp = typeof header.timestamp === "string" ? Date.parse(header.timestamp) : NaN;
|
|
74
|
-
sessions.push({
|
|
121
|
+
const files = isFile(root)
|
|
122
|
+
? [{ file: root, key: basename(root) }]
|
|
123
|
+
: listJsonlFiles(root, options.limits ?? DEFAULT_DISCOVERY_LIMITS).map((file) => ({
|
|
75
124
|
file,
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
125
|
+
key: relative(root, file).split(sep).join("/"),
|
|
126
|
+
}));
|
|
127
|
+
for (const { file, key } of files) {
|
|
128
|
+
const result = readPiSessionHeader(file, key);
|
|
129
|
+
if ("reason" in result)
|
|
130
|
+
unrecognized.push(result);
|
|
131
|
+
else
|
|
132
|
+
sessions.push(result);
|
|
81
133
|
}
|
|
82
134
|
// Chronological order, oldest first; files without a timestamp sort last.
|
|
83
135
|
sessions.sort((a, b) => (a.timestamp ?? Infinity) - (b.timestamp ?? Infinity));
|
|
@@ -7,7 +7,7 @@ import type { ImportPathMap } from "./importer.js";
|
|
|
7
7
|
* external API and key).
|
|
8
8
|
*/
|
|
9
9
|
export type ImportHost = "pi" | "opencode";
|
|
10
|
-
export type ImportSurface = "session" | "cli";
|
|
10
|
+
export type ImportSurface = "session" | "cli" | "web";
|
|
11
11
|
export interface HistoryImportArgs {
|
|
12
12
|
help: boolean;
|
|
13
13
|
dryRun: boolean;
|
|
@@ -39,6 +39,28 @@ export declare function parseHistoryImportArgs(tokens: string[], options: {
|
|
|
39
39
|
host: ImportHost;
|
|
40
40
|
surface: ImportSurface;
|
|
41
41
|
}): HistoryImportArgs;
|
|
42
|
+
/**
|
|
43
|
+
* Options the Settings page sends. Sessions come from the page's selection
|
|
44
|
+
* and the source from a signed token, so there is no `session`,
|
|
45
|
+
* `maxSessions`, or `source` here.
|
|
46
|
+
*/
|
|
47
|
+
export interface WebImportOptions {
|
|
48
|
+
dryRun?: boolean;
|
|
49
|
+
force?: boolean;
|
|
50
|
+
skipMemories?: boolean;
|
|
51
|
+
skipProfile?: boolean;
|
|
52
|
+
model?: string;
|
|
53
|
+
scope?: HistoryImportArgs["scope"];
|
|
54
|
+
project?: string;
|
|
55
|
+
/** Epoch ms from the browser's local day start, or an ISO date. */
|
|
56
|
+
since?: string | number;
|
|
57
|
+
/** Epoch ms from the browser's local day end, or an ISO date. */
|
|
58
|
+
until?: string | number;
|
|
59
|
+
profileBatch?: number;
|
|
60
|
+
pathMaps?: ImportPathMap[];
|
|
61
|
+
}
|
|
62
|
+
/** Convert page fields to the shared CLI grammar, which performs validation. */
|
|
63
|
+
export declare function webImportTokens(options: WebImportOptions, _host: ImportHost): string[];
|
|
42
64
|
/** True when the run calls a model: a real import of memories or the profile. */
|
|
43
65
|
export declare function importNeedsModel(args: HistoryImportArgs): boolean;
|
|
44
66
|
export declare function historyImportUsage(host: ImportHost, surface: ImportSurface): string;
|
|
@@ -168,7 +168,7 @@ export function parseHistoryImportArgs(tokens, options) {
|
|
|
168
168
|
if (result.project !== undefined && result.scope === "all-projects") {
|
|
169
169
|
result.errors.push("--project cannot be combined with --scope=all-projects");
|
|
170
170
|
}
|
|
171
|
-
if (options.surface
|
|
171
|
+
if (options.surface !== "cli" && result.model !== undefined) {
|
|
172
172
|
const separator = result.model.indexOf("/");
|
|
173
173
|
if (separator < 1 || separator === result.model.length - 1) {
|
|
174
174
|
result.errors.push(`--model must be provider/id, got "${result.model}"`);
|
|
@@ -176,6 +176,34 @@ export function parseHistoryImportArgs(tokens, options) {
|
|
|
176
176
|
}
|
|
177
177
|
return result;
|
|
178
178
|
}
|
|
179
|
+
/** Convert page fields to the shared CLI grammar, which performs validation. */
|
|
180
|
+
export function webImportTokens(options, _host) {
|
|
181
|
+
const tokens = [];
|
|
182
|
+
for (const [field, flag] of [
|
|
183
|
+
["dryRun", "--dry-run"],
|
|
184
|
+
["force", "--force"],
|
|
185
|
+
["skipMemories", "--skip-memories"],
|
|
186
|
+
["skipProfile", "--skip-profile"],
|
|
187
|
+
]) {
|
|
188
|
+
if (options[field])
|
|
189
|
+
tokens.push(flag);
|
|
190
|
+
}
|
|
191
|
+
for (const [field, flag] of [
|
|
192
|
+
["model", "--model"],
|
|
193
|
+
["scope", "--scope"],
|
|
194
|
+
["project", "--project"],
|
|
195
|
+
["since", "--since"],
|
|
196
|
+
["until", "--until"],
|
|
197
|
+
["profileBatch", "--profile-batch"],
|
|
198
|
+
]) {
|
|
199
|
+
const value = options[field];
|
|
200
|
+
if (value !== undefined)
|
|
201
|
+
tokens.push(flag, String(value));
|
|
202
|
+
}
|
|
203
|
+
for (const map of options.pathMaps ?? [])
|
|
204
|
+
tokens.push("--map", `${map.from}=${map.to}`);
|
|
205
|
+
return tokens;
|
|
206
|
+
}
|
|
179
207
|
/** True when the run calls a model: a real import of memories or the profile. */
|
|
180
208
|
export function importNeedsModel(args) {
|
|
181
209
|
return !args.dryRun && (!args.skipMemories || !args.skipProfile);
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { ImportPathMap } from "./importer.js";
|
|
2
|
+
/** How a recorded session directory was turned into a project directory. */
|
|
3
|
+
export type ProjectResolution = "mapped" | "recorded" | "worktree" | "unresolved";
|
|
4
|
+
export interface ResolvedImportProject {
|
|
5
|
+
directory: string | null;
|
|
6
|
+
via: ProjectResolution;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* One rule for both hosts, shared by the web session list and the importers so
|
|
10
|
+
* the listed set and the imported set always agree. An exact directory map
|
|
11
|
+
* wins, because it is an explicit user choice; a map whose target is missing
|
|
12
|
+
* leaves the session unresolved rather than silently falling back. OpenCode
|
|
13
|
+
* alone passes a project worktree, which stands in for a deleted sub-directory
|
|
14
|
+
* of the same project.
|
|
15
|
+
*/
|
|
16
|
+
export declare function resolveImportProject(recordedDirectory: string | null, maps?: ImportPathMap[], projectWorktree?: string | null): ResolvedImportProject;
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { statSync } from "node:fs";
|
|
2
|
+
function isDirectory(path) {
|
|
3
|
+
if (!path)
|
|
4
|
+
return false;
|
|
5
|
+
try {
|
|
6
|
+
return statSync(path).isDirectory();
|
|
7
|
+
}
|
|
8
|
+
catch {
|
|
9
|
+
return false;
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* One rule for both hosts, shared by the web session list and the importers so
|
|
14
|
+
* the listed set and the imported set always agree. An exact directory map
|
|
15
|
+
* wins, because it is an explicit user choice; a map whose target is missing
|
|
16
|
+
* leaves the session unresolved rather than silently falling back. OpenCode
|
|
17
|
+
* alone passes a project worktree, which stands in for a deleted sub-directory
|
|
18
|
+
* of the same project.
|
|
19
|
+
*/
|
|
20
|
+
export function resolveImportProject(recordedDirectory, maps = [], projectWorktree = null) {
|
|
21
|
+
if (!recordedDirectory)
|
|
22
|
+
return { directory: null, via: "unresolved" };
|
|
23
|
+
const map = maps.find((item) => item.from === recordedDirectory);
|
|
24
|
+
if (map) {
|
|
25
|
+
return isDirectory(map.to)
|
|
26
|
+
? { directory: map.to, via: "mapped" }
|
|
27
|
+
: { directory: null, via: "unresolved" };
|
|
28
|
+
}
|
|
29
|
+
if (isDirectory(recordedDirectory))
|
|
30
|
+
return { directory: recordedDirectory, via: "recorded" };
|
|
31
|
+
if (projectWorktree && projectWorktree !== "/" && isDirectory(projectWorktree)) {
|
|
32
|
+
return { directory: projectWorktree, via: "worktree" };
|
|
33
|
+
}
|
|
34
|
+
return { directory: null, via: "unresolved" };
|
|
35
|
+
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What a real web import could use right now, checked inside the OpenCode
|
|
3
|
+
* process that runs it. `env://` and `file://` keys were resolved when the
|
|
4
|
+
* config loaded, so a variable set only in the user's shell shows as missing.
|
|
5
|
+
* "Ready" means configured, not tested: the Health section runs test calls.
|
|
6
|
+
*/
|
|
7
|
+
export type ExternalApiState = "ready" | "missing-model" | "missing-url" | "missing-key" | "unsupported-provider";
|
|
8
|
+
export interface ImportReadiness {
|
|
9
|
+
external: {
|
|
10
|
+
state: ExternalApiState;
|
|
11
|
+
provider: string;
|
|
12
|
+
model: string | null;
|
|
13
|
+
};
|
|
14
|
+
opencode: {
|
|
15
|
+
available: boolean;
|
|
16
|
+
models: Array<{
|
|
17
|
+
provider: string;
|
|
18
|
+
model: string;
|
|
19
|
+
name: string;
|
|
20
|
+
}>;
|
|
21
|
+
};
|
|
22
|
+
piReader: {
|
|
23
|
+
available: boolean;
|
|
24
|
+
reason?: string;
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
export interface ReadinessDeps {
|
|
28
|
+
supportedProviders?: () => Promise<string[]>;
|
|
29
|
+
listOpencodeModels?: () => Promise<{
|
|
30
|
+
available: boolean;
|
|
31
|
+
models?: Array<{
|
|
32
|
+
provider: string;
|
|
33
|
+
model: string;
|
|
34
|
+
name: string;
|
|
35
|
+
}>;
|
|
36
|
+
}>;
|
|
37
|
+
loadPiSdk?: () => Promise<unknown>;
|
|
38
|
+
}
|
|
39
|
+
/** Mirrors the checks in `selectImportModel`, without building a provider. */
|
|
40
|
+
export declare function externalApiState(supportedProviders?: () => Promise<string[]>): Promise<ExternalApiState>;
|
|
41
|
+
export declare function importReadiness(deps?: ReadinessDeps): Promise<ImportReadiness>;
|
|
42
|
+
/** Why a job cannot start, or `null` when it can. */
|
|
43
|
+
export declare function importBlockedReason(readiness: ImportReadiness, request: {
|
|
44
|
+
host: "pi" | "opencode";
|
|
45
|
+
needsModel: boolean;
|
|
46
|
+
modelChoice?: string;
|
|
47
|
+
}): string | null;
|