om-memory-system 3.2.0-next.11 → 3.2.0-next.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/opencode/import-command.d.ts +3 -7
- package/dist/adapters/opencode/import-command.js +3 -54
- package/dist/adapters/pi/profile.js +2 -1
- package/dist/config.d.ts +11 -1
- package/dist/config.js +82 -9
- package/dist/core/capture.js +2 -1
- package/dist/core/profile-analysis.js +9 -1
- package/dist/importer/discovery.d.ts +30 -5
- package/dist/importer/discovery.js +88 -36
- package/dist/importer/import-args.d.ts +23 -1
- package/dist/importer/import-args.js +29 -1
- package/dist/importer/import-project.d.ts +16 -0
- package/dist/importer/import-project.js +35 -0
- package/dist/importer/import-readiness.d.ts +47 -0
- package/dist/importer/import-readiness.js +54 -0
- package/dist/importer/import-sessions.d.ts +98 -0
- package/dist/importer/import-sessions.js +222 -0
- package/dist/importer/import-sources.d.ts +47 -0
- package/dist/importer/import-sources.js +147 -0
- package/dist/importer/importer.d.ts +31 -1
- package/dist/importer/importer.js +82 -24
- package/dist/importer/opencode-import.d.ts +11 -1
- package/dist/importer/opencode-import.js +133 -69
- package/dist/importer/opencode-project.d.ts +1 -4
- package/dist/importer/opencode-project.js +3 -36
- package/dist/importer/opencode-reader.d.ts +25 -5
- package/dist/importer/opencode-reader.js +92 -106
- package/dist/importer/opencode-snapshot.d.ts +82 -0
- package/dist/importer/opencode-snapshot.js +276 -0
- package/dist/importer/profile-import.d.ts +1 -0
- package/dist/importer/profile-import.js +7 -1
- package/dist/importer/run-import.d.ts +10 -0
- package/dist/importer/run-import.js +21 -0
- package/dist/importer/settings-health.d.ts +25 -0
- package/dist/importer/settings-health.js +136 -0
- package/dist/importer/web-import-api.d.ts +8 -0
- package/dist/importer/web-import-api.js +8 -0
- package/dist/importer/web-import-jobs.d.ts +53 -0
- package/dist/importer/web-import-jobs.js +227 -0
- package/dist/index.js +1 -0
- package/dist/services/ai/live-model-choice.js +6 -3
- package/dist/services/ai/opencode-import-models.d.ts +11 -0
- package/dist/services/ai/opencode-import-models.js +55 -0
- package/dist/services/capture-attempt-store.d.ts +21 -0
- package/dist/services/capture-attempt-store.js +108 -0
- package/dist/services/capture-diagnostics.js +3 -0
- package/dist/services/cleanup-service.js +9 -0
- package/dist/services/global-config-writer.d.ts +10 -0
- package/dist/services/global-config-writer.js +106 -0
- package/dist/services/settings-log.d.ts +5 -0
- package/dist/services/settings-log.js +36 -0
- package/dist/services/settings-models.d.ts +35 -0
- package/dist/services/settings-models.js +39 -0
- package/dist/services/settings-snapshot.d.ts +24 -0
- package/dist/services/settings-snapshot.js +71 -0
- package/dist/services/settings-traces.d.ts +7 -0
- package/dist/services/settings-traces.js +39 -0
- package/dist/services/user-memory-learning.js +2 -1
- package/dist/services/web-server.d.ts +5 -0
- package/dist/services/web-server.js +176 -3
- package/dist/web/assets/index-BO6gokx9.js +124 -0
- package/dist/web/assets/index-D5ak5KsI.css +2 -0
- package/dist/web/index.html +2 -2
- package/package.json +2 -1
- package/dist/web/assets/index-DPjEpD1h.css +0 -2
- package/dist/web/assets/index-do26c4JJ.js +0 -122
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
import { createHmac, randomBytes, timingSafeEqual } from "node:crypto";
|
|
2
|
+
import { closeSync, existsSync, openSync, readdirSync, readSync, realpathSync, statSync, } from "node:fs";
|
|
3
|
+
import { homedir } from "node:os";
|
|
4
|
+
import { dirname, isAbsolute, join } from "node:path";
|
|
5
|
+
import { DEFAULT_PI_SESSION_ROOT, readPiSessionHeader } from "./discovery.js";
|
|
6
|
+
import { DEFAULT_OPENCODE_DB } from "./opencode-reader.js";
|
|
7
|
+
export class ImportSourceError extends Error {
|
|
8
|
+
status;
|
|
9
|
+
constructor(message, status = 400) {
|
|
10
|
+
super(message);
|
|
11
|
+
this.status = status;
|
|
12
|
+
this.name = "ImportSourceError";
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
// Tokens only need to outlive the server process that issued them.
|
|
16
|
+
const TOKEN_KEY = randomBytes(32);
|
|
17
|
+
function sign(payload) {
|
|
18
|
+
return createHmac("sha256", TOKEN_KEY).update(payload).digest("base64url");
|
|
19
|
+
}
|
|
20
|
+
export function signImportSource(identity) {
|
|
21
|
+
const payload = Buffer.from(JSON.stringify(identity)).toString("base64url");
|
|
22
|
+
return `${payload}.${sign(payload)}`;
|
|
23
|
+
}
|
|
24
|
+
export function readImportSourceToken(token, host) {
|
|
25
|
+
if (typeof token !== "string")
|
|
26
|
+
throw new ImportSourceError("Choose a history source");
|
|
27
|
+
const [payload = "", signature = ""] = token.split(".");
|
|
28
|
+
const expected = Buffer.from(sign(payload));
|
|
29
|
+
const actual = Buffer.from(signature);
|
|
30
|
+
if (expected.length !== actual.length || !timingSafeEqual(expected, actual)) {
|
|
31
|
+
throw new ImportSourceError("The history source is no longer valid. Choose it again.");
|
|
32
|
+
}
|
|
33
|
+
const identity = JSON.parse(Buffer.from(payload, "base64url").toString());
|
|
34
|
+
if (identity.host !== host)
|
|
35
|
+
throw new ImportSourceError("The source belongs to the other host");
|
|
36
|
+
return identity;
|
|
37
|
+
}
|
|
38
|
+
/** Stable key for sharing snapshots and computing revisions. */
|
|
39
|
+
export function importSourceKey(identity) {
|
|
40
|
+
return `${identity.realPath}:${identity.dev}:${identity.ino}`;
|
|
41
|
+
}
|
|
42
|
+
/** Refuse a source whose path now names another file, such as a swapped-in symlink. */
|
|
43
|
+
export function assertImportSourceUnchanged(identity) {
|
|
44
|
+
let current;
|
|
45
|
+
try {
|
|
46
|
+
current = statSync(identity.realPath);
|
|
47
|
+
}
|
|
48
|
+
catch {
|
|
49
|
+
throw new ImportSourceError("The history source no longer exists. Choose it again.", 409);
|
|
50
|
+
}
|
|
51
|
+
if (current.dev !== identity.dev || current.ino !== identity.ino) {
|
|
52
|
+
throw new ImportSourceError("The history source changed. Choose it again.", 409);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
function checkedPath(path) {
|
|
56
|
+
if (typeof path !== "string" || !path.trim())
|
|
57
|
+
throw new ImportSourceError("Enter a path");
|
|
58
|
+
if (!isAbsolute(path))
|
|
59
|
+
throw new ImportSourceError("Enter an absolute path");
|
|
60
|
+
if (path.split(/[\\/]/).includes("..")) {
|
|
61
|
+
throw new ImportSourceError("The path must not contain '..' segments");
|
|
62
|
+
}
|
|
63
|
+
try {
|
|
64
|
+
// Symlinks at or above the chosen path are fine: macOS /tmp, a linked ~/.pi.
|
|
65
|
+
return realpathSync.native(path);
|
|
66
|
+
}
|
|
67
|
+
catch {
|
|
68
|
+
throw new ImportSourceError("Path not found");
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
function isSqliteFile(path) {
|
|
72
|
+
const fd = openSync(path, "r");
|
|
73
|
+
try {
|
|
74
|
+
const buffer = Buffer.alloc(16);
|
|
75
|
+
readSync(fd, buffer, 0, 16, 0);
|
|
76
|
+
return buffer.toString("latin1") === "SQLite format 3\0";
|
|
77
|
+
}
|
|
78
|
+
finally {
|
|
79
|
+
closeSync(fd);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
export function defaultImportSourcePath(host) {
|
|
83
|
+
return host === "pi" ? DEFAULT_PI_SESSION_ROOT : DEFAULT_OPENCODE_DB;
|
|
84
|
+
}
|
|
85
|
+
/** Check an absolute path's kind and format; never returns file contents. */
|
|
86
|
+
export function validateImportSource(host, path) {
|
|
87
|
+
const realPath = checkedPath(path);
|
|
88
|
+
const info = statSync(realPath);
|
|
89
|
+
let kind;
|
|
90
|
+
if (host === "pi") {
|
|
91
|
+
if (info.isDirectory())
|
|
92
|
+
kind = "pi-folder";
|
|
93
|
+
else if (info.isFile() && realPath.endsWith(".jsonl")) {
|
|
94
|
+
if ("reason" in readPiSessionHeader(realPath, "")) {
|
|
95
|
+
throw new ImportSourceError("The file is not a Pi session file");
|
|
96
|
+
}
|
|
97
|
+
kind = "pi-file";
|
|
98
|
+
}
|
|
99
|
+
else
|
|
100
|
+
throw new ImportSourceError("Choose a Pi sessions folder or one .jsonl session file");
|
|
101
|
+
}
|
|
102
|
+
else {
|
|
103
|
+
if (!info.isFile() || !isSqliteFile(realPath)) {
|
|
104
|
+
throw new ImportSourceError("Choose an OpenCode database file");
|
|
105
|
+
}
|
|
106
|
+
kind = "opencode-db";
|
|
107
|
+
}
|
|
108
|
+
const identity = { host, kind, realPath, dev: info.dev, ino: info.ino };
|
|
109
|
+
return { kind, displayPath: realPath, sourceToken: signImportSource(identity) };
|
|
110
|
+
}
|
|
111
|
+
const BROWSE_LIMIT = 500;
|
|
112
|
+
/**
|
|
113
|
+
* One folder at a time, with no recursion: subfolders and files the host can
|
|
114
|
+
* import. Symlinked entries are hidden. Only offered on loopback binds.
|
|
115
|
+
*/
|
|
116
|
+
export function browseImportSources(host, path) {
|
|
117
|
+
let start = path;
|
|
118
|
+
if (start === undefined || start === null || start === "") {
|
|
119
|
+
const preferred = host === "pi" ? DEFAULT_PI_SESSION_ROOT : dirname(DEFAULT_OPENCODE_DB);
|
|
120
|
+
start = existsSync(preferred) ? preferred : homedir();
|
|
121
|
+
}
|
|
122
|
+
const folder = checkedPath(start);
|
|
123
|
+
if (!statSync(folder).isDirectory())
|
|
124
|
+
throw new ImportSourceError("Choose a folder to browse");
|
|
125
|
+
const entries = [];
|
|
126
|
+
for (const entry of readdirSync(folder, { withFileTypes: true })) {
|
|
127
|
+
if (entry.isSymbolicLink())
|
|
128
|
+
continue;
|
|
129
|
+
const full = join(folder, entry.name);
|
|
130
|
+
if (entry.isDirectory())
|
|
131
|
+
entries.push({ name: entry.name, path: full, kind: "folder" });
|
|
132
|
+
else if (entry.isFile() && host === "pi" && entry.name.endsWith(".jsonl")) {
|
|
133
|
+
entries.push({ name: entry.name, path: full, kind: "pi-file" });
|
|
134
|
+
}
|
|
135
|
+
else if (entry.isFile() && host === "opencode" && entry.name.endsWith(".db")) {
|
|
136
|
+
entries.push({ name: entry.name, path: full, kind: "opencode-db" });
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
entries.sort((a, b) => Number(b.kind === "folder") - Number(a.kind === "folder") || a.name.localeCompare(b.name));
|
|
140
|
+
const parent = dirname(folder);
|
|
141
|
+
return {
|
|
142
|
+
path: folder,
|
|
143
|
+
parent: parent === folder ? null : parent,
|
|
144
|
+
entries: entries.slice(0, BROWSE_LIMIT),
|
|
145
|
+
truncated: entries.length > BROWSE_LIMIT,
|
|
146
|
+
};
|
|
147
|
+
}
|
|
@@ -29,6 +29,13 @@ export interface ImportFilters {
|
|
|
29
29
|
skipMemories?: boolean;
|
|
30
30
|
pathMaps?: ImportPathMap[];
|
|
31
31
|
root?: string;
|
|
32
|
+
/**
|
|
33
|
+
* Web selection: only these discovery keys (paths relative to `root`).
|
|
34
|
+
* Never combined with `session` or `maxSessions`.
|
|
35
|
+
*/
|
|
36
|
+
selectionKeys?: string[];
|
|
37
|
+
/** Web selection: user turns after this epoch ms are held back for a later run. */
|
|
38
|
+
cutoff?: number;
|
|
32
39
|
}
|
|
33
40
|
export type ImportUnitStatus = "imported" | "would-import" | "skipped" | "failed" | "already-handled";
|
|
34
41
|
export interface ImportUnitReport {
|
|
@@ -72,11 +79,16 @@ export interface ImportReport {
|
|
|
72
79
|
projects: ImportProjectReport[];
|
|
73
80
|
units: ImportUnitReport[];
|
|
74
81
|
profile?: ProfileImportReport;
|
|
82
|
+
/** Turns newer than the web listing time, left for a later run. */
|
|
83
|
+
unitsHeldBack?: number;
|
|
84
|
+
/** Turns without a timestamp; date limits cannot exclude them. */
|
|
85
|
+
unitsUntimed?: number;
|
|
75
86
|
}
|
|
76
87
|
export interface ImporterDeps {
|
|
77
88
|
loadSession: (file: string) => LoadedPiSession;
|
|
78
89
|
provider: CaptureSummaryProvider;
|
|
79
90
|
onProgress?: (processed: number, total: number, promptPreview: string) => void;
|
|
91
|
+
signal?: AbortSignal;
|
|
80
92
|
ledger?: PiImportLedger;
|
|
81
93
|
profile?: {
|
|
82
94
|
model?: ModelPort;
|
|
@@ -106,6 +118,24 @@ export declare function findMemoryIdByImportId(projectHash: string, importId: st
|
|
|
106
118
|
* /private/var) name one project, so compare through the real path.
|
|
107
119
|
*/
|
|
108
120
|
export declare function projectFilterTag(directory: string): string;
|
|
121
|
+
/**
|
|
122
|
+
* Keep the windows a run would import: inside the date range and not newer
|
|
123
|
+
* than the web listing cutoff. Held-back windows are only counted; they never
|
|
124
|
+
* reach the ledger, so a later run imports them under the same keys.
|
|
125
|
+
*/
|
|
126
|
+
export declare function selectImportWindows<T extends ImportWindow>(windows: T[], filters: Pick<ImportFilters, "since" | "until" | "cutoff">): {
|
|
127
|
+
kept: T[];
|
|
128
|
+
heldBack: number;
|
|
129
|
+
untimed: number;
|
|
130
|
+
};
|
|
131
|
+
/** Units of one session that `importHistorySource` would consider, for a counting pass. */
|
|
132
|
+
export declare function countImportableUnits(session: ImportSourceSession, host: MemoryHost, filters: Pick<ImportFilters, "since" | "until">): number;
|
|
133
|
+
/** Sessions produced on demand, so a large source never holds every turn at once. */
|
|
134
|
+
export interface LazyImportSource {
|
|
135
|
+
/** Units that pass the date and key filters, counted in an earlier pass. */
|
|
136
|
+
total: number;
|
|
137
|
+
open: () => AsyncIterable<ImportSourceSession>;
|
|
138
|
+
}
|
|
109
139
|
/** Process host-labelled work units through the shared capture and ledger pipeline. */
|
|
110
|
-
export declare function importHistorySource(source: AsyncIterable<ImportSourceSession> | Iterable<ImportSourceSession
|
|
140
|
+
export declare function importHistorySource(source: AsyncIterable<ImportSourceSession> | Iterable<ImportSourceSession> | LazyImportSource, host: MemoryHost, deps: Pick<ImporterDeps, "provider" | "ledger" | "onProgress" | "signal">, filters: Pick<ImportFilters, "dryRun" | "force" | "since" | "until">, report: ImportReport): Promise<void>;
|
|
111
141
|
export declare function importPiHistory(deps: ImporterDeps, filters: ImportFilters): Promise<ImportReport>;
|
|
@@ -7,6 +7,7 @@ import { tursoConnectionManager } from "../services/turso/connection-manager.js"
|
|
|
7
7
|
import { tursoShardManager } from "../services/turso/shard-manager.js";
|
|
8
8
|
import { extractPiConversationWindows, } from "../adapters/pi/conversation.js";
|
|
9
9
|
import { discoverPiSessions } from "./discovery.js";
|
|
10
|
+
import { resolveImportProject } from "./import-project.js";
|
|
10
11
|
import { PiImportLedger, importLedgerDbPath } from "./ledger.js";
|
|
11
12
|
export function buildImportKey(sessionId, window, host = "pi") {
|
|
12
13
|
const terminalEntryId = window.sourceEntryIds?.[window.sourceEntryIds.length - 1];
|
|
@@ -46,13 +47,6 @@ export function projectFilterTag(directory) {
|
|
|
46
47
|
}
|
|
47
48
|
return getTags(real).project.tag;
|
|
48
49
|
}
|
|
49
|
-
function resolveSessionDirectory(cwd, pathMaps) {
|
|
50
|
-
if (!cwd)
|
|
51
|
-
return null;
|
|
52
|
-
const mapped = pathMaps.find((map) => map.from === cwd)?.to;
|
|
53
|
-
const directory = mapped ?? cwd;
|
|
54
|
-
return existsSync(directory) ? directory : null;
|
|
55
|
-
}
|
|
56
50
|
function preview(text) {
|
|
57
51
|
const trimmed = text.trim().replace(/\s+/g, " ");
|
|
58
52
|
return trimmed.length > 60 ? `${trimmed.slice(0, 60)}...` : trimmed;
|
|
@@ -66,6 +60,38 @@ function windowWithinDateRange(window, filters) {
|
|
|
66
60
|
return false;
|
|
67
61
|
return true;
|
|
68
62
|
}
|
|
63
|
+
/**
|
|
64
|
+
* Keep the windows a run would import: inside the date range and not newer
|
|
65
|
+
* than the web listing cutoff. Held-back windows are only counted; they never
|
|
66
|
+
* reach the ledger, so a later run imports them under the same keys.
|
|
67
|
+
*/
|
|
68
|
+
export function selectImportWindows(windows, filters) {
|
|
69
|
+
const kept = [];
|
|
70
|
+
let heldBack = 0;
|
|
71
|
+
let untimed = 0;
|
|
72
|
+
for (const window of windows) {
|
|
73
|
+
if (!windowWithinDateRange(window, filters))
|
|
74
|
+
continue;
|
|
75
|
+
if (filters.cutoff !== undefined &&
|
|
76
|
+
window.userTimestamp !== undefined &&
|
|
77
|
+
window.userTimestamp > filters.cutoff) {
|
|
78
|
+
heldBack++;
|
|
79
|
+
continue;
|
|
80
|
+
}
|
|
81
|
+
if (window.userTimestamp === undefined)
|
|
82
|
+
untimed++;
|
|
83
|
+
kept.push(window);
|
|
84
|
+
}
|
|
85
|
+
return { kept, heldBack, untimed };
|
|
86
|
+
}
|
|
87
|
+
/** Units of one session that `importHistorySource` would consider, for a counting pass. */
|
|
88
|
+
export function countImportableUnits(session, host, filters) {
|
|
89
|
+
return session.units.filter((window) => windowWithinDateRange(window, filters) &&
|
|
90
|
+
buildImportKey(session.sessionId, window, host) !== null).length;
|
|
91
|
+
}
|
|
92
|
+
function isLazySource(source) {
|
|
93
|
+
return typeof source.open === "function";
|
|
94
|
+
}
|
|
69
95
|
async function lookupExistingMemory(projectHash, key, options) {
|
|
70
96
|
if (!options.storageAccessible)
|
|
71
97
|
return null;
|
|
@@ -91,21 +117,34 @@ export async function importHistorySource(source, host, deps, filters, report) {
|
|
|
91
117
|
throw new Error(`Memory system is not ready for import: ${embeddingInitError}. Start the extension and retry.`);
|
|
92
118
|
}
|
|
93
119
|
}
|
|
94
|
-
const
|
|
95
|
-
for await (const session of source) {
|
|
120
|
+
const sessionCandidates = (session) => {
|
|
96
121
|
const { hash } = extractScopeFromContainerTag(getTags(session.directory).project.tag);
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
122
|
+
return session.units
|
|
123
|
+
.filter((window) => windowWithinDateRange(window, filters) &&
|
|
124
|
+
buildImportKey(session.sessionId, window, host) !== null)
|
|
125
|
+
.map((window) => ({ ...session, hash, window }));
|
|
126
|
+
};
|
|
127
|
+
// A lazy source is read session by session; an eager one is buffered as before.
|
|
128
|
+
let candidates;
|
|
129
|
+
if (isLazySource(source)) {
|
|
130
|
+
report.unitsTotal = source.total;
|
|
131
|
+
candidates = (async function* () {
|
|
132
|
+
for await (const session of source.open())
|
|
133
|
+
yield* sessionCandidates(session);
|
|
134
|
+
})();
|
|
135
|
+
}
|
|
136
|
+
else {
|
|
137
|
+
const all = [];
|
|
138
|
+
for await (const session of source)
|
|
139
|
+
all.push(...sessionCandidates(session));
|
|
140
|
+
report.unitsTotal = all.length;
|
|
141
|
+
candidates = all;
|
|
104
142
|
}
|
|
105
|
-
report.unitsTotal = candidates.length;
|
|
106
143
|
const storageAccessible = ledgerFileExists;
|
|
107
144
|
let processed = 0;
|
|
108
|
-
for (const candidate of candidates) {
|
|
145
|
+
for await (const candidate of candidates) {
|
|
146
|
+
if (deps.signal?.aborted)
|
|
147
|
+
break;
|
|
109
148
|
const { sessionId, directory, sourceFile, hash, window } = candidate;
|
|
110
149
|
const key = buildImportKey(sessionId, window, host);
|
|
111
150
|
processed++;
|
|
@@ -223,10 +262,19 @@ export async function importPiHistory(deps, filters) {
|
|
|
223
262
|
skipReasons: {},
|
|
224
263
|
projects: [],
|
|
225
264
|
units: [],
|
|
265
|
+
unitsHeldBack: 0,
|
|
266
|
+
unitsUntimed: 0,
|
|
226
267
|
};
|
|
268
|
+
const selected = filters.selectionKeys ? new Set(filters.selectionKeys) : null;
|
|
227
269
|
const projectAggregates = new Map();
|
|
228
270
|
const candidates = [];
|
|
229
271
|
for (const discovered of discovery.sessions) {
|
|
272
|
+
if (deps.signal?.aborted)
|
|
273
|
+
break;
|
|
274
|
+
if (selected && !selected.has(discovered.key)) {
|
|
275
|
+
report.sessionsFilteredOut++;
|
|
276
|
+
continue;
|
|
277
|
+
}
|
|
230
278
|
if (filters.session) {
|
|
231
279
|
const wanted = filters.session;
|
|
232
280
|
if (discovered.sessionId !== wanted && discovered.file !== wanted) {
|
|
@@ -245,7 +293,16 @@ export async function importPiHistory(deps, filters) {
|
|
|
245
293
|
});
|
|
246
294
|
continue;
|
|
247
295
|
}
|
|
248
|
-
|
|
296
|
+
// The header ID names the session in the page's list; a file that loads as
|
|
297
|
+
// another session would be imported under keys the user never saw.
|
|
298
|
+
if (discovered.sessionId && discovered.sessionId !== session.sessionId) {
|
|
299
|
+
report.loadErrors.push({
|
|
300
|
+
file: discovered.file,
|
|
301
|
+
error: "Loaded session ID does not match the file header",
|
|
302
|
+
});
|
|
303
|
+
continue;
|
|
304
|
+
}
|
|
305
|
+
const directory = resolveImportProject(session.cwd, pathMaps).directory;
|
|
249
306
|
if (!directory) {
|
|
250
307
|
report.unresolvableSessions.push({ file: session.sourceFile, cwd: session.cwd });
|
|
251
308
|
continue;
|
|
@@ -269,10 +326,10 @@ export async function importPiHistory(deps, filters) {
|
|
|
269
326
|
projectAggregates.set(tags.project.tag, aggregate);
|
|
270
327
|
}
|
|
271
328
|
aggregate.sessions.add(session.sessionId);
|
|
272
|
-
const windows = extractPiConversationWindows(session.branch);
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
329
|
+
const windows = selectImportWindows(extractPiConversationWindows(session.branch), filters);
|
|
330
|
+
report.unitsHeldBack += windows.heldBack;
|
|
331
|
+
report.unitsUntimed += windows.untimed;
|
|
332
|
+
for (const window of windows.kept) {
|
|
276
333
|
const key = buildImportKey(session.sessionId, window);
|
|
277
334
|
if (!key)
|
|
278
335
|
continue;
|
|
@@ -311,10 +368,11 @@ export async function importPiHistory(deps, filters) {
|
|
|
311
368
|
sessions: aggregate.sessions.size,
|
|
312
369
|
units: aggregate.units,
|
|
313
370
|
}));
|
|
314
|
-
if (deps.profile) {
|
|
371
|
+
if (deps.profile && !deps.signal?.aborted) {
|
|
315
372
|
const { importProfileFromHistory } = await import("./profile-import.js");
|
|
316
373
|
report.profile = await importProfileFromHistory(sourceSessions, {
|
|
317
374
|
host: "pi",
|
|
375
|
+
signal: deps.signal,
|
|
318
376
|
dryRun,
|
|
319
377
|
model: deps.profile.model,
|
|
320
378
|
batchSize: deps.profile.batchSize,
|
|
@@ -1,14 +1,22 @@
|
|
|
1
1
|
import type { CaptureSummaryProvider } from "../core/host.js";
|
|
2
2
|
import type { ModelPort } from "../core/profile-analysis.js";
|
|
3
3
|
import { type ImportPathMap, type ImportReport } from "./importer.js";
|
|
4
|
-
import {
|
|
4
|
+
import type { UnresolvedProject } from "./opencode-project.js";
|
|
5
|
+
import { DEFAULT_OPENCODE_DB, type OpencodeOpenOptions } from "./opencode-reader.js";
|
|
5
6
|
import { type ProfileImportReport } from "./profile-import.js";
|
|
7
|
+
export { DEFAULT_OPENCODE_DB };
|
|
6
8
|
export interface OpencodeImportOptions {
|
|
7
9
|
dbPath?: string;
|
|
8
10
|
dryRun?: boolean;
|
|
9
11
|
since?: number;
|
|
10
12
|
until?: number;
|
|
11
13
|
session?: string;
|
|
14
|
+
/** Web selection by session ID; never combined with `session` or `maxSessions`. */
|
|
15
|
+
selectionKeys?: string[];
|
|
16
|
+
/** Web selection: user turns after this epoch ms are held back for a later run. */
|
|
17
|
+
cutoff?: number;
|
|
18
|
+
/** Web jobs share the listing's snapshot of a live database. */
|
|
19
|
+
snapshot?: OpencodeOpenOptions["shared"];
|
|
12
20
|
project?: string;
|
|
13
21
|
maxSessions?: number;
|
|
14
22
|
pathMaps?: ImportPathMap[];
|
|
@@ -18,6 +26,8 @@ export interface OpencodeImportOptions {
|
|
|
18
26
|
profileBatch?: number;
|
|
19
27
|
provider?: CaptureSummaryProvider;
|
|
20
28
|
profileModel?: ModelPort;
|
|
29
|
+
onProgress?: (processed: number, total: number, promptPreview: string) => void;
|
|
30
|
+
signal?: AbortSignal;
|
|
21
31
|
}
|
|
22
32
|
export interface OpencodeImportReport extends ImportReport {
|
|
23
33
|
childSessionsFolded: number;
|
|
@@ -1,11 +1,12 @@
|
|
|
1
|
-
import { homedir } from "node:os";
|
|
2
|
-
import { join } from "node:path";
|
|
3
1
|
import { getTags } from "../services/tags.js";
|
|
4
2
|
import { extractScopeFromContainerTag } from "../services/memory-scope.js";
|
|
5
|
-
import { importHistorySource, projectFilterTag, } from "./importer.js";
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
3
|
+
import { countImportableUnits, importHistorySource, projectFilterTag, selectImportWindows, } from "./importer.js";
|
|
4
|
+
import { resolveImportProject } from "./import-project.js";
|
|
5
|
+
import { DEFAULT_OPENCODE_DB, openOpencodeHistory, } from "./opencode-reader.js";
|
|
8
6
|
import { importProfileFromHistory } from "./profile-import.js";
|
|
7
|
+
export { DEFAULT_OPENCODE_DB };
|
|
8
|
+
/** Let the web server and live capture run between sessions of a long read. */
|
|
9
|
+
const yieldToEventLoop = () => new Promise((resolve) => setImmediate(resolve));
|
|
9
10
|
/** Import resolved OpenCode history through the shared memory and profile steps. */
|
|
10
11
|
export async function importOpencodeHistory(options) {
|
|
11
12
|
if (!options.dryRun && !options.skipMemories && !options.provider) {
|
|
@@ -14,74 +15,137 @@ export async function importOpencodeHistory(options) {
|
|
|
14
15
|
if (!options.dryRun && !options.skipProfile && !options.profileModel) {
|
|
15
16
|
throw new Error("OpenCode history import needs a profile model");
|
|
16
17
|
}
|
|
17
|
-
const dbPath = options.dbPath ??
|
|
18
|
-
// Resolve the filter before opening the reader: the reader only closes once iterated.
|
|
18
|
+
const dbPath = options.dbPath ?? DEFAULT_OPENCODE_DB;
|
|
19
19
|
const projectTag = options.project ? projectFilterTag(options.project) : null;
|
|
20
|
-
const
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
20
|
+
const history = await openOpencodeHistory(dbPath, {
|
|
21
|
+
...(options.since !== undefined ? { since: options.since } : {}),
|
|
22
|
+
...(options.until !== undefined ? { until: options.until } : {}),
|
|
23
|
+
...(options.session ? { session: options.session } : {}),
|
|
24
|
+
...(options.selectionKeys ? { sessionIds: options.selectionKeys } : {}),
|
|
25
|
+
...(options.maxSessions ? { maxSessions: options.maxSessions } : {}),
|
|
26
|
+
}, {
|
|
27
|
+
...(options.snapshot ? { shared: options.snapshot } : {}),
|
|
28
|
+
...(options.signal ? { signal: options.signal } : {}),
|
|
29
|
+
});
|
|
30
|
+
try {
|
|
31
|
+
const unresolved = new Map();
|
|
32
|
+
const projects = new Map();
|
|
33
|
+
const selected = [];
|
|
34
|
+
let filtered = 0;
|
|
35
|
+
for (const session of history.sessions) {
|
|
36
|
+
const { directory } = resolveImportProject(session.recordedDirectory, options.pathMaps ?? [], session.projectWorktree);
|
|
37
|
+
if (!directory) {
|
|
38
|
+
const entry = unresolved.get(session.recordedDirectory) ?? {
|
|
39
|
+
directory: session.recordedDirectory,
|
|
40
|
+
sessions: 0,
|
|
41
|
+
units: 0,
|
|
42
|
+
};
|
|
43
|
+
entry.sessions++;
|
|
44
|
+
entry.units += history.loadUnits(session.sessionId).length;
|
|
45
|
+
unresolved.set(entry.directory, entry);
|
|
46
|
+
continue;
|
|
47
|
+
}
|
|
48
|
+
if (projectTag && projectTag !== projectFilterTag(directory)) {
|
|
49
|
+
filtered++;
|
|
50
|
+
continue;
|
|
51
|
+
}
|
|
52
|
+
selected.push({ ...session, directory });
|
|
30
53
|
}
|
|
31
|
-
const
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
54
|
+
const build = (session) => {
|
|
55
|
+
const windows = selectImportWindows(history.loadUnits(session.sessionId), options);
|
|
56
|
+
const source = {
|
|
57
|
+
sessionId: session.sessionId,
|
|
58
|
+
directory: session.directory,
|
|
59
|
+
sourceFile: dbPath,
|
|
60
|
+
units: windows.kept,
|
|
61
|
+
};
|
|
62
|
+
return { source, windows };
|
|
38
63
|
};
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
64
|
+
// Counting pass: totals for progress and the report, one session in memory at a time.
|
|
65
|
+
let total = 0;
|
|
66
|
+
let kept = 0;
|
|
67
|
+
let heldBack = 0;
|
|
68
|
+
let untimed = 0;
|
|
69
|
+
for (const session of selected) {
|
|
70
|
+
if (options.signal?.aborted)
|
|
71
|
+
break;
|
|
72
|
+
const { source, windows } = build(session);
|
|
73
|
+
const info = getTags(session.directory).project;
|
|
74
|
+
const project = projects.get(info.tag) ?? {
|
|
75
|
+
tag: info.tag,
|
|
76
|
+
hash: extractScopeFromContainerTag(info.tag).hash,
|
|
77
|
+
directory: session.directory,
|
|
78
|
+
sessions: 0,
|
|
79
|
+
units: 0,
|
|
80
|
+
sessionIds: new Set(),
|
|
81
|
+
};
|
|
82
|
+
project.sessionIds.add(session.sessionId);
|
|
83
|
+
project.sessions = project.sessionIds.size;
|
|
84
|
+
project.units += windows.kept.length;
|
|
85
|
+
projects.set(info.tag, project);
|
|
86
|
+
total += countImportableUnits(source, "opencode", options);
|
|
87
|
+
kept += windows.kept.length;
|
|
88
|
+
heldBack += windows.heldBack;
|
|
89
|
+
untimed += windows.untimed;
|
|
90
|
+
await yieldToEventLoop();
|
|
91
|
+
}
|
|
92
|
+
const open = () => (async function* () {
|
|
93
|
+
for (const session of selected) {
|
|
94
|
+
if (options.signal?.aborted)
|
|
95
|
+
return;
|
|
96
|
+
await yieldToEventLoop();
|
|
97
|
+
yield build(session).source;
|
|
98
|
+
}
|
|
99
|
+
})();
|
|
100
|
+
const report = {
|
|
101
|
+
dryRun: Boolean(options.dryRun),
|
|
102
|
+
root: dbPath,
|
|
103
|
+
sessionsDiscovered: history.topLevelSessions,
|
|
104
|
+
sessionsLoaded: selected.length,
|
|
105
|
+
sessionsFilteredOut: filtered,
|
|
106
|
+
sessionsUnrecognized: 0,
|
|
107
|
+
unresolvableSessions: [],
|
|
108
|
+
loadErrors: [],
|
|
109
|
+
unitsTotal: 0,
|
|
110
|
+
unitsImported: 0,
|
|
111
|
+
unitsWouldImport: 0,
|
|
112
|
+
unitsSkipped: 0,
|
|
113
|
+
unitsFailed: 0,
|
|
114
|
+
unitsAlreadyHandled: 0,
|
|
115
|
+
unitsHeldBack: heldBack,
|
|
116
|
+
unitsUntimed: untimed,
|
|
117
|
+
skipReasons: {},
|
|
118
|
+
units: [],
|
|
119
|
+
projects: [...projects.values()].map(({ sessionIds: _ids, ...project }) => project),
|
|
120
|
+
childSessionsFolded: history.childSessions,
|
|
121
|
+
unresolvedProjects: [...unresolved.values()],
|
|
122
|
+
};
|
|
123
|
+
if (!options.skipMemories) {
|
|
124
|
+
await importHistorySource({ total, open }, "opencode", {
|
|
125
|
+
provider: options.provider ?? {
|
|
126
|
+
summarize: async () => {
|
|
127
|
+
throw new Error("A dry-run cannot call the capture model");
|
|
128
|
+
},
|
|
71
129
|
},
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
130
|
+
...(options.signal ? { signal: options.signal } : {}),
|
|
131
|
+
...(options.onProgress ? { onProgress: options.onProgress } : {}),
|
|
132
|
+
}, options, report);
|
|
133
|
+
}
|
|
134
|
+
else {
|
|
135
|
+
report.unitsTotal = kept;
|
|
136
|
+
}
|
|
137
|
+
if (!options.skipProfile && !options.signal?.aborted) {
|
|
138
|
+
report.profile = await importProfileFromHistory(open(), {
|
|
139
|
+
host: "opencode",
|
|
140
|
+
signal: options.signal,
|
|
141
|
+
dryRun: options.dryRun,
|
|
142
|
+
batchSize: options.profileBatch,
|
|
143
|
+
model: options.profileModel,
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
return report;
|
|
77
147
|
}
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
host: "opencode",
|
|
81
|
-
dryRun: options.dryRun,
|
|
82
|
-
batchSize: options.profileBatch,
|
|
83
|
-
model: options.profileModel,
|
|
84
|
-
});
|
|
148
|
+
finally {
|
|
149
|
+
await history.close();
|
|
85
150
|
}
|
|
86
|
-
return report;
|
|
87
151
|
}
|
|
@@ -1,11 +1,8 @@
|
|
|
1
1
|
import type { ImportPathMap } from "./importer.js";
|
|
2
|
-
|
|
3
|
-
/** Prefer the recorded directory, then an explicit map, then the project root. */
|
|
2
|
+
/** A directory map first, then the recorded directory, then the project root. */
|
|
4
3
|
export declare function resolveOpencodeProject(recordedDirectory: string, projectWorktree: string | null, maps?: ImportPathMap[]): string | null;
|
|
5
4
|
export interface UnresolvedProject {
|
|
6
5
|
directory: string;
|
|
7
6
|
sessions: number;
|
|
8
7
|
units: number;
|
|
9
8
|
}
|
|
10
|
-
/** Resolve each session without buffering its conversation windows. */
|
|
11
|
-
export declare function resolveOpencodeSessions(sessions: AsyncIterable<OpencodeSourceSession>, maps: ImportPathMap[], unresolved: Map<string, UnresolvedProject>): AsyncGenerator<OpencodeSourceSession>;
|