om-memory-system 3.4.0-next.23 → 3.4.0-next.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -7
- package/dist/adapters/opencode/backfill-models.js +5 -1
- package/dist/adapters/opencode/backfill-startup.d.ts +6 -0
- package/dist/adapters/opencode/backfill-startup.js +5 -0
- package/dist/adapters/opencode/import-command.js +1 -0
- package/dist/adapters/opencode/user-prompt.d.ts +2 -1
- package/dist/adapters/opencode/user-prompt.js +2 -9
- package/dist/adapters/pi/backfill-models.d.ts +4 -0
- package/dist/adapters/pi/backfill-models.js +3 -0
- package/dist/adapters/pi/capture.d.ts +1 -1
- package/dist/adapters/pi/capture.js +1 -1
- package/dist/adapters/pi/extension.js +7 -0
- package/dist/adapters/pi/import-command.js +1 -0
- package/dist/cli/index.js +13 -1
- package/dist/config.d.ts +11 -4
- package/dist/config.js +29 -6
- package/dist/core/internal-prompt.d.ts +6 -0
- package/dist/core/internal-prompt.js +11 -0
- package/dist/importer/auto-backfill.d.ts +5 -0
- package/dist/importer/auto-backfill.js +29 -10
- package/dist/importer/backfill-controls.d.ts +37 -0
- package/dist/importer/backfill-controls.js +119 -0
- package/dist/importer/backfill-lock.d.ts +1 -0
- package/dist/importer/backfill-lock.js +2 -2
- package/dist/importer/backfill-model.d.ts +1 -1
- package/dist/importer/backfill-model.js +2 -0
- package/dist/importer/external-api-test.d.ts +18 -0
- package/dist/importer/external-api-test.js +50 -0
- package/dist/importer/external-backfill-models.d.ts +15 -0
- package/dist/importer/external-backfill-models.js +14 -0
- package/dist/importer/import-path-maps.d.ts +14 -0
- package/dist/importer/import-path-maps.js +49 -0
- package/dist/importer/import-progress.d.ts +27 -0
- package/dist/importer/import-progress.js +38 -0
- package/dist/importer/import-runs.d.ts +43 -0
- package/dist/importer/import-runs.js +141 -0
- package/dist/importer/import-sessions.js +9 -9
- package/dist/importer/importer.d.ts +2 -1
- package/dist/importer/importer.js +2 -2
- package/dist/importer/map-suggestions.d.ts +39 -0
- package/dist/importer/map-suggestions.js +133 -0
- package/dist/importer/opencode-import.d.ts +2 -1
- package/dist/{adapters/pi/conversation.d.ts → importer/pi-conversation.d.ts} +1 -1
- package/dist/importer/profile-import.js +3 -2
- package/dist/importer/run-import.d.ts +21 -3
- package/dist/importer/run-import.js +82 -2
- package/dist/importer/session-loader.d.ts +1 -1
- package/dist/importer/web-import-api.d.ts +3 -0
- package/dist/importer/web-import-api.js +3 -0
- package/dist/importer/web-import-jobs.js +1 -0
- package/dist/index.js +9 -4
- package/dist/services/ai/live-model-choice.d.ts +4 -1
- package/dist/services/ai/live-model-choice.js +18 -0
- package/dist/services/api-handlers.d.ts +6 -1
- package/dist/services/api-handlers.js +71 -62
- package/dist/services/backfill-state.d.ts +30 -0
- package/dist/services/backfill-state.js +47 -0
- package/dist/services/capture-diagnostics.d.ts +2 -8
- package/dist/services/capture-diagnostics.js +3 -49
- package/dist/services/global-config-writer.d.ts +2 -0
- package/dist/services/global-config-writer.js +37 -6
- package/dist/services/global-version.d.ts +31 -0
- package/dist/services/global-version.js +78 -0
- package/dist/services/memory-key-source.d.ts +35 -0
- package/dist/services/memory-key-source.js +123 -0
- package/dist/services/package-version.d.ts +2 -0
- package/dist/services/package-version.js +21 -0
- package/dist/services/private-path.d.ts +13 -0
- package/dist/services/private-path.js +54 -0
- package/dist/services/settings-snapshot.d.ts +17 -0
- package/dist/services/settings-snapshot.js +51 -14
- package/dist/services/turso/vector-search.d.ts +1 -1
- package/dist/services/turso/vector-search.js +5 -2
- package/dist/services/web-server.d.ts +2 -0
- package/dist/services/web-server.js +74 -1
- package/dist/web/assets/index-DvZhF1BJ.css +2 -0
- package/dist/web/assets/index-jqmX5i_i.js +124 -0
- package/dist/web/index.html +2 -2
- package/package.json +6 -1
- package/dist/web/assets/index-B-_gMSn9.js +0 -124
- package/dist/web/assets/index-D5ak5KsI.css +0 -2
- /package/dist/{adapters/pi/conversation.js → importer/pi-conversation.js} +0 -0
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import { CONFIG, refreshConfigIfChanged } from "../config.js";
|
|
2
2
|
import { log } from "../services/logger.js";
|
|
3
3
|
import { tryAcquireBackfillLock } from "./backfill-lock.js";
|
|
4
|
-
import { getBackfillCutoff, updateBackfillStatus, } from "../services/backfill-state.js";
|
|
4
|
+
import { getBackfillCutoff, recordUnresolvedDirectories, unresolvedDirectoriesOf, updateBackfillStatus, } from "../services/backfill-state.js";
|
|
5
5
|
import { runHistoryImport, } from "./run-import.js";
|
|
6
6
|
import { isManualImportRunning } from "./manual-import-guard.js";
|
|
7
|
+
import { isBackfillPaused } from "./import-runs.js";
|
|
8
|
+
import { workProgress } from "./import-progress.js";
|
|
7
9
|
const zero = () => ({ imported: 0, skipped: 0, failed: 0, pending: 0, unresolved: 0 });
|
|
8
10
|
function counts(report) {
|
|
9
11
|
return {
|
|
@@ -27,14 +29,19 @@ const delay = (signal) => new Promise((resolve) => {
|
|
|
27
29
|
});
|
|
28
30
|
/** Schedule one host's history import after startup; callers do not await it. */
|
|
29
31
|
export async function scheduleAutoBackfill(options) {
|
|
30
|
-
|
|
32
|
+
if (!options.userStarted)
|
|
33
|
+
await (options.wait ?? (() => delay(options.signal)))();
|
|
31
34
|
if (options.signal?.aborted)
|
|
32
35
|
return;
|
|
33
|
-
|
|
34
|
-
(() => CONFIG.autoBackfill && process.env.OMMS_DISABLE_AUTO_BACKFILL !== "1")
|
|
36
|
+
const enabled = options.enabled ??
|
|
37
|
+
(() => CONFIG.autoBackfill && process.env.OMMS_DISABLE_AUTO_BACKFILL !== "1");
|
|
38
|
+
if (!options.userStarted && !enabled())
|
|
35
39
|
return;
|
|
36
40
|
if (isManualImportRunning(options.host))
|
|
37
41
|
return;
|
|
42
|
+
// A paused backfill waits for Resume, across host starts.
|
|
43
|
+
if (await isBackfillPaused(options.host).catch(() => false))
|
|
44
|
+
return;
|
|
38
45
|
const release = await tryAcquireBackfillLock(options.host);
|
|
39
46
|
if (!release)
|
|
40
47
|
return;
|
|
@@ -72,6 +79,10 @@ export async function scheduleAutoBackfill(options) {
|
|
|
72
79
|
signal: controller.signal,
|
|
73
80
|
});
|
|
74
81
|
current = counts(dry);
|
|
82
|
+
await recordUnresolvedDirectories(options.host, unresolvedDirectoriesOf(dry)).catch((error) => log("Backfill unresolved directories write failed", {
|
|
83
|
+
host: options.host,
|
|
84
|
+
error: error instanceof Error ? error.message : String(error),
|
|
85
|
+
}));
|
|
75
86
|
if (controller.signal.aborted) {
|
|
76
87
|
state = "stopped";
|
|
77
88
|
return;
|
|
@@ -122,16 +133,22 @@ export async function scheduleAutoBackfill(options) {
|
|
|
122
133
|
cwd: options.cwd,
|
|
123
134
|
models,
|
|
124
135
|
signal: controller.signal,
|
|
125
|
-
|
|
126
|
-
|
|
136
|
+
track: {
|
|
137
|
+
surface: options.surface ?? "auto",
|
|
138
|
+
lockHeld: true,
|
|
139
|
+
expectedTotal: dry.unitsWouldImport,
|
|
140
|
+
},
|
|
141
|
+
onProgress: (done, total, _preview, alreadyHandled = 0) => {
|
|
142
|
+
// Count only units that needed work; ledger hits are not pending work.
|
|
143
|
+
processed = workProgress(done, total, alreadyHandled, dry.unitsWouldImport).done;
|
|
127
144
|
const now = clock();
|
|
128
145
|
if (now - checkedAt < 5_000)
|
|
129
146
|
return;
|
|
130
147
|
checkedAt = now;
|
|
131
148
|
refreshConfigIfChanged(options.cwd);
|
|
132
|
-
if (!CONFIG.autoBackfill)
|
|
149
|
+
if (!options.userStarted && !CONFIG.autoBackfill)
|
|
133
150
|
controller.abort();
|
|
134
|
-
current = { ...current, pending: Math.max(0,
|
|
151
|
+
current = { ...current, pending: Math.max(0, dry.unitsWouldImport - processed) };
|
|
135
152
|
pendingWrite = pendingWrite
|
|
136
153
|
.then(() => updateBackfillStatus(options.host, { state: "running", model, counts: current }))
|
|
137
154
|
.catch((error) => {
|
|
@@ -143,8 +160,10 @@ export async function scheduleAutoBackfill(options) {
|
|
|
143
160
|
},
|
|
144
161
|
});
|
|
145
162
|
await pendingWrite;
|
|
146
|
-
|
|
147
|
-
|
|
163
|
+
const finished = real.unitsImported + real.unitsSkipped + real.unitsFailed;
|
|
164
|
+
current = { ...counts(real), pending: Math.max(0, dry.unitsWouldImport - finished) };
|
|
165
|
+
const paused = await isBackfillPaused(options.host).catch(() => false);
|
|
166
|
+
state = controller.signal.aborted || paused ? "stopped" : real.unitsFailed ? "failed" : "done";
|
|
148
167
|
if (!options.signal?.aborted) {
|
|
149
168
|
options.notify(`${options.host} automatic import ${state}: ${current.imported} imported, ${current.failed} failed. autoBackfill controls the next run.`);
|
|
150
169
|
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { type BackfillHost } from "./backfill-model.js";
|
|
2
|
+
import { type ImportRun } from "./import-runs.js";
|
|
3
|
+
import type { HistoryImportModels } from "./run-import.js";
|
|
4
|
+
type ResolveModels = () => Promise<{
|
|
5
|
+
model: string;
|
|
6
|
+
models: HistoryImportModels;
|
|
7
|
+
}>;
|
|
8
|
+
export declare function registerHostBackfillModels(host: BackfillHost, resolve: ResolveModels): void;
|
|
9
|
+
/** A control request the page cannot carry out; `status` is the HTTP status. */
|
|
10
|
+
export declare class BackfillControlError extends Error {
|
|
11
|
+
readonly status: number;
|
|
12
|
+
constructor(message: string, status: number);
|
|
13
|
+
}
|
|
14
|
+
/** True when the host's backfill model is the external API, directly or through the live rule. */
|
|
15
|
+
export declare function backfillUsesExternal(host: BackfillHost): boolean;
|
|
16
|
+
/** Why Run now is not available for the host in this process, or null when it is. */
|
|
17
|
+
export declare function runNowUnavailableReason(host: BackfillHost): string | null;
|
|
18
|
+
export interface BackfillHostStatus {
|
|
19
|
+
run: ImportRun | null;
|
|
20
|
+
runNowUnavailable: string | null;
|
|
21
|
+
}
|
|
22
|
+
/** Run now, Pause, and Resume for each host's backfill, run in this process. */
|
|
23
|
+
export declare class BackfillControls {
|
|
24
|
+
private readonly controllers;
|
|
25
|
+
status(): Promise<Record<BackfillHost, BackfillHostStatus>>;
|
|
26
|
+
runNow(host: BackfillHost, cwd: string): Promise<{
|
|
27
|
+
started: true;
|
|
28
|
+
}>;
|
|
29
|
+
/** Stop after the current exchange; a run in another process sees the flag at its next write. */
|
|
30
|
+
pause(host: BackfillHost): Promise<{
|
|
31
|
+
paused: true;
|
|
32
|
+
}>;
|
|
33
|
+
resume(host: BackfillHost, cwd: string): Promise<{
|
|
34
|
+
started: true;
|
|
35
|
+
}>;
|
|
36
|
+
}
|
|
37
|
+
export {};
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import { CONFIG } from "../config.js";
|
|
2
|
+
import { log } from "../services/logger.js";
|
|
3
|
+
import { getAutoCaptureProviderStatus, resolvePiLiveModel, } from "../services/ai/live-model-choice.js";
|
|
4
|
+
import { parseBackfillModel } from "./backfill-model.js";
|
|
5
|
+
import { readImportRun, setBackfillPaused } from "./import-runs.js";
|
|
6
|
+
/** A host runtime in this process that can resolve its signed-in backfill models. */
|
|
7
|
+
const hostResolvers = new Map();
|
|
8
|
+
export function registerHostBackfillModels(host, resolve) {
|
|
9
|
+
hostResolvers.set(host, resolve);
|
|
10
|
+
}
|
|
11
|
+
/** A control request the page cannot carry out; `status` is the HTTP status. */
|
|
12
|
+
export class BackfillControlError extends Error {
|
|
13
|
+
status;
|
|
14
|
+
constructor(message, status) {
|
|
15
|
+
super(message);
|
|
16
|
+
this.status = status;
|
|
17
|
+
this.name = "BackfillControlError";
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
const label = (host) => (host === "pi" ? "Pi" : "OpenCode");
|
|
21
|
+
/** True when the host's backfill model is the external API, directly or through the live rule. */
|
|
22
|
+
export function backfillUsesExternal(host) {
|
|
23
|
+
const choice = parseBackfillModel(CONFIG, host);
|
|
24
|
+
if (choice === "external")
|
|
25
|
+
return true;
|
|
26
|
+
if (choice !== "inherit")
|
|
27
|
+
return false;
|
|
28
|
+
if (host === "pi")
|
|
29
|
+
return resolvePiLiveModel(CONFIG).kind === "manual";
|
|
30
|
+
const live = getAutoCaptureProviderStatus(CONFIG);
|
|
31
|
+
return live.ready && live.mode === "manual";
|
|
32
|
+
}
|
|
33
|
+
/** Why Run now is not available for the host in this process, or null when it is. */
|
|
34
|
+
export function runNowUnavailableReason(host) {
|
|
35
|
+
if (hostResolvers.has(host))
|
|
36
|
+
return null;
|
|
37
|
+
if (!backfillUsesExternal(host)) {
|
|
38
|
+
return `Open ${label(host)}, or choose the external API for ${label(host)}'s backfill`;
|
|
39
|
+
}
|
|
40
|
+
return null;
|
|
41
|
+
}
|
|
42
|
+
async function resolveModels(host) {
|
|
43
|
+
const hostResolver = hostResolvers.get(host);
|
|
44
|
+
if (hostResolver && !backfillUsesExternal(host))
|
|
45
|
+
return hostResolver();
|
|
46
|
+
const { resolveExternalBackfillModels } = await import("./external-backfill-models.js");
|
|
47
|
+
return resolveExternalBackfillModels(host, CONFIG);
|
|
48
|
+
}
|
|
49
|
+
/** Run now, Pause, and Resume for each host's backfill, run in this process. */
|
|
50
|
+
export class BackfillControls {
|
|
51
|
+
controllers = new Map();
|
|
52
|
+
async status() {
|
|
53
|
+
const entry = async (host) => ({
|
|
54
|
+
run: await readImportRun(host),
|
|
55
|
+
runNowUnavailable: runNowUnavailableReason(host),
|
|
56
|
+
});
|
|
57
|
+
return { pi: await entry("pi"), opencode: await entry("opencode") };
|
|
58
|
+
}
|
|
59
|
+
async runNow(host, cwd) {
|
|
60
|
+
const reason = runNowUnavailableReason(host);
|
|
61
|
+
if (reason)
|
|
62
|
+
throw new BackfillControlError(reason, 409);
|
|
63
|
+
const current = await readImportRun(host);
|
|
64
|
+
if (current?.state === "running" || this.controllers.has(host)) {
|
|
65
|
+
throw new BackfillControlError(`${host === "pi" ? "A" : "An"} ${label(host)} import is already running`, 409);
|
|
66
|
+
}
|
|
67
|
+
if (current?.paused) {
|
|
68
|
+
throw new BackfillControlError(`${label(host)}'s backfill is paused; resume it first`, 409);
|
|
69
|
+
}
|
|
70
|
+
// Fail fast with the missing setting instead of starting a run that cannot call a model.
|
|
71
|
+
if (backfillUsesExternal(host) || !hostResolvers.has(host)) {
|
|
72
|
+
const { externalModelIssues } = await import("../services/ai/live-model-choice.js");
|
|
73
|
+
const issues = externalModelIssues(CONFIG);
|
|
74
|
+
if (issues.length > 0)
|
|
75
|
+
throw new BackfillControlError(issues.join("; "), 409);
|
|
76
|
+
}
|
|
77
|
+
const controller = new AbortController();
|
|
78
|
+
this.controllers.set(host, controller);
|
|
79
|
+
const { scheduleAutoBackfill } = await import("./auto-backfill.js");
|
|
80
|
+
void scheduleAutoBackfill({
|
|
81
|
+
host,
|
|
82
|
+
cwd,
|
|
83
|
+
signal: controller.signal,
|
|
84
|
+
surface: "web",
|
|
85
|
+
userStarted: true,
|
|
86
|
+
resolveModels: () => resolveModels(host),
|
|
87
|
+
notify: (message) => log("Backfill from the Settings page", { host, message: message.slice(0, 200) }),
|
|
88
|
+
})
|
|
89
|
+
.catch((error) => log("Backfill from the Settings page failed", {
|
|
90
|
+
host,
|
|
91
|
+
error: error instanceof Error ? error.name : "unknown",
|
|
92
|
+
}))
|
|
93
|
+
.finally(() => {
|
|
94
|
+
if (this.controllers.get(host) === controller)
|
|
95
|
+
this.controllers.delete(host);
|
|
96
|
+
});
|
|
97
|
+
return { started: true };
|
|
98
|
+
}
|
|
99
|
+
/** Stop after the current exchange; a run in another process sees the flag at its next write. */
|
|
100
|
+
async pause(host) {
|
|
101
|
+
await setBackfillPaused(host, true);
|
|
102
|
+
this.controllers.get(host)?.abort();
|
|
103
|
+
return { paused: true };
|
|
104
|
+
}
|
|
105
|
+
async resume(host, cwd) {
|
|
106
|
+
const reason = runNowUnavailableReason(host);
|
|
107
|
+
if (reason)
|
|
108
|
+
throw new BackfillControlError(reason, 409);
|
|
109
|
+
await setBackfillPaused(host, false);
|
|
110
|
+
try {
|
|
111
|
+
return await this.runNow(host, cwd);
|
|
112
|
+
}
|
|
113
|
+
catch (error) {
|
|
114
|
+
// Run now refused (a run is active, or a setting is missing): stay paused.
|
|
115
|
+
await setBackfillPaused(host, true);
|
|
116
|
+
throw error;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
1
|
import type { BackfillHost } from "./backfill-model.js";
|
|
2
|
+
export declare function isProcessAlive(pid: number): boolean;
|
|
2
3
|
/** Claim one host's import without holding a database transaction during the run. */
|
|
3
4
|
export declare function tryAcquireBackfillLock(host: BackfillHost, storagePath?: string): Promise<(() => Promise<void>) | null>;
|
|
@@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto";
|
|
|
2
2
|
import { join } from "node:path";
|
|
3
3
|
import { CONFIG } from "../config.js";
|
|
4
4
|
import { tursoConnectionManager } from "../services/turso/connection-manager.js";
|
|
5
|
-
function
|
|
5
|
+
export function isProcessAlive(pid) {
|
|
6
6
|
if (!Number.isSafeInteger(pid) || pid <= 0)
|
|
7
7
|
return false;
|
|
8
8
|
try {
|
|
@@ -26,7 +26,7 @@ export async function tryAcquireBackfillLock(host, storagePath = CONFIG.storageP
|
|
|
26
26
|
const holder = await db.get("SELECT pid, token FROM backfill_locks WHERE host = ?", [host]);
|
|
27
27
|
if (!holder)
|
|
28
28
|
continue;
|
|
29
|
-
if (
|
|
29
|
+
if (isProcessAlive(holder.pid))
|
|
30
30
|
return null;
|
|
31
31
|
const replaced = await db.run("UPDATE backfill_locks SET pid = ?, token = ? WHERE host = ? AND pid = ? AND token = ?", [state.pid, state.token, host, holder.pid, holder.token]);
|
|
32
32
|
if (!replaced)
|
|
@@ -3,6 +3,8 @@ export function parseBackfillModel(config, host) {
|
|
|
3
3
|
const value = host === "pi" ? config.piBackfillModel : config.opencodeBackfillModel;
|
|
4
4
|
if (value === undefined || value === "inherit")
|
|
5
5
|
return "inherit";
|
|
6
|
+
if (value === "external")
|
|
7
|
+
return "external";
|
|
6
8
|
if (typeof value !== "string")
|
|
7
9
|
throw new Error(`Invalid ${host}BackfillModel config`);
|
|
8
10
|
const slash = value.indexOf("/");
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { CONFIG } from "../config.js";
|
|
2
|
+
export type ExternalApiTestResult = {
|
|
3
|
+
ok: true;
|
|
4
|
+
model: string;
|
|
5
|
+
} | {
|
|
6
|
+
ok: false;
|
|
7
|
+
error: string;
|
|
8
|
+
};
|
|
9
|
+
/**
|
|
10
|
+
* One small call to the saved external API from this process, with a small
|
|
11
|
+
* output limit. The error never contains the key.
|
|
12
|
+
*/
|
|
13
|
+
export declare function testExternalApi(config?: typeof CONFIG, createProvider?: (provider: string, providerConfig: Record<string, unknown>) => {
|
|
14
|
+
executeToolCall: (...args: any[]) => Promise<{
|
|
15
|
+
success: boolean;
|
|
16
|
+
error?: string;
|
|
17
|
+
}>;
|
|
18
|
+
}): Promise<ExternalApiTestResult>;
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { CONFIG } from "../config.js";
|
|
3
|
+
import { safeHealthError } from "../services/safe-health-error.js";
|
|
4
|
+
const OUTPUT_LIMIT = 64;
|
|
5
|
+
const TIMEOUT_MS = 20_000;
|
|
6
|
+
/** Provider-specific output caps; Anthropic-style providers read `maxTokens`. */
|
|
7
|
+
function outputLimitParams(provider) {
|
|
8
|
+
if (provider === "openai-chat" || provider === "orcarouter")
|
|
9
|
+
return { max_tokens: OUTPUT_LIMIT };
|
|
10
|
+
if (provider === "openai-responses")
|
|
11
|
+
return { max_output_tokens: OUTPUT_LIMIT };
|
|
12
|
+
return {};
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* One small call to the saved external API from this process, with a small
|
|
16
|
+
* output limit. The error never contains the key.
|
|
17
|
+
*/
|
|
18
|
+
export async function testExternalApi(config = CONFIG, createProvider) {
|
|
19
|
+
const secrets = [config.memoryApiKey];
|
|
20
|
+
try {
|
|
21
|
+
const { buildMemoryProviderConfig } = await import("../services/ai/provider-config.js");
|
|
22
|
+
const base = buildMemoryProviderConfig(config, {
|
|
23
|
+
maxIterations: 1,
|
|
24
|
+
iterationTimeout: TIMEOUT_MS,
|
|
25
|
+
});
|
|
26
|
+
const providerConfig = {
|
|
27
|
+
...base,
|
|
28
|
+
maxTokens: OUTPUT_LIMIT,
|
|
29
|
+
extraParams: { ...base.extraParams, ...outputLimitParams(config.memoryProvider) },
|
|
30
|
+
};
|
|
31
|
+
const create = createProvider ??
|
|
32
|
+
(await import("../services/ai/ai-provider-factory.js")).AIProviderFactory.createProvider;
|
|
33
|
+
const provider = create(config.memoryProvider, providerConfig);
|
|
34
|
+
const tool = {
|
|
35
|
+
type: "function",
|
|
36
|
+
function: {
|
|
37
|
+
name: "acknowledge",
|
|
38
|
+
description: "Confirm the connection works",
|
|
39
|
+
parameters: { type: "object", properties: { ok: { type: "boolean" } }, required: ["ok"] },
|
|
40
|
+
},
|
|
41
|
+
};
|
|
42
|
+
const result = await provider.executeToolCall("You are testing a connection. Call the acknowledge tool with ok set to true.", "Acknowledge.", tool, `external-api-test-${randomUUID()}`);
|
|
43
|
+
if (!result.success)
|
|
44
|
+
throw new Error(result.error ?? "The external API call failed");
|
|
45
|
+
return { ok: true, model: `${config.memoryProvider}/${config.memoryModel ?? "default"}` };
|
|
46
|
+
}
|
|
47
|
+
catch (error) {
|
|
48
|
+
return { ok: false, error: safeHealthError(error, secrets).slice(0, 500) };
|
|
49
|
+
}
|
|
50
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { BackfillHost } from "./backfill-model.js";
|
|
2
|
+
import type { SelectedImportModel } from "./model-selection.js";
|
|
3
|
+
import type { HistoryImportModels } from "./run-import.js";
|
|
4
|
+
interface ExternalConfig {
|
|
5
|
+
memoryProvider?: string;
|
|
6
|
+
memoryModel?: string;
|
|
7
|
+
memoryApiUrl?: string;
|
|
8
|
+
memoryApiKey?: string;
|
|
9
|
+
}
|
|
10
|
+
/** A backfill set to `external`: the external API, or an error naming the missing settings. */
|
|
11
|
+
export declare function resolveExternalBackfillModels(host: BackfillHost, config: ExternalConfig, externalModels?: (flags: Record<string, never>) => SelectedImportModel): Promise<{
|
|
12
|
+
model: string;
|
|
13
|
+
models: HistoryImportModels;
|
|
14
|
+
}>;
|
|
15
|
+
export {};
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { externalModelIssues } from "../services/ai/live-model-choice.js";
|
|
2
|
+
/** A backfill set to `external`: the external API, or an error naming the missing settings. */
|
|
3
|
+
export async function resolveExternalBackfillModels(host, config, externalModels) {
|
|
4
|
+
const issues = externalModelIssues(config);
|
|
5
|
+
const label = host === "pi" ? "Pi" : "OpenCode";
|
|
6
|
+
if (issues.length > 0)
|
|
7
|
+
throw new Error(`${label} backfill: ${issues.join("; ")}`);
|
|
8
|
+
const select = externalModels ?? (await import("./model-selection.js")).selectImportModel;
|
|
9
|
+
const external = select({});
|
|
10
|
+
return {
|
|
11
|
+
model: `${external.provider}/${external.modelId}`,
|
|
12
|
+
models: { capture: external.capture, profile: external.profile },
|
|
13
|
+
};
|
|
14
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { ImportPathMap } from "./importer.js";
|
|
2
|
+
/**
|
|
3
|
+
* Validate and normalise `importPathMaps` from the global config. Each entry
|
|
4
|
+
* needs absolute `from` and `to` directories after `~` expansion. Throws on
|
|
5
|
+
* the first invalid entry, like the rest of config validation.
|
|
6
|
+
*/
|
|
7
|
+
export declare function parseImportPathMaps(value: unknown): ImportPathMap[];
|
|
8
|
+
/** Saved maps with the run's own maps on top; a run map wins for the same `from`. */
|
|
9
|
+
export declare function mergeImportPathMaps(saved: readonly ImportPathMap[], run: readonly ImportPathMap[]): ImportPathMap[];
|
|
10
|
+
/**
|
|
11
|
+
* The maps one import run uses: the saved `importPathMaps` with the run's
|
|
12
|
+
* maps on top, run targets resolved against `cwd`.
|
|
13
|
+
*/
|
|
14
|
+
export declare function runPathMaps(runMaps: readonly ImportPathMap[], cwd: string, saved?: readonly ImportPathMap[]): Promise<ImportPathMap[]>;
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { homedir } from "node:os";
|
|
2
|
+
import { isAbsolute, join, resolve } from "node:path";
|
|
3
|
+
function expandHome(path) {
|
|
4
|
+
if (path === "~")
|
|
5
|
+
return homedir();
|
|
6
|
+
if (path.startsWith("~/") || path.startsWith("~\\"))
|
|
7
|
+
return join(homedir(), path.slice(2));
|
|
8
|
+
return path;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Validate and normalise `importPathMaps` from the global config. Each entry
|
|
12
|
+
* needs absolute `from` and `to` directories after `~` expansion. Throws on
|
|
13
|
+
* the first invalid entry, like the rest of config validation.
|
|
14
|
+
*/
|
|
15
|
+
export function parseImportPathMaps(value) {
|
|
16
|
+
if (value === undefined)
|
|
17
|
+
return [];
|
|
18
|
+
if (!Array.isArray(value))
|
|
19
|
+
throw new Error("Invalid importPathMaps config: must be a list");
|
|
20
|
+
return value.map((entry, index) => {
|
|
21
|
+
const from = entry?.from;
|
|
22
|
+
const to = entry?.to;
|
|
23
|
+
if (typeof from !== "string" || typeof to !== "string" || !from.trim() || !to.trim()) {
|
|
24
|
+
throw new Error(`Invalid importPathMaps config: entry ${index} needs "from" and "to"`);
|
|
25
|
+
}
|
|
26
|
+
const expanded = { from: expandHome(from.trim()), to: expandHome(to.trim()) };
|
|
27
|
+
if (!isAbsolute(expanded.from) || !isAbsolute(expanded.to)) {
|
|
28
|
+
throw new Error(`Invalid importPathMaps config: entry ${index} needs absolute paths`);
|
|
29
|
+
}
|
|
30
|
+
return { from: resolve(expanded.from), to: resolve(expanded.to) };
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
/** Saved maps with the run's own maps on top; a run map wins for the same `from`. */
|
|
34
|
+
export function mergeImportPathMaps(saved, run) {
|
|
35
|
+
const byFrom = new Map();
|
|
36
|
+
for (const map of saved)
|
|
37
|
+
byFrom.set(map.from, map);
|
|
38
|
+
for (const map of run)
|
|
39
|
+
byFrom.set(map.from, map);
|
|
40
|
+
return [...byFrom.values()];
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* The maps one import run uses: the saved `importPathMaps` with the run's
|
|
44
|
+
* maps on top, run targets resolved against `cwd`.
|
|
45
|
+
*/
|
|
46
|
+
export async function runPathMaps(runMaps, cwd, saved) {
|
|
47
|
+
const savedMaps = saved ?? (await import("../config.js")).CONFIG.importPathMaps ?? [];
|
|
48
|
+
return mergeImportPathMaps(savedMaps, runMaps.map((map) => ({ from: map.from, to: resolve(cwd, map.to) })));
|
|
49
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/** One `(time, done)` reading of a running import. */
|
|
2
|
+
export interface ProgressSample {
|
|
3
|
+
at: number;
|
|
4
|
+
done: number;
|
|
5
|
+
}
|
|
6
|
+
export declare const MAX_PROGRESS_SAMPLES = 20;
|
|
7
|
+
/** Keep the most recent samples; a sample with no new units only moves the clock. */
|
|
8
|
+
export declare function addProgressSample(samples: readonly ProgressSample[], sample: ProgressSample): ProgressSample[];
|
|
9
|
+
export interface ProgressEstimate {
|
|
10
|
+
percent: number;
|
|
11
|
+
/** Null until the recent window has 5 samples spanning a minute and a non-zero rate. */
|
|
12
|
+
minutesLeft: number | null;
|
|
13
|
+
/** Units a minute over the recent window, or null while unknown. */
|
|
14
|
+
ratePerMinute: number | null;
|
|
15
|
+
}
|
|
16
|
+
/** Percentage done and minutes left from the recent sample window, not the whole run. */
|
|
17
|
+
export declare function estimateProgress(total: number, done: number, samples: readonly ProgressSample[]): ProgressEstimate;
|
|
18
|
+
/**
|
|
19
|
+
* Turn the importer's progress callback into finished work units. The
|
|
20
|
+
* importer reports a unit as it starts, and `alreadyHandled` (units found in
|
|
21
|
+
* the ledger, which need no model call) does not yet include that unit, so
|
|
22
|
+
* finished work is `processed - 1 - alreadyHandled`.
|
|
23
|
+
*/
|
|
24
|
+
export declare function workProgress(processed: number, total: number, alreadyHandled?: number, expectedTotal?: number): {
|
|
25
|
+
done: number;
|
|
26
|
+
total: number;
|
|
27
|
+
};
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
export const MAX_PROGRESS_SAMPLES = 20;
|
|
2
|
+
const MIN_SAMPLES = 5;
|
|
3
|
+
const MIN_WINDOW_MS = 60_000;
|
|
4
|
+
/** Keep the most recent samples; a sample with no new units only moves the clock. */
|
|
5
|
+
export function addProgressSample(samples, sample) {
|
|
6
|
+
return [...samples, sample].slice(-MAX_PROGRESS_SAMPLES);
|
|
7
|
+
}
|
|
8
|
+
/** Percentage done and minutes left from the recent sample window, not the whole run. */
|
|
9
|
+
export function estimateProgress(total, done, samples) {
|
|
10
|
+
const percent = total > 0 ? Math.min(100, Math.floor((done / total) * 100)) : 0;
|
|
11
|
+
if (samples.length < MIN_SAMPLES)
|
|
12
|
+
return { percent, minutesLeft: null, ratePerMinute: null };
|
|
13
|
+
const first = samples[0];
|
|
14
|
+
const last = samples[samples.length - 1];
|
|
15
|
+
const span = last.at - first.at;
|
|
16
|
+
if (span < MIN_WINDOW_MS || last.done <= first.done) {
|
|
17
|
+
return { percent, minutesLeft: null, ratePerMinute: null };
|
|
18
|
+
}
|
|
19
|
+
const ratePerMinute = (last.done - first.done) / (span / 60_000);
|
|
20
|
+
return {
|
|
21
|
+
percent,
|
|
22
|
+
minutesLeft: Math.max(0, Math.round((total - done) / ratePerMinute)),
|
|
23
|
+
ratePerMinute,
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Turn the importer's progress callback into finished work units. The
|
|
28
|
+
* importer reports a unit as it starts, and `alreadyHandled` (units found in
|
|
29
|
+
* the ledger, which need no model call) does not yet include that unit, so
|
|
30
|
+
* finished work is `processed - 1 - alreadyHandled`.
|
|
31
|
+
*/
|
|
32
|
+
export function workProgress(processed, total, alreadyHandled = 0, expectedTotal) {
|
|
33
|
+
const workTotal = Math.max(0, expectedTotal ?? total - alreadyHandled);
|
|
34
|
+
return {
|
|
35
|
+
done: Math.min(workTotal, Math.max(0, processed - 1 - alreadyHandled)),
|
|
36
|
+
total: workTotal,
|
|
37
|
+
};
|
|
38
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import type { BackfillHost } from "./backfill-model.js";
|
|
2
|
+
import { type ProgressEstimate } from "./import-progress.js";
|
|
3
|
+
export type ImportSurface = "auto" | "web" | "cli" | "slash";
|
|
4
|
+
export type ImportRunState = "running" | "paused" | "stopped" | "done" | "failed";
|
|
5
|
+
export interface ImportRunCounts {
|
|
6
|
+
total: number;
|
|
7
|
+
done: number;
|
|
8
|
+
imported: number;
|
|
9
|
+
skipped: number;
|
|
10
|
+
failed: number;
|
|
11
|
+
}
|
|
12
|
+
export interface ImportRun extends ImportRunCounts, ProgressEstimate {
|
|
13
|
+
host: BackfillHost;
|
|
14
|
+
surface: ImportSurface | null;
|
|
15
|
+
state: ImportRunState | null;
|
|
16
|
+
pid: number | null;
|
|
17
|
+
startedAt: number | null;
|
|
18
|
+
updatedAt: number | null;
|
|
19
|
+
error: string | null;
|
|
20
|
+
/** The user paused this host's backfill; automatic runs wait for Resume. */
|
|
21
|
+
paused: boolean;
|
|
22
|
+
}
|
|
23
|
+
/** Read a host's run without creating the store; a running row whose process is gone reads as stopped. */
|
|
24
|
+
export declare function readImportRun(host: BackfillHost): Promise<ImportRun | null>;
|
|
25
|
+
export declare function isBackfillPaused(host: BackfillHost): Promise<boolean>;
|
|
26
|
+
export declare function setBackfillPaused(host: BackfillHost, paused: boolean): Promise<void>;
|
|
27
|
+
export interface ImportRunRecorder {
|
|
28
|
+
/** Feed from `onProgress`; writes at most once per interval and checks the paused flag. */
|
|
29
|
+
progress(done: number, total: number): void;
|
|
30
|
+
/** True after a progress write saw the paused flag. */
|
|
31
|
+
readonly pauseRequested: boolean;
|
|
32
|
+
finish(state: Exclude<ImportRunState, "running">, counts: Partial<ImportRunCounts>, error?: unknown): Promise<void>;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Start a host's run record. Only numbers, the surface, and the PID are
|
|
36
|
+
* stored; `promptPreview` and all other conversation content never are.
|
|
37
|
+
*/
|
|
38
|
+
export declare function startImportRun(host: BackfillHost, surface: ImportSurface, options?: {
|
|
39
|
+
now?: () => number;
|
|
40
|
+
intervalMs?: number;
|
|
41
|
+
sampleIntervalMs?: number;
|
|
42
|
+
onPause?: () => void;
|
|
43
|
+
}): Promise<ImportRunRecorder>;
|