om-memory-system 3.3.0-next.17 → 3.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -5
- package/dist/adapters/opencode/backfill-models.d.ts +25 -0
- package/dist/adapters/opencode/backfill-models.js +39 -0
- package/dist/adapters/opencode/backfill-startup.d.ts +7 -0
- package/dist/adapters/opencode/backfill-startup.js +12 -0
- package/dist/adapters/opencode/import-command.js +3 -0
- package/dist/adapters/pi/backfill-models.d.ts +9 -0
- package/dist/adapters/pi/backfill-models.js +30 -0
- package/dist/adapters/pi/extension.js +50 -0
- package/dist/adapters/pi/import-command.js +3 -0
- package/dist/cli/index.js +13 -1
- package/dist/cli/web-command.d.ts +3 -0
- package/dist/cli/web-command.js +72 -0
- package/dist/config.d.ts +13 -1
- package/dist/config.js +28 -0
- package/dist/importer/auto-backfill.d.ts +20 -0
- package/dist/importer/auto-backfill.js +165 -0
- package/dist/importer/backfill-lock.d.ts +3 -0
- package/dist/importer/backfill-lock.js +44 -0
- package/dist/importer/backfill-model.d.ts +10 -0
- package/dist/importer/backfill-model.js +13 -0
- package/dist/importer/importer.js +14 -0
- package/dist/importer/manual-import-guard.d.ts +4 -0
- package/dist/importer/manual-import-guard.js +9 -0
- package/dist/importer/settings-health.d.ts +1 -2
- package/dist/importer/settings-health.js +2 -11
- package/dist/index.js +30 -0
- package/dist/services/backfill-state.d.ts +25 -0
- package/dist/services/backfill-state.js +68 -0
- package/dist/services/capture-diagnostics.d.ts +9 -0
- package/dist/services/capture-diagnostics.js +73 -6
- package/dist/services/global-config-writer.js +5 -1
- package/dist/services/live-captured-entries.d.ts +3 -0
- package/dist/services/live-captured-entries.js +34 -0
- package/dist/services/safe-health-error.d.ts +2 -0
- package/dist/services/safe-health-error.js +11 -0
- package/dist/services/settings-snapshot.js +14 -2
- package/dist/services/web-autostart.d.ts +29 -0
- package/dist/services/web-autostart.js +208 -0
- package/dist/services/web-server.js +11 -0
- package/dist/web/assets/index-B-_gMSn9.js +124 -0
- package/dist/web/index.html +1 -1
- package/package.json +1 -1
- package/dist/web/assets/index-BMkPht7g.js +0 -124
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
import { CONFIG, refreshConfigIfChanged } from "../config.js";
|
|
2
|
+
import { log } from "../services/logger.js";
|
|
3
|
+
import { tryAcquireBackfillLock } from "./backfill-lock.js";
|
|
4
|
+
import { getBackfillCutoff, updateBackfillStatus, } from "../services/backfill-state.js";
|
|
5
|
+
import { runHistoryImport, } from "./run-import.js";
|
|
6
|
+
import { isManualImportRunning } from "./manual-import-guard.js";
|
|
7
|
+
const zero = () => ({ imported: 0, skipped: 0, failed: 0, pending: 0, unresolved: 0 });
|
|
8
|
+
function counts(report) {
|
|
9
|
+
return {
|
|
10
|
+
imported: report.unitsImported,
|
|
11
|
+
skipped: report.unitsSkipped,
|
|
12
|
+
failed: report.unitsFailed,
|
|
13
|
+
pending: report.unitsWouldImport,
|
|
14
|
+
unresolved: (report.unresolvedProjects?.length ?? 0) + report.unresolvableSessions.length,
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
const delay = (signal) => new Promise((resolve) => {
|
|
18
|
+
if (signal?.aborted)
|
|
19
|
+
return resolve();
|
|
20
|
+
const done = () => {
|
|
21
|
+
clearTimeout(timer);
|
|
22
|
+
signal?.removeEventListener("abort", done);
|
|
23
|
+
resolve();
|
|
24
|
+
};
|
|
25
|
+
const timer = setTimeout(done, 30_000);
|
|
26
|
+
signal?.addEventListener("abort", done, { once: true });
|
|
27
|
+
});
|
|
28
|
+
/** Schedule one host's history import after startup; callers do not await it. */
|
|
29
|
+
export async function scheduleAutoBackfill(options) {
|
|
30
|
+
await (options.wait ?? (() => delay(options.signal)))();
|
|
31
|
+
if (options.signal?.aborted)
|
|
32
|
+
return;
|
|
33
|
+
if (!(options.enabled ??
|
|
34
|
+
(() => CONFIG.autoBackfill && process.env.OMMS_DISABLE_AUTO_BACKFILL !== "1"))())
|
|
35
|
+
return;
|
|
36
|
+
if (isManualImportRunning(options.host))
|
|
37
|
+
return;
|
|
38
|
+
const release = await tryAcquireBackfillLock(options.host);
|
|
39
|
+
if (!release)
|
|
40
|
+
return;
|
|
41
|
+
const clock = options.now ?? Date.now;
|
|
42
|
+
const run = options.run ?? runHistoryImport;
|
|
43
|
+
const controller = new AbortController();
|
|
44
|
+
const onAbort = () => controller.abort();
|
|
45
|
+
options.signal?.addEventListener("abort", onAbort, { once: true });
|
|
46
|
+
if (options.signal?.aborted)
|
|
47
|
+
controller.abort();
|
|
48
|
+
let state = "failed";
|
|
49
|
+
let model = null;
|
|
50
|
+
let current = zero();
|
|
51
|
+
let lastError;
|
|
52
|
+
try {
|
|
53
|
+
if (controller.signal.aborted) {
|
|
54
|
+
state = "stopped";
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
57
|
+
const cutoff = await getBackfillCutoff(options.host, clock());
|
|
58
|
+
const args = {
|
|
59
|
+
help: false,
|
|
60
|
+
dryRun: true,
|
|
61
|
+
force: false,
|
|
62
|
+
skipMemories: false,
|
|
63
|
+
skipProfile: false,
|
|
64
|
+
scope: "all-projects",
|
|
65
|
+
pathMaps: [],
|
|
66
|
+
until: cutoff,
|
|
67
|
+
errors: [],
|
|
68
|
+
};
|
|
69
|
+
const dry = await run(options.host, args, {
|
|
70
|
+
cwd: options.cwd,
|
|
71
|
+
models: {},
|
|
72
|
+
signal: controller.signal,
|
|
73
|
+
});
|
|
74
|
+
current = counts(dry);
|
|
75
|
+
if (controller.signal.aborted) {
|
|
76
|
+
state = "stopped";
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
79
|
+
if (!current.pending) {
|
|
80
|
+
state = "done";
|
|
81
|
+
return;
|
|
82
|
+
}
|
|
83
|
+
const resolved = await options.resolveModels();
|
|
84
|
+
if (controller.signal.aborted) {
|
|
85
|
+
state = "stopped";
|
|
86
|
+
return;
|
|
87
|
+
}
|
|
88
|
+
model = resolved.model;
|
|
89
|
+
options.notify(`${options.host} automatic import: ${current.pending} exchanges with ${model}. Set autoBackfill to false to stop it.`);
|
|
90
|
+
state = "running";
|
|
91
|
+
await updateBackfillStatus(options.host, { state, model, counts: current });
|
|
92
|
+
if (controller.signal.aborted) {
|
|
93
|
+
state = "stopped";
|
|
94
|
+
return;
|
|
95
|
+
}
|
|
96
|
+
let failures = 0;
|
|
97
|
+
const capture = resolved.models.capture;
|
|
98
|
+
const models = capture
|
|
99
|
+
? {
|
|
100
|
+
...resolved.models,
|
|
101
|
+
capture: {
|
|
102
|
+
summarize: async (request) => {
|
|
103
|
+
try {
|
|
104
|
+
const result = await capture.summarize(request);
|
|
105
|
+
failures = 0;
|
|
106
|
+
return result;
|
|
107
|
+
}
|
|
108
|
+
catch (error) {
|
|
109
|
+
lastError = error;
|
|
110
|
+
if (++failures >= 5)
|
|
111
|
+
controller.abort();
|
|
112
|
+
throw error;
|
|
113
|
+
}
|
|
114
|
+
},
|
|
115
|
+
},
|
|
116
|
+
}
|
|
117
|
+
: resolved.models;
|
|
118
|
+
let checkedAt = clock();
|
|
119
|
+
let processed = 0;
|
|
120
|
+
let pendingWrite = Promise.resolve();
|
|
121
|
+
const real = await run(options.host, { ...args, dryRun: false }, {
|
|
122
|
+
cwd: options.cwd,
|
|
123
|
+
models,
|
|
124
|
+
signal: controller.signal,
|
|
125
|
+
onProgress: (done, total) => {
|
|
126
|
+
processed = done;
|
|
127
|
+
const now = clock();
|
|
128
|
+
if (now - checkedAt < 5_000)
|
|
129
|
+
return;
|
|
130
|
+
checkedAt = now;
|
|
131
|
+
refreshConfigIfChanged(options.cwd);
|
|
132
|
+
if (!CONFIG.autoBackfill)
|
|
133
|
+
controller.abort();
|
|
134
|
+
current = { ...current, pending: Math.max(0, total - done) };
|
|
135
|
+
pendingWrite = pendingWrite
|
|
136
|
+
.then(() => updateBackfillStatus(options.host, { state: "running", model, counts: current }))
|
|
137
|
+
.catch((error) => {
|
|
138
|
+
log("Backfill status write failed", {
|
|
139
|
+
host: options.host,
|
|
140
|
+
error: error instanceof Error ? error.message : String(error),
|
|
141
|
+
});
|
|
142
|
+
});
|
|
143
|
+
},
|
|
144
|
+
});
|
|
145
|
+
await pendingWrite;
|
|
146
|
+
current = { ...counts(real), pending: Math.max(0, dry.unitsWouldImport - processed) };
|
|
147
|
+
state = controller.signal.aborted ? "stopped" : real.unitsFailed ? "failed" : "done";
|
|
148
|
+
if (!options.signal?.aborted) {
|
|
149
|
+
options.notify(`${options.host} automatic import ${state}: ${current.imported} imported, ${current.failed} failed. autoBackfill controls the next run.`);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
catch (error) {
|
|
153
|
+
lastError = error;
|
|
154
|
+
state = controller.signal.aborted ? "stopped" : "failed";
|
|
155
|
+
}
|
|
156
|
+
finally {
|
|
157
|
+
try {
|
|
158
|
+
await updateBackfillStatus(options.host, { state, model, counts: current, error: lastError });
|
|
159
|
+
}
|
|
160
|
+
finally {
|
|
161
|
+
options.signal?.removeEventListener("abort", onAbort);
|
|
162
|
+
await release();
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { CONFIG } from "../config.js";
|
|
4
|
+
import { tursoConnectionManager } from "../services/turso/connection-manager.js";
|
|
5
|
+
function live(pid) {
|
|
6
|
+
if (!Number.isSafeInteger(pid) || pid <= 0)
|
|
7
|
+
return false;
|
|
8
|
+
try {
|
|
9
|
+
process.kill(pid, 0);
|
|
10
|
+
return true;
|
|
11
|
+
}
|
|
12
|
+
catch (error) {
|
|
13
|
+
return error.code === "EPERM";
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
/** Claim one host's import without holding a database transaction during the run. */
|
|
17
|
+
export async function tryAcquireBackfillLock(host, storagePath = CONFIG.storagePath) {
|
|
18
|
+
const db = await tursoConnectionManager.getConnection(join(storagePath, "import-ledger.db"));
|
|
19
|
+
await db.run(`CREATE TABLE IF NOT EXISTS backfill_locks (
|
|
20
|
+
host TEXT PRIMARY KEY, pid INTEGER NOT NULL, token TEXT NOT NULL
|
|
21
|
+
)`);
|
|
22
|
+
const state = { pid: process.pid, token: randomUUID() };
|
|
23
|
+
for (let attempt = 0; attempt < 3; attempt++) {
|
|
24
|
+
const inserted = await db.run("INSERT OR IGNORE INTO backfill_locks (host, pid, token) VALUES (?, ?, ?)", [host, state.pid, state.token]);
|
|
25
|
+
if (!inserted) {
|
|
26
|
+
const holder = await db.get("SELECT pid, token FROM backfill_locks WHERE host = ?", [host]);
|
|
27
|
+
if (!holder)
|
|
28
|
+
continue;
|
|
29
|
+
if (live(holder.pid))
|
|
30
|
+
return null;
|
|
31
|
+
const replaced = await db.run("UPDATE backfill_locks SET pid = ?, token = ? WHERE host = ? AND pid = ? AND token = ?", [state.pid, state.token, host, holder.pid, holder.token]);
|
|
32
|
+
if (!replaced)
|
|
33
|
+
continue;
|
|
34
|
+
}
|
|
35
|
+
let released = false;
|
|
36
|
+
return async () => {
|
|
37
|
+
if (released)
|
|
38
|
+
return;
|
|
39
|
+
released = true;
|
|
40
|
+
await db.run("DELETE FROM backfill_locks WHERE host = ? AND token = ?", [host, state.token]);
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
return null;
|
|
44
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export type BackfillHost = "pi" | "opencode";
|
|
2
|
+
export type BackfillModel = "inherit" | {
|
|
3
|
+
provider: string;
|
|
4
|
+
model: string;
|
|
5
|
+
};
|
|
6
|
+
/** Parse the global backfill model without importing the runtime config. */
|
|
7
|
+
export declare function parseBackfillModel(config: {
|
|
8
|
+
piBackfillModel?: unknown;
|
|
9
|
+
opencodeBackfillModel?: unknown;
|
|
10
|
+
}, host: BackfillHost): BackfillModel;
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/** Parse the global backfill model without importing the runtime config. */
|
|
2
|
+
export function parseBackfillModel(config, host) {
|
|
3
|
+
const value = host === "pi" ? config.piBackfillModel : config.opencodeBackfillModel;
|
|
4
|
+
if (value === undefined || value === "inherit")
|
|
5
|
+
return "inherit";
|
|
6
|
+
if (typeof value !== "string")
|
|
7
|
+
throw new Error(`Invalid ${host}BackfillModel config`);
|
|
8
|
+
const slash = value.indexOf("/");
|
|
9
|
+
if (slash <= 0 || slash === value.length - 1) {
|
|
10
|
+
throw new Error(`Invalid ${host}BackfillModel config`);
|
|
11
|
+
}
|
|
12
|
+
return { provider: value.slice(0, slash), model: value.slice(slash + 1) };
|
|
13
|
+
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { existsSync, realpathSync } from "node:fs";
|
|
2
2
|
import { captureConversation } from "../core/capture.js";
|
|
3
3
|
import { memoryClient } from "../services/client.js";
|
|
4
|
+
import { findLiveCapturedEntryIds } from "../services/live-captured-entries.js";
|
|
4
5
|
import { extractScopeFromContainerTag } from "../services/memory-scope.js";
|
|
5
6
|
import { getTags } from "../services/tags.js";
|
|
6
7
|
import { tursoConnectionManager } from "../services/turso/connection-manager.js";
|
|
@@ -171,6 +172,19 @@ export async function importHistorySource(source, host, deps, filters, report) {
|
|
|
171
172
|
report.units.push(unit);
|
|
172
173
|
continue;
|
|
173
174
|
}
|
|
175
|
+
const liveEntryIds = await findLiveCapturedEntryIds(hash, host, sessionId);
|
|
176
|
+
if (liveEntryIds.has(window.userEntryId)) {
|
|
177
|
+
if (!dryRun) {
|
|
178
|
+
await ledger.begin({ key, sessionId, sourceFile, projectHash: hash });
|
|
179
|
+
await ledger.skip(key, "live-captured");
|
|
180
|
+
}
|
|
181
|
+
unit.status = "skipped";
|
|
182
|
+
unit.reason = "live-captured";
|
|
183
|
+
report.unitsSkipped++;
|
|
184
|
+
report.skipReasons["live-captured"] = (report.skipReasons["live-captured"] ?? 0) + 1;
|
|
185
|
+
report.units.push(unit);
|
|
186
|
+
continue;
|
|
187
|
+
}
|
|
174
188
|
// Crash reconciliation: the memory landed but the ledger never committed.
|
|
175
189
|
const storedMemoryId = await lookupExistingMemory(hash, key, { storageAccessible });
|
|
176
190
|
if (storedMemoryId) {
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { BackfillHost } from "./backfill-model.js";
|
|
2
|
+
/** Keep an automatic run from starting during a manual session import. */
|
|
3
|
+
export declare function beginManualImport(host: BackfillHost): () => void;
|
|
4
|
+
export declare function isManualImportRunning(host: BackfillHost): boolean;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
const running = new Set();
|
|
2
|
+
/** Keep an automatic run from starting during a manual session import. */
|
|
3
|
+
export function beginManualImport(host) {
|
|
4
|
+
running.add(host);
|
|
5
|
+
return () => running.delete(host);
|
|
6
|
+
}
|
|
7
|
+
export function isManualImportRunning(host) {
|
|
8
|
+
return running.has(host);
|
|
9
|
+
}
|
|
@@ -1,10 +1,9 @@
|
|
|
1
|
+
export { safeHealthError } from "../services/safe-health-error.js";
|
|
1
2
|
export type HealthRow = {
|
|
2
3
|
check: string;
|
|
3
4
|
status: "pass" | "warn" | "fail";
|
|
4
5
|
reason: string;
|
|
5
6
|
};
|
|
6
|
-
/** Remove configured credentials before returning a model error to the browser. */
|
|
7
|
-
export declare function safeHealthError(error: unknown, secrets: Array<string | undefined>): string;
|
|
8
7
|
export declare function captureFailureHealth(byModel: Array<{
|
|
9
8
|
total: number;
|
|
10
9
|
failed: number;
|
|
@@ -1,18 +1,9 @@
|
|
|
1
1
|
import { CONFIG } from "../config.js";
|
|
2
2
|
import { getAutoCaptureProviderStatus, resolvePiLiveModel, } from "../services/ai/live-model-choice.js";
|
|
3
3
|
import { isLoopbackHost } from "../services/web-api-auth.js";
|
|
4
|
+
import { safeHealthError } from "../services/safe-health-error.js";
|
|
5
|
+
export { safeHealthError } from "../services/safe-health-error.js";
|
|
4
6
|
const PROBE = "Reply with a short acknowledgement.";
|
|
5
|
-
/** Remove configured credentials before returning a model error to the browser. */
|
|
6
|
-
export function safeHealthError(error, secrets) {
|
|
7
|
-
let message = error instanceof Error ? error.message : String(error);
|
|
8
|
-
for (const secret of secrets) {
|
|
9
|
-
if (secret && secret.length > 2)
|
|
10
|
-
message = message.replaceAll(secret, "[redacted]");
|
|
11
|
-
}
|
|
12
|
-
return message
|
|
13
|
-
.replace(/(Bearer\s+)[^\s]+/gi, "$1[redacted]")
|
|
14
|
-
.replace(/(api[_-]?key[=:]\s*)[^\s&]+/gi, "$1[redacted]");
|
|
15
|
-
}
|
|
16
7
|
export function captureFailureHealth(byModel, byReason) {
|
|
17
8
|
const total = byModel.reduce((count, row) => count + row.total, 0);
|
|
18
9
|
const failed = byModel.reduce((count, row) => count + row.failed, 0);
|
package/dist/index.js
CHANGED
|
@@ -183,6 +183,13 @@ export const OmmsPlugin = async (ctx) => {
|
|
|
183
183
|
initConfigWithLegacyMigration(directory);
|
|
184
184
|
// Runs even with tracing off, so turning it off does not leave old traces behind.
|
|
185
185
|
pruneTraces(CONFIG);
|
|
186
|
+
if (CONFIG.webServerAutoStart !== undefined && process.env.OMMS_DISABLE_WEB_AUTOSTART !== "1") {
|
|
187
|
+
void import("./services/web-autostart.js")
|
|
188
|
+
.then(({ reconcileWebAutostart }) => reconcileWebAutostart(CONFIG))
|
|
189
|
+
.catch((error) => log("OpenCode login item reconciliation failed", {
|
|
190
|
+
code: error instanceof Error ? error.name : "unknown",
|
|
191
|
+
}));
|
|
192
|
+
}
|
|
186
193
|
logAutoCaptureProviderStatus();
|
|
187
194
|
const tags = getTags(directory);
|
|
188
195
|
const autoCaptureHost = createOpenCodeAutoCaptureHost(ctx);
|
|
@@ -203,9 +210,11 @@ export const OmmsPlugin = async (ctx) => {
|
|
|
203
210
|
}
|
|
204
211
|
await configureOpencodeHostTransport(ctx);
|
|
205
212
|
(async () => {
|
|
213
|
+
let connected = [];
|
|
206
214
|
try {
|
|
207
215
|
const providerResult = await ctx.client.provider.list();
|
|
208
216
|
if (providerResult.data?.connected) {
|
|
217
|
+
connected = providerResult.data.connected;
|
|
209
218
|
const { setConnectedProviders } = await loadOpencodeProvider();
|
|
210
219
|
setConnectedProviders(providerResult.data.connected);
|
|
211
220
|
log("opencode providers connected", {
|
|
@@ -222,6 +231,27 @@ export const OmmsPlugin = async (ctx) => {
|
|
|
222
231
|
catch (error) {
|
|
223
232
|
log("Failed to initialize opencode provider state", { error: String(error) });
|
|
224
233
|
}
|
|
234
|
+
if (CONFIG.autoBackfill && process.env.OMMS_DISABLE_AUTO_BACKFILL !== "1" && isConfigured()) {
|
|
235
|
+
void import("./adapters/opencode/backfill-startup.js")
|
|
236
|
+
.then(({ startOpencodeBackfill }) => startOpencodeBackfill({
|
|
237
|
+
connected,
|
|
238
|
+
directory,
|
|
239
|
+
configModel: async () => {
|
|
240
|
+
const response = await ctx.client.config.get();
|
|
241
|
+
return unwrapSdkData(response)?.model ?? null;
|
|
242
|
+
},
|
|
243
|
+
notify: (message) => {
|
|
244
|
+
void ctx.client.tui
|
|
245
|
+
?.showToast({
|
|
246
|
+
body: { title: "Automatic import", message, variant: "info", duration: 5000 },
|
|
247
|
+
})
|
|
248
|
+
.catch(() => { });
|
|
249
|
+
},
|
|
250
|
+
}))
|
|
251
|
+
.catch((error) => log("OpenCode backfill failed", {
|
|
252
|
+
error: error instanceof Error ? error.message : String(error),
|
|
253
|
+
}));
|
|
254
|
+
}
|
|
225
255
|
})();
|
|
226
256
|
let tursoReadyForWeb = !isConfigured();
|
|
227
257
|
if (CONFIG.webServerEnabled && isConfigured()) {
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
type BackfillHost = "pi" | "opencode";
|
|
2
|
+
export type BackfillStatus = {
|
|
3
|
+
host: BackfillHost;
|
|
4
|
+
cutoff: number;
|
|
5
|
+
state: "not-started" | "running" | "stopped" | "done" | "failed";
|
|
6
|
+
model: string | null;
|
|
7
|
+
counts: {
|
|
8
|
+
imported: number;
|
|
9
|
+
skipped: number;
|
|
10
|
+
failed: number;
|
|
11
|
+
pending: number;
|
|
12
|
+
unresolved: number;
|
|
13
|
+
};
|
|
14
|
+
error: string | null;
|
|
15
|
+
updatedAt: number;
|
|
16
|
+
};
|
|
17
|
+
/** Record a fixed per-host cutoff before the first import. */
|
|
18
|
+
export declare function getBackfillCutoff(host: BackfillHost, now?: number): Promise<number>;
|
|
19
|
+
/** Read progress without creating a store or a ledger table. */
|
|
20
|
+
export declare function readBackfillStatus(host: BackfillHost): Promise<BackfillStatus | null>;
|
|
21
|
+
/** Save only numeric counts and a redacted error, never report units or prompts. */
|
|
22
|
+
export declare function updateBackfillStatus(host: BackfillHost, update: Pick<BackfillStatus, "state" | "model" | "counts"> & {
|
|
23
|
+
error?: unknown;
|
|
24
|
+
}): Promise<void>;
|
|
25
|
+
export {};
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import { existsSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { CONFIG } from "../config.js";
|
|
4
|
+
import { tursoConnectionManager } from "./turso/connection-manager.js";
|
|
5
|
+
import { safeHealthError } from "./safe-health-error.js";
|
|
6
|
+
const importLedgerDbPath = () => join(CONFIG.storagePath, "import-ledger.db");
|
|
7
|
+
const emptyCounts = () => ({ imported: 0, skipped: 0, failed: 0, pending: 0, unresolved: 0 });
|
|
8
|
+
async function table() {
|
|
9
|
+
const db = await tursoConnectionManager.getConnection(importLedgerDbPath());
|
|
10
|
+
await db.run(`CREATE TABLE IF NOT EXISTS backfill_state (
|
|
11
|
+
host TEXT PRIMARY KEY, cutoff INTEGER NOT NULL, state TEXT,
|
|
12
|
+
model TEXT, counts TEXT, error TEXT, updated_at INTEGER
|
|
13
|
+
)`);
|
|
14
|
+
return db;
|
|
15
|
+
}
|
|
16
|
+
/** Record a fixed per-host cutoff before the first import. */
|
|
17
|
+
export async function getBackfillCutoff(host, now = Date.now()) {
|
|
18
|
+
const db = await table();
|
|
19
|
+
await db.run(`INSERT OR IGNORE INTO backfill_state (host, cutoff, state, counts, updated_at)
|
|
20
|
+
VALUES (?, ?, 'not-started', ?, ?)`, [host, now, JSON.stringify(emptyCounts()), now]);
|
|
21
|
+
const row = await db.get("SELECT cutoff FROM backfill_state WHERE host = ?", [
|
|
22
|
+
host,
|
|
23
|
+
]);
|
|
24
|
+
return Number(row.cutoff);
|
|
25
|
+
}
|
|
26
|
+
/** Read progress without creating a store or a ledger table. */
|
|
27
|
+
export async function readBackfillStatus(host) {
|
|
28
|
+
if (!existsSync(importLedgerDbPath()))
|
|
29
|
+
return null;
|
|
30
|
+
const db = await tursoConnectionManager.getConnection(importLedgerDbPath());
|
|
31
|
+
const exists = await db.get("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'backfill_state'");
|
|
32
|
+
if (!exists)
|
|
33
|
+
return null;
|
|
34
|
+
const row = await db.get("SELECT * FROM backfill_state WHERE host = ?", [
|
|
35
|
+
host,
|
|
36
|
+
]);
|
|
37
|
+
if (!row)
|
|
38
|
+
return null;
|
|
39
|
+
return {
|
|
40
|
+
host,
|
|
41
|
+
cutoff: Number(row.cutoff),
|
|
42
|
+
state: row.state,
|
|
43
|
+
model: row.model === null ? null : String(row.model),
|
|
44
|
+
counts: row.counts
|
|
45
|
+
? JSON.parse(String(row.counts))
|
|
46
|
+
: emptyCounts(),
|
|
47
|
+
error: row.error === null ? null : String(row.error),
|
|
48
|
+
updatedAt: Number(row.updated_at),
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
/** Save only numeric counts and a redacted error, never report units or prompts. */
|
|
52
|
+
export async function updateBackfillStatus(host, update) {
|
|
53
|
+
const counts = Object.fromEntries(Object.keys(emptyCounts()).map((key) => [
|
|
54
|
+
key,
|
|
55
|
+
Number.isSafeInteger(update.counts[key]) && update.counts[key] >= 0 ? update.counts[key] : 0,
|
|
56
|
+
]));
|
|
57
|
+
const db = await table();
|
|
58
|
+
await db.run(`UPDATE backfill_state SET state = ?, model = ?, counts = ?, error = ?, updated_at = ? WHERE host = ?`, [
|
|
59
|
+
update.state,
|
|
60
|
+
update.model,
|
|
61
|
+
JSON.stringify(counts),
|
|
62
|
+
update.error == null
|
|
63
|
+
? null
|
|
64
|
+
: safeHealthError(update.error, [CONFIG.memoryApiKey]).slice(0, 500),
|
|
65
|
+
Date.now(),
|
|
66
|
+
host,
|
|
67
|
+
]);
|
|
68
|
+
}
|
|
@@ -52,6 +52,14 @@ export declare function getTraceDirectory(): string;
|
|
|
52
52
|
* that do not match the trace name pattern are never touched.
|
|
53
53
|
*/
|
|
54
54
|
export declare function pruneTraces(config: CaptureDiagnosticsConfig, now?: Date): number;
|
|
55
|
+
type WindowsAclRunner = (command: string, args: string[], options: {
|
|
56
|
+
env: NodeJS.ProcessEnv;
|
|
57
|
+
timeout: number;
|
|
58
|
+
windowsHide: boolean;
|
|
59
|
+
stdio: "ignore";
|
|
60
|
+
}) => void;
|
|
61
|
+
/** Restrict a trace path to the current user on both Windows and POSIX. */
|
|
62
|
+
export declare function protectTracePath(path: string, mode: number, platform?: NodeJS.Platform, run?: WindowsAclRunner): void;
|
|
55
63
|
/**
|
|
56
64
|
* Append one trace entry to the day's JSON Lines file. A single append per
|
|
57
65
|
* entry keeps lines whole when Pi and OpenCode write at the same time.
|
|
@@ -64,3 +72,4 @@ export declare function writeTraceEntry(record: CaptureAttemptRecord, diagnostic
|
|
|
64
72
|
export declare function emitCaptureAttempt(record: CaptureAttemptRecord, diagnostics: CaptureAttemptDiagnostics, config: CaptureDiagnosticsConfig, now?: Date): void;
|
|
65
73
|
/** Test hook: forget which day the last trace was written. */
|
|
66
74
|
export declare function resetTraceStateForTests(): void;
|
|
75
|
+
export {};
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { execFileSync } from "node:child_process";
|
|
2
|
+
import { appendFileSync, chmodSync, existsSync, lstatSync, mkdirSync, readdirSync, unlinkSync, } from "fs";
|
|
2
3
|
import { join } from "path";
|
|
3
4
|
import { getLogDirPath } from "./log-path.js";
|
|
4
5
|
import { log } from "./logger.js";
|
|
@@ -112,6 +113,54 @@ export function pruneTraces(config, now = new Date()) {
|
|
|
112
113
|
return removed;
|
|
113
114
|
}
|
|
114
115
|
let lastTraceDate = null;
|
|
116
|
+
const WINDOWS_TRACE_ACL = `$ErrorActionPreference = 'Stop'
|
|
117
|
+
$path = $env:OMMS_TRACE_ACL_PATH
|
|
118
|
+
$isDirectory = [System.IO.Directory]::Exists($path)
|
|
119
|
+
$acl = if ($isDirectory) {
|
|
120
|
+
[System.IO.Directory]::GetAccessControl($path)
|
|
121
|
+
} else {
|
|
122
|
+
[System.IO.File]::GetAccessControl($path)
|
|
123
|
+
}
|
|
124
|
+
$currentUser = [System.Security.Principal.WindowsIdentity]::GetCurrent().User
|
|
125
|
+
$owner = $acl.GetOwner([System.Security.Principal.SecurityIdentifier])
|
|
126
|
+
if ($owner.Value -ne $currentUser.Value) {
|
|
127
|
+
throw 'Trace path has a different owner'
|
|
128
|
+
}
|
|
129
|
+
$acl.SetAccessRuleProtection($true, $false)
|
|
130
|
+
foreach ($rule in @($acl.Access)) {
|
|
131
|
+
[void]$acl.RemoveAccessRuleSpecific($rule)
|
|
132
|
+
}
|
|
133
|
+
$inheritance = if ($isDirectory) {
|
|
134
|
+
[System.Security.AccessControl.InheritanceFlags]'ContainerInherit, ObjectInherit'
|
|
135
|
+
} else {
|
|
136
|
+
[System.Security.AccessControl.InheritanceFlags]::None
|
|
137
|
+
}
|
|
138
|
+
$rule = [System.Security.AccessControl.FileSystemAccessRule]::new(
|
|
139
|
+
$currentUser,
|
|
140
|
+
[System.Security.AccessControl.FileSystemRights]::FullControl,
|
|
141
|
+
$inheritance,
|
|
142
|
+
[System.Security.AccessControl.PropagationFlags]::None,
|
|
143
|
+
[System.Security.AccessControl.AccessControlType]::Allow
|
|
144
|
+
)
|
|
145
|
+
$acl.AddAccessRule($rule)
|
|
146
|
+
if ($isDirectory) {
|
|
147
|
+
[System.IO.Directory]::SetAccessControl($path, $acl)
|
|
148
|
+
} else {
|
|
149
|
+
[System.IO.File]::SetAccessControl($path, $acl)
|
|
150
|
+
}`;
|
|
151
|
+
/** Restrict a trace path to the current user on both Windows and POSIX. */
|
|
152
|
+
export function protectTracePath(path, mode, platform = process.platform, run = execFileSync) {
|
|
153
|
+
if (platform !== "win32") {
|
|
154
|
+
chmodSync(path, mode);
|
|
155
|
+
return;
|
|
156
|
+
}
|
|
157
|
+
run("powershell.exe", ["-NoProfile", "-NonInteractive", "-Command", WINDOWS_TRACE_ACL], {
|
|
158
|
+
env: { ...process.env, OMMS_TRACE_ACL_PATH: path },
|
|
159
|
+
timeout: 10_000,
|
|
160
|
+
windowsHide: true,
|
|
161
|
+
stdio: "ignore",
|
|
162
|
+
});
|
|
163
|
+
}
|
|
115
164
|
/**
|
|
116
165
|
* Append one trace entry to the day's JSON Lines file. A single append per
|
|
117
166
|
* entry keeps lines whole when Pi and OpenCode write at the same time.
|
|
@@ -125,7 +174,18 @@ export function writeTraceEntry(record, diagnostics, config, now = new Date()) {
|
|
|
125
174
|
}
|
|
126
175
|
const dir = getTraceDirectory();
|
|
127
176
|
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
128
|
-
|
|
177
|
+
if (!lstatSync(dir).isDirectory())
|
|
178
|
+
throw new Error("Trace directory is not a real directory");
|
|
179
|
+
protectTracePath(dir, 0o700);
|
|
180
|
+
const dailyFile = `capture-${today}.jsonl`;
|
|
181
|
+
for (const name of readdirSync(dir)) {
|
|
182
|
+
if (!TRACE_FILE.test(name) || name === dailyFile)
|
|
183
|
+
continue;
|
|
184
|
+
const retained = join(dir, name);
|
|
185
|
+
if (!lstatSync(retained).isFile())
|
|
186
|
+
throw new Error("Trace path is not a regular file");
|
|
187
|
+
protectTracePath(retained, 0o600);
|
|
188
|
+
}
|
|
129
189
|
const redact = (value) => value === undefined ? null : redactTraceText(value, config);
|
|
130
190
|
const entry = {
|
|
131
191
|
schemaVersion: TRACE_SCHEMA_VERSION,
|
|
@@ -135,11 +195,18 @@ export function writeTraceEntry(record, diagnostics, config, now = new Date()) {
|
|
|
135
195
|
userPrompt: redact(diagnostics.userPrompt),
|
|
136
196
|
reply: redact(diagnostics.rawReply),
|
|
137
197
|
};
|
|
138
|
-
const file = join(dir,
|
|
198
|
+
const file = join(dir, dailyFile);
|
|
199
|
+
const existingFile = lstatSync(file, { throwIfNoEntry: false });
|
|
200
|
+
if (existingFile && !existingFile.isFile())
|
|
201
|
+
throw new Error("Trace path is not a regular file");
|
|
202
|
+
// Narrow an existing file before appending sensitive content.
|
|
203
|
+
const existingWindowsFile = process.platform === "win32" && Boolean(existingFile);
|
|
204
|
+
if (existingWindowsFile)
|
|
205
|
+
protectTracePath(file, 0o600);
|
|
139
206
|
appendFileSync(file, `${JSON.stringify(entry)}\n`, { mode: 0o600 });
|
|
140
|
-
//
|
|
141
|
-
|
|
142
|
-
|
|
207
|
+
// A new Windows file inherits the private directory ACL until this call.
|
|
208
|
+
if (!existingWindowsFile)
|
|
209
|
+
protectTracePath(file, 0o600);
|
|
143
210
|
}
|
|
144
211
|
catch (error) {
|
|
145
212
|
log("Capture trace write failed", { code: errorCode(error) });
|
|
@@ -11,6 +11,10 @@ const keys = new Set([
|
|
|
11
11
|
"captureTrace",
|
|
12
12
|
"captureTraceRetentionDays",
|
|
13
13
|
"captureAttemptRetentionDays",
|
|
14
|
+
"autoBackfill",
|
|
15
|
+
"opencodeBackfillModel",
|
|
16
|
+
"piBackfillModel",
|
|
17
|
+
"webServerAutoStart",
|
|
14
18
|
]);
|
|
15
19
|
export class ConfigConflictError extends Error {
|
|
16
20
|
status = 409;
|
|
@@ -46,7 +50,7 @@ export async function writeGlobalConfigKeys(edits, expectedRevision) {
|
|
|
46
50
|
for (const [key, value] of Object.entries(edits)) {
|
|
47
51
|
if (!keys.has(key))
|
|
48
52
|
throw new Error(`Setting ${key} cannot be edited here`);
|
|
49
|
-
if (key === "captureTrace"
|
|
53
|
+
if (key === "captureTrace" || key === "autoBackfill" || key === "webServerAutoStart"
|
|
50
54
|
? typeof value !== "boolean"
|
|
51
55
|
: key.endsWith("RetentionDays")
|
|
52
56
|
? !Number.isSafeInteger(value) || value < 1
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { existsSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { CONFIG } from "../config.js";
|
|
4
|
+
import { tursoConnectionManager } from "./turso/connection-manager.js";
|
|
5
|
+
import { tursoShardManager } from "./turso/shard-manager.js";
|
|
6
|
+
/** Read user and assistant IDs saved by live capture in one project session. */
|
|
7
|
+
export async function findLiveCapturedEntryIds(projectHash, host, sessionId) {
|
|
8
|
+
const entries = new Set();
|
|
9
|
+
if (!existsSync(join(CONFIG.storagePath, "metadata.db")))
|
|
10
|
+
return entries;
|
|
11
|
+
const shards = await tursoShardManager.getAllShards("project", projectHash);
|
|
12
|
+
for (const shard of shards) {
|
|
13
|
+
const db = await tursoConnectionManager.getConnection(shard.dbPath);
|
|
14
|
+
const rows = await db.all(`SELECT metadata FROM memories WHERE json_valid(metadata)
|
|
15
|
+
AND json_extract(metadata, '$.host') = ?
|
|
16
|
+
AND json_extract(metadata, '$.hostSessionId') = ?
|
|
17
|
+
AND json_extract(metadata, '$.sourceType') != 'history-import'`, [host, sessionId]);
|
|
18
|
+
for (const row of rows) {
|
|
19
|
+
const metadata = JSON.parse(row.metadata);
|
|
20
|
+
if (!metadata || typeof metadata !== "object")
|
|
21
|
+
continue;
|
|
22
|
+
const provenance = metadata;
|
|
23
|
+
if (typeof provenance.promptId === "string")
|
|
24
|
+
entries.add(provenance.promptId);
|
|
25
|
+
const ids = provenance.sourceEntryIds;
|
|
26
|
+
if (Array.isArray(ids)) {
|
|
27
|
+
for (const id of ids)
|
|
28
|
+
if (typeof id === "string")
|
|
29
|
+
entries.add(id);
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
return entries;
|
|
34
|
+
}
|