@openpond/evals 0.11.17 → 0.11.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/connected-evidence/contracts.js +14 -4
- package/dist/connected-evidence/imports.js +77 -17
- package/dist/connected-evidence/normalize.js +4 -3
- package/dist/connected-evidence/sources/claude-branches.js +52 -0
- package/dist/connected-evidence/sources/claude-code.js +7 -29
- package/dist/connected-evidence/sources/codex-authorization.js +40 -0
- package/dist/connected-evidence/sources/codex.js +7 -3
- package/dist/connected-evidence/sources/grok-build.js +137 -0
- package/dist/connected-evidence/sources/opencode.js +101 -0
- package/dist/connected-evidence/sources/pi.js +187 -0
- package/dist/graders.js +11 -1
- package/dist/learning/budgeted-reward-fixtures.js +1 -0
- package/dist/learning/contracts.js +10 -0
- package/dist/learning/index.js +1 -0
- package/dist/learning/model-judge.js +22 -5
- package/dist/learning/reward-calibration-closure.js +110 -0
- package/dist/learning/reward-calibration.js +20 -10
- package/dist/learning/reward-checks.js +1 -0
- package/dist/native-conversations/appimage-runtime.js +131 -0
- package/dist/native-conversations/collector-branches.js +79 -0
- package/dist/native-conversations/collector-contracts.js +9 -0
- package/dist/native-conversations/collector-destinations.js +27 -0
- package/dist/native-conversations/collector-errors.js +42 -0
- package/dist/native-conversations/collector-progress.js +114 -0
- package/dist/native-conversations/collector-store.js +247 -0
- package/dist/native-conversations/collector-sync-requests.js +52 -0
- package/dist/native-conversations/collector.js +322 -0
- package/dist/native-conversations/contracts.js +11 -0
- package/dist/native-conversations/database.js +149 -0
- package/dist/native-conversations/discovery.js +135 -0
- package/dist/native-conversations/history.js +298 -0
- package/dist/native-conversations/index.js +11 -0
- package/dist/native-conversations/location.js +29 -0
- package/dist/native-conversations/openclaw-cold-archive.js +91 -0
- package/dist/native-conversations/openclaw-database.js +130 -0
- package/dist/native-conversations/source-locations.js +112 -0
- package/dist/native-conversations/supervisor.js +208 -0
- package/dist/rewards.js +1 -1
- package/dist/types/connected-evidence/contracts.d.ts +15 -0
- package/dist/types/connected-evidence/contracts.d.ts.map +1 -1
- package/dist/types/connected-evidence/imports.d.ts +6 -0
- package/dist/types/connected-evidence/imports.d.ts.map +1 -1
- package/dist/types/connected-evidence/normalize.d.ts +4 -0
- package/dist/types/connected-evidence/normalize.d.ts.map +1 -1
- package/dist/types/connected-evidence/sources/claude-branches.d.ts +19 -0
- package/dist/types/connected-evidence/sources/claude-branches.d.ts.map +1 -0
- package/dist/types/connected-evidence/sources/claude-code.d.ts +5 -1
- package/dist/types/connected-evidence/sources/claude-code.d.ts.map +1 -1
- package/dist/types/connected-evidence/sources/codex-authorization.d.ts +18 -0
- package/dist/types/connected-evidence/sources/codex-authorization.d.ts.map +1 -0
- package/dist/types/connected-evidence/sources/codex.d.ts +5 -1
- package/dist/types/connected-evidence/sources/codex.d.ts.map +1 -1
- package/dist/types/connected-evidence/sources/grok-build.d.ts +79 -0
- package/dist/types/connected-evidence/sources/grok-build.d.ts.map +1 -0
- package/dist/types/connected-evidence/sources/history.d.ts +10 -2
- package/dist/types/connected-evidence/sources/history.d.ts.map +1 -1
- package/dist/types/connected-evidence/sources/opencode.d.ts +78 -0
- package/dist/types/connected-evidence/sources/opencode.d.ts.map +1 -0
- package/dist/types/connected-evidence/sources/pi.d.ts +79 -0
- package/dist/types/connected-evidence/sources/pi.d.ts.map +1 -0
- package/dist/types/graders.d.ts +81 -0
- package/dist/types/graders.d.ts.map +1 -1
- package/dist/types/learning/budgeted-reward-fixtures.d.ts +2 -1
- package/dist/types/learning/budgeted-reward-fixtures.d.ts.map +1 -1
- package/dist/types/learning/contracts.d.ts +167 -0
- package/dist/types/learning/contracts.d.ts.map +1 -1
- package/dist/types/learning/index.d.ts +1 -0
- package/dist/types/learning/index.d.ts.map +1 -1
- package/dist/types/learning/judge-budget.d.ts +3 -0
- package/dist/types/learning/judge-budget.d.ts.map +1 -1
- package/dist/types/learning/model-judge.d.ts +6 -1
- package/dist/types/learning/model-judge.d.ts.map +1 -1
- package/dist/types/learning/operations.d.ts +44 -0
- package/dist/types/learning/operations.d.ts.map +1 -1
- package/dist/types/learning/repository.d.ts +81 -0
- package/dist/types/learning/repository.d.ts.map +1 -1
- package/dist/types/learning/reward-calibration-closure.d.ts +2323 -0
- package/dist/types/learning/reward-calibration-closure.d.ts.map +1 -0
- package/dist/types/learning/reward-calibration.d.ts +1 -1
- package/dist/types/learning/reward-calibration.d.ts.map +1 -1
- package/dist/types/learning/reward-checks.d.ts +35 -0
- package/dist/types/learning/reward-checks.d.ts.map +1 -1
- package/dist/types/learning/transport.d.ts +103 -0
- package/dist/types/learning/transport.d.ts.map +1 -1
- package/dist/types/native-conversations/appimage-runtime.d.ts +12 -0
- package/dist/types/native-conversations/appimage-runtime.d.ts.map +1 -0
- package/dist/types/native-conversations/collector-branches.d.ts +22 -0
- package/dist/types/native-conversations/collector-branches.d.ts.map +1 -0
- package/dist/types/native-conversations/collector-contracts.d.ts +88 -0
- package/dist/types/native-conversations/collector-contracts.d.ts.map +1 -0
- package/dist/types/native-conversations/collector-destinations.d.ts +18 -0
- package/dist/types/native-conversations/collector-destinations.d.ts.map +1 -0
- package/dist/types/native-conversations/collector-errors.d.ts +13 -0
- package/dist/types/native-conversations/collector-errors.d.ts.map +1 -0
- package/dist/types/native-conversations/collector-progress.d.ts +24 -0
- package/dist/types/native-conversations/collector-progress.d.ts.map +1 -0
- package/dist/types/native-conversations/collector-store.d.ts +31 -0
- package/dist/types/native-conversations/collector-store.d.ts.map +1 -0
- package/dist/types/native-conversations/collector-sync-requests.d.ts +25 -0
- package/dist/types/native-conversations/collector-sync-requests.d.ts.map +1 -0
- package/dist/types/native-conversations/collector.d.ts +8 -0
- package/dist/types/native-conversations/collector.d.ts.map +1 -0
- package/dist/types/native-conversations/contracts.d.ts +47 -0
- package/dist/types/native-conversations/contracts.d.ts.map +1 -0
- package/dist/types/native-conversations/database.d.ts +14 -0
- package/dist/types/native-conversations/database.d.ts.map +1 -0
- package/dist/types/native-conversations/discovery.d.ts +9 -0
- package/dist/types/native-conversations/discovery.d.ts.map +1 -0
- package/dist/types/native-conversations/history.d.ts +23 -0
- package/dist/types/native-conversations/history.d.ts.map +1 -0
- package/dist/types/native-conversations/index.d.ts +12 -0
- package/dist/types/native-conversations/index.d.ts.map +1 -0
- package/dist/types/native-conversations/location.d.ts +3 -0
- package/dist/types/native-conversations/location.d.ts.map +1 -0
- package/dist/types/native-conversations/openclaw-cold-archive.d.ts +8 -0
- package/dist/types/native-conversations/openclaw-cold-archive.d.ts.map +1 -0
- package/dist/types/native-conversations/openclaw-database.d.ts +14 -0
- package/dist/types/native-conversations/openclaw-database.d.ts.map +1 -0
- package/dist/types/native-conversations/source-locations.d.ts +8 -0
- package/dist/types/native-conversations/source-locations.d.ts.map +1 -0
- package/dist/types/native-conversations/supervisor.d.ts +21 -0
- package/dist/types/native-conversations/supervisor.d.ts.map +1 -0
- package/dist/types/rewards.d.ts +50 -0
- package/dist/types/rewards.d.ts.map +1 -1
- package/dist/types/telemetry-analysis.d.ts +1 -1
- package/dist/types/telemetry-bundle.d.ts +2 -2
- package/dist/types/telemetry-catalog.d.ts +21 -21
- package/dist/types/telemetry.d.ts +2 -2
- package/package.json +15 -3
- package/schemas/learning/v1/command-request.schema.json +122 -91
- package/schemas/learning/v1/decision.schema.json +63 -0
- package/schemas/learning/v1/grade.schema.json +91 -0
- package/schemas/learning/v1/reward_check.schema.json +91 -0
- package/schemas/learning/v1/source.schema.json +28 -0
|
@@ -0,0 +1,322 @@
|
|
|
1
|
+
import { agentImportNormalizerRevision } from "../connected-evidence/imports.js";
|
|
2
|
+
import { watch } from "node:fs";
|
|
3
|
+
import { contentHash } from "@openpond/harness";
|
|
4
|
+
import { randomUUID } from "node:crypto";
|
|
5
|
+
import { listSessions, readSession } from "./history.js";
|
|
6
|
+
import { CollectorStore } from "./collector-store.js";
|
|
7
|
+
import { CollectorErrors, } from "./collector-errors.js";
|
|
8
|
+
import { collectorBranchAnchor } from "./collector-branches.js";
|
|
9
|
+
import { CollectorSyncRequests } from "./collector-sync-requests.js";
|
|
10
|
+
import { COLLECTOR_DEFAULTS, } from "./collector-contracts.js";
|
|
11
|
+
/** Supervised single owner: acquisition and admission checkpoints survive monitor/service exits. */
|
|
12
|
+
export async function runCollector(input) {
|
|
13
|
+
const store = await CollectorStore.open(input.directory), nonce = randomUUID();
|
|
14
|
+
store.database.exec("BEGIN IMMEDIATE");
|
|
15
|
+
try {
|
|
16
|
+
const retained = store.setting("owner");
|
|
17
|
+
if (retained) {
|
|
18
|
+
const owner = JSON.parse(retained);
|
|
19
|
+
let alive = true;
|
|
20
|
+
try {
|
|
21
|
+
process.kill(owner.pid, 0);
|
|
22
|
+
}
|
|
23
|
+
catch (error) {
|
|
24
|
+
alive = error.code !== "ESRCH";
|
|
25
|
+
}
|
|
26
|
+
if (alive)
|
|
27
|
+
throw new Error("The collector already has a live owner.");
|
|
28
|
+
}
|
|
29
|
+
store.set("owner", JSON.stringify({ pid: process.pid, nonce }));
|
|
30
|
+
store.database.exec("COMMIT");
|
|
31
|
+
}
|
|
32
|
+
catch (error) {
|
|
33
|
+
store.database.exec("ROLLBACK");
|
|
34
|
+
store.close();
|
|
35
|
+
throw error;
|
|
36
|
+
}
|
|
37
|
+
if (store.setting("desiredState") !== "running") {
|
|
38
|
+
store.set("owner", "");
|
|
39
|
+
store.close();
|
|
40
|
+
return;
|
|
41
|
+
}
|
|
42
|
+
store.set("pid", String(process.pid));
|
|
43
|
+
let dirty = true, lastReconcile = 0, lastHeartbeat = 0;
|
|
44
|
+
const watchers = new Map();
|
|
45
|
+
const errors = new CollectorErrors(store);
|
|
46
|
+
const syncRequests = new CollectorSyncRequests(store);
|
|
47
|
+
const reconciledRevisions = new Map();
|
|
48
|
+
function retained(connection) {
|
|
49
|
+
return (store.connections().find((item) => item.id === connection.id) ??
|
|
50
|
+
connection);
|
|
51
|
+
}
|
|
52
|
+
function heartbeatInput(connection) {
|
|
53
|
+
const pendingOperations = store.queued(connection.id);
|
|
54
|
+
const error = errors.message(connection.id, "heartbeat");
|
|
55
|
+
return {
|
|
56
|
+
pendingOperations,
|
|
57
|
+
error,
|
|
58
|
+
// A durable older completion may coexist with newly queued live work.
|
|
59
|
+
// Keep its receipt, but do not send an ineligible ACK that the server
|
|
60
|
+
// must reject while the current acquisition/admission state is unhealthy.
|
|
61
|
+
completedSyncRevision: !pendingOperations && !error ? connection.completedSyncRevision ?? 0 : 0,
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
function applyRemote(connection, remote) {
|
|
65
|
+
const current = retained(connection);
|
|
66
|
+
if (current.state === "disconnected" || remote.revision < current.revision)
|
|
67
|
+
return current;
|
|
68
|
+
const sync = syncRequests.observe(current, remote);
|
|
69
|
+
if (remote.revision > current.revision)
|
|
70
|
+
errors.set(current.id, "control", null);
|
|
71
|
+
const next = { ...current, ...sync, ...(remote.revision > current.revision ? { revision: remote.revision, state: remote.state } : {}), ...(remote.destinations ? { destinations: remote.destinations } : {}) };
|
|
72
|
+
store.put(next);
|
|
73
|
+
return next;
|
|
74
|
+
}
|
|
75
|
+
function recordFailure(connection, error, phase) {
|
|
76
|
+
const message = error instanceof Error ? error.message : "Collection failed.";
|
|
77
|
+
errors.set(connection.id, phase, message);
|
|
78
|
+
const status = error && typeof error === "object" && "status" in error
|
|
79
|
+
? Number(error.status)
|
|
80
|
+
: 0;
|
|
81
|
+
const current = retained(connection);
|
|
82
|
+
// Authentication loss requires explicit reconnection; a timer or restart
|
|
83
|
+
// must never turn a revoked connection back into a paused/resumable one.
|
|
84
|
+
if (current.state !== "disconnected" && [400, 401, 403].includes(status))
|
|
85
|
+
store.put({
|
|
86
|
+
...current,
|
|
87
|
+
state: status === 400 ? "paused" : "disconnected",
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
async function acquire(connection) {
|
|
91
|
+
// A request arriving mid-scan must not be acknowledged by the older scan.
|
|
92
|
+
const controlRevision = connection.revision;
|
|
93
|
+
const requestedSyncRevision = connection.requestedSyncRevision ?? 0;
|
|
94
|
+
let cursor;
|
|
95
|
+
const failures = [];
|
|
96
|
+
do {
|
|
97
|
+
if (Date.now() - lastHeartbeat >= COLLECTOR_DEFAULTS.heartbeatMs) {
|
|
98
|
+
lastHeartbeat = Date.now();
|
|
99
|
+
store.set("heartbeatAt", new Date().toISOString());
|
|
100
|
+
for (const current of store.connections()) {
|
|
101
|
+
if (current.state === "disconnected")
|
|
102
|
+
continue;
|
|
103
|
+
try {
|
|
104
|
+
const remote = await input.transport.heartbeat(current, heartbeatInput(current));
|
|
105
|
+
applyRemote(current, remote);
|
|
106
|
+
errors.set(current.id, "heartbeat", null);
|
|
107
|
+
}
|
|
108
|
+
catch (error) {
|
|
109
|
+
recordFailure(current, error, "heartbeat");
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
connection =
|
|
114
|
+
store.connections().find((item) => item.id === connection.id) ??
|
|
115
|
+
connection;
|
|
116
|
+
if (connection.state !== "active")
|
|
117
|
+
return;
|
|
118
|
+
if (input.signal.aborted || store.setting("desiredState") !== "running")
|
|
119
|
+
return;
|
|
120
|
+
const page = await listSessions(connection.source, {
|
|
121
|
+
since: connection.since ?? undefined,
|
|
122
|
+
cursor,
|
|
123
|
+
limit: 50,
|
|
124
|
+
});
|
|
125
|
+
for (const native of page.items) {
|
|
126
|
+
const scanKey = JSON.stringify([native.path, native.nativeSessionId]);
|
|
127
|
+
const scanRevision = native.storageRevision
|
|
128
|
+
? contentHash([native.storageRevision, agentImportNormalizerRevision(connection.source.source)])
|
|
129
|
+
: null;
|
|
130
|
+
store.progress.select(connection.id, scanKey);
|
|
131
|
+
if (input.signal.aborted ||
|
|
132
|
+
store.setting("desiredState") !== "running" ||
|
|
133
|
+
store.connections().find((item) => item.id === connection.id)
|
|
134
|
+
?.state !== "active")
|
|
135
|
+
return;
|
|
136
|
+
try {
|
|
137
|
+
if (scanRevision &&
|
|
138
|
+
!store.sourceChanged(connection.id, scanKey, scanRevision) &&
|
|
139
|
+
!store.progress.tracks(connection.id, scanKey))
|
|
140
|
+
continue;
|
|
141
|
+
const { files, preview, branchLeafId } = await readSession(connection.source, native, {
|
|
142
|
+
branchAnchor: collectorBranchAnchor(store, connection.id, native.nativeSessionId),
|
|
143
|
+
});
|
|
144
|
+
let eligible = false;
|
|
145
|
+
for (const session of preview.sessions) {
|
|
146
|
+
const key = JSON.stringify([
|
|
147
|
+
connection.source.instanceId,
|
|
148
|
+
session.sessionId,
|
|
149
|
+
session.branchId,
|
|
150
|
+
]);
|
|
151
|
+
store.progress.retained(connection.id, scanKey, key);
|
|
152
|
+
const selectedBoundaries = session.boundaries.filter((boundary) => boundary.projection === "turn" &&
|
|
153
|
+
boundary.terminal !== "unknown" &&
|
|
154
|
+
(!connection.since ||
|
|
155
|
+
Date.parse(session.events[boundary.start].occurredAt ??
|
|
156
|
+
native.updatedAt) >= Date.parse(connection.since)));
|
|
157
|
+
eligible ||= selectedBoundaries.length > 0;
|
|
158
|
+
const changed = selectedBoundaries.filter((boundary) => store.changed(connection.id, key, boundary.id, contentHash([boundary.inputHash, boundary.revisionHash])));
|
|
159
|
+
for (let offset = 0; offset < changed.length; offset += COLLECTOR_DEFAULTS.batchCases) {
|
|
160
|
+
const selected = changed.slice(offset, offset + COLLECTOR_DEFAULTS.batchCases);
|
|
161
|
+
const operationId = `sync-${contentHash([connection.id, key, selected.map((boundary) => [boundary.id, boundary.inputHash, boundary.revisionHash])]).slice(0, 48)}`;
|
|
162
|
+
store.enqueue({
|
|
163
|
+
operationId,
|
|
164
|
+
connectionId: connection.id,
|
|
165
|
+
sessionKey: key,
|
|
166
|
+
contentHash: session.contentHash,
|
|
167
|
+
files,
|
|
168
|
+
boundaryIds: selected.map((boundary) => boundary.id),
|
|
169
|
+
...(branchLeafId ? { branchLeafId } : {}),
|
|
170
|
+
}, selected.map((boundary) => ({
|
|
171
|
+
id: boundary.id,
|
|
172
|
+
revision: contentHash([
|
|
173
|
+
boundary.inputHash,
|
|
174
|
+
boundary.revisionHash,
|
|
175
|
+
]),
|
|
176
|
+
})), scanKey);
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
store.progress.read(connection.id, scanKey, eligible);
|
|
180
|
+
if (scanRevision)
|
|
181
|
+
store.scanned(connection.id, scanKey, scanRevision);
|
|
182
|
+
}
|
|
183
|
+
catch (error) {
|
|
184
|
+
store.progress.failed(connection.id, scanKey, error instanceof Error ? error.message : "Unable to read session");
|
|
185
|
+
failures.push(`${native.nativeSessionId}: ${error instanceof Error ? error.message : "Unable to read session"}`);
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
cursor = page.nextCursor ?? undefined;
|
|
189
|
+
} while (cursor);
|
|
190
|
+
store.progress.discovered(connection.id);
|
|
191
|
+
reconciledRevisions.set(connection.id, controlRevision);
|
|
192
|
+
syncRequests.acquired(connection.id, { controlRevision, requestedSyncRevision, succeeded: failures.length === 0 });
|
|
193
|
+
errors.set(connection.id, "source", failures.length
|
|
194
|
+
? `${failures.length} session(s) need attention. ${failures[0]}`
|
|
195
|
+
: null);
|
|
196
|
+
}
|
|
197
|
+
try {
|
|
198
|
+
while (!input.signal.aborted &&
|
|
199
|
+
store.setting("desiredState") === "running") {
|
|
200
|
+
const connections = store.connections();
|
|
201
|
+
store.set("heartbeatAt", new Date().toISOString());
|
|
202
|
+
for (const connection of connections) {
|
|
203
|
+
if (connection.state === "disconnected")
|
|
204
|
+
continue;
|
|
205
|
+
if (!watchers.has(connection.source.root))
|
|
206
|
+
try {
|
|
207
|
+
const watcher = watch(connection.source.root, { recursive: connection.source.acquisition !== "sqlite" }, () => {
|
|
208
|
+
dirty = true;
|
|
209
|
+
});
|
|
210
|
+
watcher.on("error", () => {
|
|
211
|
+
dirty = true;
|
|
212
|
+
});
|
|
213
|
+
watchers.set(connection.source.root, watcher);
|
|
214
|
+
}
|
|
215
|
+
catch {
|
|
216
|
+
/* Reconciliation remains authoritative when platform watch is unavailable. */
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
const heartbeat = Date.now() - lastHeartbeat >= COLLECTOR_DEFAULTS.heartbeatMs;
|
|
220
|
+
if (heartbeat)
|
|
221
|
+
lastHeartbeat = Date.now();
|
|
222
|
+
const reconcile = dirty ||
|
|
223
|
+
Date.now() - lastReconcile >= COLLECTOR_DEFAULTS.reconcileMs ||
|
|
224
|
+
store.setting("syncNow") === "yes";
|
|
225
|
+
if (reconcile) {
|
|
226
|
+
dirty = false;
|
|
227
|
+
lastReconcile = Date.now();
|
|
228
|
+
store.set("syncNow", "no");
|
|
229
|
+
}
|
|
230
|
+
for (let connection of connections) {
|
|
231
|
+
connection = retained(connection);
|
|
232
|
+
if (connection.state === "disconnected")
|
|
233
|
+
continue;
|
|
234
|
+
let phase = "heartbeat";
|
|
235
|
+
try {
|
|
236
|
+
if (heartbeat) {
|
|
237
|
+
const remote = await input.transport.heartbeat(connection, heartbeatInput(connection));
|
|
238
|
+
connection = applyRemote(connection, remote);
|
|
239
|
+
errors.set(connection.id, "heartbeat", null);
|
|
240
|
+
}
|
|
241
|
+
if (connection.state !== "active") {
|
|
242
|
+
reconciledRevisions.delete(connection.id);
|
|
243
|
+
continue;
|
|
244
|
+
}
|
|
245
|
+
phase = "source";
|
|
246
|
+
// Resume must scan before retained one-time completion can pause again,
|
|
247
|
+
// even when the ordinary reconcile interval has not elapsed.
|
|
248
|
+
if (reconcile || reconciledRevisions.get(connection.id) !== connection.revision || syncRequests.needsAcquisition(connection))
|
|
249
|
+
await acquire(connection);
|
|
250
|
+
connection =
|
|
251
|
+
store.connections().find((item) => item.id === connection.id) ??
|
|
252
|
+
connection;
|
|
253
|
+
if (connection.state !== "active")
|
|
254
|
+
continue;
|
|
255
|
+
if (input.signal.aborted || store.setting("desiredState") !== "running")
|
|
256
|
+
continue;
|
|
257
|
+
phase = "admission";
|
|
258
|
+
const next = store.next(connection.id);
|
|
259
|
+
if (next)
|
|
260
|
+
try {
|
|
261
|
+
await input.transport.admit(connection, next.entry);
|
|
262
|
+
store.acknowledge(next.entry);
|
|
263
|
+
errors.set(connection.id, "admission", null);
|
|
264
|
+
}
|
|
265
|
+
catch (error) {
|
|
266
|
+
store.retry(next.entry.operationId, next.attempts);
|
|
267
|
+
throw error;
|
|
268
|
+
}
|
|
269
|
+
connection = retained(connection);
|
|
270
|
+
if (input.signal.aborted || store.setting("desiredState") !== "running" || connection.state !== "active")
|
|
271
|
+
continue;
|
|
272
|
+
connection = syncRequests.complete(connection, errors.message(connection.id, "heartbeat"));
|
|
273
|
+
if (syncRequests.needsAcknowledgement(connection) && heartbeatInput(connection).completedSyncRevision) {
|
|
274
|
+
phase = "heartbeat";
|
|
275
|
+
const remote = await input.transport.heartbeat(connection, heartbeatInput(connection));
|
|
276
|
+
connection = applyRemote(connection, remote);
|
|
277
|
+
errors.set(connection.id, "heartbeat", null);
|
|
278
|
+
}
|
|
279
|
+
if (!input.signal.aborted &&
|
|
280
|
+
store.setting("desiredState") === "running" &&
|
|
281
|
+
connection.state === "active" &&
|
|
282
|
+
!connection.keepSyncing &&
|
|
283
|
+
reconciledRevisions.get(connection.id) === connection.revision &&
|
|
284
|
+
store.queued(connection.id) === 0 &&
|
|
285
|
+
store.progress.status(connection.id).stage === "complete" &&
|
|
286
|
+
!errors.source(connection.id) &&
|
|
287
|
+
syncRequests.confirmed(connection.id) >= (connection.requestedSyncRevision ?? 0)) {
|
|
288
|
+
phase = "control";
|
|
289
|
+
if (!input.transport.pause)
|
|
290
|
+
throw new Error("This transport cannot acknowledge completion of a one-time import.");
|
|
291
|
+
const paused = await input.transport.pause(connection);
|
|
292
|
+
applyRemote(connection, paused);
|
|
293
|
+
errors.set(connection.id, "control", null);
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
catch (error) {
|
|
297
|
+
recordFailure(connection, error, phase);
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
await new Promise((resolve) => {
|
|
301
|
+
const done = () => {
|
|
302
|
+
clearTimeout(timer);
|
|
303
|
+
input.signal.removeEventListener("abort", done);
|
|
304
|
+
resolve();
|
|
305
|
+
};
|
|
306
|
+
const timer = setTimeout(done, 1000);
|
|
307
|
+
input.signal.addEventListener("abort", done, { once: true });
|
|
308
|
+
});
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
finally {
|
|
312
|
+
for (const watcher of watchers.values())
|
|
313
|
+
watcher.close();
|
|
314
|
+
const owner = JSON.parse(store.setting("owner") || "{}");
|
|
315
|
+
if (owner.nonce === nonce) {
|
|
316
|
+
store.set("pid", "");
|
|
317
|
+
store.set("heartbeatAt", "");
|
|
318
|
+
store.set("owner", "");
|
|
319
|
+
}
|
|
320
|
+
store.close();
|
|
321
|
+
}
|
|
322
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export const NATIVE_SOURCE_NAMES = {
|
|
2
|
+
codex: "Codex",
|
|
3
|
+
claude_code: "Claude Code",
|
|
4
|
+
hermes: "Hermes",
|
|
5
|
+
openclaw: "OpenClaw",
|
|
6
|
+
opencode: "OpenCode",
|
|
7
|
+
grok_build: "Grok Build",
|
|
8
|
+
pi: "Pi",
|
|
9
|
+
oh_my_pi: "Oh My Pi",
|
|
10
|
+
};
|
|
11
|
+
export const NATIVE_READ_LIMIT = 32 * 1024 * 1024;
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
import { DatabaseSync } from "node:sqlite";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { listOpenClawSessions, readOpenClawSession, } from "./openclaw-database.js";
|
|
4
|
+
import { NATIVE_READ_LIMIT } from "./contracts.js";
|
|
5
|
+
function open(source) {
|
|
6
|
+
const file = source.root.endsWith(".db")
|
|
7
|
+
? source.root
|
|
8
|
+
: join(source.root, source.source === "opencode" ? "opencode.db" : "state.db");
|
|
9
|
+
const database = new DatabaseSync(file, {
|
|
10
|
+
readOnly: true,
|
|
11
|
+
allowExtension: false,
|
|
12
|
+
enableDoubleQuotedStringLiterals: false,
|
|
13
|
+
});
|
|
14
|
+
database.exec("PRAGMA query_only=ON; PRAGMA busy_timeout=1000; BEGIN");
|
|
15
|
+
return database;
|
|
16
|
+
}
|
|
17
|
+
export function listDatabaseSessions(source, input) {
|
|
18
|
+
if (source.source === "openclaw")
|
|
19
|
+
return listOpenClawSessions(source, input);
|
|
20
|
+
const database = open(source);
|
|
21
|
+
try {
|
|
22
|
+
const since = input.since ? Date.parse(input.since) : 0;
|
|
23
|
+
const rows = source.source === "opencode"
|
|
24
|
+
? database
|
|
25
|
+
.prepare("SELECT id,title,directory AS cwd,time_updated AS updated, time_updated || ':' || COALESCE((SELECT MAX(time_updated) FROM message WHERE session_id=session.id),0) || ':' || COALESCE((SELECT MAX(time_updated) FROM part WHERE session_id=session.id),0) || ':' || (SELECT COUNT(*) FROM part WHERE session_id=session.id) AS revision FROM session WHERE id > ? AND time_updated >= ? ORDER BY id LIMIT ?")
|
|
26
|
+
.all(input.cursor ?? "", since, input.limit + 1)
|
|
27
|
+
: database
|
|
28
|
+
.prepare("SELECT id,title,cwd,MAX(COALESCE(last_activity_at,0),started_at) * 1000 AS updated FROM sessions WHERE id > ? AND MAX(COALESCE(last_activity_at,0),started_at) * 1000 >= ? ORDER BY id LIMIT ?")
|
|
29
|
+
.all(input.cursor ?? "", since, input.limit + 1);
|
|
30
|
+
const items = rows.slice(0, input.limit).map((row) => ({
|
|
31
|
+
nativeSessionId: String(row.id),
|
|
32
|
+
sourceInstanceId: source.instanceId,
|
|
33
|
+
path: String(row.id),
|
|
34
|
+
title: typeof row.title === "string" ? row.title : String(row.id),
|
|
35
|
+
cwd: typeof row.cwd === "string" ? row.cwd : null,
|
|
36
|
+
updatedAt: new Date(Number(row.updated)).toISOString(),
|
|
37
|
+
storageRevision: source.source === "opencode" ? String(row.revision) : undefined,
|
|
38
|
+
}));
|
|
39
|
+
return {
|
|
40
|
+
items,
|
|
41
|
+
nextCursor: rows.length > input.limit ? items.at(-1).nativeSessionId : null,
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
finally {
|
|
45
|
+
database.close();
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
export function readDatabaseSession(source, sessionId) {
|
|
49
|
+
if (source.source === "openclaw")
|
|
50
|
+
return readOpenClawSession(source, sessionId);
|
|
51
|
+
const database = open(source);
|
|
52
|
+
try {
|
|
53
|
+
let result;
|
|
54
|
+
if (source.source === "opencode") {
|
|
55
|
+
const info = database
|
|
56
|
+
.prepare("SELECT id,title,directory,parent_id,version,time_created,time_updated FROM session WHERE id = ?")
|
|
57
|
+
.get(sessionId);
|
|
58
|
+
if (!info)
|
|
59
|
+
throw new Error("Native session is unavailable.");
|
|
60
|
+
const total = database
|
|
61
|
+
.prepare("SELECT (SELECT COALESCE(SUM(length(CAST(data AS BLOB))),0) FROM message WHERE session_id=?) + (SELECT COALESCE(SUM(length(CAST(data AS BLOB))),0) FROM part WHERE session_id=?) AS bytes")
|
|
62
|
+
.get(sessionId, sessionId);
|
|
63
|
+
if (Number(total?.bytes) > NATIVE_READ_LIMIT)
|
|
64
|
+
throw new Error("Native session exceeds the retained byte limit.");
|
|
65
|
+
const rows = database
|
|
66
|
+
.prepare("SELECT id,data FROM message WHERE session_id = ? ORDER BY time_created,id LIMIT 100001")
|
|
67
|
+
.all(sessionId);
|
|
68
|
+
if (rows.length > 100000)
|
|
69
|
+
throw new Error("Native session exceeds the event limit.");
|
|
70
|
+
let bytes = 0;
|
|
71
|
+
const messages = rows.map((row) => {
|
|
72
|
+
bytes += Buffer.byteLength(String(row.data));
|
|
73
|
+
if (bytes > NATIVE_READ_LIMIT)
|
|
74
|
+
throw new Error("Native session exceeds the retained byte limit.");
|
|
75
|
+
const parts = database
|
|
76
|
+
.prepare("SELECT id,data FROM part WHERE session_id = ? AND message_id = ? ORDER BY time_created,id LIMIT 100001")
|
|
77
|
+
.all(sessionId, String(row.id));
|
|
78
|
+
if (parts.length > 100000)
|
|
79
|
+
throw new Error("Native parts exceed the event limit.");
|
|
80
|
+
return {
|
|
81
|
+
info: {
|
|
82
|
+
...JSON.parse(String(row.data)),
|
|
83
|
+
id: row.id,
|
|
84
|
+
sessionID: sessionId,
|
|
85
|
+
},
|
|
86
|
+
parts: parts.map((part) => {
|
|
87
|
+
bytes += Buffer.byteLength(String(part.data));
|
|
88
|
+
if (bytes > NATIVE_READ_LIMIT)
|
|
89
|
+
throw new Error("Native session exceeds the retained byte limit.");
|
|
90
|
+
return {
|
|
91
|
+
...JSON.parse(String(part.data)),
|
|
92
|
+
id: part.id,
|
|
93
|
+
sessionID: sessionId,
|
|
94
|
+
messageID: row.id,
|
|
95
|
+
};
|
|
96
|
+
}),
|
|
97
|
+
};
|
|
98
|
+
});
|
|
99
|
+
result = {
|
|
100
|
+
info: {
|
|
101
|
+
...info,
|
|
102
|
+
parentID: info.parent_id,
|
|
103
|
+
time: { created: info.time_created, updated: info.time_updated },
|
|
104
|
+
},
|
|
105
|
+
messages,
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
else {
|
|
109
|
+
const info = database
|
|
110
|
+
.prepare("SELECT id,title,parent_session_id,system_prompt FROM sessions WHERE id = ?")
|
|
111
|
+
.get(sessionId);
|
|
112
|
+
if (!info)
|
|
113
|
+
throw new Error("Native session is unavailable.");
|
|
114
|
+
const total = database
|
|
115
|
+
.prepare("SELECT COALESCE(SUM(length(CAST(COALESCE(content,'') AS BLOB))+length(CAST(COALESCE(tool_calls,'') AS BLOB))),0) AS bytes FROM messages WHERE session_id=? AND active=1")
|
|
116
|
+
.get(sessionId);
|
|
117
|
+
if (Number(total?.bytes) > NATIVE_READ_LIMIT)
|
|
118
|
+
throw new Error("Native session exceeds the retained byte limit.");
|
|
119
|
+
const rows = database
|
|
120
|
+
.prepare("SELECT id,role,content,tool_call_id,tool_calls,timestamp,finish_reason,compacted FROM messages WHERE session_id = ? AND active = 1 ORDER BY id LIMIT 100001")
|
|
121
|
+
.all(sessionId);
|
|
122
|
+
if (rows.length > 100000)
|
|
123
|
+
throw new Error("Native session exceeds the event limit.");
|
|
124
|
+
let bytes = 0;
|
|
125
|
+
result = {
|
|
126
|
+
...info,
|
|
127
|
+
messages: rows.map((row) => {
|
|
128
|
+
bytes += Buffer.byteLength(JSON.stringify(row));
|
|
129
|
+
if (bytes > NATIVE_READ_LIMIT)
|
|
130
|
+
throw new Error("Native session exceeds the retained byte limit.");
|
|
131
|
+
return {
|
|
132
|
+
...row,
|
|
133
|
+
tool_calls: typeof row.tool_calls === "string"
|
|
134
|
+
? JSON.parse(row.tool_calls)
|
|
135
|
+
: null,
|
|
136
|
+
};
|
|
137
|
+
}),
|
|
138
|
+
message_count: rows.length,
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
const text = JSON.stringify(result);
|
|
142
|
+
if (Buffer.byteLength(text) > NATIVE_READ_LIMIT)
|
|
143
|
+
throw new Error("Native session exceeds the retained byte limit.");
|
|
144
|
+
return { path: `${sessionId}.json`, text };
|
|
145
|
+
}
|
|
146
|
+
finally {
|
|
147
|
+
database.close();
|
|
148
|
+
}
|
|
149
|
+
}
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
import { homedir } from "node:os";
|
|
2
|
+
import { realpath, stat, readdir, readFile } from "node:fs/promises";
|
|
3
|
+
import { resolve, join, isAbsolute } from "node:path";
|
|
4
|
+
import { contentHash } from "@openpond/harness";
|
|
5
|
+
import { hermesHome, hermesLocations, ompLocations } from "./source-locations.js";
|
|
6
|
+
import { NATIVE_SOURCE_NAMES, } from "./contracts.js";
|
|
7
|
+
/** Discovery returns locations/capabilities only. Transcript reads require a selected source. */
|
|
8
|
+
export async function discoverSources(input) {
|
|
9
|
+
const home = input.home ?? homedir(), env = input.environment ?? process.env;
|
|
10
|
+
const defaults = {
|
|
11
|
+
codex: env.CODEX_HOME || join(home, ".codex"),
|
|
12
|
+
claude_code: env.CLAUDE_CONFIG_DIR || join(home, ".claude"),
|
|
13
|
+
hermes: hermesHome(home, env),
|
|
14
|
+
openclaw: env.OPENCLAW_STATE_DIR || join(home, ".openclaw"),
|
|
15
|
+
opencode: join(env.XDG_DATA_HOME || join(home, ".local/share"), "opencode"),
|
|
16
|
+
grok_build: env.GROK_HOME || join(home, ".grok"),
|
|
17
|
+
pi: env.PI_CODING_AGENT_DIR || join(home, ".pi/agent"),
|
|
18
|
+
oh_my_pi: join(home, ".omp/agent"),
|
|
19
|
+
};
|
|
20
|
+
// Only documented non-secret source-location settings are read, and never emitted.
|
|
21
|
+
const piHome = defaults.pi;
|
|
22
|
+
let piRelativeSetting = false;
|
|
23
|
+
if (env.PI_CODING_AGENT_SESSION_DIR)
|
|
24
|
+
defaults.pi = resolve(env.PI_CODING_AGENT_SESSION_DIR);
|
|
25
|
+
else if (!input.locations?.pi) {
|
|
26
|
+
try {
|
|
27
|
+
const file = join(piHome, "settings.json");
|
|
28
|
+
if ((await stat(file)).size <= 1024 * 1024) {
|
|
29
|
+
const setting = JSON.parse(await readFile(file, "utf8"));
|
|
30
|
+
if (typeof setting.sessionDir === "string" &&
|
|
31
|
+
setting.sessionDir.trim()) {
|
|
32
|
+
if (isAbsolute(setting.sessionDir))
|
|
33
|
+
defaults.pi = setting.sessionDir;
|
|
34
|
+
else
|
|
35
|
+
piRelativeSetting = true;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
catch {
|
|
40
|
+
/* Missing/invalid settings do not authorize broader filesystem discovery. */
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
const result = [];
|
|
44
|
+
for (const source of Object.keys(NATIVE_SOURCE_NAMES)) {
|
|
45
|
+
const locations = input.locations?.[source]
|
|
46
|
+
? [{ root: input.locations[source] }]
|
|
47
|
+
: source === "hermes"
|
|
48
|
+
? await hermesLocations(resolve(defaults.hermes))
|
|
49
|
+
: source === "oh_my_pi"
|
|
50
|
+
? await ompLocations(home, env)
|
|
51
|
+
: [{ root: defaults[source] }];
|
|
52
|
+
for (const location of locations) {
|
|
53
|
+
let root = resolve(location.root), available = false, file = false;
|
|
54
|
+
try {
|
|
55
|
+
root = await realpath(root);
|
|
56
|
+
const info = await stat(root);
|
|
57
|
+
file = info.isFile();
|
|
58
|
+
available = info.isDirectory() || file;
|
|
59
|
+
}
|
|
60
|
+
catch {
|
|
61
|
+
/* Absent or unreadable source is shown explicitly. */
|
|
62
|
+
}
|
|
63
|
+
if (location.reason)
|
|
64
|
+
available = false;
|
|
65
|
+
if (source === "openclaw" && available && !file) {
|
|
66
|
+
const agentsRoot = join(root, "agents");
|
|
67
|
+
const agents = await readdir(agentsRoot, { withFileTypes: true }).catch(() => []);
|
|
68
|
+
const databases = [];
|
|
69
|
+
for (const agent of agents.slice(0, 100)) {
|
|
70
|
+
if (!agent.isDirectory() || agent.isSymbolicLink())
|
|
71
|
+
continue;
|
|
72
|
+
const database = join(agentsRoot, agent.name, "agent", "openclaw-agent.sqlite");
|
|
73
|
+
if (await stat(database)
|
|
74
|
+
.then((info) => info.isFile())
|
|
75
|
+
.catch(() => false))
|
|
76
|
+
databases.push(await realpath(database));
|
|
77
|
+
}
|
|
78
|
+
if (databases.length) {
|
|
79
|
+
for (const database of databases)
|
|
80
|
+
result.push({
|
|
81
|
+
source,
|
|
82
|
+
root: database,
|
|
83
|
+
machineId: input.machineId,
|
|
84
|
+
instanceId: `native-${contentHash([input.machineId, source, database]).slice(0, 40)}`,
|
|
85
|
+
acquisition: "sqlite",
|
|
86
|
+
available: true,
|
|
87
|
+
capabilities: { history: true, live: true, nativeResume: false },
|
|
88
|
+
});
|
|
89
|
+
continue;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
if (source === "pi" && piRelativeSetting && !input.locations?.pi) {
|
|
93
|
+
result.push({
|
|
94
|
+
source,
|
|
95
|
+
root,
|
|
96
|
+
machineId: input.machineId,
|
|
97
|
+
instanceId: `native-${contentHash([input.machineId, source, root]).slice(0, 40)}`,
|
|
98
|
+
acquisition: "files",
|
|
99
|
+
available: false,
|
|
100
|
+
capabilities: { history: false, live: false, nativeResume: false },
|
|
101
|
+
reason: "Pi config uses a working-directory-relative sessionDir. Select its resolved directory with --source-path.",
|
|
102
|
+
});
|
|
103
|
+
continue;
|
|
104
|
+
}
|
|
105
|
+
result.push({
|
|
106
|
+
source,
|
|
107
|
+
root,
|
|
108
|
+
machineId: input.machineId,
|
|
109
|
+
instanceId: `native-${contentHash([input.machineId, source, root]).slice(0, 40)}`,
|
|
110
|
+
acquisition: file && !/\.(?:db|sqlite)$/u.test(root)
|
|
111
|
+
? "files"
|
|
112
|
+
: source === "hermes" ||
|
|
113
|
+
source === "opencode" ||
|
|
114
|
+
(source === "openclaw" && file)
|
|
115
|
+
? "sqlite"
|
|
116
|
+
: source === "openclaw"
|
|
117
|
+
? "bundle"
|
|
118
|
+
: "files",
|
|
119
|
+
available,
|
|
120
|
+
capabilities: {
|
|
121
|
+
history: available,
|
|
122
|
+
live: available && (source !== "openclaw" || file),
|
|
123
|
+
nativeResume: false,
|
|
124
|
+
},
|
|
125
|
+
...(!available
|
|
126
|
+
? {
|
|
127
|
+
reason: location.reason ??
|
|
128
|
+
"Source location is missing or unreadable. Select a custom location.",
|
|
129
|
+
}
|
|
130
|
+
: {}),
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
return result.filter((item, index) => result.findIndex((candidate) => candidate.instanceId === item.instanceId) === index);
|
|
135
|
+
}
|