@indigoai-us/hq-cloud 6.16.2 → 6.16.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-events.test.js +22 -12
- package/dist/bin/sync-runner-events.test.js.map +1 -1
- package/dist/bin/sync-runner-heap-telemetry.d.ts +238 -84
- package/dist/bin/sync-runner-heap-telemetry.d.ts.map +1 -1
- package/dist/bin/sync-runner-heap-telemetry.js +385 -131
- package/dist/bin/sync-runner-heap-telemetry.js.map +1 -1
- package/dist/bin/sync-runner-heap-telemetry.test.js +539 -204
- package/dist/bin/sync-runner-heap-telemetry.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +65 -15
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.test.js +47 -9
- package/dist/bin/sync-runner-watch-loop.test.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +32 -14
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/cli/reindex.d.ts.map +1 -1
- package/dist/cli/reindex.js +2 -253
- package/dist/cli/reindex.js.map +1 -1
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +10 -0
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +67 -1
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/object-io.d.ts +2 -0
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +2 -0
- package/dist/object-io.js.map +1 -1
- package/dist/object-io.test.js +5 -1
- package/dist/object-io.test.js.map +1 -1
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +3 -1
- package/dist/personal-vault.js.map +1 -1
- package/dist/personal-vault.test.js +6 -0
- package/dist/personal-vault.test.js.map +1 -1
- package/dist/s3.d.ts +2 -0
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +1 -0
- package/dist/s3.js.map +1 -1
- package/dist/s3.test.js +22 -0
- package/dist/s3.test.js.map +1 -1
- package/dist/session-log-capture.d.ts +20 -0
- package/dist/session-log-capture.d.ts.map +1 -0
- package/dist/session-log-capture.js +648 -0
- package/dist/session-log-capture.js.map +1 -0
- package/dist/session-log-capture.test.d.ts +2 -0
- package/dist/session-log-capture.test.d.ts.map +1 -0
- package/dist/session-log-capture.test.js +276 -0
- package/dist/session-log-capture.test.js.map +1 -0
- package/dist/sync/state-store.js +5 -1
- package/dist/sync/state-store.js.map +1 -1
- package/dist/vault-client.d.ts +5 -0
- package/dist/vault-client.d.ts.map +1 -1
- package/dist/vault-client.js +1 -0
- package/dist/vault-client.js.map +1 -1
- package/dist/vault-client.test.js +15 -0
- package/dist/vault-client.test.js.map +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,648 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Source-authoritative capture of coding-harness session logs.
|
|
3
|
+
*
|
|
4
|
+
* Captured files remain individual, byte-identical files under
|
|
5
|
+
* workspace/.session-logs. The state stored here deliberately records the
|
|
6
|
+
* source, not the destination: an archived, absent, or unreadable destination
|
|
7
|
+
* must never turn an unchanged transcript into another copy.
|
|
8
|
+
*/
|
|
9
|
+
import * as crypto from "node:crypto";
|
|
10
|
+
import * as fs from "node:fs";
|
|
11
|
+
import * as os from "node:os";
|
|
12
|
+
import * as path from "node:path";
|
|
13
|
+
import { getStateDir } from "./journal.js";
|
|
14
|
+
import { acquireOperationLock, OperationLockedError, OperationLockUnwritableError, } from "./operation-lock.js";
|
|
15
|
+
import { StateStore, StateStoreCorruptionError, } from "./sync/state-store.js";
|
|
16
|
+
const SESSION_LOG_DEST_PREFIX = "workspace/.session-logs/";
|
|
17
|
+
const SESSION_LOG_SCOPE_PREFIX = "session-log-capture:v1:";
|
|
18
|
+
const PREPARE_RECORD = 1;
|
|
19
|
+
const COMMIT_RECORD = 2;
|
|
20
|
+
const COPY_BUFFER_BYTES = 1024 * 1024;
|
|
21
|
+
const COPY_ATTEMPTS = 3;
|
|
22
|
+
const CAPTURE_BATCH_SIZE = 128;
|
|
23
|
+
const CAPTURE_COMPACTION_RECORDS = 256;
|
|
24
|
+
let captureTestHook;
|
|
25
|
+
let captureTestMaxRecordsBeforeCompaction;
|
|
26
|
+
/** Test-only crash seam for recovery coverage. */
|
|
27
|
+
export function setSessionLogCaptureTestHook(hook) {
|
|
28
|
+
captureTestHook = hook;
|
|
29
|
+
}
|
|
30
|
+
/** Test-only seam for forcing the append/compaction crash window. */
|
|
31
|
+
export function setSessionLogCaptureMaxRecordsBeforeCompactionForTest(maxRecordsBeforeCompaction) {
|
|
32
|
+
captureTestMaxRecordsBeforeCompaction = maxRecordsBeforeCompaction;
|
|
33
|
+
}
|
|
34
|
+
function warning(summary, message) {
|
|
35
|
+
summary.warnings.push(message);
|
|
36
|
+
process.stderr.write(`reindex: ${message}\n`);
|
|
37
|
+
}
|
|
38
|
+
function isDirectory(file) {
|
|
39
|
+
try {
|
|
40
|
+
return fs.statSync(file).isDirectory();
|
|
41
|
+
}
|
|
42
|
+
catch {
|
|
43
|
+
return false;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
function canonicalRoot(root) {
|
|
47
|
+
const resolved = path.resolve(root);
|
|
48
|
+
try {
|
|
49
|
+
return fs.realpathSync.native(resolved);
|
|
50
|
+
}
|
|
51
|
+
catch {
|
|
52
|
+
return resolved;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
function sourceSignature(file) {
|
|
56
|
+
try {
|
|
57
|
+
const stat = fs.statSync(file, { bigint: true });
|
|
58
|
+
if (!stat.isFile())
|
|
59
|
+
return null;
|
|
60
|
+
return {
|
|
61
|
+
dev: stat.dev.toString(),
|
|
62
|
+
ino: stat.ino.toString(),
|
|
63
|
+
size: stat.size.toString(),
|
|
64
|
+
mtimeNs: stat.mtimeNs.toString(),
|
|
65
|
+
ctimeNs: stat.ctimeNs.toString(),
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
catch {
|
|
69
|
+
return null;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
function signaturesEqual(left, right) {
|
|
73
|
+
return left.dev === right.dev &&
|
|
74
|
+
left.ino === right.ino &&
|
|
75
|
+
left.size === right.size &&
|
|
76
|
+
left.mtimeNs === right.mtimeNs &&
|
|
77
|
+
left.ctimeNs === right.ctimeNs;
|
|
78
|
+
}
|
|
79
|
+
function hash(value) {
|
|
80
|
+
return crypto.createHash("sha256").update(value).digest("hex");
|
|
81
|
+
}
|
|
82
|
+
function captureId(harness, source, destinationKey) {
|
|
83
|
+
return hash(`${harness}\0${source}\0${destinationKey}`);
|
|
84
|
+
}
|
|
85
|
+
function captureScope(root) {
|
|
86
|
+
return `${SESSION_LOG_SCOPE_PREFIX}${hash(root)}`;
|
|
87
|
+
}
|
|
88
|
+
function isSourceSignature(value) {
|
|
89
|
+
if (!value || typeof value !== "object")
|
|
90
|
+
return false;
|
|
91
|
+
const signature = value;
|
|
92
|
+
return typeof signature.dev === "string" &&
|
|
93
|
+
typeof signature.ino === "string" &&
|
|
94
|
+
typeof signature.size === "string" &&
|
|
95
|
+
typeof signature.mtimeNs === "string" &&
|
|
96
|
+
typeof signature.ctimeNs === "string";
|
|
97
|
+
}
|
|
98
|
+
function isCaptureRecordPayload(value) {
|
|
99
|
+
if (!value || typeof value !== "object")
|
|
100
|
+
return false;
|
|
101
|
+
const { id, capture } = value;
|
|
102
|
+
return typeof id === "string" &&
|
|
103
|
+
!!capture && typeof capture === "object" &&
|
|
104
|
+
typeof capture.destinationKey === "string" &&
|
|
105
|
+
typeof capture.hash === "string" &&
|
|
106
|
+
typeof capture.tempName === "string" &&
|
|
107
|
+
isSourceSignature(capture.signature);
|
|
108
|
+
}
|
|
109
|
+
function captureRecords(value) {
|
|
110
|
+
if (isCaptureRecordPayload(value))
|
|
111
|
+
return [value];
|
|
112
|
+
if (!value || typeof value !== "object")
|
|
113
|
+
return null;
|
|
114
|
+
const captures = value.captures;
|
|
115
|
+
if (!Array.isArray(captures) || captures.length === 0 || !captures.every(isCaptureRecordPayload))
|
|
116
|
+
return null;
|
|
117
|
+
const ids = new Set(captures.map((capture) => capture.id));
|
|
118
|
+
return ids.size === captures.length ? captures : null;
|
|
119
|
+
}
|
|
120
|
+
function validateCaptureRecord(record) {
|
|
121
|
+
const captures = captureRecords(record.payload);
|
|
122
|
+
if ((record.type !== PREPARE_RECORD && record.type !== COMMIT_RECORD) || captures === null) {
|
|
123
|
+
throw new StateStoreCorruptionError("invalid session-log capture record");
|
|
124
|
+
}
|
|
125
|
+
return captures;
|
|
126
|
+
}
|
|
127
|
+
function reduceCaptureState(state, record) {
|
|
128
|
+
const captures = validateCaptureRecord(record);
|
|
129
|
+
if (record.type === PREPARE_RECORD) {
|
|
130
|
+
for (const { id, capture } of captures)
|
|
131
|
+
state.prepared[id] = capture;
|
|
132
|
+
return state;
|
|
133
|
+
}
|
|
134
|
+
for (const { id, capture } of captures) {
|
|
135
|
+
const { tempName: _tempName, ...committed } = capture;
|
|
136
|
+
delete state.prepared[id];
|
|
137
|
+
state.committed[id] = committed;
|
|
138
|
+
}
|
|
139
|
+
return state;
|
|
140
|
+
}
|
|
141
|
+
function destinationPath(root, destinationKey) {
|
|
142
|
+
if (!destinationKey.startsWith(SESSION_LOG_DEST_PREFIX))
|
|
143
|
+
return null;
|
|
144
|
+
const destinationRoot = path.resolve(root, "workspace", ".session-logs");
|
|
145
|
+
const destination = path.resolve(root, destinationKey);
|
|
146
|
+
return destination.startsWith(destinationRoot + path.sep) ? destination : null;
|
|
147
|
+
}
|
|
148
|
+
function fsyncDirectory(directory) {
|
|
149
|
+
let fd;
|
|
150
|
+
try {
|
|
151
|
+
fd = fs.openSync(directory, "r");
|
|
152
|
+
fs.fsyncSync(fd);
|
|
153
|
+
}
|
|
154
|
+
catch (error) {
|
|
155
|
+
const code = error.code;
|
|
156
|
+
// Windows and a few virtual filesystems cannot fsync a directory. The
|
|
157
|
+
// rename is still atomic there; leave the next state-store open to recover
|
|
158
|
+
// a crash window instead of turning a best-effort reindex into a failure.
|
|
159
|
+
if (code !== "EINVAL" && code !== "ENOTSUP" && code !== "EOPNOTSUPP" && code !== "EPERM") {
|
|
160
|
+
throw error;
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
finally {
|
|
164
|
+
if (fd !== undefined)
|
|
165
|
+
fs.closeSync(fd);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
function copyAndHash(source, temp) {
|
|
169
|
+
const sourceFd = fs.openSync(source, "r");
|
|
170
|
+
let tempFd;
|
|
171
|
+
const digest = crypto.createHash("sha256");
|
|
172
|
+
try {
|
|
173
|
+
tempFd = fs.openSync(temp, "wx", 0o600);
|
|
174
|
+
const buffer = Buffer.allocUnsafe(COPY_BUFFER_BYTES);
|
|
175
|
+
let position = 0;
|
|
176
|
+
while (true) {
|
|
177
|
+
const read = fs.readSync(sourceFd, buffer, 0, buffer.length, position);
|
|
178
|
+
if (read === 0)
|
|
179
|
+
break;
|
|
180
|
+
digest.update(buffer.subarray(0, read));
|
|
181
|
+
let written = 0;
|
|
182
|
+
while (written < read) {
|
|
183
|
+
written += fs.writeSync(tempFd, buffer, written, read - written);
|
|
184
|
+
}
|
|
185
|
+
position += read;
|
|
186
|
+
}
|
|
187
|
+
fs.fsyncSync(tempFd);
|
|
188
|
+
return digest.digest("hex");
|
|
189
|
+
}
|
|
190
|
+
finally {
|
|
191
|
+
if (tempFd !== undefined)
|
|
192
|
+
fs.closeSync(tempFd);
|
|
193
|
+
fs.closeSync(sourceFd);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
function safeRemove(file) {
|
|
197
|
+
try {
|
|
198
|
+
fs.unlinkSync(file);
|
|
199
|
+
}
|
|
200
|
+
catch (error) {
|
|
201
|
+
if (error.code !== "ENOENT")
|
|
202
|
+
throw error;
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
function tempNameFor(destination) {
|
|
206
|
+
return `.${path.basename(destination)}.hq-session-log-${process.pid}-${crypto.randomBytes(8).toString("hex")}.tmp`;
|
|
207
|
+
}
|
|
208
|
+
function tempExists(temp) {
|
|
209
|
+
try {
|
|
210
|
+
fs.lstatSync(temp);
|
|
211
|
+
return true;
|
|
212
|
+
}
|
|
213
|
+
catch (error) {
|
|
214
|
+
if (error.code === "ENOENT")
|
|
215
|
+
return false;
|
|
216
|
+
throw error;
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
function appendCaptureBatches(store, type, captures) {
|
|
220
|
+
for (let start = 0; start < captures.length; start += CAPTURE_BATCH_SIZE) {
|
|
221
|
+
store.append(type, { captures: captures.slice(start, start + CAPTURE_BATCH_SIZE) });
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
/**
|
|
225
|
+
* Batches the durable state transitions, while retaining the per-file temp
|
|
226
|
+
* protocol. A batch PREPARE is durable before any member rename and a batch
|
|
227
|
+
* COMMIT contains only files that were renamed (or metadata-only updates).
|
|
228
|
+
*/
|
|
229
|
+
class CaptureBatchWriter {
|
|
230
|
+
store;
|
|
231
|
+
summary;
|
|
232
|
+
prepared = [];
|
|
233
|
+
metadataOnly = [];
|
|
234
|
+
constructor(store, summary) {
|
|
235
|
+
this.store = store;
|
|
236
|
+
this.summary = summary;
|
|
237
|
+
}
|
|
238
|
+
queuePrepared(capture) {
|
|
239
|
+
this.prepared.push(capture);
|
|
240
|
+
this.flushIfFull();
|
|
241
|
+
}
|
|
242
|
+
queueMetadataOnly(capture) {
|
|
243
|
+
this.metadataOnly.push(capture);
|
|
244
|
+
this.flushIfFull();
|
|
245
|
+
}
|
|
246
|
+
flush() {
|
|
247
|
+
if (this.prepared.length === 0 && this.metadataOnly.length === 0)
|
|
248
|
+
return;
|
|
249
|
+
const prepared = this.prepared;
|
|
250
|
+
const metadataOnly = this.metadataOnly;
|
|
251
|
+
this.prepared = [];
|
|
252
|
+
this.metadataOnly = [];
|
|
253
|
+
const commits = [...metadataOnly];
|
|
254
|
+
let renamed = 0;
|
|
255
|
+
if (prepared.length > 0) {
|
|
256
|
+
// Once append() begins, it may have made PREPARE durable before a
|
|
257
|
+
// compaction maintenance failure reaches us. Do not remove these temps
|
|
258
|
+
// on an exception: recovery's temp probe must retain the only evidence
|
|
259
|
+
// that the destination has not been renamed yet.
|
|
260
|
+
this.store.append(PREPARE_RECORD, {
|
|
261
|
+
captures: prepared.map(({ id, capture }) => ({ id, capture })),
|
|
262
|
+
});
|
|
263
|
+
captureTestHook?.("after-prepare-batch", prepared[0].source);
|
|
264
|
+
for (const entry of prepared)
|
|
265
|
+
captureTestHook?.("after-prepare", entry.source);
|
|
266
|
+
}
|
|
267
|
+
// The actual per-file rename loop is kept outside the PREPARE append so a
|
|
268
|
+
// failing destination retains just its own prepared row and does not hold
|
|
269
|
+
// Claude, Codex, Grok, or the other entries in this batch hostage.
|
|
270
|
+
for (const entry of prepared) {
|
|
271
|
+
const temp = path.join(path.dirname(entry.destination), entry.capture.tempName);
|
|
272
|
+
try {
|
|
273
|
+
fs.renameSync(temp, entry.destination);
|
|
274
|
+
fsyncDirectory(path.dirname(entry.destination));
|
|
275
|
+
captureTestHook?.("after-rename", entry.source);
|
|
276
|
+
captureTestHook?.("before-commit", entry.source);
|
|
277
|
+
commits.push({ id: entry.id, capture: entry.capture });
|
|
278
|
+
renamed++;
|
|
279
|
+
}
|
|
280
|
+
catch (error) {
|
|
281
|
+
this.summary.deferred++;
|
|
282
|
+
warning(this.summary, `session-log recovery deferred '${entry.source}' (${error.message}); retaining its prepared copy.`);
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
if (commits.length > 0) {
|
|
286
|
+
appendCaptureBatches(this.store, COMMIT_RECORD, commits);
|
|
287
|
+
captureTestHook?.("after-commit-batch", prepared[0]?.source ?? "");
|
|
288
|
+
this.summary.copied += renamed;
|
|
289
|
+
this.summary.metadataOnly += metadataOnly.length;
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
flushIfFull() {
|
|
293
|
+
if (this.prepared.length + this.metadataOnly.length >= CAPTURE_BATCH_SIZE)
|
|
294
|
+
this.flush();
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
function recoverPrepared(root, store, summary) {
|
|
298
|
+
const recovered = [];
|
|
299
|
+
for (const [id, prepared] of Object.entries(store.getState().prepared)) {
|
|
300
|
+
const destination = destinationPath(root, prepared.destinationKey);
|
|
301
|
+
if (!destination || path.basename(prepared.tempName) !== prepared.tempName) {
|
|
302
|
+
throw new StateStoreCorruptionError("prepared session-log capture escaped its destination");
|
|
303
|
+
}
|
|
304
|
+
const temp = path.join(path.dirname(destination), prepared.tempName);
|
|
305
|
+
// The temp's presence is the sole recovery probe. If it is absent, the
|
|
306
|
+
// previous process either renamed it already or removed it after a rename;
|
|
307
|
+
// do NOT inspect/recreate the final destination.
|
|
308
|
+
try {
|
|
309
|
+
if (tempExists(temp)) {
|
|
310
|
+
fs.renameSync(temp, destination);
|
|
311
|
+
fsyncDirectory(path.dirname(destination));
|
|
312
|
+
}
|
|
313
|
+
recovered.push({ id, capture: prepared });
|
|
314
|
+
}
|
|
315
|
+
catch (error) {
|
|
316
|
+
summary.deferred++;
|
|
317
|
+
warning(summary, `session-log recovery deferred '${prepared.destinationKey}' (${error.message}); retaining its prepared copy.`);
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
appendCaptureBatches(store, COMMIT_RECORD, recovered);
|
|
321
|
+
summary.copied += recovered.length;
|
|
322
|
+
}
|
|
323
|
+
function captureOne(root, store, harness, source, destinationKey, summary, batches) {
|
|
324
|
+
const destination = destinationPath(root, destinationKey);
|
|
325
|
+
if (!destination)
|
|
326
|
+
throw new Error(`invalid session-log destination ${destinationKey}`);
|
|
327
|
+
const canonicalSource = canonicalRoot(source);
|
|
328
|
+
const id = captureId(harness, canonicalSource, destinationKey);
|
|
329
|
+
const before = sourceSignature(canonicalSource);
|
|
330
|
+
if (!before)
|
|
331
|
+
return;
|
|
332
|
+
if (store.getState().prepared[id]) {
|
|
333
|
+
summary.deferred++;
|
|
334
|
+
return;
|
|
335
|
+
}
|
|
336
|
+
const previous = store.getState().committed[id];
|
|
337
|
+
if (previous && signaturesEqual(before, previous.signature)) {
|
|
338
|
+
summary.unchanged++;
|
|
339
|
+
return;
|
|
340
|
+
}
|
|
341
|
+
for (let attempt = 0; attempt < COPY_ATTEMPTS; attempt++) {
|
|
342
|
+
const currentBefore = sourceSignature(canonicalSource);
|
|
343
|
+
if (!currentBefore)
|
|
344
|
+
return;
|
|
345
|
+
// A peer may have committed this source while we waited on the dedicated
|
|
346
|
+
// capture lock. Re-check state, never the destination.
|
|
347
|
+
const committed = store.getState().committed[id];
|
|
348
|
+
if (committed && signaturesEqual(currentBefore, committed.signature)) {
|
|
349
|
+
summary.unchanged++;
|
|
350
|
+
return;
|
|
351
|
+
}
|
|
352
|
+
if (store.getState().prepared[id]) {
|
|
353
|
+
summary.deferred++;
|
|
354
|
+
return;
|
|
355
|
+
}
|
|
356
|
+
fs.mkdirSync(path.dirname(destination), { recursive: true });
|
|
357
|
+
const tempName = tempNameFor(destination);
|
|
358
|
+
const temp = path.join(path.dirname(destination), tempName);
|
|
359
|
+
let retainedForBatch = false;
|
|
360
|
+
try {
|
|
361
|
+
const contentHash = copyAndHash(canonicalSource, temp);
|
|
362
|
+
captureTestHook?.("after-copy", canonicalSource);
|
|
363
|
+
const after = sourceSignature(canonicalSource);
|
|
364
|
+
if (!after || !signaturesEqual(currentBefore, after)) {
|
|
365
|
+
safeRemove(temp);
|
|
366
|
+
continue;
|
|
367
|
+
}
|
|
368
|
+
if (committed && contentHash === committed.hash) {
|
|
369
|
+
safeRemove(temp);
|
|
370
|
+
const metadataOnly = {
|
|
371
|
+
destinationKey,
|
|
372
|
+
hash: contentHash,
|
|
373
|
+
signature: after,
|
|
374
|
+
tempName,
|
|
375
|
+
};
|
|
376
|
+
batches.queueMetadataOnly({ id, capture: metadataOnly });
|
|
377
|
+
return;
|
|
378
|
+
}
|
|
379
|
+
const prepared = {
|
|
380
|
+
destinationKey,
|
|
381
|
+
hash: contentHash,
|
|
382
|
+
signature: after,
|
|
383
|
+
tempName,
|
|
384
|
+
};
|
|
385
|
+
retainedForBatch = true;
|
|
386
|
+
batches.queuePrepared({ id, capture: prepared, source: canonicalSource, destination });
|
|
387
|
+
return;
|
|
388
|
+
}
|
|
389
|
+
finally {
|
|
390
|
+
if (!retainedForBatch) {
|
|
391
|
+
try {
|
|
392
|
+
safeRemove(temp);
|
|
393
|
+
}
|
|
394
|
+
catch {
|
|
395
|
+
// A failed copy remains best-effort. Prepared rows intentionally keep
|
|
396
|
+
// their temp for recovery; this only clears pre-prepare debris.
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
summary.deferred++;
|
|
402
|
+
warning(summary, `session log '${source}' changed while being captured; deferring.`);
|
|
403
|
+
}
|
|
404
|
+
function rootCandidates(root) {
|
|
405
|
+
const canonical = canonicalRoot(root);
|
|
406
|
+
return canonical === root ? [root] : [root, canonical];
|
|
407
|
+
}
|
|
408
|
+
function entries(directory) {
|
|
409
|
+
try {
|
|
410
|
+
return fs.readdirSync(directory).filter((entry) => !entry.startsWith(".")).sort();
|
|
411
|
+
}
|
|
412
|
+
catch {
|
|
413
|
+
return [];
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
function walkRegularFiles(directory) {
|
|
417
|
+
const files = [];
|
|
418
|
+
const stack = [directory];
|
|
419
|
+
while (stack.length > 0) {
|
|
420
|
+
const current = stack.pop();
|
|
421
|
+
let children;
|
|
422
|
+
try {
|
|
423
|
+
// Harness root selection has shell-glob semantics and deliberately omits
|
|
424
|
+
// dot names. Descendant traversal is different: the historic capture
|
|
425
|
+
// walked raw directory entries, including hidden transcript files.
|
|
426
|
+
children = fs.readdirSync(current).sort();
|
|
427
|
+
}
|
|
428
|
+
catch {
|
|
429
|
+
continue;
|
|
430
|
+
}
|
|
431
|
+
for (const entry of children) {
|
|
432
|
+
const absolute = path.join(current, entry);
|
|
433
|
+
try {
|
|
434
|
+
const stat = fs.lstatSync(absolute);
|
|
435
|
+
if (stat.isDirectory())
|
|
436
|
+
stack.push(absolute);
|
|
437
|
+
else if (stat.isFile())
|
|
438
|
+
files.push(absolute);
|
|
439
|
+
}
|
|
440
|
+
catch {
|
|
441
|
+
// Source discovery is best-effort, matching historic reindex behavior.
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
return files;
|
|
446
|
+
}
|
|
447
|
+
function firstLine(file, cap = 4 * 1024 * 1024) {
|
|
448
|
+
let fd;
|
|
449
|
+
try {
|
|
450
|
+
fd = fs.openSync(file, "r");
|
|
451
|
+
const chunks = [];
|
|
452
|
+
const chunk = Buffer.allocUnsafe(64 * 1024);
|
|
453
|
+
let position = 0;
|
|
454
|
+
while (position < cap) {
|
|
455
|
+
const read = fs.readSync(fd, chunk, 0, Math.min(chunk.length, cap - position), position);
|
|
456
|
+
if (read === 0)
|
|
457
|
+
break;
|
|
458
|
+
const newline = chunk.indexOf(0x0a, 0);
|
|
459
|
+
if (newline !== -1 && newline < read) {
|
|
460
|
+
chunks.push(Buffer.from(chunk.subarray(0, newline)));
|
|
461
|
+
return Buffer.concat(chunks).toString("utf8");
|
|
462
|
+
}
|
|
463
|
+
chunks.push(Buffer.from(chunk.subarray(0, read)));
|
|
464
|
+
position += read;
|
|
465
|
+
}
|
|
466
|
+
return chunks.length > 0 ? Buffer.concat(chunks).toString("utf8") : null;
|
|
467
|
+
}
|
|
468
|
+
catch {
|
|
469
|
+
return null;
|
|
470
|
+
}
|
|
471
|
+
finally {
|
|
472
|
+
if (fd !== undefined)
|
|
473
|
+
fs.closeSync(fd);
|
|
474
|
+
}
|
|
475
|
+
}
|
|
476
|
+
function codexCwd(file) {
|
|
477
|
+
const line = firstLine(file);
|
|
478
|
+
if (line === null)
|
|
479
|
+
return null;
|
|
480
|
+
try {
|
|
481
|
+
const parsed = JSON.parse(line);
|
|
482
|
+
const cwd = parsed.payload?.cwd ?? parsed.cwd;
|
|
483
|
+
return typeof cwd === "string" ? cwd : null;
|
|
484
|
+
}
|
|
485
|
+
catch {
|
|
486
|
+
return null;
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
function captureClaude(root, store, summary, candidateRoots) {
|
|
490
|
+
const projects = path.join(process.env.CLAUDE_CONFIG_DIR || path.join(os.homedir(), ".claude"), "projects");
|
|
491
|
+
if (!isDirectory(projects))
|
|
492
|
+
return;
|
|
493
|
+
const slugs = new Set(candidateRoots.map((candidate) => candidate.replace(/[^a-zA-Z0-9]/g, "-")));
|
|
494
|
+
const batches = new CaptureBatchWriter(store, summary);
|
|
495
|
+
try {
|
|
496
|
+
for (const project of entries(projects)) {
|
|
497
|
+
if (![...slugs].some((slug) => project === slug || project.startsWith(`${slug}-`)))
|
|
498
|
+
continue;
|
|
499
|
+
const sourceRoot = path.join(projects, project);
|
|
500
|
+
if (!isDirectory(sourceRoot))
|
|
501
|
+
continue;
|
|
502
|
+
for (const source of walkRegularFiles(sourceRoot)) {
|
|
503
|
+
const relative = path.relative(sourceRoot, source).split(path.sep).join("/");
|
|
504
|
+
captureOne(root, store, "claude", source, `${SESSION_LOG_DEST_PREFIX}claude/${project}/${relative}`, summary, batches);
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
finally {
|
|
509
|
+
batches.flush();
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
function captureGrok(root, store, summary, candidateRoots) {
|
|
513
|
+
const sessions = path.join(process.env.GROK_HOME || path.join(os.homedir(), ".grok"), "sessions");
|
|
514
|
+
if (!isDirectory(sessions))
|
|
515
|
+
return;
|
|
516
|
+
const keys = new Set(candidateRoots.map((candidate) => encodeURIComponent(candidate)));
|
|
517
|
+
const batches = new CaptureBatchWriter(store, summary);
|
|
518
|
+
try {
|
|
519
|
+
for (const session of entries(sessions)) {
|
|
520
|
+
if (![...keys].some((key) => session === key || session.startsWith(`${key}%2F`)))
|
|
521
|
+
continue;
|
|
522
|
+
const sourceRoot = path.join(sessions, session);
|
|
523
|
+
if (!isDirectory(sourceRoot))
|
|
524
|
+
continue;
|
|
525
|
+
for (const source of walkRegularFiles(sourceRoot)) {
|
|
526
|
+
const relative = path.relative(sourceRoot, source).split(path.sep).join("/");
|
|
527
|
+
captureOne(root, store, "grok", source, `${SESSION_LOG_DEST_PREFIX}grok/${session}/${relative}`, summary, batches);
|
|
528
|
+
}
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
finally {
|
|
532
|
+
batches.flush();
|
|
533
|
+
}
|
|
534
|
+
}
|
|
535
|
+
function captureCodex(root, store, summary, candidates) {
|
|
536
|
+
const sessions = path.join(process.env.CODEX_HOME || path.join(os.homedir(), ".codex"), "sessions");
|
|
537
|
+
if (!isDirectory(sessions))
|
|
538
|
+
return;
|
|
539
|
+
const prefixes = candidates.map((candidate) => candidate.endsWith(path.sep) ? candidate : `${candidate}${path.sep}`);
|
|
540
|
+
const batches = new CaptureBatchWriter(store, summary);
|
|
541
|
+
try {
|
|
542
|
+
for (const source of walkRegularFiles(sessions)) {
|
|
543
|
+
if (!source.endsWith(".jsonl"))
|
|
544
|
+
continue;
|
|
545
|
+
const relative = path.relative(sessions, source).split(path.sep).join("/");
|
|
546
|
+
const destinationKey = `${SESSION_LOG_DEST_PREFIX}codex/${relative}`;
|
|
547
|
+
const sourceId = captureId("codex", canonicalRoot(source), destinationKey);
|
|
548
|
+
const signature = sourceSignature(source);
|
|
549
|
+
if (!signature)
|
|
550
|
+
continue;
|
|
551
|
+
const committed = store.getState().committed[sourceId];
|
|
552
|
+
// This fast path happens before parsing the rollout and never consults the
|
|
553
|
+
// destination. New/changed rollouts alone need their metadata read to
|
|
554
|
+
// prove they belong to this HQ tree.
|
|
555
|
+
if (committed && signaturesEqual(signature, committed.signature)) {
|
|
556
|
+
summary.unchanged++;
|
|
557
|
+
continue;
|
|
558
|
+
}
|
|
559
|
+
const cwd = codexCwd(source);
|
|
560
|
+
if (cwd === null || (!candidates.includes(cwd) && !prefixes.some((prefix) => cwd.startsWith(prefix))))
|
|
561
|
+
continue;
|
|
562
|
+
captureOne(root, store, "codex", source, destinationKey, summary, batches);
|
|
563
|
+
}
|
|
564
|
+
}
|
|
565
|
+
finally {
|
|
566
|
+
batches.flush();
|
|
567
|
+
}
|
|
568
|
+
}
|
|
569
|
+
function quarantineMarker(stateDir, scope) {
|
|
570
|
+
return path.join(stateDir, "session-log-capture-quarantine", `${hash(scope)}.json`);
|
|
571
|
+
}
|
|
572
|
+
function quarantineCorruptStore(stateDir, scope, error) {
|
|
573
|
+
const marker = quarantineMarker(stateDir, scope);
|
|
574
|
+
fs.mkdirSync(path.dirname(marker), { recursive: true });
|
|
575
|
+
const statePath = path.join(stateDir, "sync-state-v3", hash(scope));
|
|
576
|
+
try {
|
|
577
|
+
if (fs.existsSync(statePath)) {
|
|
578
|
+
fs.renameSync(statePath, `${statePath}.corrupt-${Date.now()}-${process.pid}`);
|
|
579
|
+
}
|
|
580
|
+
}
|
|
581
|
+
catch {
|
|
582
|
+
// The marker remains the important fail-closed guard even if the damaged
|
|
583
|
+
// store cannot be moved (for example an administrator-owned state dir).
|
|
584
|
+
}
|
|
585
|
+
fs.writeFileSync(marker, JSON.stringify({ scope, error: String(error), at: new Date().toISOString() }) + "\n", { mode: 0o600 });
|
|
586
|
+
return marker;
|
|
587
|
+
}
|
|
588
|
+
/**
|
|
589
|
+
* Capture supported harness logs for one HQ root. This operation is deliberately
|
|
590
|
+
* best-effort to retain reindex's historic contract, except a corrupt capture
|
|
591
|
+
* store refuses the pass rather than silently re-copying every transcript.
|
|
592
|
+
*/
|
|
593
|
+
export function captureSessionLogs(root) {
|
|
594
|
+
const summary = { copied: 0, unchanged: 0, metadataOnly: 0, deferred: 0, warnings: [] };
|
|
595
|
+
const candidateRoots = rootCandidates(path.resolve(root));
|
|
596
|
+
const canonical = canonicalRoot(root);
|
|
597
|
+
const scope = captureScope(canonical);
|
|
598
|
+
const stateDir = getStateDir();
|
|
599
|
+
const marker = quarantineMarker(stateDir, scope);
|
|
600
|
+
if (fs.existsSync(marker)) {
|
|
601
|
+
warning(summary, `session-log capture state is quarantined (${marker}); refusing to recopy logs.`);
|
|
602
|
+
return summary;
|
|
603
|
+
}
|
|
604
|
+
let lock;
|
|
605
|
+
try {
|
|
606
|
+
lock = acquireOperationLock(canonical, "session-log-capture", { scope: "session-log-capture", timeoutSec: 0 });
|
|
607
|
+
}
|
|
608
|
+
catch (error) {
|
|
609
|
+
if (error instanceof OperationLockedError || error instanceof OperationLockUnwritableError) {
|
|
610
|
+
warning(summary, `session-log capture skipped (${error.message}).`);
|
|
611
|
+
return summary;
|
|
612
|
+
}
|
|
613
|
+
throw error;
|
|
614
|
+
}
|
|
615
|
+
try {
|
|
616
|
+
const store = StateStore.open({
|
|
617
|
+
rootDir: stateDir,
|
|
618
|
+
scopeId: scope,
|
|
619
|
+
initialState: { committed: {}, prepared: {} },
|
|
620
|
+
reduce: reduceCaptureState,
|
|
621
|
+
validateRecord: validateCaptureRecord,
|
|
622
|
+
maxRecordsBeforeCompaction: captureTestMaxRecordsBeforeCompaction ?? CAPTURE_COMPACTION_RECORDS,
|
|
623
|
+
});
|
|
624
|
+
recoverPrepared(canonical, store, summary);
|
|
625
|
+
for (const capture of [captureClaude, captureCodex, captureGrok]) {
|
|
626
|
+
try {
|
|
627
|
+
capture(canonical, store, summary, candidateRoots);
|
|
628
|
+
}
|
|
629
|
+
catch (error) {
|
|
630
|
+
warning(summary, `session-log capture step failed (${error.message}); skipping.`);
|
|
631
|
+
}
|
|
632
|
+
}
|
|
633
|
+
return summary;
|
|
634
|
+
}
|
|
635
|
+
catch (error) {
|
|
636
|
+
if (error instanceof StateStoreCorruptionError) {
|
|
637
|
+
const quarantined = quarantineCorruptStore(stateDir, scope, error);
|
|
638
|
+
warning(summary, `session-log capture state is corrupt and was quarantined at ${quarantined}; refusing to recopy logs.`);
|
|
639
|
+
return summary;
|
|
640
|
+
}
|
|
641
|
+
warning(summary, `session-log capture state could not be used (${error.message}); skipping.`);
|
|
642
|
+
return summary;
|
|
643
|
+
}
|
|
644
|
+
finally {
|
|
645
|
+
lock.release();
|
|
646
|
+
}
|
|
647
|
+
}
|
|
648
|
+
//# sourceMappingURL=session-log-capture.js.map
|