@turns/dev 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +74 -0
- package/dist/agent-anchor-client.js +319 -0
- package/dist/agent-anchor.js +1685 -0
- package/dist/auth.js +258 -0
- package/dist/daemon.js +134 -0
- package/dist/deepseek-trajectories.js +196 -0
- package/dist/directory-policy.js +41 -0
- package/dist/file-browsing.js +241 -0
- package/dist/git-association.js +136 -0
- package/dist/harness-registry.js +323 -0
- package/dist/index.js +300 -0
- package/dist/local-session-sources.js +1448 -0
- package/dist/native-record-compaction.js +84 -0
- package/dist/path-resolution.js +298 -0
- package/dist/question-mcp.js +126 -0
- package/dist/queue-worker.js +1053 -0
- package/dist/secret-redaction.js +181 -0
- package/dist/upload-sessions.js +2016 -0
- package/package.json +35 -0
|
@@ -0,0 +1,2016 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import crypto from 'node:crypto';
|
|
3
|
+
import fs from 'node:fs';
|
|
4
|
+
import os from 'node:os';
|
|
5
|
+
import path from 'node:path';
|
|
6
|
+
import process from 'node:process';
|
|
7
|
+
import { fileURLToPath } from 'node:url';
|
|
8
|
+
import WebSocket from 'ws';
|
|
9
|
+
import { createPathResolver, pathCandidates } from "./path-resolution.js";
|
|
10
|
+
import { streamDeepseekTrajectoryLines } from "./deepseek-trajectories.js";
|
|
11
|
+
import { normalizeAllowedDirectories, workingDirectoryIsAllowed } from "./directory-policy.js";
|
|
12
|
+
import { discoverGitAssociation } from "./git-association.js";
|
|
13
|
+
import { discoverLocalSessions, externallyManagedNativeDescriptorKeys, nativeDescriptorAgentSessionId, nativeDescriptorLineage, nativeDescriptorWorkingDirectory, nativeSessionShallowWatchRoots, nativeSessionWatchRoots, } from "./local-session-sources.js";
|
|
14
|
+
import { compactNativeRaw } from "./native-record-compaction.js";
|
|
15
|
+
import { httpBaseUrl, watchQueuedPrompts } from "./queue-worker.js";
|
|
16
|
+
import { createSecretRedactor, REDACTION_VERSION } from "./secret-redaction.js";
|
|
17
|
+
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
|
18
|
+
const DEFAULT_RUNS = path.resolve(HERE, '../scaffolding/scenario/runs');
|
|
19
|
+
// Keep a single User DO transaction short enough for multiplexed queue,
|
|
20
|
+
// presence, and ingestion requests to interleave during archive replay.
|
|
21
|
+
const MAX_BATCH_BYTES = 256_000;
|
|
22
|
+
const MAX_BATCH_ROWS = 100;
|
|
23
|
+
/** Rows the server accepts in one runtime append. */
|
|
24
|
+
const MAX_RUNTIME_EVENT_BATCH = 250;
|
|
25
|
+
const RPC_TIMEOUT_MS = 5 * 60_000;
|
|
26
|
+
const MAX_SOCKET_RETRIES = 8;
|
|
27
|
+
const MAX_NATIVE_SETTLE_PASSES = 4;
|
|
28
|
+
const MAX_EXTERNAL_STATUS_BATCH = 1_000;
|
|
29
|
+
const NATIVE_STATE_VERSION = 1;
|
|
30
|
+
const NATIVE_DESCRIPTOR_VERSION = 3;
|
|
31
|
+
const NATIVE_DISCOVERY_INTERVAL_MS = Math.max(30_000, Number(process.env.TURNS_NATIVE_DISCOVERY_INTERVAL_MS) || 5 * 60_000);
|
|
32
|
+
const NATIVE_POLL_INTERVAL_MS = Math.max(10_000, Number(process.env.TURNS_NATIVE_POLL_INTERVAL_MS) || 30_000);
|
|
33
|
+
const NATIVE_WATCH_INTERVAL_MS = Math.max(1_000, Number(process.env.TURNS_NATIVE_WATCH_INTERVAL_MS) || 10_000);
|
|
34
|
+
const NATIVE_CHANGE_DEBOUNCE_MS = 100;
|
|
35
|
+
const NATIVE_DISCOVERY_DEBOUNCE_MS = 250;
|
|
36
|
+
// Historical redaction replays are deliberately cooperative: one bounded DO
|
|
37
|
+
// transaction, then a pause and another scheduler pass. A file that changes
|
|
38
|
+
// during backfill is promoted and receives a larger quantum immediately.
|
|
39
|
+
const HISTORICAL_REPLAY_BATCHES_PER_SLICE = 1;
|
|
40
|
+
const ACTIVE_REPLAY_BATCHES_PER_SLICE = 4;
|
|
41
|
+
const RECENT_SESSION_PRIORITY_MS = 5 * 60_000;
|
|
42
|
+
const HISTORICAL_REPLAY_DELAY_MS = Math.max(100, Number(process.env.TURNS_HISTORICAL_REPLAY_DELAY_MS) || 1_500);
|
|
43
|
+
const FAILURE_RETRY_BASE_MS = 5_000;
|
|
44
|
+
const FAILURE_RETRY_MAX_MS = 5 * 60_000;
|
|
45
|
+
const DEFAULT_REDACTOR = createSecretRedactor();
|
|
46
|
+
const TIME_FIELDS = [
|
|
47
|
+
'ts',
|
|
48
|
+
'timestamp',
|
|
49
|
+
'time',
|
|
50
|
+
'created_at',
|
|
51
|
+
'createdAt',
|
|
52
|
+
'created_timestamp',
|
|
53
|
+
'time_created',
|
|
54
|
+
'recorded_at',
|
|
55
|
+
'occurred_at_ms',
|
|
56
|
+
];
|
|
57
|
+
const TIME_CONTAINERS = ['data', 'payload', 'message', 'chat_message', 'node'];
|
|
58
|
+
function epochToIso(value) {
|
|
59
|
+
const milliseconds = value > 1e14 ? value / 1000 : value > 1e11 ? value : value > 1e9 ? value * 1000 : NaN;
|
|
60
|
+
if (!Number.isFinite(milliseconds))
|
|
61
|
+
return undefined;
|
|
62
|
+
const date = new Date(milliseconds);
|
|
63
|
+
return Number.isNaN(+date) ? undefined : date.toISOString();
|
|
64
|
+
}
|
|
65
|
+
/** Extract the source record's creation time without importing the reducer. */
|
|
66
|
+
export function documentTime(document, depth = 0) {
|
|
67
|
+
if (!document || typeof document !== 'object' || Array.isArray(document))
|
|
68
|
+
return undefined;
|
|
69
|
+
for (const field of TIME_FIELDS) {
|
|
70
|
+
const value = document[field];
|
|
71
|
+
if (typeof value === 'number') {
|
|
72
|
+
const iso = epochToIso(value);
|
|
73
|
+
if (iso)
|
|
74
|
+
return iso;
|
|
75
|
+
}
|
|
76
|
+
if (typeof value === 'string' && /^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}/u.test(value)) {
|
|
77
|
+
const zoned = /(Z|[+-]\d{2}:?\d{2})$/u.test(value) ? value : `${value.replace(' ', 'T')}Z`;
|
|
78
|
+
const date = new Date(zoned);
|
|
79
|
+
if (!Number.isNaN(+date))
|
|
80
|
+
return date.toISOString();
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
if (depth >= 2)
|
|
84
|
+
return undefined;
|
|
85
|
+
for (const field of TIME_CONTAINERS) {
|
|
86
|
+
const nested = document[field];
|
|
87
|
+
let parsed = nested;
|
|
88
|
+
if (typeof nested === 'string') {
|
|
89
|
+
try {
|
|
90
|
+
parsed = JSON.parse(nested);
|
|
91
|
+
}
|
|
92
|
+
catch {
|
|
93
|
+
parsed = null;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
const found = documentTime(parsed, depth + 1);
|
|
97
|
+
if (found)
|
|
98
|
+
return found;
|
|
99
|
+
}
|
|
100
|
+
return undefined;
|
|
101
|
+
}
|
|
102
|
+
function socketError(message, cause) {
|
|
103
|
+
return Object.assign(new Error(message, cause ? { cause } : undefined), { code: 'SOCKET_CLOSED' });
|
|
104
|
+
}
|
|
105
|
+
function authenticationError(message) {
|
|
106
|
+
return Object.assign(new Error(message), { code: 'AUTH_REJECTED' });
|
|
107
|
+
}
|
|
108
|
+
export function isSocketError(error) {
|
|
109
|
+
return error?.code === 'SOCKET_CLOSED' || error?.code === 'SOCKET_TIMEOUT';
|
|
110
|
+
}
|
|
111
|
+
function reconnectDelay(attempt) {
|
|
112
|
+
return Math.min(15_000, 750 * 2 ** Math.min(attempt, 5)) + Math.floor(Math.random() * 250);
|
|
113
|
+
}
|
|
114
|
+
function wait(milliseconds) {
|
|
115
|
+
return new Promise((resolve) => setTimeout(resolve, milliseconds));
|
|
116
|
+
}
|
|
117
|
+
function reportingRedactor(tokens) {
|
|
118
|
+
const counts = new Map();
|
|
119
|
+
const redactor = createSecretRedactor({
|
|
120
|
+
literals: [...(Array.isArray(tokens) ? tokens : [tokens]), process.env.TURNS_UPLOAD_TOKEN].filter(Boolean),
|
|
121
|
+
onFinding: ({ rule, count }) => counts.set(rule, (counts.get(rule) || 0) + count),
|
|
122
|
+
});
|
|
123
|
+
const report = () => {
|
|
124
|
+
if (!counts.size)
|
|
125
|
+
return;
|
|
126
|
+
const rules = Object.fromEntries([...counts].sort((left, right) => right[1] - left[1]));
|
|
127
|
+
counts.clear();
|
|
128
|
+
process.stderr.write(`Secret filtering applied (v${REDACTION_VERSION}): ${JSON.stringify(rules)}\n`);
|
|
129
|
+
};
|
|
130
|
+
const timer = setInterval(report, 60_000);
|
|
131
|
+
timer.unref?.();
|
|
132
|
+
return {
|
|
133
|
+
redactor,
|
|
134
|
+
close() {
|
|
135
|
+
clearInterval(timer);
|
|
136
|
+
report();
|
|
137
|
+
},
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
function argumentsFrom(argv, { allowMissingToken = false } = {}) {
|
|
141
|
+
const options = {
|
|
142
|
+
url: process.env.TURNS_URL || 'wss://turns.dev/api/socket',
|
|
143
|
+
token: process.env.TURNS_UPLOAD_TOKEN || '',
|
|
144
|
+
runs: DEFAULT_RUNS,
|
|
145
|
+
native: false,
|
|
146
|
+
dryRun: false,
|
|
147
|
+
watch: false,
|
|
148
|
+
only: '',
|
|
149
|
+
rewriteSession: '',
|
|
150
|
+
nativeStateFile: process.env.TURNS_NATIVE_STATE_FILE || path.join(HERE, '.native-upload-state.json'),
|
|
151
|
+
machineIdFile: process.env.TURNS_MACHINE_ID_FILE || path.join(HERE, '.machine-id'),
|
|
152
|
+
directory: '',
|
|
153
|
+
all: false,
|
|
154
|
+
devUser: '',
|
|
155
|
+
allowedDirectories: [],
|
|
156
|
+
};
|
|
157
|
+
for (let index = 0; index < argv.length; index += 1) {
|
|
158
|
+
const argument = argv[index];
|
|
159
|
+
const next = () => {
|
|
160
|
+
const value = argv[++index];
|
|
161
|
+
if (!value)
|
|
162
|
+
throw new Error(`${argument} requires a value`);
|
|
163
|
+
return value;
|
|
164
|
+
};
|
|
165
|
+
if (argument === '--url')
|
|
166
|
+
options.url = next();
|
|
167
|
+
else if (argument === '--token')
|
|
168
|
+
options.token = next();
|
|
169
|
+
else if (argument === '--runs')
|
|
170
|
+
options.runs = path.resolve(next());
|
|
171
|
+
else if (argument === '--native')
|
|
172
|
+
options.native = true;
|
|
173
|
+
else if (argument === '--dry-run')
|
|
174
|
+
options.dryRun = true;
|
|
175
|
+
else if (argument === '--only')
|
|
176
|
+
options.only = next();
|
|
177
|
+
else if (argument === '--rewrite-session')
|
|
178
|
+
options.rewriteSession = next();
|
|
179
|
+
else if (argument === '--native-state-file')
|
|
180
|
+
options.nativeStateFile = path.resolve(next());
|
|
181
|
+
else if (argument === '--machine-id-file')
|
|
182
|
+
options.machineIdFile = path.resolve(next());
|
|
183
|
+
else if (argument === '--directory')
|
|
184
|
+
options.directory = path.resolve(next());
|
|
185
|
+
else if (argument === '--all')
|
|
186
|
+
options.all = true;
|
|
187
|
+
else if (argument === '--dev-user')
|
|
188
|
+
options.devUser = next().trim().toLowerCase();
|
|
189
|
+
else if (argument === '--allow-directory')
|
|
190
|
+
options.allowedDirectories.push(next());
|
|
191
|
+
else if (argument === '--watch')
|
|
192
|
+
options.watch = true;
|
|
193
|
+
else if (argument === '--help') {
|
|
194
|
+
process.stdout.write(`Usage: node upload-sessions.ts [options]\n\n` +
|
|
195
|
+
` --watch keep rescanning and uploading changed sessions\n` +
|
|
196
|
+
` --native upload native harness session stores, not captures\n` +
|
|
197
|
+
` --rewrite-session ID reset and fully replay one exact native session\n` +
|
|
198
|
+
` --dry-run inventory native sessions without uploading\n` +
|
|
199
|
+
` --runs PATH capture root (default: ${DEFAULT_RUNS})\n` +
|
|
200
|
+
` --only TEXT upload paths containing TEXT\n` +
|
|
201
|
+
` --directory PATH upload sessions whose working directory is PATH\n` +
|
|
202
|
+
` --all upload sessions across all working directories\n` +
|
|
203
|
+
` --allow-directory P upload/launch only inside directory prefix P (repeatable)\n` +
|
|
204
|
+
` --dev-user LOGIN authenticate as a local Wrangler DEV_USER uploader\n` +
|
|
205
|
+
` --url WS_URL websocket endpoint\n` +
|
|
206
|
+
` --token TOKEN internal/test bearer token override\n` +
|
|
207
|
+
` --machine-id-file PATH persistent machine identity\n` +
|
|
208
|
+
` --native-state-file PATH persistent native upload cursor\n`);
|
|
209
|
+
process.exit(0);
|
|
210
|
+
}
|
|
211
|
+
else
|
|
212
|
+
throw new Error(`Unknown argument: ${argument}`);
|
|
213
|
+
}
|
|
214
|
+
if (options.rewriteSession && !options.native) {
|
|
215
|
+
throw new Error('--rewrite-session requires --native.');
|
|
216
|
+
}
|
|
217
|
+
if (options.directory && options.all)
|
|
218
|
+
throw new Error('--directory cannot be combined with --all.');
|
|
219
|
+
if (options.allowedDirectories.length && (options.directory || options.all)) {
|
|
220
|
+
throw new Error('--allow-directory cannot be combined with --directory or --all.');
|
|
221
|
+
}
|
|
222
|
+
options.allowedDirectories = normalizeAllowedDirectories(options.allowedDirectories);
|
|
223
|
+
if (!options.dryRun && !options.token && !options.devUser && !allowMissingToken) {
|
|
224
|
+
throw new Error('No uploader credential provider. Run uploads through the `turns` CLI.');
|
|
225
|
+
}
|
|
226
|
+
return options;
|
|
227
|
+
}
|
|
228
|
+
function machineId(file = path.join(HERE, '.machine-id')) {
|
|
229
|
+
if (fs.existsSync(file))
|
|
230
|
+
return fs.readFileSync(file, 'utf8').trim();
|
|
231
|
+
const id = crypto.randomUUID();
|
|
232
|
+
fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 });
|
|
233
|
+
fs.writeFileSync(file, `${id}\n`, { mode: 0o600 });
|
|
234
|
+
return id;
|
|
235
|
+
}
|
|
236
|
+
export function discover(root, only = '') {
|
|
237
|
+
const files = [];
|
|
238
|
+
for (const folder of ['', 'imports', 'archive']) {
|
|
239
|
+
const directory = path.join(root, folder);
|
|
240
|
+
if (!fs.existsSync(directory))
|
|
241
|
+
continue;
|
|
242
|
+
for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {
|
|
243
|
+
if (!entry.isFile() || !entry.name.endsWith('.jsonl'))
|
|
244
|
+
continue;
|
|
245
|
+
const file = path.join(directory, entry.name);
|
|
246
|
+
const relative = path.relative(root, file);
|
|
247
|
+
if (only && !relative.toLowerCase().includes(only.toLowerCase()))
|
|
248
|
+
continue;
|
|
249
|
+
files.push({ file, relative });
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
return files.sort((left, right) => left.relative.localeCompare(right.relative));
|
|
253
|
+
}
|
|
254
|
+
export function completeLines(file) {
|
|
255
|
+
const source = fs.readFileSync(file, 'utf8');
|
|
256
|
+
const lines = source.split('\n');
|
|
257
|
+
if (!source.endsWith('\n'))
|
|
258
|
+
lines.pop();
|
|
259
|
+
while (lines.at(-1) === '')
|
|
260
|
+
lines.pop();
|
|
261
|
+
const rows = [];
|
|
262
|
+
const raw = [];
|
|
263
|
+
for (const line of lines) {
|
|
264
|
+
try {
|
|
265
|
+
rows.push(JSON.parse(line));
|
|
266
|
+
raw.push(line);
|
|
267
|
+
}
|
|
268
|
+
catch {
|
|
269
|
+
break;
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
return { rows, raw };
|
|
273
|
+
}
|
|
274
|
+
function harnessOf(relative, rows) {
|
|
275
|
+
const fromRow = String(rows[0]?.harness || '').trim();
|
|
276
|
+
if (fromRow)
|
|
277
|
+
return fromRow;
|
|
278
|
+
return path.basename(relative, '.jsonl').split('__').at(-1) || 'unknown';
|
|
279
|
+
}
|
|
280
|
+
export function stableNativeSessionId(machine, harness, sourcePath) {
|
|
281
|
+
const fingerprint = crypto
|
|
282
|
+
.createHash('sha256')
|
|
283
|
+
.update(`${machine}\0native\0${harness}\0${sourcePath}`)
|
|
284
|
+
.digest('hex')
|
|
285
|
+
.slice(0, 24);
|
|
286
|
+
return `${harness}-native-${fingerprint}`.slice(0, 240);
|
|
287
|
+
}
|
|
288
|
+
/** Resolve harness-native parent keys to the machine-stable turns.dev IDs. */
|
|
289
|
+
export function linkNativeSessionParents(machine, descriptors) {
|
|
290
|
+
const records = descriptors.map((descriptor) => ({
|
|
291
|
+
descriptor,
|
|
292
|
+
lineage: nativeDescriptorLineage(descriptor),
|
|
293
|
+
}));
|
|
294
|
+
const byNativeId = new Map();
|
|
295
|
+
for (const record of records) {
|
|
296
|
+
if (!record.lineage.nativeSessionId)
|
|
297
|
+
continue;
|
|
298
|
+
const key = `${record.descriptor.harness}\0${record.lineage.nativeSessionId}`;
|
|
299
|
+
const candidates = byNativeId.get(key) || [];
|
|
300
|
+
candidates.push(record.descriptor);
|
|
301
|
+
byNativeId.set(key, candidates);
|
|
302
|
+
}
|
|
303
|
+
const storeNamespace = (descriptor) => {
|
|
304
|
+
const sourcePath = String(descriptor?.sourcePath || '').replaceAll('\\', '/');
|
|
305
|
+
if (descriptor?.harness === 'gemini')
|
|
306
|
+
return /^(.*?\/chats)(?:\/|$)/iu.exec(sourcePath)?.[1] || '';
|
|
307
|
+
return '';
|
|
308
|
+
};
|
|
309
|
+
return records.flatMap(({ descriptor, lineage }) => {
|
|
310
|
+
// Internal harness sidechains such as Claude's compaction worker are still
|
|
311
|
+
// children for identity/resume safety, but never become uploaded sessions.
|
|
312
|
+
if (lineage.uploadable === false)
|
|
313
|
+
return [];
|
|
314
|
+
const candidates = lineage.parentNativeSessionId
|
|
315
|
+
? byNativeId.get(`${descriptor.harness}\0${lineage.parentNativeSessionId}`) || []
|
|
316
|
+
: [];
|
|
317
|
+
const namespace = storeNamespace(descriptor);
|
|
318
|
+
const localCandidates = namespace
|
|
319
|
+
? candidates.filter((candidate) => storeNamespace(candidate) === namespace)
|
|
320
|
+
: candidates;
|
|
321
|
+
// Native IDs are not globally unique for every harness. Prefer the child's
|
|
322
|
+
// own store. Gemini can retain multiple timestamped files for one native
|
|
323
|
+
// session in that store; those aliases converge server-side, so use its
|
|
324
|
+
// newest file deterministically. Do not guess for unnamespaced collisions.
|
|
325
|
+
const parent = localCandidates.length === 1
|
|
326
|
+
? localCandidates[0]
|
|
327
|
+
: namespace && localCandidates.length > 1
|
|
328
|
+
? [...localCandidates].sort((left, right) => Number(right.mtimeMs || 0) - Number(left.mtimeMs || 0) ||
|
|
329
|
+
String(right.sourcePath).localeCompare(String(left.sourcePath)))[0]
|
|
330
|
+
: null;
|
|
331
|
+
// An exact structural parent path is stronger than a harness-wide native
|
|
332
|
+
// ID match. This matters for Claude descendants, whose short runtime agent
|
|
333
|
+
// IDs can recur in another project while the adjacent sidecar names the
|
|
334
|
+
// correct transcript in this session's flat subagents directory.
|
|
335
|
+
const parentSourcePath = lineage.parentSourcePath || parent?.sourcePath;
|
|
336
|
+
// An unresolved child cannot satisfy the sidebar contract. Leave it out of
|
|
337
|
+
// native upload until its store exposes enough lineage to nest it safely.
|
|
338
|
+
if (lineage.isSubagent && !parentSourcePath)
|
|
339
|
+
return [];
|
|
340
|
+
return [
|
|
341
|
+
{
|
|
342
|
+
...descriptor,
|
|
343
|
+
isSubagent: lineage.isSubagent,
|
|
344
|
+
...(lineage.nativeTitle ? { nativeTitle: lineage.nativeTitle } : {}),
|
|
345
|
+
...(lineage.nativeSubagentType ? { nativeSubagentType: lineage.nativeSubagentType } : {}),
|
|
346
|
+
...(lineage.nativeSubagentDepth ? { nativeSubagentDepth: lineage.nativeSubagentDepth } : {}),
|
|
347
|
+
...(parentSourcePath
|
|
348
|
+
? {
|
|
349
|
+
parentSessionId: stableNativeSessionId(machine, descriptor.harness, parentSourcePath),
|
|
350
|
+
}
|
|
351
|
+
: {}),
|
|
352
|
+
},
|
|
353
|
+
];
|
|
354
|
+
});
|
|
355
|
+
}
|
|
356
|
+
export function nativeDescriptorFingerprint(descriptor) {
|
|
357
|
+
return JSON.stringify([
|
|
358
|
+
Number(descriptor.size) || 0,
|
|
359
|
+
Number(descriptor.mtimeMs) || 0,
|
|
360
|
+
nativeDescriptorWorkingDirectory(descriptor),
|
|
361
|
+
String(descriptor.parentSessionId || ''),
|
|
362
|
+
NATIVE_DESCRIPTOR_VERSION,
|
|
363
|
+
nativeDescriptorMetadataFingerprint(descriptor),
|
|
364
|
+
]);
|
|
365
|
+
}
|
|
366
|
+
function nativeDescriptorMetadataFingerprint(descriptor) {
|
|
367
|
+
return JSON.stringify([
|
|
368
|
+
String(descriptor.nativeTitle || ''),
|
|
369
|
+
String(descriptor.nativeSubagentType || ''),
|
|
370
|
+
Number(descriptor.nativeSubagentDepth) || 0,
|
|
371
|
+
]);
|
|
372
|
+
}
|
|
373
|
+
function nativeDescriptorKey(descriptor) {
|
|
374
|
+
return `${descriptor.harness}\0${descriptor.sourcePath}`;
|
|
375
|
+
}
|
|
376
|
+
export function nativeExternalManagementStatuses(machine, descriptors, managedKeys, observedAt) {
|
|
377
|
+
return descriptors.map((descriptor) => ({
|
|
378
|
+
sessionId: stableNativeSessionId(machine, descriptor.harness, descriptor.sourcePath),
|
|
379
|
+
externallyManaged: managedKeys.has(nativeDescriptorKey(descriptor)),
|
|
380
|
+
observedAt,
|
|
381
|
+
}));
|
|
382
|
+
}
|
|
383
|
+
export function nativeTitle(descriptor) {
|
|
384
|
+
const fallback = path.basename(descriptor.sourcePath.split('#', 1)[0]);
|
|
385
|
+
const firstLine = String(descriptor.nativeTitle || fallback || `${descriptor.harness} session`)
|
|
386
|
+
.trim()
|
|
387
|
+
.split('\n')[0];
|
|
388
|
+
return firstLine.length > 120 ? `${firstLine.slice(0, 119)}…` : firstLine;
|
|
389
|
+
}
|
|
390
|
+
function nativeTimestamp(raw, descriptor, rowIndex) {
|
|
391
|
+
try {
|
|
392
|
+
const timestamp = documentTime(JSON.parse(raw));
|
|
393
|
+
if (timestamp)
|
|
394
|
+
return timestamp;
|
|
395
|
+
}
|
|
396
|
+
catch { }
|
|
397
|
+
const base = Date.parse(descriptor.createdAt) || descriptor.mtimeMs || Date.now();
|
|
398
|
+
return new Date(base + rowIndex).toISOString();
|
|
399
|
+
}
|
|
400
|
+
async function* completeNativeFileLines(file, startOffset, endOffset) {
|
|
401
|
+
if (endOffset < startOffset)
|
|
402
|
+
return;
|
|
403
|
+
let carry = Buffer.alloc(0);
|
|
404
|
+
let carryOffset = startOffset;
|
|
405
|
+
const input = fs.createReadStream(file, { start: startOffset, end: endOffset });
|
|
406
|
+
try {
|
|
407
|
+
for await (const chunk of input) {
|
|
408
|
+
const buffer = carry.length ? Buffer.concat([carry, chunk]) : chunk;
|
|
409
|
+
const bufferOffset = carryOffset;
|
|
410
|
+
let lineStart = 0;
|
|
411
|
+
for (;;) {
|
|
412
|
+
const newline = buffer.indexOf(0x0a, lineStart);
|
|
413
|
+
if (newline < 0)
|
|
414
|
+
break;
|
|
415
|
+
let line = buffer.subarray(lineStart, newline);
|
|
416
|
+
if (line.at(-1) === 0x0d)
|
|
417
|
+
line = line.subarray(0, -1);
|
|
418
|
+
yield {
|
|
419
|
+
raw: line.toString('utf8'),
|
|
420
|
+
endOffset: bufferOffset + newline + 1,
|
|
421
|
+
};
|
|
422
|
+
lineStart = newline + 1;
|
|
423
|
+
}
|
|
424
|
+
carry = buffer.subarray(lineStart);
|
|
425
|
+
carryOffset = bufferOffset + lineStart;
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
finally {
|
|
429
|
+
input.destroy();
|
|
430
|
+
}
|
|
431
|
+
}
|
|
432
|
+
async function* nativeSourceRecords(descriptor, cursor) {
|
|
433
|
+
if (descriptor.harness === 'deepseek') {
|
|
434
|
+
for await (const raw of streamDeepseekTrajectoryLines(descriptor.file)) {
|
|
435
|
+
yield { raw, endOffset: Number(descriptor.size) || 0 };
|
|
436
|
+
}
|
|
437
|
+
return;
|
|
438
|
+
}
|
|
439
|
+
if (descriptor.file.endsWith('.json')) {
|
|
440
|
+
let list = [];
|
|
441
|
+
try {
|
|
442
|
+
const parsed = JSON.parse(fs.readFileSync(descriptor.file, 'utf8'));
|
|
443
|
+
const candidate = Array.isArray(parsed) ? parsed : parsed?.messages || parsed?.conversation || [];
|
|
444
|
+
if (Array.isArray(candidate))
|
|
445
|
+
list = candidate;
|
|
446
|
+
}
|
|
447
|
+
catch { }
|
|
448
|
+
for (const value of list) {
|
|
449
|
+
yield { raw: JSON.stringify(value), endOffset: Number(descriptor.size) || 0 };
|
|
450
|
+
}
|
|
451
|
+
return;
|
|
452
|
+
}
|
|
453
|
+
yield* completeNativeFileLines(descriptor.file, Number(cursor.offset) || 0, Math.max(-1, (Number(descriptor.size) || 0) - 1));
|
|
454
|
+
}
|
|
455
|
+
/** Stream canonical raw events while retaining only one WebSocket batch. The
|
|
456
|
+
* mutable cursor is byte-accurate for append-only JSONL stores and lets the
|
|
457
|
+
* daemon resume at the first unseen source byte instead of reparsing history. */
|
|
458
|
+
export async function* nativeCaptureRowStream(descriptor, cursor = {}) {
|
|
459
|
+
let rowIndex = Number(cursor.rowCount) || 0;
|
|
460
|
+
let lineNumber = Number(cursor.lineNumber) || 0;
|
|
461
|
+
let offset = Number(cursor.offset) || 0;
|
|
462
|
+
for await (const record of nativeSourceRecords(descriptor, { offset })) {
|
|
463
|
+
lineNumber += 1;
|
|
464
|
+
offset = record.endOffset;
|
|
465
|
+
cursor.offset = offset;
|
|
466
|
+
cursor.lineNumber = lineNumber;
|
|
467
|
+
if (!record.raw)
|
|
468
|
+
continue;
|
|
469
|
+
const compacted = compactNativeRaw(record.raw);
|
|
470
|
+
if (descriptor.harness === 'deepseek') {
|
|
471
|
+
try {
|
|
472
|
+
const document = JSON.parse(compacted);
|
|
473
|
+
if (document?.type === 'session/title' && String(document?.data?.title || '').trim()) {
|
|
474
|
+
descriptor.nativeTitle = String(document.data.title).trim();
|
|
475
|
+
}
|
|
476
|
+
}
|
|
477
|
+
catch { }
|
|
478
|
+
}
|
|
479
|
+
const row = {
|
|
480
|
+
seq: rowIndex,
|
|
481
|
+
ts: nativeTimestamp(compacted, descriptor, lineNumber),
|
|
482
|
+
elapsedMs: rowIndex,
|
|
483
|
+
runId: 'native',
|
|
484
|
+
harness: descriptor.harness,
|
|
485
|
+
kind: 'transcript.line',
|
|
486
|
+
phase: 'native',
|
|
487
|
+
path: descriptor.sourcePath,
|
|
488
|
+
lineNumber,
|
|
489
|
+
mutation: 'append',
|
|
490
|
+
source: descriptor.sourcePath.includes('.db#') ? 'sqlite' : 'file',
|
|
491
|
+
scope: 'root',
|
|
492
|
+
value: compacted,
|
|
493
|
+
};
|
|
494
|
+
rowIndex += 1;
|
|
495
|
+
cursor.rowCount = rowIndex;
|
|
496
|
+
yield { row, raw: JSON.stringify(row), endOffset: offset, lineNumber, rowCount: rowIndex };
|
|
497
|
+
}
|
|
498
|
+
cursor.offset = offset;
|
|
499
|
+
cursor.lineNumber = lineNumber;
|
|
500
|
+
cursor.rowCount = rowIndex;
|
|
501
|
+
}
|
|
502
|
+
// Kept as a small compatibility helper for fixtures and one-shot callers.
|
|
503
|
+
export async function nativeCaptureRows(descriptor) {
|
|
504
|
+
const rows = [];
|
|
505
|
+
for await (const capture of nativeCaptureRowStream(descriptor, {}))
|
|
506
|
+
rows.push(capture.row);
|
|
507
|
+
return rows;
|
|
508
|
+
}
|
|
509
|
+
export function sessionIdOf(machine, relative, rows, harness) {
|
|
510
|
+
const runId = String(rows[0]?.runId || path.basename(relative, '.jsonl').split('__')[0] || 'session');
|
|
511
|
+
const fingerprint = crypto
|
|
512
|
+
.createHash('sha256')
|
|
513
|
+
.update(`${machine}\0${relative}\0${runId}\0${harness}`)
|
|
514
|
+
.digest('hex')
|
|
515
|
+
.slice(0, 24);
|
|
516
|
+
return `${harness}-${runId}-${fingerprint}`.slice(0, 240);
|
|
517
|
+
}
|
|
518
|
+
function titleOf(relative) {
|
|
519
|
+
const firstLine = path.basename(relative, '.jsonl');
|
|
520
|
+
return firstLine.length > 120 ? `${firstLine.slice(0, 119)}…` : firstLine;
|
|
521
|
+
}
|
|
522
|
+
export class RpcSocket {
|
|
523
|
+
pending;
|
|
524
|
+
listeners;
|
|
525
|
+
closed;
|
|
526
|
+
resolveClosed;
|
|
527
|
+
ready;
|
|
528
|
+
socket;
|
|
529
|
+
constructor(url, token, devUser = '') {
|
|
530
|
+
this.pending = new Map();
|
|
531
|
+
this.listeners = new Set();
|
|
532
|
+
this.closed = new Promise((resolve) => {
|
|
533
|
+
this.resolveClosed = resolve;
|
|
534
|
+
});
|
|
535
|
+
this.ready = new Promise((resolve, reject) => {
|
|
536
|
+
const headers = devUser
|
|
537
|
+
? { 'x-turns-dev-user': devUser, 'x-turns-dev-role': 'uploader' }
|
|
538
|
+
: { Authorization: `Bearer ${token}` };
|
|
539
|
+
this.socket = new WebSocket(url, { headers });
|
|
540
|
+
this.socket.once('unexpected-response', (_request, response) => {
|
|
541
|
+
response.resume?.();
|
|
542
|
+
if (response.statusCode === 401 || response.statusCode === 403) {
|
|
543
|
+
reject(authenticationError(`Turns rejected uploader authentication (HTTP ${response.statusCode}).`));
|
|
544
|
+
}
|
|
545
|
+
else {
|
|
546
|
+
reject(socketError(`Unexpected WebSocket response (HTTP ${response.statusCode}).`));
|
|
547
|
+
}
|
|
548
|
+
});
|
|
549
|
+
this.socket.once('error', (error) => reject(socketError(`WebSocket connection failed: ${error.message}`, error)));
|
|
550
|
+
this.socket.on('message', (data) => this.onMessage(String(data), resolve, reject));
|
|
551
|
+
this.socket.on('close', (code, reason) => {
|
|
552
|
+
const error = socketError(`WebSocket closed (${code}): ${String(reason)}`);
|
|
553
|
+
for (const request of this.pending.values()) {
|
|
554
|
+
clearTimeout(request.timer);
|
|
555
|
+
request.reject(error);
|
|
556
|
+
}
|
|
557
|
+
this.pending.clear();
|
|
558
|
+
this.resolveClosed?.({ code, reason: String(reason) });
|
|
559
|
+
});
|
|
560
|
+
});
|
|
561
|
+
}
|
|
562
|
+
onMessage(raw, resolveReady, rejectReady) {
|
|
563
|
+
let message;
|
|
564
|
+
try {
|
|
565
|
+
message = JSON.parse(raw);
|
|
566
|
+
}
|
|
567
|
+
catch {
|
|
568
|
+
return;
|
|
569
|
+
}
|
|
570
|
+
if (message.type === 'ready') {
|
|
571
|
+
if (message.role !== 'uploader')
|
|
572
|
+
rejectReady(new Error('Endpoint did not grant uploader access.'));
|
|
573
|
+
else
|
|
574
|
+
resolveReady(message);
|
|
575
|
+
return;
|
|
576
|
+
}
|
|
577
|
+
if (!message.requestId) {
|
|
578
|
+
for (const listener of this.listeners)
|
|
579
|
+
listener(message);
|
|
580
|
+
return;
|
|
581
|
+
}
|
|
582
|
+
const request = this.pending.get(message.requestId);
|
|
583
|
+
if (!request) {
|
|
584
|
+
for (const listener of this.listeners)
|
|
585
|
+
listener(message);
|
|
586
|
+
return;
|
|
587
|
+
}
|
|
588
|
+
this.pending.delete(message.requestId);
|
|
589
|
+
clearTimeout(request.timer);
|
|
590
|
+
if (message.type === 'error')
|
|
591
|
+
request.reject(Object.assign(new Error(message.message), { code: message.code }));
|
|
592
|
+
else
|
|
593
|
+
request.resolve(message);
|
|
594
|
+
}
|
|
595
|
+
async request(message, timeoutMs = RPC_TIMEOUT_MS) {
|
|
596
|
+
await this.ready;
|
|
597
|
+
if (this.socket.readyState !== WebSocket.OPEN)
|
|
598
|
+
throw socketError('WebSocket is not open.');
|
|
599
|
+
const id = message.requestId || crypto.randomUUID();
|
|
600
|
+
const payload = { ...message, requestId: id };
|
|
601
|
+
const promise = new Promise((resolve, reject) => {
|
|
602
|
+
const timer = setTimeout(() => {
|
|
603
|
+
this.pending.delete(id);
|
|
604
|
+
reject(Object.assign(new Error(`WebSocket request timed out after ${timeoutMs / 1000} seconds.`), {
|
|
605
|
+
code: 'SOCKET_TIMEOUT',
|
|
606
|
+
}));
|
|
607
|
+
this.socket.terminate();
|
|
608
|
+
}, timeoutMs);
|
|
609
|
+
this.pending.set(id, { resolve, reject, timer });
|
|
610
|
+
});
|
|
611
|
+
try {
|
|
612
|
+
this.socket.send(JSON.stringify(payload));
|
|
613
|
+
}
|
|
614
|
+
catch (error) {
|
|
615
|
+
const request = this.pending.get(id);
|
|
616
|
+
if (request)
|
|
617
|
+
clearTimeout(request.timer);
|
|
618
|
+
this.pending.delete(id);
|
|
619
|
+
throw socketError('WebSocket send failed.', error);
|
|
620
|
+
}
|
|
621
|
+
return promise;
|
|
622
|
+
}
|
|
623
|
+
notify(message) {
|
|
624
|
+
if (this.socket.readyState !== WebSocket.OPEN)
|
|
625
|
+
throw socketError('WebSocket is not open.');
|
|
626
|
+
this.socket.send(JSON.stringify(message));
|
|
627
|
+
}
|
|
628
|
+
onEvent(listener) {
|
|
629
|
+
this.listeners.add(listener);
|
|
630
|
+
return () => this.listeners.delete(listener);
|
|
631
|
+
}
|
|
632
|
+
close() {
|
|
633
|
+
if (this.socket.readyState === WebSocket.OPEN)
|
|
634
|
+
this.socket.close(1000, 'upload complete');
|
|
635
|
+
else if (this.socket.readyState === WebSocket.CONNECTING)
|
|
636
|
+
this.socket.terminate();
|
|
637
|
+
}
|
|
638
|
+
}
|
|
639
|
+
export async function connectUploader(options, machine, reconnect = false) {
|
|
640
|
+
let authenticationRecoveries = 0;
|
|
641
|
+
for (let attempt = 0; attempt <= MAX_SOCKET_RETRIES; attempt += 1) {
|
|
642
|
+
if (options.credentialProvider) {
|
|
643
|
+
const credentials = await options.credentialProvider.refreshIfDue();
|
|
644
|
+
options.token = credentials.accessToken;
|
|
645
|
+
}
|
|
646
|
+
const rpc = new RpcSocket(options.url, options.token, options.devUser);
|
|
647
|
+
try {
|
|
648
|
+
const ready = await rpc.ready;
|
|
649
|
+
process.stdout.write(`${reconnect ? 'Reconnected' : 'Connected'} to ${options.url} as @${ready.user.login} ` +
|
|
650
|
+
`from ${os.hostname()} (${machine})\n`);
|
|
651
|
+
return rpc;
|
|
652
|
+
}
|
|
653
|
+
catch (error) {
|
|
654
|
+
rpc.close();
|
|
655
|
+
if (error?.code === 'AUTH_REJECTED' && options.credentialProvider && authenticationRecoveries < 2) {
|
|
656
|
+
authenticationRecoveries += 1;
|
|
657
|
+
const credentials = await options.credentialProvider.recover();
|
|
658
|
+
options.token = credentials.accessToken;
|
|
659
|
+
process.stderr.write('Uploader authentication was rejected; refreshed credentials.\n');
|
|
660
|
+
attempt -= 1;
|
|
661
|
+
continue;
|
|
662
|
+
}
|
|
663
|
+
if (!isSocketError(error) || attempt === MAX_SOCKET_RETRIES)
|
|
664
|
+
throw error;
|
|
665
|
+
const delay = reconnectDelay(attempt);
|
|
666
|
+
process.stderr.write(` WebSocket connection failed: ${error.message}; retrying in ` +
|
|
667
|
+
`${(delay / 1000).toFixed(1)}s (${attempt + 1}/${MAX_SOCKET_RETRIES})\n`);
|
|
668
|
+
await wait(delay);
|
|
669
|
+
}
|
|
670
|
+
}
|
|
671
|
+
throw socketError('Could not establish an uploader WebSocket.');
|
|
672
|
+
}
|
|
673
|
+
/** One reconnecting, multiplexed uploader transport for ingestion, queue
|
|
674
|
+
* control, live rows, ownership, and machine presence. RpcSocket request IDs
|
|
675
|
+
* already make concurrent calls independent, so separate TCP/WebSocket
|
|
676
|
+
* connections only duplicate authentication and presence state. */
|
|
677
|
+
export class SharedUploaderConnection {
|
|
678
|
+
options;
|
|
679
|
+
machine;
|
|
680
|
+
connectImpl;
|
|
681
|
+
rpc;
|
|
682
|
+
connecting;
|
|
683
|
+
stopped;
|
|
684
|
+
hasConnected;
|
|
685
|
+
listeners;
|
|
686
|
+
connectListeners;
|
|
687
|
+
unsubscribeEvents;
|
|
688
|
+
reconnectTimer;
|
|
689
|
+
constructor(options, machine, connect = connectUploader) {
|
|
690
|
+
this.options = options;
|
|
691
|
+
this.machine = machine;
|
|
692
|
+
this.connectImpl = connect;
|
|
693
|
+
this.rpc = null;
|
|
694
|
+
this.connecting = null;
|
|
695
|
+
this.stopped = false;
|
|
696
|
+
this.hasConnected = false;
|
|
697
|
+
this.listeners = new Set();
|
|
698
|
+
this.connectListeners = new Set();
|
|
699
|
+
this.unsubscribeEvents = null;
|
|
700
|
+
this.reconnectTimer = null;
|
|
701
|
+
}
|
|
702
|
+
invalidate(rpc) {
|
|
703
|
+
if (this.rpc !== rpc)
|
|
704
|
+
return;
|
|
705
|
+
this.unsubscribeEvents?.();
|
|
706
|
+
this.unsubscribeEvents = null;
|
|
707
|
+
this.rpc = null;
|
|
708
|
+
}
|
|
709
|
+
scheduleReconnect() {
|
|
710
|
+
if (this.stopped || this.reconnectTimer)
|
|
711
|
+
return;
|
|
712
|
+
this.reconnectTimer = setTimeout(() => {
|
|
713
|
+
this.reconnectTimer = null;
|
|
714
|
+
void this.current().catch((error) => {
|
|
715
|
+
process.stderr.write(`Shared uploader reconnect failed: ${error?.message || error}\n`);
|
|
716
|
+
this.scheduleReconnect();
|
|
717
|
+
});
|
|
718
|
+
}, 1_000);
|
|
719
|
+
this.reconnectTimer.unref?.();
|
|
720
|
+
}
|
|
721
|
+
async current() {
|
|
722
|
+
if (this.stopped)
|
|
723
|
+
throw socketError('Shared uploader connection is closed.');
|
|
724
|
+
if (this.rpc)
|
|
725
|
+
return this.rpc;
|
|
726
|
+
if (this.connecting)
|
|
727
|
+
return this.connecting;
|
|
728
|
+
this.connecting = this.connectImpl(this.options, this.machine, this.hasConnected)
|
|
729
|
+
.then((rpc) => {
|
|
730
|
+
if (this.stopped) {
|
|
731
|
+
rpc.close();
|
|
732
|
+
throw socketError('Shared uploader connection is closed.');
|
|
733
|
+
}
|
|
734
|
+
this.hasConnected = true;
|
|
735
|
+
this.rpc = rpc;
|
|
736
|
+
this.unsubscribeEvents = rpc.onEvent((message) => {
|
|
737
|
+
for (const listener of this.listeners)
|
|
738
|
+
listener(message);
|
|
739
|
+
});
|
|
740
|
+
void rpc.closed.then(() => {
|
|
741
|
+
this.invalidate(rpc);
|
|
742
|
+
this.scheduleReconnect();
|
|
743
|
+
});
|
|
744
|
+
for (const listener of this.connectListeners) {
|
|
745
|
+
queueMicrotask(() => listener(rpc));
|
|
746
|
+
}
|
|
747
|
+
return rpc;
|
|
748
|
+
})
|
|
749
|
+
.finally(() => {
|
|
750
|
+
this.connecting = null;
|
|
751
|
+
});
|
|
752
|
+
return this.connecting;
|
|
753
|
+
}
|
|
754
|
+
async request(message, timeoutMs = RPC_TIMEOUT_MS) {
|
|
755
|
+
for (let attempt = 0; attempt <= MAX_SOCKET_RETRIES; attempt += 1) {
|
|
756
|
+
const rpc = await this.current();
|
|
757
|
+
try {
|
|
758
|
+
return await rpc.request(message, timeoutMs);
|
|
759
|
+
}
|
|
760
|
+
catch (error) {
|
|
761
|
+
if (!isSocketError(error) || attempt === MAX_SOCKET_RETRIES)
|
|
762
|
+
throw error;
|
|
763
|
+
this.invalidate(rpc);
|
|
764
|
+
rpc.close();
|
|
765
|
+
await wait(reconnectDelay(attempt));
|
|
766
|
+
}
|
|
767
|
+
}
|
|
768
|
+
throw socketError('Shared uploader request exhausted its reconnect attempts.');
|
|
769
|
+
}
|
|
770
|
+
notify(message) {
|
|
771
|
+
if (!this.rpc)
|
|
772
|
+
throw socketError('Shared uploader WebSocket is reconnecting.');
|
|
773
|
+
this.rpc.notify(message);
|
|
774
|
+
}
|
|
775
|
+
onEvent(listener) {
|
|
776
|
+
this.listeners.add(listener);
|
|
777
|
+
return () => this.listeners.delete(listener);
|
|
778
|
+
}
|
|
779
|
+
onConnect(listener) {
|
|
780
|
+
this.connectListeners.add(listener);
|
|
781
|
+
if (this.rpc)
|
|
782
|
+
queueMicrotask(() => listener(this.rpc));
|
|
783
|
+
return () => this.connectListeners.delete(listener);
|
|
784
|
+
}
|
|
785
|
+
close() {
|
|
786
|
+
this.stopped = true;
|
|
787
|
+
clearTimeout(this.reconnectTimer);
|
|
788
|
+
this.reconnectTimer = null;
|
|
789
|
+
const rpc = this.rpc;
|
|
790
|
+
this.invalidate(rpc);
|
|
791
|
+
rpc?.close();
|
|
792
|
+
}
|
|
793
|
+
}
|
|
794
|
+
function* batches(rows, raw, startRow, redactor = DEFAULT_REDACTOR) {
|
|
795
|
+
let heldRows = [];
|
|
796
|
+
let heldBytes = 0;
|
|
797
|
+
for (let index = startRow; index < rows.length; index += 1) {
|
|
798
|
+
const row = rows[index];
|
|
799
|
+
const sourceLine = raw[index];
|
|
800
|
+
if (!row || sourceLine === undefined)
|
|
801
|
+
continue;
|
|
802
|
+
const line = redactor.redactJsonLine(sourceLine);
|
|
803
|
+
const bytes = Buffer.byteLength(line) + 1;
|
|
804
|
+
const event = {
|
|
805
|
+
index,
|
|
806
|
+
seq: Number(row.seq) || 0,
|
|
807
|
+
kind: String(row.kind || 'unknown'),
|
|
808
|
+
...(row.phase ? { phase: String(row.phase) } : {}),
|
|
809
|
+
...(row.elapsedMs !== undefined ? { elapsedMs: Number(row.elapsedMs) } : {}),
|
|
810
|
+
...(row.ts ? { ts: String(row.ts) } : {}),
|
|
811
|
+
raw: line,
|
|
812
|
+
bytes,
|
|
813
|
+
};
|
|
814
|
+
const estimated = bytes;
|
|
815
|
+
if (heldRows.length && (heldRows.length >= MAX_BATCH_ROWS || heldBytes + estimated > MAX_BATCH_BYTES)) {
|
|
816
|
+
yield { startRow: heldRows[0].index, rows: heldRows };
|
|
817
|
+
heldRows = [];
|
|
818
|
+
heldBytes = 0;
|
|
819
|
+
}
|
|
820
|
+
heldRows.push(event);
|
|
821
|
+
heldBytes += estimated;
|
|
822
|
+
}
|
|
823
|
+
if (heldRows.length)
|
|
824
|
+
yield { startRow: heldRows[0].index, rows: heldRows };
|
|
825
|
+
}
|
|
826
|
+
async function uploadFile(rpc, machine, root, capture, redactor = DEFAULT_REDACTOR) {
|
|
827
|
+
const { rows, raw } = completeLines(capture.file);
|
|
828
|
+
if (!rows.length)
|
|
829
|
+
return { rows: 0, sent: 0, skipped: true };
|
|
830
|
+
const harness = harnessOf(capture.relative, rows);
|
|
831
|
+
const sessionId = sessionIdOf(machine, capture.relative, rows, harness);
|
|
832
|
+
const opened = await rpc.request({
|
|
833
|
+
type: 'session.open',
|
|
834
|
+
session: {
|
|
835
|
+
sessionId,
|
|
836
|
+
machineId: machine,
|
|
837
|
+
machineHostname: os.hostname(),
|
|
838
|
+
harness,
|
|
839
|
+
sourcePath: path.join(root, capture.relative),
|
|
840
|
+
title: redactor.redactText(titleOf(capture.relative)),
|
|
841
|
+
createdAt: String(rows[0]?.ts || new Date(fs.statSync(capture.file).birthtimeMs).toISOString()),
|
|
842
|
+
redactionVersion: REDACTION_VERSION,
|
|
843
|
+
},
|
|
844
|
+
});
|
|
845
|
+
if (opened.type === 'session_tombstoned')
|
|
846
|
+
return { rows: 0, sent: 0, skipped: true, tombstoned: true };
|
|
847
|
+
let nextRow = Number(opened.nextRow) || 0;
|
|
848
|
+
if (Number(opened.redactionVersion || 0) !== REDACTION_VERSION) {
|
|
849
|
+
await rpc.request({ type: 'session.reset', sessionId, redactionVersion: REDACTION_VERSION });
|
|
850
|
+
nextRow = 0;
|
|
851
|
+
}
|
|
852
|
+
if (nextRow > rows.length)
|
|
853
|
+
throw new Error(`${capture.relative}: remote has ${nextRow} rows, local file has ${rows.length}`);
|
|
854
|
+
if (nextRow === rows.length)
|
|
855
|
+
return { rows: rows.length, sent: 0, skipped: true };
|
|
856
|
+
let sent = 0;
|
|
857
|
+
for (const batch of batches(rows, raw, nextRow, redactor)) {
|
|
858
|
+
let acknowledgement;
|
|
859
|
+
try {
|
|
860
|
+
acknowledgement = await rpc.request({ type: 'ingest', sessionId, ...batch });
|
|
861
|
+
}
|
|
862
|
+
catch (error) {
|
|
863
|
+
const offset = /^ROW_OFFSET:(\d+)$/u.exec(String(error.code || ''));
|
|
864
|
+
if (!offset)
|
|
865
|
+
throw error;
|
|
866
|
+
nextRow = Number(offset[1]);
|
|
867
|
+
if (nextRow >= batch.startRow + batch.rows.length)
|
|
868
|
+
continue;
|
|
869
|
+
throw error;
|
|
870
|
+
}
|
|
871
|
+
nextRow = Number(acknowledgement.nextRow);
|
|
872
|
+
sent += batch.rows.length;
|
|
873
|
+
process.stdout.write(` ${capture.relative}: ${nextRow}/${rows.length} rows · revision ${acknowledgement.revision}${acknowledgement.archived ? ' · archived' : ''}\n`);
|
|
874
|
+
}
|
|
875
|
+
return { rows: rows.length, sent, skipped: false };
|
|
876
|
+
}
|
|
877
|
+
function nativeSourceIdentity(file) {
|
|
878
|
+
try {
|
|
879
|
+
const stat = fs.statSync(file);
|
|
880
|
+
return `${stat.dev}:${stat.ino}`;
|
|
881
|
+
}
|
|
882
|
+
catch {
|
|
883
|
+
return '';
|
|
884
|
+
}
|
|
885
|
+
}
|
|
886
|
+
function canResumeNativeCursor(descriptor, cursor, nextRow) {
|
|
887
|
+
return Boolean(cursor &&
|
|
888
|
+
!descriptor.file.endsWith('.json') &&
|
|
889
|
+
descriptor.harness !== 'deepseek' &&
|
|
890
|
+
!descriptor.sourcePath.includes('.db#') &&
|
|
891
|
+
cursor.file === descriptor.file &&
|
|
892
|
+
cursor.sourceIdentity === nativeSourceIdentity(descriptor.file) &&
|
|
893
|
+
Number(cursor.rowCount) === nextRow &&
|
|
894
|
+
Number(cursor.offset) >= 0 &&
|
|
895
|
+
Number(cursor.offset) <= Number(descriptor.size));
|
|
896
|
+
}
|
|
897
|
+
function gitAssociationForSession(cwd, redactor) {
|
|
898
|
+
const found = discoverGitAssociation(cwd);
|
|
899
|
+
if (!found)
|
|
900
|
+
return null;
|
|
901
|
+
return {
|
|
902
|
+
root: found.root,
|
|
903
|
+
remotes: found.remotes.map((remote) => ({
|
|
904
|
+
name: remote.name,
|
|
905
|
+
url: redactor?.redactText ? redactor.redactText(remote.url) : remote.url,
|
|
906
|
+
})),
|
|
907
|
+
};
|
|
908
|
+
}
|
|
909
|
+
/**
|
|
910
|
+
* @param {any} rpc
|
|
911
|
+
* @param {string} machine
|
|
912
|
+
* @param {any} descriptor
|
|
913
|
+
* @param {any} [cursor]
|
|
914
|
+
* @param {(progress: any) => void} [onProgress]
|
|
915
|
+
* @param {any} [redactor]
|
|
916
|
+
* @param {object} [options]
|
|
917
|
+
* @param {number} [options.maxBatches]
|
|
918
|
+
* @param {() => boolean} [options.shouldYield]
|
|
919
|
+
* @param {boolean} [options.recoverRemoteAhead]
|
|
920
|
+
*/
|
|
921
|
+
export async function uploadNativeSession(rpc, machine, descriptor, cursor = {}, onProgress = () => { }, redactor = DEFAULT_REDACTOR, { maxBatches = Number.POSITIVE_INFINITY, shouldYield = () => false, recoverRemoteAhead = true, onPathCandidates = (_sessionId, _workingDirectory, _candidates) => { }, } = {}) {
|
|
922
|
+
const workingDirectory = await nativeDescriptorWorkingDirectory(descriptor);
|
|
923
|
+
const agentSessionId = await nativeDescriptorAgentSessionId(descriptor);
|
|
924
|
+
const git = gitAssociationForSession(workingDirectory, redactor);
|
|
925
|
+
const sessionId = stableNativeSessionId(machine, descriptor.harness, descriptor.sourcePath);
|
|
926
|
+
const opened = await rpc.request({
|
|
927
|
+
type: 'session.open',
|
|
928
|
+
session: {
|
|
929
|
+
sessionId,
|
|
930
|
+
machineId: machine,
|
|
931
|
+
machineHostname: descriptor.machineHostname || os.hostname(),
|
|
932
|
+
...(workingDirectory ? { workingDirectory } : {}),
|
|
933
|
+
...(git ? { git } : {}),
|
|
934
|
+
harness: descriptor.harness,
|
|
935
|
+
sourcePath: descriptor.sourcePath,
|
|
936
|
+
title: redactor.redactText(nativeTitle(descriptor)),
|
|
937
|
+
createdAt: descriptor.createdAt || new Date(descriptor.mtimeMs).toISOString(),
|
|
938
|
+
redactionVersion: REDACTION_VERSION,
|
|
939
|
+
...(descriptor.readOnly === true ? { readOnly: true } : {}),
|
|
940
|
+
...(agentSessionId ? { agentSessionId } : {}),
|
|
941
|
+
...(descriptor.parentSessionId ? { parentSessionId: descriptor.parentSessionId } : {}),
|
|
942
|
+
},
|
|
943
|
+
});
|
|
944
|
+
if (opened.type === 'session_tombstoned') {
|
|
945
|
+
return { rows: 0, sent: 0, skipped: true, tombstoned: true };
|
|
946
|
+
}
|
|
947
|
+
const uploadSessionId = String(opened.sessionId || sessionId);
|
|
948
|
+
let nextRow = Number(opened.nextRow) || 0;
|
|
949
|
+
if (Number(opened.redactionVersion || 0) !== REDACTION_VERSION) {
|
|
950
|
+
await rpc.request({ type: 'session.reset', sessionId: uploadSessionId, redactionVersion: REDACTION_VERSION });
|
|
951
|
+
nextRow = 0;
|
|
952
|
+
for (const key of Object.keys(cursor))
|
|
953
|
+
delete cursor[key];
|
|
954
|
+
}
|
|
955
|
+
const resumable = canResumeNativeCursor(descriptor, cursor, nextRow);
|
|
956
|
+
const scanCursor = resumable
|
|
957
|
+
? { offset: Number(cursor.offset), rowCount: Number(cursor.rowCount), lineNumber: Number(cursor.lineNumber) || 0 }
|
|
958
|
+
: { offset: 0, rowCount: 0, lineNumber: 0 };
|
|
959
|
+
const sourceIdentity = nativeSourceIdentity(descriptor.file);
|
|
960
|
+
let held = [];
|
|
961
|
+
let heldBytes = 0;
|
|
962
|
+
let pathMentions = [];
|
|
963
|
+
let sent = 0;
|
|
964
|
+
let batchesSent = 0;
|
|
965
|
+
let sawRows = false;
|
|
966
|
+
const commitCursor = (capture) => {
|
|
967
|
+
Object.assign(cursor, {
|
|
968
|
+
file: descriptor.file,
|
|
969
|
+
sourceIdentity,
|
|
970
|
+
offset: capture.endOffset,
|
|
971
|
+
rowCount: capture.rowCount,
|
|
972
|
+
lineNumber: capture.lineNumber,
|
|
973
|
+
});
|
|
974
|
+
};
|
|
975
|
+
const flush = async () => {
|
|
976
|
+
if (!held.length)
|
|
977
|
+
return;
|
|
978
|
+
const rows = held.map((entry) => entry.event);
|
|
979
|
+
const acknowledgement = await rpc.request({
|
|
980
|
+
type: 'ingest',
|
|
981
|
+
sessionId: uploadSessionId,
|
|
982
|
+
startRow: rows[0].index,
|
|
983
|
+
rows,
|
|
984
|
+
});
|
|
985
|
+
nextRow = Number(acknowledgement.nextRow);
|
|
986
|
+
sent += rows.length;
|
|
987
|
+
batchesSent += 1;
|
|
988
|
+
// Only after the rows are acknowledged: a resolution is appended to the
|
|
989
|
+
// session's own log and refers to prose in it, so the sentence has to be
|
|
990
|
+
// stored before what it named is.
|
|
991
|
+
if (pathMentions.length) {
|
|
992
|
+
onPathCandidates(uploadSessionId, workingDirectory || '', pathMentions);
|
|
993
|
+
pathMentions = [];
|
|
994
|
+
}
|
|
995
|
+
const last = held.at(-1).capture;
|
|
996
|
+
if (nextRow >= last.rowCount)
|
|
997
|
+
commitCursor(last);
|
|
998
|
+
held = [];
|
|
999
|
+
heldBytes = 0;
|
|
1000
|
+
onProgress({
|
|
1001
|
+
nextRow,
|
|
1002
|
+
total: Math.max(nextRow, Number(scanCursor.rowCount) || 0),
|
|
1003
|
+
revision: Number(acknowledgement.revision),
|
|
1004
|
+
archived: Boolean(acknowledgement.archived),
|
|
1005
|
+
});
|
|
1006
|
+
};
|
|
1007
|
+
for await (const capture of nativeCaptureRowStream(descriptor, scanCursor)) {
|
|
1008
|
+
sawRows = true;
|
|
1009
|
+
if (capture.rowCount <= nextRow) {
|
|
1010
|
+
commitCursor(capture);
|
|
1011
|
+
continue;
|
|
1012
|
+
}
|
|
1013
|
+
const outboundRaw = redactor.redactJsonLine(capture.raw);
|
|
1014
|
+
for (const candidate of pathCandidates(outboundRaw)) {
|
|
1015
|
+
if (!pathMentions.includes(candidate))
|
|
1016
|
+
pathMentions.push(candidate);
|
|
1017
|
+
}
|
|
1018
|
+
const bytes = Buffer.byteLength(outboundRaw) + 1;
|
|
1019
|
+
if (held.length && (held.length >= MAX_BATCH_ROWS || heldBytes + bytes > MAX_BATCH_BYTES)) {
|
|
1020
|
+
await flush();
|
|
1021
|
+
if (batchesSent >= maxBatches || shouldYield()) {
|
|
1022
|
+
return { rows: nextRow, sent, skipped: false, partial: true };
|
|
1023
|
+
}
|
|
1024
|
+
}
|
|
1025
|
+
held.push({
|
|
1026
|
+
capture,
|
|
1027
|
+
event: {
|
|
1028
|
+
index: capture.row.seq,
|
|
1029
|
+
seq: capture.row.seq,
|
|
1030
|
+
kind: capture.row.kind,
|
|
1031
|
+
phase: capture.row.phase,
|
|
1032
|
+
elapsedMs: capture.row.elapsedMs,
|
|
1033
|
+
ts: capture.row.ts,
|
|
1034
|
+
raw: outboundRaw,
|
|
1035
|
+
bytes,
|
|
1036
|
+
},
|
|
1037
|
+
});
|
|
1038
|
+
heldBytes += bytes;
|
|
1039
|
+
}
|
|
1040
|
+
await flush();
|
|
1041
|
+
if (nextRow > scanCursor.rowCount) {
|
|
1042
|
+
if (recoverRemoteAhead) {
|
|
1043
|
+
// Harnesses such as Grok rewrite their native transcript in place during
|
|
1044
|
+
// compaction. Rewind only the native source cursor: the canonical rows
|
|
1045
|
+
// are an append-only audit log and include runtime events that cannot be
|
|
1046
|
+
// reconstructed from the rewritten file.
|
|
1047
|
+
await rpc.request({
|
|
1048
|
+
type: 'session.native.rewind',
|
|
1049
|
+
sessionId: uploadSessionId,
|
|
1050
|
+
redactionVersion: REDACTION_VERSION,
|
|
1051
|
+
});
|
|
1052
|
+
for (const key of Object.keys(cursor))
|
|
1053
|
+
delete cursor[key];
|
|
1054
|
+
return uploadNativeSession(rpc, machine, descriptor, cursor, onProgress, redactor, {
|
|
1055
|
+
maxBatches,
|
|
1056
|
+
shouldYield,
|
|
1057
|
+
recoverRemoteAhead: false,
|
|
1058
|
+
onPathCandidates,
|
|
1059
|
+
});
|
|
1060
|
+
}
|
|
1061
|
+
const error = /** @type {Error & { code?: string }} */ (new Error(`${descriptor.sourcePath}: remote has ${nextRow} rows, local session has ${scanCursor.rowCount}`));
|
|
1062
|
+
error.code = 'REMOTE_AHEAD';
|
|
1063
|
+
throw error;
|
|
1064
|
+
}
|
|
1065
|
+
if (nextRow === scanCursor.rowCount) {
|
|
1066
|
+
Object.assign(cursor, {
|
|
1067
|
+
file: descriptor.file,
|
|
1068
|
+
sourceIdentity,
|
|
1069
|
+
offset: scanCursor.offset,
|
|
1070
|
+
rowCount: scanCursor.rowCount,
|
|
1071
|
+
lineNumber: scanCursor.lineNumber,
|
|
1072
|
+
});
|
|
1073
|
+
}
|
|
1074
|
+
if (!sawRows && scanCursor.rowCount === 0) {
|
|
1075
|
+
return { rows: 0, sent: 0, skipped: true, empty: true };
|
|
1076
|
+
}
|
|
1077
|
+
return { rows: scanCursor.rowCount, sent, skipped: sent === 0 };
|
|
1078
|
+
}
|
|
1079
|
+
export function nativeFailureRetryDelay(attempts) {
|
|
1080
|
+
const exponent = Math.max(0, Math.min(16, Math.floor(Number(attempts) || 1) - 1));
|
|
1081
|
+
return Math.min(FAILURE_RETRY_MAX_MS, FAILURE_RETRY_BASE_MS * 2 ** exponent);
|
|
1082
|
+
}
|
|
1083
|
+
export function createNativeUploadState(file, machine) {
|
|
1084
|
+
let document = { version: NATIVE_STATE_VERSION, machineId: machine, sessions: {} };
|
|
1085
|
+
try {
|
|
1086
|
+
const parsed = JSON.parse(fs.readFileSync(file, 'utf8'));
|
|
1087
|
+
if (parsed?.version === NATIVE_STATE_VERSION &&
|
|
1088
|
+
parsed.machineId === machine &&
|
|
1089
|
+
parsed.sessions &&
|
|
1090
|
+
typeof parsed.sessions === 'object')
|
|
1091
|
+
document = parsed;
|
|
1092
|
+
}
|
|
1093
|
+
catch { }
|
|
1094
|
+
let timer = null;
|
|
1095
|
+
let dirty = false;
|
|
1096
|
+
const persist = () => {
|
|
1097
|
+
if (timer)
|
|
1098
|
+
clearTimeout(timer);
|
|
1099
|
+
timer = null;
|
|
1100
|
+
if (!dirty)
|
|
1101
|
+
return;
|
|
1102
|
+
dirty = false;
|
|
1103
|
+
fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 });
|
|
1104
|
+
const temporary = `${file}.${process.pid}.tmp`;
|
|
1105
|
+
fs.writeFileSync(temporary, `${JSON.stringify(document)}\n`, { mode: 0o600 });
|
|
1106
|
+
fs.renameSync(temporary, file);
|
|
1107
|
+
};
|
|
1108
|
+
return {
|
|
1109
|
+
get(sessionId) {
|
|
1110
|
+
return document.sessions[sessionId] || null;
|
|
1111
|
+
},
|
|
1112
|
+
isTombstoned(sessionId) {
|
|
1113
|
+
return document.sessions[sessionId]?.tombstoned === true;
|
|
1114
|
+
},
|
|
1115
|
+
tombstone(sessionId) {
|
|
1116
|
+
document.sessions[sessionId] = { tombstoned: true };
|
|
1117
|
+
dirty = true;
|
|
1118
|
+
// A deletion acknowledgement must survive an immediate daemon restart;
|
|
1119
|
+
// delaying this write could resurrect the session during that window.
|
|
1120
|
+
persist();
|
|
1121
|
+
},
|
|
1122
|
+
set(sessionId, fingerprint, cursor, redactionVersion = REDACTION_VERSION) {
|
|
1123
|
+
if (document.sessions[sessionId]?.tombstoned === true)
|
|
1124
|
+
return;
|
|
1125
|
+
document.sessions[sessionId] = { fingerprint, cursor: { ...cursor }, redactionVersion };
|
|
1126
|
+
dirty = true;
|
|
1127
|
+
if (!timer) {
|
|
1128
|
+
timer = setTimeout(persist, 15_000);
|
|
1129
|
+
timer.unref?.();
|
|
1130
|
+
}
|
|
1131
|
+
},
|
|
1132
|
+
retain(sessionIds) {
|
|
1133
|
+
const keep = new Set(sessionIds);
|
|
1134
|
+
let changed = false;
|
|
1135
|
+
for (const sessionId of Object.keys(document.sessions)) {
|
|
1136
|
+
if (keep.has(sessionId) || document.sessions[sessionId]?.tombstoned === true)
|
|
1137
|
+
continue;
|
|
1138
|
+
delete document.sessions[sessionId];
|
|
1139
|
+
changed = true;
|
|
1140
|
+
}
|
|
1141
|
+
if (changed)
|
|
1142
|
+
dirty = true;
|
|
1143
|
+
},
|
|
1144
|
+
persist,
|
|
1145
|
+
close() {
|
|
1146
|
+
persist();
|
|
1147
|
+
},
|
|
1148
|
+
};
|
|
1149
|
+
}
|
|
1150
|
+
/** Recursive directory notifications provide low-latency append detection;
|
|
1151
|
+
* callers retain stat polling and periodic discovery as correctness fallbacks. */
|
|
1152
|
+
export function watchNativeSessionDirectories({ roots, shallowRoots = [], files = [], onFile, onDirectory }) {
|
|
1153
|
+
const watchers = new Map();
|
|
1154
|
+
const resolvedRoots = roots.map((root) => path.resolve(root));
|
|
1155
|
+
const resolvedShallowRoots = new Set(shallowRoots.map((root) => path.resolve(root)));
|
|
1156
|
+
const recursiveRoots = new Set();
|
|
1157
|
+
const trackedFiles = new Set(files.map((file) => path.resolve(file)));
|
|
1158
|
+
let closed = false;
|
|
1159
|
+
const addDirectory = (directory, recursive = false) => {
|
|
1160
|
+
const resolved = path.resolve(directory);
|
|
1161
|
+
if (closed || watchers.has(resolved))
|
|
1162
|
+
return false;
|
|
1163
|
+
try {
|
|
1164
|
+
const watcher = fs.watch(resolved, { persistent: false, recursive }, (_event, filename) => {
|
|
1165
|
+
if (!filename) {
|
|
1166
|
+
onDirectory(resolved);
|
|
1167
|
+
return;
|
|
1168
|
+
}
|
|
1169
|
+
const changed = path.join(resolved, filename.toString());
|
|
1170
|
+
let stat;
|
|
1171
|
+
try {
|
|
1172
|
+
stat = fs.statSync(changed);
|
|
1173
|
+
}
|
|
1174
|
+
catch { }
|
|
1175
|
+
if (stat?.isDirectory()) {
|
|
1176
|
+
// Watch the new level immediately. A following inventory refresh
|
|
1177
|
+
// adds only the descendant directories that actually contain a
|
|
1178
|
+
// transcript, avoiding tens of thousands of unrelated watchers.
|
|
1179
|
+
if (!recursive)
|
|
1180
|
+
addDirectory(changed);
|
|
1181
|
+
onDirectory(changed);
|
|
1182
|
+
}
|
|
1183
|
+
else {
|
|
1184
|
+
onFile(changed);
|
|
1185
|
+
}
|
|
1186
|
+
});
|
|
1187
|
+
watcher.on('error', () => {
|
|
1188
|
+
watcher.close();
|
|
1189
|
+
watchers.delete(resolved);
|
|
1190
|
+
if (recursive) {
|
|
1191
|
+
recursiveRoots.delete(resolved);
|
|
1192
|
+
addDirectory(resolved);
|
|
1193
|
+
for (const file of trackedFiles)
|
|
1194
|
+
addFile(file);
|
|
1195
|
+
}
|
|
1196
|
+
});
|
|
1197
|
+
watchers.set(resolved, watcher);
|
|
1198
|
+
if (recursive)
|
|
1199
|
+
recursiveRoots.add(resolved);
|
|
1200
|
+
return true;
|
|
1201
|
+
}
|
|
1202
|
+
catch {
|
|
1203
|
+
return false;
|
|
1204
|
+
}
|
|
1205
|
+
};
|
|
1206
|
+
const addFile = (file) => {
|
|
1207
|
+
let directory = path.dirname(path.resolve(file));
|
|
1208
|
+
const root = resolvedRoots.find((candidate) => directory === candidate || directory.startsWith(`${candidate}${path.sep}`));
|
|
1209
|
+
if (!root)
|
|
1210
|
+
return;
|
|
1211
|
+
if (recursiveRoots.has(root))
|
|
1212
|
+
return;
|
|
1213
|
+
for (;;) {
|
|
1214
|
+
addDirectory(directory);
|
|
1215
|
+
if (directory === root)
|
|
1216
|
+
break;
|
|
1217
|
+
const parent = path.dirname(directory);
|
|
1218
|
+
if (parent === directory)
|
|
1219
|
+
break;
|
|
1220
|
+
directory = parent;
|
|
1221
|
+
}
|
|
1222
|
+
};
|
|
1223
|
+
for (const root of resolvedRoots) {
|
|
1224
|
+
if (resolvedShallowRoots.has(root) || !addDirectory(root, true))
|
|
1225
|
+
addDirectory(root);
|
|
1226
|
+
}
|
|
1227
|
+
for (const file of trackedFiles)
|
|
1228
|
+
addFile(file);
|
|
1229
|
+
return {
|
|
1230
|
+
get count() {
|
|
1231
|
+
return watchers.size;
|
|
1232
|
+
},
|
|
1233
|
+
addFiles(nextFiles) {
|
|
1234
|
+
for (const file of nextFiles) {
|
|
1235
|
+
trackedFiles.add(path.resolve(file));
|
|
1236
|
+
addFile(file);
|
|
1237
|
+
}
|
|
1238
|
+
},
|
|
1239
|
+
close() {
|
|
1240
|
+
closed = true;
|
|
1241
|
+
for (const watcher of watchers.values())
|
|
1242
|
+
watcher.close();
|
|
1243
|
+
watchers.clear();
|
|
1244
|
+
},
|
|
1245
|
+
};
|
|
1246
|
+
}
|
|
1247
|
+
const nativeWorkingDirectoryCache = new Map();
|
|
1248
|
+
export async function filterNativeSessionsByDirectory(sessions, directory, resolveWorkingDirectory = nativeDescriptorWorkingDirectory) {
|
|
1249
|
+
if (!directory)
|
|
1250
|
+
return sessions;
|
|
1251
|
+
const selected = path.resolve(directory);
|
|
1252
|
+
const scoped = [];
|
|
1253
|
+
for (const descriptor of sessions) {
|
|
1254
|
+
const key = `${descriptor.harness}\0${descriptor.sourcePath}`;
|
|
1255
|
+
let workingDirectory = nativeWorkingDirectoryCache.get(key) || '';
|
|
1256
|
+
if (!workingDirectory) {
|
|
1257
|
+
workingDirectory = await resolveWorkingDirectory(descriptor);
|
|
1258
|
+
if (workingDirectory)
|
|
1259
|
+
nativeWorkingDirectoryCache.set(key, workingDirectory);
|
|
1260
|
+
}
|
|
1261
|
+
if (workingDirectory && path.resolve(workingDirectory) === selected)
|
|
1262
|
+
scoped.push(descriptor);
|
|
1263
|
+
}
|
|
1264
|
+
return scoped;
|
|
1265
|
+
}
|
|
1266
|
+
export async function filterNativeSessionsByAllowedDirectories(sessions, allowedDirectories, resolveWorkingDirectory = nativeDescriptorWorkingDirectory) {
|
|
1267
|
+
if (!allowedDirectories?.length)
|
|
1268
|
+
return sessions;
|
|
1269
|
+
const scoped = [];
|
|
1270
|
+
for (const descriptor of sessions) {
|
|
1271
|
+
const key = `${descriptor.harness}\0${descriptor.sourcePath}`;
|
|
1272
|
+
let workingDirectory = nativeWorkingDirectoryCache.get(key) || '';
|
|
1273
|
+
if (!workingDirectory) {
|
|
1274
|
+
workingDirectory = await resolveWorkingDirectory(descriptor);
|
|
1275
|
+
if (workingDirectory)
|
|
1276
|
+
nativeWorkingDirectoryCache.set(key, workingDirectory);
|
|
1277
|
+
}
|
|
1278
|
+
if (workingDirectoryIsAllowed(workingDirectory, allowedDirectories))
|
|
1279
|
+
scoped.push(descriptor);
|
|
1280
|
+
}
|
|
1281
|
+
return scoped;
|
|
1282
|
+
}
|
|
1283
|
+
async function nativeInventory(options, machine) {
|
|
1284
|
+
let sessions = linkNativeSessionParents(machine, discoverLocalSessions(options.rewriteSession ? '' : options.only));
|
|
1285
|
+
sessions = await filterNativeSessionsByDirectory(sessions, options.directory);
|
|
1286
|
+
sessions = await filterNativeSessionsByAllowedDirectories(sessions, options.allowedDirectories);
|
|
1287
|
+
if (!options.rewriteSession)
|
|
1288
|
+
return sessions;
|
|
1289
|
+
const exact = sessions.filter((descriptor) => stableNativeSessionId(machine, descriptor.harness, descriptor.sourcePath) === options.rewriteSession);
|
|
1290
|
+
if (exact.length !== 1) {
|
|
1291
|
+
throw new Error(`--rewrite-session matched ${exact.length} local sessions; expected exactly one.`);
|
|
1292
|
+
}
|
|
1293
|
+
return exact;
|
|
1294
|
+
}
|
|
1295
|
+
async function reportExternalManagement(rpc, machine, descriptors, previous = new Map()) {
|
|
1296
|
+
let managedKeys;
|
|
1297
|
+
try {
|
|
1298
|
+
managedKeys = externallyManagedNativeDescriptorKeys(descriptors);
|
|
1299
|
+
}
|
|
1300
|
+
catch (error) {
|
|
1301
|
+
process.stderr.write(`External ownership scan failed: ${error?.message || error}\n`);
|
|
1302
|
+
return { rpc, managed: null };
|
|
1303
|
+
}
|
|
1304
|
+
const observedAt = new Date().toISOString();
|
|
1305
|
+
const allStatuses = nativeExternalManagementStatuses(machine, descriptors, managedKeys, observedAt);
|
|
1306
|
+
// Positive leases must be renewed. False values are sent once for newly
|
|
1307
|
+
// discovered sessions and once on a true -> false transition; continuously
|
|
1308
|
+
// retransmitting thousands of unchanged false rows only burns CPU/network.
|
|
1309
|
+
const statuses = allStatuses.filter((status) => status.externallyManaged || previous.get(status.sessionId) !== false);
|
|
1310
|
+
for (let start = 0; start < statuses.length; start += MAX_EXTERNAL_STATUS_BATCH) {
|
|
1311
|
+
const batch = statuses.slice(start, start + MAX_EXTERNAL_STATUS_BATCH);
|
|
1312
|
+
try {
|
|
1313
|
+
await rpc.request({
|
|
1314
|
+
type: 'session.external.update',
|
|
1315
|
+
watcherId: machine,
|
|
1316
|
+
statuses: batch,
|
|
1317
|
+
});
|
|
1318
|
+
}
|
|
1319
|
+
catch (error) {
|
|
1320
|
+
process.stderr.write(`External ownership upload failed: ${error?.message || error}\n`);
|
|
1321
|
+
return { managed: null, previous };
|
|
1322
|
+
}
|
|
1323
|
+
}
|
|
1324
|
+
for (const status of allStatuses)
|
|
1325
|
+
previous.set(status.sessionId, status.externallyManaged);
|
|
1326
|
+
return { managed: managedKeys.size, previous };
|
|
1327
|
+
}
|
|
1328
|
+
async function watchExternalManagement(rpc, machine, getDescriptors) {
|
|
1329
|
+
let lastManagedCount = -1;
|
|
1330
|
+
let previous = new Map();
|
|
1331
|
+
for (;;) {
|
|
1332
|
+
try {
|
|
1333
|
+
const reported = await reportExternalManagement(rpc, machine, getDescriptors(), previous);
|
|
1334
|
+
previous = reported.previous;
|
|
1335
|
+
if (reported.managed !== null && reported.managed !== lastManagedCount) {
|
|
1336
|
+
lastManagedCount = reported.managed;
|
|
1337
|
+
process.stdout.write(`External ownership: ${reported.managed} managed session${reported.managed === 1 ? '' : 's'}\n`);
|
|
1338
|
+
}
|
|
1339
|
+
}
|
|
1340
|
+
catch (error) {
|
|
1341
|
+
process.stderr.write(`External ownership watcher failed: ${error?.message || error}\n`);
|
|
1342
|
+
}
|
|
1343
|
+
await wait(NATIVE_WATCH_INTERVAL_MS);
|
|
1344
|
+
}
|
|
1345
|
+
}
|
|
1346
|
+
export function nativeUploadProgressSnapshot({ machine, descriptors, seen, fingerprints, cursors, redactionPending, rewritePending, tombstoned = new Set(), }) {
|
|
1347
|
+
let totalBytes = 0;
|
|
1348
|
+
let completedBytes = 0;
|
|
1349
|
+
let remainingSessions = 0;
|
|
1350
|
+
let totalSessions = 0;
|
|
1351
|
+
for (const descriptor of descriptors) {
|
|
1352
|
+
const sessionId = stableNativeSessionId(machine, descriptor.harness, descriptor.sourcePath);
|
|
1353
|
+
if (tombstoned.has(sessionId))
|
|
1354
|
+
continue;
|
|
1355
|
+
totalSessions += 1;
|
|
1356
|
+
const size = Math.max(0, Math.floor(Number(descriptor.size) || 0));
|
|
1357
|
+
totalBytes += size;
|
|
1358
|
+
const pending = rewritePending.has(sessionId) ||
|
|
1359
|
+
redactionPending.has(sessionId) ||
|
|
1360
|
+
seen.get(sessionId) !== fingerprints.get(sessionId);
|
|
1361
|
+
if (!pending) {
|
|
1362
|
+
completedBytes += size;
|
|
1363
|
+
continue;
|
|
1364
|
+
}
|
|
1365
|
+
remainingSessions += 1;
|
|
1366
|
+
const cursor = cursors.get(sessionId);
|
|
1367
|
+
if (cursor?.file === descriptor.file) {
|
|
1368
|
+
completedBytes += Math.max(0, Math.min(size, Math.floor(Number(cursor.offset) || 0)));
|
|
1369
|
+
}
|
|
1370
|
+
}
|
|
1371
|
+
const safe = (value) => Math.min(Number.MAX_SAFE_INTEGER, Math.max(0, Math.floor(value)));
|
|
1372
|
+
return {
|
|
1373
|
+
totalSessions,
|
|
1374
|
+
remainingSessions,
|
|
1375
|
+
totalBytes: safe(totalBytes),
|
|
1376
|
+
completedBytes: safe(Math.min(totalBytes, completedBytes)),
|
|
1377
|
+
};
|
|
1378
|
+
}
|
|
1379
|
+
async function uploadNative(options, machine, redactor = DEFAULT_REDACTOR) {
|
|
1380
|
+
let initialSessions = await nativeInventory(options, machine);
|
|
1381
|
+
const descriptorRegistry = { sessions: initialSessions };
|
|
1382
|
+
const byHarness = new Map();
|
|
1383
|
+
for (const session of initialSessions)
|
|
1384
|
+
byHarness.set(session.harness, (byHarness.get(session.harness) || 0) + 1);
|
|
1385
|
+
process.stdout.write(`Native inventory: ${initialSessions.length} sessions across ${byHarness.size} harnesses\n`);
|
|
1386
|
+
for (const [harness, count] of [...byHarness].sort())
|
|
1387
|
+
process.stdout.write(` ${harness}: ${count}\n`);
|
|
1388
|
+
if (options.dryRun)
|
|
1389
|
+
return;
|
|
1390
|
+
const state = createNativeUploadState(options.nativeStateFile, machine);
|
|
1391
|
+
const seen = new Map();
|
|
1392
|
+
const cursors = new Map();
|
|
1393
|
+
const fingerprints = new Map();
|
|
1394
|
+
const descriptorBySession = new Map();
|
|
1395
|
+
const sessionByFile = new Map();
|
|
1396
|
+
const physicalStats = new Map();
|
|
1397
|
+
const redactionPending = new Set();
|
|
1398
|
+
const redactionStarted = new Set();
|
|
1399
|
+
const livePriority = new Set();
|
|
1400
|
+
const tombstoned = new Set();
|
|
1401
|
+
let nativeMonitor = null;
|
|
1402
|
+
const rememberInventory = (sessions, retainState = false, forceMetadataDirectories = new Set()) => {
|
|
1403
|
+
const previousFingerprints = new Map(fingerprints);
|
|
1404
|
+
descriptorBySession.clear();
|
|
1405
|
+
sessionByFile.clear();
|
|
1406
|
+
fingerprints.clear();
|
|
1407
|
+
physicalStats.clear();
|
|
1408
|
+
for (const descriptor of sessions) {
|
|
1409
|
+
const sessionId = stableNativeSessionId(machine, descriptor.harness, descriptor.sourcePath);
|
|
1410
|
+
descriptorBySession.set(sessionId, descriptor);
|
|
1411
|
+
sessionByFile.set(path.resolve(descriptor.file), sessionId);
|
|
1412
|
+
try {
|
|
1413
|
+
const stat = fs.statSync(descriptor.file);
|
|
1414
|
+
physicalStats.set(sessionId, { size: stat.size, mtimeMs: stat.mtimeMs });
|
|
1415
|
+
}
|
|
1416
|
+
catch { }
|
|
1417
|
+
const saved = state.get(sessionId);
|
|
1418
|
+
const previousFingerprint = previousFingerprints.get(sessionId) || String(saved?.fingerprint || '');
|
|
1419
|
+
let fingerprint = '';
|
|
1420
|
+
try {
|
|
1421
|
+
const previous = JSON.parse(previousFingerprint);
|
|
1422
|
+
const metadataForced = forceMetadataDirectories.has(path.dirname(path.resolve(descriptor.file)));
|
|
1423
|
+
if (!metadataForced &&
|
|
1424
|
+
Number(previous?.[0]) === Number(descriptor.size) &&
|
|
1425
|
+
Number(previous?.[1]) === Number(descriptor.mtimeMs) &&
|
|
1426
|
+
String(previous?.[3] || '') === String(descriptor.parentSessionId || '') &&
|
|
1427
|
+
Number(previous?.[4]) === NATIVE_DESCRIPTOR_VERSION &&
|
|
1428
|
+
String(previous?.[5] || '') === nativeDescriptorMetadataFingerprint(descriptor)) {
|
|
1429
|
+
fingerprint = previousFingerprint;
|
|
1430
|
+
}
|
|
1431
|
+
}
|
|
1432
|
+
catch { }
|
|
1433
|
+
fingerprints.set(sessionId, fingerprint || nativeDescriptorFingerprint(descriptor));
|
|
1434
|
+
if (saved?.tombstoned === true) {
|
|
1435
|
+
tombstoned.add(sessionId);
|
|
1436
|
+
continue;
|
|
1437
|
+
}
|
|
1438
|
+
const redactionCurrent = Number(saved?.redactionVersion || 0) === REDACTION_VERSION;
|
|
1439
|
+
if (saved && redactionCurrent && !seen.has(sessionId)) {
|
|
1440
|
+
seen.set(sessionId, String(saved.fingerprint || ''));
|
|
1441
|
+
}
|
|
1442
|
+
if (saved?.cursor && redactionCurrent && !cursors.has(sessionId)) {
|
|
1443
|
+
cursors.set(sessionId, { ...saved.cursor });
|
|
1444
|
+
}
|
|
1445
|
+
if (saved && !redactionCurrent) {
|
|
1446
|
+
redactionPending.add(sessionId);
|
|
1447
|
+
// A stale-version session is urgent at startup only when its source is
|
|
1448
|
+
// genuinely recent. Fingerprints can differ for old metadata/schema
|
|
1449
|
+
// reasons and must not promote the entire historical fleet.
|
|
1450
|
+
if (Date.now() - Number(descriptor.mtimeMs || 0) <= RECENT_SESSION_PRIORITY_MS) {
|
|
1451
|
+
livePriority.add(sessionId);
|
|
1452
|
+
}
|
|
1453
|
+
}
|
|
1454
|
+
else if (saved && redactionCurrent && !saved.fingerprint) {
|
|
1455
|
+
// An interrupted replay stores a current version with an empty
|
|
1456
|
+
// fingerprint. Resume it cooperatively rather than treating it as an
|
|
1457
|
+
// ordinary live append after a daemon restart.
|
|
1458
|
+
redactionPending.add(sessionId);
|
|
1459
|
+
redactionStarted.add(sessionId);
|
|
1460
|
+
}
|
|
1461
|
+
}
|
|
1462
|
+
if (retainState && !options.only && !options.rewriteSession)
|
|
1463
|
+
state.retain(descriptorBySession.keys());
|
|
1464
|
+
descriptorRegistry.sessions = sessions;
|
|
1465
|
+
nativeMonitor?.addFiles(sessions.map((descriptor) => descriptor.file));
|
|
1466
|
+
};
|
|
1467
|
+
rememberInventory(initialSessions);
|
|
1468
|
+
// A single request-id-multiplexed transport carries ingestion, ownership,
|
|
1469
|
+
// queue control, and live rows. These tasks remain concurrent without
|
|
1470
|
+
// duplicating authentication or machine-presence sockets.
|
|
1471
|
+
let rpc = options.watch ? new SharedUploaderConnection(options, machine) : await connectUploader(options, machine);
|
|
1472
|
+
// Only the shared connection has a `current()` handle, and only the watch path builds one.
|
|
1473
|
+
// Only the shared connection has a `current()` handle, and only the watch path builds one.
|
|
1474
|
+
if (options.watch)
|
|
1475
|
+
await rpc.current();
|
|
1476
|
+
if (options.watch)
|
|
1477
|
+
void watchExternalManagement(rpc, machine, () => descriptorRegistry.sessions);
|
|
1478
|
+
// Paths an agent wrote about are resolved from the rows themselves and
|
|
1479
|
+
// recorded on the session's own log, in both the one-shot and watching
|
|
1480
|
+
// uploaders: a session uploaded once and never revisited still says what its
|
|
1481
|
+
// prose referred to.
|
|
1482
|
+
const resolvedPaths = createPathResolver({
|
|
1483
|
+
options,
|
|
1484
|
+
machine,
|
|
1485
|
+
baseUrl: httpBaseUrl(options.url),
|
|
1486
|
+
redactor,
|
|
1487
|
+
appendResolutions: async (sessionId, resolutions) => {
|
|
1488
|
+
// Resolutions are ordinary rows in the session's log, written the way the
|
|
1489
|
+
// anchor writes its own: one request per bounded batch.
|
|
1490
|
+
for (let start = 0; start < resolutions.length; start += MAX_RUNTIME_EVENT_BATCH) {
|
|
1491
|
+
await rpc.request({
|
|
1492
|
+
type: 'session.runtime.append',
|
|
1493
|
+
sessionId,
|
|
1494
|
+
events: resolutions.slice(start, start + MAX_RUNTIME_EVENT_BATCH).map((resolution) => ({
|
|
1495
|
+
// Keyed by what it resolved, so replaying a rewritten transcript
|
|
1496
|
+
// records the same substring once.
|
|
1497
|
+
eventKey: `path:${resolution.text}`.slice(0, 400),
|
|
1498
|
+
kind: 'session.path_resolution',
|
|
1499
|
+
value: JSON.stringify({ paths: [resolution] }),
|
|
1500
|
+
})),
|
|
1501
|
+
});
|
|
1502
|
+
}
|
|
1503
|
+
},
|
|
1504
|
+
});
|
|
1505
|
+
if (options.watch) {
|
|
1506
|
+
void watchQueuedPrompts({
|
|
1507
|
+
rpc,
|
|
1508
|
+
options,
|
|
1509
|
+
machine,
|
|
1510
|
+
getDescriptors: () => descriptorRegistry.sessions,
|
|
1511
|
+
stableSessionId: stableNativeSessionId,
|
|
1512
|
+
redactor,
|
|
1513
|
+
}).catch((error) => {
|
|
1514
|
+
process.stderr.write(`Queue worker stopped unexpectedly: ${error?.stack || error}\n`);
|
|
1515
|
+
});
|
|
1516
|
+
}
|
|
1517
|
+
const totals = { current: 0, uploaded: 0, rows: 0, empty: 0, failed: 0 };
|
|
1518
|
+
const failures = new Map();
|
|
1519
|
+
const rewritePending = new Set(options.rewriteSession ? [options.rewriteSession] : []);
|
|
1520
|
+
let uploadProgressTimer = null;
|
|
1521
|
+
let lastUploadProgress = '';
|
|
1522
|
+
let uploadProgressError = '';
|
|
1523
|
+
const reportUploadProgress = async (force = false) => {
|
|
1524
|
+
const progress = nativeUploadProgressSnapshot({
|
|
1525
|
+
machine,
|
|
1526
|
+
descriptors: descriptorRegistry.sessions,
|
|
1527
|
+
seen,
|
|
1528
|
+
fingerprints,
|
|
1529
|
+
cursors,
|
|
1530
|
+
redactionPending,
|
|
1531
|
+
rewritePending,
|
|
1532
|
+
tombstoned,
|
|
1533
|
+
});
|
|
1534
|
+
const signature = JSON.stringify(progress);
|
|
1535
|
+
if (!force && signature === lastUploadProgress)
|
|
1536
|
+
return;
|
|
1537
|
+
lastUploadProgress = signature;
|
|
1538
|
+
try {
|
|
1539
|
+
await rpc.request({
|
|
1540
|
+
type: 'machine.upload_progress',
|
|
1541
|
+
machineId: machine,
|
|
1542
|
+
...progress,
|
|
1543
|
+
});
|
|
1544
|
+
uploadProgressError = '';
|
|
1545
|
+
}
|
|
1546
|
+
catch (error) {
|
|
1547
|
+
lastUploadProgress = '';
|
|
1548
|
+
const message = error?.message || String(error);
|
|
1549
|
+
if (message !== uploadProgressError) {
|
|
1550
|
+
uploadProgressError = message;
|
|
1551
|
+
process.stderr.write(`Upload progress report failed: ${message}\n`);
|
|
1552
|
+
}
|
|
1553
|
+
}
|
|
1554
|
+
};
|
|
1555
|
+
const scheduleUploadProgress = () => {
|
|
1556
|
+
if (uploadProgressTimer)
|
|
1557
|
+
return;
|
|
1558
|
+
uploadProgressTimer = setTimeout(() => {
|
|
1559
|
+
uploadProgressTimer = null;
|
|
1560
|
+
void reportUploadProgress();
|
|
1561
|
+
}, 500);
|
|
1562
|
+
uploadProgressTimer.unref?.();
|
|
1563
|
+
};
|
|
1564
|
+
if (rpc instanceof SharedUploaderConnection) {
|
|
1565
|
+
rpc.onConnect(() => {
|
|
1566
|
+
lastUploadProgress = '';
|
|
1567
|
+
scheduleUploadProgress();
|
|
1568
|
+
});
|
|
1569
|
+
}
|
|
1570
|
+
rpc.onEvent((message) => {
|
|
1571
|
+
if (message.type !== 'session_tombstoned' || !message.sessionId)
|
|
1572
|
+
return;
|
|
1573
|
+
resolvedPaths.forget(message.sessionId);
|
|
1574
|
+
tombstoned.add(message.sessionId);
|
|
1575
|
+
redactionPending.delete(message.sessionId);
|
|
1576
|
+
redactionStarted.delete(message.sessionId);
|
|
1577
|
+
rewritePending.delete(message.sessionId);
|
|
1578
|
+
livePriority.delete(message.sessionId);
|
|
1579
|
+
failures.delete(message.sessionId);
|
|
1580
|
+
state.tombstone(message.sessionId);
|
|
1581
|
+
scheduleUploadProgress();
|
|
1582
|
+
});
|
|
1583
|
+
await reportUploadProgress(true);
|
|
1584
|
+
/**
|
|
1585
|
+
* @param {any[]} changed
|
|
1586
|
+
* @param {object} [options]
|
|
1587
|
+
* @param {(sessionId: string) => number} [options.batchLimitFor]
|
|
1588
|
+
* @param {(sessionId: string) => boolean} [options.shouldYieldFor]
|
|
1589
|
+
*/
|
|
1590
|
+
const processChanged = async (changed, { batchLimitFor = (_sessionId) => Number.POSITIVE_INFINITY, shouldYieldFor = (_sessionId) => false, } = {}) => {
|
|
1591
|
+
for (const [index, descriptor] of changed.entries()) {
|
|
1592
|
+
const sessionId = stableNativeSessionId(machine, descriptor.harness, descriptor.sourcePath);
|
|
1593
|
+
const fingerprint = fingerprints.get(sessionId) || nativeDescriptorFingerprint(descriptor);
|
|
1594
|
+
let cursor = cursors.get(sessionId);
|
|
1595
|
+
if (!cursor) {
|
|
1596
|
+
cursor = {};
|
|
1597
|
+
cursors.set(sessionId, cursor);
|
|
1598
|
+
}
|
|
1599
|
+
let result;
|
|
1600
|
+
let failure;
|
|
1601
|
+
for (let attempt = 0; attempt <= MAX_SOCKET_RETRIES; attempt += 1) {
|
|
1602
|
+
try {
|
|
1603
|
+
if (rewritePending.has(sessionId) || (redactionPending.has(sessionId) && !redactionStarted.has(sessionId))) {
|
|
1604
|
+
const redactionUpgrade = redactionPending.has(sessionId);
|
|
1605
|
+
await rpc.request({ type: 'session.reset', sessionId, redactionVersion: REDACTION_VERSION });
|
|
1606
|
+
resolvedPaths.forget(sessionId);
|
|
1607
|
+
rewritePending.delete(sessionId);
|
|
1608
|
+
if (redactionUpgrade)
|
|
1609
|
+
redactionStarted.add(sessionId);
|
|
1610
|
+
cursor = {};
|
|
1611
|
+
cursors.set(sessionId, cursor);
|
|
1612
|
+
seen.delete(sessionId);
|
|
1613
|
+
process.stdout.write(` ${descriptor.harness} ${descriptor.sourcePath}: reset for ${redactionUpgrade ? 'secret-filtered' : 'full'} replay\n`);
|
|
1614
|
+
}
|
|
1615
|
+
result = await uploadNativeSession(rpc, machine, descriptor, cursor, (progress) => {
|
|
1616
|
+
// Persist an explicitly incomplete fingerprint while a long replay
|
|
1617
|
+
// is in flight. A daemon restart can resume from this byte cursor,
|
|
1618
|
+
// but the empty fingerprint still schedules the unfinished tail.
|
|
1619
|
+
state.set(sessionId, '', cursor, REDACTION_VERSION);
|
|
1620
|
+
scheduleUploadProgress();
|
|
1621
|
+
process.stdout.write(` ${index + 1}/${changed.length} ${descriptor.harness}: ` +
|
|
1622
|
+
`${progress.nextRow}/${progress.total} rows` +
|
|
1623
|
+
`${progress.archived ? ' · archived' : ''}\n`);
|
|
1624
|
+
}, redactor, {
|
|
1625
|
+
maxBatches: batchLimitFor(sessionId),
|
|
1626
|
+
shouldYield: () => shouldYieldFor(sessionId),
|
|
1627
|
+
onPathCandidates: (uploadSessionId, workingDirectory, candidates) => {
|
|
1628
|
+
void resolvedPaths.enqueue(uploadSessionId, workingDirectory, candidates);
|
|
1629
|
+
},
|
|
1630
|
+
});
|
|
1631
|
+
break;
|
|
1632
|
+
}
|
|
1633
|
+
catch (error) {
|
|
1634
|
+
if (!isSocketError(error) || attempt === MAX_SOCKET_RETRIES) {
|
|
1635
|
+
failure = error;
|
|
1636
|
+
break;
|
|
1637
|
+
}
|
|
1638
|
+
if (rpc instanceof SharedUploaderConnection) {
|
|
1639
|
+
// The shared transport already performed bounded reconnection for
|
|
1640
|
+
// every concurrent consumer. Leave this session pending for the
|
|
1641
|
+
// next scheduler sweep rather than replacing the shared object.
|
|
1642
|
+
failure = error;
|
|
1643
|
+
break;
|
|
1644
|
+
}
|
|
1645
|
+
process.stderr.write(` ${index + 1}/${changed.length} ${descriptor.harness}: ${error.message}; ` +
|
|
1646
|
+
`reconnecting (${attempt + 1}/${MAX_SOCKET_RETRIES})\n`);
|
|
1647
|
+
rpc.close();
|
|
1648
|
+
await wait(reconnectDelay(attempt));
|
|
1649
|
+
rpc = await connectUploader(options, machine, true);
|
|
1650
|
+
}
|
|
1651
|
+
}
|
|
1652
|
+
if (!failure && result) {
|
|
1653
|
+
if (result.tombstoned) {
|
|
1654
|
+
tombstoned.add(sessionId);
|
|
1655
|
+
redactionPending.delete(sessionId);
|
|
1656
|
+
redactionStarted.delete(sessionId);
|
|
1657
|
+
rewritePending.delete(sessionId);
|
|
1658
|
+
livePriority.delete(sessionId);
|
|
1659
|
+
failures.delete(sessionId);
|
|
1660
|
+
state.tombstone(sessionId);
|
|
1661
|
+
totals.current += 1;
|
|
1662
|
+
process.stdout.write(` ${index + 1}/${changed.length} ${descriptor.harness}: deleted on turns.dev\n`);
|
|
1663
|
+
scheduleUploadProgress();
|
|
1664
|
+
continue;
|
|
1665
|
+
}
|
|
1666
|
+
if (result.partial) {
|
|
1667
|
+
state.set(sessionId, '', cursor, REDACTION_VERSION);
|
|
1668
|
+
totals.rows += result.sent;
|
|
1669
|
+
scheduleUploadProgress();
|
|
1670
|
+
continue;
|
|
1671
|
+
}
|
|
1672
|
+
seen.set(sessionId, fingerprint);
|
|
1673
|
+
state.set(sessionId, fingerprint, cursor, REDACTION_VERSION);
|
|
1674
|
+
redactionPending.delete(sessionId);
|
|
1675
|
+
redactionStarted.delete(sessionId);
|
|
1676
|
+
livePriority.delete(sessionId);
|
|
1677
|
+
failures.delete(sessionId);
|
|
1678
|
+
if (result.empty)
|
|
1679
|
+
totals.empty += 1;
|
|
1680
|
+
else if (result.skipped)
|
|
1681
|
+
totals.current += 1;
|
|
1682
|
+
else {
|
|
1683
|
+
totals.uploaded += 1;
|
|
1684
|
+
totals.rows += result.sent;
|
|
1685
|
+
}
|
|
1686
|
+
if (!result.skipped || (index + 1) % 50 === 0 || index + 1 === changed.length) {
|
|
1687
|
+
process.stdout.write(` ${index + 1}/${changed.length} ${descriptor.harness} ${descriptor.sourcePath}: ` +
|
|
1688
|
+
`${result.empty ? 'no conversation rows' : result.skipped ? `current (${result.rows} rows)` : `${result.sent} rows uploaded`}\n`);
|
|
1689
|
+
}
|
|
1690
|
+
scheduleUploadProgress();
|
|
1691
|
+
}
|
|
1692
|
+
else {
|
|
1693
|
+
const error = failure || new Error('Upload ended without a result.');
|
|
1694
|
+
const attempts = Number(failures.get(sessionId)?.attempts || 0) + 1;
|
|
1695
|
+
failures.set(sessionId, {
|
|
1696
|
+
harness: descriptor.harness,
|
|
1697
|
+
sourcePath: descriptor.sourcePath,
|
|
1698
|
+
error: error?.message || String(error),
|
|
1699
|
+
attempts,
|
|
1700
|
+
retryAt: Date.now() + nativeFailureRetryDelay(attempts),
|
|
1701
|
+
});
|
|
1702
|
+
process.stderr.write(` ${index + 1}/${changed.length} ${descriptor.harness} ${descriptor.sourcePath}: ${error?.message || error}\n`);
|
|
1703
|
+
scheduleUploadProgress();
|
|
1704
|
+
}
|
|
1705
|
+
}
|
|
1706
|
+
};
|
|
1707
|
+
const changedSessions = () => descriptorRegistry.sessions.filter((descriptor) => {
|
|
1708
|
+
const sessionId = stableNativeSessionId(machine, descriptor.harness, descriptor.sourcePath);
|
|
1709
|
+
if (tombstoned.has(sessionId))
|
|
1710
|
+
return false;
|
|
1711
|
+
return (rewritePending.has(sessionId) ||
|
|
1712
|
+
redactionPending.has(sessionId) ||
|
|
1713
|
+
seen.get(sessionId) !== fingerprints.get(sessionId));
|
|
1714
|
+
});
|
|
1715
|
+
if (!options.watch) {
|
|
1716
|
+
let settlePass = 0;
|
|
1717
|
+
for (;;) {
|
|
1718
|
+
const changed = changedSessions();
|
|
1719
|
+
await processChanged(changed);
|
|
1720
|
+
if (!changed.length)
|
|
1721
|
+
break;
|
|
1722
|
+
settlePass += 1;
|
|
1723
|
+
if (settlePass >= MAX_NATIVE_SETTLE_PASSES) {
|
|
1724
|
+
const sessions = await nativeInventory(options, machine);
|
|
1725
|
+
rememberInventory(sessions);
|
|
1726
|
+
const unsettled = changedSessions();
|
|
1727
|
+
if (unsettled.length) {
|
|
1728
|
+
process.stderr.write(`${unsettled.length} native sessions kept changing; use --watch to follow them continuously.\n`);
|
|
1729
|
+
}
|
|
1730
|
+
break;
|
|
1731
|
+
}
|
|
1732
|
+
await wait(250);
|
|
1733
|
+
const sessions = await nativeInventory(options, machine);
|
|
1734
|
+
rememberInventory(sessions);
|
|
1735
|
+
}
|
|
1736
|
+
state.close();
|
|
1737
|
+
if (uploadProgressTimer) {
|
|
1738
|
+
clearTimeout(uploadProgressTimer);
|
|
1739
|
+
uploadProgressTimer = null;
|
|
1740
|
+
}
|
|
1741
|
+
await reportUploadProgress(true);
|
|
1742
|
+
rpc.close();
|
|
1743
|
+
totals.failed = failures.size;
|
|
1744
|
+
process.stdout.write(`Native upload complete: ${JSON.stringify(totals)}\n`);
|
|
1745
|
+
if (failures.size) {
|
|
1746
|
+
process.stderr.write(`${JSON.stringify({ failures: [...failures.values()] }, null, 2)}\n`);
|
|
1747
|
+
process.exitCode = 1;
|
|
1748
|
+
}
|
|
1749
|
+
return;
|
|
1750
|
+
}
|
|
1751
|
+
let wake = null;
|
|
1752
|
+
let sweepPending = true;
|
|
1753
|
+
let discoveryPending = false;
|
|
1754
|
+
let discoveryDebounce = null;
|
|
1755
|
+
const pendingMetadataDirectories = new Set();
|
|
1756
|
+
const fileDebounces = new Map();
|
|
1757
|
+
const requestSweep = () => {
|
|
1758
|
+
sweepPending = true;
|
|
1759
|
+
if (wake) {
|
|
1760
|
+
const resolve = wake;
|
|
1761
|
+
wake = null;
|
|
1762
|
+
resolve();
|
|
1763
|
+
}
|
|
1764
|
+
};
|
|
1765
|
+
const waitForSweep = () => sweepPending
|
|
1766
|
+
? Promise.resolve()
|
|
1767
|
+
: new Promise((resolve) => {
|
|
1768
|
+
wake = resolve;
|
|
1769
|
+
});
|
|
1770
|
+
const waitForSweepOrTimeout = (milliseconds) => sweepPending
|
|
1771
|
+
? Promise.resolve()
|
|
1772
|
+
: new Promise((resolve) => {
|
|
1773
|
+
let timer;
|
|
1774
|
+
const resume = () => {
|
|
1775
|
+
clearTimeout(timer);
|
|
1776
|
+
if (wake === resume)
|
|
1777
|
+
wake = null;
|
|
1778
|
+
resolve();
|
|
1779
|
+
};
|
|
1780
|
+
timer = setTimeout(resume, milliseconds);
|
|
1781
|
+
timer.unref?.();
|
|
1782
|
+
wake = resume;
|
|
1783
|
+
});
|
|
1784
|
+
const scheduleDiscovery = (changedPath) => {
|
|
1785
|
+
if (changedPath)
|
|
1786
|
+
pendingMetadataDirectories.add(path.dirname(path.resolve(changedPath)));
|
|
1787
|
+
if (discoveryDebounce)
|
|
1788
|
+
return;
|
|
1789
|
+
discoveryDebounce = setTimeout(() => {
|
|
1790
|
+
discoveryDebounce = null;
|
|
1791
|
+
discoveryPending = true;
|
|
1792
|
+
requestSweep();
|
|
1793
|
+
}, NATIVE_DISCOVERY_DEBOUNCE_MS);
|
|
1794
|
+
discoveryDebounce.unref?.();
|
|
1795
|
+
};
|
|
1796
|
+
const refreshDescriptor = (sessionId) => {
|
|
1797
|
+
const descriptor = descriptorBySession.get(sessionId);
|
|
1798
|
+
if (!descriptor)
|
|
1799
|
+
return;
|
|
1800
|
+
let stat;
|
|
1801
|
+
try {
|
|
1802
|
+
stat = fs.statSync(descriptor.file);
|
|
1803
|
+
}
|
|
1804
|
+
catch {
|
|
1805
|
+
return;
|
|
1806
|
+
}
|
|
1807
|
+
if (!stat.isFile())
|
|
1808
|
+
return;
|
|
1809
|
+
const previous = physicalStats.get(sessionId);
|
|
1810
|
+
if (previous && stat.size === previous.size && stat.mtimeMs === previous.mtimeMs)
|
|
1811
|
+
return;
|
|
1812
|
+
physicalStats.set(sessionId, { size: stat.size, mtimeMs: stat.mtimeMs });
|
|
1813
|
+
failures.delete(sessionId);
|
|
1814
|
+
descriptor.size = stat.size;
|
|
1815
|
+
descriptor.mtimeMs = stat.mtimeMs;
|
|
1816
|
+
let fingerprint = '';
|
|
1817
|
+
try {
|
|
1818
|
+
const previousFingerprint = JSON.parse(fingerprints.get(sessionId) || 'null');
|
|
1819
|
+
const workingDirectory = String(previousFingerprint?.[2] || '');
|
|
1820
|
+
if (workingDirectory) {
|
|
1821
|
+
fingerprint = JSON.stringify([
|
|
1822
|
+
stat.size,
|
|
1823
|
+
stat.mtimeMs,
|
|
1824
|
+
workingDirectory,
|
|
1825
|
+
String(descriptor.parentSessionId || ''),
|
|
1826
|
+
NATIVE_DESCRIPTOR_VERSION,
|
|
1827
|
+
String(previousFingerprint?.[5] || nativeDescriptorMetadataFingerprint(descriptor)),
|
|
1828
|
+
]);
|
|
1829
|
+
}
|
|
1830
|
+
}
|
|
1831
|
+
catch { }
|
|
1832
|
+
// Once discovered, cwd is immutable for a native session. Sessions whose
|
|
1833
|
+
// cwd has not appeared yet retain the bounded metadata probe on append.
|
|
1834
|
+
fingerprints.set(sessionId, fingerprint || nativeDescriptorFingerprint(descriptor));
|
|
1835
|
+
livePriority.add(sessionId);
|
|
1836
|
+
requestSweep();
|
|
1837
|
+
};
|
|
1838
|
+
const refreshDescriptorFile = (changedFile) => {
|
|
1839
|
+
const sessionId = sessionByFile.get(path.resolve(changedFile));
|
|
1840
|
+
if (!sessionId) {
|
|
1841
|
+
scheduleDiscovery(changedFile);
|
|
1842
|
+
return;
|
|
1843
|
+
}
|
|
1844
|
+
if (fileDebounces.has(sessionId))
|
|
1845
|
+
return;
|
|
1846
|
+
const timer = setTimeout(() => {
|
|
1847
|
+
fileDebounces.delete(sessionId);
|
|
1848
|
+
refreshDescriptor(sessionId);
|
|
1849
|
+
}, NATIVE_CHANGE_DEBOUNCE_MS);
|
|
1850
|
+
timer.unref?.();
|
|
1851
|
+
fileDebounces.set(sessionId, timer);
|
|
1852
|
+
};
|
|
1853
|
+
nativeMonitor = watchNativeSessionDirectories({
|
|
1854
|
+
roots: nativeSessionWatchRoots(),
|
|
1855
|
+
shallowRoots: nativeSessionShallowWatchRoots(),
|
|
1856
|
+
files: descriptorRegistry.sessions.map((descriptor) => descriptor.file),
|
|
1857
|
+
onFile: refreshDescriptorFile,
|
|
1858
|
+
onDirectory: (directory) => scheduleDiscovery(path.join(directory, '.directory')),
|
|
1859
|
+
});
|
|
1860
|
+
const pollTimer = setInterval(() => {
|
|
1861
|
+
for (const descriptor of descriptorRegistry.sessions) {
|
|
1862
|
+
// Database projections are refreshed from their authoritative store by
|
|
1863
|
+
// discovery. Polling the projection's synthetic mtime is misleading.
|
|
1864
|
+
if (descriptor.sourcePath.includes('.db#'))
|
|
1865
|
+
continue;
|
|
1866
|
+
const sessionId = stableNativeSessionId(machine, descriptor.harness, descriptor.sourcePath);
|
|
1867
|
+
refreshDescriptor(sessionId);
|
|
1868
|
+
}
|
|
1869
|
+
}, NATIVE_POLL_INTERVAL_MS);
|
|
1870
|
+
pollTimer.unref?.();
|
|
1871
|
+
const discoveryTimer = setInterval(() => {
|
|
1872
|
+
discoveryPending = true;
|
|
1873
|
+
requestSweep();
|
|
1874
|
+
}, NATIVE_DISCOVERY_INTERVAL_MS);
|
|
1875
|
+
discoveryTimer.unref?.();
|
|
1876
|
+
const shutdown = () => {
|
|
1877
|
+
state.close();
|
|
1878
|
+
nativeMonitor.close();
|
|
1879
|
+
clearInterval(pollTimer);
|
|
1880
|
+
clearInterval(discoveryTimer);
|
|
1881
|
+
};
|
|
1882
|
+
process.once('SIGTERM', () => {
|
|
1883
|
+
shutdown();
|
|
1884
|
+
process.exit(0);
|
|
1885
|
+
});
|
|
1886
|
+
process.once('SIGINT', () => {
|
|
1887
|
+
shutdown();
|
|
1888
|
+
process.exit(0);
|
|
1889
|
+
});
|
|
1890
|
+
process.stdout.write(`Watching ${initialSessions.length} native sessions via ${nativeMonitor.count} directories; ` +
|
|
1891
|
+
`polling known files every ${NATIVE_POLL_INTERVAL_MS / 1000}s and auditing every ` +
|
|
1892
|
+
`${NATIVE_DISCOVERY_INTERVAL_MS / 1000}s…\n`);
|
|
1893
|
+
let historicalSessionId = '';
|
|
1894
|
+
for (;;) {
|
|
1895
|
+
await waitForSweep();
|
|
1896
|
+
sweepPending = false;
|
|
1897
|
+
if (discoveryPending) {
|
|
1898
|
+
discoveryPending = false;
|
|
1899
|
+
try {
|
|
1900
|
+
initialSessions = await nativeInventory(options, machine);
|
|
1901
|
+
const forceMetadataDirectories = new Set(pendingMetadataDirectories);
|
|
1902
|
+
pendingMetadataDirectories.clear();
|
|
1903
|
+
rememberInventory(initialSessions, true, forceMetadataDirectories);
|
|
1904
|
+
scheduleUploadProgress();
|
|
1905
|
+
}
|
|
1906
|
+
catch (error) {
|
|
1907
|
+
process.stderr.write(`Native inventory refresh failed: ${error?.message || error}\n`);
|
|
1908
|
+
}
|
|
1909
|
+
}
|
|
1910
|
+
const changed = changedSessions();
|
|
1911
|
+
const active = changed
|
|
1912
|
+
.filter((descriptor) => {
|
|
1913
|
+
const sessionId = stableNativeSessionId(machine, descriptor.harness, descriptor.sourcePath);
|
|
1914
|
+
if (Number(failures.get(sessionId)?.retryAt || 0) > Date.now())
|
|
1915
|
+
return false;
|
|
1916
|
+
return rewritePending.has(sessionId) || livePriority.has(sessionId) || !redactionPending.has(sessionId);
|
|
1917
|
+
})
|
|
1918
|
+
.sort((left, right) => Number(right.mtimeMs || 0) - Number(left.mtimeMs || 0));
|
|
1919
|
+
let processed;
|
|
1920
|
+
if (active.length) {
|
|
1921
|
+
processed = active;
|
|
1922
|
+
await processChanged(active, {
|
|
1923
|
+
batchLimitFor: (sessionId) => redactionPending.has(sessionId) ? ACTIVE_REPLAY_BATCHES_PER_SLICE : Number.POSITIVE_INFINITY,
|
|
1924
|
+
shouldYieldFor: (sessionId) => [...livePriority].some((other) => other !== sessionId),
|
|
1925
|
+
});
|
|
1926
|
+
}
|
|
1927
|
+
else {
|
|
1928
|
+
const historical = changed.filter((descriptor) => {
|
|
1929
|
+
const sessionId = stableNativeSessionId(machine, descriptor.harness, descriptor.sourcePath);
|
|
1930
|
+
return redactionPending.has(sessionId) && Number(failures.get(sessionId)?.retryAt || 0) <= Date.now();
|
|
1931
|
+
});
|
|
1932
|
+
let descriptor = historical.find((candidate) => stableNativeSessionId(machine, candidate.harness, candidate.sourcePath) === historicalSessionId);
|
|
1933
|
+
if (!descriptor)
|
|
1934
|
+
descriptor = historical[0];
|
|
1935
|
+
processed = descriptor ? [descriptor] : [];
|
|
1936
|
+
if (descriptor) {
|
|
1937
|
+
historicalSessionId = stableNativeSessionId(machine, descriptor.harness, descriptor.sourcePath);
|
|
1938
|
+
await processChanged([descriptor], {
|
|
1939
|
+
batchLimitFor: () => HISTORICAL_REPLAY_BATCHES_PER_SLICE,
|
|
1940
|
+
shouldYieldFor: () => livePriority.size > 0,
|
|
1941
|
+
});
|
|
1942
|
+
if (!redactionPending.has(historicalSessionId))
|
|
1943
|
+
historicalSessionId = '';
|
|
1944
|
+
}
|
|
1945
|
+
}
|
|
1946
|
+
// A file may grow while its upload is in flight. Refresh only those files;
|
|
1947
|
+
// filesystem events normally already did this, and the stat is a cheap
|
|
1948
|
+
// correctness fallback.
|
|
1949
|
+
for (const descriptor of processed)
|
|
1950
|
+
refreshDescriptorFile(descriptor.file);
|
|
1951
|
+
if (changedSessions().length) {
|
|
1952
|
+
// The backfill delay is interruptible: a filesystem event sets
|
|
1953
|
+
// sweepPending and wakes this promise immediately, so active sessions do
|
|
1954
|
+
// not inherit the deliberately slow historical cadence.
|
|
1955
|
+
if (!active.length)
|
|
1956
|
+
await waitForSweepOrTimeout(HISTORICAL_REPLAY_DELAY_MS);
|
|
1957
|
+
requestSweep();
|
|
1958
|
+
}
|
|
1959
|
+
}
|
|
1960
|
+
}
|
|
1961
|
+
export async function runUploader(argv = process.argv.slice(2), { credentialProvider = null } = {}) {
|
|
1962
|
+
const options = argumentsFrom(argv, { allowMissingToken: Boolean(credentialProvider) });
|
|
1963
|
+
options.credentialProvider = credentialProvider;
|
|
1964
|
+
const initialCredentials = credentialProvider ? await credentialProvider.refreshIfDue() : null;
|
|
1965
|
+
if (initialCredentials?.accessToken)
|
|
1966
|
+
options.token = initialCredentials.accessToken;
|
|
1967
|
+
if (!options.dryRun && !options.token && !options.devUser) {
|
|
1968
|
+
throw new Error('Turns authentication did not provide an access token or local DEV_USER.');
|
|
1969
|
+
}
|
|
1970
|
+
const machine = machineId(options.machineIdFile);
|
|
1971
|
+
const filtering = reportingRedactor([options.token, initialCredentials?.refreshToken]);
|
|
1972
|
+
const unsubscribeCredentials = credentialProvider?.onUpdate?.((credentials) => {
|
|
1973
|
+
options.token = credentials.accessToken;
|
|
1974
|
+
filtering.redactor.addLiterals([credentials.accessToken, credentials.refreshToken]);
|
|
1975
|
+
});
|
|
1976
|
+
try {
|
|
1977
|
+
if (options.native) {
|
|
1978
|
+
await uploadNative(options, machine, filtering.redactor);
|
|
1979
|
+
return;
|
|
1980
|
+
}
|
|
1981
|
+
const rpc = await connectUploader(options, machine);
|
|
1982
|
+
const seen = new Map();
|
|
1983
|
+
const sweep = async () => {
|
|
1984
|
+
for (const capture of discover(options.runs, options.only)) {
|
|
1985
|
+
const stat = fs.statSync(capture.file);
|
|
1986
|
+
const fingerprint = `${stat.size}:${stat.mtimeMs}`;
|
|
1987
|
+
if (seen.get(capture.file) === fingerprint)
|
|
1988
|
+
continue;
|
|
1989
|
+
const result = await uploadFile(rpc, machine, options.runs, capture, filtering.redactor);
|
|
1990
|
+
seen.set(capture.file, fingerprint);
|
|
1991
|
+
if (result.skipped)
|
|
1992
|
+
process.stdout.write(` ${capture.relative}: current (${result.rows} rows)\n`);
|
|
1993
|
+
}
|
|
1994
|
+
};
|
|
1995
|
+
await sweep();
|
|
1996
|
+
if (!options.watch) {
|
|
1997
|
+
rpc.close();
|
|
1998
|
+
return;
|
|
1999
|
+
}
|
|
2000
|
+
process.stdout.write('Watching for appended capture rows…\n');
|
|
2001
|
+
for (;;) {
|
|
2002
|
+
await new Promise((resolve) => setTimeout(resolve, 2_000));
|
|
2003
|
+
await sweep();
|
|
2004
|
+
}
|
|
2005
|
+
}
|
|
2006
|
+
finally {
|
|
2007
|
+
unsubscribeCredentials?.();
|
|
2008
|
+
filtering.close();
|
|
2009
|
+
}
|
|
2010
|
+
}
|
|
2011
|
+
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
|
2012
|
+
runUploader().catch((error) => {
|
|
2013
|
+
process.stderr.write(`${error?.stack || error}\n`);
|
|
2014
|
+
process.exitCode = 1;
|
|
2015
|
+
});
|
|
2016
|
+
}
|