@turri-dev/cli 0.1.8-next.e553476 → 0.1.9-next.73bfddf
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -3
- package/dist/cli.js +6 -2
- package/dist/config-edit.d.ts +1 -1
- package/dist/config-edit.js +4 -0
- package/dist/config.d.ts +27 -0
- package/dist/config.js +28 -1
- package/dist/extension.js +75 -9
- package/dist/genome.js +2 -0
- package/dist/host.d.ts +3 -1
- package/dist/host.js +4 -1
- package/dist/init-config.d.ts +3 -0
- package/dist/init-config.js +15 -0
- package/dist/ledger-cli.d.ts +1 -0
- package/dist/ledger-cli.js +50 -0
- package/dist/ledger-tools.d.ts +10 -0
- package/dist/ledger-tools.js +64 -0
- package/dist/loop-cli.js +4 -0
- package/dist/pi.d.ts +2 -0
- package/dist/pi.js +24 -4
- package/dist/regenerate/activity.d.ts +9 -0
- package/dist/regenerate/activity.js +11 -0
- package/dist/regenerate/agent-env.d.ts +7 -0
- package/dist/regenerate/agent-env.js +126 -5
- package/dist/regenerate/agent-shell.js +5 -0
- package/dist/regenerate/chunk-plan.d.ts +28 -9
- package/dist/regenerate/chunk-plan.js +141 -17
- package/dist/regenerate/conformance-agents.js +2 -2
- package/dist/regenerate/conformance.js +6 -3
- package/dist/regenerate/debug-log.d.ts +3 -1
- package/dist/regenerate/debug-log.js +10 -2
- package/dist/regenerate/diagnose.js +12 -2
- package/dist/regenerate/environment-error.d.ts +6 -0
- package/dist/regenerate/environment-error.js +19 -1
- package/dist/regenerate/job-log.d.ts +3 -1
- package/dist/regenerate/job-log.js +8 -5
- package/dist/regenerate/jobs-tree.d.ts +7 -3
- package/dist/regenerate/jobs-tree.js +21 -6
- package/dist/regenerate/jobs.d.ts +9 -0
- package/dist/regenerate/jobs.js +51 -5
- package/dist/regenerate/ledger-view.d.ts +92 -0
- package/dist/regenerate/ledger-view.js +345 -0
- package/dist/regenerate/ledger.d.ts +275 -0
- package/dist/regenerate/ledger.js +349 -0
- package/dist/regenerate/lineage.d.ts +9 -1
- package/dist/regenerate/lineage.js +64 -12
- package/dist/regenerate/loop.js +12 -3
- package/dist/regenerate/pulse-files.d.ts +14 -0
- package/dist/regenerate/pulse-files.js +63 -0
- package/dist/regenerate/regenerate.js +7 -5
- package/dist/regenerate/regression.js +4 -2
- package/dist/regenerate/services.js +3 -0
- package/dist/regenerate/side-panel.d.ts +2 -0
- package/dist/regenerate/side-panel.js +4 -3
- package/dist/regenerate/test-scripts.js +18 -5
- package/dist/regenerate/testing.js +3 -0
- package/dist/regenerate/unit-review.js +3 -0
- package/dist/run-report.js +9 -5
- package/docs/config.md +23 -2
- package/docs/development.md +3 -1
- package/docs/headless.md +5 -4
- package/docs/hosting.md +29 -0
- package/docs/installation.md +1 -0
- package/docs/regenerating.md +11 -9
- package/docs/sandbox.md +5 -5
- package/docs/using.md +36 -5
- package/package.json +1 -1
package/dist/regenerate/jobs.js
CHANGED
|
@@ -3,8 +3,10 @@ import { availableParallelism } from "node:os";
|
|
|
3
3
|
import { withPulse } from "./shell.js";
|
|
4
4
|
import { describeEnvironmentError, EnvironmentError, missingCredentials } from "./environment-error.js";
|
|
5
5
|
import { addAgentReport, describeAgents, JobStoppedError, withActivity } from "./activity.js";
|
|
6
|
-
import { startJobLog } from "./job-log.js";
|
|
7
|
-
import {
|
|
6
|
+
import { projectOfLanes, startJobLog } from "./job-log.js";
|
|
7
|
+
import { firstLine, healing, record } from "./ledger.js";
|
|
8
|
+
import { chosenStack, debugMode, withStack } from "../config.js";
|
|
9
|
+
import { pulseEnded, pulseWrote } from "./pulse-files.js";
|
|
8
10
|
const DEFAULT_TAIL_CHARS = 4000;
|
|
9
11
|
const ACTIVITY_LINES = 10;
|
|
10
12
|
/** A long render's full transcript is kept, up to about this much text. */
|
|
@@ -88,6 +90,20 @@ export function createJobRegistry(options) {
|
|
|
88
90
|
};
|
|
89
91
|
jobs.set(job.id, job);
|
|
90
92
|
let transcriptChars = 0;
|
|
93
|
+
// Where what it does is recorded: its project, from the modules it writes.
|
|
94
|
+
const project = projectOfLanes(claimed);
|
|
95
|
+
const place = { job: id, ...(project !== undefined ? { project } : {}), ...(input.module !== undefined ? { module: input.module } : {}) };
|
|
96
|
+
record({
|
|
97
|
+
kind: "job.queued",
|
|
98
|
+
jobKind: input.kind,
|
|
99
|
+
summary: firstLine(input.summary, 300),
|
|
100
|
+
...(input.pulse !== undefined ? { pulse: input.pulse } : {}),
|
|
101
|
+
...(input.title !== undefined ? { title: input.title } : {}),
|
|
102
|
+
...(input.model !== undefined ? { model: input.model } : {}),
|
|
103
|
+
...(stack !== undefined ? { stack } : {}),
|
|
104
|
+
waitingOn: blockers,
|
|
105
|
+
...(input.follows !== undefined ? { follows: input.follows } : {}),
|
|
106
|
+
}, place);
|
|
91
107
|
const changed = () => {
|
|
92
108
|
// Like onFinish: a display that throws must not take the job down.
|
|
93
109
|
try {
|
|
@@ -105,6 +121,8 @@ export function createJobRegistry(options) {
|
|
|
105
121
|
const sink = {
|
|
106
122
|
signal: controller.signal,
|
|
107
123
|
job: { id, kind: input.kind, summary: input.summary },
|
|
124
|
+
...(project !== undefined ? { project } : {}),
|
|
125
|
+
...(input.module !== undefined ? { module: input.module } : {}),
|
|
108
126
|
specConflict(conflict) {
|
|
109
127
|
job.specConflict = conflict;
|
|
110
128
|
this.transcript({ kind: "stage", text: `the spec contradicts itself: functionalities ${conflict.specs.map((spec) => `${spec.module} ${spec.frid}`).join(" and ")} cannot both hold` });
|
|
@@ -112,6 +130,7 @@ export function createJobRegistry(options) {
|
|
|
112
130
|
},
|
|
113
131
|
stage(label) {
|
|
114
132
|
job.stage = label;
|
|
133
|
+
record({ kind: "job.stage", label: firstLine(label, 300) }, place);
|
|
115
134
|
this.transcript({ kind: "stage", text: label });
|
|
116
135
|
changed();
|
|
117
136
|
},
|
|
@@ -213,22 +232,24 @@ export function createJobRegistry(options) {
|
|
|
213
232
|
// and the person is told. What the harness found itself is worked out at once; what an agent said goes as it is.
|
|
214
233
|
if (error instanceof EnvironmentError) {
|
|
215
234
|
job.failedForGood = true;
|
|
235
|
+
record({ kind: "environment.error", missing: firstLine(error.missing, 300), source: error.source, sandbox: error.sandbox !== undefined }, place);
|
|
216
236
|
if (error.source === "harness" && options?.diagnose !== undefined) {
|
|
217
237
|
sink.stage("the environment lacks something - working out what to do about it");
|
|
218
|
-
job.diagnosis = await withActivity(sink, () => options.diagnose(job, [error.message, describeEnvironmentError(error, input.rerun)], log?.file)
|
|
238
|
+
job.diagnosis = await withActivity(sink, () => healing("diagnosis", () => options.diagnose(job, [error.message, describeEnvironmentError(error, input.rerun)], log?.file), (found) => found));
|
|
219
239
|
}
|
|
220
240
|
job.environment = describeEnvironmentError(error, input.rerun);
|
|
221
241
|
throw new Error([job.environment, ...(job.diagnosis !== undefined ? ["", job.diagnosis] : [])].join("\n"));
|
|
222
242
|
}
|
|
223
243
|
failures.push(error !== undefined ? (error instanceof Error ? error.message : String(error)) : outcome.report);
|
|
224
244
|
if (attempt < retries) {
|
|
245
|
+
record({ kind: "job.retried", attempt: attempt + 1, of: retries, failure: firstLine(failures.at(-1), 300) }, place);
|
|
225
246
|
sink.stage(`the pulse failed - trying it once more (${attempt + 1} of ${retries})`);
|
|
226
247
|
continue;
|
|
227
248
|
}
|
|
228
249
|
job.failedForGood = true;
|
|
229
250
|
if (options?.diagnose !== undefined) {
|
|
230
251
|
sink.stage("the pulse failed again - working out why, and what has to be done");
|
|
231
|
-
job.diagnosis = await withActivity(sink, () => options.diagnose(job, failures, log?.file)
|
|
252
|
+
job.diagnosis = await withActivity(sink, () => healing("diagnosis", () => options.diagnose(job, failures, log?.file), (found) => found));
|
|
232
253
|
}
|
|
233
254
|
throw new Error([
|
|
234
255
|
`Failed ${failures.length} time${failures.length === 1 ? "" : "s"}, so it and every pulse queued behind it have stopped.`,
|
|
@@ -241,6 +262,8 @@ export function createJobRegistry(options) {
|
|
|
241
262
|
};
|
|
242
263
|
/** It holds one of the run slots, to give back when it ends. */
|
|
243
264
|
let holding = false;
|
|
265
|
+
/** It started, holding its lanes: what it wrote, and what its chain installed, are its to clear as it ends. */
|
|
266
|
+
let ran = false;
|
|
244
267
|
const promise = turn
|
|
245
268
|
.then(async () => {
|
|
246
269
|
// Stopped while it waited: it never starts.
|
|
@@ -275,8 +298,14 @@ export function createJobRegistry(options) {
|
|
|
275
298
|
changed();
|
|
276
299
|
}
|
|
277
300
|
log = startJobLog(job, claimed);
|
|
278
|
-
|
|
301
|
+
ran = true;
|
|
302
|
+
if (log !== undefined) {
|
|
279
303
|
job.logFile = log.file;
|
|
304
|
+
pulseWrote(log.file, id);
|
|
305
|
+
}
|
|
306
|
+
// Its log is named only where it is kept: otherwise it goes as the pulse ends.
|
|
307
|
+
const named = log !== undefined && project !== undefined && debugMode(project);
|
|
308
|
+
record({ kind: "job.started", waitedMs: job.startedAt - job.queuedAt, ...(named ? { logFile: log.file } : {}) }, place);
|
|
280
309
|
// What the job's agents start runs until it ends, and no longer (withPulse) - with the stack it was queued
|
|
281
310
|
// with, whatever is chosen while it waits or runs.
|
|
282
311
|
return withStack(stack, () => (guarded ? runGuarded() : runOnce()));
|
|
@@ -303,6 +332,23 @@ export function createJobRegistry(options) {
|
|
|
303
332
|
}
|
|
304
333
|
finished.finishedAt = Date.now();
|
|
305
334
|
log?.end(finished.status === "succeeded" ? "finished" : finished.status === "stopped" ? "stopped" : "finished - failed", finished.announce ?? finished.report ?? finished.error, describeAgents(finished.agents));
|
|
335
|
+
// What it wrote for a person goes now, outside debug mode - its log too, which nothing names from then on.
|
|
336
|
+
if (ran && project !== undefined && !pulseEnded({ job: id, project, ...(input.module !== undefined ? { module: input.module } : {}), lanes: claimed }))
|
|
337
|
+
delete finished.logFile;
|
|
338
|
+
const outcome = finished.details;
|
|
339
|
+
const stopReason = stopReasons.get(id);
|
|
340
|
+
record({
|
|
341
|
+
kind: "job.finished",
|
|
342
|
+
status: finished.status,
|
|
343
|
+
...(typeof outcome?.landed === "boolean" ? { landed: outcome.landed } : {}),
|
|
344
|
+
...(typeof outcome?.passed === "boolean" ? { passed: outcome.passed } : {}),
|
|
345
|
+
ms: finished.finishedAt - finished.startedAt,
|
|
346
|
+
usage: finished.usage,
|
|
347
|
+
...(finished.failedForGood === true ? { failedForGood: true } : {}),
|
|
348
|
+
...(finished.status === "stopped" ? { stoppedBy: stopReason !== undefined ? "turri" : "person" } : {}),
|
|
349
|
+
...(finished.error !== undefined ? { error: firstLine(finished.error, 300) } : {}),
|
|
350
|
+
...(finished.logFile !== undefined ? { logFile: finished.logFile } : {}),
|
|
351
|
+
}, place);
|
|
306
352
|
// A failing listener must not take the job down with it; the result is
|
|
307
353
|
// already recorded and can still be collected by asking.
|
|
308
354
|
try {
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import { type UsageReport } from "./activity.ts";
|
|
2
|
+
import { type Job } from "./jobs.ts";
|
|
3
|
+
import { type LedgerEntry } from "./ledger.ts";
|
|
4
|
+
/**
|
|
5
|
+
* What the ledger says of pulses, read back: one pulse - from this session or an earlier one - as the chat and
|
|
6
|
+
* `turri ledger` are told it, and the runs a project has had, each with its pulses.
|
|
7
|
+
*/
|
|
8
|
+
type Of<K extends LedgerEntry["kind"]> = Extract<LedgerEntry, {
|
|
9
|
+
kind: K;
|
|
10
|
+
}>;
|
|
11
|
+
/** The projects whose ledgers a folder's pulses may be in: its own, and those of the projects in folders under it. */
|
|
12
|
+
export declare function ledgerProjects(cwd: string, depth?: number): string[];
|
|
13
|
+
/** One pulse, as its events tell it. */
|
|
14
|
+
export interface PulseView {
|
|
15
|
+
id: string;
|
|
16
|
+
run: string;
|
|
17
|
+
/** How its run was started - chat, host, regenerate - and when. */
|
|
18
|
+
runMode?: string;
|
|
19
|
+
queued?: Of<"job.queued"> & {
|
|
20
|
+
module?: string;
|
|
21
|
+
};
|
|
22
|
+
started?: Of<"job.started">;
|
|
23
|
+
finished?: Of<"job.finished">;
|
|
24
|
+
stages: string[];
|
|
25
|
+
retried: Array<Of<"job.retried">>;
|
|
26
|
+
rounds: Map<string, {
|
|
27
|
+
round: number;
|
|
28
|
+
of: number;
|
|
29
|
+
}>;
|
|
30
|
+
scripts: Array<Of<"script.run">>;
|
|
31
|
+
judge: Map<string, number>;
|
|
32
|
+
reviews: Array<Of<"review.verdict">>;
|
|
33
|
+
healers: Array<Of<"healer.ended"> | Of<"healer.failed">>;
|
|
34
|
+
provider: {
|
|
35
|
+
retries: number;
|
|
36
|
+
stalls: number;
|
|
37
|
+
errors: string[];
|
|
38
|
+
};
|
|
39
|
+
refused: Map<string, number>;
|
|
40
|
+
stopped: string[];
|
|
41
|
+
history: Array<Of<"history.committed"> | Of<"history.failed"> | Of<"history.reset"> | Of<"history.rolledBack">>;
|
|
42
|
+
environment?: Of<"environment.error">;
|
|
43
|
+
usage?: UsageReport;
|
|
44
|
+
}
|
|
45
|
+
/** A pulse's events, gathered: undefined when the ledger has none of it. `entries` are its run's, oldest first. */
|
|
46
|
+
export declare function pulseView(id: string, entries: LedgerEntry[]): PulseView | undefined;
|
|
47
|
+
/** A pulse from the ledger, for the chat and `turri ledger <pulse>`: what happened in it, as facts. */
|
|
48
|
+
export declare function describePulseView(view: PulseView): string;
|
|
49
|
+
/** A pulse the ledger of any of these projects knows, described - or undefined. */
|
|
50
|
+
export declare function describePastPulse(projects: string[], id: string): string | undefined;
|
|
51
|
+
/** A run of the project's, and its pulses: the most recent first. */
|
|
52
|
+
export interface RunView {
|
|
53
|
+
run: string;
|
|
54
|
+
started?: Of<"run.started">;
|
|
55
|
+
ended?: Of<"run.ended">;
|
|
56
|
+
pulses: PulseView[];
|
|
57
|
+
}
|
|
58
|
+
export declare function runViews(project: string): RunView[];
|
|
59
|
+
/** The project's last runs, a line a pulse: what `turri ledger` prints. */
|
|
60
|
+
export declare function describeRuns(runs: RunView[]): string;
|
|
61
|
+
/**
|
|
62
|
+
* The pulses of earlier sessions, for a list beside this session's: the last `runs` runs of these projects that are not
|
|
63
|
+
* this one, the most recent first, each pulse as a job of its own marked `earlier` - none `known` to this session's
|
|
64
|
+
* registry. `more`: how many runs the ledger has beyond them.
|
|
65
|
+
*/
|
|
66
|
+
export declare function earlierPulses(projects: string[], known: Set<string>, runs?: number): {
|
|
67
|
+
jobs: Job[];
|
|
68
|
+
runs: RunView[];
|
|
69
|
+
more: number;
|
|
70
|
+
};
|
|
71
|
+
/** A pulse of the ledger's, as the lists show a job: finished as it ended, its time and cost - and nothing to run. */
|
|
72
|
+
export declare function asJob(view: PulseView): Job;
|
|
73
|
+
/** What to count in the ledger: kinds - whole, or a group by what comes before the dot - a module, since when. */
|
|
74
|
+
export interface EventQuery {
|
|
75
|
+
/** `healer.failed`, or `healer` for every healer event. */
|
|
76
|
+
kinds?: string[];
|
|
77
|
+
module?: string;
|
|
78
|
+
/** Only events at or after this time. */
|
|
79
|
+
since?: Date;
|
|
80
|
+
/** How many of the most recent to list, a line each. */
|
|
81
|
+
limit?: number;
|
|
82
|
+
}
|
|
83
|
+
/** One event in a line: when, its pulse and module, its kind, its own fields - the agent's role when it has one. */
|
|
84
|
+
export declare function eventLine(entry: LedgerEntry): string;
|
|
85
|
+
/**
|
|
86
|
+
* The events of these projects' ledgers that `query` asks for: how many of each kind, and the most recent, a line each -
|
|
87
|
+
* or, when no kind asked for is in the ledger, the kinds it has.
|
|
88
|
+
*/
|
|
89
|
+
export declare function describeEvents(projects: string[], query?: EventQuery): string;
|
|
90
|
+
/** `24h`, `7d`, `30m`, or a date and time: the moment it means, or undefined when it means none. */
|
|
91
|
+
export declare function sinceOf(text: string, now?: number): Date | undefined;
|
|
92
|
+
export {};
|
|
@@ -0,0 +1,345 @@
|
|
|
1
|
+
import { existsSync, readdirSync } from "node:fs";
|
|
2
|
+
import { join, resolve } from "node:path";
|
|
3
|
+
import { describeUsage } from "./activity.js";
|
|
4
|
+
import { formatElapsed } from "./jobs.js";
|
|
5
|
+
import { LEDGER_DIR, ledgerRun, readLedger } from "./ledger.js";
|
|
6
|
+
/** The projects whose ledgers a folder's pulses may be in: its own, and those of the projects in folders under it. */
|
|
7
|
+
export function ledgerProjects(cwd, depth = 3) {
|
|
8
|
+
const found = [];
|
|
9
|
+
const visit = (dir, left) => {
|
|
10
|
+
if (existsSync(join(dir, ".turri", LEDGER_DIR)))
|
|
11
|
+
found.push(dir);
|
|
12
|
+
if (left === 0)
|
|
13
|
+
return;
|
|
14
|
+
let names;
|
|
15
|
+
try {
|
|
16
|
+
names = readdirSync(dir, { withFileTypes: true }).filter((entry) => entry.isDirectory()).map((entry) => entry.name);
|
|
17
|
+
}
|
|
18
|
+
catch {
|
|
19
|
+
return;
|
|
20
|
+
}
|
|
21
|
+
for (const name of names) {
|
|
22
|
+
if (name.startsWith(".") || name === "node_modules" || name === "plain_modules" || name === "dist")
|
|
23
|
+
continue;
|
|
24
|
+
visit(join(dir, name), left - 1);
|
|
25
|
+
}
|
|
26
|
+
};
|
|
27
|
+
visit(resolve(cwd), depth);
|
|
28
|
+
return found;
|
|
29
|
+
}
|
|
30
|
+
/** A pulse's events, gathered: undefined when the ledger has none of it. `entries` are its run's, oldest first. */
|
|
31
|
+
export function pulseView(id, entries) {
|
|
32
|
+
const own = entries.filter((entry) => entry.job !== undefined && entry.job.startsWith(id));
|
|
33
|
+
if (own.length === 0)
|
|
34
|
+
return undefined;
|
|
35
|
+
const fullId = own[0].job;
|
|
36
|
+
const view = {
|
|
37
|
+
id: fullId,
|
|
38
|
+
run: own[0].run,
|
|
39
|
+
stages: [],
|
|
40
|
+
retried: [],
|
|
41
|
+
rounds: new Map(),
|
|
42
|
+
scripts: [],
|
|
43
|
+
judge: new Map(),
|
|
44
|
+
reviews: [],
|
|
45
|
+
healers: [],
|
|
46
|
+
provider: { retries: 0, stalls: 0, errors: [] },
|
|
47
|
+
refused: new Map(),
|
|
48
|
+
stopped: [],
|
|
49
|
+
history: [],
|
|
50
|
+
};
|
|
51
|
+
const started = entries.find((entry) => entry.kind === "run.started" && entry.run === view.run);
|
|
52
|
+
if (started !== undefined)
|
|
53
|
+
view.runMode = `${started.mode}, ${started.ts.slice(0, 16).replace("T", " ")}`;
|
|
54
|
+
for (const entry of own) {
|
|
55
|
+
if (entry.job !== fullId)
|
|
56
|
+
continue;
|
|
57
|
+
switch (entry.kind) {
|
|
58
|
+
case "job.queued":
|
|
59
|
+
view.queued = { ...entry, ...(entry.module !== undefined ? { module: entry.module } : {}) };
|
|
60
|
+
break;
|
|
61
|
+
case "job.started":
|
|
62
|
+
view.started = entry;
|
|
63
|
+
break;
|
|
64
|
+
case "job.finished":
|
|
65
|
+
view.finished = entry;
|
|
66
|
+
view.usage = entry.usage;
|
|
67
|
+
break;
|
|
68
|
+
case "job.stage":
|
|
69
|
+
view.stages.push(entry.label);
|
|
70
|
+
break;
|
|
71
|
+
case "job.retried":
|
|
72
|
+
view.retried.push(entry);
|
|
73
|
+
break;
|
|
74
|
+
case "gate.round":
|
|
75
|
+
view.rounds.set(entry.gate, { round: Math.max(entry.round, view.rounds.get(entry.gate)?.round ?? 0), of: entry.of });
|
|
76
|
+
break;
|
|
77
|
+
case "script.run":
|
|
78
|
+
view.scripts.push(entry);
|
|
79
|
+
break;
|
|
80
|
+
case "judge.verdict":
|
|
81
|
+
view.judge.set(entry.culprit, (view.judge.get(entry.culprit) ?? 0) + 1);
|
|
82
|
+
break;
|
|
83
|
+
case "review.verdict":
|
|
84
|
+
view.reviews.push(entry);
|
|
85
|
+
break;
|
|
86
|
+
case "healer.ended":
|
|
87
|
+
case "healer.failed":
|
|
88
|
+
view.healers.push(entry);
|
|
89
|
+
break;
|
|
90
|
+
case "provider.retry":
|
|
91
|
+
view.provider.retries += 1;
|
|
92
|
+
break;
|
|
93
|
+
case "provider.stall":
|
|
94
|
+
view.provider.stalls += 1;
|
|
95
|
+
break;
|
|
96
|
+
case "provider.error":
|
|
97
|
+
view.provider.errors.push(entry.error);
|
|
98
|
+
break;
|
|
99
|
+
case "guard.refused":
|
|
100
|
+
view.refused.set(entry.rule, (view.refused.get(entry.rule) ?? 0) + 1);
|
|
101
|
+
break;
|
|
102
|
+
case "command.refused":
|
|
103
|
+
view.refused.set("command check", (view.refused.get("command check") ?? 0) + 1);
|
|
104
|
+
break;
|
|
105
|
+
case "turn.stopped":
|
|
106
|
+
view.stopped.push(entry.reason);
|
|
107
|
+
break;
|
|
108
|
+
case "history.committed":
|
|
109
|
+
case "history.failed":
|
|
110
|
+
case "history.reset":
|
|
111
|
+
case "history.rolledBack":
|
|
112
|
+
view.history.push(entry);
|
|
113
|
+
break;
|
|
114
|
+
case "environment.error":
|
|
115
|
+
view.environment = entry;
|
|
116
|
+
break;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
return view;
|
|
120
|
+
}
|
|
121
|
+
/** What a pulse is called: its module, kind and number - as pulseName says it for a live one. */
|
|
122
|
+
function nameOf(view) {
|
|
123
|
+
const queued = view.queued;
|
|
124
|
+
if (queued === undefined)
|
|
125
|
+
return view.id;
|
|
126
|
+
if (queued.module === undefined)
|
|
127
|
+
return queued.summary;
|
|
128
|
+
return queued.pulse !== undefined ? `${queued.module} · ${queued.jobKind} pulse ${queued.pulse}` : `${queued.module} ${queued.jobKind}`;
|
|
129
|
+
}
|
|
130
|
+
function outcomeOf(view) {
|
|
131
|
+
const finished = view.finished;
|
|
132
|
+
if (finished === undefined)
|
|
133
|
+
return view.started !== undefined ? "never ended - its run stopped while it ran" : "never started";
|
|
134
|
+
const landed = finished.landed === true ? ", landed" : finished.landed === false ? ", did not land" : "";
|
|
135
|
+
const stopped = finished.stoppedBy !== undefined ? ` by ${finished.stoppedBy === "turri" ? "turri" : "the person"}` : "";
|
|
136
|
+
return `${finished.status}${stopped}${landed}, after ${formatElapsed(finished.ms)}`;
|
|
137
|
+
}
|
|
138
|
+
/** A pulse from the ledger, for the chat and `turri ledger <pulse>`: what happened in it, as facts. */
|
|
139
|
+
export function describePulseView(view) {
|
|
140
|
+
const lines = [`${view.id} · ${nameOf(view)}${view.queued?.title !== undefined ? ` - ${view.queued.title}` : ""}`];
|
|
141
|
+
lines.push(` run ${view.run}${view.runMode !== undefined ? ` (${view.runMode})` : ""}`);
|
|
142
|
+
lines.push(` ${outcomeOf(view)}${view.usage !== undefined && describeUsage(view.usage) !== "" ? ` · ${describeUsage(view.usage)}` : ""}${view.queued?.model !== undefined ? ` · ${view.queued.model}` : ""}`);
|
|
143
|
+
if (view.finished?.error !== undefined)
|
|
144
|
+
lines.push(` ended with: ${view.finished.error}`);
|
|
145
|
+
if (view.environment !== undefined)
|
|
146
|
+
lines.push(` the environment lacks ${view.environment.missing} (found by ${view.environment.source === "harness" ? "turri" : "an agent"})`);
|
|
147
|
+
for (const retry of view.retried)
|
|
148
|
+
lines.push(` tried again (${retry.attempt} of ${retry.of}) after: ${retry.failure}`);
|
|
149
|
+
if (view.rounds.size > 0)
|
|
150
|
+
lines.push(` rounds: ${[...view.rounds].map(([gate, { round, of }]) => `${gate} ${round} of ${of}`).join("; ")}`);
|
|
151
|
+
// can_it_fail's runs, on code changed to break it, apart: one failing is a test that caught the bug.
|
|
152
|
+
const mutations = view.scripts.filter((run) => run.mutated === true);
|
|
153
|
+
const tests = view.scripts.filter((run) => run.mutated !== true);
|
|
154
|
+
if (tests.length > 0) {
|
|
155
|
+
const failing = tests.filter((run) => run.exit !== 0);
|
|
156
|
+
const signatures = new Map();
|
|
157
|
+
for (const run of failing)
|
|
158
|
+
if (run.signature !== undefined)
|
|
159
|
+
signatures.set(run.signature, (signatures.get(run.signature) ?? 0) + 1);
|
|
160
|
+
const most = Math.max(0, ...signatures.values());
|
|
161
|
+
const timedOut = tests.filter((run) => run.timedOut).length;
|
|
162
|
+
lines.push(` test runs: ${tests.length}, ${failing.length} failing` +
|
|
163
|
+
(most > 1 ? ` - ${most} of them the same way` : "") +
|
|
164
|
+
(timedOut > 0 ? `, ${timedOut} stopped at the time limit` : ""));
|
|
165
|
+
}
|
|
166
|
+
if (mutations.length > 0) {
|
|
167
|
+
const caught = mutations.filter((run) => run.exit !== 0 && !run.timedOut).length;
|
|
168
|
+
lines.push(` can_it_fail: ${mutations.length} run${mutations.length === 1 ? "" : "s"} on code broken on purpose, ${caught} caught by the tests`);
|
|
169
|
+
}
|
|
170
|
+
if (view.judge.size > 0)
|
|
171
|
+
lines.push(` the judge blamed: ${[...view.judge].map(([culprit, times]) => `${culprit} ${times}x`).join(", ")}`);
|
|
172
|
+
if (view.reviews.length > 0)
|
|
173
|
+
lines.push(` reviews: ${view.reviews.map((review) => (review.pass ? "passed" : `${review.problems} problem${review.problems === 1 ? "" : "s"}`)).join(", ")}`);
|
|
174
|
+
// The same outcome again is counted, not said again: a debugger failing for want of a key fails every time.
|
|
175
|
+
const healed = new Map();
|
|
176
|
+
for (const healer of view.healers) {
|
|
177
|
+
const line = healer.kind === "healer.failed" ? `the ${healer.healer} FAILED: ${healer.error}` : `the ${healer.healer}${healer.found !== undefined ? `: ${healer.found}` : " found nothing"}`;
|
|
178
|
+
healed.set(line, (healed.get(line) ?? 0) + 1);
|
|
179
|
+
}
|
|
180
|
+
for (const [line, times] of healed)
|
|
181
|
+
lines.push(` ${times > 1 ? line.replace(/^(the \S+( FAILED)?)/, `$1 ${times}x`) : line}`);
|
|
182
|
+
const provider = view.provider;
|
|
183
|
+
if (provider.retries + provider.stalls + provider.errors.length > 0) {
|
|
184
|
+
lines.push(` the model provider: ${provider.retries} retries, ${provider.stalls} stalls${provider.errors.length > 0 ? `, ${provider.errors.length} errors - last: ${provider.errors.at(-1)}` : ""}`);
|
|
185
|
+
}
|
|
186
|
+
if (view.refused.size > 0)
|
|
187
|
+
lines.push(` turri refused calls: ${[...view.refused].map(([rule, times]) => `${rule} ${times}x`).join(", ")}`);
|
|
188
|
+
if (view.stopped.length > 0)
|
|
189
|
+
lines.push(` turri stopped ${view.stopped.length} turn${view.stopped.length === 1 ? "" : "s"} - last: ${view.stopped.at(-1)}`);
|
|
190
|
+
for (const event of view.history) {
|
|
191
|
+
if (event.kind === "history.committed")
|
|
192
|
+
lines.push(` committed: ${event.subject}`);
|
|
193
|
+
else if (event.kind === "history.failed")
|
|
194
|
+
lines.push(` a commit FAILED${event.landing ? " (landing work)" : ""}: ${event.said}`);
|
|
195
|
+
else if (event.kind === "history.reset")
|
|
196
|
+
lines.push(` put back to the last commit: ${event.undone} change${event.undone === 1 ? "" : "s"} undone`);
|
|
197
|
+
else
|
|
198
|
+
lines.push(` rolled back: ${event.why}`);
|
|
199
|
+
}
|
|
200
|
+
if (view.stages.length > 0)
|
|
201
|
+
lines.push(" last stages:", ...view.stages.slice(-8).map((stage) => ` ${stage}`));
|
|
202
|
+
const log = view.finished?.logFile ?? view.started?.logFile;
|
|
203
|
+
if (log !== undefined)
|
|
204
|
+
lines.push(` log: ${log}`);
|
|
205
|
+
return lines.join("\n");
|
|
206
|
+
}
|
|
207
|
+
/** A pulse the ledger of any of these projects knows, described - or undefined. */
|
|
208
|
+
export function describePastPulse(projects, id) {
|
|
209
|
+
for (const project of projects) {
|
|
210
|
+
const entries = readLedger(project);
|
|
211
|
+
const view = pulseView(id, entries);
|
|
212
|
+
if (view !== undefined)
|
|
213
|
+
return describePulseView(view);
|
|
214
|
+
}
|
|
215
|
+
return undefined;
|
|
216
|
+
}
|
|
217
|
+
export function runViews(project) {
|
|
218
|
+
const entries = readLedger(project);
|
|
219
|
+
const runs = new Map();
|
|
220
|
+
for (const entry of entries)
|
|
221
|
+
runs.set(entry.run, [...(runs.get(entry.run) ?? []), entry]);
|
|
222
|
+
return [...runs]
|
|
223
|
+
.map(([run, own]) => {
|
|
224
|
+
const ids = [...new Set(own.filter((entry) => entry.kind === "job.queued").map((entry) => entry.job))];
|
|
225
|
+
const started = own.find((entry) => entry.kind === "run.started");
|
|
226
|
+
const ended = own.find((entry) => entry.kind === "run.ended");
|
|
227
|
+
return { run, ...(started !== undefined ? { started } : {}), ...(ended !== undefined ? { ended } : {}), pulses: ids.map((id) => pulseView(id, own)).filter(Boolean) };
|
|
228
|
+
})
|
|
229
|
+
.reverse();
|
|
230
|
+
}
|
|
231
|
+
/** The project's last runs, a line a pulse: what `turri ledger` prints. */
|
|
232
|
+
export function describeRuns(runs) {
|
|
233
|
+
if (runs.length === 0)
|
|
234
|
+
return "The ledger is empty: nothing has run here since it began.";
|
|
235
|
+
return runs
|
|
236
|
+
.map((view) => {
|
|
237
|
+
const started = view.started;
|
|
238
|
+
const when = started?.ts !== undefined ? started.ts.slice(0, 19).replace("T", " ") : "";
|
|
239
|
+
const version = started?.version !== undefined && started.version !== null ? ` · turri ${started.version}${started.commit !== null && started.commit !== undefined ? ` (${started.commit.slice(0, 7)}${started.dirty === true ? ", changed" : ""})` : ""}` : "";
|
|
240
|
+
const head = `run ${view.run} · ${started?.mode ?? "?"} · ${when}${view.ended !== undefined ? ` · ${formatElapsed(view.ended.ms)}` : " · still running, or killed"}${version}`;
|
|
241
|
+
const pulses = view.pulses.map((pulse) => ` ${pulse.id} ${nameOf(pulse).padEnd(32)} ${outcomeOf(pulse)}${pulse.usage !== undefined && describeUsage(pulse.usage) !== "" ? ` · ${describeUsage(pulse.usage)}` : ""}`);
|
|
242
|
+
return [head, ...(pulses.length > 0 ? pulses : [" no pulses"])].join("\n");
|
|
243
|
+
})
|
|
244
|
+
.join("\n\n");
|
|
245
|
+
}
|
|
246
|
+
/**
|
|
247
|
+
* The pulses of earlier sessions, for a list beside this session's: the last `runs` runs of these projects that are not
|
|
248
|
+
* this one, the most recent first, each pulse as a job of its own marked `earlier` - none `known` to this session's
|
|
249
|
+
* registry. `more`: how many runs the ledger has beyond them.
|
|
250
|
+
*/
|
|
251
|
+
export function earlierPulses(projects, known, runs = 3) {
|
|
252
|
+
const own = ledgerRun();
|
|
253
|
+
const all = projects
|
|
254
|
+
.flatMap((project) => runViews(project))
|
|
255
|
+
.filter((view) => view.run !== own && view.pulses.some((pulse) => !known.has(pulse.id)))
|
|
256
|
+
.sort((a, b) => (b.started?.ts ?? "").localeCompare(a.started?.ts ?? ""));
|
|
257
|
+
const shown = all.slice(0, runs);
|
|
258
|
+
return { jobs: shown.flatMap((view) => view.pulses.filter((pulse) => !known.has(pulse.id)).map(asJob)), runs: shown, more: all.length - shown.length };
|
|
259
|
+
}
|
|
260
|
+
/** A pulse of the ledger's, as the lists show a job: finished as it ended, its time and cost - and nothing to run. */
|
|
261
|
+
export function asJob(view) {
|
|
262
|
+
const at = (entry) => (entry?.ts !== undefined ? Date.parse(entry.ts) : undefined);
|
|
263
|
+
const queuedAt = at(view.queued) ?? 0;
|
|
264
|
+
const startedAt = at(view.started) ?? queuedAt;
|
|
265
|
+
const finished = view.finished;
|
|
266
|
+
// A pulse whose run ended before it did never finished: it was stopped with its run.
|
|
267
|
+
const status = finished !== undefined && ["succeeded", "failed", "stopped"].includes(finished.status) ? finished.status : "stopped";
|
|
268
|
+
const logFile = finished?.logFile ?? view.started?.logFile;
|
|
269
|
+
return {
|
|
270
|
+
id: view.id,
|
|
271
|
+
kind: view.queued?.jobKind ?? "regenerate",
|
|
272
|
+
summary: view.queued?.summary ?? view.id,
|
|
273
|
+
...(view.queued?.module !== undefined ? { module: view.queued.module } : {}),
|
|
274
|
+
...(view.queued?.pulse !== undefined ? { pulse: view.queued.pulse } : {}),
|
|
275
|
+
...(view.queued?.title !== undefined ? { title: view.queued.title } : {}),
|
|
276
|
+
...(view.queued?.model !== undefined ? { model: view.queued.model } : {}),
|
|
277
|
+
status,
|
|
278
|
+
sequence: 0,
|
|
279
|
+
queuedAt,
|
|
280
|
+
startedAt,
|
|
281
|
+
finishedAt: finished !== undefined ? startedAt + finished.ms : startedAt,
|
|
282
|
+
waitingOn: [],
|
|
283
|
+
tail: "",
|
|
284
|
+
activity: [],
|
|
285
|
+
usage: view.usage ?? { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, cost: 0 },
|
|
286
|
+
agents: [],
|
|
287
|
+
transcript: [],
|
|
288
|
+
...(logFile !== undefined ? { logFile } : {}),
|
|
289
|
+
...(finished?.error !== undefined ? { error: finished.error } : {}),
|
|
290
|
+
details: { ...(finished?.landed !== undefined ? { landed: finished.landed } : {}), ...(finished?.passed !== undefined ? { passed: finished.passed } : {}) },
|
|
291
|
+
earlier: { run: view.run, ...(view.runMode !== undefined ? { runMode: view.runMode } : {}), lines: describePulseView(view).split("\n").slice(1).map((line) => line.replace(/^ /, "")) },
|
|
292
|
+
};
|
|
293
|
+
}
|
|
294
|
+
/** The envelope's fields, said apart from an event's own. */
|
|
295
|
+
const ENVELOPE = new Set(["v", "ts", "seq", "run", "job", "module", "agent", "kind"]);
|
|
296
|
+
/** One event in a line: when, its pulse and module, its kind, its own fields - the agent's role when it has one. */
|
|
297
|
+
export function eventLine(entry) {
|
|
298
|
+
const fields = Object.entries(entry)
|
|
299
|
+
.filter(([key]) => !ENVELOPE.has(key))
|
|
300
|
+
.map(([key, value]) => {
|
|
301
|
+
const text = typeof value === "string" ? JSON.stringify(value.length > 160 ? `${value.slice(0, 159)}…` : value) : JSON.stringify(value);
|
|
302
|
+
return `${key}=${text}`;
|
|
303
|
+
});
|
|
304
|
+
const where = [entry.job, entry.module, entry.agent?.role ?? entry.agent?.label].filter((part) => part !== undefined);
|
|
305
|
+
return [entry.ts.slice(0, 19).replace("T", " "), ...where, entry.kind, ...fields].join(" · ");
|
|
306
|
+
}
|
|
307
|
+
/**
|
|
308
|
+
* The events of these projects' ledgers that `query` asks for: how many of each kind, and the most recent, a line each -
|
|
309
|
+
* or, when no kind asked for is in the ledger, the kinds it has.
|
|
310
|
+
*/
|
|
311
|
+
export function describeEvents(projects, query = {}) {
|
|
312
|
+
const all = projects.flatMap((project) => readLedger(project, query.since !== undefined ? { since: query.since } : {}));
|
|
313
|
+
const inModule = query.module !== undefined ? all.filter((entry) => entry.module === query.module) : all;
|
|
314
|
+
const wanted = query.kinds?.map((kind) => kind.trim()).filter((kind) => kind !== "");
|
|
315
|
+
const matches = (kind) => wanted === undefined || wanted.length === 0 || wanted.some((asked) => kind === asked || kind.startsWith(`${asked}.`));
|
|
316
|
+
const found = inModule.filter((entry) => matches(entry.kind));
|
|
317
|
+
const scope = [query.module !== undefined ? `in ${query.module}` : "", query.since !== undefined ? `since ${query.since.toISOString().slice(0, 16).replace("T", " ")}` : ""].filter(Boolean).join(", ");
|
|
318
|
+
if (found.length === 0) {
|
|
319
|
+
const kinds = [...new Set(inModule.map((entry) => entry.kind))].sort();
|
|
320
|
+
return `No ${wanted !== undefined && wanted.length > 0 ? wanted.join(", ") : ""} events${scope !== "" ? ` ${scope}` : ""} in the ledger.${kinds.length > 0 ? ` It has: ${kinds.join(", ")}.` : ""}`;
|
|
321
|
+
}
|
|
322
|
+
const counts = new Map();
|
|
323
|
+
for (const entry of found)
|
|
324
|
+
counts.set(entry.kind, (counts.get(entry.kind) ?? 0) + 1);
|
|
325
|
+
const pulses = new Set(found.map((entry) => entry.job).filter((job) => job !== undefined)).size;
|
|
326
|
+
const runs = new Set(found.map((entry) => entry.run)).size;
|
|
327
|
+
const limit = Math.max(1, query.limit ?? 20);
|
|
328
|
+
const recent = found.slice(-limit);
|
|
329
|
+
return [
|
|
330
|
+
`${found.length} event${found.length === 1 ? "" : "s"}${scope !== "" ? ` ${scope}` : ""}, in ${runs} run${runs === 1 ? "" : "s"} and ${pulses} pulse${pulses === 1 ? "" : "s"}: ${[...counts].sort((a, b) => b[1] - a[1]).map(([kind, times]) => `${kind} ${times}`).join(", ")}`,
|
|
331
|
+
"",
|
|
332
|
+
recent.length === found.length ? "Each, oldest first:" : `The last ${recent.length}, oldest first:`,
|
|
333
|
+
...recent.map(eventLine),
|
|
334
|
+
].join("\n");
|
|
335
|
+
}
|
|
336
|
+
/** `24h`, `7d`, `30m`, or a date and time: the moment it means, or undefined when it means none. */
|
|
337
|
+
export function sinceOf(text, now = Date.now()) {
|
|
338
|
+
const relative = /^(\d+)\s*([mhd])$/i.exec(text.trim());
|
|
339
|
+
if (relative !== null) {
|
|
340
|
+
const unit = { m: 60_000, h: 3_600_000, d: 86_400_000 }[relative[2].toLowerCase()];
|
|
341
|
+
return new Date(now - Number(relative[1]) * unit);
|
|
342
|
+
}
|
|
343
|
+
const date = new Date(text.trim());
|
|
344
|
+
return Number.isNaN(date.getTime()) ? undefined : date;
|
|
345
|
+
}
|