@foldrun/core 2.4.0 → 2.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (149) hide show
  1. package/dist/src/agent-activity.d.ts +5 -0
  2. package/dist/src/agent-activity.js +31 -0
  3. package/dist/src/agent-import.js +2 -2
  4. package/dist/src/agent-tools.js +2 -2
  5. package/dist/src/agents-md.d.ts +5 -1
  6. package/dist/src/agents-md.js +42 -18
  7. package/dist/src/alerts.d.ts +60 -0
  8. package/dist/src/alerts.js +161 -0
  9. package/dist/src/browser-pod.js +4 -0
  10. package/dist/src/capture.d.ts +23 -0
  11. package/dist/src/capture.js +80 -0
  12. package/dist/src/compare.d.ts +57 -0
  13. package/dist/src/compare.js +116 -0
  14. package/dist/src/completions.js +26 -3
  15. package/dist/src/deploy.d.ts +36 -1
  16. package/dist/src/deploy.js +237 -13
  17. package/dist/src/egress.d.ts +16 -1
  18. package/dist/src/evals.d.ts +27 -2
  19. package/dist/src/evals.js +62 -19
  20. package/dist/src/flow-lint.d.ts +14 -0
  21. package/dist/src/flow-lint.js +35 -0
  22. package/dist/src/flow-patterns.js +29 -14
  23. package/dist/src/frontmatter-fields.d.ts +27 -0
  24. package/dist/src/frontmatter-fields.js +60 -3
  25. package/dist/src/frontmatter-guard.d.ts +19 -0
  26. package/dist/src/frontmatter-guard.js +67 -0
  27. package/dist/src/frontmatter-schema.d.ts +2 -2
  28. package/dist/src/frontmatter-schema.js +33 -2
  29. package/dist/src/frontmatter-text.d.ts +31 -0
  30. package/dist/src/frontmatter-text.js +102 -0
  31. package/dist/src/history.d.ts +4 -0
  32. package/dist/src/history.js +15 -0
  33. package/dist/src/hooks.js +1 -1
  34. package/dist/src/insights.d.ts +30 -0
  35. package/dist/src/insights.js +152 -0
  36. package/dist/src/issues.d.ts +140 -0
  37. package/dist/src/issues.js +302 -0
  38. package/dist/src/judge-verdict.d.ts +19 -0
  39. package/dist/src/judge-verdict.js +54 -0
  40. package/dist/src/library.js +1 -1
  41. package/dist/src/limits.d.ts +8 -0
  42. package/dist/src/limits.js +36 -5
  43. package/dist/src/linkable.js +1 -1
  44. package/dist/src/matter.d.ts +42 -0
  45. package/dist/src/matter.js +132 -0
  46. package/dist/src/model-credential.d.ts +14 -0
  47. package/dist/src/model-credential.js +40 -0
  48. package/dist/src/network.d.ts +12 -1
  49. package/dist/src/network.js +35 -8
  50. package/dist/src/notify.d.ts +16 -0
  51. package/dist/src/notify.js +17 -7
  52. package/dist/src/okf.js +1 -1
  53. package/dist/src/online-checks.d.ts +76 -0
  54. package/dist/src/online-checks.js +432 -0
  55. package/dist/src/operator.d.ts +277 -0
  56. package/dist/src/operator.js +176 -2
  57. package/dist/src/otel-config.d.ts +18 -0
  58. package/dist/src/otel-config.js +69 -0
  59. package/dist/src/otel.d.ts +38 -0
  60. package/dist/src/otel.js +85 -0
  61. package/dist/src/outcome.d.ts +40 -0
  62. package/dist/src/outcome.js +77 -0
  63. package/dist/src/package.js +9 -2
  64. package/dist/src/platform.d.ts +1 -1
  65. package/dist/src/prompt-layers.d.ts +30 -0
  66. package/dist/src/prompt-layers.js +77 -0
  67. package/dist/src/provider-health.js +2 -2
  68. package/dist/src/providers.d.ts +5 -0
  69. package/dist/src/providers.js +2 -2
  70. package/dist/src/run-container.d.ts +25 -2
  71. package/dist/src/run-container.js +61 -6
  72. package/dist/src/runner.d.ts +53 -2
  73. package/dist/src/runner.js +471 -102
  74. package/dist/src/runs-filter.d.ts +48 -0
  75. package/dist/src/runs-filter.js +73 -0
  76. package/dist/src/scan.d.ts +30 -1
  77. package/dist/src/scan.js +145 -14
  78. package/dist/src/starter.js +27 -2
  79. package/dist/src/step-exec.d.ts +39 -3
  80. package/dist/src/step-exec.js +118 -15
  81. package/dist/src/store.d.ts +144 -3
  82. package/dist/src/store.js +150 -7
  83. package/dist/src/tokens.d.ts +38 -0
  84. package/dist/src/tokens.js +57 -0
  85. package/dist/src/tool-names.js +1 -1
  86. package/dist/src/tool-programs.js +1 -1
  87. package/dist/src/trace-tree.d.ts +40 -0
  88. package/dist/src/trace-tree.js +156 -0
  89. package/dist/src/translator.d.ts +8 -0
  90. package/dist/src/translator.js +7 -2
  91. package/dist/src/trigger-log.d.ts +1 -1
  92. package/dist/src/workspace-check.d.ts +18 -0
  93. package/dist/src/workspace-check.js +34 -0
  94. package/package.json +2 -2
  95. package/src/agent-activity.ts +32 -0
  96. package/src/agent-import.ts +2 -2
  97. package/src/agent-tools.ts +2 -2
  98. package/src/agents-md.ts +39 -14
  99. package/src/alerts.ts +190 -0
  100. package/src/browser-pod.ts +3 -0
  101. package/src/capture.ts +88 -0
  102. package/src/compare.ts +146 -0
  103. package/src/completions.ts +27 -3
  104. package/src/deploy.ts +238 -12
  105. package/src/egress.ts +12 -2
  106. package/src/evals.ts +64 -19
  107. package/src/flow-lint.ts +36 -0
  108. package/src/flow-patterns.ts +26 -13
  109. package/src/frontmatter-fields.ts +83 -3
  110. package/src/frontmatter-guard.ts +70 -0
  111. package/src/frontmatter-schema.ts +33 -4
  112. package/src/frontmatter-text.ts +109 -0
  113. package/src/history.ts +13 -0
  114. package/src/hooks.ts +1 -1
  115. package/src/insights.ts +174 -0
  116. package/src/issues.ts +396 -0
  117. package/src/judge-verdict.ts +52 -0
  118. package/src/library.ts +1 -1
  119. package/src/limits.ts +38 -5
  120. package/src/linkable.ts +1 -1
  121. package/src/matter.ts +145 -0
  122. package/src/model-credential.ts +45 -0
  123. package/src/network.ts +35 -7
  124. package/src/notify.ts +22 -7
  125. package/src/okf.ts +1 -1
  126. package/src/online-checks.ts +450 -0
  127. package/src/operator.ts +213 -3
  128. package/src/otel-config.ts +67 -0
  129. package/src/otel.ts +100 -0
  130. package/src/outcome.ts +90 -0
  131. package/src/package.ts +8 -2
  132. package/src/platform.ts +1 -1
  133. package/src/prompt-layers.ts +90 -0
  134. package/src/provider-health.ts +2 -2
  135. package/src/providers.ts +7 -2
  136. package/src/run-container.ts +77 -8
  137. package/src/runner.ts +477 -79
  138. package/src/runs-filter.ts +88 -0
  139. package/src/scan.ts +153 -14
  140. package/src/starter.ts +27 -2
  141. package/src/step-exec.ts +168 -17
  142. package/src/store.ts +268 -9
  143. package/src/tokens.ts +68 -0
  144. package/src/tool-names.ts +1 -1
  145. package/src/tool-programs.ts +1 -1
  146. package/src/trace-tree.ts +187 -0
  147. package/src/translator.ts +13 -2
  148. package/src/trigger-log.ts +1 -0
  149. package/src/workspace-check.ts +41 -0
@@ -1,10 +1,12 @@
1
1
  // Checking frontmatter against frontmatter-fields.ts: the values, not just
2
2
  // the names. Node-side (it reads store's parsers); the browser takes the
3
3
  // field list alone from frontmatter-fields.ts.
4
- import { FRONTMATTER } from "./frontmatter-fields.js";
4
+ import { FRONTMATTER, JOBS } from "./frontmatter-fields.js";
5
+ import { jobFold } from "./matter.js";
5
6
  import { resolveEffort, parseDuration, parseInstant, parseRate, STEP_OPTION_KEYS } from "./store.js";
6
7
  import { budgetProblem } from "./budget.js";
7
8
  import { cronProblem } from "./cron.js";
9
+ import { modelSecretCollision } from "./model-credential.js";
8
10
  export { FRONTMATTER, schemaKindOf } from "./frontmatter-fields.js";
9
11
  /** A value for a message: a scalar as written, anything else by its shape.
10
12
  * Never String() of a map — its `toString` is whatever the YAML says. */
@@ -29,6 +31,7 @@ const CHECKS = {
29
31
  apis: apisProblem,
30
32
  budget: (v) => budgetProblem(v, ["run"]),
31
33
  ask: (v) => askProblem(v),
34
+ secrets: modelSecretCollision,
32
35
  },
33
36
  "agents-md": {
34
37
  budget: (v) => budgetProblem(v, ["day", "week", "month"]),
@@ -52,6 +55,7 @@ const CHECKS = {
52
55
  transport: (v) => (["http", "script", "mcp"].includes(String(v).toLowerCase()) ? null : `"${String(v)}" is not a transport — http, script or mcp`),
53
56
  methods: methodsProblem,
54
57
  rate: (v) => (parseRate(v) ? null : `"${shown(v)}" is not a rate — N/s, N/minute, N/hour, or a number per second, above 0`),
58
+ secrets: modelSecretCollision,
55
59
  },
56
60
  "eval": {
57
61
  effort: (v) => (resolveEffort(v) ? null : `"${String(v)}" is not an effort — low, medium, high, xhigh or max; the judge runs at none`),
@@ -77,7 +81,33 @@ export function frontmatterProblems(kind, data, near) {
77
81
  const fields = FRONTMATTER[kind];
78
82
  const known = fields.map((f) => f.key);
79
83
  const out = [];
84
+ // The four job blocks (matter.ts folds them): a key set both ways, and a
85
+ // key left in a block that does not hold it.
86
+ const fold = jobFold(data);
87
+ for (const [key, job] of fold?.conflicts ?? []) {
88
+ out.push({ key, rule: "config/conflict", level: "error", message: `\`${key}:\` is set twice — at the top and inside \`${job}:\`. Only the top one would be read; keep one` });
89
+ }
90
+ for (const { name: job } of JOBS) {
91
+ const block = data[job];
92
+ if (!block || typeof block !== "object" || Array.isArray(block))
93
+ continue;
94
+ for (const inner of Object.keys(block)) {
95
+ const spec = fields.find((f) => f.key === inner);
96
+ const message = spec?.job
97
+ ? `\`${inner}:\` is a ${spec.job} field, not a ${job} one — move it into \`${spec.job}:\` or to the top. Inside \`${job}:\` it is not read`
98
+ : spec
99
+ ? `\`${inner}:\` belongs to none of the four jobs — write it at the top. Inside \`${job}:\` it is not read`
100
+ : `\`${inner}:\` inside \`${job}:\` is not a field foldrun reads${belongsIn(inner, kind) ? ` here — it is read in ${belongsIn(inner, kind)}` : ""}; it is ignored`;
101
+ // A real setting in the wrong block is a setting silently not applied
102
+ // — a guardrail written under governance: guards nothing — so it is an
103
+ // error (Codex debate). A word foldrun does not read at all is a note.
104
+ out.push({ key: inner, rule: "config/misplaced", level: spec ? "error" : "info", message });
105
+ }
106
+ }
80
107
  for (const [key, value] of Object.entries(data)) {
108
+ // A block holding only misplaced keys is reported above, key by key.
109
+ if (JOBS.some((j) => j.name === key) && value && typeof value === "object" && !Array.isArray(value))
110
+ continue;
81
111
  const spec = fields.find((f) => f.key === key);
82
112
  const check = CHECKS[kind]?.[key];
83
113
  if (!spec) {
@@ -92,7 +122,8 @@ export function frontmatterProblems(kind, data, near) {
92
122
  out.push({ key, rule: "config/unknown-field", level: "warn", message: `\`${key}:\` is not read on an agent — it is a step option: indent it under the step in the flow (\` ${key}: ${shown(value)}\`). Here it is ignored` });
93
123
  }
94
124
  else if (home) {
95
- out.push({ key, rule: "config/unknown-field", level: "warn", message: `\`${key}:\` is read in ${home}, not in ${kind === "agents-md" ? "AGENTS.md" : kind === "eval" ? "an eval" : `a ${kind}`} — here it is ignored` });
125
+ const where = fold?.folded.get(key);
126
+ out.push({ key, rule: "config/unknown-field", level: "warn", message: `\`${key}:\`${where ? ` (inside \`${where}:\`)` : ""} is read in ${home}, not in ${kind === "agents-md" ? "AGENTS.md" : kind === "eval" ? "an eval" : `a ${kind}`} — here it is ignored` });
96
127
  }
97
128
  else {
98
129
  out.push({ key, rule: "config/unknown-field", level: "info", message: `\`${key}:\` is not a field foldrun reads; it is ignored` });
@@ -0,0 +1,31 @@
1
+ export interface KeySpan {
2
+ /** The key's line. */
3
+ at: number;
4
+ /** One past its last line. */
5
+ end: number;
6
+ /** What its line starts with. */
7
+ indent: string;
8
+ /** The block header's line, when it is inside one. */
9
+ block: number | null;
10
+ }
11
+ /** Thrown for a key a writer cannot edit in place: one that is, or may be,
12
+ * inside a block not written one key per line (`guardrail: {…}`, an alias).
13
+ * Rewriting it would mean re-serialising the person's YAML, and adding the
14
+ * key at the top beside it would be a config/conflict — so the writer says
15
+ * what to write instead. */
16
+ export declare class InlineBlockError extends Error {
17
+ status: number;
18
+ constructor(key: string, block: string);
19
+ }
20
+ /**
21
+ * Where a key is written in frontmatter lines: at the top, or a direct child
22
+ * of one of the four job blocks (`guardrail:` and so on). Every writer that
23
+ * edits a key as text reads it here, so a file written in blocks is edited
24
+ * where it is — never given a second, flat copy that check would refuse as
25
+ * config/conflict. A key inside an inline block map throws InlineBlockError.
26
+ */
27
+ export declare function keySpan(lines: string[], key: string): KeySpan | null;
28
+ /** A block header left with nothing under it is removed with its last key.
29
+ * Blank lines and comments between it and its next key do not count as
30
+ * nothing: the header stays while any indented key follows them. */
31
+ export declare function dropEmptyBlock(lines: string[], header: number): string[];
@@ -0,0 +1,102 @@
1
+ // Frontmatter edited as text, where it is written: at the top or inside one
2
+ // of the four job blocks. No node imports — the flow canvas (flow-patterns.ts)
3
+ // runs in the browser and edits frontmatter with it too.
4
+ /** A block header on its own line — an anchor and a trailing `# comment`
5
+ * allowed. */
6
+ const JOB_BLOCK = /^["']?(governance|guardrail|observability|evaluation)["']?\s*:\s*(&\S+\s*)?(#.*)?$/;
7
+ /** Any other block header: an inline map, an alias, a tag — a form these
8
+ * writers do not edit. */
9
+ const JOB_OTHER = /^["']?(governance|guardrail|observability|evaluation)["']?\s*:\s*\S/;
10
+ /** An inline map closed on its own line: `guardrail: {network: [a.com]}`. */
11
+ const JOB_ONE_LINE = /^["']?(governance|guardrail|observability|evaluation)["']?\s*:\s*\{.*\}\s*(#.*)?$/;
12
+ const indentOf = (l) => l.match(/^[ \t]*/)?.[0].length ?? 0;
13
+ /** Thrown for a key a writer cannot edit in place: one that is, or may be,
14
+ * inside a block not written one key per line (`guardrail: {…}`, an alias).
15
+ * Rewriting it would mean re-serialising the person's YAML, and adding the
16
+ * key at the top beside it would be a config/conflict — so the writer says
17
+ * what to write instead. */
18
+ export class InlineBlockError extends Error {
19
+ status = 409;
20
+ constructor(key, block) {
21
+ super(`\`${key}\` may be inside \`${block}:\`, which is not written one key per line — write \`${block}:\` one key per line to change it here, or edit the file`);
22
+ }
23
+ }
24
+ /** A comment or blank line: neither ends a block nor is one of its keys. */
25
+ const filler = (l) => l.trim() === "" || /^\s*#/.test(l);
26
+ /**
27
+ * Where a key is written in frontmatter lines: at the top, or a direct child
28
+ * of one of the four job blocks (`guardrail:` and so on). Every writer that
29
+ * edits a key as text reads it here, so a file written in blocks is edited
30
+ * where it is — never given a second, flat copy that check would refuse as
31
+ * config/conflict. A key inside an inline block map throws InlineBlockError.
32
+ */
33
+ export function keySpan(lines, key) {
34
+ const k = key.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
35
+ const span = (at, indent, block) => {
36
+ let end = at + 1;
37
+ // The key continues while lines are indented deeper, are an item of its
38
+ // list written at its own indent (YAML allows `key:\n- a`), or are blank
39
+ // lines and comments inside it.
40
+ while (end < lines.length &&
41
+ (filler(lines[end]) ||
42
+ indentOf(lines[end]) > indent.length ||
43
+ (indentOf(lines[end]) === indent.length && /^[ \t]*-(\s|$)/.test(lines[end]))))
44
+ end++;
45
+ // Blank lines and comments at its foot, no deeper than the key, belong
46
+ // to what follows.
47
+ while (end > at + 1 && filler(lines[end - 1]) && (lines[end - 1].trim() === "" || indentOf(lines[end - 1]) <= indent.length))
48
+ end--;
49
+ return { at, end, indent, block };
50
+ };
51
+ // A key may be quoted: `"network": none`.
52
+ const named = `["']?${k}["']?\\s*:`;
53
+ const top = lines.findIndex((l) => new RegExp(`^${named}`).test(l));
54
+ if (top !== -1)
55
+ return span(top, "", null);
56
+ for (let h = 0; h < lines.length; h++) {
57
+ const other = !JOB_BLOCK.test(lines[h]) && JOB_OTHER.exec(lines[h]);
58
+ if (other) {
59
+ // One closed line can be read: refuse only if the key is in it. Any
60
+ // other form (a map over several lines, an alias) cannot be, so any
61
+ // key not found at the top is refused rather than added beside it.
62
+ if (!JOB_ONE_LINE.test(lines[h]) || new RegExp(`[{,]\\s*${named}`).test(lines[h]))
63
+ throw new InlineBlockError(key, other[1]);
64
+ continue;
65
+ }
66
+ if (!JOB_BLOCK.test(lines[h]))
67
+ continue;
68
+ // Its children sit at the indent of its first indented line; a deeper
69
+ // line of the same name belongs to one of them, not to the block.
70
+ // Comments and blank lines, indented or not, do not end it.
71
+ let child = -1;
72
+ let merged = false;
73
+ for (let n = h + 1; n < lines.length && (/^[ \t]/.test(lines[n]) || filler(lines[n])); n++) {
74
+ if (filler(lines[n]))
75
+ continue;
76
+ if (child === -1)
77
+ child = indentOf(lines[n]);
78
+ if (indentOf(lines[n]) !== child)
79
+ continue;
80
+ const m = new RegExp(`^([ \t]+)${named}`).exec(lines[n]);
81
+ if (m)
82
+ return span(n, m[1], h);
83
+ if (/^[ \t]+<<\s*:/.test(lines[n]))
84
+ merged = true;
85
+ }
86
+ // `<<: *policy` may bring the key in from elsewhere: not one this text
87
+ // edit can see, so it is refused rather than added beside it.
88
+ if (merged)
89
+ throw new InlineBlockError(key, JOB_BLOCK.exec(lines[h])[1]);
90
+ }
91
+ return null;
92
+ }
93
+ /** A block header left with nothing under it is removed with its last key.
94
+ * Blank lines and comments between it and its next key do not count as
95
+ * nothing: the header stays while any indented key follows them. */
96
+ export function dropEmptyBlock(lines, header) {
97
+ let n = header + 1;
98
+ while (n < lines.length && filler(lines[n]))
99
+ n++;
100
+ const next = lines[n];
101
+ return next === undefined || !/^[ \t]+\S/.test(next) ? [...lines.slice(0, header), ...lines.slice(header + 1)] : lines;
102
+ }
@@ -86,6 +86,10 @@ export declare function listRevisions(tenant: string, scope: string, opts?: {
86
86
  limit?: number;
87
87
  }): RevisionSummary[];
88
88
  export declare function readRevision(tenant: string, scope: string, id: string, forPath?: string): Revision | null;
89
+ /** The files changed after revision `from` up to and including `to`, in a
90
+ * scope — what differs between the versions two runs ran (FOL-72). Null
91
+ * when either is unknown or `to` is not after `from`. */
92
+ export declare function changedPathsBetween(tenant: string, scope: string, from: string | null, to: string | null): string[] | null;
89
93
  /** The most recent revision id for a scope — what a change is attributed
90
94
  * to when nothing more specific (a deployed commit) is on record. */
91
95
  export declare function latestRevisionId(tenant: string, scope: string): string | null;
@@ -344,6 +344,21 @@ export function readRevision(tenant, scope, id, forPath) {
344
344
  const rev = JSON.parse(fs.readFileSync(file, "utf8"));
345
345
  return forPath ? { ...rev, files: rev.files.filter((f) => f.path === forPath) } : rev;
346
346
  }
347
+ /** The files changed after revision `from` up to and including `to`, in a
348
+ * scope — what differs between the versions two runs ran (FOL-72). Null
349
+ * when either is unknown or `to` is not after `from`. */
350
+ export function changedPathsBetween(tenant, scope, from, to) {
351
+ if (!from || !to)
352
+ return null;
353
+ if (from === to)
354
+ return [];
355
+ const revs = listRevisions(tenant, scope, { limit: 500 }); // newest first
356
+ const ti = revs.findIndex((r) => r.id === to || r.commit === to);
357
+ const fi = revs.findIndex((r) => r.id === from || r.commit === from);
358
+ if (ti < 0 || fi < 0 || ti > fi)
359
+ return null;
360
+ return [...new Set(revs.slice(ti, fi).flatMap((r) => r.paths))].sort();
361
+ }
347
362
  /** The most recent revision id for a scope — what a change is attributed
348
363
  * to when nothing more specific (a deployed commit) is on record. */
349
364
  export function latestRevisionId(tenant, scope) {
package/dist/src/hooks.js CHANGED
@@ -30,7 +30,7 @@ import fs from "node:fs";
30
30
  import path from "node:path";
31
31
  import os from "node:os";
32
32
  import { spawn } from "node:child_process";
33
- import matter from "gray-matter";
33
+ import matter from "./matter.js";
34
34
  export const HOOK_EVENTS = ["tool.call", "tool.result", "step.start", "step.end"];
35
35
  /** Events that decide something before it happens: a failure refuses. */
36
36
  const GATES = new Set(["tool.call", "step.start"]);
@@ -0,0 +1,30 @@
1
+ import type { RunRecord } from "./store.ts";
2
+ export type InsightKind = "cache" | "big-result" | "retry-loop" | "timeout";
3
+ export interface Insight {
4
+ id: string;
5
+ kind: InsightKind;
6
+ workspace: string;
7
+ agent: string;
8
+ tool?: string;
9
+ title: string;
10
+ /** Why it happens, in a sentence. */
11
+ why: string;
12
+ fix: string;
13
+ /** About how many credits it cost in the window; null when not money. */
14
+ credits: number | null;
15
+ /** How many steps showed it. */
16
+ steps: number;
17
+ /** The newest runs it was seen in, at most 5. */
18
+ evidence: {
19
+ workspace: string;
20
+ runId: string;
21
+ step: number;
22
+ }[];
23
+ }
24
+ type Run = RunRecord & {
25
+ workspace: string;
26
+ };
27
+ export declare function findInsights(runs: Run[], opts?: {
28
+ minSteps?: number;
29
+ }): Insight[];
30
+ export {};
@@ -0,0 +1,152 @@
1
+ // Insights (FOL-72 step 8): the problems that recur without failing anything,
2
+ // found in the runs with no setup — each with its evidence, what it costs and
3
+ // what to do. The operator's "3 problems, here is why" instead of a list of
4
+ // runs. Pure: the API, the Observe page and `foldrun insights` read the same.
5
+ //
6
+ // cache an agent re-reads a big prompt uncached, step after step
7
+ // big-result a tool's result is large and every later model call sends it again
8
+ // retry-loop a tool errors again and again inside one step
9
+ // timeout steps that use most of their `timeout:` — the next bigger input fails
10
+ //
11
+ // Costs are estimates in credits from the steps' own token costs (a credit
12
+ // is a cent of USD), said as "about".
13
+ /** A cached input token costs about a tenth of a fresh one. */
14
+ const CACHE_SAVING = 0.9;
15
+ /** Characters to tokens, roughly, for text in a conversation. */
16
+ const CHARS_PER_TOKEN = 4;
17
+ const BIG_RESULT = 20_000;
18
+ const stepRefs = (hits) => [...hits].sort((a, b) => b.run.startedAt.localeCompare(a.run.startedAt)).slice(0, 5).map((h) => ({ workspace: h.run.workspace, runId: h.run.id, step: h.index + 1 }));
19
+ const credits = (usd) => Math.round(usd * 100);
20
+ export function findInsights(runs, opts = {}) {
21
+ const min = opts.minSteps ?? 3;
22
+ const steps = [];
23
+ for (const run of runs)
24
+ run.steps.forEach((s, index) => {
25
+ if (s.status === "completed" || s.status === "failed")
26
+ steps.push({ run, index, s });
27
+ });
28
+ const out = [];
29
+ const byAgent = new Map();
30
+ for (const x of steps) {
31
+ const k = `${x.run.workspace}\u0000${x.s.agent}`;
32
+ (byAgent.get(k) ?? byAgent.set(k, []).get(k)).push(x);
33
+ }
34
+ for (const [k, list] of byAgent) {
35
+ const [workspace, agent] = k.split("\u0000");
36
+ // cache: big inputs, little of it from the cache, on most of its steps.
37
+ const measured = list.filter((x) => x.s.tokens && x.s.tokens.cacheRead !== undefined && x.s.tokens.input >= 20_000);
38
+ const poor = measured.filter((x) => (x.s.tokens.cacheRead ?? 0) / x.s.tokens.input < 0.3);
39
+ if (poor.length >= min && poor.length >= measured.length / 2) {
40
+ const usd = poor.reduce((n, x) => {
41
+ const t = x.s.tokens;
42
+ const uncached = Math.max(0, t.input - (t.cacheRead ?? 0)) / Math.max(1, t.input);
43
+ return n + (x.s.costUsd ?? 0) * uncached * CACHE_SAVING * 0.5;
44
+ }, 0);
45
+ const rate = Math.round((100 * poor.reduce((n, x) => n + (x.s.tokens.cacheRead ?? 0), 0)) / Math.max(1, poor.reduce((n, x) => n + x.s.tokens.input, 0)));
46
+ out.push({
47
+ id: `cache/${workspace}/${agent}`,
48
+ kind: "cache",
49
+ workspace,
50
+ agent,
51
+ title: `${agent} reads its prompt mostly uncached (${rate}% from the cache)`,
52
+ why: "Each call sends a large prompt that the model's cache rarely serves: something near its start changes from call to call (a date, a run id, a file listing), or the steps are spaced further apart than the cache lasts.",
53
+ fix: "Keep what changes (dates, ids, the task) at the end of the agent's prompt and the large fixed part first; run its steps closer together.",
54
+ credits: credits(usd),
55
+ steps: poor.length,
56
+ evidence: stepRefs(poor),
57
+ });
58
+ }
59
+ // big-result: a tool's result over 20k characters with calls after it,
60
+ // each of which sends it again.
61
+ const big = new Map();
62
+ for (const x of list) {
63
+ const callsAfter = (at) => (x.s.calls ?? []).filter((c) => c.at > at).length;
64
+ const turns = x.s.calls?.length ?? x.s.turns ?? 0;
65
+ for (const e of x.s.events) {
66
+ if (e.type !== "tool" || e.ms == null || (e.bytes ?? 0) < BIG_RESULT)
67
+ continue;
68
+ const after = x.s.calls ? callsAfter(e.t) : Math.max(0, turns - 1);
69
+ if (after < 2)
70
+ continue;
71
+ const tokensResent = ((e.bytes ?? 0) / CHARS_PER_TOKEN) * after;
72
+ const input = Math.max(1, x.s.tokens?.input ?? 0);
73
+ const share = Math.min(1, tokensResent / input);
74
+ const b = big.get(e.text) ?? { hits: [], usd: 0, chars: 0 };
75
+ if (!b.hits.includes(x))
76
+ b.hits.push(x);
77
+ b.usd += (x.s.costUsd ?? 0) * share;
78
+ b.chars = Math.max(b.chars, e.bytes ?? 0);
79
+ big.set(e.text, b);
80
+ }
81
+ }
82
+ for (const [tool, b] of big) {
83
+ if (b.hits.length < min)
84
+ continue;
85
+ out.push({
86
+ id: `big-result/${workspace}/${agent}/${tool}`,
87
+ kind: "big-result",
88
+ workspace,
89
+ agent,
90
+ tool,
91
+ title: `${tool} hands ${agent} results up to ${Math.round(b.chars / 1000)}k characters, sent again on every later call`,
92
+ why: "A tool result stays in the conversation, so every model call after it pays for it again.",
93
+ fix: `Have ${tool} return less (filter, a summary, or write the bulk to a file and return the path), or split the step so the big read ends it.`,
94
+ credits: credits(b.usd),
95
+ steps: b.hits.length,
96
+ evidence: stepRefs(b.hits),
97
+ });
98
+ }
99
+ // retry-loop: one tool erroring three or more times in a step.
100
+ const loops = new Map();
101
+ for (const x of list) {
102
+ const errs = new Map();
103
+ for (const e of x.s.events)
104
+ if (e.type === "tool" && e.ms != null && e.err)
105
+ errs.set(e.text, (errs.get(e.text) ?? 0) + 1);
106
+ for (const [tool, n] of errs)
107
+ if (n >= 3)
108
+ (loops.get(tool) ?? loops.set(tool, []).get(tool)).push(x);
109
+ }
110
+ for (const [tool, hits] of loops) {
111
+ if (hits.length < Math.min(min, 2))
112
+ continue;
113
+ out.push({
114
+ id: `retry-loop/${workspace}/${agent}/${tool}`,
115
+ kind: "retry-loop",
116
+ workspace,
117
+ agent,
118
+ tool,
119
+ title: `${agent} keeps retrying ${tool} after it errors`,
120
+ why: "The tool fails the same way and the agent tries again instead of stopping — every attempt is a model call and a tool call.",
121
+ fix: `Read one of the errors (the trace's input / output): fix the tool or its input, or tell ${agent} in its prompt to stop after one failure and say BLOCKED with why.`,
122
+ credits: credits(hits.reduce((n, x) => n + (x.s.costUsd ?? 0) * 0.3, 0)),
123
+ steps: hits.length,
124
+ evidence: stepRefs(hits),
125
+ });
126
+ }
127
+ // timeout: steps that took 80% or more of their timeout.
128
+ const tight = list.filter((x) => {
129
+ if (!x.s.timeout || !x.s.startedAt || !x.s.finishedAt)
130
+ return false;
131
+ const secs = (Date.parse(x.s.finishedAt) - Date.parse(x.s.startedAt)) / 1000;
132
+ return secs >= 0.8 * x.s.timeout;
133
+ });
134
+ if (tight.length >= Math.min(min, 2)) {
135
+ const limit = tight[0].s.timeout;
136
+ out.push({
137
+ id: `timeout/${workspace}/${agent}`,
138
+ kind: "timeout",
139
+ workspace,
140
+ agent,
141
+ title: `${agent} uses most of its ${limit}s timeout`,
142
+ why: "Steps finish just inside their limit; a slightly bigger input or a slow site and they are cut off.",
143
+ fix: "Raise timeout: on the step, or give the agent less to do per step (each: to fan out, a smaller batch).",
144
+ credits: null,
145
+ steps: tight.length,
146
+ evidence: stepRefs(tight),
147
+ });
148
+ }
149
+ }
150
+ const order = { "retry-loop": 0, timeout: 1, "big-result": 2, cache: 3 };
151
+ return out.sort((a, b) => (b.credits ?? -1) - (a.credits ?? -1) || order[a.kind] - order[b.kind]);
152
+ }
@@ -0,0 +1,140 @@
1
+ import type { Outcome } from "./outcome.ts";
2
+ export type IssueKind = "platform" | "failed" | "blocked" | "bad" | "missed";
3
+ export type IssueStatus = "open" | "acknowledged" | "snoozed" | "resolved";
4
+ export type IssueAction = "acknowledge" | "snooze" | "resolve" | "reopen";
5
+ /** As much of a run as an issue needs (the overview's RunLite carries it). */
6
+ export interface IssueRun {
7
+ workspace: string;
8
+ id: string;
9
+ /** The flow's name, or "agent: x" / "eval: y". */
10
+ what: string;
11
+ outcome: Outcome;
12
+ startedAt: string;
13
+ finishedAt?: string | null;
14
+ failure?: {
15
+ agent: string;
16
+ reason: string;
17
+ } | null;
18
+ headline?: string | null;
19
+ /** The revision the run ran (RunRecord.revision), for "regressed since". */
20
+ revision?: string | null;
21
+ /** Not a run: a scheduled fire that started none (FOL-72 step 7). Its
22
+ * `startedAt` is when it was due; its id is `missed-<when>`. */
23
+ missed?: boolean;
24
+ }
25
+ export interface IssueEvent {
26
+ at: string;
27
+ /** Who: a person's email, or a key's label. */
28
+ by: string;
29
+ action: IssueAction;
30
+ /** Snoozed until, ISO. */
31
+ until?: string;
32
+ /** Or snoozed until it happens this many more times. */
33
+ untilCount?: number;
34
+ note?: string;
35
+ }
36
+ /** What people did to one issue. Stored; the rest is derived. */
37
+ export interface IssueState {
38
+ history: IssueEvent[];
39
+ }
40
+ export interface Issue {
41
+ id: string;
42
+ kind: IssueKind;
43
+ status: IssueStatus;
44
+ /** The desk; for a platform issue, the newest desk it hit. */
45
+ workspace: string;
46
+ /** For a platform issue, every desk it hit. */
47
+ workspaces: string[];
48
+ /** The flow (or agent run); for a platform issue, what broke. */
49
+ what: string;
50
+ /** The agent the failure was at, for a failed issue. */
51
+ agent: string | null;
52
+ /** Why, from the newest run: its failure, or its summary line. */
53
+ reason: string;
54
+ /** What to do about it, in plain words. */
55
+ action: string;
56
+ /** Since it was last resolved (or ever): when the first and the latest of
57
+ * its runs went wrong (their end, else their start), how many. */
58
+ firstSeen: string;
59
+ lastSeen: string;
60
+ count: number;
61
+ /** The newest runs it covers, newest first, at most 20; a missed
62
+ * schedule's are the fires that started nothing (`missed`). */
63
+ runs: {
64
+ workspace: string;
65
+ id: string;
66
+ at: string;
67
+ missed?: boolean;
68
+ }[];
69
+ /** The newest run of the same flow went fine after the last occurrence. */
70
+ recoveredAt: string | null;
71
+ acknowledged: {
72
+ at: string;
73
+ by: string;
74
+ } | null;
75
+ snoozedUntil: string | null;
76
+ /** Snoozed until it happens this many more times: how many are left. */
77
+ snoozedFor: number | null;
78
+ /** It was resolved before and came back: this episode follows a resolve. */
79
+ regressed: boolean;
80
+ /** The revision the first run of this episode ran — "since abc1234". */
81
+ sinceRevision: string | null;
82
+ resolved: {
83
+ at: string;
84
+ by: string;
85
+ } | null;
86
+ history: IssueEvent[];
87
+ }
88
+ /** The model refusing the credential, as the runner sees it (runner.ts
89
+ * isAuthRefusal reads this one pattern, so a refusal the runner retries on
90
+ * is the same refusal an issue groups by — Codex review). */
91
+ export declare const AUTH_REFUSAL: RegExp;
92
+ /** A failure that is the platform's, not the desk's. */
93
+ export interface PlatformCause {
94
+ key: string;
95
+ /** What broke, in the words a person would use. */
96
+ label: string;
97
+ /** What to do about it. None of it is editing the desk. */
98
+ fix: string;
99
+ }
100
+ /** Whether a failure reason is the platform's, and which kind. */
101
+ export declare function platformCause(reason: string | null | undefined): PlatformCause | null;
102
+ /** Which issue a run belongs to, or null when it is not a problem. */
103
+ export declare function issueKeyOf(r: IssueRun): {
104
+ id: string;
105
+ kind: IssueKind;
106
+ agent: string | null;
107
+ cause: PlatformCause | null;
108
+ } | null;
109
+ /**
110
+ * Every issue the runs show, with what people did to them folded in.
111
+ * `states` is keyed by issue id. Resolved issues are kept (with their last
112
+ * occurrences) while their runs are still in the window, so their history
113
+ * can be read; a status filter is the caller's.
114
+ */
115
+ export declare function foldIssues(runs: Iterable<IssueRun>, states: Record<string, IssueState>, now: number,
116
+ /** `autoClose(workspace, flow)`: the flow says `issues: auto-close`, so a
117
+ * run that goes fine after a failure closes it (FOL-72 step 7). Never a
118
+ * platform cause or a missed schedule — those stay until a person says. */
119
+ opts?: {
120
+ autoClose?: (workspace: string, what: string) => boolean;
121
+ }): Issue[];
122
+ /** An issue still asking for someone: open or acknowledged, not snoozed or resolved. */
123
+ export declare const isActiveIssue: (i: Issue) => boolean;
124
+ /**
125
+ * What the run doctor needs to look at an issue: its newest run, the
126
+ * workspace that run is in (the assistant works inside one), and the
127
+ * question the run page's Doctor button asks — so the issue's button, the
128
+ * CLI's `issues doctor` and the run page put it the same way. Null for a
129
+ * missed schedule, which has no run to read.
130
+ */
131
+ export declare function issueDoctor(issue: Pick<Issue, "kind" | "runs">): {
132
+ workspace: string;
133
+ runId: string;
134
+ prompt: string;
135
+ } | null;
136
+ /** Parse "30m", "4h", "1d", "1w" into milliseconds; null if not one. */
137
+ export declare function snoozeMs(s: string): number | null;
138
+ /** One person's act on an issue, appended to its history. Refuses a snooze
139
+ * with no future `until`. Pure: the caller stores the result. */
140
+ export declare function applyIssueAction(state: IssueState | null, ev: IssueEvent, now: number): IssueState;