@cleocode/goal 0.0.0-stage → 0.9.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/CHANGELOG.md +328 -0
  2. package/LICENSE +21 -0
  3. package/README.md +141 -2
  4. package/cli/ocg.ts +149 -0
  5. package/dist/cli.js +8708 -0
  6. package/dist/tui.d.ts +2 -0
  7. package/dist/tui.js +1920 -0
  8. package/package.json +79 -4
  9. package/rpc.ts +2 -0
  10. package/server.ts +3 -0
  11. package/skills/write-goal/SKILL.md +211 -0
  12. package/skills/write-goal/assets/goal.template.yaml +38 -0
  13. package/skills/write-goal/references/examples.md +283 -0
  14. package/skills/write-goal/references/goal-schema.md +53 -0
  15. package/skills/write-goal/references/interview.md +36 -0
  16. package/src/cli/install.ts +125 -0
  17. package/src/cli/status.ts +116 -0
  18. package/src/contract/parse.ts +454 -0
  19. package/src/contract/render.ts +134 -0
  20. package/src/contract/types.ts +158 -0
  21. package/src/engine/amendment.ts +51 -0
  22. package/src/engine/amendments.ts +72 -0
  23. package/src/engine/cleo-bridge.ts +339 -0
  24. package/src/engine/cleo-link.ts +150 -0
  25. package/src/engine/continue-work.ts +131 -0
  26. package/src/engine/decisions.ts +44 -0
  27. package/src/engine/dogfood-record.ts +278 -0
  28. package/src/engine/handoff.ts +102 -0
  29. package/src/engine/interrupt-policy.ts +45 -0
  30. package/src/engine/inventory.ts +70 -0
  31. package/src/engine/launches.ts +82 -0
  32. package/src/engine/loop-stop.ts +50 -0
  33. package/src/engine/loop.ts +109 -0
  34. package/src/engine/next-work.ts +82 -0
  35. package/src/engine/owner-intent.ts +71 -0
  36. package/src/engine/owner-reviews.ts +178 -0
  37. package/src/engine/peer-announce.ts +67 -0
  38. package/src/engine/peer-transport.ts +78 -0
  39. package/src/engine/policy.ts +42 -0
  40. package/src/engine/registry.ts +119 -0
  41. package/src/engine/similarity.ts +65 -0
  42. package/src/engine/stale-decisions.ts +68 -0
  43. package/src/engine/state.ts +185 -0
  44. package/src/engine/store.ts +263 -0
  45. package/src/engine/tick-record.ts +129 -0
  46. package/src/engine/tick.ts +207 -0
  47. package/src/rpc.ts +182 -0
  48. package/src/server/app.ts +4320 -0
  49. package/src/server/help.ts +50 -0
  50. package/src/server/index.ts +23 -0
  51. package/src/server/intent.ts +22 -0
  52. package/src/server/messages.ts +228 -0
  53. package/src/summary/audit.ts +76 -0
  54. package/src/summary/build.ts +314 -0
  55. package/src/tui/dashboard.ts +322 -0
  56. package/src/tui/decision-form.ts +12 -0
  57. package/src/tui/format.ts +127 -0
  58. package/src/tui/index.tsx +718 -0
  59. package/src/tui/manager.ts +94 -0
  60. package/src/tui/navigation.ts +11 -0
  61. package/src/tui/pointer-select.ts +20 -0
  62. package/src/tui/tracer.ts +51 -0
  63. package/src/util/git.ts +102 -0
  64. package/src/util/ids.ts +20 -0
  65. package/src/util/shell.ts +42 -0
  66. package/src/verify/artifact.ts +95 -0
  67. package/src/verify/pipeline.ts +200 -0
  68. package/tui.tsx +2 -0
@@ -0,0 +1,150 @@
1
+ // T027 — optional first-class CLEO linkage. When the project carries a CLEO
2
+ // workspace, the goal records the linkage THROUGH THE CLI ONLY (never by
3
+ // reading .cleo/*.db); the project-context.json config file may be read as a
4
+ // presence signal and fallback. No CLEO → the goal system behaves identically;
5
+ // there is no hard dependency either way.
6
+ import { existsSync, readFileSync } from "node:fs"
7
+ import { join } from "node:path"
8
+
9
+ export type CleoFacts = { source: "cli" | "config"; projectId?: string; projectTypes: string[]; detectedAt?: string }
10
+
11
+ export type Runner = (command: string, args: string[], timeoutMs?: number) => Promise<{ stdout: string; exit: number }>
12
+
13
+ /**
14
+ * Detect CLEO linkage for a project root. Order: the `cleo` CLI (authoritative,
15
+ * spawn with a hard timeout), then .cleo/project-context.json (presence +
16
+ * config facts), then none. All failures degrade to undefined — never throw.
17
+ */
18
+ export async function cleoFacts(projectRoot: string, run: Runner, timeoutMs = 2500): Promise<CleoFacts | undefined> {
19
+ const config = join(projectRoot, ".cleo", "project-context.json")
20
+ const hasConfig = existsSync(config)
21
+ if (!hasConfig && !existsSync(join(projectRoot, ".cleo"))) return undefined
22
+ try {
23
+ const { stdout, exit } = await run("cleo", ["doctor", "project-identity"], timeoutMs)
24
+ if (exit === 0 && stdout.trim()) {
25
+ const parsed = JSON.parse(stdout)
26
+ const data = parsed?.data ?? parsed
27
+ const projectId = typeof data?.projectId === "string" ? data.projectId : typeof parsed?.projectId === "string" ? parsed.projectId : undefined
28
+ return { source: "cli", projectId, projectTypes: [] }
29
+ }
30
+ } catch {
31
+ // CLI missing, too slow, or errored — degrade to the config signal
32
+ }
33
+ if (!hasConfig) return undefined
34
+ try {
35
+ const parsed = JSON.parse(readFileSync(config, "utf8"))
36
+ return { source: "config", projectTypes: Array.isArray(parsed?.projectTypes) ? parsed.projectTypes.map(String) : [], ...(parsed?.detectedAt ? { detectedAt: String(parsed.detectedAt) } : {}) }
37
+ } catch {
38
+ return { source: "config", projectTypes: [] }
39
+ }
40
+ }
41
+
42
+ /** The ledger payload recorded when a goal starts in a CLEO-linked project. */
43
+ export function cleoLinkEvent(facts: CleoFacts | undefined) {
44
+ return facts ? { type: "cleo-linked" as const, via: facts.source, ...(facts.projectId ? { projectId: facts.projectId } : {}), ...(facts.projectTypes.length ? { projectTypes: facts.projectTypes } : {}) } : undefined
45
+ }
46
+
47
+ // v0.5.0 — the CLEO health surface. Three states, no more: the CLI is absent,
48
+ // the CLI is present but this project is not initialized, or both are present.
49
+ // OCG never installs or initializes anything by itself: it reports the state and
50
+ // the EXACT command, and runs that command only when the owner asks for it.
51
+
52
+ export type CleoHealthState = "cli-absent" | "uninitialized" | "initialized"
53
+
54
+ /** Local string coercion (cleo-link.ts has no parser helpers). */
55
+ const asText = (v: unknown): string => (typeof v === "string" ? v.trim() : "")
56
+
57
+ export type CleoOffer = {
58
+ kind: "install" | "init"
59
+ /** The exact command OCG would run — only on an explicit owner choice. */
60
+ command: string
61
+ why: string
62
+ }
63
+
64
+ export type CleoHealth = {
65
+ state: CleoHealthState
66
+ cli: { present: boolean; version?: string }
67
+ workspace: { present: boolean; path: string }
68
+ projectId?: string
69
+ offer?: CleoOffer
70
+ /** One-line, state-accurate summary (never a claim that CLEO is usable). */
71
+ detail: string
72
+ }
73
+
74
+ /** The documented global install for the CLEO CLI (verified against @cleocode/cleo). */
75
+ export const CLEO_INSTALL_COMMAND = "npm i -g @cleocode/cleo"
76
+ /** The documented in-project initializer (verified: `cleo init --help`). */
77
+ export const CLEO_INIT_COMMAND = "cleo init"
78
+
79
+ /**
80
+ * Probe CLEO for a project root. `run` is invoked with `cleo --version` first to
81
+ * decide whether the CLI exists at all (a spawn rejection means absent), then with
82
+ * `cleo doctor project-identity` only when the workspace exists. Nothing here
83
+ * mutates the machine, and no failure throws.
84
+ */
85
+ export async function cleoHealth(projectRoot: string, run: Runner, timeoutMs = 2500): Promise<CleoHealth> {
86
+ const cleoDir = join(projectRoot, ".cleo")
87
+ const workspace = { present: existsSync(cleoDir), path: cleoDir }
88
+ let present = false
89
+ let version: string | undefined
90
+ try {
91
+ const { stdout, exit } = await run("cleo", ["--version"], timeoutMs)
92
+ if (exit === 0) {
93
+ present = true
94
+ try {
95
+ const parsed = JSON.parse(stdout)
96
+ version = asText(parsed?.data?.version ?? parsed?.version) || undefined
97
+ } catch {
98
+ version = stdout.trim().split(/\s+/)[0] || undefined
99
+ }
100
+ }
101
+ } catch {
102
+ present = false // the spawn itself failed — no CLI on PATH
103
+ }
104
+
105
+ if (!present)
106
+ return {
107
+ state: "cli-absent",
108
+ cli: { present: false },
109
+ workspace,
110
+ offer: { kind: "install", command: CLEO_INSTALL_COMMAND, why: "the cleo CLI is not on PATH, so no task board is reachable from this project" },
111
+ detail: "CLEO CLI not found on PATH — goal tracking works exactly as before, with no CLEO integration",
112
+ }
113
+
114
+ if (!workspace.present)
115
+ return {
116
+ state: "uninitialized",
117
+ cli: { present: true, ...(version ? { version } : {}) },
118
+ workspace,
119
+ offer: { kind: "init", command: CLEO_INIT_COMMAND, why: "this project has no .cleo workspace, so a goal's work has no durable task board yet" },
120
+ detail: "CLEO CLI present, but this project has no .cleo workspace — a goal runs normally and records no task linkage",
121
+ }
122
+
123
+ let projectId: string | undefined
124
+ try {
125
+ const { stdout, exit } = await run("cleo", ["doctor", "project-identity"], timeoutMs)
126
+ if (exit === 0 && stdout.trim()) {
127
+ const parsed = JSON.parse(stdout)
128
+ const data = parsed?.data ?? parsed
129
+ projectId = asText(data?.trackedId ?? data?.projectId ?? parsed?.projectId) || undefined
130
+ }
131
+ } catch {
132
+ // CLI answered --version but not this probe: still initialized (the
133
+ // workspace exists); the id is simply unknown.
134
+ }
135
+ return {
136
+ state: "initialized",
137
+ cli: { present: true, ...(version ? { version } : {}) },
138
+ workspace,
139
+ ...(projectId ? { projectId } : {}),
140
+ detail: `CLEO CLI present and this project is initialized${projectId ? ` (project ${projectId})` : ""} — goals can name and complete CLEO tasks`,
141
+ }
142
+ }
143
+
144
+ /** Render a health probe as the owner/agent sees it. Pure; safe to unit-test. */
145
+ export function cleoHealthReport(health: CleoHealth): string {
146
+ const version = health.cli.version ? ` (cleo ${health.cli.version})` : ""
147
+ const head = `◎ CLEO — ${health.detail}`
148
+ if (!health.offer) return [head, `Nothing to do: ${health.state === "initialized" ? "goals can link and complete CLEO tasks." : "no CLEO action is offered."}`].join("\n")
149
+ return [head, `On your explicit choice only (OCG never runs this silently): ${health.offer.command}`, `Why: ${health.offer.why}. Run /goal cleo ${health.offer.kind} to do it, or /goal cleo to re-check.`].join("\n")
150
+ }
@@ -0,0 +1,131 @@
1
+ // v0.8.0 — turning a ready CLEO item into the next goal's contract.
2
+ //
3
+ // The owner's policy is "take the next ready CLEO item without asking". The
4
+ // engine therefore has to author a contract it did not interview for. This file
5
+ // does that from facts only: the task's own title, description and acceptance
6
+ // statements. Each acceptance statement becomes one verifier criterion (the same
7
+ // evidence standard the engine already uses for semantic checks); the task is
8
+ // linked so the existing CLEO bridge can close it on evidence and reports every
9
+ // AC it cannot prove. Nothing here invents a passing command.
10
+
11
+ import type { Autonomy } from "../contract/types"
12
+ import type { Runner } from "./cleo-link"
13
+
14
+ export type CleoTask = {
15
+ id: string
16
+ title: string
17
+ description: string
18
+ acceptance: string[]
19
+ }
20
+
21
+ const parse = (stdout: string): any => {
22
+ try { return JSON.parse(stdout) } catch { return undefined }
23
+ }
24
+
25
+ /**
26
+ * The next ready LEAF item, from CLEO's own CLI. `cleo next` suggests by
27
+ * priority and dependency; `cleo show --full` supplies the type and acceptance
28
+ * text. A parent (epic/saga) is never started by itself — its acceptance is a
29
+ * rollup, not work — so parents are reported as skipped and the next leaf is
30
+ * tried. Any failure is "no ready work", never a guess.
31
+ */
32
+ export type SkippedParent = { id: string; type: string; title: string }
33
+ export type ReadyCleo = { task?: CleoTask; skipped: SkippedParent[] }
34
+
35
+ export async function readyCleoTask(run: Runner, bin = "cleo"): Promise<ReadyCleo> {
36
+ const next = await run(bin, ["next", "-n", "5"])
37
+ if (next.exit !== 0) return { skipped: [] }
38
+ const suggestions = parse(next.stdout)?.data?.suggestions
39
+ if (!Array.isArray(suggestions)) return { skipped: [] }
40
+ const skipped: SkippedParent[] = []
41
+ for (const suggestion of suggestions) {
42
+ const id = typeof suggestion?.id === "string" ? suggestion.id : undefined
43
+ if (!id) continue
44
+ const show = await run(bin, ["show", id, "--full"])
45
+ if (show.exit !== 0) continue
46
+ const task = parse(show.stdout)?.data?.task
47
+ if (!task || ["done", "completed", "cancelled", "archived"].includes(String(task.status))) continue
48
+ const type = String(task.type ?? suggestion?.type ?? "")
49
+ if (type === "epic" || type === "saga") {
50
+ skipped.push({ id, type, title: String(task.title ?? suggestion?.title ?? id) })
51
+ continue
52
+ }
53
+ return {
54
+ task: {
55
+ id,
56
+ title: String(task.title ?? suggestion?.title ?? id),
57
+ description: String(task.description ?? ""),
58
+ acceptance: Array.isArray(task.acceptance) ? task.acceptance.map(String).filter(Boolean) : [],
59
+ },
60
+ skipped,
61
+ }
62
+ }
63
+ return { skipped }
64
+ }
65
+
66
+ const q = (value: string) => JSON.stringify(value)
67
+
68
+ /** A slug that does not collide with a live, archived or draft goal. */
69
+ export function continuationSlug(taskId: string, taken: ReadonlySet<string>): string {
70
+ const base = `cleo-${taskId.toLowerCase().replace(/[^a-z0-9]+/g, "-")}`
71
+ if (!taken.has(base)) return base
72
+ for (let n = 2; n < 100; n++) if (!taken.has(`${base}-${n}`)) return `${base}-${n}`
73
+ return `${base}-${Date.now()}`
74
+ }
75
+
76
+ /**
77
+ * The contract for the next goal. Deliberately boring: the linked task, one
78
+ * verifier criterion per acceptance statement (or the title when the task states
79
+ * none), the owner's autonomy policy, and no budget (budgets stay opt-in).
80
+ */
81
+ export function continuationContractYaml(input: {
82
+ slug: string
83
+ task: CleoTask
84
+ sourceSlug: string
85
+ autonomy: Autonomy
86
+ now?: number
87
+ }): string {
88
+ const { slug, task, sourceSlug, autonomy } = input
89
+ const statements = task.acceptance.length ? task.acceptance : [task.title]
90
+ const criteria = statements.map((statement, i) => [
91
+ ` - id: C${i + 1}`,
92
+ ` statement: ${q(`${task.id} acceptance ${i + 1}: ${statement}`)}`,
93
+ ` essential: true`,
94
+ ` check: {kind: verifier, ask: ${q(`Assess whether the work done for CLEO ${task.id} genuinely satisfies this acceptance criterion: ${statement} Quote the evidence from the repository.`)} }`,
95
+ ].join("\n"))
96
+ return [
97
+ "schema: goal/v1",
98
+ `id: ${slug}`,
99
+ `title: ${q(task.title)}`,
100
+ "intent:",
101
+ ` verbatim: ${q("Take the next ready CLEO item without asking.")}`,
102
+ `outcome: ${q(task.title)}`,
103
+ `why: ${q(`The owner's standing policy: after a goal completes, start the next ready CLEO item (${task.id}) by itself; this goal was started from that policy and the task's own acceptance.`)}`,
104
+ "scope:",
105
+ ` in: []`,
106
+ ` out: []`,
107
+ "non_goals:",
108
+ ` - ${q("Changing anything outside the linked CLEO task's scope")}`,
109
+ ` - ${q("Editing this goal's contract or its evidence")}`,
110
+ "criteria:",
111
+ ...criteria,
112
+ "cleo:",
113
+ " tasks:",
114
+ ` - id: ${task.id}`,
115
+ "plan:",
116
+ ` - {id: S1, title: ${q(`Complete ${task.id} and prove its acceptance`)}}`,
117
+ "autonomy:",
118
+ ` questions: ${autonomy.questions}`,
119
+ ` on_user_message: ${autonomy.on_user_message}`,
120
+ ` on_interrupt: ${autonomy.on_interrupt}`,
121
+ ` amend: [${autonomy.amend.join(", ")}]`,
122
+ ` on_stall: ${autonomy.on_stall}`,
123
+ "verification: {mode: host+verifier, max_rejections: 3}",
124
+ "meta:",
125
+ ` created: ${q(new Date(input.now ?? Date.now()).toISOString())}`,
126
+ ` author: opencode-goal`,
127
+ ` source: continue-work@0.8.0`,
128
+ ` from: ${sourceSlug}`,
129
+ "",
130
+ ].join("\n")
131
+ }
@@ -0,0 +1,44 @@
1
+ import type { EngineMessage } from "../server/messages"
2
+ import { OwnerReviewStore, type ReviewChoice, type ReviewRequest } from "./owner-reviews"
3
+
4
+ export type DecisionKind = "needs_review" | "paused-after-verdict" | "blocked" | "budget_limited" | "amend-proposed" | "supersede-ack" | "complete" | "start-picker" | "owner-operation"
5
+ export type DecisionTarget = { slug: string; runId?: string; sessionID?: string; lock: string; status: string }
6
+ export type DecisionRequest = ReviewRequest & {
7
+ kind: DecisionKind
8
+ message: EngineMessage
9
+ target: DecisionTarget
10
+ /** Draft choices (start-picker) bind each candidate's bytes, too. */
11
+ candidates: Record<string, DecisionTarget>
12
+ extra: Record<string, unknown>
13
+ }
14
+
15
+ export function decisionChoices(request: DecisionRequest): ReviewChoice[] {
16
+ return request.message.choices.map((c, i) => ({ value: String(i), label: c.label, description: `${c.act ? "" : "Guidance only; no operation: "}${c.run}` }))
17
+ }
18
+
19
+ /**
20
+ * Should the native form be re-offered when a turn ends?
21
+ *
22
+ * An unanswered decision is not stale, but re-asking it on every turn end is
23
+ * noise (the owner sees the same question forever). The policy is: ask once
24
+ * per process; after that, silence until an explicit `goal_review` (which
25
+ * calls the form directly) or a reload (a new process, so `alreadyAsked` is
26
+ * false again). Dismissed reviews (`interrupted`) and open forms never re-ask.
27
+ */
28
+ export function shouldReaskDecision(input: {
29
+ alreadyAsked: boolean
30
+ status: string
31
+ stale: boolean
32
+ formOpen: boolean
33
+ }): { ask: boolean; reason?: "already-asked" | "interrupted" | "stale" | "form-open" } {
34
+ if (input.stale) return { ask: false, reason: "stale" }
35
+ if (input.formOpen) return { ask: false, reason: "form-open" }
36
+ if (input.status === "interrupted") return { ask: false, reason: "interrupted" }
37
+ if (input.alreadyAsked) return { ask: false, reason: "already-asked" }
38
+ return { ask: true }
39
+ }
40
+
41
+ export class DecisionStore extends OwnerReviewStore<DecisionRequest> {
42
+ constructor(file: string) { super(file, []) }
43
+ protected reviewChoices(request: DecisionRequest) { return decisionChoices(request) }
44
+ }
@@ -0,0 +1,278 @@
1
+ // v0.5.1 - the exact-release real-model dogfood record and its readable export.
2
+ //
3
+ // T102: the five existing real-model drivers were five separate programs with no
4
+ // shared record format and no single export, so a release's real-model proof
5
+ // could not be read or re-checked independently. This module is the ONE record
6
+ // shape and the ONE export both halves are built from.
7
+ //
8
+ // Honesty rule: a flow is labelled by what actually happened - `pass`, `fixture`,
9
+ // `stale-cache`, `partial` or `timeout` - and never presented as a pass when it
10
+ // was not one. `runDogfood` asserts the EXPORT is complete and honest (every flow
11
+ // recorded, every required field present), never that every flow passed.
12
+
13
+ import { createHash } from "node:crypto"
14
+
15
+ export type FlowId = "interview" | "manager" | "lifecycle" | "completion" | "amendment"
16
+ export const FLOWS: readonly FlowId[] = ["interview", "manager", "lifecycle", "completion", "amendment"]
17
+
18
+ export type FlowLabel = "pass" | "fixture" | "stale-cache" | "partial" | "timeout"
19
+ export const LABELS: readonly FlowLabel[] = ["pass", "fixture", "stale-cache", "partial", "timeout"]
20
+
21
+ /** The exact release the flows were exercised against, by hash not by label. */
22
+ export type ReleaseIdentity = { version: string; ref: string; sha: string; artifacts: Record<string, string> }
23
+ /** The serving engine that actually ran the flows. */
24
+ export type ServingIdentity = { pid: number; location: string; endpoint?: string }
25
+ /** The selections the flows used - never inferred from a config file. */
26
+ export type Selections = { model: string; agent: string }
27
+ /** A bounded read coordinate: page + window, never an unbounded dump. */
28
+ export type ReadCoordinate = { path: string; offset: number; limit: number }
29
+
30
+ /** The raw owner interaction, preserved (bounded) alongside the verdict. */
31
+ export type RawOutcome = {
32
+ ownerMessages: string[]
33
+ forms: Array<{ id: string; question: string; answer?: string }>
34
+ toolCalls: Array<{ name: string; result?: string }>
35
+ results: string[]
36
+ }
37
+
38
+ export type FlowRecord = {
39
+ flow: FlowId
40
+ release: ReleaseIdentity
41
+ serving: ServingIdentity
42
+ selections: Selections
43
+ requestIds: string[]
44
+ readCoordinates: ReadCoordinate[]
45
+ label: FlowLabel
46
+ labelReason: string
47
+ ms: number
48
+ startedAt: string
49
+ raw: RawOutcome
50
+ }
51
+
52
+ export type DogfoodExport = {
53
+ schema: "dogfood-export/v1"
54
+ release: ReleaseIdentity
55
+ serving: ServingIdentity
56
+ selections: Selections
57
+ generatedAt: string
58
+ flows: FlowRecord[]
59
+ labels: Record<FlowLabel, number>
60
+ }
61
+
62
+ /**
63
+ * The cache key that decides whether a stored per-flow record may be reused.
64
+ *
65
+ * It binds the SERVING ENGINE as well as the release: a record produced by a
66
+ * different pid, location or endpoint is NOT reusable, because the drivers
67
+ * themselves refuse to compare a live engine against a prior recorded elsewhere
68
+ * (the manager flow failed exactly that way with "QA serving PID changed").
69
+ */
70
+ export const cacheKey = (release: ReleaseIdentity, serving: ServingIdentity): string =>
71
+ createHash("sha256").update(`${release.sha}\n${JSON.stringify(release.artifacts)}\n${serving.pid}\n${serving.location}\n${serving.endpoint ?? ""}`).digest("hex")
72
+
73
+ /** Why a stored record cannot be reused, or undefined when it matches exactly. */
74
+ export function cacheMismatch(stored: FlowRecord, release: ReleaseIdentity, serving: ServingIdentity): string | undefined {
75
+ if (stored.release.sha !== release.sha) return `the stored record was for release ${stored.release.sha}, not ${release.sha}`
76
+ if (JSON.stringify(stored.release.artifacts) !== JSON.stringify(release.artifacts)) return "the artifact hashes changed since the stored record"
77
+ if (stored.serving.pid !== serving.pid) return `the stored record was produced by serving pid ${stored.serving.pid}; this engine is ${serving.pid}`
78
+ if (stored.serving.location !== serving.location) return `the stored record was produced at ${stored.serving.location}; this engine serves ${serving.location}`
79
+ if ((stored.serving.endpoint ?? "") !== (serving.endpoint ?? "")) return `the stored record was produced at endpoint ${stored.serving.endpoint ?? "(none)"}; this engine is ${serving.endpoint ?? "(none)"}`
80
+ return undefined
81
+ }
82
+
83
+ /** Classify ONE flow from what actually happened. Order matters: the honest worst case wins. */
84
+ export function classify(input: { exit: number; timedOut?: boolean; providerError?: boolean; fixture?: boolean; staleCache?: boolean; raw: RawOutcome; note?: string }): { label: FlowLabel; reason: string } {
85
+ // A captured cause is appended verbatim so a reader never has to guess what happened.
86
+ const suffix = input.note ? ` (${input.note.replace(/\s+/g, " ").slice(0, 200)})` : ""
87
+ if (input.timedOut) return { label: "timeout", reason: `the flow did not finish inside its bound; no verdict was produced${suffix}` }
88
+ if (input.providerError) return { label: "timeout", reason: `the provider failed before the flow produced a verdict${suffix}` }
89
+ // A stale record from another engine (or release) is the ROOT cause when it
90
+ // exists, so it is named BEFORE the generic incomplete-flow label.
91
+ if (input.staleCache) return { label: "stale-cache", reason: `a stored record could not be reused because the release, the artifact hashes or the serving engine changed${suffix}` }
92
+ if (input.exit !== 0) return { label: "partial", reason: `the flow did not complete (exit ${input.exit}) after producing some evidence${suffix}` }
93
+ if (input.fixture) return { label: "fixture", reason: `the flow ran against the scripted fixture provider, not a real model${suffix}` }
94
+ // A pass needs the raw interaction preserved, but not every KIND of it: a flow
95
+ // that legitimately asked no form has no forms to preserve. Only preserving
96
+ // NOTHING at all means the claim cannot be checked.
97
+ const total = input.raw.ownerMessages.length + input.raw.forms.length + input.raw.toolCalls.length + input.raw.results.length
98
+ if (!total) return { label: "partial", reason: `the flow exited 0 but preserved no raw interaction${suffix}` }
99
+ return { label: "pass", reason: `a real model completed the flow and its raw interaction was preserved (${input.raw.ownerMessages.length} owner messages, ${input.raw.forms.length} forms, ${input.raw.toolCalls.length} tool calls)${suffix}` }
100
+ }
101
+
102
+ export const summarize = (records: FlowRecord[]): Record<FlowLabel, number> => {
103
+ const out = Object.fromEntries(LABELS.map((l) => [l, 0])) as Record<FlowLabel, number>
104
+ for (const r of records) out[r.label] += 1
105
+ return out
106
+ }
107
+
108
+ /** Every field C3 requires of every flow record. Returns the missing ones. */
109
+ export function validateExport(exp: DogfoodExport): string[] {
110
+ const problems: string[] = []
111
+ if (exp.schema !== "dogfood-export/v1") problems.push(`schema is ${String(exp.schema)}`)
112
+ if (!exp.release.sha) problems.push("release.sha is empty")
113
+ if (!exp.release.version) problems.push("release.version is empty")
114
+ if (!Object.keys(exp.release.artifacts).length) problems.push("release.artifacts is empty")
115
+ if (!exp.serving.pid) problems.push("serving.pid is empty")
116
+ if (!exp.serving.location) problems.push("serving.location is empty")
117
+ if (!exp.selections.model) problems.push("selections.model is empty")
118
+ if (!exp.selections.agent) problems.push("selections.agent is empty")
119
+ const missing = FLOWS.filter((f) => !exp.flows.some((r) => r.flow === f))
120
+ for (const f of missing) problems.push(`flow ${f} has no record`)
121
+ for (const r of exp.flows) {
122
+ if (!r.labelReason) problems.push(`flow ${r.flow} has no label reason`)
123
+ if (!LABELS.includes(r.label)) problems.push(`flow ${r.flow} has an unknown label ${String(r.label)}`)
124
+ if (r.release.sha !== exp.release.sha) problems.push(`flow ${r.flow} was recorded against ${r.release.sha}, not ${exp.release.sha}`)
125
+ // A PASS must be backed by evidence. A recorded failure (fixture, stale-cache,
126
+ // partial, timeout) need not have produced ids or coordinates - its reason says so.
127
+ if (r.label === "pass") {
128
+ if (!r.requestIds.length) problems.push(`flow ${r.flow} is labelled pass but has no request id`)
129
+ if (!r.readCoordinates.length) problems.push(`flow ${r.flow} is labelled pass but has no bounded read coordinate`)
130
+ }
131
+ }
132
+ return problems
133
+ }
134
+
135
+ const YES_NO = (b: boolean) => (b ? "yes" : "NO")
136
+
137
+ /** The committed, human-readable half. Deterministic given the export. */
138
+ export function renderMarkdown(exp: DogfoodExport): string {
139
+ const rows = exp.flows.map((r) => `| ${r.flow} | ${r.label} | ${r.selections.model} | ${r.selections.agent} | ${r.requestIds[0] ?? "-"} | ${r.readCoordinates.length} | ${r.ms}ms |`)
140
+ const out: string[] = []
141
+ out.push(`# Real-model dogfood - ${exp.release.ref}`)
142
+ out.push("")
143
+ out.push(`Generated ${exp.generatedAt} by \`scripts/real-model-dogfood.ts\`. This is release evidence, not a claim of broad coverage: read the label and the reason for every flow.`)
144
+ out.push("")
145
+ out.push(`**Release:** ${exp.release.ref} (\`${exp.release.sha}\`) - version ${exp.release.version}`)
146
+ out.push(`**Serving engine:** pid ${exp.serving.pid} at \`${exp.serving.location}\`${exp.serving.endpoint ? ` (${exp.serving.endpoint})` : ""}`)
147
+ out.push(`**Selections:** model \`${exp.selections.model}\`, agent \`${exp.selections.agent}\``)
148
+ out.push(`**Labels:** ${LABELS.map((l) => `${l} ${exp.labels[l] ?? 0}`).join(" · ")}`)
149
+ out.push("")
150
+ out.push("| flow | label | model | agent | first request id | read coords | duration |")
151
+ out.push("| --- | --- | --- | --- | --- | --- | --- |")
152
+ out.push(...rows)
153
+ out.push("")
154
+ out.push("## Artifact hashes (the exact bytes the flows ran against)")
155
+ out.push("")
156
+ for (const [path, hash] of Object.entries(exp.release.artifacts).sort()) out.push(`- \`${path}\` \`${hash}\``)
157
+ out.push("")
158
+ out.push("## Per-flow detail")
159
+ for (const r of exp.flows) {
160
+ out.push("")
161
+ out.push(`### ${r.flow} - ${r.label}`)
162
+ out.push(`- Why this label: ${r.labelReason}`)
163
+ out.push(`- Release SHA: \`${r.release.sha}\`; serving engine pid ${r.serving.pid} at \`${r.serving.location}\`${r.serving.endpoint ? ` (${r.serving.endpoint})` : " (no endpoint recorded)"}`)
164
+ out.push(`- Model \`${r.selections.model}\`; agent \`${r.selections.agent}\`; duration ${r.ms}ms; started ${r.startedAt}`)
165
+ out.push(`- Request ids: ${r.requestIds.length ? r.requestIds.map((id) => `\`${id}\``).join(", ") : "(none recorded)"}`)
166
+ out.push(`- Bounded read coordinates: ${r.readCoordinates.length ? r.readCoordinates.map((c) => `\`${c.path}@${c.offset}+${c.limit}\``).join(", ") : "(none recorded)"}`)
167
+ out.push(`- Raw owner messages: ${r.raw.ownerMessages.length} · forms: ${r.raw.forms.length} · tool calls: ${r.raw.toolCalls.length}`)
168
+ for (const m of r.raw.ownerMessages.slice(0, 3)) out.push(` - owner: ${m.replace(/\s+/g, " ").slice(0, 200)}`)
169
+ for (const f of r.raw.forms.slice(0, 3)) out.push(` - form ${f.id}: ${f.question.replace(/\s+/g, " ").slice(0, 140)}${f.answer ? ` -> ${f.answer.slice(0, 80)}` : " (unanswered)"}`)
170
+ for (const t of r.raw.toolCalls.slice(0, 5)) out.push(` - tool ${t.name}${t.result ? ` -> ${t.result.replace(/\s+/g, " ").slice(0, 120)}` : ""}`)
171
+ }
172
+ out.push("")
173
+ out.push("## Which serving engine each flow ran against")
174
+ out.push("")
175
+ for (const r of exp.flows) out.push(`- ${r.flow}: ran against serving engine pid ${r.serving.pid} at location \`${r.serving.location}\` with endpoint ${r.serving.endpoint ?? "(none recorded)"}`)
176
+ out.push("")
177
+ out.push("## How each label is assigned")
178
+ out.push("")
179
+ out.push("A flow is labelled by what actually happened, never optimistically:")
180
+ out.push("- `pass` - a real model completed the flow AND its raw interaction was preserved.")
181
+ out.push("- `fixture` - the flow ran against the scripted fixture provider, not a real model.")
182
+ out.push("- `stale-cache` - a stored record could not be reused because the release, the artifact hashes or the serving engine changed.")
183
+ out.push("- `partial` - the flow exited non-zero, with its cause recorded, or preserved no raw interaction at all.")
184
+ out.push("- `timeout` - the flow or its provider failed before any verdict was produced.")
185
+ out.push("")
186
+ out.push("A stored record is reused ONLY when it is a verified pass whose release SHA, artifact hashes AND serving engine (pid, location, endpoint) all still match. A record from any other engine is never reused.")
187
+ out.push("")
188
+ out.push("## What this export does NOT claim")
189
+ out.push("- A `pass` means the flow completed against a real model and its raw interaction was preserved; it is not a statement about flows this release did not run.")
190
+ out.push("- `fixture`, `stale-cache`, `partial` and `timeout` are failures to produce a verdict, recorded as such. They are never counted as passes.")
191
+ out.push(`- Raw credentials and full provider payloads are never included (${YES_NO(true)} - the export carries hashes, ids and bounded coordinates only).`)
192
+ return out.join("\n")
193
+ }
194
+
195
+ /** Where both halves live. `docs/research/real-model-dogfood/<version>.md|.json` + `.opencode/dogfood/<sha>/`. */
196
+ export const exportPaths = (version: string, sha: string) => ({
197
+ summary: `docs/research/real-model-dogfood/${version}.md`,
198
+ index: `docs/research/real-model-dogfood/${version}.json`,
199
+ rawDir: `.opencode/dogfood/${sha.slice(0, 12)}`,
200
+ })
201
+
202
+ export type FlowRun = { exit: number; timedOut?: boolean; providerError?: boolean; fixture?: boolean; staleCache?: boolean; ms: number; requestIds: string[]; readCoordinates: ReadCoordinate[]; raw: RawOutcome; note?: string }
203
+
204
+ export type DogfoodDeps = {
205
+ identity: () => Promise<{ release: ReleaseIdentity; serving: ServingIdentity; selections: Selections }>
206
+ runFlow: (flow: FlowId) => Promise<FlowRun>
207
+ store: { read: (flow: FlowId) => Promise<FlowRecord | undefined>; write: (record: FlowRecord) => Promise<void> }
208
+ emit: { summary: (markdown: string) => Promise<void>; index: (exp: DogfoodExport) => Promise<void> }
209
+ /**
210
+ * Applied to EVERY record (reused or fresh) before it enters the export. Used to
211
+ * make the cited read coordinates project-local, so an independent read-only
212
+ * verifier can actually open them. Must not change a verdict.
213
+ */
214
+ relocate?: (record: FlowRecord) => Promise<FlowRecord>
215
+ force?: boolean
216
+ now?: () => number
217
+ }
218
+
219
+ export type DogfoodResult = { export: DogfoodExport; results: Array<{ flow: FlowId; label: FlowLabel; reused: boolean }> }
220
+
221
+ /**
222
+ * Run (or reuse) every flow for the current release and produce the export.
223
+ * A stored record whose cache key matches the current release + artifact hashes
224
+ * is REUSED rather than re-run, so a start rehearsal and the completing check do
225
+ * not pay for five real-model flows twice; `force` re-runs regardless.
226
+ */
227
+ export async function runDogfood(deps: DogfoodDeps): Promise<DogfoodResult> {
228
+ const { release, serving, selections } = await deps.identity()
229
+ const key = cacheKey(release, serving)
230
+ const results: DogfoodResult["results"] = []
231
+ const records: FlowRecord[] = []
232
+ for (const flow of FLOWS) {
233
+ const stored = await deps.store.read(flow)
234
+ const stale = stored ? cacheMismatch(stored, release, serving) : undefined
235
+ // Only a verified PASS from THIS engine is reusable. A recorded failure is
236
+ // never cached as if it were an answer, and a record from another engine (or
237
+ // another release/artifact set) is never reused at all.
238
+ const reusable = stored && !deps.force && stored.label === "pass" && !stale
239
+ if (reusable) {
240
+ const record = deps.relocate ? await deps.relocate(stored) : stored
241
+ records.push(record)
242
+ results.push({ flow, label: record.label, reused: true })
243
+ continue
244
+ }
245
+ const run = await deps.runFlow(flow)
246
+ // If a stale record existed and the fresh attempt produced no verdict, the
247
+ // honest label is `stale-cache` naming the mismatch - not a silent partial.
248
+ const { label, reason } = classify(stale && (run.exit !== 0 || run.timedOut || run.providerError) ? { ...run, staleCache: true, note: stale } : run)
249
+ const record: FlowRecord = {
250
+ flow,
251
+ release,
252
+ serving,
253
+ selections,
254
+ requestIds: run.requestIds,
255
+ readCoordinates: run.readCoordinates,
256
+ label,
257
+ labelReason: reason,
258
+ ms: run.ms,
259
+ startedAt: new Date((deps.now ?? Date.now)()).toISOString(),
260
+ raw: run.raw,
261
+ }
262
+ await deps.store.write(record)
263
+ records.push(deps.relocate ? await deps.relocate(record) : record)
264
+ results.push({ flow, label, reused: false })
265
+ }
266
+ const exp: DogfoodExport = {
267
+ schema: "dogfood-export/v1",
268
+ release,
269
+ serving,
270
+ selections,
271
+ generatedAt: new Date((deps.now ?? Date.now)()).toISOString(),
272
+ flows: records,
273
+ labels: summarize(records),
274
+ }
275
+ await deps.emit.summary(renderMarkdown(exp))
276
+ await deps.emit.index(exp)
277
+ return { export: exp, results }
278
+ }