@cleocode/goal 0.0.0-stage → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +319 -0
- package/LICENSE +21 -0
- package/README.md +141 -2
- package/cli/ocg.ts +149 -0
- package/dist/cli.js +8708 -0
- package/dist/tui.d.ts +2 -0
- package/dist/tui.js +1920 -0
- package/package.json +79 -4
- package/rpc.ts +2 -0
- package/server.ts +3 -0
- package/skills/write-goal/SKILL.md +211 -0
- package/skills/write-goal/assets/goal.template.yaml +38 -0
- package/skills/write-goal/references/examples.md +283 -0
- package/skills/write-goal/references/goal-schema.md +53 -0
- package/skills/write-goal/references/interview.md +36 -0
- package/src/cli/install.ts +125 -0
- package/src/cli/status.ts +116 -0
- package/src/contract/parse.ts +454 -0
- package/src/contract/render.ts +134 -0
- package/src/contract/types.ts +158 -0
- package/src/engine/amendment.ts +51 -0
- package/src/engine/amendments.ts +72 -0
- package/src/engine/cleo-bridge.ts +339 -0
- package/src/engine/cleo-link.ts +150 -0
- package/src/engine/continue-work.ts +131 -0
- package/src/engine/decisions.ts +44 -0
- package/src/engine/dogfood-record.ts +278 -0
- package/src/engine/handoff.ts +102 -0
- package/src/engine/interrupt-policy.ts +45 -0
- package/src/engine/inventory.ts +70 -0
- package/src/engine/launches.ts +82 -0
- package/src/engine/loop-stop.ts +50 -0
- package/src/engine/loop.ts +109 -0
- package/src/engine/next-work.ts +82 -0
- package/src/engine/owner-intent.ts +71 -0
- package/src/engine/owner-reviews.ts +178 -0
- package/src/engine/peer-announce.ts +67 -0
- package/src/engine/peer-transport.ts +78 -0
- package/src/engine/policy.ts +42 -0
- package/src/engine/registry.ts +119 -0
- package/src/engine/similarity.ts +65 -0
- package/src/engine/stale-decisions.ts +68 -0
- package/src/engine/state.ts +185 -0
- package/src/engine/store.ts +263 -0
- package/src/engine/tick-record.ts +129 -0
- package/src/engine/tick.ts +207 -0
- package/src/rpc.ts +182 -0
- package/src/server/app.ts +4320 -0
- package/src/server/help.ts +50 -0
- package/src/server/index.ts +23 -0
- package/src/server/intent.ts +22 -0
- package/src/server/messages.ts +228 -0
- package/src/summary/audit.ts +76 -0
- package/src/summary/build.ts +314 -0
- package/src/tui/dashboard.ts +322 -0
- package/src/tui/decision-form.ts +12 -0
- package/src/tui/format.ts +127 -0
- package/src/tui/index.tsx +718 -0
- package/src/tui/manager.ts +94 -0
- package/src/tui/navigation.ts +11 -0
- package/src/tui/pointer-select.ts +20 -0
- package/src/tui/tracer.ts +51 -0
- package/src/util/git.ts +102 -0
- package/src/util/ids.ts +20 -0
- package/src/util/shell.ts +42 -0
- package/src/verify/artifact.ts +95 -0
- package/src/verify/pipeline.ts +200 -0
- package/tui.tsx +2 -0
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
// T027 — optional first-class CLEO linkage. When the project carries a CLEO
|
|
2
|
+
// workspace, the goal records the linkage THROUGH THE CLI ONLY (never by
|
|
3
|
+
// reading .cleo/*.db); the project-context.json config file may be read as a
|
|
4
|
+
// presence signal and fallback. No CLEO → the goal system behaves identically;
|
|
5
|
+
// there is no hard dependency either way.
|
|
6
|
+
import { existsSync, readFileSync } from "node:fs"
|
|
7
|
+
import { join } from "node:path"
|
|
8
|
+
|
|
9
|
+
export type CleoFacts = { source: "cli" | "config"; projectId?: string; projectTypes: string[]; detectedAt?: string }
|
|
10
|
+
|
|
11
|
+
export type Runner = (command: string, args: string[], timeoutMs?: number) => Promise<{ stdout: string; exit: number }>
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Detect CLEO linkage for a project root. Order: the `cleo` CLI (authoritative,
|
|
15
|
+
* spawn with a hard timeout), then .cleo/project-context.json (presence +
|
|
16
|
+
* config facts), then none. All failures degrade to undefined — never throw.
|
|
17
|
+
*/
|
|
18
|
+
export async function cleoFacts(projectRoot: string, run: Runner, timeoutMs = 2500): Promise<CleoFacts | undefined> {
|
|
19
|
+
const config = join(projectRoot, ".cleo", "project-context.json")
|
|
20
|
+
const hasConfig = existsSync(config)
|
|
21
|
+
if (!hasConfig && !existsSync(join(projectRoot, ".cleo"))) return undefined
|
|
22
|
+
try {
|
|
23
|
+
const { stdout, exit } = await run("cleo", ["doctor", "project-identity"], timeoutMs)
|
|
24
|
+
if (exit === 0 && stdout.trim()) {
|
|
25
|
+
const parsed = JSON.parse(stdout)
|
|
26
|
+
const data = parsed?.data ?? parsed
|
|
27
|
+
const projectId = typeof data?.projectId === "string" ? data.projectId : typeof parsed?.projectId === "string" ? parsed.projectId : undefined
|
|
28
|
+
return { source: "cli", projectId, projectTypes: [] }
|
|
29
|
+
}
|
|
30
|
+
} catch {
|
|
31
|
+
// CLI missing, too slow, or errored — degrade to the config signal
|
|
32
|
+
}
|
|
33
|
+
if (!hasConfig) return undefined
|
|
34
|
+
try {
|
|
35
|
+
const parsed = JSON.parse(readFileSync(config, "utf8"))
|
|
36
|
+
return { source: "config", projectTypes: Array.isArray(parsed?.projectTypes) ? parsed.projectTypes.map(String) : [], ...(parsed?.detectedAt ? { detectedAt: String(parsed.detectedAt) } : {}) }
|
|
37
|
+
} catch {
|
|
38
|
+
return { source: "config", projectTypes: [] }
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** The ledger payload recorded when a goal starts in a CLEO-linked project. */
|
|
43
|
+
export function cleoLinkEvent(facts: CleoFacts | undefined) {
|
|
44
|
+
return facts ? { type: "cleo-linked" as const, via: facts.source, ...(facts.projectId ? { projectId: facts.projectId } : {}), ...(facts.projectTypes.length ? { projectTypes: facts.projectTypes } : {}) } : undefined
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// v0.5.0 — the CLEO health surface. Three states, no more: the CLI is absent,
|
|
48
|
+
// the CLI is present but this project is not initialized, or both are present.
|
|
49
|
+
// OCG never installs or initializes anything by itself: it reports the state and
|
|
50
|
+
// the EXACT command, and runs that command only when the owner asks for it.
|
|
51
|
+
|
|
52
|
+
export type CleoHealthState = "cli-absent" | "uninitialized" | "initialized"
|
|
53
|
+
|
|
54
|
+
/** Local string coercion (cleo-link.ts has no parser helpers). */
|
|
55
|
+
const asText = (v: unknown): string => (typeof v === "string" ? v.trim() : "")
|
|
56
|
+
|
|
57
|
+
export type CleoOffer = {
|
|
58
|
+
kind: "install" | "init"
|
|
59
|
+
/** The exact command OCG would run — only on an explicit owner choice. */
|
|
60
|
+
command: string
|
|
61
|
+
why: string
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export type CleoHealth = {
|
|
65
|
+
state: CleoHealthState
|
|
66
|
+
cli: { present: boolean; version?: string }
|
|
67
|
+
workspace: { present: boolean; path: string }
|
|
68
|
+
projectId?: string
|
|
69
|
+
offer?: CleoOffer
|
|
70
|
+
/** One-line, state-accurate summary (never a claim that CLEO is usable). */
|
|
71
|
+
detail: string
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** The documented global install for the CLEO CLI (verified against @cleocode/cleo). */
|
|
75
|
+
export const CLEO_INSTALL_COMMAND = "npm i -g @cleocode/cleo"
|
|
76
|
+
/** The documented in-project initializer (verified: `cleo init --help`). */
|
|
77
|
+
export const CLEO_INIT_COMMAND = "cleo init"
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Probe CLEO for a project root. `run` is invoked with `cleo --version` first to
|
|
81
|
+
* decide whether the CLI exists at all (a spawn rejection means absent), then with
|
|
82
|
+
* `cleo doctor project-identity` only when the workspace exists. Nothing here
|
|
83
|
+
* mutates the machine, and no failure throws.
|
|
84
|
+
*/
|
|
85
|
+
export async function cleoHealth(projectRoot: string, run: Runner, timeoutMs = 2500): Promise<CleoHealth> {
|
|
86
|
+
const cleoDir = join(projectRoot, ".cleo")
|
|
87
|
+
const workspace = { present: existsSync(cleoDir), path: cleoDir }
|
|
88
|
+
let present = false
|
|
89
|
+
let version: string | undefined
|
|
90
|
+
try {
|
|
91
|
+
const { stdout, exit } = await run("cleo", ["--version"], timeoutMs)
|
|
92
|
+
if (exit === 0) {
|
|
93
|
+
present = true
|
|
94
|
+
try {
|
|
95
|
+
const parsed = JSON.parse(stdout)
|
|
96
|
+
version = asText(parsed?.data?.version ?? parsed?.version) || undefined
|
|
97
|
+
} catch {
|
|
98
|
+
version = stdout.trim().split(/\s+/)[0] || undefined
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
} catch {
|
|
102
|
+
present = false // the spawn itself failed — no CLI on PATH
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
if (!present)
|
|
106
|
+
return {
|
|
107
|
+
state: "cli-absent",
|
|
108
|
+
cli: { present: false },
|
|
109
|
+
workspace,
|
|
110
|
+
offer: { kind: "install", command: CLEO_INSTALL_COMMAND, why: "the cleo CLI is not on PATH, so no task board is reachable from this project" },
|
|
111
|
+
detail: "CLEO CLI not found on PATH — goal tracking works exactly as before, with no CLEO integration",
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
if (!workspace.present)
|
|
115
|
+
return {
|
|
116
|
+
state: "uninitialized",
|
|
117
|
+
cli: { present: true, ...(version ? { version } : {}) },
|
|
118
|
+
workspace,
|
|
119
|
+
offer: { kind: "init", command: CLEO_INIT_COMMAND, why: "this project has no .cleo workspace, so a goal's work has no durable task board yet" },
|
|
120
|
+
detail: "CLEO CLI present, but this project has no .cleo workspace — a goal runs normally and records no task linkage",
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
let projectId: string | undefined
|
|
124
|
+
try {
|
|
125
|
+
const { stdout, exit } = await run("cleo", ["doctor", "project-identity"], timeoutMs)
|
|
126
|
+
if (exit === 0 && stdout.trim()) {
|
|
127
|
+
const parsed = JSON.parse(stdout)
|
|
128
|
+
const data = parsed?.data ?? parsed
|
|
129
|
+
projectId = asText(data?.trackedId ?? data?.projectId ?? parsed?.projectId) || undefined
|
|
130
|
+
}
|
|
131
|
+
} catch {
|
|
132
|
+
// CLI answered --version but not this probe: still initialized (the
|
|
133
|
+
// workspace exists); the id is simply unknown.
|
|
134
|
+
}
|
|
135
|
+
return {
|
|
136
|
+
state: "initialized",
|
|
137
|
+
cli: { present: true, ...(version ? { version } : {}) },
|
|
138
|
+
workspace,
|
|
139
|
+
...(projectId ? { projectId } : {}),
|
|
140
|
+
detail: `CLEO CLI present and this project is initialized${projectId ? ` (project ${projectId})` : ""} — goals can name and complete CLEO tasks`,
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/** Render a health probe as the owner/agent sees it. Pure; safe to unit-test. */
|
|
145
|
+
export function cleoHealthReport(health: CleoHealth): string {
|
|
146
|
+
const version = health.cli.version ? ` (cleo ${health.cli.version})` : ""
|
|
147
|
+
const head = `◎ CLEO — ${health.detail}`
|
|
148
|
+
if (!health.offer) return [head, `Nothing to do: ${health.state === "initialized" ? "goals can link and complete CLEO tasks." : "no CLEO action is offered."}`].join("\n")
|
|
149
|
+
return [head, `On your explicit choice only (OCG never runs this silently): ${health.offer.command}`, `Why: ${health.offer.why}. Run /goal cleo ${health.offer.kind} to do it, or /goal cleo to re-check.`].join("\n")
|
|
150
|
+
}
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
// v0.8.0 — turning a ready CLEO item into the next goal's contract.
|
|
2
|
+
//
|
|
3
|
+
// The owner's policy is "take the next ready CLEO item without asking". The
|
|
4
|
+
// engine therefore has to author a contract it did not interview for. This file
|
|
5
|
+
// does that from facts only: the task's own title, description and acceptance
|
|
6
|
+
// statements. Each acceptance statement becomes one verifier criterion (the same
|
|
7
|
+
// evidence standard the engine already uses for semantic checks); the task is
|
|
8
|
+
// linked so the existing CLEO bridge can close it on evidence and reports every
|
|
9
|
+
// AC it cannot prove. Nothing here invents a passing command.
|
|
10
|
+
|
|
11
|
+
import type { Autonomy } from "../contract/types"
|
|
12
|
+
import type { Runner } from "./cleo-link"
|
|
13
|
+
|
|
14
|
+
export type CleoTask = {
|
|
15
|
+
id: string
|
|
16
|
+
title: string
|
|
17
|
+
description: string
|
|
18
|
+
acceptance: string[]
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
const parse = (stdout: string): any => {
|
|
22
|
+
try { return JSON.parse(stdout) } catch { return undefined }
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* The next ready LEAF item, from CLEO's own CLI. `cleo next` suggests by
|
|
27
|
+
* priority and dependency; `cleo show --full` supplies the type and acceptance
|
|
28
|
+
* text. A parent (epic/saga) is never started by itself — its acceptance is a
|
|
29
|
+
* rollup, not work — so parents are reported as skipped and the next leaf is
|
|
30
|
+
* tried. Any failure is "no ready work", never a guess.
|
|
31
|
+
*/
|
|
32
|
+
export type SkippedParent = { id: string; type: string; title: string }
|
|
33
|
+
export type ReadyCleo = { task?: CleoTask; skipped: SkippedParent[] }
|
|
34
|
+
|
|
35
|
+
export async function readyCleoTask(run: Runner, bin = "cleo"): Promise<ReadyCleo> {
|
|
36
|
+
const next = await run(bin, ["next", "-n", "5"])
|
|
37
|
+
if (next.exit !== 0) return { skipped: [] }
|
|
38
|
+
const suggestions = parse(next.stdout)?.data?.suggestions
|
|
39
|
+
if (!Array.isArray(suggestions)) return { skipped: [] }
|
|
40
|
+
const skipped: SkippedParent[] = []
|
|
41
|
+
for (const suggestion of suggestions) {
|
|
42
|
+
const id = typeof suggestion?.id === "string" ? suggestion.id : undefined
|
|
43
|
+
if (!id) continue
|
|
44
|
+
const show = await run(bin, ["show", id, "--full"])
|
|
45
|
+
if (show.exit !== 0) continue
|
|
46
|
+
const task = parse(show.stdout)?.data?.task
|
|
47
|
+
if (!task || ["done", "completed", "cancelled", "archived"].includes(String(task.status))) continue
|
|
48
|
+
const type = String(task.type ?? suggestion?.type ?? "")
|
|
49
|
+
if (type === "epic" || type === "saga") {
|
|
50
|
+
skipped.push({ id, type, title: String(task.title ?? suggestion?.title ?? id) })
|
|
51
|
+
continue
|
|
52
|
+
}
|
|
53
|
+
return {
|
|
54
|
+
task: {
|
|
55
|
+
id,
|
|
56
|
+
title: String(task.title ?? suggestion?.title ?? id),
|
|
57
|
+
description: String(task.description ?? ""),
|
|
58
|
+
acceptance: Array.isArray(task.acceptance) ? task.acceptance.map(String).filter(Boolean) : [],
|
|
59
|
+
},
|
|
60
|
+
skipped,
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
return { skipped }
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const q = (value: string) => JSON.stringify(value)
|
|
67
|
+
|
|
68
|
+
/** A slug that does not collide with a live, archived or draft goal. */
|
|
69
|
+
export function continuationSlug(taskId: string, taken: ReadonlySet<string>): string {
|
|
70
|
+
const base = `cleo-${taskId.toLowerCase().replace(/[^a-z0-9]+/g, "-")}`
|
|
71
|
+
if (!taken.has(base)) return base
|
|
72
|
+
for (let n = 2; n < 100; n++) if (!taken.has(`${base}-${n}`)) return `${base}-${n}`
|
|
73
|
+
return `${base}-${Date.now()}`
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* The contract for the next goal. Deliberately boring: the linked task, one
|
|
78
|
+
* verifier criterion per acceptance statement (or the title when the task states
|
|
79
|
+
* none), the owner's autonomy policy, and no budget (budgets stay opt-in).
|
|
80
|
+
*/
|
|
81
|
+
export function continuationContractYaml(input: {
|
|
82
|
+
slug: string
|
|
83
|
+
task: CleoTask
|
|
84
|
+
sourceSlug: string
|
|
85
|
+
autonomy: Autonomy
|
|
86
|
+
now?: number
|
|
87
|
+
}): string {
|
|
88
|
+
const { slug, task, sourceSlug, autonomy } = input
|
|
89
|
+
const statements = task.acceptance.length ? task.acceptance : [task.title]
|
|
90
|
+
const criteria = statements.map((statement, i) => [
|
|
91
|
+
` - id: C${i + 1}`,
|
|
92
|
+
` statement: ${q(`${task.id} acceptance ${i + 1}: ${statement}`)}`,
|
|
93
|
+
` essential: true`,
|
|
94
|
+
` check: {kind: verifier, ask: ${q(`Assess whether the work done for CLEO ${task.id} genuinely satisfies this acceptance criterion: ${statement} Quote the evidence from the repository.`)} }`,
|
|
95
|
+
].join("\n"))
|
|
96
|
+
return [
|
|
97
|
+
"schema: goal/v1",
|
|
98
|
+
`id: ${slug}`,
|
|
99
|
+
`title: ${q(task.title)}`,
|
|
100
|
+
"intent:",
|
|
101
|
+
` verbatim: ${q("Take the next ready CLEO item without asking.")}`,
|
|
102
|
+
`outcome: ${q(task.title)}`,
|
|
103
|
+
`why: ${q(`The owner's standing policy: after a goal completes, start the next ready CLEO item (${task.id}) by itself; this goal was started from that policy and the task's own acceptance.`)}`,
|
|
104
|
+
"scope:",
|
|
105
|
+
` in: []`,
|
|
106
|
+
` out: []`,
|
|
107
|
+
"non_goals:",
|
|
108
|
+
` - ${q("Changing anything outside the linked CLEO task's scope")}`,
|
|
109
|
+
` - ${q("Editing this goal's contract or its evidence")}`,
|
|
110
|
+
"criteria:",
|
|
111
|
+
...criteria,
|
|
112
|
+
"cleo:",
|
|
113
|
+
" tasks:",
|
|
114
|
+
` - id: ${task.id}`,
|
|
115
|
+
"plan:",
|
|
116
|
+
` - {id: S1, title: ${q(`Complete ${task.id} and prove its acceptance`)}}`,
|
|
117
|
+
"autonomy:",
|
|
118
|
+
` questions: ${autonomy.questions}`,
|
|
119
|
+
` on_user_message: ${autonomy.on_user_message}`,
|
|
120
|
+
` on_interrupt: ${autonomy.on_interrupt}`,
|
|
121
|
+
` amend: [${autonomy.amend.join(", ")}]`,
|
|
122
|
+
` on_stall: ${autonomy.on_stall}`,
|
|
123
|
+
"verification: {mode: host+verifier, max_rejections: 3}",
|
|
124
|
+
"meta:",
|
|
125
|
+
` created: ${q(new Date(input.now ?? Date.now()).toISOString())}`,
|
|
126
|
+
` author: opencode-goal`,
|
|
127
|
+
` source: continue-work@0.8.0`,
|
|
128
|
+
` from: ${sourceSlug}`,
|
|
129
|
+
"",
|
|
130
|
+
].join("\n")
|
|
131
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import type { EngineMessage } from "../server/messages"
|
|
2
|
+
import { OwnerReviewStore, type ReviewChoice, type ReviewRequest } from "./owner-reviews"
|
|
3
|
+
|
|
4
|
+
export type DecisionKind = "needs_review" | "paused-after-verdict" | "blocked" | "budget_limited" | "amend-proposed" | "supersede-ack" | "complete" | "start-picker" | "owner-operation"
|
|
5
|
+
export type DecisionTarget = { slug: string; runId?: string; sessionID?: string; lock: string; status: string }
|
|
6
|
+
export type DecisionRequest = ReviewRequest & {
|
|
7
|
+
kind: DecisionKind
|
|
8
|
+
message: EngineMessage
|
|
9
|
+
target: DecisionTarget
|
|
10
|
+
/** Draft choices (start-picker) bind each candidate's bytes, too. */
|
|
11
|
+
candidates: Record<string, DecisionTarget>
|
|
12
|
+
extra: Record<string, unknown>
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export function decisionChoices(request: DecisionRequest): ReviewChoice[] {
|
|
16
|
+
return request.message.choices.map((c, i) => ({ value: String(i), label: c.label, description: `${c.act ? "" : "Guidance only; no operation: "}${c.run}` }))
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Should the native form be re-offered when a turn ends?
|
|
21
|
+
*
|
|
22
|
+
* An unanswered decision is not stale, but re-asking it on every turn end is
|
|
23
|
+
* noise (the owner sees the same question forever). The policy is: ask once
|
|
24
|
+
* per process; after that, silence until an explicit `goal_review` (which
|
|
25
|
+
* calls the form directly) or a reload (a new process, so `alreadyAsked` is
|
|
26
|
+
* false again). Dismissed reviews (`interrupted`) and open forms never re-ask.
|
|
27
|
+
*/
|
|
28
|
+
export function shouldReaskDecision(input: {
|
|
29
|
+
alreadyAsked: boolean
|
|
30
|
+
status: string
|
|
31
|
+
stale: boolean
|
|
32
|
+
formOpen: boolean
|
|
33
|
+
}): { ask: boolean; reason?: "already-asked" | "interrupted" | "stale" | "form-open" } {
|
|
34
|
+
if (input.stale) return { ask: false, reason: "stale" }
|
|
35
|
+
if (input.formOpen) return { ask: false, reason: "form-open" }
|
|
36
|
+
if (input.status === "interrupted") return { ask: false, reason: "interrupted" }
|
|
37
|
+
if (input.alreadyAsked) return { ask: false, reason: "already-asked" }
|
|
38
|
+
return { ask: true }
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export class DecisionStore extends OwnerReviewStore<DecisionRequest> {
|
|
42
|
+
constructor(file: string) { super(file, []) }
|
|
43
|
+
protected reviewChoices(request: DecisionRequest) { return decisionChoices(request) }
|
|
44
|
+
}
|
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
// v0.5.1 - the exact-release real-model dogfood record and its readable export.
|
|
2
|
+
//
|
|
3
|
+
// T102: the five existing real-model drivers were five separate programs with no
|
|
4
|
+
// shared record format and no single export, so a release's real-model proof
|
|
5
|
+
// could not be read or re-checked independently. This module is the ONE record
|
|
6
|
+
// shape and the ONE export both halves are built from.
|
|
7
|
+
//
|
|
8
|
+
// Honesty rule: a flow is labelled by what actually happened - `pass`, `fixture`,
|
|
9
|
+
// `stale-cache`, `partial` or `timeout` - and never presented as a pass when it
|
|
10
|
+
// was not one. `runDogfood` asserts the EXPORT is complete and honest (every flow
|
|
11
|
+
// recorded, every required field present), never that every flow passed.
|
|
12
|
+
|
|
13
|
+
import { createHash } from "node:crypto"
|
|
14
|
+
|
|
15
|
+
export type FlowId = "interview" | "manager" | "lifecycle" | "completion" | "amendment"
|
|
16
|
+
export const FLOWS: readonly FlowId[] = ["interview", "manager", "lifecycle", "completion", "amendment"]
|
|
17
|
+
|
|
18
|
+
export type FlowLabel = "pass" | "fixture" | "stale-cache" | "partial" | "timeout"
|
|
19
|
+
export const LABELS: readonly FlowLabel[] = ["pass", "fixture", "stale-cache", "partial", "timeout"]
|
|
20
|
+
|
|
21
|
+
/** The exact release the flows were exercised against, by hash not by label. */
|
|
22
|
+
export type ReleaseIdentity = { version: string; ref: string; sha: string; artifacts: Record<string, string> }
|
|
23
|
+
/** The serving engine that actually ran the flows. */
|
|
24
|
+
export type ServingIdentity = { pid: number; location: string; endpoint?: string }
|
|
25
|
+
/** The selections the flows used - never inferred from a config file. */
|
|
26
|
+
export type Selections = { model: string; agent: string }
|
|
27
|
+
/** A bounded read coordinate: page + window, never an unbounded dump. */
|
|
28
|
+
export type ReadCoordinate = { path: string; offset: number; limit: number }
|
|
29
|
+
|
|
30
|
+
/** The raw owner interaction, preserved (bounded) alongside the verdict. */
|
|
31
|
+
export type RawOutcome = {
|
|
32
|
+
ownerMessages: string[]
|
|
33
|
+
forms: Array<{ id: string; question: string; answer?: string }>
|
|
34
|
+
toolCalls: Array<{ name: string; result?: string }>
|
|
35
|
+
results: string[]
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export type FlowRecord = {
|
|
39
|
+
flow: FlowId
|
|
40
|
+
release: ReleaseIdentity
|
|
41
|
+
serving: ServingIdentity
|
|
42
|
+
selections: Selections
|
|
43
|
+
requestIds: string[]
|
|
44
|
+
readCoordinates: ReadCoordinate[]
|
|
45
|
+
label: FlowLabel
|
|
46
|
+
labelReason: string
|
|
47
|
+
ms: number
|
|
48
|
+
startedAt: string
|
|
49
|
+
raw: RawOutcome
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export type DogfoodExport = {
|
|
53
|
+
schema: "dogfood-export/v1"
|
|
54
|
+
release: ReleaseIdentity
|
|
55
|
+
serving: ServingIdentity
|
|
56
|
+
selections: Selections
|
|
57
|
+
generatedAt: string
|
|
58
|
+
flows: FlowRecord[]
|
|
59
|
+
labels: Record<FlowLabel, number>
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* The cache key that decides whether a stored per-flow record may be reused.
|
|
64
|
+
*
|
|
65
|
+
* It binds the SERVING ENGINE as well as the release: a record produced by a
|
|
66
|
+
* different pid, location or endpoint is NOT reusable, because the drivers
|
|
67
|
+
* themselves refuse to compare a live engine against a prior recorded elsewhere
|
|
68
|
+
* (the manager flow failed exactly that way with "QA serving PID changed").
|
|
69
|
+
*/
|
|
70
|
+
export const cacheKey = (release: ReleaseIdentity, serving: ServingIdentity): string =>
|
|
71
|
+
createHash("sha256").update(`${release.sha}\n${JSON.stringify(release.artifacts)}\n${serving.pid}\n${serving.location}\n${serving.endpoint ?? ""}`).digest("hex")
|
|
72
|
+
|
|
73
|
+
/** Why a stored record cannot be reused, or undefined when it matches exactly. */
|
|
74
|
+
export function cacheMismatch(stored: FlowRecord, release: ReleaseIdentity, serving: ServingIdentity): string | undefined {
|
|
75
|
+
if (stored.release.sha !== release.sha) return `the stored record was for release ${stored.release.sha}, not ${release.sha}`
|
|
76
|
+
if (JSON.stringify(stored.release.artifacts) !== JSON.stringify(release.artifacts)) return "the artifact hashes changed since the stored record"
|
|
77
|
+
if (stored.serving.pid !== serving.pid) return `the stored record was produced by serving pid ${stored.serving.pid}; this engine is ${serving.pid}`
|
|
78
|
+
if (stored.serving.location !== serving.location) return `the stored record was produced at ${stored.serving.location}; this engine serves ${serving.location}`
|
|
79
|
+
if ((stored.serving.endpoint ?? "") !== (serving.endpoint ?? "")) return `the stored record was produced at endpoint ${stored.serving.endpoint ?? "(none)"}; this engine is ${serving.endpoint ?? "(none)"}`
|
|
80
|
+
return undefined
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** Classify ONE flow from what actually happened. Order matters: the honest worst case wins. */
|
|
84
|
+
export function classify(input: { exit: number; timedOut?: boolean; providerError?: boolean; fixture?: boolean; staleCache?: boolean; raw: RawOutcome; note?: string }): { label: FlowLabel; reason: string } {
|
|
85
|
+
// A captured cause is appended verbatim so a reader never has to guess what happened.
|
|
86
|
+
const suffix = input.note ? ` (${input.note.replace(/\s+/g, " ").slice(0, 200)})` : ""
|
|
87
|
+
if (input.timedOut) return { label: "timeout", reason: `the flow did not finish inside its bound; no verdict was produced${suffix}` }
|
|
88
|
+
if (input.providerError) return { label: "timeout", reason: `the provider failed before the flow produced a verdict${suffix}` }
|
|
89
|
+
// A stale record from another engine (or release) is the ROOT cause when it
|
|
90
|
+
// exists, so it is named BEFORE the generic incomplete-flow label.
|
|
91
|
+
if (input.staleCache) return { label: "stale-cache", reason: `a stored record could not be reused because the release, the artifact hashes or the serving engine changed${suffix}` }
|
|
92
|
+
if (input.exit !== 0) return { label: "partial", reason: `the flow did not complete (exit ${input.exit}) after producing some evidence${suffix}` }
|
|
93
|
+
if (input.fixture) return { label: "fixture", reason: `the flow ran against the scripted fixture provider, not a real model${suffix}` }
|
|
94
|
+
// A pass needs the raw interaction preserved, but not every KIND of it: a flow
|
|
95
|
+
// that legitimately asked no form has no forms to preserve. Only preserving
|
|
96
|
+
// NOTHING at all means the claim cannot be checked.
|
|
97
|
+
const total = input.raw.ownerMessages.length + input.raw.forms.length + input.raw.toolCalls.length + input.raw.results.length
|
|
98
|
+
if (!total) return { label: "partial", reason: `the flow exited 0 but preserved no raw interaction${suffix}` }
|
|
99
|
+
return { label: "pass", reason: `a real model completed the flow and its raw interaction was preserved (${input.raw.ownerMessages.length} owner messages, ${input.raw.forms.length} forms, ${input.raw.toolCalls.length} tool calls)${suffix}` }
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
export const summarize = (records: FlowRecord[]): Record<FlowLabel, number> => {
|
|
103
|
+
const out = Object.fromEntries(LABELS.map((l) => [l, 0])) as Record<FlowLabel, number>
|
|
104
|
+
for (const r of records) out[r.label] += 1
|
|
105
|
+
return out
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/** Every field C3 requires of every flow record. Returns the missing ones. */
|
|
109
|
+
export function validateExport(exp: DogfoodExport): string[] {
|
|
110
|
+
const problems: string[] = []
|
|
111
|
+
if (exp.schema !== "dogfood-export/v1") problems.push(`schema is ${String(exp.schema)}`)
|
|
112
|
+
if (!exp.release.sha) problems.push("release.sha is empty")
|
|
113
|
+
if (!exp.release.version) problems.push("release.version is empty")
|
|
114
|
+
if (!Object.keys(exp.release.artifacts).length) problems.push("release.artifacts is empty")
|
|
115
|
+
if (!exp.serving.pid) problems.push("serving.pid is empty")
|
|
116
|
+
if (!exp.serving.location) problems.push("serving.location is empty")
|
|
117
|
+
if (!exp.selections.model) problems.push("selections.model is empty")
|
|
118
|
+
if (!exp.selections.agent) problems.push("selections.agent is empty")
|
|
119
|
+
const missing = FLOWS.filter((f) => !exp.flows.some((r) => r.flow === f))
|
|
120
|
+
for (const f of missing) problems.push(`flow ${f} has no record`)
|
|
121
|
+
for (const r of exp.flows) {
|
|
122
|
+
if (!r.labelReason) problems.push(`flow ${r.flow} has no label reason`)
|
|
123
|
+
if (!LABELS.includes(r.label)) problems.push(`flow ${r.flow} has an unknown label ${String(r.label)}`)
|
|
124
|
+
if (r.release.sha !== exp.release.sha) problems.push(`flow ${r.flow} was recorded against ${r.release.sha}, not ${exp.release.sha}`)
|
|
125
|
+
// A PASS must be backed by evidence. A recorded failure (fixture, stale-cache,
|
|
126
|
+
// partial, timeout) need not have produced ids or coordinates - its reason says so.
|
|
127
|
+
if (r.label === "pass") {
|
|
128
|
+
if (!r.requestIds.length) problems.push(`flow ${r.flow} is labelled pass but has no request id`)
|
|
129
|
+
if (!r.readCoordinates.length) problems.push(`flow ${r.flow} is labelled pass but has no bounded read coordinate`)
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
return problems
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
const YES_NO = (b: boolean) => (b ? "yes" : "NO")
|
|
136
|
+
|
|
137
|
+
/** The committed, human-readable half. Deterministic given the export. */
|
|
138
|
+
export function renderMarkdown(exp: DogfoodExport): string {
|
|
139
|
+
const rows = exp.flows.map((r) => `| ${r.flow} | ${r.label} | ${r.selections.model} | ${r.selections.agent} | ${r.requestIds[0] ?? "-"} | ${r.readCoordinates.length} | ${r.ms}ms |`)
|
|
140
|
+
const out: string[] = []
|
|
141
|
+
out.push(`# Real-model dogfood - ${exp.release.ref}`)
|
|
142
|
+
out.push("")
|
|
143
|
+
out.push(`Generated ${exp.generatedAt} by \`scripts/real-model-dogfood.ts\`. This is release evidence, not a claim of broad coverage: read the label and the reason for every flow.`)
|
|
144
|
+
out.push("")
|
|
145
|
+
out.push(`**Release:** ${exp.release.ref} (\`${exp.release.sha}\`) - version ${exp.release.version}`)
|
|
146
|
+
out.push(`**Serving engine:** pid ${exp.serving.pid} at \`${exp.serving.location}\`${exp.serving.endpoint ? ` (${exp.serving.endpoint})` : ""}`)
|
|
147
|
+
out.push(`**Selections:** model \`${exp.selections.model}\`, agent \`${exp.selections.agent}\``)
|
|
148
|
+
out.push(`**Labels:** ${LABELS.map((l) => `${l} ${exp.labels[l] ?? 0}`).join(" · ")}`)
|
|
149
|
+
out.push("")
|
|
150
|
+
out.push("| flow | label | model | agent | first request id | read coords | duration |")
|
|
151
|
+
out.push("| --- | --- | --- | --- | --- | --- | --- |")
|
|
152
|
+
out.push(...rows)
|
|
153
|
+
out.push("")
|
|
154
|
+
out.push("## Artifact hashes (the exact bytes the flows ran against)")
|
|
155
|
+
out.push("")
|
|
156
|
+
for (const [path, hash] of Object.entries(exp.release.artifacts).sort()) out.push(`- \`${path}\` \`${hash}\``)
|
|
157
|
+
out.push("")
|
|
158
|
+
out.push("## Per-flow detail")
|
|
159
|
+
for (const r of exp.flows) {
|
|
160
|
+
out.push("")
|
|
161
|
+
out.push(`### ${r.flow} - ${r.label}`)
|
|
162
|
+
out.push(`- Why this label: ${r.labelReason}`)
|
|
163
|
+
out.push(`- Release SHA: \`${r.release.sha}\`; serving engine pid ${r.serving.pid} at \`${r.serving.location}\`${r.serving.endpoint ? ` (${r.serving.endpoint})` : " (no endpoint recorded)"}`)
|
|
164
|
+
out.push(`- Model \`${r.selections.model}\`; agent \`${r.selections.agent}\`; duration ${r.ms}ms; started ${r.startedAt}`)
|
|
165
|
+
out.push(`- Request ids: ${r.requestIds.length ? r.requestIds.map((id) => `\`${id}\``).join(", ") : "(none recorded)"}`)
|
|
166
|
+
out.push(`- Bounded read coordinates: ${r.readCoordinates.length ? r.readCoordinates.map((c) => `\`${c.path}@${c.offset}+${c.limit}\``).join(", ") : "(none recorded)"}`)
|
|
167
|
+
out.push(`- Raw owner messages: ${r.raw.ownerMessages.length} · forms: ${r.raw.forms.length} · tool calls: ${r.raw.toolCalls.length}`)
|
|
168
|
+
for (const m of r.raw.ownerMessages.slice(0, 3)) out.push(` - owner: ${m.replace(/\s+/g, " ").slice(0, 200)}`)
|
|
169
|
+
for (const f of r.raw.forms.slice(0, 3)) out.push(` - form ${f.id}: ${f.question.replace(/\s+/g, " ").slice(0, 140)}${f.answer ? ` -> ${f.answer.slice(0, 80)}` : " (unanswered)"}`)
|
|
170
|
+
for (const t of r.raw.toolCalls.slice(0, 5)) out.push(` - tool ${t.name}${t.result ? ` -> ${t.result.replace(/\s+/g, " ").slice(0, 120)}` : ""}`)
|
|
171
|
+
}
|
|
172
|
+
out.push("")
|
|
173
|
+
out.push("## Which serving engine each flow ran against")
|
|
174
|
+
out.push("")
|
|
175
|
+
for (const r of exp.flows) out.push(`- ${r.flow}: ran against serving engine pid ${r.serving.pid} at location \`${r.serving.location}\` with endpoint ${r.serving.endpoint ?? "(none recorded)"}`)
|
|
176
|
+
out.push("")
|
|
177
|
+
out.push("## How each label is assigned")
|
|
178
|
+
out.push("")
|
|
179
|
+
out.push("A flow is labelled by what actually happened, never optimistically:")
|
|
180
|
+
out.push("- `pass` - a real model completed the flow AND its raw interaction was preserved.")
|
|
181
|
+
out.push("- `fixture` - the flow ran against the scripted fixture provider, not a real model.")
|
|
182
|
+
out.push("- `stale-cache` - a stored record could not be reused because the release, the artifact hashes or the serving engine changed.")
|
|
183
|
+
out.push("- `partial` - the flow exited non-zero, with its cause recorded, or preserved no raw interaction at all.")
|
|
184
|
+
out.push("- `timeout` - the flow or its provider failed before any verdict was produced.")
|
|
185
|
+
out.push("")
|
|
186
|
+
out.push("A stored record is reused ONLY when it is a verified pass whose release SHA, artifact hashes AND serving engine (pid, location, endpoint) all still match. A record from any other engine is never reused.")
|
|
187
|
+
out.push("")
|
|
188
|
+
out.push("## What this export does NOT claim")
|
|
189
|
+
out.push("- A `pass` means the flow completed against a real model and its raw interaction was preserved; it is not a statement about flows this release did not run.")
|
|
190
|
+
out.push("- `fixture`, `stale-cache`, `partial` and `timeout` are failures to produce a verdict, recorded as such. They are never counted as passes.")
|
|
191
|
+
out.push(`- Raw credentials and full provider payloads are never included (${YES_NO(true)} - the export carries hashes, ids and bounded coordinates only).`)
|
|
192
|
+
return out.join("\n")
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/** Where both halves live. `docs/research/real-model-dogfood/<version>.md|.json` + `.opencode/dogfood/<sha>/`. */
|
|
196
|
+
export const exportPaths = (version: string, sha: string) => ({
|
|
197
|
+
summary: `docs/research/real-model-dogfood/${version}.md`,
|
|
198
|
+
index: `docs/research/real-model-dogfood/${version}.json`,
|
|
199
|
+
rawDir: `.opencode/dogfood/${sha.slice(0, 12)}`,
|
|
200
|
+
})
|
|
201
|
+
|
|
202
|
+
export type FlowRun = { exit: number; timedOut?: boolean; providerError?: boolean; fixture?: boolean; staleCache?: boolean; ms: number; requestIds: string[]; readCoordinates: ReadCoordinate[]; raw: RawOutcome; note?: string }
|
|
203
|
+
|
|
204
|
+
export type DogfoodDeps = {
|
|
205
|
+
identity: () => Promise<{ release: ReleaseIdentity; serving: ServingIdentity; selections: Selections }>
|
|
206
|
+
runFlow: (flow: FlowId) => Promise<FlowRun>
|
|
207
|
+
store: { read: (flow: FlowId) => Promise<FlowRecord | undefined>; write: (record: FlowRecord) => Promise<void> }
|
|
208
|
+
emit: { summary: (markdown: string) => Promise<void>; index: (exp: DogfoodExport) => Promise<void> }
|
|
209
|
+
/**
|
|
210
|
+
* Applied to EVERY record (reused or fresh) before it enters the export. Used to
|
|
211
|
+
* make the cited read coordinates project-local, so an independent read-only
|
|
212
|
+
* verifier can actually open them. Must not change a verdict.
|
|
213
|
+
*/
|
|
214
|
+
relocate?: (record: FlowRecord) => Promise<FlowRecord>
|
|
215
|
+
force?: boolean
|
|
216
|
+
now?: () => number
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
export type DogfoodResult = { export: DogfoodExport; results: Array<{ flow: FlowId; label: FlowLabel; reused: boolean }> }
|
|
220
|
+
|
|
221
|
+
/**
|
|
222
|
+
* Run (or reuse) every flow for the current release and produce the export.
|
|
223
|
+
* A stored record whose cache key matches the current release + artifact hashes
|
|
224
|
+
* is REUSED rather than re-run, so a start rehearsal and the completing check do
|
|
225
|
+
* not pay for five real-model flows twice; `force` re-runs regardless.
|
|
226
|
+
*/
|
|
227
|
+
export async function runDogfood(deps: DogfoodDeps): Promise<DogfoodResult> {
|
|
228
|
+
const { release, serving, selections } = await deps.identity()
|
|
229
|
+
const key = cacheKey(release, serving)
|
|
230
|
+
const results: DogfoodResult["results"] = []
|
|
231
|
+
const records: FlowRecord[] = []
|
|
232
|
+
for (const flow of FLOWS) {
|
|
233
|
+
const stored = await deps.store.read(flow)
|
|
234
|
+
const stale = stored ? cacheMismatch(stored, release, serving) : undefined
|
|
235
|
+
// Only a verified PASS from THIS engine is reusable. A recorded failure is
|
|
236
|
+
// never cached as if it were an answer, and a record from another engine (or
|
|
237
|
+
// another release/artifact set) is never reused at all.
|
|
238
|
+
const reusable = stored && !deps.force && stored.label === "pass" && !stale
|
|
239
|
+
if (reusable) {
|
|
240
|
+
const record = deps.relocate ? await deps.relocate(stored) : stored
|
|
241
|
+
records.push(record)
|
|
242
|
+
results.push({ flow, label: record.label, reused: true })
|
|
243
|
+
continue
|
|
244
|
+
}
|
|
245
|
+
const run = await deps.runFlow(flow)
|
|
246
|
+
// If a stale record existed and the fresh attempt produced no verdict, the
|
|
247
|
+
// honest label is `stale-cache` naming the mismatch - not a silent partial.
|
|
248
|
+
const { label, reason } = classify(stale && (run.exit !== 0 || run.timedOut || run.providerError) ? { ...run, staleCache: true, note: stale } : run)
|
|
249
|
+
const record: FlowRecord = {
|
|
250
|
+
flow,
|
|
251
|
+
release,
|
|
252
|
+
serving,
|
|
253
|
+
selections,
|
|
254
|
+
requestIds: run.requestIds,
|
|
255
|
+
readCoordinates: run.readCoordinates,
|
|
256
|
+
label,
|
|
257
|
+
labelReason: reason,
|
|
258
|
+
ms: run.ms,
|
|
259
|
+
startedAt: new Date((deps.now ?? Date.now)()).toISOString(),
|
|
260
|
+
raw: run.raw,
|
|
261
|
+
}
|
|
262
|
+
await deps.store.write(record)
|
|
263
|
+
records.push(deps.relocate ? await deps.relocate(record) : record)
|
|
264
|
+
results.push({ flow, label, reused: false })
|
|
265
|
+
}
|
|
266
|
+
const exp: DogfoodExport = {
|
|
267
|
+
schema: "dogfood-export/v1",
|
|
268
|
+
release,
|
|
269
|
+
serving,
|
|
270
|
+
selections,
|
|
271
|
+
generatedAt: new Date((deps.now ?? Date.now)()).toISOString(),
|
|
272
|
+
flows: records,
|
|
273
|
+
labels: summarize(records),
|
|
274
|
+
}
|
|
275
|
+
await deps.emit.summary(renderMarkdown(exp))
|
|
276
|
+
await deps.emit.index(exp)
|
|
277
|
+
return { export: exp, results }
|
|
278
|
+
}
|