fdeops 3.9.12 → 3.9.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/bin/check.js CHANGED
@@ -63,6 +63,7 @@ const requiredReferences = [
63
63
  'debug.md', 'rescue.md', 'ship.md', 'sketch.md', 'close.md', 'dashboard.md',
64
64
  'debrief.md', 'status.md', 'demo-prep.md',
65
65
  'healthcare.md', 'fintech.md', 'gov.md',
66
+ 'ai.md', 'eval-pack.md',
66
67
  ]
67
68
  for (const f of requiredReferences) {
68
69
  const p = path.join(root, 'skills', 'fde', 'references', f)
package/bin/fde.js CHANGED
@@ -843,8 +843,11 @@ function cmdResume(args) {
843
843
  const fdeDir = path.join(engRoot, '.fde')
844
844
  const existed = fs.existsSync(fdeDir)
845
845
 
846
+ // Optional stubs (AI eval pack, …) stay in templates/ for copy-on-use — not day-1 scaffold.
847
+ const SKIP_INIT_TEMPLATES = new Set(['evals.md'])
846
848
  const fillTemplates = (destFde) => {
847
849
  for (const f of fs.readdirSync(tpl)) {
850
+ if (SKIP_INIT_TEMPLATES.has(f)) continue
848
851
  const src = path.join(tpl, f); const dst = path.join(destFde, f)
849
852
  if (fs.statSync(src).isDirectory()) fs.mkdirSync(dst, { recursive: true })
850
853
  else if (!fs.existsSync(dst)) fs.copyFileSync(src, dst)
@@ -1501,6 +1504,18 @@ function collectDoctorIssues(eng) {
1501
1504
  `phase is ${s.phase} with ${s.openRisks} open risk(s) - retire, hand off, or move still-live ones before calling the embed done`
1502
1505
  )
1503
1506
  }
1507
+ if (s.phase === 'close' || s.phase === 'ship') {
1508
+ if (!hasValueBucket(eng)) {
1509
+ issues.push(
1510
+ `phase is ${s.phase} with no value bucket (cost-save | risk-mitigation | revenue-uplift) in success.md or delivery value ledger`
1511
+ )
1512
+ }
1513
+ if (engagementTouchesAI(eng) && !hasEvalReceipt(eng)) {
1514
+ issues.push(
1515
+ `phase is ${s.phase} with AI in scope but no eval receipt (evals.md Verdict or delivery Eval / Ship receipts) — required before green ship/close`
1516
+ )
1517
+ }
1518
+ }
1504
1519
  const dupes = findDuplicateOpenRisks(eng)
1505
1520
  if (dupes.length) {
1506
1521
  const sample = (dupes[0][0] || '').replace(/\s+/g, ' ').trim().slice(0, 60)
@@ -1511,6 +1526,71 @@ function collectDoctorIssues(eng) {
1511
1526
  return issues
1512
1527
  }
1513
1528
 
1529
+ // Strip template comments / italic *(hints)* so doctor does not treat stubs as filled.
1530
+ function stripTemplateNoise(md) {
1531
+ return String(md || '')
1532
+ .replace(/<!--[\s\S]*?-->/g, '')
1533
+ .replace(/\*\([^)]*\)\*/g, '')
1534
+ }
1535
+
1536
+ const VALUE_BUCKET_RE = /(cost[- ]?save|risk[- ]?mitigat|revenue[- ]?uplift)/i
1537
+
1538
+ function hasValueBucket(eng) {
1539
+ const success = stripTemplateNoise(readClean(eng, 'success.md'))
1540
+ const bucketLine = success.match(/\*\*Primary value bucket:\*\*\s*(.+)/i)
1541
+ if (bucketLine && VALUE_BUCKET_RE.test(bucketLine[1].trim())) return true
1542
+ if (!/\*\*Primary value bucket:\*\*/i.test(success) && VALUE_BUCKET_RE.test(success)) return true
1543
+
1544
+ const ledger = stripTemplateNoise(sectionBody(readClean(eng, 'delivery.md'), 'Value ledger') || '')
1545
+ const table = parseMdTable(ledger)
1546
+ if (table) {
1547
+ const bIdx = colIndex(table.headers, /bucket/i)
1548
+ if (bIdx !== -1) {
1549
+ for (const row of table.rows) {
1550
+ const cell = String(row[bIdx] || '').trim()
1551
+ if (cell && VALUE_BUCKET_RE.test(cell)) return true
1552
+ }
1553
+ } else if (VALUE_BUCKET_RE.test(ledger)) {
1554
+ return true
1555
+ }
1556
+ } else if (VALUE_BUCKET_RE.test(ledger)) {
1557
+ return true
1558
+ }
1559
+ return false
1560
+ }
1561
+
1562
+ // AI in scope for ship/close hygiene — delivery/decisions/trust evidence only.
1563
+ // Do not scan terrain.md: its template headers mention LLM and would false-positive every ship.
1564
+ function engagementTouchesAI(eng) {
1565
+ const trust = readClean(eng, 'trust-profile.md')
1566
+ const aiSec = stripTemplateNoise(sectionBody(trust, 'AI policy') || '')
1567
+ if (aiSec.trim().length > 20) return true
1568
+ const blob = stripTemplateNoise([
1569
+ readClean(eng, 'delivery.md'),
1570
+ readClean(eng, 'decisions.md'),
1571
+ ].join('\n'))
1572
+ return /\b(llm|rag|embedding|inference|model card|agentic|openai|anthropic|vector database|vector db)\b/i.test(blob)
1573
+ }
1574
+
1575
+ function hasEvalReceipt(eng) {
1576
+ const evalsPath = path.join(eng, 'evals.md')
1577
+ if (fs.existsSync(evalsPath)) {
1578
+ const e = stripTemplateNoise(readClean(eng, 'evals.md'))
1579
+ // Empty G1 stub + "Pass / fail" heading is not a receipt — need a real verdict/run/result.
1580
+ if (/\*\*Verdict:\*\*\s*SHIP\b/i.test(e) || /(?:^|\n)\s*-\s*\*\*Verdict:\*\*\s*SHIP\b/i.test(e)) return true
1581
+ if (/\bLast run:\s*\d{4}-\d{2}-\d{2}/i.test(e)) return true
1582
+ if (/\|\s*G\d+\s*\|[^|\n]+\|[^|\n]+\|[^|\n]+\|[^|\n]+\|\s*pass\s*\|/i.test(e)) return true
1583
+ }
1584
+ const del = stripTemplateNoise(readClean(eng, 'delivery.md'))
1585
+ if (/#{1,6}\s+Eval\b/i.test(del) && /\b(pass|SHIP|\d+\/\d+)\b/i.test(sectionBody(del, 'Eval') || del)) return true
1586
+ if (/\beval (pack|receipt)[:\s].*\b(pass|SHIP)\b/i.test(del)) return true
1587
+ const receipts = sectionBody(del, 'Ship receipts') || ''
1588
+ if (/\bevals\.md\b/i.test(receipts) && /\b(pass|SHIP)\b/i.test(receipts) && !/\*\([^)]*evals\.md[^)]*\)\*/i.test(receipts)) {
1589
+ return true
1590
+ }
1591
+ return false
1592
+ }
1593
+
1514
1594
  // Lean line for session-start TRIAGE - count + top issue + NL cue. Omitted when clean.
1515
1595
  function hygieneTriageLines(eng) {
1516
1596
  const issues = collectDoctorIssues(eng)
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "fdeops",
3
- "version": "3.9.12",
3
+ "version": "3.9.13",
4
4
  "description": "Field kit for engineers embedded in client work - a real CLI (recon, memory, portfolio), one @fde skill with field judgment on top, and hooks that make it automatic. Claude Code plugin and any agent that loads skills.",
5
5
  "bin": {
6
6
  "fdeops": "bin/install.js",
@@ -270,6 +270,7 @@ Running the engagement and ending it well.
270
270
  | Signal | Overlay |
271
271
  |--------|---------|
272
272
  | AI, ML, LLM, model, embeddings, RAG, agents, fine-tuning, inference, drift | `references/ai.md` |
273
+ | Golden set, eval suite, eval pack, pass/fail before AI ship, HITL gate for model | `references/eval-pack.md` (+ `ai.md`) |
273
274
  | Deck, slides, report, governance framework, compliance pack, ADR, PDF | `references/artifacts.md` |
274
275
  | Patient data, PHI, HIPAA, EHR, clinical | `references/healthcare.md` |
275
276
  | Payments, cardholder data, PCI-DSS, anything that moves money | `references/fintech.md` |
@@ -36,6 +36,19 @@ Never start with the most powerful model. Start with the cheapest that meets the
36
36
 
37
37
  Write model selection rationale to `decisions.md`. Include: models tested, test set size, scores, cost comparison.
38
38
 
39
+ ## Engagement eval pack (before AI ships)
40
+
41
+ When any slice touches a model, embeddings, RAG, or an agent: create or update `.fde/evals.md` **before** ship. Full method: `references/eval-pack.md`. This is the engagement-local test set — not unit tests.
42
+
43
+ **Minimum pack (do not grow until the minimum exists):**
44
+ 1. **Component + quality bar** — one sentence each; kill switch / fallback named.
45
+ 2. **Golden cases** — 5–20 representative inputs with expected outputs and a pass rule. Prefer real production-shaped data (sanitized).
46
+ 3. **Failure modes** — at least the silent ones: hallucination/ungrounded, retrieval miss (if RAG), drift, cost runaway.
47
+ 4. **Pass/fail** — dated run; Verdict **SHIP** or **NO-SHIP**; critical fails must be 0.
48
+ 5. **HITL gate** — which decisions need human review before action (align with `trust-profile.md`). Empty when policy requires review → NO-SHIP.
49
+
50
+ **When to write:** plan seeds the pack; sketch/build grows goldens; ship requires Verdict SHIP and a receipt in `delivery.md` → `## Ship receipts`. Non-AI work skips this file entirely.
51
+
39
52
  ## RAG architecture (retrieval-augmented generation)
40
53
 
41
54
  When the AI needs to answer questions about the client's data:
@@ -78,12 +91,13 @@ When the AI takes actions (not just generates text):
78
91
 
79
92
  ## Writes
80
93
 
81
- `trust-profile.md` - AI policy, data classification, model hosting, human-in-the-loop requirements. `decisions.md` - model selection rationale, architecture choices. `risks.md` - bias findings, drift observations, cost projections. `delivery.md` - AI component inventory with kill switches documented.
94
+ `trust-profile.md` - AI policy, data classification, model hosting, human-in-the-loop requirements. `evals.md` - golden cases, failure modes, SHIP/NO-SHIP, HITL. `decisions.md` - model selection rationale, architecture choices. `risks.md` - bias findings, drift observations, cost projections. `delivery.md` - AI component inventory with kill switches + eval receipt on ship.
82
95
 
83
96
  ## Principles
84
97
 
85
98
  - AI degrades silently. Monitor outputs, not just uptime.
86
99
  - Start with the cheapest model that meets the quality bar.
100
+ - No golden set, no AI ship (`evals.md` Verdict SHIP).
87
101
  - Every AI component needs a kill switch and a fallback path.
88
102
  - Log reasoning, not just results. Debug AI from its decisions.
89
103
  - Drift is inevitable. Define the detection method before shipping.
@@ -127,7 +127,7 @@ The FDE's job is to make themselves replaceable. Not at handoff - every day. A c
127
127
 
128
128
  - **`decisions.md`** - each significant choice: what, alternatives considered, why this one. For non-trivial architecture decisions, present three options to the FDE (safe / pragmatic / aggressive) with costs and a recommendation - three options is a real decision; one option is a request for trust. Integration contracts go here too.
129
129
  - **`risks.md`** - new risks discovered while building.
130
- - **`delivery.md`** - append a **value ledger** row for every ship: Date | Slice | Promised | Measured | Evidence | Rollback. "Measured" may be `pending` until the pulse exists - never skip the promised column. Narrative under Shipped is optional color; the ledger is the record status and close read.
130
+ - **`delivery.md`** - append a **value ledger** row for every ship: Date | Slice | Bucket | Promised | Measured | Evidence | Rollback. Bucket is `cost-save` / `risk-mitigation` / `revenue-uplift`. "Measured" may be `pending` until the pulse exists - never skip the promised column. Narrative under Shipped is optional color; the ledger is the record status and close read.
131
131
 
132
132
  ## Checkpoint
133
133
 
@@ -17,6 +17,12 @@ The engagement doesn't end at ship. It ends when the customer can maintain what
17
17
  - Which risk almost became real?
18
18
  - AI components: did they behave in production? What failure modes did the prototype hide? Is the team equipped to maintain them?
19
19
 
20
+ **1b. Value + receipts close gate (refuse green close if any fail):**
21
+ - Primary value bucket in `success.md` matches what the sponsor funded; at least one ledger row has **Measured** (not forever-`pending`) with evidence for that bucket — or the retrospective explicitly records “not measured; sponsor accepted pending.”
22
+ - Audit receipt exists for the final shipped path (exceptions/operating map walked; cite file).
23
+ - Eval receipt: **n/a if no AI**, else final golden/eval result + HITL owner recorded; kill switch / fallback named in `handoff.md`.
24
+ - One line in the retrospective: which bucket moved, by how much, vs baseline.
25
+
20
26
  **2. The pattern.** Anything that happened here and will happen again - a compliance approach, a migration pattern, a stakeholder dynamic - gets encoded for reuse. **If you do it twice, encode it.**
21
27
 
22
28
  **3. The handoff.** Operational knowledge for the person woken at 2am, not technical documentation: the 3 things that will break and the fix for each · who holds the tribal knowledge · what each alert means · deploy and rollback in plain language. AI components additionally: model version, what normal output looks like (so drift is recognisable), fallback behaviour, who owns retraining, **how to disable the AI path without taking down the feature** - without this the team turns it off at the first misbehaviour and it stays off.
@@ -33,11 +39,12 @@ The engagement doesn't end at ship. It ends when the customer can maintain what
33
39
 
34
40
  ## Checkpoint
35
41
 
36
- Direct assessment to the FDE: did the engagement achieve `success.md` · 2–3 lessons that matter · is the pattern worth encoding · is the handoff complete or where are the gaps. Honest - a gap named now is cheaper than a callback in six weeks.
42
+ Direct assessment to the FDE: did the engagement achieve `success.md` · 2–3 lessons that matter · is the pattern worth encoding · is the handoff complete or where are the gaps. Also: value bucket + audit receipt green; eval **n/a or green**. Pending Measured without sponsor acceptance = gap, not green close. Honest - a gap named now is cheaper than a callback in six weeks.
37
43
 
38
44
  ## Principles
39
45
 
40
46
  - Done = the customer operates without you.
47
+ - No named value bucket moved (or sponsor-accepted pending) = not a green close.
41
48
  - The retrospective is an investment in the next engagement, not a post-mortem.
42
49
  - Encode what repeated. The same lesson learned twice is a process failure.
43
50
  - Write the handoff for 2am.
@@ -92,6 +92,7 @@ The real spec is what people **do** when the system fails - not what the slide d
92
92
  - **The hesitation.** When someone says "well, there's also this other thing we do…" - stop them, ask them to finish. The main story is what they're comfortable explaining; the hesitation is the real problem.
93
93
  - **"Which part of the codebase do you least want to touch?"** The answer is unanimous and it's the load-bearing wall. Check it against your churn scan - when the human answer and the churn data agree, that's your first map landmark.
94
94
  - **Shadow AI.** Someone pasting data into ChatGPT to cope = a real unmet need + an uncontrolled data risk. Note both.
95
+ - **Exception-led operating map.** For each real break (not the slide-deck process): what fails, who notices first, what they do today, and which artifact is trusted in that moment. Prefer exceptions over happy-path swimlanes — the workaround is the operating system. Write rows under `terrain.md` → `## Operating map (exception-led)`. If the section is missing on an older engagement, add it; never regenerate the rest of terrain. When AI is in play, also fill `## Intelligence placement` (deterministic vs LLM judgement vs human approve).
95
96
 
96
97
  ## Method - part 3: workshop facilitation
97
98
 
@@ -165,6 +166,11 @@ Score every candidate use case before anything gets prototyped:
165
166
  **Data flow:** <entry → transform → store → exit>
166
167
  **Test landscape:** <covered / gaps / lies>
167
168
  **Unknowns:** <named explicitly - an honest gap beats a confident guess>
169
+
170
+ ## Operating map (exception-led)
171
+ | Exception / break | Who notices first | What they do today | System of record then | Blast | Evidence |
172
+ |-------------------|-------------------|--------------------|----------------------|-------|----------|
173
+ | <break> | <role> | <workaround> | <sheet/DB/person> | CRITICAL / LOAD-BEARING / CONVENIENCE | <who/day> |
168
174
  ```
169
175
 
170
176
  Every line carries its evidence. `(churn: 47/90d)` `(ops lead, Day 5)` `(stated, unverified)`.
@@ -173,11 +179,12 @@ Every line carries its evidence. `(churn: 47/90d)` `(ops lead, Day 5)` `(stated,
173
179
 
174
180
  ## Checkpoint (before any build)
175
181
 
176
- Present to the FDE, four things, one paragraph each - no padding:
182
+ Present to the FDE, five things, one paragraph each - no padding:
177
183
  1. The real problem, with the two strongest pieces of evidence.
178
184
  2. The top 3 risk areas of the codebase, one line of why each.
179
185
  3. What must not be touched without characterisation tests.
180
- 4. The recommendation: confirm brief / descope / rescope - and the decision it puts in front of the sponsor.
186
+ 4. The exception-led operating map: the two breaks that matter most, who owns the workaround, and where shadow systems live.
187
+ 5. The recommendation: confirm brief / descope / rescope - and the decision it puts in front of the sponsor.
181
188
 
182
189
  If discovery revealed the problem is 3× the brief: the FDE tells the customer **before** telling themselves it's manageable. Lead with evidence, offer three paths (descope / rescope / pause-and-plan), confirm any reset in writing - update `success.md` and `brief.md` before continuing.
183
190
 
@@ -0,0 +1,43 @@
1
+ # eval-pack - prove the system before it acts
2
+
3
+ **Enter when:** the work touches AI/LLM/agents/RAG, or ship/close is blocked because there is no evidence the non-deterministic path is safe. Activate alongside `ai.md`, `sketch`, `build`, or `ship` — not instead of them.
4
+
5
+ **Read first:** `trust-profile.md` (AI policy + HITL), `terrain.md` (operating map), `delivery.md`. Create or extend `evals.md`.
6
+
7
+ Non-AI engagements skip this pack entirely.
8
+
9
+ ## Why this exists
10
+
11
+ Intelligence without evidence is token-maxing with a nicer name. An FDE earns trust by showing: golden cases, failure modes, and a human gate before action.
12
+
13
+ ## Method (you do this work)
14
+
15
+ **1. Scope the judgement surface.** One sentence: which step uses model judgement, and what must never be autonomous.
16
+
17
+ **2. Build a golden set (minimum 5–20 for a slice; prefer 50–100 before broad scale).** For each case:
18
+ - input (sanitized — no `<private>` raw values)
19
+ - expected outcome or expert-approved acceptance note
20
+ - pass rule (exact / contains / short rubric)
21
+ - source: real historical example / expert label / staged fixture
22
+
23
+ **3. Score pass/fail, not vibes.** Run the suite. Record count pass / fail. Failures get a failure-mode tag (missing data, wrong record, format drift, hallucination, retrieval miss, unsafe action, other).
24
+
25
+ **4. Human-in-the-loop gate.** Name which outcomes require human approve before side effects. If none, write why that is allowed under `trust-profile.md` AI policy — do not invent permission.
26
+
27
+ **5. Ship rule.** Until `evals.md` shows Verdict **SHIP** with a dated run (critical fails = 0) and HITL filled when policy requires it, AI-touching ship stays **fix-first**. Log a one-line eval receipt in `delivery.md` → `## Ship receipts`.
28
+
29
+ ## Artifact — `evals.md`
30
+
31
+ Create on first AI-touching slice (not at `resume --init`). Use the stub in `templates/.fde/evals.md`. Every claim needs a source. Missing evidence → leave the cell `unknown - ask:`, never invent scores.
32
+
33
+ ## Checkpoint
34
+
35
+ Present to the FDE: suite size, pass rate, top failure mode, HITL gate, Verdict SHIP/NO-SHIP. If they want to ship without a run: say no, and offer the smallest suite that would unblock.
36
+
37
+ ## Principles
38
+
39
+ - No golden set, no AI ship.
40
+ - Pass/fail beats “looks good.”
41
+ - Failure modes are the product — the happy path is table stakes.
42
+ - HITL is a gate, not a slide.
43
+ - Non-AI work does not need this file.
@@ -65,6 +65,7 @@ Let silence sit. If their fear doesn't match the written brief, the brief is wro
65
65
  - **The previous attempt** - "we tried something similar last year" is the most important sentence in the first meeting. Who was involved? Still there and protective, or gone because of it?
66
66
  - **The passed-over internal team** - they know exactly what's wrong, and they resent the FDE's presence. Find them before the first standup, ask what they tried, use their language in every meeting. Make them look right and they protect you; ignore them and they wait for the mistake.
67
67
  - **The sacred thing** - "Is there anything in this environment I should treat as untouchable?" The hesitation before the answer is the answer.
68
+ - **Exception path (operating map seed)** - "When the happy path breaks this week, what do people actually do — who do they call, what spreadsheet opens, what do they skip?" Capture the break → workaround → who owns it. Do not build a full map on day 1; seed rows later in `terrain.md` → `## Operating map (exception-led)` during discover. Unknowns stay `unknown - ask:`.
68
69
  - **AI posture and policy** - tools already in use (sanctioned or shadow), and: "Does your organisation have a policy on AI-generated code? Are there decisions where you would not be comfortable with AI involvement?"
69
70
  - **Boundaries in multi-vendor rooms** - who owns what surface, who signs off before a change crosses it.
70
71
 
@@ -76,7 +77,7 @@ Before the end of day 1, ship one visible thing: a small bug fix, a cleanup the
76
77
 
77
78
  **`brief.md`** - what they said, who sent the FDE, the timeline, **and the gap list**.
78
79
 
79
- **`success.md`** - what done looks like, how it's measured, who actually signs off, what is explicitly out of scope. Agreed with the customer, not assumed.
80
+ **`success.md`** - what done looks like, **primary value bucket** (`cost-save` | `risk-mitigation` | `revenue-uplift`), baseline → target, who actually signs off, what is explicitly out of scope. Agreed with the customer, not assumed.
80
81
 
81
82
  **`stakeholders.md`**:
82
83
  ```markdown
@@ -100,7 +101,7 @@ One falsifiable hypothesis about the real problem also goes at the bottom of `br
100
101
 
101
102
  ## Checkpoint
102
103
 
103
- One page back to the FDE: success + sign-off owner, out-of-scope boundary, sacred data, stakeholder map with veto power, AI posture, the hypothesis, and the top CRITICAL assumptions still OPEN. If it doesn't fit one page, the engagement isn't understood yet.
104
+ One page back to the FDE: success + value bucket + sign-off owner, out-of-scope boundary, sacred data, stakeholder map with veto power, AI posture, the hypothesis, the top CRITICAL assumptions still OPEN, and any exception-path seeds heard (break → workaround → owner) for discover to map into `terrain.md`. If it doesn't fit one page, the engagement isn't understood yet.
104
105
 
105
106
  If remote: trust-building takes ~40% longer - push for a short video call before anything asynchronous.
106
107
 
@@ -2,7 +2,7 @@
2
2
 
3
3
  **Enter when:** a slice is built, reviewed, and ready to deploy.
4
4
 
5
- **Read first:** `context.md`, `delivery.md`. Load `trust-profile.md` if the deploy touches regulated data or needs an approval chain.
5
+ **Read first:** `context.md`, `delivery.md`, `success.md`. Load `trust-profile.md` if the deploy touches regulated data or needs an approval chain. Load `evals.md` when the deploy touches AI/ML/LLM/RAG/agents.
6
6
 
7
7
  Opening question, calm tech lead voice: **has anyone actually *run* the rollback, or is it still a slide?** If only planned, that's today's work - say so plainly.
8
8
 
@@ -42,10 +42,26 @@ Score each dimension green/amber/red. This is the gate, not a suggestion:
42
42
  | Runbook | Exists and someone other than you has read it | Exists but unreviewed | Missing |
43
43
  | Monitoring | Alerts configured, owner named, dashboard live | Alerts configured, no named owner | No monitoring |
44
44
 
45
+ ### Value + receipts gate (score with the table above)
46
+
47
+ | Dimension | Green | Amber | Red |
48
+ |-----------|-------|-------|-----|
49
+ | **Value bucket** | `success.md` names primary bucket (`cost-save` \| `risk-mitigation` \| `revenue-uplift`) and a baseline→target metric; this slice’s value-ledger row has **Bucket** + **Promised** | Bucket named; **Measured** still `pending` with a pulse date | No bucket, or Promised empty / ticket-theater only |
50
+ | **Audit receipt** | Dated line in `delivery.md` (`## Ship receipts` or ledger Evidence) proving exceptions/operating path were walked — cite `terrain.md` / `reality.md` / `audit.md` | Path described, not verified this ship | No audit receipt for this slice |
51
+ | **Eval receipt** | **n/a** (no AI on this slice) **or** `evals.md` Verdict SHIP with dated golden run + HITL gate named | Eval pack exists; known fails open with owner + date | AI in scope and no eval receipt |
52
+ | **AI eval pack** | `.fde/evals.md` Verdict SHIP; goldens run this change; critical fails 0; HITL filled if policy requires | Pack exists; run stale vs change log | AI-touching deploy and pack missing / NO-SHIP / HITL required but empty |
53
+
45
54
  **Any RED = stop. Do not deploy. Fix the red dimension first.**
46
55
  **2+ AMBER = sponsor conversation before deploying.** Present the ambers and get explicit "proceed" or "fix first."
47
56
 
48
- Write the readiness score to `delivery.md` before deploying. The score is the evidence if anything goes wrong.
57
+ **AI-touching deploys (model, embeddings, RAG, agent, or inference path):**
58
+ 1. Read `.fde/evals.md`. If missing → **RED. Do not deploy.** Create the pack (`eval-pack` / `ai` overlay) and re-score.
59
+ 2. If Verdict is not **SHIP**, or Last run is older than the latest change-log row → **RED.**
60
+ 3. If `trust-profile.md` requires human-in-the-loop and the HITL gate has no reviewer → **RED.**
61
+ 4. Log in `delivery.md` → `## Ship receipts` before deploy: audit cite + eval receipt.
62
+ 5. Non-AI deploys: Eval = **n/a** — do not invent an empty pack.
63
+
64
+ Write the readiness score (including value + receipts) to `delivery.md` before deploying. The score is the evidence if anything goes wrong.
49
65
 
50
66
  ## Pre-blast challenge (before the deploy button)
51
67
 
@@ -146,13 +162,14 @@ Adoption isn't a handoff-stage problem - it starts during build. Software that l
146
162
 
147
163
  ## Checkpoint
148
164
 
149
- Before 100%: canary clean, business metric verified, pulse written into `delivery.md`. Any item unconfirmedthe deploy waits. For enterprise-scale: scale-readiness gate passed before broad rollout.
165
+ Before 100%: canary clean, business metric verified, pulse written into `delivery.md`. Also green: value bucket named, audit receipt dated, eval receipt **n/a or pass**. Missing any of those not green. For enterprise-scale: scale-readiness gate passed before broad rollout.
150
166
 
151
167
  ## Principles
152
168
 
153
169
  - A deployment without a tested rollback is reckless.
154
170
  - Roll back on any canary anomaly; investigate safely.
155
171
  - Verify the business metric, not just the technical one.
156
- - No pulse, no done.
172
+ - No value bucket, no green ship. No pulse, no done.
173
+ - AI path without eval receipt = fix-first; non-AI ships leave eval as n/a.
157
174
  - Scale readiness is organizational, not just technical. Check all 8 dimensions.
158
175
  - Adoption is measured from day one, not hoped for at launch.
@@ -4,9 +4,17 @@
4
4
 
5
5
  ## Value ledger
6
6
 
7
- | Date | Slice | Promised | Measured | Evidence | Rollback |
8
- |------|-------|----------|----------|----------|----------|
9
- | | | *(what we said it would change)* | *(what actually changed, or pending)* | *(who/when/metric)* | |
7
+ | Date | Slice | Bucket | Promised | Measured | Evidence | Rollback |
8
+ |------|-------|--------|----------|----------|----------|----------|
9
+ | | | *(cost-save / risk-mitigation / revenue-uplift)* | *(what we said it would change)* | *(what actually changed, or pending)* | *(who/when/metric)* | |
10
+
11
+ ## Ship receipts
12
+
13
+ <!-- Fill before green ship. Eval = n/a unless AI touches the slice. -->
14
+
15
+ | Date | Slice | Audit receipt | Eval receipt |
16
+ |------|-------|---------------|--------------|
17
+ | | | *(dated cite: exceptions/operating path verified — terrain/reality/audit)* | *(n/a \| evals.md pass + HITL owner)* |
10
18
 
11
19
  ## Shipped
12
20
 
@@ -0,0 +1,42 @@
1
+ # Engagement eval pack
2
+
3
+ <!-- AI-touching work only. Fill before ship. Empty pack = do not deploy AI path. Non-AI engagements: leave unused or delete. -->
4
+
5
+ ## Component
6
+ - **Name / slice:**
7
+ - **Model / stack:** <!-- rules | small model | frontier | RAG | agent -->
8
+ - **Quality bar:**
9
+ - **Kill switch / fallback:**
10
+ - **Owner:**
11
+
12
+ ## Golden cases
13
+
14
+ | ID | Input (sanitized) | Expected | Pass rule | Last run | Result |
15
+ |----|-------------------|----------|-----------|----------|--------|
16
+ | G1 | | | | | |
17
+
18
+ ## Failure modes
19
+
20
+ | Mode | How it shows up | Detection | Mitigation |
21
+ |------|-----------------|-----------|------------|
22
+ | | | | |
23
+
24
+ ## Pass / fail (this ship)
25
+ - **Golden:** _/_ pass (threshold: _)
26
+ - **Critical fails (must be 0):**
27
+ - **Verdict:** <!-- SHIP | NO-SHIP -->
28
+ - **Evidence:** <!-- who/when -->
29
+
30
+ ## Human-in-the-loop gate
31
+
32
+ | Decision / action | Autonomous OK? | Reviewer role | Escalation |
33
+ |-------------------|----------------|---------------|------------|
34
+ | | | | |
35
+
36
+ **HITL rule for this ship:**
37
+
38
+ ## Change log
39
+
40
+ | Date | What changed | Pack re-run? | Notes |
41
+ |------|--------------|--------------|-------|
42
+ | | | | |
@@ -3,5 +3,7 @@
3
3
  <!-- Agreed definition of done. Out-of-scope is as important as in-scope. -->
4
4
 
5
5
  **Done when:**
6
+ **Primary value bucket:** <!-- cost-save | risk-mitigation | revenue-uplift (pick one) -->
7
+ **Baseline → target:** <!-- metric, number, by when -->
6
8
  **Explicitly out of scope:**
7
9
  **Stakeholder who signs off:**
@@ -5,3 +5,21 @@
5
5
  **Stack:**
6
6
  **Hotspots (handle with care):**
7
7
  **Test gaps:**
8
+
9
+ ## Operating map (exception-led)
10
+
11
+ <!-- How work actually runs when the happy path fails. Fill in discover; leave blank until heard/seen. -->
12
+
13
+ | Exception / break | Who notices first | What they do today (workaround) | System of record then | Blast if wrong | Evidence |
14
+ |-------------------|-------------------|---------------------------------|-----------------------|----------------|----------|
15
+ | | | | | | |
16
+
17
+ **Shadow systems / silent workarounds:**
18
+ **Sacred / untouchable in ops:**
19
+ **Previous attempt residue:**
20
+
21
+ ## Intelligence placement (when AI is in play)
22
+
23
+ | Step | Deterministic | Model judgement | Human approve |
24
+ |------|---------------|-----------------|---------------|
25
+ | | | | |