fdeops 3.9.12 → 3.9.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/check.js +1 -0
- package/bin/fde.js +80 -0
- package/package.json +1 -1
- package/skills/fde/SKILL.md +1 -0
- package/skills/fde/references/ai.md +15 -1
- package/skills/fde/references/build.md +1 -1
- package/skills/fde/references/close.md +8 -1
- package/skills/fde/references/discover.md +9 -2
- package/skills/fde/references/eval-pack.md +43 -0
- package/skills/fde/references/land.md +3 -2
- package/skills/fde/references/ship.md +21 -4
- package/templates/.fde/delivery.md +11 -3
- package/templates/.fde/evals.md +42 -0
- package/templates/.fde/success.md +2 -0
- package/templates/.fde/terrain.md +18 -0
package/bin/check.js
CHANGED
|
@@ -63,6 +63,7 @@ const requiredReferences = [
|
|
|
63
63
|
'debug.md', 'rescue.md', 'ship.md', 'sketch.md', 'close.md', 'dashboard.md',
|
|
64
64
|
'debrief.md', 'status.md', 'demo-prep.md',
|
|
65
65
|
'healthcare.md', 'fintech.md', 'gov.md',
|
|
66
|
+
'ai.md', 'eval-pack.md',
|
|
66
67
|
]
|
|
67
68
|
for (const f of requiredReferences) {
|
|
68
69
|
const p = path.join(root, 'skills', 'fde', 'references', f)
|
package/bin/fde.js
CHANGED
|
@@ -843,8 +843,11 @@ function cmdResume(args) {
|
|
|
843
843
|
const fdeDir = path.join(engRoot, '.fde')
|
|
844
844
|
const existed = fs.existsSync(fdeDir)
|
|
845
845
|
|
|
846
|
+
// Optional stubs (AI eval pack, …) stay in templates/ for copy-on-use — not day-1 scaffold.
|
|
847
|
+
const SKIP_INIT_TEMPLATES = new Set(['evals.md'])
|
|
846
848
|
const fillTemplates = (destFde) => {
|
|
847
849
|
for (const f of fs.readdirSync(tpl)) {
|
|
850
|
+
if (SKIP_INIT_TEMPLATES.has(f)) continue
|
|
848
851
|
const src = path.join(tpl, f); const dst = path.join(destFde, f)
|
|
849
852
|
if (fs.statSync(src).isDirectory()) fs.mkdirSync(dst, { recursive: true })
|
|
850
853
|
else if (!fs.existsSync(dst)) fs.copyFileSync(src, dst)
|
|
@@ -1501,6 +1504,18 @@ function collectDoctorIssues(eng) {
|
|
|
1501
1504
|
`phase is ${s.phase} with ${s.openRisks} open risk(s) - retire, hand off, or move still-live ones before calling the embed done`
|
|
1502
1505
|
)
|
|
1503
1506
|
}
|
|
1507
|
+
if (s.phase === 'close' || s.phase === 'ship') {
|
|
1508
|
+
if (!hasValueBucket(eng)) {
|
|
1509
|
+
issues.push(
|
|
1510
|
+
`phase is ${s.phase} with no value bucket (cost-save | risk-mitigation | revenue-uplift) in success.md or delivery value ledger`
|
|
1511
|
+
)
|
|
1512
|
+
}
|
|
1513
|
+
if (engagementTouchesAI(eng) && !hasEvalReceipt(eng)) {
|
|
1514
|
+
issues.push(
|
|
1515
|
+
`phase is ${s.phase} with AI in scope but no eval receipt (evals.md Verdict or delivery Eval / Ship receipts) — required before green ship/close`
|
|
1516
|
+
)
|
|
1517
|
+
}
|
|
1518
|
+
}
|
|
1504
1519
|
const dupes = findDuplicateOpenRisks(eng)
|
|
1505
1520
|
if (dupes.length) {
|
|
1506
1521
|
const sample = (dupes[0][0] || '').replace(/\s+/g, ' ').trim().slice(0, 60)
|
|
@@ -1511,6 +1526,71 @@ function collectDoctorIssues(eng) {
|
|
|
1511
1526
|
return issues
|
|
1512
1527
|
}
|
|
1513
1528
|
|
|
1529
|
+
// Strip template comments / italic *(hints)* so doctor does not treat stubs as filled.
|
|
1530
|
+
function stripTemplateNoise(md) {
|
|
1531
|
+
return String(md || '')
|
|
1532
|
+
.replace(/<!--[\s\S]*?-->/g, '')
|
|
1533
|
+
.replace(/\*\([^)]*\)\*/g, '')
|
|
1534
|
+
}
|
|
1535
|
+
|
|
1536
|
+
const VALUE_BUCKET_RE = /(cost[- ]?save|risk[- ]?mitigat|revenue[- ]?uplift)/i
|
|
1537
|
+
|
|
1538
|
+
function hasValueBucket(eng) {
|
|
1539
|
+
const success = stripTemplateNoise(readClean(eng, 'success.md'))
|
|
1540
|
+
const bucketLine = success.match(/\*\*Primary value bucket:\*\*\s*(.+)/i)
|
|
1541
|
+
if (bucketLine && VALUE_BUCKET_RE.test(bucketLine[1].trim())) return true
|
|
1542
|
+
if (!/\*\*Primary value bucket:\*\*/i.test(success) && VALUE_BUCKET_RE.test(success)) return true
|
|
1543
|
+
|
|
1544
|
+
const ledger = stripTemplateNoise(sectionBody(readClean(eng, 'delivery.md'), 'Value ledger') || '')
|
|
1545
|
+
const table = parseMdTable(ledger)
|
|
1546
|
+
if (table) {
|
|
1547
|
+
const bIdx = colIndex(table.headers, /bucket/i)
|
|
1548
|
+
if (bIdx !== -1) {
|
|
1549
|
+
for (const row of table.rows) {
|
|
1550
|
+
const cell = String(row[bIdx] || '').trim()
|
|
1551
|
+
if (cell && VALUE_BUCKET_RE.test(cell)) return true
|
|
1552
|
+
}
|
|
1553
|
+
} else if (VALUE_BUCKET_RE.test(ledger)) {
|
|
1554
|
+
return true
|
|
1555
|
+
}
|
|
1556
|
+
} else if (VALUE_BUCKET_RE.test(ledger)) {
|
|
1557
|
+
return true
|
|
1558
|
+
}
|
|
1559
|
+
return false
|
|
1560
|
+
}
|
|
1561
|
+
|
|
1562
|
+
// AI in scope for ship/close hygiene — delivery/decisions/trust evidence only.
|
|
1563
|
+
// Do not scan terrain.md: its template headers mention LLM and would false-positive every ship.
|
|
1564
|
+
function engagementTouchesAI(eng) {
|
|
1565
|
+
const trust = readClean(eng, 'trust-profile.md')
|
|
1566
|
+
const aiSec = stripTemplateNoise(sectionBody(trust, 'AI policy') || '')
|
|
1567
|
+
if (aiSec.trim().length > 20) return true
|
|
1568
|
+
const blob = stripTemplateNoise([
|
|
1569
|
+
readClean(eng, 'delivery.md'),
|
|
1570
|
+
readClean(eng, 'decisions.md'),
|
|
1571
|
+
].join('\n'))
|
|
1572
|
+
return /\b(llm|rag|embedding|inference|model card|agentic|openai|anthropic|vector database|vector db)\b/i.test(blob)
|
|
1573
|
+
}
|
|
1574
|
+
|
|
1575
|
+
function hasEvalReceipt(eng) {
|
|
1576
|
+
const evalsPath = path.join(eng, 'evals.md')
|
|
1577
|
+
if (fs.existsSync(evalsPath)) {
|
|
1578
|
+
const e = stripTemplateNoise(readClean(eng, 'evals.md'))
|
|
1579
|
+
// Empty G1 stub + "Pass / fail" heading is not a receipt — need a real verdict/run/result.
|
|
1580
|
+
if (/\*\*Verdict:\*\*\s*SHIP\b/i.test(e) || /(?:^|\n)\s*-\s*\*\*Verdict:\*\*\s*SHIP\b/i.test(e)) return true
|
|
1581
|
+
if (/\bLast run:\s*\d{4}-\d{2}-\d{2}/i.test(e)) return true
|
|
1582
|
+
if (/\|\s*G\d+\s*\|[^|\n]+\|[^|\n]+\|[^|\n]+\|[^|\n]+\|\s*pass\s*\|/i.test(e)) return true
|
|
1583
|
+
}
|
|
1584
|
+
const del = stripTemplateNoise(readClean(eng, 'delivery.md'))
|
|
1585
|
+
if (/#{1,6}\s+Eval\b/i.test(del) && /\b(pass|SHIP|\d+\/\d+)\b/i.test(sectionBody(del, 'Eval') || del)) return true
|
|
1586
|
+
if (/\beval (pack|receipt)[:\s].*\b(pass|SHIP)\b/i.test(del)) return true
|
|
1587
|
+
const receipts = sectionBody(del, 'Ship receipts') || ''
|
|
1588
|
+
if (/\bevals\.md\b/i.test(receipts) && /\b(pass|SHIP)\b/i.test(receipts) && !/\*\([^)]*evals\.md[^)]*\)\*/i.test(receipts)) {
|
|
1589
|
+
return true
|
|
1590
|
+
}
|
|
1591
|
+
return false
|
|
1592
|
+
}
|
|
1593
|
+
|
|
1514
1594
|
// Lean line for session-start TRIAGE - count + top issue + NL cue. Omitted when clean.
|
|
1515
1595
|
function hygieneTriageLines(eng) {
|
|
1516
1596
|
const issues = collectDoctorIssues(eng)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "fdeops",
|
|
3
|
-
"version": "3.9.
|
|
3
|
+
"version": "3.9.13",
|
|
4
4
|
"description": "Field kit for engineers embedded in client work - a real CLI (recon, memory, portfolio), one @fde skill with field judgment on top, and hooks that make it automatic. Claude Code plugin and any agent that loads skills.",
|
|
5
5
|
"bin": {
|
|
6
6
|
"fdeops": "bin/install.js",
|
package/skills/fde/SKILL.md
CHANGED
|
@@ -270,6 +270,7 @@ Running the engagement and ending it well.
|
|
|
270
270
|
| Signal | Overlay |
|
|
271
271
|
|--------|---------|
|
|
272
272
|
| AI, ML, LLM, model, embeddings, RAG, agents, fine-tuning, inference, drift | `references/ai.md` |
|
|
273
|
+
| Golden set, eval suite, eval pack, pass/fail before AI ship, HITL gate for model | `references/eval-pack.md` (+ `ai.md`) |
|
|
273
274
|
| Deck, slides, report, governance framework, compliance pack, ADR, PDF | `references/artifacts.md` |
|
|
274
275
|
| Patient data, PHI, HIPAA, EHR, clinical | `references/healthcare.md` |
|
|
275
276
|
| Payments, cardholder data, PCI-DSS, anything that moves money | `references/fintech.md` |
|
|
@@ -36,6 +36,19 @@ Never start with the most powerful model. Start with the cheapest that meets the
|
|
|
36
36
|
|
|
37
37
|
Write model selection rationale to `decisions.md`. Include: models tested, test set size, scores, cost comparison.
|
|
38
38
|
|
|
39
|
+
## Engagement eval pack (before AI ships)
|
|
40
|
+
|
|
41
|
+
When any slice touches a model, embeddings, RAG, or an agent: create or update `.fde/evals.md` **before** ship. Full method: `references/eval-pack.md`. This is the engagement-local test set — not unit tests.
|
|
42
|
+
|
|
43
|
+
**Minimum pack (do not grow until the minimum exists):**
|
|
44
|
+
1. **Component + quality bar** — one sentence each; kill switch / fallback named.
|
|
45
|
+
2. **Golden cases** — 5–20 representative inputs with expected outputs and a pass rule. Prefer real production-shaped data (sanitized).
|
|
46
|
+
3. **Failure modes** — at least the silent ones: hallucination/ungrounded, retrieval miss (if RAG), drift, cost runaway.
|
|
47
|
+
4. **Pass/fail** — dated run; Verdict **SHIP** or **NO-SHIP**; critical fails must be 0.
|
|
48
|
+
5. **HITL gate** — which decisions need human review before action (align with `trust-profile.md`). Empty when policy requires review → NO-SHIP.
|
|
49
|
+
|
|
50
|
+
**When to write:** plan seeds the pack; sketch/build grows goldens; ship requires Verdict SHIP and a receipt in `delivery.md` → `## Ship receipts`. Non-AI work skips this file entirely.
|
|
51
|
+
|
|
39
52
|
## RAG architecture (retrieval-augmented generation)
|
|
40
53
|
|
|
41
54
|
When the AI needs to answer questions about the client's data:
|
|
@@ -78,12 +91,13 @@ When the AI takes actions (not just generates text):
|
|
|
78
91
|
|
|
79
92
|
## Writes
|
|
80
93
|
|
|
81
|
-
`trust-profile.md` - AI policy, data classification, model hosting, human-in-the-loop requirements. `decisions.md` - model selection rationale, architecture choices. `risks.md` - bias findings, drift observations, cost projections. `delivery.md` - AI component inventory with kill switches
|
|
94
|
+
`trust-profile.md` - AI policy, data classification, model hosting, human-in-the-loop requirements. `evals.md` - golden cases, failure modes, SHIP/NO-SHIP, HITL. `decisions.md` - model selection rationale, architecture choices. `risks.md` - bias findings, drift observations, cost projections. `delivery.md` - AI component inventory with kill switches + eval receipt on ship.
|
|
82
95
|
|
|
83
96
|
## Principles
|
|
84
97
|
|
|
85
98
|
- AI degrades silently. Monitor outputs, not just uptime.
|
|
86
99
|
- Start with the cheapest model that meets the quality bar.
|
|
100
|
+
- No golden set, no AI ship (`evals.md` Verdict SHIP).
|
|
87
101
|
- Every AI component needs a kill switch and a fallback path.
|
|
88
102
|
- Log reasoning, not just results. Debug AI from its decisions.
|
|
89
103
|
- Drift is inevitable. Define the detection method before shipping.
|
|
@@ -127,7 +127,7 @@ The FDE's job is to make themselves replaceable. Not at handoff - every day. A c
|
|
|
127
127
|
|
|
128
128
|
- **`decisions.md`** - each significant choice: what, alternatives considered, why this one. For non-trivial architecture decisions, present three options to the FDE (safe / pragmatic / aggressive) with costs and a recommendation - three options is a real decision; one option is a request for trust. Integration contracts go here too.
|
|
129
129
|
- **`risks.md`** - new risks discovered while building.
|
|
130
|
-
- **`delivery.md`** - append a **value ledger** row for every ship: Date | Slice | Promised | Measured | Evidence | Rollback. "Measured" may be `pending` until the pulse exists - never skip the promised column. Narrative under Shipped is optional color; the ledger is the record status and close read.
|
|
130
|
+
- **`delivery.md`** - append a **value ledger** row for every ship: Date | Slice | Bucket | Promised | Measured | Evidence | Rollback. Bucket is `cost-save` / `risk-mitigation` / `revenue-uplift`. "Measured" may be `pending` until the pulse exists - never skip the promised column. Narrative under Shipped is optional color; the ledger is the record status and close read.
|
|
131
131
|
|
|
132
132
|
## Checkpoint
|
|
133
133
|
|
|
@@ -17,6 +17,12 @@ The engagement doesn't end at ship. It ends when the customer can maintain what
|
|
|
17
17
|
- Which risk almost became real?
|
|
18
18
|
- AI components: did they behave in production? What failure modes did the prototype hide? Is the team equipped to maintain them?
|
|
19
19
|
|
|
20
|
+
**1b. Value + receipts close gate (refuse green close if any fail):**
|
|
21
|
+
- Primary value bucket in `success.md` matches what the sponsor funded; at least one ledger row has **Measured** (not forever-`pending`) with evidence for that bucket — or the retrospective explicitly records “not measured; sponsor accepted pending.”
|
|
22
|
+
- Audit receipt exists for the final shipped path (exceptions/operating map walked; cite file).
|
|
23
|
+
- Eval receipt: **n/a if no AI**, else final golden/eval result + HITL owner recorded; kill switch / fallback named in `handoff.md`.
|
|
24
|
+
- One line in the retrospective: which bucket moved, by how much, vs baseline.
|
|
25
|
+
|
|
20
26
|
**2. The pattern.** Anything that happened here and will happen again - a compliance approach, a migration pattern, a stakeholder dynamic - gets encoded for reuse. **If you do it twice, encode it.**
|
|
21
27
|
|
|
22
28
|
**3. The handoff.** Operational knowledge for the person woken at 2am, not technical documentation: the 3 things that will break and the fix for each · who holds the tribal knowledge · what each alert means · deploy and rollback in plain language. AI components additionally: model version, what normal output looks like (so drift is recognisable), fallback behaviour, who owns retraining, **how to disable the AI path without taking down the feature** - without this the team turns it off at the first misbehaviour and it stays off.
|
|
@@ -33,11 +39,12 @@ The engagement doesn't end at ship. It ends when the customer can maintain what
|
|
|
33
39
|
|
|
34
40
|
## Checkpoint
|
|
35
41
|
|
|
36
|
-
Direct assessment to the FDE: did the engagement achieve `success.md` · 2–3 lessons that matter · is the pattern worth encoding · is the handoff complete or where are the gaps. Honest - a gap named now is cheaper than a callback in six weeks.
|
|
42
|
+
Direct assessment to the FDE: did the engagement achieve `success.md` · 2–3 lessons that matter · is the pattern worth encoding · is the handoff complete or where are the gaps. Also: value bucket + audit receipt green; eval **n/a or green**. Pending Measured without sponsor acceptance = gap, not green close. Honest - a gap named now is cheaper than a callback in six weeks.
|
|
37
43
|
|
|
38
44
|
## Principles
|
|
39
45
|
|
|
40
46
|
- Done = the customer operates without you.
|
|
47
|
+
- No named value bucket moved (or sponsor-accepted pending) = not a green close.
|
|
41
48
|
- The retrospective is an investment in the next engagement, not a post-mortem.
|
|
42
49
|
- Encode what repeated. The same lesson learned twice is a process failure.
|
|
43
50
|
- Write the handoff for 2am.
|
|
@@ -92,6 +92,7 @@ The real spec is what people **do** when the system fails - not what the slide d
|
|
|
92
92
|
- **The hesitation.** When someone says "well, there's also this other thing we do…" - stop them, ask them to finish. The main story is what they're comfortable explaining; the hesitation is the real problem.
|
|
93
93
|
- **"Which part of the codebase do you least want to touch?"** The answer is unanimous and it's the load-bearing wall. Check it against your churn scan - when the human answer and the churn data agree, that's your first map landmark.
|
|
94
94
|
- **Shadow AI.** Someone pasting data into ChatGPT to cope = a real unmet need + an uncontrolled data risk. Note both.
|
|
95
|
+
- **Exception-led operating map.** For each real break (not the slide-deck process): what fails, who notices first, what they do today, and which artifact is trusted in that moment. Prefer exceptions over happy-path swimlanes — the workaround is the operating system. Write rows under `terrain.md` → `## Operating map (exception-led)`. If the section is missing on an older engagement, add it; never regenerate the rest of terrain. When AI is in play, also fill `## Intelligence placement` (deterministic vs LLM judgement vs human approve).
|
|
95
96
|
|
|
96
97
|
## Method - part 3: workshop facilitation
|
|
97
98
|
|
|
@@ -165,6 +166,11 @@ Score every candidate use case before anything gets prototyped:
|
|
|
165
166
|
**Data flow:** <entry → transform → store → exit>
|
|
166
167
|
**Test landscape:** <covered / gaps / lies>
|
|
167
168
|
**Unknowns:** <named explicitly - an honest gap beats a confident guess>
|
|
169
|
+
|
|
170
|
+
## Operating map (exception-led)
|
|
171
|
+
| Exception / break | Who notices first | What they do today | System of record then | Blast | Evidence |
|
|
172
|
+
|-------------------|-------------------|--------------------|----------------------|-------|----------|
|
|
173
|
+
| <break> | <role> | <workaround> | <sheet/DB/person> | CRITICAL / LOAD-BEARING / CONVENIENCE | <who/day> |
|
|
168
174
|
```
|
|
169
175
|
|
|
170
176
|
Every line carries its evidence. `(churn: 47/90d)` `(ops lead, Day 5)` `(stated, unverified)`.
|
|
@@ -173,11 +179,12 @@ Every line carries its evidence. `(churn: 47/90d)` `(ops lead, Day 5)` `(stated,
|
|
|
173
179
|
|
|
174
180
|
## Checkpoint (before any build)
|
|
175
181
|
|
|
176
|
-
Present to the FDE,
|
|
182
|
+
Present to the FDE, five things, one paragraph each - no padding:
|
|
177
183
|
1. The real problem, with the two strongest pieces of evidence.
|
|
178
184
|
2. The top 3 risk areas of the codebase, one line of why each.
|
|
179
185
|
3. What must not be touched without characterisation tests.
|
|
180
|
-
4. The
|
|
186
|
+
4. The exception-led operating map: the two breaks that matter most, who owns the workaround, and where shadow systems live.
|
|
187
|
+
5. The recommendation: confirm brief / descope / rescope - and the decision it puts in front of the sponsor.
|
|
181
188
|
|
|
182
189
|
If discovery revealed the problem is 3× the brief: the FDE tells the customer **before** telling themselves it's manageable. Lead with evidence, offer three paths (descope / rescope / pause-and-plan), confirm any reset in writing - update `success.md` and `brief.md` before continuing.
|
|
183
190
|
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# eval-pack - prove the system before it acts
|
|
2
|
+
|
|
3
|
+
**Enter when:** the work touches AI/LLM/agents/RAG, or ship/close is blocked because there is no evidence the non-deterministic path is safe. Activate alongside `ai.md`, `sketch`, `build`, or `ship` — not instead of them.
|
|
4
|
+
|
|
5
|
+
**Read first:** `trust-profile.md` (AI policy + HITL), `terrain.md` (operating map), `delivery.md`. Create or extend `evals.md`.
|
|
6
|
+
|
|
7
|
+
Non-AI engagements skip this pack entirely.
|
|
8
|
+
|
|
9
|
+
## Why this exists
|
|
10
|
+
|
|
11
|
+
Intelligence without evidence is token-maxing with a nicer name. An FDE earns trust by showing: golden cases, failure modes, and a human gate before action.
|
|
12
|
+
|
|
13
|
+
## Method (you do this work)
|
|
14
|
+
|
|
15
|
+
**1. Scope the judgement surface.** One sentence: which step uses model judgement, and what must never be autonomous.
|
|
16
|
+
|
|
17
|
+
**2. Build a golden set (minimum 5–20 for a slice; prefer 50–100 before broad scale).** For each case:
|
|
18
|
+
- input (sanitized — no `<private>` raw values)
|
|
19
|
+
- expected outcome or expert-approved acceptance note
|
|
20
|
+
- pass rule (exact / contains / short rubric)
|
|
21
|
+
- source: real historical example / expert label / staged fixture
|
|
22
|
+
|
|
23
|
+
**3. Score pass/fail, not vibes.** Run the suite. Record count pass / fail. Failures get a failure-mode tag (missing data, wrong record, format drift, hallucination, retrieval miss, unsafe action, other).
|
|
24
|
+
|
|
25
|
+
**4. Human-in-the-loop gate.** Name which outcomes require human approve before side effects. If none, write why that is allowed under `trust-profile.md` AI policy — do not invent permission.
|
|
26
|
+
|
|
27
|
+
**5. Ship rule.** Until `evals.md` shows Verdict **SHIP** with a dated run (critical fails = 0) and HITL filled when policy requires it, AI-touching ship stays **fix-first**. Log a one-line eval receipt in `delivery.md` → `## Ship receipts`.
|
|
28
|
+
|
|
29
|
+
## Artifact — `evals.md`
|
|
30
|
+
|
|
31
|
+
Create on first AI-touching slice (not at `resume --init`). Use the stub in `templates/.fde/evals.md`. Every claim needs a source. Missing evidence → leave the cell `unknown - ask:`, never invent scores.
|
|
32
|
+
|
|
33
|
+
## Checkpoint
|
|
34
|
+
|
|
35
|
+
Present to the FDE: suite size, pass rate, top failure mode, HITL gate, Verdict SHIP/NO-SHIP. If they want to ship without a run: say no, and offer the smallest suite that would unblock.
|
|
36
|
+
|
|
37
|
+
## Principles
|
|
38
|
+
|
|
39
|
+
- No golden set, no AI ship.
|
|
40
|
+
- Pass/fail beats “looks good.”
|
|
41
|
+
- Failure modes are the product — the happy path is table stakes.
|
|
42
|
+
- HITL is a gate, not a slide.
|
|
43
|
+
- Non-AI work does not need this file.
|
|
@@ -65,6 +65,7 @@ Let silence sit. If their fear doesn't match the written brief, the brief is wro
|
|
|
65
65
|
- **The previous attempt** - "we tried something similar last year" is the most important sentence in the first meeting. Who was involved? Still there and protective, or gone because of it?
|
|
66
66
|
- **The passed-over internal team** - they know exactly what's wrong, and they resent the FDE's presence. Find them before the first standup, ask what they tried, use their language in every meeting. Make them look right and they protect you; ignore them and they wait for the mistake.
|
|
67
67
|
- **The sacred thing** - "Is there anything in this environment I should treat as untouchable?" The hesitation before the answer is the answer.
|
|
68
|
+
- **Exception path (operating map seed)** - "When the happy path breaks this week, what do people actually do — who do they call, what spreadsheet opens, what do they skip?" Capture the break → workaround → who owns it. Do not build a full map on day 1; seed rows later in `terrain.md` → `## Operating map (exception-led)` during discover. Unknowns stay `unknown - ask:`.
|
|
68
69
|
- **AI posture and policy** - tools already in use (sanctioned or shadow), and: "Does your organisation have a policy on AI-generated code? Are there decisions where you would not be comfortable with AI involvement?"
|
|
69
70
|
- **Boundaries in multi-vendor rooms** - who owns what surface, who signs off before a change crosses it.
|
|
70
71
|
|
|
@@ -76,7 +77,7 @@ Before the end of day 1, ship one visible thing: a small bug fix, a cleanup the
|
|
|
76
77
|
|
|
77
78
|
**`brief.md`** - what they said, who sent the FDE, the timeline, **and the gap list**.
|
|
78
79
|
|
|
79
|
-
**`success.md`** - what done looks like,
|
|
80
|
+
**`success.md`** - what done looks like, **primary value bucket** (`cost-save` | `risk-mitigation` | `revenue-uplift`), baseline → target, who actually signs off, what is explicitly out of scope. Agreed with the customer, not assumed.
|
|
80
81
|
|
|
81
82
|
**`stakeholders.md`**:
|
|
82
83
|
```markdown
|
|
@@ -100,7 +101,7 @@ One falsifiable hypothesis about the real problem also goes at the bottom of `br
|
|
|
100
101
|
|
|
101
102
|
## Checkpoint
|
|
102
103
|
|
|
103
|
-
One page back to the FDE: success + sign-off owner, out-of-scope boundary, sacred data, stakeholder map with veto power, AI posture, the hypothesis,
|
|
104
|
+
One page back to the FDE: success + value bucket + sign-off owner, out-of-scope boundary, sacred data, stakeholder map with veto power, AI posture, the hypothesis, the top CRITICAL assumptions still OPEN, and any exception-path seeds heard (break → workaround → owner) for discover to map into `terrain.md`. If it doesn't fit one page, the engagement isn't understood yet.
|
|
104
105
|
|
|
105
106
|
If remote: trust-building takes ~40% longer - push for a short video call before anything asynchronous.
|
|
106
107
|
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
**Enter when:** a slice is built, reviewed, and ready to deploy.
|
|
4
4
|
|
|
5
|
-
**Read first:** `context.md`, `delivery.md`. Load `trust-profile.md` if the deploy touches regulated data or needs an approval chain.
|
|
5
|
+
**Read first:** `context.md`, `delivery.md`, `success.md`. Load `trust-profile.md` if the deploy touches regulated data or needs an approval chain. Load `evals.md` when the deploy touches AI/ML/LLM/RAG/agents.
|
|
6
6
|
|
|
7
7
|
Opening question, calm tech lead voice: **has anyone actually *run* the rollback, or is it still a slide?** If only planned, that's today's work - say so plainly.
|
|
8
8
|
|
|
@@ -42,10 +42,26 @@ Score each dimension green/amber/red. This is the gate, not a suggestion:
|
|
|
42
42
|
| Runbook | Exists and someone other than you has read it | Exists but unreviewed | Missing |
|
|
43
43
|
| Monitoring | Alerts configured, owner named, dashboard live | Alerts configured, no named owner | No monitoring |
|
|
44
44
|
|
|
45
|
+
### Value + receipts gate (score with the table above)
|
|
46
|
+
|
|
47
|
+
| Dimension | Green | Amber | Red |
|
|
48
|
+
|-----------|-------|-------|-----|
|
|
49
|
+
| **Value bucket** | `success.md` names primary bucket (`cost-save` \| `risk-mitigation` \| `revenue-uplift`) and a baseline→target metric; this slice’s value-ledger row has **Bucket** + **Promised** | Bucket named; **Measured** still `pending` with a pulse date | No bucket, or Promised empty / ticket-theater only |
|
|
50
|
+
| **Audit receipt** | Dated line in `delivery.md` (`## Ship receipts` or ledger Evidence) proving exceptions/operating path were walked — cite `terrain.md` / `reality.md` / `audit.md` | Path described, not verified this ship | No audit receipt for this slice |
|
|
51
|
+
| **Eval receipt** | **n/a** (no AI on this slice) **or** `evals.md` Verdict SHIP with dated golden run + HITL gate named | Eval pack exists; known fails open with owner + date | AI in scope and no eval receipt |
|
|
52
|
+
| **AI eval pack** | `.fde/evals.md` Verdict SHIP; goldens run this change; critical fails 0; HITL filled if policy requires | Pack exists; run stale vs change log | AI-touching deploy and pack missing / NO-SHIP / HITL required but empty |
|
|
53
|
+
|
|
45
54
|
**Any RED = stop. Do not deploy. Fix the red dimension first.**
|
|
46
55
|
**2+ AMBER = sponsor conversation before deploying.** Present the ambers and get explicit "proceed" or "fix first."
|
|
47
56
|
|
|
48
|
-
|
|
57
|
+
**AI-touching deploys (model, embeddings, RAG, agent, or inference path):**
|
|
58
|
+
1. Read `.fde/evals.md`. If missing → **RED. Do not deploy.** Create the pack (`eval-pack` / `ai` overlay) and re-score.
|
|
59
|
+
2. If Verdict is not **SHIP**, or Last run is older than the latest change-log row → **RED.**
|
|
60
|
+
3. If `trust-profile.md` requires human-in-the-loop and the HITL gate has no reviewer → **RED.**
|
|
61
|
+
4. Log in `delivery.md` → `## Ship receipts` before deploy: audit cite + eval receipt.
|
|
62
|
+
5. Non-AI deploys: Eval = **n/a** — do not invent an empty pack.
|
|
63
|
+
|
|
64
|
+
Write the readiness score (including value + receipts) to `delivery.md` before deploying. The score is the evidence if anything goes wrong.
|
|
49
65
|
|
|
50
66
|
## Pre-blast challenge (before the deploy button)
|
|
51
67
|
|
|
@@ -146,13 +162,14 @@ Adoption isn't a handoff-stage problem - it starts during build. Software that l
|
|
|
146
162
|
|
|
147
163
|
## Checkpoint
|
|
148
164
|
|
|
149
|
-
Before 100%: canary clean, business metric verified, pulse written into `delivery.md`.
|
|
165
|
+
Before 100%: canary clean, business metric verified, pulse written into `delivery.md`. Also green: value bucket named, audit receipt dated, eval receipt **n/a or pass**. Missing any of those → not green. For enterprise-scale: scale-readiness gate passed before broad rollout.
|
|
150
166
|
|
|
151
167
|
## Principles
|
|
152
168
|
|
|
153
169
|
- A deployment without a tested rollback is reckless.
|
|
154
170
|
- Roll back on any canary anomaly; investigate safely.
|
|
155
171
|
- Verify the business metric, not just the technical one.
|
|
156
|
-
- No pulse, no done.
|
|
172
|
+
- No value bucket, no green ship. No pulse, no done.
|
|
173
|
+
- AI path without eval receipt = fix-first; non-AI ships leave eval as n/a.
|
|
157
174
|
- Scale readiness is organizational, not just technical. Check all 8 dimensions.
|
|
158
175
|
- Adoption is measured from day one, not hoped for at launch.
|
|
@@ -4,9 +4,17 @@
|
|
|
4
4
|
|
|
5
5
|
## Value ledger
|
|
6
6
|
|
|
7
|
-
| Date | Slice | Promised | Measured | Evidence | Rollback |
|
|
8
|
-
|
|
9
|
-
| | | *(what we said it would change)* | *(what actually changed, or pending)* | *(who/when/metric)* | |
|
|
7
|
+
| Date | Slice | Bucket | Promised | Measured | Evidence | Rollback |
|
|
8
|
+
|------|-------|--------|----------|----------|----------|----------|
|
|
9
|
+
| | | *(cost-save / risk-mitigation / revenue-uplift)* | *(what we said it would change)* | *(what actually changed, or pending)* | *(who/when/metric)* | |
|
|
10
|
+
|
|
11
|
+
## Ship receipts
|
|
12
|
+
|
|
13
|
+
<!-- Fill before green ship. Eval = n/a unless AI touches the slice. -->
|
|
14
|
+
|
|
15
|
+
| Date | Slice | Audit receipt | Eval receipt |
|
|
16
|
+
|------|-------|---------------|--------------|
|
|
17
|
+
| | | *(dated cite: exceptions/operating path verified — terrain/reality/audit)* | *(n/a \| evals.md pass + HITL owner)* |
|
|
10
18
|
|
|
11
19
|
## Shipped
|
|
12
20
|
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# Engagement eval pack
|
|
2
|
+
|
|
3
|
+
<!-- AI-touching work only. Fill before ship. Empty pack = do not deploy AI path. Non-AI engagements: leave unused or delete. -->
|
|
4
|
+
|
|
5
|
+
## Component
|
|
6
|
+
- **Name / slice:**
|
|
7
|
+
- **Model / stack:** <!-- rules | small model | frontier | RAG | agent -->
|
|
8
|
+
- **Quality bar:**
|
|
9
|
+
- **Kill switch / fallback:**
|
|
10
|
+
- **Owner:**
|
|
11
|
+
|
|
12
|
+
## Golden cases
|
|
13
|
+
|
|
14
|
+
| ID | Input (sanitized) | Expected | Pass rule | Last run | Result |
|
|
15
|
+
|----|-------------------|----------|-----------|----------|--------|
|
|
16
|
+
| G1 | | | | | |
|
|
17
|
+
|
|
18
|
+
## Failure modes
|
|
19
|
+
|
|
20
|
+
| Mode | How it shows up | Detection | Mitigation |
|
|
21
|
+
|------|-----------------|-----------|------------|
|
|
22
|
+
| | | | |
|
|
23
|
+
|
|
24
|
+
## Pass / fail (this ship)
|
|
25
|
+
- **Golden:** _/_ pass (threshold: _)
|
|
26
|
+
- **Critical fails (must be 0):**
|
|
27
|
+
- **Verdict:** <!-- SHIP | NO-SHIP -->
|
|
28
|
+
- **Evidence:** <!-- who/when -->
|
|
29
|
+
|
|
30
|
+
## Human-in-the-loop gate
|
|
31
|
+
|
|
32
|
+
| Decision / action | Autonomous OK? | Reviewer role | Escalation |
|
|
33
|
+
|-------------------|----------------|---------------|------------|
|
|
34
|
+
| | | | |
|
|
35
|
+
|
|
36
|
+
**HITL rule for this ship:**
|
|
37
|
+
|
|
38
|
+
## Change log
|
|
39
|
+
|
|
40
|
+
| Date | What changed | Pack re-run? | Notes |
|
|
41
|
+
|------|--------------|--------------|-------|
|
|
42
|
+
| | | | |
|
|
@@ -3,5 +3,7 @@
|
|
|
3
3
|
<!-- Agreed definition of done. Out-of-scope is as important as in-scope. -->
|
|
4
4
|
|
|
5
5
|
**Done when:**
|
|
6
|
+
**Primary value bucket:** <!-- cost-save | risk-mitigation | revenue-uplift (pick one) -->
|
|
7
|
+
**Baseline → target:** <!-- metric, number, by when -->
|
|
6
8
|
**Explicitly out of scope:**
|
|
7
9
|
**Stakeholder who signs off:**
|
|
@@ -5,3 +5,21 @@
|
|
|
5
5
|
**Stack:**
|
|
6
6
|
**Hotspots (handle with care):**
|
|
7
7
|
**Test gaps:**
|
|
8
|
+
|
|
9
|
+
## Operating map (exception-led)
|
|
10
|
+
|
|
11
|
+
<!-- How work actually runs when the happy path fails. Fill in discover; leave blank until heard/seen. -->
|
|
12
|
+
|
|
13
|
+
| Exception / break | Who notices first | What they do today (workaround) | System of record then | Blast if wrong | Evidence |
|
|
14
|
+
|-------------------|-------------------|---------------------------------|-----------------------|----------------|----------|
|
|
15
|
+
| | | | | | |
|
|
16
|
+
|
|
17
|
+
**Shadow systems / silent workarounds:**
|
|
18
|
+
**Sacred / untouchable in ops:**
|
|
19
|
+
**Previous attempt residue:**
|
|
20
|
+
|
|
21
|
+
## Intelligence placement (when AI is in play)
|
|
22
|
+
|
|
23
|
+
| Step | Deterministic | Model judgement | Human approve |
|
|
24
|
+
|------|---------------|-----------------|---------------|
|
|
25
|
+
| | | | |
|