@installagent/sdk 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. package/dist/agent-mirror.d.ts +69 -0
  2. package/dist/agent-mirror.js +275 -0
  3. package/dist/agent-templates.d.ts +36 -2
  4. package/dist/agent-templates.js +173 -3
  5. package/dist/claude-hours.d.ts +118 -0
  6. package/dist/claude-hours.js +523 -0
  7. package/dist/client-drive.d.ts +3 -0
  8. package/dist/client-drive.js +2 -2
  9. package/dist/client-linear-create.d.ts +26 -0
  10. package/dist/client-linear-create.js +46 -0
  11. package/dist/client-linear.d.ts +28 -9
  12. package/dist/client-linear.js +36 -14
  13. package/dist/docs-search.d.ts +19 -6
  14. package/dist/docs-search.js +32 -11
  15. package/dist/doctor.d.ts +9 -0
  16. package/dist/doctor.js +126 -23
  17. package/dist/env-fill.d.ts +13 -0
  18. package/dist/env-fill.js +55 -13
  19. package/dist/env-push.d.ts +3 -0
  20. package/dist/env-push.js +11 -6
  21. package/dist/env-registry.d.ts +29 -0
  22. package/dist/env-registry.js +123 -9
  23. package/dist/exec.js +15 -2
  24. package/dist/git.d.ts +31 -0
  25. package/dist/git.js +59 -0
  26. package/dist/google-service-account.d.ts +20 -0
  27. package/dist/google-service-account.js +49 -0
  28. package/dist/hours-estimate.d.ts +43 -0
  29. package/dist/hours-estimate.js +159 -0
  30. package/dist/hours-export-schedule.d.ts +35 -0
  31. package/dist/hours-export-schedule.js +188 -0
  32. package/dist/hours-ledger.d.ts +129 -0
  33. package/dist/hours-ledger.js +882 -0
  34. package/dist/index.d.ts +25 -6
  35. package/dist/index.js +25 -6
  36. package/dist/linear-project.d.ts +2 -2
  37. package/dist/linear-project.js +2 -2
  38. package/dist/log.d.ts +28 -0
  39. package/dist/log.js +73 -0
  40. package/dist/n8n-executions.d.ts +85 -0
  41. package/dist/n8n-executions.js +259 -0
  42. package/dist/op-cli.d.ts +29 -0
  43. package/dist/op-cli.js +52 -0
  44. package/dist/platform.d.ts +35 -0
  45. package/dist/platform.js +39 -0
  46. package/dist/portal-admin.d.ts +108 -0
  47. package/dist/portal-admin.js +94 -1
  48. package/dist/standup.d.ts +88 -0
  49. package/dist/standup.js +235 -0
  50. package/dist/sync-repos.d.ts +54 -0
  51. package/dist/sync-repos.js +89 -0
  52. package/dist/template-sync.d.ts +11 -0
  53. package/dist/template-sync.js +16 -0
  54. package/dist/triage-backfill.d.ts +37 -0
  55. package/dist/triage-backfill.js +211 -0
  56. package/dist/triage-enable.d.ts +90 -0
  57. package/dist/triage-enable.js +428 -0
  58. package/dist/triage-gates.d.ts +252 -0
  59. package/dist/triage-gates.js +951 -0
  60. package/dist/wizard.d.ts +15 -0
  61. package/dist/wizard.js +35 -56
  62. package/dist/workspace-docs.d.ts +44 -0
  63. package/dist/workspace-docs.js +73 -0
  64. package/dist/workspace-push.d.ts +39 -0
  65. package/dist/workspace-push.js +78 -0
  66. package/dist/workspace-status.d.ts +37 -0
  67. package/dist/workspace-status.js +114 -0
  68. package/package.json +2 -2
  69. package/templates/env.example +37 -23
  70. package/templates/workspace-root/.claude/settings.json +12 -1
  71. package/templates/workspace-root/AGENT.md +7 -5
  72. package/templates/workspace-root/ONBOARDING.md +65 -8
  73. package/templates/workspace-root/skills/.github/workflows/pr-to-slack.yml +127 -0
  74. package/templates/workspace-root/skills/CLAUDE.md +29 -6
  75. package/templates/workspace-root/skills/README.md +171 -0
  76. package/templates/workspace-root/skills/ad-machine/SKILL.md +140 -0
  77. package/templates/workspace-root/skills/backtest-acceptance-gates/SKILL.md +293 -0
  78. package/templates/workspace-root/skills/backtest-acceptance-gates/backtest-acceptance-gates-guide.pdf +0 -0
  79. package/templates/workspace-root/skills/backtest-acceptance-gates/references/evidence-record-template.md +128 -0
  80. package/templates/workspace-root/skills/backtest-acceptance-gates/references/harness-checklist.md +90 -0
  81. package/templates/workspace-root/skills/client-deliverables/SKILL.md +1 -1
  82. package/templates/workspace-root/skills/client-deliverables/references/package-layout.md +3 -2
  83. package/templates/workspace-root/skills/client-latest/SKILL.md +343 -0
  84. package/templates/workspace-root/skills/client-latest/catchup.command.md +18 -0
  85. package/templates/workspace-root/skills/cloud-devops/SKILL.md +235 -0
  86. package/templates/workspace-root/skills/email-triage-rollout/SKILL.md +258 -0
  87. package/templates/workspace-root/skills/engagement-spec/SKILL.md +51 -18
  88. package/templates/workspace-root/skills/engagement-spec/references/spec-templates.md +10 -4
  89. package/templates/workspace-root/skills/gcloud-cli-setup/SKILL.md +100 -0
  90. package/templates/workspace-root/skills/github-profile/SKILL.md +106 -0
  91. package/templates/workspace-root/skills/google-chat-app-setup/SKILL.md +136 -0
  92. package/templates/workspace-root/skills/google-chat-app-setup/google-chat-app-setup-guide.pdf +0 -0
  93. package/templates/workspace-root/skills/house-voice/SKILL.md +80 -4
  94. package/templates/workspace-root/skills/ia-cli/SKILL.md +49 -7
  95. package/templates/workspace-root/skills/init_repo/SKILL.md +188 -0
  96. package/templates/workspace-root/skills/installagent-brand-kit/SKILL.md +59 -10
  97. package/templates/workspace-root/skills/installagent-brand-kit/assets/installagent-deck-template.pptx +0 -0
  98. package/templates/workspace-root/skills/installagent-brand-kit/references/deck-guide.md +181 -0
  99. package/templates/workspace-root/skills/installagent-brand-kit/scripts/deck.py +652 -0
  100. package/templates/workspace-root/skills/installagent-brand-kit/scripts/deck_lint.py +201 -0
  101. package/templates/workspace-root/skills/installagent-brand-kit/scripts/logo_raster.py +102 -0
  102. package/templates/workspace-root/skills/installagent-brand-kit/scripts/make_deck_template.py +290 -0
  103. package/templates/workspace-root/skills/salesforce-api-snapshot/SKILL.md +379 -0
  104. package/templates/workspace-root/skills/salesforce-api-snapshot/references/schema-discovery-soql.md +153 -0
  105. package/templates/workspace-root/skills/salesforce-api-snapshot/references/snapshot-storage.md +122 -0
  106. package/templates/workspace-root/skills/salesforce-api-snapshot/salesforce-api-snapshot-guide.pdf +0 -0
  107. package/templates/workspace-root/skills/slack-voice/SKILL.md +158 -0
  108. package/templates/workspace-root/skills/sso-gated-client-app/SKILL.md +337 -0
  109. package/templates/workspace-root/skills/sso-gated-client-app/sso-gated-client-app-guide.pdf +0 -0
  110. package/templates/workspace-root/skills/tech-call-prep/SKILL.md +146 -47
  111. package/templates/workspace-root/skills/vendor-feed-ingestion/SKILL.md +323 -0
  112. package/templates/workspace-root/skills/vendor-feed-ingestion/references/delivery-contract-template.md +110 -0
  113. package/templates/workspace-root/skills/vendor-feed-ingestion/references/feed-guards.md +134 -0
  114. package/templates/workspace-root/skills/vendor-feed-ingestion/vendor-feed-ingestion-guide.pdf +0 -0
@@ -0,0 +1,140 @@
1
+ ---
2
+ name: ad-machine
3
+ description: >-
4
+ The Install Agent Meta ads machine — the creative engine that turns a
5
+ script, a prompt/brief, OR a client's existing asset library into a batch
6
+ of human-realistic video ads, then hands the batch to the banked
7
+ `meta-video-ads-agent` n8n rail for paused publishing into the client's
8
+ Meta ad account. Two modes: BRIEF (script or prompt → new UGC-style /
9
+ product video ads generated via the connected media-generation MCP's ad
10
+ workflows) and RECYCLE (mine the client's Drive folder of past ads, raw
11
+ footage, and product photos — ad-multiplier variations of a winning ad,
12
+ recuts of real footage, image-to-video from product stills — so the
13
+ client never has to film again and creatives keep coming). Use this
14
+ WHENEVER someone says "make ads for [client]", "new ad batch", "turn this
15
+ script into ads", "multiply the winner", "we need more creatives",
16
+ "generate ads from their footage/assets", "Meta ads for [client]", or the
17
+ weekly ad digest names a winner to feed back. Reach for it even if the
18
+ user never says "skill". Output is always the same contract: a Drive
19
+ batch folder (videos + thumbnails + batch.json) that the n8n rail
20
+ validates — this skill never touches the Meta API itself, and nothing it
21
+ makes goes live without a human in Ads Manager.
22
+ ---
23
+
24
+ # ad-machine — the Meta ads creative engine
25
+
26
+ The two-layer split, and why it exists: **this skill is the creative
27
+ engine** (taste, brand, hooks, generation — a technician runs it in Claude
28
+ Code, quality stays human-supervised); **the banked
29
+ `agents/meta-video-ads-agent/` workflow is the delivery rail** (validation,
30
+ paused publishing, ledger, weekly insights). The seam between them is one
31
+ file: `batch.json`. This skill writes it; the rail enforces it. The
32
+ manifest schema and every hard rule (PAUSED-only, budget cap, cross-client
33
+ guard, special ad categories) live in
34
+ `agents/meta-video-ads-agent/template.md` — **that file is the source of
35
+ truth for the contract; do not restate or fork it here.**
36
+
37
+ ## Preconditions
38
+
39
+ - The media-generation MCP (Higgsfield) is connected in this session — the
40
+ `generate_*` tools and `get_workflow_instructions` are available. If not,
41
+ stop and say so; there is no fallback generator.
42
+ - Client brand facts are reachable: `team-clients/clients/<slug>/` (who
43
+ they are, offer, review themes) and the build repo's `CLIENT.md`.
44
+ Client-facing copy follows the `house-voice` skill.
45
+ - The client's ad config is known (from the rail's deployment): client
46
+ slug, landing URLs, special-ad-category decision, budget cap. If the
47
+ rail isn't deployed yet, you can still produce the batch folder — the
48
+ creatives are usable manually (stage 1 of the SOP).
49
+
50
+ ## Mode selection
51
+
52
+ | Input you were given | Mode | Generation route |
53
+ |---|---|---|
54
+ | A script (voiceover / talking-head copy) | BRIEF | `ugc-review-video` (talking head) or `ugc-product-video` (voiceover only) |
55
+ | A prompt / offer brief, no script | BRIEF | Write the scripts first (below), then as above |
56
+ | A product/store/site URL to feature on screen | BRIEF | `ugc-website-video` |
57
+ | A winning past ad (4–30s) + "more like this" | RECYCLE | `ad-multiplier` |
58
+ | Raw client footage (jobs, before/after, walkthroughs) | RECYCLE | `video-editing` recuts: hook title + cut + captions |
59
+ | Product/service photos only | RECYCLE | `product-photoshoot` stills → image-to-video where motion helps |
60
+
61
+ Always load the matching workflow with `get_workflow_instructions` before
62
+ generating — never call `generate_video` cold for an ad. The workflow
63
+ catalog changes; check it rather than assuming.
64
+
65
+ ## BRIEF mode
66
+
67
+ 1. **Ingest.** Script or prompt + brand facts + landing URL + objective.
68
+ One batch = one angle tested several ways: default **3 variants, one
69
+ hook each** (e.g. price-anchor / social-proof / question). Hooks differ
70
+ in the first 3 seconds; bodies may share 80%.
71
+ 2. **Scripts.** If given a prompt, draft the variant scripts in the
72
+ client's register (house-voice: plain concrete words, name the
73
+ objection then contrast, honest hedges on numbers). 15–30s each.
74
+ **Truth guard:** no fabricated testimonials, invented statistics,
75
+ undocumented offers, or fake urgency — the same lines the GBP posts
76
+ agent enforces. A creator in a UGC ad is a presenter delivering the
77
+ client's message, never a fake customer with a fake experience. Claims
78
+ must trace to something the client actually said or documented.
79
+ 3. **Generate** via the loaded workflow (consistent creator per batch,
80
+ client's product/branding as references where available). Then a
81
+ thumbnail per variant (`thumbnail-generation` workflow) — the rail
82
+ requires one.
83
+ 4. **Package + approve** (below).
84
+
85
+ ## RECYCLE mode — creatives keep coming without new filming
86
+
87
+ The client's Drive folder gets an asset library, one-time setup:
88
+
89
+ ```
90
+ <client Drive root>/ad-assets/
91
+ past-ads/ # every finished ad we've run (winners get re-multiplied)
92
+ raw/ # phone footage: jobs, before/after, team, workspace
93
+ product/ # product & result photos, logos already in brand kit
94
+ ```
95
+
96
+ Ask the client once, in their channel, for a phone dump into `raw/` — 20
97
+ clips of real work beats a shoot. After that the machine self-feeds:
98
+
99
+ 1. **Inventory** the library (list the folder; note durations, subjects,
100
+ what's fresh since last batch).
101
+ 2. **Pick the seam:**
102
+ - A proven winner (from the rail's Monday digest) that's 4–30s →
103
+ `ad-multiplier`: N independently edited versions (swap people,
104
+ background, on-screen text; keep the motion and timing that won).
105
+ This is the highest-yield route — prefer it when a winner exists.
106
+ - Good raw footage → `video-editing`: hook title card, tight cut,
107
+ burned captions, CTA end-card. Real footage reads as the most
108
+ "human" ad of all — spend the effort on the first 3 seconds.
109
+ - Stills only → `product-photoshoot` restyles and image-to-video.
110
+ 3. **Rights check:** only client-supplied or client-approved footage; any
111
+ recognizable person in `raw/` needs the client's confirmation they're
112
+ ok appearing in ads (record the answer in the client repo). Never pull
113
+ media from anywhere but their library and our generations.
114
+ 4. **Package + approve** (below).
115
+
116
+ ## Package + approve (both modes)
117
+
118
+ 1. Batch folder in the client's Drive: `ad-batches/<batch_id>/` with
119
+ `v1.mp4`, `v1-thumb.jpg`, …, and `batch.json` per the rail's schema
120
+ (`agents/meta-video-ads-agent/template.md`). `batch_id` =
121
+ `YYYY-MM-DD-<angle>`; variant names `v1…`; `hook_label` is the real
122
+ hook, kebab-case — it becomes the ad name and the insights join key.
123
+ 2. Post the videos + copy to the client's Slack channel for approval
124
+ (house-voice). **Wait for the yes.** This is the same gate the SOP's
125
+ stage 1 formalizes — no batch is submitted unapproved.
126
+ 3. Submit the Drive folder ID to the rail's Batch Intake form. The rail
127
+ takes it from there: validated, published PAUSED, ledgered. Activation
128
+ is a human in Ads Manager, always.
129
+ 4. Log the batch in the client repo's delivery log (one CADENCE slice:
130
+ scripts/angle chosen, approval permalink, batch id).
131
+
132
+ ## Cost + cadence
133
+
134
+ - Generation burns media-MCP credits — usage-based, passed through at cost
135
+ per our standard commercial model. Note the batch's credit spend in the
136
+ delivery log so the pass-through is honest.
137
+ - Steady state is pull, not push: the Monday digest names a winner → next
138
+ batch multiplies it; no winner → next batch tests a new angle. One batch
139
+ a week per client is plenty; ten mediocre variants lose to three sharp
140
+ ones.
@@ -0,0 +1,293 @@
1
+ ---
2
+ name: backtest-acceptance-gates
3
+ description: >-
4
+ Prove an automation actually works before a client is asked to trust it —
5
+ and before we invoice against an acceptance gate. Covers reading the gate
6
+ definition off the signed SOW rather than from memory, pinning down which
7
+ metric the gate means when several plausible ones disagree by 18 points,
8
+ reconstructing ground truth from a client's own historical behaviour
9
+ (including the bimodal-signal problem, no-look-ahead scoring, and
10
+ right-censoring), excluding contaminated events legitimately instead of
11
+ conveniently, segmenting to tell a data gap from a logic gap, discovering
12
+ that ground truth itself is only 79% complete, the evidence-record
13
+ discipline that keeps a months-long build honest about which of its own
14
+ numbers are void, and why a green test suite can mask the exact bug it was
15
+ written to catch. Use this WHENEVER an engagement has an accuracy or
16
+ acceptance criterion — "backtest it", "how accurate is it", "prove it works",
17
+ "we need to hit the gate", "run it against history", "the client wants
18
+ evidence", "is it ready for go-live" — or when an existing accuracy number
19
+ needs re-checking.
20
+ ---
21
+
22
+ # Backtesting & acceptance gates
23
+
24
+ Practice from the CarVision pricing agent
25
+ (`clients/client-carvision/agent-pricing/` + that repo's `evidence/`), where an
26
+ SOW acceptance gate had real money attached and the accuracy number was
27
+ re-derived, contradicted, retracted and rebuilt over two months.
28
+
29
+ **What this skill is really about: staying honest at speed.** Almost every
30
+ mistake below produced a *number*, and the number looked fine. The discipline
31
+ isn't producing measurements — it's knowing which of your measurements are
32
+ still true.
33
+
34
+ ---
35
+
36
+ ## Part 1 — Get the gate from the contract, and pin the metric
37
+
38
+ **Read the acceptance criterion off the signed SOW.** Not the proposal draft,
39
+ not the kickoff deck, not memory. On CarVision the team measured against 85%
40
+ for weeks; the signed SOW said **80%**, on a review sample agreed by both
41
+ parties. Every measured number was unaffected — only the bar they were compared
42
+ against, and therefore every "we're short" judgement, was wrong.
43
+
44
+ **Then pin the metric, because several plausible readings will disagree
45
+ materially.** The same 2,004 decisions scored:
46
+
47
+ | Reading | Result |
48
+ |---|---|
49
+ | Direction match (raise/hold/drop agrees) | 62.2% |
50
+ | Price within tolerance | 56.1% |
51
+ | **Direction OR price** (the lenient reading) | **74.5%** |
52
+
53
+ Eighteen points between the loosest and tightest reading of "accurate." Write
54
+ the chosen definition into the tech spec in the SOW's own words, note which
55
+ readings you're *not* using, and report all of them in the evidence record.
56
+ If you only ever quote the flattering one, someone will eventually ask, and
57
+ that conversation is much worse than the honest table.
58
+
59
+ **Also check the arithmetic isn't being inflated by trivia.** An early run
60
+ showed 82.7% — inflated by 1,046 sub-$1 events that satisfied the price band
61
+ automatically. Excluding them gave 74.5%. Any metric with a tolerance band
62
+ will be gamed by near-zero cases; find them before someone else does.
63
+
64
+ ---
65
+
66
+ ## Part 2 — Reconstruct ground truth
67
+
68
+ Ground truth is what the client's expert actually did, recovered from history.
69
+ Four problems, all of which showed up in one dataset.
70
+
71
+ **The signal is bimodal, and the split is the whole exercise.** The client
72
+ expert's repricing log contained:
73
+
74
+ | Signal | n | What it actually means |
75
+ |---|---|---|
76
+ | ±$1–2 "touch" flips | 1,093 | *"I reviewed this and held."* A **hold** decision. |
77
+ | ≥$100 moves | 2,004 | Genuine repricing decisions. |
78
+ | $3–99 middle | ~16 | Ambiguous — skipped and declared |
79
+
80
+ Reading all 3,113 as "price changes" would have been catastrophic. The $1
81
+ nudges are a review marker in a system with no review button. **Ask the client
82
+ what their idiosyncratic markers mean before interpreting their log** — this
83
+ one only surfaced because someone asked why so many changes were a dollar.
84
+
85
+ **Score at the pre-decision state, with no look-ahead.** For each decision,
86
+ feed the agent the last snapshot *before* the decision — the stale price, that
87
+ row's own last-change date — with every input filtered to `created ≤ asOf`.
88
+ Any input that includes post-decision information turns the backtest into a
89
+ demonstration that the future predicts the past.
90
+
91
+ **Handle censoring explicitly.** Outcome layers ("did the car sell?") are
92
+ right-censored: a unit still listed on the last day has no outcome, not a
93
+ negative one. Say how you treated it.
94
+
95
+ **Absence is not an event.** In a feed where a sold unit simply stops
96
+ appearing, "disappeared" has to be *defined* as an outcome, and the definition
97
+ belongs in the evidence record.
98
+
99
+ ---
100
+
101
+ ## Part 3 — Excluding contaminated events: legitimately, not conveniently
102
+
103
+ Real history contains events that aren't the behaviour you're modelling. On
104
+ CarVision, **246 of 495 raises (49.7%) were a single bulk test** — one
105
+ operator setting a batch of cars to the same price on one day at one location,
106
+ reverted after it failed. Not pricing strategy. Excluding it moved the metric
107
+ from 63.8% to 72.1% and roughly doubled the raise arm's accuracy.
108
+
109
+ That is a big move in our favour, which is exactly why the bar is high:
110
+
111
+ 1. **Name the event.** What happened, when, who did it, and how you know — the
112
+ client confirming it is the strongest form.
113
+ 2. **Encode the exclusion as a dated constant in the harness**, not a hand-
114
+ filtered dataset. `TEST_EVENT_DATE` in the build script, reviewable in the
115
+ diff.
116
+ 3. **Show the control arm is unchanged.** The drop side didn't move at all when
117
+ the raise-side contamination was removed. An exclusion that improves
118
+ everything is suspicious; one that moves only the arm it should is credible.
119
+ 4. **Write it up as its own evidence record**, with the before and after.
120
+ 5. **Report the number both ways** the first time it's used in front of the
121
+ client.
122
+
123
+ If you can't do all five, you're not excluding contamination, you're tuning the
124
+ sample.
125
+
126
+ ---
127
+
128
+ ## Part 4 — Segment, to tell a data gap from a logic gap
129
+
130
+ An aggregate number tells you whether you passed. Segmentation tells you what
131
+ to do next.
132
+
133
+ | Segment | n | GATE |
134
+ |---|---|---|
135
+ | DROP decisions | 1,509 | **94.1%** |
136
+ | RAISE decisions | 495 | **14.7%** |
137
+ | Location A (multi-month lead history) | 825 | **84.7%** |
138
+ | Location B (one month of leads) | 1,179 | **67.3%** |
139
+
140
+ Two readings that changed the plan:
141
+
142
+ - **The raise arm didn't fail on logic.** A raise requires a 7-day lead count
143
+ above a threshold; the lead export was a single late-period snapshot, so
144
+ earlier 7-day windows were structurally empty. The demand signal that
145
+ justifies a raise was *invisible*, not wrong. No amount of tuning fixes that.
146
+ - **Same code, different data density, 17 points apart.** That comparison —
147
+ identical logic across two segments — is the cleanest possible evidence that
148
+ the lever is data completeness, not algorithm quality.
149
+
150
+ **Always segment by the arm/decision type and by data density.** Those two cuts
151
+ have repeatedly turned "the model is bad" into "the feed is thin", which is a
152
+ completely different (and usually cheaper) project.
153
+
154
+ And when you report a count, **report what's inside it**. Roughly half of one
155
+ build's "drops" were units already at a cost floor — no cut was possible. A raw
156
+ drop count quoted without that qualifier overstates the finding every time.
157
+
158
+ ---
159
+
160
+ ## Part 5 — Ground truth can itself be wrong
161
+
162
+ The hardest lesson here, and the one worth carrying into every engagement.
163
+
164
+ An early investigation concluded the ingestion was faithful and the counts were
165
+ correct. It was wrong — **it audited the feed against itself, with no external
166
+ reference.** A later reconciliation against the client's own CRM report found
167
+ the feed carried **79%** of the real records, and that gap had been quietly
168
+ distorting every accuracy figure computed on top of it.
169
+
170
+ Consequences, all of which are the normal cost of finding this late:
171
+
172
+ - Every gate figure computed before the reconciliation became **provisional**,
173
+ and the evidence index had to say so at the top rather than in a footnote.
174
+ - A published headline number was marked **void** — not deleted. Deleting it
175
+ would make the record dishonest; superseding it in place makes it useful.
176
+ - One evidence record's conclusion was struck through and replaced, with the
177
+ part that *was* right (a hypothesis it raised) kept and re-labelled.
178
+
179
+ **So: reconcile your inputs against an independent source before you trust any
180
+ metric built on them.** One export from a second path is enough. Do it early —
181
+ it is much cheaper than retracting a number the client has already seen.
182
+
183
+ ---
184
+
185
+ ## Part 6 — A green test suite can hide the bug it was written to catch
186
+
187
+ A 144-assertion suite ran green while a data-joining layer contributed **zero**
188
+ records to the calculation for three days. The parity test *applied the missing
189
+ conversion itself* before comparing — so it compared two correct things and
190
+ passed, while production compared two incompatible ones.
191
+
192
+ Worse than inert: one field name happened to match across both shapes, so a
193
+ "history exists" flag was set even though the history was empty — which
194
+ **raised the confidence badge** on exactly the recommendations that had no
195
+ supporting evidence behind them.
196
+
197
+ Defences, in order of value:
198
+
199
+ 1. **Test the seam with production's own inputs**, not inputs the test
200
+ pre-normalizes. If the test does work the pipeline doesn't, it isn't testing
201
+ the pipeline.
202
+ 2. **Assert on non-zero contribution, not just on equality.** "The join
203
+ produced ≥ N records" catches silent-empty; "the outputs match" doesn't.
204
+ 3. **Replay a real historical run end to end** and diff against what production
205
+ actually produced. A replay harness that reproduces a known run exactly
206
+ (e.g. 319/319 records) is what lets you attribute a delta to one cause.
207
+ 4. **When a fix changes headline numbers, re-derive every number that depended
208
+ on it** and mark the old ones. Here: drops 111→77 and 178→83, raises 11→36
209
+ and 2→33 — every prior claim about drop volume was void the moment the fix
210
+ landed.
211
+
212
+ ---
213
+
214
+ ## Part 7 — A backtest is a proxy; the gate is a live window
215
+
216
+ **The formal acceptance in the SOW was a ~10-business-day live window with the
217
+ named client reviewer, not a historical backtest.** The backtest is our internal
218
+ de-risking proxy for it. Never let the two be confused — passing a backtest is
219
+ not passing acceptance, and saying so plainly protects everyone.
220
+
221
+ So the go-live sequence is:
222
+
223
+ 1. Backtest to the pinned metric → evidence record.
224
+ 2. **Shadow period**: the automation proposes, a named human decides, for a
225
+ defined window. Every build in these repos does this.
226
+ 3. Only then does the output get treated as decidable truth.
227
+
228
+ Also, take the win when the client changes the frame. When one client declined
229
+ to annotate historical examples and wanted to use the tool live instead, the
230
+ right response was to **retire the annotation ask and the backtest gate for
231
+ that arm**, reframing it as rule-based flags — not to keep chasing a gate the
232
+ client no longer cared about. The gate serves the engagement, not the reverse.
233
+
234
+ ---
235
+
236
+ ## Part 8 — Evidence discipline
237
+
238
+ One dated, self-contained ~1-page record per run, in the build repo's
239
+ `evidence/`. Template and conventions:
240
+ [`references/evidence-record-template.md`](references/evidence-record-template.md).
241
+
242
+ The parts that make it work over months:
243
+
244
+ - **An index table at the top of `evidence/README.md`**, newest first, one line
245
+ each with the headline number.
246
+ - **Supersession is loud and in place.** Strike through the wrong conclusion,
247
+ say what replaced it, keep what was right. Never quietly delete.
248
+ - **A standing caveat block** above the index when something invalidates a
249
+ whole class of prior numbers.
250
+ - **Mark internal-only records**, and keep client-facing ones free of raw
251
+ identifiers and personal data — aggregates only.
252
+ - **Point at the reproduce-it-yourself doc** in the build folder rather than
253
+ duplicating steps.
254
+
255
+ Harness rules: [`references/harness-checklist.md`](references/harness-checklist.md).
256
+ In short — every number comes from a committed script with a copy-pasteable
257
+ command line, thresholds live in one config file rather than scattered through
258
+ code, and the raw inputs stay out of git.
259
+
260
+ ---
261
+
262
+ ## Requirements checklist
263
+
264
+ - [ ] Gate definition read off the **signed** SOW; the number and its wording quoted in the tech spec
265
+ - [ ] Metric pinned; competing readings computed and reported alongside
266
+ - [ ] Trivial/near-zero cases identified and excluded from any tolerance-band metric
267
+ - [ ] Client's idiosyncratic markers understood before interpreting their history
268
+ - [ ] Scoring at pre-decision state; every input filtered to `≤ asOf`; no look-ahead
269
+ - [ ] Censoring and "absence" semantics defined in writing
270
+ - [ ] Contamination exclusions: named, dated, encoded in the harness, control arm shown unchanged, written up
271
+ - [ ] Segmented by decision type **and** by data density
272
+ - [ ] Inputs reconciled against an independent source before trusting any metric
273
+ - [ ] Seam tests use production's own inputs; assertions include non-zero contribution
274
+ - [ ] A real historical run replayed end to end and diffed
275
+ - [ ] Every number traceable to a committed script and a command line
276
+ - [ ] Evidence record written, indexed, and superseded-in-place where relevant
277
+ - [ ] Shadow period run with a named reviewer before go-live
278
+ - [ ] Delivery-log entry linking the evidence record
279
+
280
+ ## Known gaps to flag, not silently skip
281
+
282
+ - **A backtest measures agreement with past behaviour, not correctness.** If
283
+ the client's historical decisions were poor, high agreement means we
284
+ reproduced poor decisions. Say which one the gate is actually testing.
285
+ - **Historical data is rarely balanced across arms.** An arm with few events
286
+ and sparse inputs cannot be validated to the same confidence as the main arm.
287
+ Report per-arm sample sizes; a 14.7% on 495 events and a 94.1% on 1,509 are
288
+ not comparable evidence.
289
+ - **Thresholds tuned on a dataset and validated on the same dataset are
290
+ overfit.** Hold out a validation slice, and say whether you did.
291
+ - **The gate is a point-in-time reading.** Feed quality, client behaviour and
292
+ the market all move. Date every figure, and re-run before quoting an old one
293
+ in a new conversation.
@@ -0,0 +1,128 @@
1
+ # Evidence record — convention and template
2
+
3
+ Every backtest or validation run gets one dated, self-contained `.md` file in
4
+ the build repo's `evidence/`. These are the shareable trail — the client-facing
5
+ ones get imported to their Google Drive, so a reader with no repo access has to
6
+ be able to follow them.
7
+
8
+ Live exemplar set: `clients/client-carvision/evidence/`.
9
+
10
+ ---
11
+
12
+ ## Convention
13
+
14
+ - **One file per run**: `YYYY-MM-DD-<agent>-<what>.md`
15
+ - **~1 page.** What we tested, on what data, the headline number, the honest
16
+ read. One small table. No code — link to the reproduce-it doc instead.
17
+ - **No raw identifiers or personal data.** Aggregates only. Raw inputs stay in
18
+ the gitignored data folder.
19
+ - **Mark internal-only records explicitly** (🔒) so nobody exports one to a
20
+ client Drive by accident.
21
+ - **Index it** in `evidence/README.md`, newest first, with the headline.
22
+
23
+ ---
24
+
25
+ ## Template
26
+
27
+ ```markdown
28
+ # <Agent> — <what was tested> (<YYYY-MM-DD>)
29
+
30
+ **Status:** [client-shareable | 🔒 internal, not for Drive]
31
+ **Supersedes:** [prior record, or —]
32
+
33
+ ## What we tested
34
+
35
+ One paragraph. The question this run answers, and the metric definition being
36
+ used — in the SOW's own words if it maps to an acceptance gate.
37
+
38
+ ## Data
39
+
40
+ - Source, date range, row/entity counts.
41
+ - Known limitations of the sample (sparse periods, single-snapshot inputs,
42
+ missing columns) — stated here, not discovered in the read.
43
+ - Exclusions applied, each with its reason and the constant that encodes it.
44
+
45
+ ## Result
46
+
47
+ | Metric | Result | Bar |
48
+ |---|---:|---:|
49
+ | <the pinned gate metric> | xx.x% | xx% |
50
+ | <the stricter reading> | xx.x% | — |
51
+ | <the looser reading> | xx.x% | — |
52
+
53
+ Segmented:
54
+
55
+ | Segment | n | Result |
56
+ |---|---:|---:|
57
+ | <arm A> | | |
58
+ | <arm B> | | |
59
+ | <high data density> | | |
60
+ | <low data density> | | |
61
+
62
+ ## The honest read
63
+
64
+ Two to four sentences. What this does and does not establish. If the number is
65
+ below the bar, say what the lever is — data completeness or logic — and what
66
+ evidence supports that attribution. If the number is above the bar, say what
67
+ would make it stop being true.
68
+
69
+ ## Reproduce
70
+
71
+ `node scripts/<harness>.js --<args>`
72
+
73
+ Technical detail: `<path to the build-folder doc>`
74
+ ```
75
+
76
+ ---
77
+
78
+ ## Superseding a record
79
+
80
+ When a later run contradicts an earlier one, **edit the old record in place**;
81
+ never delete it.
82
+
83
+ - Strike through the wrong conclusion: `~~Ingestion faithful; counts correct~~`
84
+ - Immediately after it: `**⚠️ SUPERSEDED YYYY-MM-DD — the conclusion was
85
+ wrong.** <one sentence on why.>`
86
+ - **Keep what was right.** If the record raised a hypothesis that later proved
87
+ correct, say so — "the back-truncation hypothesis it raised was right; 'no
88
+ fix needed' was not."
89
+ - Update the index line to match.
90
+
91
+ When something invalidates a whole class of prior figures, put a **standing
92
+ caveat block above the index table**, not in a footnote:
93
+
94
+ ```markdown
95
+ > **Every gate figure in this folder is now provisional.** All were computed on
96
+ > lead data that the YYYY-MM-DD reconciliation showed to be ~79% of ground
97
+ > truth, and the raise arm depends on those counts almost entirely.
98
+ ```
99
+
100
+ That block is what stops a stale headline being quoted into a client
101
+ conversation six weeks later.
102
+
103
+ ---
104
+
105
+ ## Publishing a run to the client
106
+
107
+ Two surfaces, and they're separate steps:
108
+
109
+ 1. **The Drive copy** — the client-shareable records only, imported to their
110
+ engagement folder.
111
+ 2. **A dated tab in the client's working spreadsheet** — a summary block, then
112
+ one row per decision, so they can inspect individual cases rather than
113
+ taking a percentage on faith. CarVision does this via a generator script
114
+ plus a deliberately **inactive** publisher workflow driven by its test
115
+ webhook, so no permanent public endpoint lingers.
116
+
117
+ Keep the publishing secret on the workflow's own credential, never in the repo.
118
+ If one was ever committed, treat it as burned and rotate it.
119
+
120
+ ---
121
+
122
+ ## What belongs in the delivery log instead
123
+
124
+ The evidence record is the *finding*. The build repo's
125
+ `docs/00-delivery-log.md` gets a short dated entry that says what shipped,
126
+ where it runs, links the evidence file, and lists the follow-ups. Schema
127
+ spikes, audits and investigations with no pass/fail verdict belong in the
128
+ delivery log and the tech spec — `evidence/` is for runs with a verdict.
@@ -0,0 +1,90 @@
1
+ # Backtest harness — how to build one that stays trustworthy
2
+
3
+ Every number we quote should be reproducible by a colleague with one command.
4
+ Shape drawn from `clients/client-carvision/agent-pricing/scripts/`.
5
+
6
+ ---
7
+
8
+ ## The scripts you end up needing
9
+
10
+ | Script | Job |
11
+ |---|---|
12
+ | `build-<x>-history.js` | Reconstruct ground truth from raw history. Where exclusion constants live. |
13
+ | `backtest.js` | Score the pinned gate metric. Takes the history file as input. |
14
+ | `<x>-analysis.js` | Segmentation — the *why* behind the headline. |
15
+ | `reconcile-<x>.js` | Compare our inputs against an independent export (Part 5 of the skill). |
16
+ | `replay-coverage.js` | Replay a real historical run end to end and diff against what production produced. |
17
+ | `test-<component>.js` | Unit-level assertions on the scoring logic. |
18
+ | `value-of-fields.js` | What each input field is actually worth to the output — kills debates about which upstream ask matters. |
19
+
20
+ Separate **reconstruct** from **score**. When the ground-truth definition
21
+ changes you re-run one script, and the diff shows exactly what moved.
22
+
23
+ ## Rules
24
+
25
+ **One command line per number.** Every figure in an evidence record carries the
26
+ invocation that produced it:
27
+
28
+ ```
29
+ node scripts/backtest.js --history data/outcome-history.json
30
+ ```
31
+
32
+ **Exclusions are dated constants in the harness**, never a hand-edited dataset.
33
+ `const TEST_EVENT_DATE = "…"` shows up in a diff; a filtered CSV doesn't.
34
+
35
+ **Thresholds live in one config file**, with the calibration rationale in
36
+ comments next to the values, and are read by both production and the harness
37
+ from the same place. If the harness and production can drift apart on a
38
+ threshold, they will, and the backtest stops describing the thing you shipped.
39
+
40
+ **Raw inputs stay out of git.** Client data goes in a gitignored data folder;
41
+ only summaries are committed.
42
+
43
+ **Deterministic.** Same inputs, same outputs, every run. No wall-clock, no
44
+ unseeded randomness — an `asOf` is a parameter, not `now()`.
45
+
46
+ **Print the sample size next to every percentage.** Always. A percentage
47
+ without an n is not a result.
48
+
49
+ ## Tests that actually catch things
50
+
51
+ The failure to design against: a 144-assertion suite passing green while a
52
+ data-joining layer contributed zero records for three days, because the parity
53
+ test applied the missing conversion itself before comparing.
54
+
55
+ - **Feed the seam production's own input shape.** If the test normalizes the
56
+ input first, it is testing the normalizer, not the seam.
57
+ - **Assert non-zero contribution**, not only equality. `joined.length > 0` and
58
+ `contributedRecords >= N` catch silent-empty; `expect(a).toEqual(b)` does not.
59
+ - **Round-trip the real artifact.** Read the actual stored shape (the sheet
60
+ tab, the S3 object) and assert the consumer can read it — not a
61
+ hand-constructed fixture of what you believe it contains.
62
+ - **Red-then-green.** Write the failing assertion against the live bug first,
63
+ watch it fail, then fix. A test written after the fix proves nothing about
64
+ whether it would have caught it.
65
+ - **Replay a known run.** Reproducing a historical run exactly (all N records,
66
+ not most) is what lets you attribute a later delta to a single cause instead
67
+ of arguing about it.
68
+
69
+ ## When a fix moves the numbers
70
+
71
+ 1. Re-run every harness that depended on the fixed component.
72
+ 2. Diff the headline figures and record both sides.
73
+ 3. Mark every prior evidence record whose numbers are now void — in place, with
74
+ a strike-through and a pointer.
75
+ 4. Update the standing caveat block in `evidence/README.md`.
76
+ 5. Only then quote the new number to anyone.
77
+
78
+ Steps 3 and 4 are the ones that get skipped, and they are the ones that stop a
79
+ void figure resurfacing in a client conversation weeks later.
80
+
81
+ ## Before quoting any number externally
82
+
83
+ - [ ] Produced by a committed script, from committed inputs, with the command line recorded
84
+ - [ ] Sample size stated
85
+ - [ ] Metric definition stated, in the SOW's wording if it maps to a gate
86
+ - [ ] Competing metric readings computed, and reported if materially different
87
+ - [ ] Exclusions named and justified; control arm shown unchanged
88
+ - [ ] Inputs reconciled against an independent source
89
+ - [ ] No prior evidence record contradicts it un-superseded
90
+ - [ ] Qualifiers travel with the count (e.g. "half of these were already at a floor")
@@ -55,7 +55,7 @@ by its `ci/repo-lint.mjs`:
55
55
  ```
56
56
  client-<slug>/
57
57
  ├── AGENT.md / CLAUDE.md / CLIENT.md
58
- ├── onboarding.html # tabbed open-me-first page: Architecture/Client/Proposals/Public
58
+ ├── onboarding.html # tabbed open-me-first page: Architecture/Client/Access/Builds/Workflows/Proposals/Public
59
59
  ├── docs/ # internal markdown (engagement-spec): 01-overview, 02-tech-spec,
60
60
  │ # 03-client-onboarding, ① the in-depth internal HTML (tabbed),
61
61
  │ # plus ③ diagrams as self-contained HTML (<system>-<audience>-diagram.html)
@@ -19,8 +19,9 @@ client-<slug>/
19
19
  │ ├── 02-tech-spec.md # internal: names tools, the decision model
20
20
  │ ├── 03-client-onboarding.md # what we need from the client
21
21
  │ └── <system>-overview.html # ① in-depth INTERNAL html (tabbed), if built
22
- ├── onboarding.html # tabbed open-me-first page (Architecture/Client/
23
- │ # Proposals/Public — Public is client-presentable)
22
+ ├── onboarding.html # tabbed open-me-first page (Architecture/Client/Access/
23
+ │ # Builds/Workflows/Proposals/Public — Public and the
24
+ │ # Builds diagrams are client-presentable)
24
25
  ├── workflows/ # n8n JSON exports, credential IDs scrubbed
25
26
  ├── data/ # client-sent raw data — gitignored (README tracked)
26
27
  ├── evidence/ # ④ dated backtest/validation records — required,