@codyswann/lisa 2.305.4 → 2.306.0

package/ui/index.html

@@ -5227,7 +5227,7 @@
                 ],
               },
               {
-                label: "The agent's program (AC8.2–8.3 · AC4.5)",
+                label: "The agent's program (AC8.2–8.3 · AC4.5 · P10–P11)",
                 note: "Prompts, skills and models are the factory's own source code and runtime. This group asks whether that layer is under change control — or improvised per run.",
                 items: [
                   {
@@ -5270,6 +5270,24 @@
                       "None",
                     ],
                   },
+                  {
+                    q: "Do agent-program and model qualifications report repeated-run distributions?",
+                    help: "TASC P10, P11, AC8.2 and AC8.3. A single lucky run does not qualify a prompt, model or effort-level change. The evidence names the condition tested, number of repetitions, and distribution of outcomes.",
+                    kind: "select",
+                    options: [
+                      "Repeated runs — distribution reported",
+                      "Some repeats, summarized informally",
+                      "One passing run",
+                      "Vendor benchmark only",
+                    ],
+                    via: [
+                      "Evaluation harness",
+                      "Canary work items",
+                      "Golden task suite",
+                      "In-house analysis",
+                      "None",
+                    ],
+                  },
                   {
                     q: "When an agent makes a mistake, what stops the next agent repeating it?",
                     help: "TASC AC4.5 — the self-learning loop. Mistakes and drift are captured with provenance, pass a skeptical gate, and get promoted upstream into lint rules, hooks, gates and skills. A lesson that stays prose is one the next agent never reads; recurrence of a captured mistake means the loop failed.",
@@ -5282,6 +5300,16 @@
                       "Nothing",
                     ],
                   },
+                  {
+                    q: "Is the reasoning / effort level pinned alongside the model version?",
+                    help: "Effort is not monotonic — more thinking makes some tasks worse and costs more on all of them. It is part of the runtime configuration, so it belongs in the pin and in the qualification, not in whatever each agent happens to pass.",
+                    kind: "select",
+                    options: [
+                      "Model, effort and tool config all pinned",
+                      "Model version pinned only",
+                      "Neither pinned",
+                    ],
+                  },
                 ],
               },
               {
@@ -5366,7 +5394,7 @@
                 ],
               },
               {
-                label: "Correctness gates (SI1–SI3 · SI8 · AC5.4)",
+                label: "Correctness gates (SI1–SI3 · SI8–SI9 · AC5.4)",
                 items: [
                   {
                     q: "Is unit behaviour proven on every change?",
@@ -5419,6 +5447,23 @@
                     options: ["Yes — gates merges", "Yes — nightly only", "No"],
                     value: "Yes — nightly only",
                   },
+                  {
+                    q: "Do escaped defects become replayable efficacy tests?",
+                    help: "TASC SI3. A production failure that disappears into a fix note does not improve the test suite. The defect should replay in automation, or carry an accepted-risk exception explaining why replay is infeasible.",
+                    kind: "select",
+                    options: [
+                      "Always replayed or risk-accepted",
+                      "Sometimes replayed",
+                      "Fixed without replay",
+                    ],
+                    via: [
+                      "Mutation testing",
+                      "Regression harness",
+                      "Incident replay suite",
+                      "In-house",
+                      "None",
+                    ],
+                  },
                   {
                     q: "Is the system load-tested against SLOs before release?",
                     help: "TASC SI8. Functional correctness without performance is insufficient — an endpoint returning correct data in 30 seconds fails a 200ms SLO. Stress, spike, or soak per the risk of the change; no service surface is a valid answer.",
@@ -5441,6 +5486,24 @@
                       "None",
                     ],
                   },
+                  {
+                    q: "Are generated tests and synthetic user findings measured before intake?",
+                    help: "TASC SI9 and AC4.6. Generative testing is a finding source, not an oracle. Its prompts or seeds, target selection, rejection stage, false-positive rate and regression-promotion path are part of the control.",
+                    kind: "select",
+                    options: [
+                      "Measured source with rejection stage",
+                      "Generated findings reviewed ad-hoc",
+                      "Generated findings enter intake directly",
+                      "No generative testing",
+                    ],
+                    via: [
+                      "Property-based tests",
+                      "Fuzzing",
+                      "Agent exploratory tests",
+                      "Synthetic users",
+                      "None",
+                    ],
+                  },
                   {
                     q: "Can a quality threshold be quietly lowered?",
                     help: "The ratchet: thresholds may only tighten, and loosening one requires its own reviewed change. A bar anyone can edit downward in passing is a suggestion, not a bar — and an agent under pressure to go green will find it.",
@@ -5453,6 +5516,36 @@
                       "Anyone can edit the config",
                     ],
                   },
+                  {
+                    q: "Does anything generate new test inputs, or do you only re-run the same suite?",
+                    help: "TASC SI9 — generative testing. A fixed suite only ever checks the cases someone thought of. At equal compute, generating fresh cases against a stable build explores far more of the input space than replaying one suite per change, and under agent volume the unexplored space is where quality quietly goes.",
+                    kind: "select",
+                    options: [
+                      "Continuous generation, outside the per-change gate",
+                      "Property-based tests inside CI only",
+                      "A fuzzer someone ran once",
+                      "No — fixed suite only",
+                    ],
+                    via: [
+                      "fast-check",
+                      "Hypothesis",
+                      "jqwik",
+                      "libFuzzer / AFL++",
+                      "OSS-Fuzz",
+                      "In-house generator",
+                      "None",
+                    ],
+                  },
+                  {
+                    q: "Can your tests still catch the bugs you have already fixed?",
+                    help: "TASC SI3 — defect replay. Point your generator at your own fixed-bug history: the classes it cannot re-discover are the classes it is not really covering. Agent-written tests read as thorough and are characteristically empty in the adversarial cases, so count and coverage prove nothing here.",
+                    kind: "select",
+                    options: [
+                      "Every fixed defect has a permanent case, replay measured",
+                      "Most fixes get a test — no systematic replay",
+                      "We do not know",
+                    ],
+                  },
                 ],
               },
               {
@@ -5550,7 +5643,7 @@
                 ],
               },
               {
-                label: "Agent attack surface (AC3)",
+                label: "Agent attack surface (AC3 · §7)",
                 note: "The group above secures the code; this one secures the agent. Inbound, the risk is untrusted content steering it; outbound, exfiltration — which is governed by the egress lock and credential gateway you answered earlier.",
                 items: [
                   {
@@ -5644,6 +5737,16 @@
                       "No persistent memory",
                     ],
                   },
+                  {
+                    q: "Does the threat model cover agent misreporting and fabricated evidence?",
+                    help: "TASC AC3.1 and §7. The agent can be wrong about what it tested, where it ran, or whether it had access. Evidence fabrication and self-serving harness selection belong in the threat model explicitly.",
+                    kind: "select",
+                    options: [
+                      "Explicit threat class with controls",
+                      "Mentioned but not controlled",
+                      "Not covered",
+                    ],
+                  },
                   {
                     q: "If an agent escapes its sandbox, what can it reach?",
                     help: "The joint test of everything above: scoped credentials, the egress lock and the gateway together decide whether a compromised runtime is an incident or a catastrophe.",
@@ -5793,7 +5896,7 @@
                 ],
               },
               {
-                label: "Review & merge (AC8.1 · AC5)",
+                label: "Review & merge (AC8.1 · AC5 · AC5.7)",
                 items: [
                   {
                     q: "What reviews every diff before merge?",
@@ -5820,6 +5923,23 @@
                       "Findings are advisory",
                     ],
                   },
+                  {
+                    q: "What happens when review capacity is saturated?",
+                    help: "TASC AC8.1. A reviewer queue is a control with capacity. The system needs a timeout and escalation path; it must not merge through known unadjudicated findings because the reviewer is backed up.",
+                    kind: "select",
+                    options: [
+                      "Queue limit + timeout + escalation",
+                      "Waits indefinitely",
+                      "Merges after CI is green",
+                    ],
+                    via: [
+                      "CodeRabbit / review bot",
+                      "GitHub review rules",
+                      "Human review queue",
+                      "In-house reviewer",
+                      "None",
+                    ],
+                  },
                   {
                     q: "When a gate rejects agent output, what happens?",
                     help: "TASC AC8.1. The published ideal is regenerate-from-spec — a hand-patched artifact has mixed provenance nobody can cleanly attribute. Agents iterating on their own output is the common middle ground; humans quietly patching agent work is the provenance leak.",
@@ -5852,10 +5972,20 @@
                       "Never — or we can't tell",
                     ],
                   },
+                  {
+                    q: "Which of your rules live only in agent instructions — and how often are they broken?",
+                    help: "TASC AC5.7 — instruction adherence is a rate, not a property. A rule broken once per hundred agent-days is invisible across ten supervised agents and routine across a thousand unattended ones. Inventory the prose-only rules, measure the breach rate against a declared exposure unit and window, and promote anything over tolerance to a gate.",
+                    kind: "select",
+                    options: [
+                      "Inventoried, breach rate measured, over-tolerance rules promoted to gates",
+                      "Inventoried, adherence not measured",
+                      "We assume the instructions are followed",
+                    ],
+                  },
                 ],
               },
               {
-                label: "Ship & rollback (AC8.5 · AC7.2)",
+                label: "Ship & rollback (AC8.5 · AC7.2 · AC4.8)",
                 note: "Change management — the human-shaped hole. In a human SDLC a person approved the deploy and reverted the bad one; every row here is that person's named replacement.",
                 items: [
                   {
@@ -5916,10 +6046,28 @@
                       "None",
                     ],
                   },
+                  {
+                    q: "What promotes a canary or staged rollout — observed signal, or a timer?",
+                    help: "TASC AC8.5 — the fractional ship is the factory's main source of outside feedback, not just a blast-radius trick. Promotion should depend on observed production or production-like signals inside declared thresholds; elapsed time bounds damage but teaches the system nothing, and the deploying agent saying it looks healthy is not a signal.",
+                    kind: "select",
+                    options: [
+                      "Observed signals within declared thresholds",
+                      "A human eyeballs dashboards, then promotes",
+                      "On a timer, or on the agent's own assertion",
+                      "No staged promotion",
+                    ],
+                    via: [
+                      "SLO dashboard",
+                      "Error budget checks",
+                      "Feature flag metrics",
+                      "Synthetic monitors",
+                      "None",
+                    ],
+                  },
                 ],
               },
               {
-                label: "Verify & acceptance (SI5–SI6)",
+                label: "Verify & acceptance (SI5–SI6 · §7)",
                 note: "Processing integrity: green CI proves the tests passed, not that the product works. Something has to use it.",
                 items: [
                   {
@@ -5938,6 +6086,23 @@
                     kind: "select",
                     options: ["Every PRD, before it closes", "Sometimes", "No"],
                   },
+                  {
+                    q: "Who adjudicates evidence produced by the ADS itself?",
+                    help: "TASC §7. A video, trace or report from a harness the agent selected or built is asserted evidence until an independent actor confirms it exercised the claimed path.",
+                    kind: "select",
+                    options: [
+                      "Independent actor adjudicates",
+                      "Authoring agent self-adjudicates",
+                      "No adjudication",
+                    ],
+                    via: [
+                      "Separate verifier agent",
+                      "CI artifact policy",
+                      "Human verifier",
+                      "Immutable evidence store",
+                      "None",
+                    ],
+                  },
                   {
                     q: "Does each passing verification become a regression test?",
                     help: "Without codifying, UAT proves the present and protects nothing about the future.",
@@ -5948,10 +6113,21 @@
                       "No",
                     ],
                   },
+                  {
+                    q: "When an agent proves a bug or a fix, where does the proof run?",
+                    help: "TASC §7 — evidence authenticity. Asked to prove a regression, an agent will build a harness that produces the video you asked for, on a path that is not the one under claim. A convincing artifact from a self-built environment is an assertion, not observation: require the real path, and have something other than the producer check the artifact and the code that made it.",
+                    kind: "select",
+                    options: [
+                      "The real path, reproduced independently of the producer",
+                      "The real path, checked by the producing agent",
+                      "A harness the agent built for the proof",
+                      "We take the agent's word",
+                    ],
+                  },
                 ],
               },
               {
-                label: "Observe (AC4)",
+                label: "Observe (AC4.1 · AC4.3–AC4.8)",
                 items: [
                   {
                     q: "What captures production errors?",
@@ -5991,6 +6167,40 @@
                       "Dashboards only",
                     ],
                   },
+                  {
+                    q: "Do user-reported problems enter the same signal pipeline?",
+                    help: "TASC AC4.1 and AC4.4. A user report is a first-class monitoring signal, and disagreement between user pain and quiet telemetry is itself an instrumentation gap. Report counts are not impact: the affected-users-per-report ratio varies by product, severity and channel, so impact needs triangulating against an independent signal.",
+                    kind: "select",
+                    options: [
+                      "Yes — routed through validated intake",
+                      "Humans triage separately",
+                      "Not connected",
+                    ],
+                    via: [
+                      "Support inbox",
+                      "GitHub issues",
+                      "Zendesk / Intercom",
+                      "PostHog feedback",
+                      "None",
+                    ],
+                  },
+                  {
+                    q: "Are finding sources measured for false positives or rejection rate?",
+                    help: "TASC AC4.6. A sensor, audit loop, review agent or test generator that files work without a measured rejection stage is an uncalibrated source of churn — an unfiltered stream of findings is not a gift to the recipient, it is work transferred to them. Independence means a separately controlled actor or mechanism: re-running the same one reproduces its blind spots. Trace false positives to a cause in the generator instead of dismissing instances.",
+                    kind: "select",
+                    options: [
+                      "Measured over a defined window",
+                      "Sampled occasionally",
+                      "Not measured",
+                    ],
+                    via: [
+                      "Triage labels",
+                      "Adjudication queue",
+                      "Review analytics",
+                      "In-house metrics",
+                      "None",
+                    ],
+                  },
                   {
                     q: "What tells you what users actually do?",
                     help: "Product analytics is the richest Research input there is — it routes usage patterns back to the Research gate as PRD material.",
@@ -6004,6 +6214,23 @@
                       "None",
                     ],
                   },
+                  {
+                    q: "Are cost and governance metrics reconciled against an independent source?",
+                    help: "TASC AC1.6 and AC4.7. Agent-reported spend, autonomy rate, pass rate, false-positive rate and coverage-like numbers need numerator, denominator, window, exclusions and a source-of-record check before they drive decisions.",
+                    kind: "select",
+                    options: [
+                      "Reconciled with source of record",
+                      "Metric definition only",
+                      "Agent-reported only",
+                    ],
+                    via: [
+                      "Provider billing",
+                      "Telemetry warehouse",
+                      "CI history",
+                      "Tracker history",
+                      "None",
+                    ],
+                  },
                 ],
               },
               {
@@ -6045,6 +6272,23 @@
                       "None",
                     ],
                   },
+                  {
+                    q: "Do monitoring findings close on root cause instead of symptom silence?",
+                    help: "TASC AC4.8. Closing because the alert stopped is not enough; closure needs a root-cause correction or accepted-risk record, and recurrence after closure is a control failure. Symptom-only fixes destroy the signal that made the defect class findable, so sweep for siblings sharing the cause and aim the regression test there.",
+                    kind: "select",
+                    options: [
+                      "Root-cause or accepted-risk closure",
+                      "Closed when alert clears",
+                      "Closed manually without rule",
+                    ],
+                    via: [
+                      "Incident review",
+                      "Problem-management ticket",
+                      "Runbook checklist",
+                      "In-house workflow",
+                      "None",
+                    ],
+                  },
                   {
                     q: "When the pipeline itself cannot proceed, does it escalate?",
                     help: "The recovery-required outcome: a decision-ready packet naming the broken capability, not a generic failure alert — and never a silent stall.",
@@ -6086,7 +6330,8 @@
                 ],
               },
               {
-                label: "Governance & accountability (AC1 · AC9.1 · DP)",
+                label:
+                  "Governance & accountability (AC1 · AC4.7 · AC5.6 · AC9.1 · DP)",
                 note: "The audit-trail cluster: everything a human used to vouch for by being in the room.",
                 items: [
                   {
@@ -6194,6 +6439,26 @@
                       "Not retained",
                     ],
                   },
+                  {
+                    q: "Who computes the numbers your gates and dashboards depend on?",
+                    help: "TASC AC4.7 — autonomy rate, threshold compliance, canary verdicts and conformance state are all machine-computed claims. Agents are strikingly bad at data analysis and will publish impossible numbers with confidence, so the measurement path needs independence from the work it judges plus range and invariant checks. Beating your own previous version is not evidence either.",
+                    kind: "select",
+                    options: [
+                      "An independent mechanism, with range and invariant checks",
+                      "Agents compute them, a human spot-checks",
+                      "Agents compute and publish them",
+                    ],
+                  },
+                  {
+                    q: "Are instruction-level residual risks reviewed before promotion?",
+                    help: "TASC AC5.6. A prompt, skill, hook or workflow edit can broaden tool authority, weaken rejection behavior or lower evidence standards. Those residual risks need review and disposition like code risk.",
+                    kind: "select",
+                    options: [
+                      "Reviewed and tracked to disposition",
+                      "Mentioned in PR text",
+                      "Not reviewed",
+                    ],
+                  },
                 ],
               },
             ],