@codyswann/lisa 2.305.5 → 2.307.0

package/ui/index.html

@@ -5227,7 +5227,7 @@
                 ],
               },
               {
-                label: "The agent's program (AC8.2–8.3 · AC4.5 · P9–P10)",
+                label: "The agent's program (AC8.2–8.3 · AC4.5 · P10–P11)",
                 note: "Prompts, skills and models are the factory's own source code and runtime. This group asks whether that layer is under change control — or improvised per run.",
                 items: [
                   {
@@ -5272,7 +5272,7 @@
                   },
                   {
                     q: "Do agent-program and model qualifications report repeated-run distributions?",
-                    help: "TASC P9, P10, AC8.2 and AC8.3. A single lucky run does not qualify a prompt, model or effort-level change. The evidence names the condition tested, number of repetitions, and distribution of outcomes.",
+                    help: "TASC P10, P11, AC8.2 and AC8.3. A single lucky run does not qualify a prompt, model or effort-level change. The evidence names the condition tested, number of repetitions, and distribution of outcomes.",
                     kind: "select",
                     options: [
                       "Repeated runs — distribution reported",
@@ -5300,6 +5300,16 @@
                       "Nothing",
                     ],
                   },
+                  {
+                    q: "Is the reasoning / effort level pinned alongside the model version?",
+                    help: "Effort is not monotonic — more thinking makes some tasks worse and costs more on all of them. It is part of the runtime configuration, so it belongs in the pin and in the qualification, not in whatever each agent happens to pass.",
+                    kind: "select",
+                    options: [
+                      "Model, effort and tool config all pinned",
+                      "Model version pinned only",
+                      "Neither pinned",
+                    ],
+                  },
                 ],
               },
               {
@@ -5506,6 +5516,36 @@
                       "Anyone can edit the config",
                     ],
                   },
+                  {
+                    q: "Does anything generate new test inputs, or do you only re-run the same suite?",
+                    help: "TASC SI9 — generative testing. A fixed suite only ever checks the cases someone thought of. At equal compute, generating fresh cases against a stable build explores far more of the input space than replaying one suite per change, and under agent volume the unexplored space is where quality quietly goes.",
+                    kind: "select",
+                    options: [
+                      "Continuous generation, outside the per-change gate",
+                      "Property-based tests inside CI only",
+                      "A fuzzer someone ran once",
+                      "No — fixed suite only",
+                    ],
+                    via: [
+                      "fast-check",
+                      "Hypothesis",
+                      "jqwik",
+                      "libFuzzer / AFL++",
+                      "OSS-Fuzz",
+                      "In-house generator",
+                      "None",
+                    ],
+                  },
+                  {
+                    q: "Can your tests still catch the bugs you have already fixed?",
+                    help: "TASC SI3 — defect replay. Point your generator at your own fixed-bug history: the classes it cannot re-discover are the classes it is not really covering. Agent-written tests read as thorough and are characteristically empty in the adversarial cases, so count and coverage prove nothing here.",
+                    kind: "select",
+                    options: [
+                      "Every fixed defect has a permanent case, replay measured",
+                      "Most fixes get a test — no systematic replay",
+                      "We do not know",
+                    ],
+                  },
                 ],
               },
               {
@@ -5856,7 +5896,7 @@
                 ],
               },
               {
-                label: "Review & merge (AC8.1 · AC5)",
+                label: "Review & merge (AC8.1 · AC5 · AC5.7)",
                 items: [
                   {
                     q: "What reviews every diff before merge?",
@@ -5932,6 +5972,16 @@
                       "Never — or we can't tell",
                     ],
                   },
+                  {
+                    q: "Which of your rules live only in agent instructions — and how often are they broken?",
+                    help: "TASC AC5.7 — instruction adherence is a rate, not a property. A rule broken once per hundred agent-days is invisible across ten supervised agents and routine across a thousand unattended ones. Inventory the prose-only rules, measure the breach rate against a declared exposure unit and window, and promote anything over tolerance to a gate.",
+                    kind: "select",
+                    options: [
+                      "Inventoried, breach rate measured, over-tolerance rules promoted to gates",
+                      "Inventoried, adherence not measured",
+                      "We assume the instructions are followed",
+                    ],
+                  },
                 ],
               },
               {
@@ -5997,13 +6047,13 @@
                     ],
                   },
                   {
-                    q: "What evidence promotes a canary or staged rollout?",
-                    help: "TASC AC8.5. Promotion should depend on observed production or production-like signals inside declared thresholds, not the deploying agent saying the release looks healthy.",
+                    q: "What promotes a canary or staged rollout — observed signal, or a timer?",
+                    help: "TASC AC8.5 — the fractional ship is the factory's main source of outside feedback, not just a blast-radius trick. Promotion should depend on observed production or production-like signals inside declared thresholds; elapsed time bounds damage but teaches the system nothing, and the deploying agent saying it looks healthy is not a signal.",
                     kind: "select",
                     options: [
-                      "Observed signals within thresholds",
-                      "Manual spot-check",
-                      "Agent assertion",
+                      "Observed signals within declared thresholds",
+                      "A human eyeballs dashboards, then promotes",
+                      "On a timer, or on the agent's own assertion",
                       "No staged promotion",
                     ],
                     via: [
@@ -6063,10 +6113,21 @@
                       "No",
                     ],
                   },
+                  {
+                    q: "When an agent proves a bug or a fix, where does the proof run?",
+                    help: "TASC §7 — evidence authenticity. Asked to prove a regression, an agent will build a harness that produces the video you asked for, on a path that is not the one under claim. A convincing artifact from a self-built environment is an assertion, not observation: require the real path, and have something other than the producer check the artifact and the code that made it.",
+                    kind: "select",
+                    options: [
+                      "The real path, reproduced independently of the producer",
+                      "The real path, checked by the producing agent",
+                      "A harness the agent built for the proof",
+                      "We take the agent's word",
+                    ],
+                  },
                 ],
               },
               {
-                label: "Observe (AC4.1 · AC4.3–AC4.7)",
+                label: "Observe (AC4.1 · AC4.3–AC4.8)",
                 items: [
                   {
                     q: "What captures production errors?",
@@ -6108,7 +6169,7 @@
                   },
                   {
                     q: "Do user-reported problems enter the same signal pipeline?",
-                    help: "TASC AC4.1 and AC4.4. A user report is a first-class monitoring signal, and disagreement between user pain and quiet telemetry is itself an instrumentation gap.",
+                    help: "TASC AC4.1 and AC4.4. A user report is a first-class monitoring signal, and disagreement between user pain and quiet telemetry is itself an instrumentation gap. Report counts are not impact: the affected-users-per-report ratio varies by product, severity and channel, so impact needs triangulating against an independent signal.",
                     kind: "select",
                     options: [
                       "Yes — routed through validated intake",
@@ -6125,7 +6186,7 @@
                   },
                   {
                     q: "Are finding sources measured for false positives or rejection rate?",
-                    help: "TASC AC4.6. A sensor, audit loop, review agent or test generator that files work without a measured rejection stage is an uncalibrated source of churn.",
+                    help: "TASC AC4.6. A sensor, audit loop, review agent or test generator that files work without a measured rejection stage is an uncalibrated source of churn — an unfiltered stream of findings is not a gift to the recipient, it is work transferred to them. Independence means a separately controlled actor or mechanism: re-running the same one reproduces its blind spots. Trace false positives to a cause in the generator instead of dismissing instances.",
                     kind: "select",
                     options: [
                       "Measured over a defined window",
@@ -6213,7 +6274,7 @@
                   },
                   {
                     q: "Do monitoring findings close on root cause instead of symptom silence?",
-                    help: "TASC AC4.8. Closing because the alert stopped is not enough; closure needs a root-cause correction or accepted-risk record, and recurrence after closure is a control failure.",
+                    help: "TASC AC4.8. Closing because the alert stopped is not enough; closure needs a root-cause correction or accepted-risk record, and recurrence after closure is a control failure. Symptom-only fixes destroy the signal that made the defect class findable, so sweep for siblings sharing the cause and aim the regression test there.",
                     kind: "select",
                     options: [
                       "Root-cause or accepted-risk closure",
@@ -6266,11 +6327,22 @@
                       "No convention",
                     ],
                   },
+                  {
+                    q: "When an agent breaks production, does the incident record say who answers for it — and which gate should have caught it?",
+                    help: 'TASC AC7.2 — "an agent did it" describes the mechanism, it does not dispose of the incident. The record names the accountable party, the generation lineage that produced the change (procedure, prompt version, pinned model), and the control that should have prevented it; where no such control existed, that absence becomes a learning-promotion candidate.',
+                    kind: "select",
+                    options: [
+                      "Names the party, the lineage and the missing control",
+                      "Names the party and the fix",
+                      "Records the fix only",
+                      "No incident record",
+                    ],
+                  },
                 ],
               },
               {
                 label:
-                  "Governance & accountability (AC1.2 · AC1.6 · AC5.6 · AC9.1 · DP)",
+                  "Governance & accountability (AC1 · AC4.7 · AC5.6 · AC9.1 · DP)",
                 note: "The audit-trail cluster: everything a human used to vouch for by being in the room.",
                 items: [
                   {
@@ -6378,6 +6450,16 @@
                       "Not retained",
                     ],
                   },
+                  {
+                    q: "Who computes the numbers your gates and dashboards depend on?",
+                    help: "TASC AC4.7 — autonomy rate, threshold compliance, canary verdicts and conformance state are all machine-computed claims. Agents are strikingly bad at data analysis and will publish impossible numbers with confidence, so the measurement path needs independence from the work it judges plus range and invariant checks. Beating your own previous version is not evidence either.",
+                    kind: "select",
+                    options: [
+                      "An independent mechanism, with range and invariant checks",
+                      "Agents compute them, a human spot-checks",
+                      "Agents compute and publish them",
+                    ],
+                  },
                   {
                     q: "Are instruction-level residual risks reviewed before promotion?",
                     help: "TASC AC5.6. A prompt, skill, hook or workflow edit can broaden tool authority, weaken rejection behavior or lower evidence standards. Those residual risks need review and disposition like code risk.",
@@ -6388,6 +6470,30 @@
                       "Not reviewed",
                     ],
                   },
+                  {
+                    q: "Who is accountable when an agent breaks something?",
+                    help: "TASC AC1.7 — attribution and accountability are different obligations. AC1.1 tells you which agent acted; this tells you which human answers for it. Recorded per system, per loop, per deploy target and per human control, before anything goes wrong — with a backup, because accountability must never be vacant. An agent cannot hold it, and a team alias with no person behind it does not count.",
+                    kind: "select",
+                    options: [
+                      "A named person or role per scope, with a backup or succession path",
+                      "A named person or role per scope, no backup or succession",
+                      "One name for the whole system",
+                      "A team alias owns it",
+                      "An agent or automation is named as accountable",
+                      "Nobody named",
+                    ],
+                  },
+                  {
+                    q: "Who can accept a risk the system decided not to fix?",
+                    help: "TASC AC1.8 — accepted-risk records are the escape hatch in four criteria (symptom-only closure, advisory controls over tolerance, infeasible replay, unsafe reproducers). If an agent can write its own, they are not controls, they are bypasses with better paperwork. Each acceptance names the human with standing, the scope, the reason and an expiry that forces re-review.",
+                    kind: "select",
+                    options: [
+                      "A named human with declared standing, time-bounded",
+                      "A named human, no expiry",
+                      "Whoever is around when it comes up",
+                      "The agent records its own accepted risk",
+                    ],
+                  },
                 ],
               },
             ],