@codyswann/lisa 2.307.0 → 2.309.0

package/ui/index.html

@@ -5104,7 +5104,7 @@
             type: "intake",
             groups: [
               {
-                label: "The bottom line (AC7.4)",
+                label: "The bottom line (AC7.4 · AC7.5)",
                 note: "The one number the rest of this questionnaire exists to predict.",
                 items: [
                   {
@@ -5118,6 +5118,24 @@
                       "We don't measure it",
                     ],
                   },
+                  {
+                    q: "Besides autonomy rate, do you measure whether the delivered work was any good?",
+                    help: "TASC AC7.5 — autonomy rate says how much shipped without a human, not whether it was worth shipping. A factory can post 95% autonomy while producing work the gates keep rejecting. Measure gate rejection, rework, escape to production, first-pass yield and cost per delivered item, with declared targets that only tighten.",
+                    kind: "select",
+                    options: [
+                      "Rejection, rework, escape, first-pass yield and cost — with targets",
+                      "Some of those, no declared targets",
+                      "Autonomy rate only",
+                      "Neither",
+                    ],
+                    via: [
+                      "Tracker analytics",
+                      "CI history",
+                      "Metric warehouse",
+                      "In-house dashboard",
+                      "None",
+                    ],
+                  },
                 ],
               },
               {
@@ -5227,7 +5245,8 @@
                 ],
               },
               {
-                label: "The agent's program (AC8.2–8.3 · AC4.5 · P10–P11)",
+                label:
+                  "The agent's program (AC8.2–8.3 · AC8.7 · AC8.9 · AC4.5 · P10–P11)",
                 note: "Prompts, skills and models are the factory's own source code and runtime. This group asks whether that layer is under change control — or improvised per run.",
                 items: [
                   {
@@ -5310,6 +5329,35 @@
                       "Neither pinned",
                     ],
                   },
+                  {
+                    q: "Do you have your own task suite for judging agents — and can the agents see it?",
+                    help: "TASC AC8.7 — public leaderboards measure someone else's task mix, so qualification runs on tasks you own. That suite has to exist between changes, stay representative as the work drifts, and stay unreachable by the agents being judged: a suite in an instruction file, a skill, retrievable context or vendor feedback produces a score, not a measurement.",
+                    kind: "select",
+                    options: [
+                      "Maintained suite, representativeness reviewed, agents cannot reach it",
+                      "Maintained suite, contamination not controlled",
+                      "Assembled per change",
+                      "No suite — public benchmarks or judgement",
+                    ],
+                    via: [
+                      "In-house golden tasks",
+                      "promptfoo",
+                      "Braintrust",
+                      "Langfuse / LangSmith",
+                      "None",
+                    ],
+                  },
+                  {
+                    q: "When a new agent joins the roster, how does it earn trust?",
+                    help: "TASC AC8.9 — a new reviewer, sensor or worker is a system change. Run it in shadow first: output recorded and compared against the existing control but not acted on, then adversarially exercised with deliberately defective work to confirm it catches what it claims to catch. Trust must also be revocable — an agent whose measured performance degrades goes back to shadow.",
+                    kind: "select",
+                    options: [
+                      "Shadow for a declared duration or volume, compared against the existing control, adversarially exercised, trust revocable",
+                      "Shadow period, no declared duration or comparison basis",
+                      "Reviewed by a human for a while",
+                      "Switched on and trusted",
+                    ],
+                  },
                 ],
               },
               {
@@ -5643,7 +5691,7 @@
                 ],
               },
               {
-                label: "Agent attack surface (AC3 · §7)",
+                label: "Agent attack surface (AC3 · AC3.6 · §7)",
                 note: "The group above secures the code; this one secures the agent. Inbound, the risk is untrusted content steering it; outbound, exfiltration — which is governed by the egress lock and credential gateway you answered earlier.",
                 items: [
                   {
@@ -5758,6 +5806,28 @@
                       "Unknown",
                     ],
                   },
+                  {
+                    q: "Can one agent get another to do what it is not allowed to do itself?",
+                    help: "TASC AC3.6 — draw the boundary around access and actions, not around instructions. An agent that cannot deploy but can ask a peer that can deploy has deploy authority. This is a documented failure: a constrained incident-response agent, unable to deploy by design, asked a peer over chat to push its fix, and only a human gate caught it. Enumerate reachable peers, evaluate effective authority, and make sure separation of duties survives delegation.",
+                    kind: "select",
+                    options: [
+                      "Peers enumerated, effective authority evaluated, requests recorded and attributable, duties hold against delegation",
+                      "Peers enumerated, but inter-agent requests are not recorded or attributable",
+                      "Agents coordinate freely on unaudited channels",
+                      "We have not considered agent-to-agent reach",
+                    ],
+                  },
+                  {
+                    q: "Does every threat in your model map to a control, and every control to a threat?",
+                    help: "TASC AC3.1 — an unmapped threat is an accepted risk that needs someone with standing to accept it; an unmapped control is unexplained cost. Without the mapping a threat model is a document rather than a design.",
+                    kind: "select",
+                    options: [
+                      "Mapped both ways, kept current",
+                      "Threats mapped to controls only",
+                      "Threat model exists, no mapping",
+                      "No threat model",
+                    ],
+                  },
                 ],
               },
               {
@@ -5896,7 +5966,7 @@
                 ],
               },
               {
-                label: "Review & merge (AC8.1 · AC5 · AC5.7)",
+                label: "Review & merge (AC8.1 · AC5 · AC5.7 · AC8.8)",
                 items: [
                   {
                     q: "What reviews every diff before merge?",
@@ -5982,6 +6052,16 @@
                       "We assume the instructions are followed",
                     ],
                   },
+                  {
+                    q: "Is autonomy the same everywhere, or tiered by risk?",
+                    help: "TASC AC8.8 — organizations running agent-authored code at scale tier the codebase and decide per tier what agents may do unattended, with whole codebases kept under mandatory human approval. Uniform autonomy is a decision nobody made: a tier no one chose is a tier no one is accountable for.",
+                    kind: "select",
+                    options: [
+                      "Tiered, with the actions permitted unattended and the mandatory human controls declared per tier, enforced mechanically",
+                      "Tiered, but the per-tier permissions are convention rather than enforcement",
+                      "Uniform — same autonomy everywhere",
+                    ],
+                  },
                 ],
               },
               {
@@ -6127,7 +6207,7 @@
                 ],
               },
               {
-                label: "Observe (AC4.1 · AC4.3–AC4.8)",
+                label: "Observe (AC4.1 · AC4.3–AC4.9)",
                 items: [
                   {
                     q: "What captures production errors?",
@@ -6231,6 +6311,24 @@
                       "None",
                     ],
                   },
+                  {
+                    q: "Would you notice if your agents got worse without you changing anything?",
+                    help: "TASC AC4.9 — qualification fires on your changes; capability also moves when you change nothing. A vendor reroutes a pinned alias, instruction surfaces accrete, tool behavior shifts. Sample the suite on a cadence against a recorded baseline, and be able to say whether a decline came from the vendor or from your own accumulated change.",
+                    kind: "select",
+                    options: [
+                      "Suite sampled on a cadence against a recorded baseline",
+                      "Occasional spot checks",
+                      "We would find out from incidents",
+                      "No",
+                    ],
+                    via: [
+                      "Scheduled eval runs",
+                      "Canary work items",
+                      "Metric dashboards",
+                      "In-house",
+                      "None",
+                    ],
+                  },
                 ],
               },
               {
@@ -6494,6 +6592,18 @@
                       "The agent records its own accepted risk",
                     ],
                   },
+                  {
+                    q: "Does anyone re-examine a sample of the approvals your agents make?",
+                    help: "TASC AC1.9 — logging an automated approval proves it happened, not that it was sound. Re-examine a risk-weighted sample independently, at a declared rate per tier, and feed what you find back into the approving mechanism's false-positive rate rather than just fixing the instance.",
+                    kind: "select",
+                    options: [
+                      "Each approval records its signals and reasoning; a risk-weighted sample is re-examined independently at a declared per-tier rate, findings fed back",
+                      "Sample re-examined, but the rate is not declared per tier",
+                      "Occasional spot checks",
+                      "Approvals are logged but never re-examined",
+                      "Approvals are not logged with their reasoning",
+                    ],
+                  },
                 ],
               },
             ],