bmad-method-test-architecture-enterprise 3.2.0 → 3.4.0

package/cli/lib/evaluate/retain-skill-artifacts.js

@@ -0,0 +1,78 @@
+/** Preserve generated workspaces and raw observations before eval cleanup. */
+'use strict';
+const fs = require('node:fs');
+const path = require('node:path');
+const writeText = (file, text) => fs.promises.writeFile(file, text);
+/** Observe each attempt before retry cleanup can discard its workspace. */
+function captureProbe(port, record) {
+  return {
+    async probe(request, signal) {
+      try {
+        const observation = await port.probe(request, signal);
+        record(observation);
+        return observation;
+      } catch (error) {
+        record({ fault: { name: error.name, code: error.code, message: error.message, detail: error.detail } });
+        throw error;
+      }
+    },
+  };
+}
+/** Retain exact workspace bytes, prompt, raw observation and live provenance. */
+async function retainSkillArtifacts({
+  artifactsDir,
+  workspace,
+  caseId,
+  agent,
+  model,
+  repetition,
+  attempt,
+  prompt,
+  observation,
+  provenance = {},
+  preserveOnFailure = false,
+}) {
+  if (!artifactsDir) return null;
+  const root = path.resolve(artifactsDir);
+  fs.mkdirSync(root, { recursive: true });
+  const label = `${agent}-${caseId}-run-${repetition}-attempt-${attempt}`.replaceAll(/[^a-zA-Z0-9_-]/g, '_');
+  const destination = fs.mkdtempSync(path.join(root, `${label}-`));
+  try {
+    fs.cpSync(workspace.dir, path.join(destination, 'workspace'), { recursive: true });
+    await writeText(path.join(destination, 'prompt.txt'), prompt);
+    if (observation !== undefined) await writeText(path.join(destination, 'observation.json'), `${JSON.stringify(observation, null, 2)}\n`);
+    await writeText(
+      path.join(destination, 'provenance.json'),
+      `${JSON.stringify({ mode: 'live', caseId, agent, requestedModel: model, repetition, attempt, capturedAt: new Date().toISOString(), originalWorkspace: workspace.dir, ...provenance }, null, 2)}\n`,
+    );
+    process.stderr.write(`retained skill artifacts: ${destination}\n`);
+    return destination;
+  } catch (error) {
+    if (preserveOnFailure) {
+      error.retainedWorkspace = workspace.dir;
+      error.partialArchive = destination;
+    } else fs.rmSync(destination, { recursive: true, force: true });
+    throw error;
+  }
+}
+/** Clean the attempt; explicit experiment capture preserves it when retention fails. */
+async function finishSkillAttempt({ retain, workspace, runError, remove = fs.rmSync, preserveOnFailure = false }) {
+  let failure = runError;
+  let retained = false;
+  try {
+    await retain();
+    retained = true;
+  } catch (error) {
+    if (failure) process.stderr.write(`artifact retention failed: ${error.message}\n`);
+    else failure = error;
+  } finally {
+    try {
+      if (retained || !preserveOnFailure) remove(workspace.dir, { recursive: true, force: true });
+    } catch (error) {
+      if (failure) process.stderr.write(`attempt cleanup failed: ${error.message}\n`);
+      else failure = error;
+    }
+  }
+  if (failure && failure !== runError) throw failure;
+}
+module.exports = { retainSkillArtifacts, captureProbe, finishSkillAttempt };

package/cli/lib/evaluate/test-design-corpus/clean/docs/epics/epic-9-last-sync-indicator.md

@@ -0,0 +1,47 @@
+# Epic 9: Show the last successful sync time on the technician home screen
+
+## Context
+
+Technicians ask dispatch whether their device is up to date. The answer already exists on the
+device: the sync engine records the completion time of every successful sync. Nothing surfaces
+it, so the question is asked over the radio instead.
+
+This epic renders that value on the home screen. It adds one row to a screen that is already
+rendered, and it reads a field the sync engine already writes.
+
+## Stories
+
+### 9.1 Render the last successful sync time
+
+The home screen reads the existing `lastSuccessfulSyncAt` field from the local settings store
+and renders it as a relative time, for example "synced 4 minutes ago".
+
+The read is a single local key lookup on a store the screen already opens for other fields. No
+network call is added, nothing is written, and no new field is stored.
+
+When the field is absent, which is the state of a device that has never completed a sync, the
+row is not rendered at all and the rest of the screen is unchanged.
+
+### 9.2 Keep the value current while the screen is open
+
+The row re-renders when the sync engine publishes a completion event, which it already
+publishes for the existing progress spinner. No polling and no timer is added.
+
+## Constraints already satisfied
+
+- The value is a timestamp the device wrote about itself. It is shown to the person holding
+  the device, so the change exposes nothing to anyone who could not already see it, and it
+  adds no personal data to the screen.
+- The change writes nothing. The settings store is opened read-only for this field, so no
+  order, queue entry or setting can be altered or lost by it.
+- The change adds no network call and no storage. The single local key lookup runs on a screen
+  that is already open, so the screen's existing render budget is unchanged.
+- The row ships behind `home_last_sync_row`, an existing feature flag the release process
+  already toggles per organization, so it can be turned off without a new build.
+
+## Acceptance criteria
+
+- A device that has completed a sync shows the relative time of the most recent one.
+- A device that has never completed a sync shows the home screen with no sync row.
+- The displayed value updates when a sync completes while the screen is open.
+- Turning `home_last_sync_row` off restores the previous home screen exactly.

package/cli/lib/evaluate/test-design-corpus/ground-truth.json

@@ -0,0 +1,222 @@
+{
+  "$comment": [
+    "The expected answer for the test-design behavioral eval, and the file the agent must never see.",
+    "",
+    "Two fixture sets, each one epic document and each its own case. The seeded set carries five",
+    "material risks the epic actually supports and four plausible risks it rules out in as many words.",
+    "The clean set carries a feature whose genuine risk set is small, which is what stops recall alone",
+    "from rewarding a run that reports everything it can think of.",
+    "",
+    "GROUNDING IS A SUBSTRING TEST, NOT A JUDGMENT",
+    "",
+    "Every material risk names the sentence of the epic that supports it in `groundingQuote`, and every",
+    "unsupported risk names the sentence that rules it out in `exclusionQuote`. --validate-only checks",
+    "each quote is present in the epic it names, with runs of whitespace collapsed on both sides so a",
+    "quote that spans a wrapped line still resolves. A fixture claiming the description supports a risk",
+    "the description never mentions fails before a model call is spent.",
+    "",
+    "A reported risk row matches a declared risk when its category is one the risk admits and every",
+    "group in `anyOf` has at least one token present in the row's description, case-insensitively. The",
+    "groups are an AND of ORs on purpose: one distinctive word is a coincidence and two are a claim.",
+    "Material risks are matched first and a matched row is consumed, so a row that names a real risk in",
+    "words that also appear in an exclusion is never counted twice.",
+    "",
+    "SEVERITY RANKS CONSTRAIN PAIRS, NOT LEVELS",
+    "",
+    "`severityRank` is an ordering and not a priority. Priority is a separate judgment the workflow",
+    "makes from business impact, user reach and workaround availability, which the risk score informs",
+    "and does not determine; `{tea-knowledge}/test-priorities-matrix.md` states that directly and",
+    "`{tea-knowledge}/probability-impact.md` repeats it. Asserting an absolute priority per risk",
+    "would measure this fixture's opinion rather than the workflow's rule, so what is checked instead is",
+    "that every pair of risks whose ranks are strictly ordered carries priorities in the same order.",
+    "Equal ranks constrain nothing and produce no pair.",
+    "",
+    "PROVENANCE",
+    "",
+    "The six categories, the 1-3 probability and impact scales and the P x I score are the workflow's",
+    "own, read from steps-c/step-03-risk-and-testability.md section 2 and from the two knowledge",
+    "fragments it names. The five test levels are the four steps-c/step-04-coverage-plan.md section 1",
+    "declares plus Integration, which {tea-knowledge}/test-levels-framework.md treats as a first",
+    "class level throughout. Both epics were written for this corpus and describe no real system.",
+    "",
+    "EVERY SET DECLARES A CEILING",
+    "",
+    "`maxRisks` bounds how many risks a run may report, and it is on both sets. It was on the clean",
+    "control alone, which left over-reporting on the seeded set unbounded: four risks this fixture never",
+    "anticipated could be appended to an otherwise correct run and every threshold still passed, because",
+    "precision counts only the rows matching a risk the epic rules out in as many words. The seeded",
+    "ceiling is 8 against 5 material risks, which leaves room for a real risk this fixture did not",
+    "foresee and stops a register that lists everything."
+  ],
+  "riskCategories": ["TECH", "SEC", "PERF", "DATA", "BUS", "OPS"],
+  "testLevels": ["E2E", "API", "Component", "Integration", "Unit"],
+  "priorities": ["P0", "P1", "P2", "P3"],
+  "fixtureSets": [
+    {
+      "id": "seeded-offline-order-capture",
+      "root": "seeded",
+      "projectRoot": "field-order-capture",
+      "epicNum": "7",
+      "epicFile": "docs/epics/epic-7-offline-order-capture.md",
+      "maxRisks": 8,
+      "materialRisks": [
+        {
+          "id": "sec-unencrypted-offline-queue",
+          "categories": ["SEC", "DATA"],
+          "severityRank": 1,
+          "summary": "The outbound queue holds a card reference and a billing address unencrypted on the device.",
+          "groundingQuote": "It holds the full order payload, including the customer's stored card reference and billing address, and it is not encrypted at rest.",
+          "anyOf": [
+            ["unencrypted", "not encrypted", "plaintext", "plain text", "cleartext", "encryption", "encrypt"],
+            ["queue", "local", "device", "sqlite", "on-disk", "at rest", "storage", "stored"]
+          ],
+          "acceptableCoverage": ["Unit", "Integration", "E2E"],
+          "exampleDescription": "The outbound queue is stored unencrypted on the device, so anyone reaching the local file can read the stored card reference."
+        },
+        {
+          "id": "data-sync-conflict-overwrite",
+          "categories": ["DATA", "TECH", "BUS"],
+          "severityRank": 2,
+          "summary": "Two offline edits to one work order silently overwrite each other, because the server orders by arrival and checks no version.",
+          "groundingQuote": "The server applies queued edits in the order they arrive and performs no version check and no timestamp check, so a later arrival overwrites an earlier one",
+          "anyOf": [
+            ["conflict", "overwrite", "overwrit", "clobber", "erase", "discard", "last write", "last-write", "lost update"],
+            ["concurrent", "simultaneous", "two technicians", "second device", "same work order", "version", "timestamp"]
+          ],
+          "acceptableCoverage": ["API", "Integration", "E2E"],
+          "exampleDescription": "Concurrent offline edits to the same work order overwrite each other on sync, because the server applies queued edits by arrival with no version check."
+        },
+        {
+          "id": "tech-uncapped-retry-loop",
+          "categories": ["TECH", "OPS", "PERF"],
+          "severityRank": 3,
+          "summary": "A payload the server will never accept is retried forever with no backoff, cap or dead-letter path.",
+          "groundingQuote": "There is no backoff, no attempt cap, and no dead-letter path",
+          "anyOf": [
+            ["retry", "retries", "retried", "backoff", "back-off", "dead letter", "dead-letter"],
+            ["cap", "unbounded", "indefinit", "infinite", "loop", "storm", "exhaust", "no limit", "forever"]
+          ],
+          "acceptableCoverage": ["Unit", "Integration", "API"],
+          "exampleDescription": "A rejected payload is retried forever with no backoff and no attempt cap, so one bad item loops until the application is killed."
+        },
+        {
+          "id": "perf-backlog-sync-budget",
+          "categories": ["PERF", "TECH"],
+          "severityRank": 4,
+          "summary": "A full day's backlog of 2,000 queued items has to finish syncing within 30 seconds on mid-range hardware.",
+          "groundingQuote": "requires a full backlog of that size to finish syncing within 30 seconds of regaining connectivity",
+          "anyOf": [
+            ["backlog", "2,000", "2000", "bulk", "volume", "throughput", "large queue"],
+            ["30 second", "30s", "latency", "slow", "budget", "within 30", "sync time", "duration"]
+          ],
+          "acceptableCoverage": ["API", "Integration", "E2E"],
+          "exampleDescription": "A 2000 item backlog may not finish syncing inside the 30 second budget on the mid-range hardware the fleet carries."
+        },
+        {
+          "id": "ops-no-release-kill-switch",
+          "categories": ["OPS", "BUS"],
+          "severityRank": 4,
+          "summary": "The feature ships on for everyone with no flag, so recovery costs a store review of one to three days.",
+          "groundingQuote": "There is no feature flag, no staged rollout, and no way to disable it without shipping a new build",
+          "anyOf": [
+            ["feature flag", "no flag", "kill switch", "kill-switch", "toggle", "disable", "disabled", "disabling"],
+            ["rollout", "release", "rollback", "roll back", "deploy", "staged", "store review", "store build", "new build", "revert"]
+          ],
+          "acceptableCoverage": ["E2E", "Integration", "Component"],
+          "exampleDescription": "There is no feature flag, so a bad release cannot be turned off without shipping a new store build."
+        }
+      ],
+      "unsupportedRisks": [
+        {
+          "id": "payment-gateway-failure",
+          "summary": "A card-processing failure. The epic does not call the billing service and does not change when it charges.",
+          "exclusionQuote": "the existing billing service charges the stored card reference on its own nightly schedule, and this epic does not touch it, call it, or change when it runs",
+          "anyOf": [
+            ["payment gateway", "gateway", "charging", "charge fail", "settlement", "card processor", "payment provider"],
+            ["outage", "unavailable", "declined", "decline", "third-party", "third party", "timeout", "down"]
+          ],
+          "exampleDescription": "The payment gateway is unavailable when the queue uploads, so the customer is never charged for the work."
+        },
+        {
+          "id": "cross-tenant-data-leak",
+          "summary": "One organization reading another's orders. Each installation has its own database and shares no storage.",
+          "exclusionQuote": "one customer organization per installation, with its own database and its own application servers",
+          "anyOf": [
+            ["tenant", "cross-tenant", "multi-tenant", "other organizations", "another organization", "other customers"],
+            ["leak", "leakage", "isolation", "bleed", "exposure", "boundary", "segregat"]
+          ],
+          "exampleDescription": "A queued order from one organization could be applied to another organization's data, so tenant isolation has to be proven."
+        },
+        {
+          "id": "browser-and-accessibility-regression",
+          "summary": "A browser or accessibility regression. The client is a native application with no browser surface.",
+          "exclusionQuote": "There is no browser surface and no web view in this epic",
+          "anyOf": [
+            ["accessibility", "a11y", "screen reader", "wcag", "browser", "cross-browser", "safari", "firefox", "responsive layout"]
+          ],
+          "exampleDescription": "The order screen has to stay usable with a screen reader and has to render correctly in every supported browser."
+        },
+        {
+          "id": "data-residency-compliance",
+          "summary": "A residency or GDPR exposure. Residency is enforced by a routing layer this epic does not modify.",
+          "exclusionQuote": "Data residency is unchanged and is enforced by the platform's regional routing layer, which this epic does not modify.",
+          "anyOf": [
+            ["residency", "gdpr", "sovereignty", "data protection regulation"],
+            ["compli", "regulat", "jurisdiction", "cross-border", "lawful", "legal"]
+          ],
+          "exampleDescription": "Queued order data could leave its jurisdiction before it is uploaded, which is a GDPR and data residency exposure."
+        }
+      ]
+    },
+    {
+      "id": "clean-last-sync-indicator",
+      "root": "clean",
+      "projectRoot": "field-sync-indicator",
+      "epicNum": "9",
+      "epicFile": "docs/epics/epic-9-last-sync-indicator.md",
+      "maxRisks": 3,
+      "unsupportedRisks": [
+        {
+          "id": "data-loss-or-corruption",
+          "summary": "Losing or corrupting stored data. The change opens the settings store read-only and writes nothing.",
+          "exclusionQuote": "The settings store is opened read-only for this field, so no order, queue entry or setting can be altered or lost by it.",
+          "anyOf": [
+            ["corrupt", "data loss", "lose data", "lost", "overwrite", "delete", "truncat"],
+            ["store", "storage", "record", "field", "database", "setting", "queue"]
+          ],
+          "exampleDescription": "Reading the settings store could corrupt the stored record and drop a queued entry."
+        },
+        {
+          "id": "unauthorized-exposure",
+          "summary": "Exposing data to someone who should not see it. The value is the device's own timestamp shown to its holder.",
+          "exclusionQuote": "it is shown to the person holding the device, so the change exposes nothing to anyone who could not already see it, and it adds no personal data to the screen",
+          "anyOf": [
+            ["unauthori", "leak", "exposure", "expose", "pii", "personal data", "sensitive", "credential"],
+            ["access", "permission", "auth", "visible", "disclos", "privacy"]
+          ],
+          "exampleDescription": "The sync timestamp could expose sensitive personal data to an unauthorized person with access to the screen."
+        },
+        {
+          "id": "render-performance-degradation",
+          "summary": "A performance regression. One local key lookup on a screen that is already open, with no network call.",
+          "exclusionQuote": "The single local key lookup runs on a screen that is already open, so the screen's existing render budget is unchanged.",
+          "anyOf": [
+            ["performance", "slow", "latency", "jank", "frame", "degrad", "throughput", "render budget"],
+            ["load", "render", "screen", "startup", "budget", "network call", "poll"]
+          ],
+          "exampleDescription": "The extra read could slow the home screen render and degrade startup latency."
+        },
+        {
+          "id": "rollout-without-kill-switch",
+          "summary": "An unrecoverable rollout. The row ships behind an existing flag the release process already toggles.",
+          "exclusionQuote": "an existing feature flag the release process already toggles per organization, so it can be turned off without a new build",
+          "anyOf": [
+            ["kill switch", "kill-switch", "no feature flag", "cannot be disabled", "unrecoverable", "no way to disable"],
+            ["rollout", "release", "rollback", "roll back", "deploy", "revert"]
+          ],
+          "exampleDescription": "The new row cannot be disabled without a new build, so a bad rollout has no kill switch."
+        }
+      ]
+    }
+  ]
+}

package/cli/lib/evaluate/test-design-corpus/native-identity.json

@@ -0,0 +1,4 @@
+{
+  "suiteDigest": "sha256:56e7e6dd92b0e73a93d88f4211ae8cf7aa530de38dfc26bc6505302e0a0d3b1b",
+  "promptDigest": "sha256:ea2f030e94d5efae56132ff6f1d215c5f6d976752b1b0cf424009ca2936a816b"
+}

package/cli/lib/evaluate/test-design-corpus/seeded/docs/epics/epic-7-offline-order-capture.md

@@ -0,0 +1,75 @@
+# Epic 7: Offline order capture for field technicians
+
+## Context
+
+Field technicians work in basements, plant rooms and rural sites where the mobile network
+is unavailable for hours at a time. Today the order screen requires connectivity, so a
+technician either waits for signal or writes the order on paper and re-keys it that evening.
+This epic lets the technician capture and edit an order while offline and have it reach the
+server when connectivity returns.
+
+The technician client is a native Android application. There is no browser surface and no
+web view in this epic, and nothing in it changes how the order screen is rendered.
+
+The deployment is single-tenant: one customer organization per installation, with its own
+database and its own application servers. No installation can read another installation's
+data, because no storage is shared between them.
+
+Data residency is unchanged and is enforced by the platform's regional routing layer, which
+this epic does not modify.
+
+Charging is unchanged: the existing billing service charges the stored card reference on its
+own nightly schedule, and this epic does not touch it, call it, or change when it runs.
+
+## Stories
+
+### 7.1 Capture an order while offline
+
+The technician opens an existing work order, adds parts and labour lines, and saves. When the
+device has no connectivity the save writes to a local outbound queue instead of the server,
+and the order screen shows the order as captured.
+
+The outbound queue is a plain SQLite file in the application's data directory. It holds the
+full order payload, including the customer's stored card reference and billing address, and it
+is not encrypted at rest. The device's own screen lock is the only thing in front of it.
+
+### 7.2 Edit a queued order before it syncs
+
+A queued order stays editable. Each edit appends a new entry to the outbound queue rather than
+replacing the earlier one.
+
+Two technicians can be assigned to the same work order and can both be offline at once. The
+server applies queued edits in the order they arrive and performs no version check and no
+timestamp check, so a later arrival overwrites an earlier one and the earlier technician's
+lines are lost with no record that they existed.
+
+### 7.3 Sync the queue when connectivity returns
+
+The client uploads the queue when the operating system reports a usable network.
+
+A technician working a full day offline accumulates up to 2,000 queued items. The product
+requires a full backlog of that size to finish syncing within 30 seconds of regaining
+connectivity, on the mid-range devices the field fleet actually carries.
+
+A failed upload is retried immediately and indefinitely until the server accepts it. There is
+no backoff, no attempt cap, and no dead-letter path, so a payload the server will never accept
+is uploaded again as fast as the network allows for as long as the application is running.
+
+### 7.4 Release
+
+The feature ships enabled for every technician in a single release. There is no feature flag,
+no staged rollout, and no way to disable it without shipping a new build through the store
+review queue, which takes between one and three days.
+
+## Acceptance criteria
+
+- A technician with no connectivity can save an order and see it listed as captured.
+- A captured order remains editable until it has been accepted by the server.
+- The queue uploads without the technician taking any action once the network returns.
+- An order that reached the server appears on the dispatcher's web console unchanged.
+
+## Out of scope
+
+- Offline capture of anything other than work orders.
+- Any change to the dispatcher's web console.
+- Any change to how or when a card is charged.