@plainconceptsplatform/workflows 0.20.2 → 0.20.3

package/loops/actions/verify-app-errors/verify-app-errors.mjs

@@ -0,0 +1,216 @@
+// Managed by @plainconceptsplatform/workflows. Source: loops/actions/verify-app-errors/verify-app-errors.mjs. Update with workflows update --force; consumer edits may be overwritten.
+//
+// Executes the real grouping and redaction against rows shaped the way Application Insights
+// actually returns them.
+//
+// It runs the shipped module rather than grepping it. The workflow-error report's logic lives
+// inside a `script:` string in its action.yml, so nothing can execute it and its tests are
+// greps; that was the reason to put this one in a module, and this is the payoff.
+
+import {
+  FINDING_FIELDS,
+  MAX_FRAMES,
+  clip,
+  fingerprint,
+  leaks,
+  ownFrames,
+  prepare,
+  render,
+  scrub,
+  toFindings,
+  unexpectedFields,
+} from "../collect-app-errors/group-and-redact.mjs";
+
+let pass = 0;
+let fail = 0;
+
+function check(label, condition, detail = "") {
+  if (condition) {
+    pass += 1;
+    return;
+  }
+
+  fail += 1;
+  console.error(`FAIL: ${label}${detail ? `\n  ${detail}` : ""}`);
+}
+
+function section(title) {
+  console.log(`\n── ${title} ──`);
+}
+
+/** A row in the shape `az monitor log-analytics query` returns, from a real exception. */
+function row(overrides = {}) {
+  return {
+    ProblemId: "System.InvalidOperationException at Pliny.Application.Agents.RunExecutor.ExecuteAsync",
+    ExceptionType: "System.InvalidOperationException",
+    OperationName: "POST Runs/Start",
+    AppRoleName: "plinybot-pre-app-01",
+    Occurrences: 42,
+    Operations: 3,
+    FirstSeen: "2026-09-13T01:10:00Z",
+    LastSeen: "2026-09-13T06:40:00Z",
+    AnyOperationId: "6f1c1b0e9a2f4d55",
+    AnyDetails: [
+      {
+        parsedStack: [
+          { method: "Pliny.Application.Agents.RunExecutor.ExecuteAsync", fileName: "/home/vsts/work/1/s/src/RunExecutor.cs", line: 512 },
+          { method: "Microsoft.AspNetCore.Routing.EndpointMiddleware.Invoke", fileName: "/_/src/Http/Routing.cs", line: 90 },
+        ],
+      },
+    ],
+    ...overrides,
+  };
+}
+
+const options = { environment: "pre", lookbackHours: "24", ownCodePrefix: "Pliny" };
+
+section("Scrubbing");
+
+check(
+  "a GUID is masked",
+  scrub("run 6f1c1b0e-9a2f-4d55-8b3e-1f2a3b4c5d6e failed") === "run {guid} failed",
+  scrub("run 6f1c1b0e-9a2f-4d55-8b3e-1f2a3b4c5d6e failed"));
+
+check(
+  "every GUID is masked, not just the first",
+  !/[0-9a-f]{8}-[0-9a-f]{4}/i.test(scrub("a 6f1c1b0e-9a2f-4d55-8b3e-1f2a3b4c5d6e b 7f1c1b0e-9a2f-4d55-8b3e-1f2a3b4c5d6e")));
+
+check("an email address is masked", scrub("from someone@example.com") === "from {email}");
+
+check(
+  "a build-machine path is masked",
+  clip("at /home/vsts/work/1/s/src/RunExecutor.cs line 5") === "at {path} line 5",
+  clip("at /home/vsts/work/1/s/src/RunExecutor.cs line 5"));
+
+check(
+  "a query string is masked but the address survives",
+  clip("GET https://api.example.com/v1/items?token=abc123") === "GET https://api.example.com/v1/items?{query}",
+  clip("GET https://api.example.com/v1/items?token=abc123"));
+
+check(
+  "a pattern split across a newline is still caught",
+  !/example\.com/.test(scrub("mail\nto")) && clip("someone@example.com\nnext") === "{email} next",
+  clip("someone@example.com\nnext"));
+
+section("Frames");
+
+check("only our own frames survive", ownFrames(row().AnyDetails, "Pliny").length === 1);
+
+check(
+  "a frame carries the method and never the path",
+  ownFrames(row().AnyDetails, "Pliny")[0] === "Pliny.Application.Agents.RunExecutor.ExecuteAsync");
+
+check(
+  "an empty prefix publishes no frames at all",
+  ownFrames(row().AnyDetails, "").length === 0);
+
+check(
+  "frames are capped",
+  ownFrames(
+    [{ parsedStack: Array.from({ length: 40 }, (_, i) => ({ method: `Pliny.Frame${i}` })) }],
+    "Pliny",
+  ).length === MAX_FRAMES);
+
+check("a row with no details does not throw", ownFrames(undefined, "Pliny").length === 0);
+
+section("Fingerprints");
+
+check(
+  "the same problem in the same place is the same fingerprint",
+  fingerprint("a", "b", "pre") === fingerprint("a", "b", "pre"));
+
+check(
+  "the same problem in another environment is a different one",
+  fingerprint("a", "b", "pre") !== fingerprint("a", "b", "pro"));
+
+check("a fingerprint is short enough to read", fingerprint("a", "b", "pre").length === 12);
+
+section("Thresholds");
+
+check(
+  "a finding below the floor is dropped",
+  toFindings([row({ Occurrences: 2 })], { ...options, minOccurrences: 5 }).length === 0);
+
+check(
+  "a finding on the floor is kept",
+  toFindings([row({ Occurrences: 5 })], { ...options, minOccurrences: 5 }).length === 1);
+
+check(
+  "the loudest comes first",
+  toFindings([row({ Occurrences: 5, ProblemId: "quiet" }), row({ Occurrences: 99, ProblemId: "loud" })], options)[0]
+    .problemId === "loud");
+
+section("The field allowlist");
+
+check(
+  "a finding carries exactly the agreed fields",
+  unexpectedFields(toFindings([row()], options)[0]).length === 0);
+
+check(
+  "a field nobody agreed to publish is named",
+  unexpectedFields({ ...toFindings([row()], options)[0], runId: "6f1c1b0e" }).join() === "runId");
+
+check(
+  "an ordinary batch is not refused",
+  prepare([row()], options).refused === "" && prepare([row()], options).prepared.length === 1);
+
+check(
+  "run_id is not a reportable field",
+  !FINDING_FIELDS.includes("runId") && !FINDING_FIELDS.includes("run_id"));
+
+section("Rendering and the scanner");
+
+const rendered = render(toFindings([row()], options)[0], options);
+
+check("the marker is the first line", rendered.body.startsWith("<!-- pcp-app-error: "));
+
+check("the marker carries the fingerprint", rendered.body.includes(toFindings([row()], options)[0].fingerprint));
+
+check("the count is said plainly", rendered.body.includes("**42 occurrence(s)**"));
+
+check("a clean report trips nothing", leaks(`${rendered.title}\n${rendered.body}`).length === 0,
+  leaks(`${rendered.title}\n${rendered.body}`).join("; "));
+
+check(
+  "no build-machine path reaches the body",
+  !rendered.body.includes("/home/vsts"));
+
+// Two ways to have no frames, and they ask the reader for different things. Pliny-Bot #191 was
+// filed with OWN_CODE_PREFIX set to "Pliny" and still said the repository had not said which
+// assemblies were its own -- it sent a reader to configure a knob that had been set the day
+// before. Only the unconfigured case was covered here, which is why the wording survived.
+check(
+  "no prefix configured says which knob to set",
+  render(toFindings([row()], { ...options, ownCodePrefix: "" })[0], { ...options, ownCodePrefix: "" })
+    .body.includes("has not said which assemblies are its own"));
+
+check(
+  "a prefix that matched nothing does not claim the prefix is missing",
+  !render(toFindings([row()], { ...options, ownCodePrefix: "NoSuchAssembly" })[0], { ...options, ownCodePrefix: "NoSuchAssembly" })
+    .body.includes("has not said which assemblies are its own"));
+
+check(
+  "a prefix that matched nothing names the prefix it tried",
+  render(toFindings([row()], { ...options, ownCodePrefix: "NoSuchAssembly" })[0], { ...options, ownCodePrefix: "NoSuchAssembly" })
+    .body.includes("No frame in this exception belongs to `NoSuchAssembly`"));
+
+section("Fail-closed, but not fail-dead");
+
+const mixed = prepare(
+  [row(), row({ ProblemId: "Secret", ExceptionType: "AccountKey=abc123def456;Other" })],
+  options);
+
+check("the clean report is still prepared", mixed.prepared.length === 1, JSON.stringify(mixed.withheld));
+
+check("the tripped report is withheld", mixed.withheld.length === 1);
+
+check("the caller is told what matched", (mixed.withheld[0]?.matched ?? []).length > 0);
+
+check("a withheld report does not refuse the run", mixed.refused === "");
+
+check("nothing is prepared from no rows", prepare([], options).prepared.length === 0);
+
+check("a non-array does not throw", prepare(null, options).prepared.length === 0);
+
+console.log(`\n${pass} passed, ${fail} failed`);
+process.exit(fail > 0 ? 1 : 0);

package/loops/actions/verify-composite-actions/action.yml

@@ -1,9 +1,9 @@
-# Managed by @plainconceptsplatform/workflows. Source: loops/actions/verify-composite-actions/action.yml. Update with workflows update --force; consumer edits may be overwritten.
-name: Verify composite actions
-description: Check every local composite action manifest parses and uses only contexts a composite action actually has.
-runs:
-  using: composite
-  steps:
-    - name: Validate composite action manifests
-      shell: bash
-      run: bash "${GITHUB_ACTION_PATH}/verify-composite-actions.sh"
+# Managed by @plainconceptsplatform/workflows. Source: loops/actions/verify-composite-actions/action.yml. Update with workflows update --force; consumer edits may be overwritten.
+name: Verify composite actions
+description: Check every local composite action manifest parses and uses only contexts a composite action actually has.
+runs:
+  using: composite
+  steps:
+    - name: Validate composite action manifests
+      shell: bash
+      run: bash "${GITHUB_ACTION_PATH}/verify-composite-actions.sh"

package/loops/actions/verify-refine-output/action.yml

@@ -1,9 +1,9 @@
-# Managed by @plainconceptsplatform/workflows. Source: loops/actions/verify-refine-output/action.yml. Update with workflows update --force; consumer edits may be overwritten.
-name: Verify refine output validation
-description: Exercise deterministic Refine output validation regressions.
-runs:
-  using: composite
-  steps:
-    - name: Verify refinement outcomes
-      shell: bash
-      run: bash "${{ github.action_path }}/verify-refine-output.sh"
+# Managed by @plainconceptsplatform/workflows. Source: loops/actions/verify-refine-output/action.yml. Update with workflows update --force; consumer edits may be overwritten.
+name: Verify refine output validation
+description: Exercise deterministic Refine output validation regressions.
+runs:
+  using: composite
+  steps:
+    - name: Verify refinement outcomes
+      shell: bash
+      run: bash "${{ github.action_path }}/verify-refine-output.sh"

package/loops/actions/verify-route-matrix/action.yml

@@ -1,9 +1,9 @@
-# Managed by @plainconceptsplatform/workflows. Source: loops/actions/verify-route-matrix/action.yml. Update with workflows update --force; consumer edits may be overwritten.
-name: Verify route matrix
-description: Exercise the router's classifier against every supported event, and check that each route and dispatch operation has a job in work-router.yml.
-runs:
-  using: composite
-  steps:
-    - name: Run route matrix verification
-      shell: bash
-      run: bash "${GITHUB_ACTION_PATH}/verify-route-matrix.sh"
+# Managed by @plainconceptsplatform/workflows. Source: loops/actions/verify-route-matrix/action.yml. Update with workflows update --force; consumer edits may be overwritten.
+name: Verify route matrix
+description: Exercise the router's classifier against every supported event, and check that each route and dispatch operation has a job in work-router.yml.
+runs:
+  using: composite
+  steps:
+    - name: Run route matrix verification
+      shell: bash
+      run: bash "${GITHUB_ACTION_PATH}/verify-route-matrix.sh"

package/loops/actions/verify-route-matrix/verify-route-matrix.sh

@@ -2060,6 +2060,94 @@ fi
   if [ "$GATE_RUN_ID_OK" -eq 1 ]; then PASS=$((PASS + 1)); else FAIL=$((FAIL + 1)); fi
 fi
 
+echo "── App error privacy ─────────────────────────────────────────────────────"
+
+# The application error report files into the private repository rather than out of it, so
+# its danger is the opposite one: not that a name escapes, but that an identifier joining a
+# stack trace back to one person's conversation gets written down beside it.
+#
+# Four properties hold that line and each is a line an edit could drop with nothing going red.
+# Asserted the way the error report's are: call sites and set equality, never a bare symbol,
+# because a grep for a name passes against a guard that has been renamed rather than removed.
+APP_ERRORS_MJS="${HERE}/../collect-app-errors/group-and-redact.mjs"
+APP_ERRORS_SH="${HERE}/../collect-app-errors/query-app-errors.sh"
+APP_ERRORS_YML="${HERE}/../collect-app-errors/action.yml"
+
+if [ -f "$APP_ERRORS_MJS" ]; then
+  AE_OK=1
+  ae() {
+    grep -qE "$1" "$2" || { AE_OK=0; echo "FAIL: collect-app-errors ${3}" >&2; }
+  }
+
+  # 1. The query reads the two tables the application's own telemetry goes to, and no others.
+  # AppServiceConsoleLogs and ContainerAppConsoleLogs live in the same workspace and carry raw
+  # engine stdout, which the no-content rule does not govern. A union across them is an
+  # incident rather than a wider query.
+  #
+  # AppTraces is deliberate and was added after the first dry run against a real workspace
+  # returned nought exceptions on a service that had been failing all morning: nothing in
+  # these applications calls RecordException, so every failure they care about is caught in
+  # code, logged through ILogger, and lands in AppTraces alone.
+  ae '^    AppExceptions$' "$APP_ERRORS_SH" 'does not query AppExceptions by name'
+  ae '^    AppTraces$' "$APP_ERRORS_SH" 'does not query AppTraces, where caught failures land'
+  # The heredoc alone. The comment above it names the console tables in order to say they are
+  # never read, and a grep over the whole file cannot tell the warning from the offence.
+  query_body="$(sed -n '/^read -r -d .. QUERY <<KQL/,/^KQL$/p' "$APP_ERRORS_SH")"
+  if printf '%s' "$query_body" | grep -qE 'ConsoleLogs|AppRequests|AppDependencies|union \*|search '; then
+    AE_OK=0
+    echo "FAIL: collect-app-errors reads a table it has no business reading" >&2
+  fi
+
+  # 2. Counts are scaled for sampling. A pre environment samples at 0.3, so count() reads a
+  # problem that happened two hundred times as sixty, and it falls under a floor set to catch
+  # it. This is a correctness guard that looks like a style one.
+  ae 'sum\(ItemCount\)' "$APP_ERRORS_SH" 'counts rows instead of summing ItemCount'
+
+  # 3. The field allowlist is what actually holds the line, because a redaction rule filters
+  # values and cannot see a field somebody adds next year. Compared as a set, so reordering is
+  # fine and an addition is not. run_id is the one that matters: it is unhashed on this
+  # telemetry and joins to a conversation and to a person.
+  expected_fields="exceptionType frames fingerprint firstSeen lastSeen occurrences operationName operations problemId roleName"
+  actual_fields="$(sed -n '/export const FINDING_FIELDS = \[/,/\];/p' "$APP_ERRORS_MJS" |
+    grep -oE '"[a-zA-Z]+"' | tr -d '"' | sort | tr '\n' ' ' | sed 's/ $//')"
+  if [ "$actual_fields" = "$(echo "$expected_fields" | tr ' ' '\n' | sort | tr '\n' ' ' | sed 's/ $//')" ]; then
+    PASS=$((PASS + 1))
+  else
+    FAIL=$((FAIL + 1))
+    echo "FAIL: collect-app-errors reportable fields changed: ${actual_fields}" >&2
+  fi
+  # Plain grep, not the count() helper: that is `grep || true`, so an `if count -q` is true
+  # whatever it finds and the guard never fires. Three of these were written that way first
+  # and all three failed open, which is the same shape of bug this file exists to catch.
+  if sed -n '/export const FINDING_FIELDS = \[/,/\];/p' "$APP_ERRORS_MJS" | grep -qiE 'run_?id'; then
+    AE_OK=0
+    echo "FAIL: collect-app-errors lists a run id as reportable; it joins telemetry to a person" >&2
+  fi
+
+  # 4. Scrub, then scan, then refuse. Scrubbing masks what is common and safely replaceable so
+  # one unlucky operation name does not leave the job red and filing nothing forever; the
+  # scanner is the backstop for whatever that missed; and a withheld report still turns the
+  # run red so the field that carried it gets fixed.
+  ae 'export const SCRUBS = \[' "$APP_ERRORS_MJS" 'declares no scrubbing'
+  ae 'export const LEAK_CHECKS = \[' "$APP_ERRORS_MJS" 'declares no leak scanner'
+  ae 'core\.setFailed.*withheld by the leak scanner' "$APP_ERRORS_YML" 'does not go red on a withheld report'
+  ae 'not in the reportable field list' "$APP_ERRORS_MJS" 'does not refuse an unlisted field'
+  ae 'core\.setFailed\(`Nothing was filed' "$APP_ERRORS_YML" 'does not stop when the module refuses'
+
+  # A GUID must be masked on the way in, not merely detected on the way out. Both, in fact:
+  # the scrub is what keeps the job alive and the check is what keeps it honest.
+  ae '\{guid\}' "$APP_ERRORS_MJS" 'does not mask GUIDs'
+
+  # 5. No model. This is the same assertion the error report carries, for the same reason: a
+  # model summarising a stack trace paraphrases it, and a paraphrase is a wrong issue.
+  if grep -qiE '(engine:|opencode|safe-outputs|OPENAI_API_KEY)' "$APP_ERRORS_YML"; then
+    AE_OK=0
+    echo "FAIL: collect-app-errors has grown a model" >&2
+  fi
+
+  if [ "$AE_OK" -eq 1 ]; then PASS=$((PASS + 1)); else FAIL=$((FAIL + 1)); fi
+fi
+
 echo
 if [ "$FAIL" -eq 0 ]; then
   echo "Route matrix: ${PASS} passed"