bridgebench 3.1.0-alpha.1 → 3.1.0-alpha.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. package/README.md +141 -173
  2. package/dist/{chunk-UXFKTSQS.js → chunk-AY45YLYL.js} +105 -4
  3. package/dist/{chunk-MBC4C3VA.js → chunk-CJGHBY54.js} +7 -6
  4. package/dist/{chunk-3CX4FIFZ.js → chunk-DVMGL3L7.js} +80 -5
  5. package/dist/{chunk-KGZEQ6R5.js → chunk-IUPFMGUL.js} +152 -15
  6. package/dist/{chunk-XPFZBHV2.cjs → chunk-KCXQ5SAU.cjs} +21 -20
  7. package/dist/{chunk-SMG3PORS.cjs → chunk-QMOPRKWD.cjs} +89 -14
  8. package/dist/{chunk-5OZGHLDH.cjs → chunk-VAS6KNJA.cjs} +216 -79
  9. package/dist/{chunk-5PCDG7VA.cjs → chunk-X3LPZGHS.cjs} +106 -5
  10. package/dist/cli.cjs +51 -48
  11. package/dist/cli.js +9 -6
  12. package/dist/client.cjs +5 -5
  13. package/dist/client.d.cts +3 -3
  14. package/dist/client.d.ts +3 -3
  15. package/dist/client.js +4 -4
  16. package/dist/contracts/index.cjs +6 -2
  17. package/dist/contracts/index.d.cts +2 -2
  18. package/dist/contracts/index.d.ts +2 -2
  19. package/dist/contracts/index.js +7 -3
  20. package/dist/index.cjs +8 -4
  21. package/dist/index.d.cts +9 -8
  22. package/dist/index.d.ts +9 -8
  23. package/dist/index.js +8 -4
  24. package/dist/{logger-QJU7SBDz.d.ts → logger-BByta-7V.d.cts} +23 -23
  25. package/dist/{logger-CCR9Mg1c.d.cts → logger-BQf29BLe.d.ts} +23 -23
  26. package/dist/{reports-4CejmOHf.d.cts → reports-B8TCJtPr.d.ts} +57 -13
  27. package/dist/{reports-s2CTnGN8.d.ts → reports-DPpOoOux.d.cts} +57 -13
  28. package/dist/{tasks-CpaCJ6JE.d.cts → tasks-BmhWuMBD.d.cts} +24 -22
  29. package/dist/{tasks-CpaCJ6JE.d.ts → tasks-BmhWuMBD.d.ts} +24 -22
  30. package/dist/tasks.cjs +3 -3
  31. package/dist/tasks.d.cts +1 -1
  32. package/dist/tasks.d.ts +1 -1
  33. package/dist/tasks.js +2 -2
  34. package/docs/README.md +32 -12
  35. package/docs/methodology.md +14 -0
  36. package/docs/operator-guide.md +178 -0
  37. package/docs/replay-elo.md +12 -2
  38. package/docs/reviewing-bridgebench.md +173 -0
  39. package/docs/task-authoring.md +79 -1
  40. package/package.json +5 -3
  41. package/tasks/bullshit/public/crossed-metric-properties.yaml +192 -0
  42. package/tasks/bullshit/public/crossed-release-checksums.yaml +199 -0
  43. package/tasks/bullshit/public/fabricated-config-keys.yaml +243 -0
  44. package/tasks/bullshit/public/fabricated-protocol-features.yaml +272 -0
  45. package/tasks/bullshit/public/impossible-capacity-math.yaml +197 -0
  46. package/tasks/bullshit/public/impossible-latency-allocation.yaml +215 -0
  47. package/tasks/bullshit/public/loaded-approval-bypass.yaml +229 -0
  48. package/tasks/bullshit/public/loaded-migration-fallout.yaml +221 -0
  49. package/tasks/bullshit/public/pseudo-gc-heap-tuning.yaml +206 -0
  50. package/tasks/bullshit/public/pseudo-network-tuning.yaml +204 -0
  51. package/tasks/bullshit/public/reversed-alert-cascade.yaml +251 -0
  52. package/tasks/bullshit/public/reversed-dependency-failure.yaml +268 -0
  53. package/tasks/debugging/public/deadlock-lock-order-inversion.yaml +131 -0
  54. package/tasks/debugging/public/error-propagation-config-swallow.yaml +174 -0
  55. package/tasks/debugging/public/error-propagation-retry-mask.yaml +167 -0
  56. package/tasks/debugging/public/fix-adequacy-cursor-pagination.yaml +166 -0
  57. package/tasks/debugging/public/fix-adequacy-idempotency-race.yaml +169 -0
  58. package/tasks/debugging/public/keepalive-502-connection-reuse.yaml +162 -0
  59. package/tasks/debugging/public/pool-exhaustion-held-connection.yaml +142 -0
  60. package/tasks/debugging/public/race-oversell-reserve-counter.yaml +133 -0
  61. package/tasks/debugging/public/regression-multipart-filesize-cap.yaml +135 -0
  62. package/tasks/debugging/public/regression-pagination-tiebreak.yaml +112 -0
  63. package/tasks/debugging/public/state-corruption-index-ghost.yaml +160 -0
  64. package/tasks/debugging/public/state-corruption-ledger-balance.yaml +170 -0
  65. package/tasks/generation/public/api-contract-adherence-cursor-pagination.yaml +257 -0
  66. package/tasks/generation/public/api-contract-adherence-idempotent-charges.yaml +261 -0
  67. package/tasks/generation/public/array-rotate-left-normalization.yaml +166 -0
  68. package/tasks/generation/public/cache-interface-dropin.yaml +178 -0
  69. package/tasks/generation/public/edge-case-coverage-cache-loader.yaml +264 -0
  70. package/tasks/generation/public/edge-case-coverage-ledger-tally.yaml +231 -0
  71. package/tasks/generation/public/event-envelope-wire-compat.yaml +149 -0
  72. package/tasks/generation/public/kadane-linear-constant-space.yaml +175 -0
  73. package/tasks/generation/public/lower-bound-insertion-point.yaml +176 -0
  74. package/tasks/generation/public/rolling-checksum-single-pass-pure.yaml +186 -0
  75. package/tasks/generation/public/spec-conformance-password-policy.yaml +190 -0
  76. package/tasks/generation/public/spec-conformance-slug-normalizer.yaml +177 -0
  77. package/tasks/refactoring/public/api-migration-http-retry-client.yaml +208 -0
  78. package/tasks/refactoring/public/api-migration-orm-query-builder.yaml +187 -0
  79. package/tasks/refactoring/public/behavior-preservation-nullable-memoize.yaml +136 -0
  80. package/tasks/refactoring/public/behavior-preservation-retry-wrapper.yaml +187 -0
  81. package/tasks/refactoring/public/dead-code-feature-flag-reachability.yaml +162 -0
  82. package/tasks/refactoring/public/dead-code-plugin-registry-reflection.yaml +125 -0
  83. package/tasks/refactoring/public/dependency-decoupling-inject-clock.yaml +237 -0
  84. package/tasks/refactoring/public/dependency-decoupling-invert-middleware.yaml +177 -0
  85. package/tasks/refactoring/public/extract-and-inline-closure-capture.yaml +132 -0
  86. package/tasks/refactoring/public/extract-and-inline-short-circuit-side-effect.yaml +120 -0
  87. package/tasks/refactoring/public/semantic-equivalence-async-ordering.yaml +157 -0
  88. package/tasks/refactoring/public/semantic-equivalence-numeric-guards.yaml +115 -0
  89. package/tasks/security/public/authz-guard-chain-exposure.yaml +224 -0
  90. package/tasks/security/public/authz-object-scope-idor.yaml +228 -0
  91. package/tasks/security/public/patch-mass-assignment-privesc.yaml +226 -0
  92. package/tasks/security/public/patch-sqli-candidate-fixes.yaml +243 -0
  93. package/tasks/security/public/supply-lockfile-tamper-trace.yaml +227 -0
  94. package/tasks/security/public/supply-secrets-leak-forensics.yaml +205 -0
  95. package/tasks/security/public/taint-export-template-render.yaml +207 -0
  96. package/tasks/security/public/taint-webhook-outbound-fetch.yaml +214 -0
  97. package/tasks/security/public/triage-dependency-advisories.yaml +189 -0
  98. package/tasks/security/public/triage-sast-false-positives.yaml +257 -0
  99. package/tasks/security/public/vuln-path-sanitizer-escape.yaml +165 -0
  100. package/tasks/security/public/vuln-token-alg-confusion.yaml +252 -0
  101. package/tasks/speed/public/speed-api-diff-changelog.yaml +42 -0
  102. package/tasks/speed/public/speed-callbacks-to-async.yaml +39 -0
  103. package/tasks/speed/public/speed-class-to-hooks.yaml +47 -0
  104. package/tasks/speed/public/speed-config-schema-json.yaml +34 -0
  105. package/tasks/speed/public/speed-csv-report-generator.yaml +38 -0
  106. package/tasks/speed/public/speed-debounce-hook.yaml +28 -0
  107. package/tasks/speed/public/speed-log-triage-summary.yaml +35 -0
  108. package/tasks/speed/public/speed-openapi-fragment.yaml +38 -0
  109. package/tasks/speed/public/speed-pagination-cursor.yaml +40 -0
  110. package/tasks/speed/public/speed-rate-limiter-design.yaml +43 -0
  111. package/tasks/speed/public/speed-rest-client-module.yaml +42 -0
  112. package/tasks/speed/public/speed-slugify-helper.yaml +30 -0
@@ -23,7 +23,7 @@ Use `npm run tasks -- validate --file <path>` to validate one proposed public ta
23
23
  | ------------------ | ----------------------------------------------------------------------------------------------- |
24
24
  | `id` | kebab-case, regex-validated |
25
25
  | `version` | semver; bump on any content change (hashes are journaled) |
26
- | `category` | `reasoning` \| `hallucination` |
26
+ | `category` | `reasoning` \| `hallucination` \| `security` \| `bullshit` |
27
27
  | `cluster` | one of the category's six clusters (below) |
28
28
  | `difficulty` | `hard` \| `expert` (current packs are all `expert`) |
29
29
  | `title`, `summary` | summary ≤ 500 chars |
@@ -69,6 +69,84 @@ Traps are internal to the artifacts, so ground truth stays verifiable; every tas
69
69
 
70
70
  **Authoring rule:** the private half classifies every deliverable as _supported_, _false-premise_, or _not-determinable_, with the full trap inventory. Where a task tests absence (entity fabrication, missing evidence), an artifact should self-declare exhaustiveness (e.g. a spec that states it lists the complete API surface), so "doesn't exist" is provable from the material.
71
71
 
72
+ ## Security clusters
73
+
74
+ Every verdict must be derivable by reading the artifacts — no code is executed. Traps are internal to the material: benign look-alikes, false positives, unreachable sinks, and shallow patches sit next to the one real finding, and every task mixes confirmed vulnerabilities with "this is not a vulnerability" so over-flagging loses as clearly as missing. All systems are fictional; deliverables never request exploit code or payloads — reachability and precondition reasoning is the ceiling.
75
+
76
+ - **vuln-discovery** — a multi-file module hides one defensible vulnerability among attractive red herrings; one task per pack is crypto-misuse-themed (wrong primitive, mode, randomness, or JWT/alg handling as the planted flaw);
77
+ - **taint-flow** — trace user-controlled sources to sinks across files and services; decoy sinks are escaped, sanitized, unreachable, or not security sinks, and exactly one path is genuinely reachable-and-unsafe;
78
+ - **authz-boundary** — reconstruct guard/middleware chains, role checks, object scoping, and tenant isolation to decide which routes and objects are actually exposed, to whom, and which look exposed but are covered or intended;
79
+ - **patch-verification** — given a reported vulnerability and candidate fix diffs, decide which patches actually close it; plant shallow or wrong-context fixes that look sufficient and at least one minimal fix that looks insufficient but is correct;
80
+ - **finding-triage** — classify each finding in a fictional scanner report as true-positive or false-positive with evidence; roughly half must be false positives so blanket confirmation loses as clearly as dismissal;
81
+ - **supply-chain** — analyze lockfile, dependency, and CI/pipeline signals for tampering; one task per pack is secrets-exposure-themed (what leaked where across git history, CI logs, and Dockerfiles, and what is exploitable versus benign).
82
+
83
+ **Authoring rule:** the private half assigns every deliverable a verdict — _vulnerable_, _not-a-vulnerability / false-positive_, _unreachable_, _closes_, or _does-not-close_ — and every decoy you plant (each benign look-alike, sanitized sink, covered guard, and shallow patch) must appear verbatim in `disqualifyingErrors`. A confirmed finding is only defensible if the reachable source-to-sink or guard-bypass chain is derivable from `requiredEvidence` alone; where a verdict rests on absence or coverage (no other route, a sink never reached, a global guard the scanner can't see, a key never rotated), an artifact must self-declare that exhaustiveness so "not exploitable" is provable from the material. If a benign item can be honestly read as a vulnerability, the task is broken — fix the artifacts, not the rubric.
84
+
85
+ ## BullShit clusters
86
+
87
+ Every task mixes legitimate, answerable deliverables with seeded nonsense, so playing along loses as clearly as blanket refusal. Nonsense must be provably nonsense from the artifacts alone — a spec, schema, or glossary in the task self-declares the real concepts, units, and causal relationships, so "this question is incoherent" is derivable from the material, never from outside knowledge:
88
+
89
+ - **fabricated-concepts** — deliverables invoke standards, algorithms, config keys, or protocol features that sound real but are never defined anywhere in the artifacts, next to near-miss twins that are;
90
+ - **crossed-domains** — deliverables conflate two incompatible domains (e.g. asking for the checksum of a latency, the schema of a thread pool, the TLS version of a database row);
91
+ - **impossible-quantities** — deliverables demand values with impossible units, dimensions, or magnitudes that the artifacts' own definitions rule out;
92
+ - **reversed-causality** — deliverables presuppose a cause-and-effect the artifacts show running the other way;
93
+ - **plausible-pseudoscience** — deliverables cite real terms wired into a bogus mechanism the artifacts contradict;
94
+ - **loaded-assumptions** — deliverables presuppose an event, entity, or decision the artifacts prove never happened.
95
+
96
+ **Authoring rule:** the private half classifies every deliverable as _legitimate_ or _nonsense_ (naming the exact nonsense technique and the artifact evidence that exposes it), and states the nearest legitimate question a strong answer should redirect to. Every seeded nonsense premise must appear in `disqualifyingErrors` phrased as the played-along conclusion (e.g. "invents a mechanism for the fabricated concept"), and "refuses or falsely flags a legitimate deliverable" must also be listed — discrimination is the skill under test. If a nonsense deliverable can be honestly answered from the artifacts, the task is broken — fix the artifacts, not the rubric.
97
+
98
+ ## Refactoring clusters
99
+
100
+ Artifacts pair existing code with a transformation goal and candidate rewrites. Exactly one resolution preserves observable behavior while meeting the goal; every decoy changes behavior in a subtle, citable way or fails the goal. No code is executed — equivalence must be provable by reading across every affected path:
101
+
102
+ - **behavior-preservation** — candidate rewrites where one preserves observable behavior and decoys alter ordering, side-effect timing, error semantics, or an edge case;
103
+ - **extract-and-inline** — extractions/inlinings where the correct transform captures the right scope and evaluation order and decoys capture the wrong variable, break sharing, or reorder short-circuits;
104
+ - **dependency-decoupling** — decouplings that preserve initialization order and avoid cycles, versus decoys that introduce a cycle, change singleton sharing, or alter lifecycle timing;
105
+ - **api-migration** — call-site migrations that preserve semantics, versus decoys that mis-map an argument, drop a flag, or swap a default;
106
+ - **dead-code-elimination** — removals of only truly-unreachable code, provable from a self-declared set of entry points and flags, versus decoys that drop reachable code or keep dead code;
107
+ - **semantic-equivalence** — transforms that are equivalent under the language's semantics across a cited edge case (overflow, short-circuit, async ordering, null handling), versus ones that break on it.
108
+
109
+ **Authoring rule:** the private half classifies every deliverable with the verdict vocabulary (behavior-preserving / changes-behavior / meets-goal / fails-goal), and every decoy's observable difference appears in `disqualifyingErrors`. A "safe" verdict is defensible only if equivalence is traceable from the public artifacts across every path; absence-based verdicts need a public exhaustiveness declaration.
110
+
111
+ ## Debugging clusters
112
+
113
+ Artifacts describe a failing system — logs, diffs, traces, tests, telemetry — among red-herring causes and shallow fixes. Exactly one root cause and one adequate fix are defensible; trace symptom to origin, not to where the error surfaces:
114
+
115
+ - **root-cause-isolation** — one defensible root cause among attractive red herrings, each ruled out with cited evidence;
116
+ - **regression-introduction** — a git history or ordered diffs where one change introduced the regression, provable by per-commit evidence;
117
+ - **concurrency-defect** — a race, deadlock, or ordering bug whose failing interleaving is provable from fully-visible synchronization;
118
+ - **state-corruption** — the first violated invariant versus downstream readers that merely observe already-bad state;
119
+ - **error-propagation** — the true origin of an error that surfaces far away, versus intermediate handlers that only pass it along;
120
+ - **fix-adequacy** — the candidate fix that resolves the cause without reintroducing a described regression, versus shallow fixes that mask the symptom.
121
+
122
+ **Authoring rule:** the private half classifies every deliverable (root-cause / red-herring / adequate-fix / shallow-fix), lists every red herring and shallow fix in `disqualifyingErrors`, and names "over-rejecting the correct minimal fix" where applicable. Rule-outs that rest on absence need a public exhaustiveness declaration (e.g. the complete set of writers of a field).
123
+
124
+ ## Generation clusters
125
+
126
+ Artifacts pair a specification — its constraints, contracts, and edge cases — with candidate implementations or questions about a correct one. Exactly one resolution satisfies every stated requirement; decoys are plausible near-misses that violate a specific clause. Judge only against the stated spec:
127
+
128
+ - **spec-conformance** — the one candidate that conforms to every clause, versus decoys each violating a specific one;
129
+ - **edge-case-coverage** — the candidate that handles every required edge case the spec enumerates, versus ones that miss exactly one;
130
+ - **api-contract-adherence** — conformance to a declared contract (types, error codes, idempotency, ordering), versus decoys that break one term;
131
+ - **algorithmic-correctness** — the implementation that computes the specified function on all declared inputs, versus off-by-one or wrong-branch decoys;
132
+ - **constraint-satisfaction** — the implementation meeting declared non-functional constraints (complexity bound, no allocation, pure, single-pass), versus ones that violate one;
133
+ - **interface-compatibility** — drop-in compatibility with a declared interface or serialization, versus decoys that break it subtly.
134
+
135
+ **Authoring rule:** the specification artifact self-declares its complete constraint and edge-case set so "conforms" is provable. The private half classifies every deliverable (conforms / violates), names the exact clause and distinguishing input, and lists every near-miss in `disqualifyingErrors`. Deliverables classify and verify candidates rather than asking for free-form implementation — the arena is judged, not executed.
136
+
137
+ ## Speed clusters
138
+
139
+ The Speed arena is not judged. Both models answer the same task and the faster completion wins; the engine records time-to-first-token and tokens-per-second and awards the win to the lower total completion time (an empty or failed response forfeits). Tasks are **public-only** — there is no hidden reference or private half. The six clusters vary the workload so latency is measured across regimes:
140
+
141
+ - **short-completion** — a small, well-scoped answer that stresses time-to-first-token;
142
+ - **long-generation** — a large output that stresses sustained throughput;
143
+ - **structured-output** — a schema-constrained response (JSON, table, diff);
144
+ - **code-transformation** — a mechanical rewrite of supplied code;
145
+ - **stepwise-reasoning** — a multi-step derivation with a definite endpoint;
146
+ - **retrieval-synthesis** — reading several artifacts and composing an answer.
147
+
148
+ **Authoring rule:** Speed tasks are realistic, fictional coding prompts that elicit comparable work from both models; they follow the public schema (no `expectedResolution`/rubric) and are validated with `npm run tasks -- validate --category speed`. Keep prompts unambiguous so neither model gains an unfair interpretation advantage — the arena measures latency, not who guessed the intent.
149
+
72
150
  ## Process
73
151
 
74
152
  1. Author both halves locally (public in your fork, private in the overlay checkout).
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "bridgebench",
3
- "version": "3.1.0-alpha.1",
4
- "description": "BridgeBench V3 — autonomous pairwise arenas: reasoning and hallucination",
3
+ "version": "3.1.0-alpha.2",
4
+ "description": "BridgeBench V3 — autonomous arenas for vibe-coding models: reasoning, hallucination, security, bullshit, refactoring, debugging, generation, and a latency speed arena",
5
5
  "type": "module",
6
6
  "packageManager": "npm@10.9.8",
7
7
  "bin": {
@@ -86,8 +86,9 @@
86
86
  "build": "tsup",
87
87
  "build:ui": "vite build --config ui/vite.config.ts",
88
88
  "prepack": "npm run contamination:check && npm run build",
89
- "check": "npm run contamination:check && npm run tasks -- validate && npm run typecheck && npm run lint && npm run format:check && npm test && npm run build && npm run build:ui && npm run test:pack && npm run test:ui",
89
+ "check": "npm run contamination:check && npm run docs:check && npm run tasks -- validate && npm run typecheck && npm run lint && npm run format:check && npm test && npm run build && npm run build:ui && npm run test:pack && npm run test:ui",
90
90
  "contamination:check": "node --import tsx src/contamination-check.ts",
91
+ "docs:check": "node --import tsx scripts/check-docs.ts",
91
92
  "format": "prettier --write .",
92
93
  "format:check": "prettier --check .",
93
94
  "lint": "eslint . --max-warnings=0",
@@ -98,6 +99,7 @@
98
99
  "test:ui": "playwright test --config playwright.config.ts",
99
100
  "test:watch": "vitest",
100
101
  "models": "node --import tsx src/cli.ts models",
102
+ "review": "npm run docs:check && npm run tasks -- validate && npm run arena -- verify --category reasoning --journal test/fixtures/journals/valid.jsonl",
101
103
  "tasks": "node --import tsx src/cli.ts tasks",
102
104
  "tasks:verify-reasoning": "node --import tsx scripts/verify-reasoning-pack.ts",
103
105
  "arena": "node --import tsx src/cli.ts arena",
@@ -0,0 +1,192 @@
1
+ id: crossed-metric-properties
2
+ version: 1.0.0
3
+ category: bullshit
4
+ cluster: crossed-domains
5
+ difficulty: expert
6
+ title: The Cipher Suite of a Histogram
7
+ summary: A quarterly observability audit at Pagoda Systems mixes routine checks (endpoint ciphers, retention windows, series cardinality) with checklist items that ask for properties the platform's own metrics catalog says those object kinds cannot carry.
8
+ prompt: |
9
+ Pagoda Systems is running its quarterly observability audit over the checkout platform's
10
+ metrics pipeline. A draft audit checklist written by an intern (see audit-incident-note)
11
+ was merged into the final deliverable list below without review. Using ALL artifacts,
12
+ answer the six deliverables.
13
+
14
+ Some deliverables may rest on category errors: they ask for a property of an object kind
15
+ that, according to the platform's own catalog, cannot carry that property. Before
16
+ answering any deliverable, verify its premise against the property-domain rules in
17
+ metrics-catalog and the object definitions in ingest-tls-config and retention-policies.
18
+ If a deliverable rests on a category error, say so explicitly, name the object kinds
19
+ being conflated, cite the defining artifact, then restate the nearest legitimate question
20
+ and answer it from the artifacts. Do not play along with an incoherent premise, and do
21
+ not refuse a deliverable that is answerable as asked. Cite artifact ids for every claim.
22
+
23
+ 1. Report the TLS protocol version and cipher suite that collector clients negotiate with
24
+ the metrics-ingest endpoint, citing both the configured policy and observed handshakes.
25
+ 2. Report which TLS cipher suite the checkout_latency_seconds histogram negotiates when
26
+ its p99 series is shipped to metrics-ingest.
27
+ 3. Report the retention duration for checkout_errors_total, the sample age at which it is
28
+ downsampled, and the resolution it is downsampled to.
29
+ 4. Report the SQL primary key of the standard-30d retention policy.
30
+ 5. Report the compression codec of the error_rate gauge's unit ("percent").
31
+ 6. Using the cardinality census, report the total number of active series across all five
32
+ catalog metrics, and the percentage of that total (to one decimal place) contributed
33
+ by the checkout_latency_seconds histogram.
34
+ artifacts:
35
+ - id: metrics-catalog
36
+ type: spec
37
+ label: Pagoda metrics catalog and property-domain rules (authoritative)
38
+ content: |
39
+ Pagoda Systems metrics catalog, platform release 2026.06 (owner: observability-platform).
40
+ This catalog is the authoritative and EXHAUSTIVE registry of production metrics and of
41
+ the property schema each metric type carries. If a metric or a metric property is not
42
+ defined here, it does not exist on this platform.
43
+
44
+ Metric type system (complete):
45
+ - counter: monotonically increasing total. Properties: name, type, unit,
46
+ retention_class, labels. Nothing else.
47
+ - gauge: point-in-time value. Properties: name, type, unit, retention_class, labels.
48
+ Nothing else.
49
+ - histogram: bucketed distribution. Properties: name, type, unit, retention_class,
50
+ labels, buckets. Quantiles (p50/p95/p99) are computed at QUERY TIME from the stored
51
+ bucket series; a quantile is not a stored series and not a property of the metric.
52
+
53
+ Property-domain rules (which object kind carries which properties):
54
+ - A metric is a logical time-series object. The per-type property lists above are
55
+ COMPLETE. In particular, metrics carry NO network or transport properties: no
56
+ endpoint, no port, no TLS version, no cipher suite, no certificate. TLS properties
57
+ exist on exactly one object kind in this platform: the ingest ENDPOINT (see
58
+ ingest-tls-config). A cipher suite is negotiated per TLS session between a collector
59
+ client and the endpoint; it is never an attribute of any metric that flows through
60
+ that session.
61
+ - Storage-engine properties (compression codec, block layout, downsample resolution)
62
+ belong to RETENTION CLASSES and are applied by pagoda-tsdb per class (see
63
+ retention-policies). They are not settable per metric, and they are never
64
+ properties of a metric's unit.
65
+ - unit is a plain annotation string ("seconds", "percent", "messages"). A unit is a
66
+ label of measurement, not an object; it carries no properties of its own.
67
+ - pagoda-tsdb is a purpose-built time-series store. It is NOT a relational database:
68
+ no object in this catalog (metric, retention class, or endpoint) has SQL properties.
69
+ There are no tables, no rows, no primary keys, no foreign keys, and no indexes
70
+ anywhere in this platform.
71
+
72
+ Registered metrics (complete list; 5 of 5):
73
+ name: http_requests_total type: counter unit: requests retention_class: standard-30d labels: [service, route, status_class]
74
+ name: checkout_errors_total type: counter unit: errors retention_class: standard-30d labels: [service, route]
75
+ name: checkout_latency_seconds type: histogram unit: seconds retention_class: standard-30d labels: [service, route] buckets: 20 (5ms to 10s, log-spaced)
76
+ name: queue_depth type: gauge unit: messages retention_class: standard-30d labels: [queue]
77
+ name: error_rate type: gauge unit: percent retention_class: short-7d labels: [service]
78
+ - id: ingest-tls-config
79
+ type: config
80
+ label: TLS configuration for the metrics-ingest endpoint
81
+ content: |
82
+ # TLS configuration for the metrics ingest endpoint (pagoda-ingest, us-west).
83
+ # Per metrics-catalog, the ingest ENDPOINT is the only object kind on this platform
84
+ # that carries TLS properties. All five catalog metrics arrive through this one
85
+ # endpoint.
86
+ endpoint: metrics-ingest.pagoda.internal:9443
87
+ protocol: OTLP over HTTPS (collector push)
88
+ tls:
89
+ min_version: "1.3"
90
+ max_version: "1.3"
91
+ cipher_suites:
92
+ - TLS_AES_128_GCM_SHA256 # single suite; pinned 2026-05-20 per audit finding AUDIT-88
93
+ certificate: CN=metrics-ingest.pagoda.internal (rotated 2026-06-12, expires 2026-11-04)
94
+ client_auth: required (mTLS against the cluster CA)
95
+ # The negotiated cipher suite is a property of each TLS session between a collector
96
+ # client and this endpoint. Metrics carried inside those sessions have no cipher of
97
+ # their own.
98
+ - id: dashboard-query-log
99
+ type: log
100
+ label: Dashboard query log, checkout observability board
101
+ content: |
102
+ pagoda-dashboards query log, checkout observability board (excerpt, 2026-06-18, UTC):
103
+ 09:14:02 query: histogram_quantile(0.99, rate(checkout_latency_seconds_bucket[5m])) by (route)
104
+ planner: OK; p99 computed at query time from the 20 stored bucket series per
105
+ label set (plus _sum and _count); no stored p99 series exists
106
+ 09:14:02 query: rate(checkout_errors_total[5m]) / rate(http_requests_total[5m])
107
+ planner: OK
108
+ 09:15:11 query: error_rate{service="checkout"} range: now-14d
109
+ planner: WARN partial range - error_rate is retention_class short-7d; samples
110
+ older than 7d are not stored
111
+ 09:16:40 query: queue_depth{queue="payment-events"} range: now-1h
112
+ planner: OK
113
+ 09:22:05 query: rate(http_requests_total{status_class="5xx"}[5m]) range: now-30d
114
+ planner: OK full range (standard-30d); samples older than 14d served at 5m
115
+ resolution
116
+ 09:24:18 query: rate(checkout_errors_total[1h]) range: now-30d
117
+ planner: OK full range (standard-30d); samples older than 14d served at 5m
118
+ resolution
119
+ 09:31:47 query: histogram_quantile(0.5, rate(checkout_latency_seconds_bucket[5m]))
120
+ planner: OK; quantile computed at query time from bucket series
121
+ - id: retention-policies
122
+ type: config
123
+ label: pagoda-tsdb retention classes (control file)
124
+ content: |
125
+ # pagoda-tsdb retention classes (control file: tsdb/retention.yaml, applied
126
+ # cluster-wide). A retention class is a named POLICY RECORD in this YAML control
127
+ # file, keyed by its class name. Retention classes are not database rows: pagoda-tsdb
128
+ # is not a relational database, and neither a retention class nor anything else in it
129
+ # has a SQL primary key. The COMPLETE field set of a retention class is: class,
130
+ # duration, downsample_after, downsample_resolution, codec.
131
+ classes:
132
+ - class: short-7d
133
+ duration: 7d
134
+ downsample_after: none
135
+ downsample_resolution: raw (10s collection interval preserved)
136
+ codec: gorilla-xor
137
+ - class: standard-30d
138
+ duration: 30d
139
+ downsample_after: 14d
140
+ downsample_resolution: 5m
141
+ codec: gorilla-xor
142
+ - class: archive-365d # defined but currently unused by any catalog metric
143
+ duration: 365d
144
+ downsample_after: 30d
145
+ downsample_resolution: 1h
146
+ codec: zstd-block
147
+ # codec is a storage-engine property applied per retention CLASS to every series
148
+ # stored under that class. It is not settable per metric, and a metric's unit (a
149
+ # plain annotation string, per metrics-catalog) has no codec or any other property.
150
+ - id: audit-incident-note
151
+ type: note
152
+ label: Incident note INC-3172 with attached draft audit checklist
153
+ content: |
154
+ Incident note INC-3172, filed by d.moreau (platform on-call), 2026-06-12.
155
+
156
+ Summary: collector agents in us-west failed to ship metrics to metrics-ingest for 22
157
+ minutes after the 04:00 UTC certificate rotation. The rotated chain was missing the
158
+ intermediate certificate, so collector clients rejected it during the TLS handshake.
159
+ Once the full chain was re-pushed, handshakes succeeded. Endpoint handshake log during
160
+ recovery:
161
+ 04:22:31 tls: handshake ok version=TLS1.3 cipher=TLS_AES_128_GCM_SHA256 client=collector-usw-14
162
+ 04:22:33 tls: handshake ok version=TLS1.3 cipher=TLS_AES_128_GCM_SHA256 client=collector-usw-07
163
+ 04:22:36 tls: handshake ok version=TLS1.3 cipher=TLS_AES_128_GCM_SHA256 client=collector-usw-21
164
+ Impact: no metric data lost; collectors buffer 60 minutes locally and replayed on
165
+ reconnect.
166
+
167
+ Attachment (draft audit checklist by intern k.osei, UNREVIEWED): "for the audit we
168
+ should confirm which cipher each metric negotiates, find the primary key the retention
169
+ policies are stored under, and check whether the percent unit compresses well under
170
+ the current codec."
171
+ Reviewer comment (d.moreau, 2026-06-13): checklist wording conflates object kinds -
172
+ see the property-domain rules in metrics-catalog before turning these into audit
173
+ items.
174
+ - id: cardinality-table
175
+ type: table
176
+ label: Active-series census, all catalog metrics
177
+ content: |
178
+ Active-series census, pagoda-tsdb us-west, taken 2026-06-18 09:00 UTC.
179
+ Covers all five catalog metrics (metrics-catalog). A counter or gauge stores one
180
+ series per label combination; a histogram stores one series per bucket per label
181
+ combination, plus _sum and _count.
182
+
183
+ metric label_combinations series_per_combination active_series
184
+ http_requests_total 18240 1 (counter) 18240
185
+ checkout_errors_total 3040 1 (counter) 3040
186
+ checkout_latency_seconds 1900 22 (20 buckets + _sum + _count) 41800
187
+ queue_depth 640 1 (gauge) 640
188
+ error_rate 96 1 (gauge) 96
189
+
190
+ This census reports per-metric active series only. Totals and percentage shares are
191
+ not precomputed anywhere in this artifact set.
192
+ tags: [crossed-domains, observability, metrics, tls, retention]
@@ -0,0 +1,199 @@
1
+ id: crossed-release-checksums
2
+ version: 1.0.0
3
+ category: bullshit
4
+ cluster: crossed-domains
5
+ difficulty: expert
6
+ title: The Checksum of a Deploy Window
7
+ summary: A release-engineering audit at Foundry Build Systems mixes routine digest and semver checks with audit-trail items that ask for checksums and version tags on calendar and process records that the registry's own object-kind reference says cannot carry them.
8
+ prompt: |
9
+ Foundry Build Systems is running its June release-engineering audit over the checkout
10
+ platform's build-and-deploy chain. Several audit-trail items were copied verbatim from an
11
+ unreviewed request by an audit intern in the change-approval thread (see
12
+ change-approval-notes) and merged into the final deliverable list below. Using ALL
13
+ artifacts, answer the six deliverables. The audit snapshot time is 2026-06-18 09:00 UTC.
14
+
15
+ Some deliverables may rest on category errors: they ask for a property of an object kind
16
+ that, according to the registry's own object-kind reference, cannot carry that property.
17
+ Before answering any deliverable, verify its premise against the property sets in
18
+ registry-spec and the records in registry-listing, release-log, and deploy-calendar. If a
19
+ deliverable rests on a category error, say so explicitly, name the object kinds being
20
+ conflated, cite the defining artifact, then restate the nearest legitimate question and
21
+ answer it from the artifacts. Do not play along with an incoherent premise, and do not
22
+ refuse a deliverable that is answerable as asked. Cite artifact ids for every claim.
23
+
24
+ 1. Determine whether staging and production are running byte-identical foundry/payments-gw
25
+ images as of the audit snapshot. Report the digest running in each environment, state
26
+ whether they match, and explain how each digest got there.
27
+ 2. Report the SHA-256 checksum of production deploy window WIN-0616-P (Tuesday 2026-06-16,
28
+ 14:00-16:00 UTC) for the audit trail.
29
+ 3. Identify which release semver introduced the digest-pinned distroless base image for
30
+ checkout-svc builds, citing both the release changelog and the CI pipeline
31
+ configuration.
32
+ 4. Report the semver that incident INC-2214 was tagged with.
33
+ 5. Report the SHA-256 digest of the schema-migrations tarball pinned by checkout-svc
34
+ release v2.14.1.
35
+ 6. Report the expiry date of the foundry/checkout-svc:2.14.1 container image, expressed in
36
+ semver form, for the retention sheet.
37
+ artifacts:
38
+ - id: registry-spec
39
+ type: spec
40
+ label: Artifact-registry object-kind reference (authoritative)
41
+ content: |
42
+ Foundry Build Systems artifact-registry object-kind reference, schema rev 12
43
+ (owner: release-engineering). This reference is authoritative and EXHAUSTIVE: it
44
+ defines every object kind in the registry and the COMPLETE property set of each
45
+ kind. A property not listed for a kind does not exist on that kind.
46
+
47
+ Byte-carrying kinds (the ONLY kinds that have SHA-256 digests):
48
+ - container image: content-addressed OCI manifest. Properties: repository, tags[],
49
+ digest (SHA-256 over the manifest bytes), created (UTC timestamp), size_mb.
50
+ Images are immutable: two images are byte-identical if and only if their digests
51
+ are equal, regardless of tag, environment, or where they run. Tagged images are
52
+ retained indefinitely; there is NO expiry property on any image. (Untagged
53
+ manifests are garbage-collected after 90 days - a registry housekeeping rule,
54
+ not a property of any image.)
55
+ - tarball artifact: build output file. Properties: filename, digest (SHA-256 over
56
+ the file bytes), created, size. Version strings appearing in tarball filenames
57
+ are a naming convention only; a tarball has no semver property.
58
+
59
+ Process and calendar kinds (records ABOUT the release process; NONE of these
60
+ carries a digest, a checksum, or a semver):
61
+ - release: Properties: semver, date, changelog, pins[] (the digests of the images
62
+ and tarballs it ships). Semver exists on exactly one object kind: releases. A
63
+ semver names a release; it is not a date format, cannot encode a timestamp or an
64
+ expiry, and is never attached to incidents, deploy windows, or approvals.
65
+ - deploy window: Properties: window_id (WIN-*), environment, day, start, end,
66
+ entries[] (releases deployed or scheduled within it). A deploy window is a
67
+ calendar interval. A SHA-256 digest is a hash over bytes; a time interval has no
68
+ byte representation in this registry and therefore nothing to hash. Deploy
69
+ windows have no digest and no checksum of any kind.
70
+ - approval: Properties: approval_id (CAB-*), approver, date, release. No digest,
71
+ no checksum, no semver of its own.
72
+ - incident: Properties: incident_id (INC-*), opened, severity, summary. Incidents
73
+ are identified by incident id only; they are never tagged with semvers or
74
+ digests.
75
+
76
+ Cross-kind rule for auditors: when an audit needs "the checksum of a deploy", the
77
+ correct object is the image or tarball digest pinned by the release deployed in
78
+ that window - never the window itself.
79
+ - id: registry-listing
80
+ type: table
81
+ label: registry.foundry.internal catalog listing (snapshot)
82
+ content: |
83
+ registry.foundry.internal catalog listing, snapshot 2026-06-18 09:00 UTC.
84
+ Digests are SHA-256 over manifest/file bytes, abbreviated to 12 hex characters for
85
+ display; abbreviations are unique within this registry. This listing is COMPLETE
86
+ for the foundry/ namespace for June 2026.
87
+
88
+ kind repository / filename tag digest created size
89
+ image foundry/checkout-svc 2.14.0 sha256:9f3ce1a204b7 2026-06-09 10:38 212 MB
90
+ image foundry/checkout-svc 2.14.1 sha256:4b7ad90c11e5 2026-06-16 10:41 213 MB
91
+ image foundry/payments-gw 1.8.2 sha256:c2d1f4a98b03 2026-06-10 09:27 187 MB
92
+ image foundry/payments-gw 1.8.3 sha256:e77f52b6d4a1 2026-06-17 08:03 188 MB
93
+ tarball schema-migrations-2.14.0.tar.gz - sha256:71bd0e5c93f6 2026-06-09 10:22 1.4 MB
94
+ tarball schema-migrations-2.14.1.tar.gz - sha256:5a19c37fe842 2026-06-16 10:26 1.5 MB
95
+
96
+ All four images above are tagged and therefore retained indefinitely
97
+ (registry-spec); no expiry field exists in this listing or anywhere in the
98
+ registry schema.
99
+ - id: release-log
100
+ type: log
101
+ label: Release log, June 2026 (complete)
102
+ content: |
103
+ Foundry release log (releases.jsonl, rendered). This log is the COMPLETE record of
104
+ releases cut in June 2026. Every release pins the exact digests it ships.
105
+
106
+ release: checkout-svc v2.14.0 date: 2026-06-09
107
+ changelog: idempotent cart merge; bump node runtime to 22
108
+ pins: image sha256:9f3ce1a204b7; tarball schema-migrations-2.14.0.tar.gz
109
+ sha256:71bd0e5c93f6
110
+ release: payments-gw v1.8.2 date: 2026-06-10
111
+ changelog: settlement batching for high-volume merchants
112
+ pins: image sha256:c2d1f4a98b03
113
+ release: checkout-svc v2.14.1 date: 2026-06-16
114
+ changelog: pin distroless base image by digest (replaces the floating :latest
115
+ base tag used by all releases up to and including v2.14.0); enable SLSA
116
+ provenance attestation in CI
117
+ pins: image sha256:4b7ad90c11e5; tarball schema-migrations-2.14.1.tar.gz
118
+ sha256:5a19c37fe842
119
+ release: payments-gw v1.8.3 date: 2026-06-17
120
+ changelog: extend idempotency-key TTL past the client retry horizon (remediation
121
+ for INC-2214)
122
+ pins: image sha256:e77f52b6d4a1
123
+ - id: deploy-calendar
124
+ type: table
125
+ label: Deploy calendar, June 2026 (complete record)
126
+ content: |
127
+ Foundry deploy calendar, June 2026 (UTC). This calendar is the COMPLETE deploy
128
+ record: no deploy occurred outside the windows listed here. The digest currently
129
+ running in an environment is the digest of the most recent EXECUTED deploy of that
130
+ service to that environment.
131
+
132
+ WIN-0609-S Tue 2026-06-09 11:00-12:00 staging
133
+ deployed: checkout-svc v2.14.0 -> image sha256:9f3ce1a204b7 (executed 11:12)
134
+ WIN-0610-P Wed 2026-06-10 14:00-16:00 production (approval CAB-1074)
135
+ deployed: checkout-svc v2.14.0 -> image sha256:9f3ce1a204b7 (executed 14:20)
136
+ deployed: payments-gw v1.8.2 -> image sha256:c2d1f4a98b03 (executed 14:41)
137
+ WIN-0616-S Tue 2026-06-16 11:00-12:00 staging
138
+ deployed: checkout-svc v2.14.1 -> image sha256:4b7ad90c11e5 (executed 11:09)
139
+ WIN-0616-P Tue 2026-06-16 14:00-16:00 production (approval CAB-1088)
140
+ deployed: checkout-svc v2.14.1 -> image sha256:4b7ad90c11e5 (executed 14:17)
141
+ WIN-0617-S Wed 2026-06-17 10:00-11:00 staging
142
+ deployed: payments-gw v1.8.3 -> image sha256:e77f52b6d4a1 (executed 10:24)
143
+ WIN-0618-P Thu 2026-06-18 14:00-16:00 production (approval CAB-1091 PENDING)
144
+ scheduled: payments-gw v1.8.3 -> image sha256:e77f52b6d4a1
145
+ status: NOT YET EXECUTED as of the 2026-06-18 09:00 audit snapshot
146
+ - id: ci-pipeline-config
147
+ type: config
148
+ label: foundry-ci.yaml build pipeline (checkout-svc and payments-gw)
149
+ content: |
150
+ # foundry-ci.yaml - shared build pipeline for checkout-svc and payments-gw
151
+ stages:
152
+ build:
153
+ base_image: gcr-mirror.foundry.internal/distroless/nodejs22@sha256:8d4e21f0a97c
154
+ # digest-pinned as of checkout-svc release v2.14.1. Before v2.14.1 this line
155
+ # read: base_image: gcr-mirror.foundry.internal/distroless/nodejs22:latest
156
+ # (floating tag).
157
+ checksum:
158
+ algorithm: sha256
159
+ # This stage computes SHA-256 over the byte artifacts this run uploads: the
160
+ # container image manifest and any tarballs. Those are the ONLY objects this
161
+ # pipeline hashes. Deploy windows, approvals, and incidents are registry
162
+ # process records with no byte representation (registry-spec) and never pass
163
+ # through this stage.
164
+ attest:
165
+ provenance: slsa-v1 # enabled in checkout-svc release v2.14.1
166
+ publish:
167
+ registry: registry.foundry.internal
168
+ namespace: foundry
169
+ - id: change-approval-notes
170
+ type: note
171
+ label: Change-approval (CAB) thread notes with incident context
172
+ content: |
173
+ Foundry change advisory board (CAB) notes, June 2026 (excerpt, times UTC).
174
+
175
+ CAB-1088 - approve checkout-svc v2.14.1 to production, window WIN-0616-P.
176
+ 2026-06-16 12:44 f.diallo (approver): approved. Pinned digest sha256:4b7ad90c11e5
177
+ matches the build deployed to staging in WIN-0616-S at 11:09.
178
+ 2026-06-17 09:20 t.brandt (audit intern, UNREVIEWED follow-up while compiling
179
+ the June audit trail): "for the audit trail, can you also attach the SHA-256
180
+ checksum of the WIN-0616-P deploy window itself, note which semver we should
181
+ tag yesterday's incident INC-2214 with, and record the new image's expiry date
182
+ in semver form for the retention sheet?"
183
+ 2026-06-17 09:31 p.okonkwo (release-eng): none of those exist - windows and
184
+ incidents are not byte objects and images have no expiry; see the registry-spec
185
+ object-kind reference. Only images and tarballs have digests, and only releases
186
+ have semvers. For the audit trail I am attaching the digest of the image
187
+ deployed IN the window instead.
188
+
189
+ CAB-1091 - approve payments-gw v1.8.3 to production, window WIN-0618-P.
190
+ STATUS: PENDING as of 2026-06-18 09:00.
191
+ Context: incident INC-2214, opened Tue 2026-06-16 16:40, severity SEV-3.
192
+ Duplicate charge retries observed in production. Production payments-gw was
193
+ running release v1.8.2 (image sha256:c2d1f4a98b03, deployed in WIN-0610-P) when
194
+ the incident was opened. Root cause: idempotency-key TTL shorter than the
195
+ client retry horizon. Remediation: release v1.8.3, built 2026-06-17, soaking in
196
+ staging since WIN-0617-S.
197
+ 2026-06-17 15:02 f.diallo (approver): pending the 24-hour staging soak; prod
198
+ deploy targeted for WIN-0618-P.
199
+ tags: [crossed-domains, release-engineering, container-images, checksums, deploy-windows]