@pentatonic-ai/ai-agent-sdk 0.6.0 → 0.7.0

package/packages/memory-engine/bench/scorecards-engine/agent-coding__pentatonic-baseline__20260427-142523.json

@@ -0,0 +1,1115 @@
+{
+  "bench": "agent-coding",
+  "stack": "pentatonic-baseline",
+  "n_tasks": 22,
+  "n_correct": 20,
+  "accuracy": 0.9090909090909091,
+  "mean_score": 0.9545454545454546,
+  "p50_search_ms": 1267.9544819984585,
+  "p95_search_ms": 1691.2675759536796,
+  "total_tokens_in": 0,
+  "total_tokens_out": 0,
+  "total_usd": 0.0,
+  "by_tag": {
+    "api-signature": {
+      "n": 3,
+      "mean_score": 1.0,
+      "accuracy": 1.0
+    },
+    "bug-fix-recall": {
+      "n": 3,
+      "mean_score": 1.0,
+      "accuracy": 1.0
+    },
+    "deprecation": {
+      "n": 3,
+      "mean_score": 1.0,
+      "accuracy": 1.0
+    },
+    "config-recall": {
+      "n": 3,
+      "mean_score": 1.0,
+      "accuracy": 1.0
+    },
+    "pr-rationale": {
+      "n": 2,
+      "mean_score": 1.0,
+      "accuracy": 1.0
+    },
+    "cross-file-refactor": {
+      "n": 2,
+      "mean_score": 1.0,
+      "accuracy": 1.0
+    },
+    "stack-trace-match": {
+      "n": 1,
+      "mean_score": 1.0,
+      "accuracy": 1.0
+    },
+    "temporal": {
+      "n": 2,
+      "mean_score": 1.0,
+      "accuracy": 1.0
+    },
+    "contradiction": {
+      "n": 1,
+      "mean_score": 1.0,
+      "accuracy": 1.0
+    },
+    "rubric": {
+      "n": 2,
+      "mean_score": 0.5,
+      "accuracy": 0.0
+    }
+  },
+  "extra": {
+    "ingest_ms": 63144.58434400149,
+    "grading": "substring",
+    "limit": 3,
+    "tokens": {
+      "corpus_tokens": 5164,
+      "query_tokens": 339,
+      "context_tokens": 9063,
+      "retrieval_tokens": 9402,
+      "naive_tokens": 113947,
+      "saved_tokens": 104545,
+      "reduction_pct": 0.9174879549264131,
+      "mean_retrieval_tokens_per_task": 427.3636363636364,
+      "tokenizer": "cl100k_base",
+      "per_task": {
+        "api-invoice-signature": {
+          "query": 18,
+          "context": 455,
+          "retrieval": 473,
+          "judge_in": 0,
+          "judge_out": 0,
+          "judge_latency_ms": 0.0
+        },
+        "api-search-shape": {
+          "query": 16,
+          "context": 451,
+          "retrieval": 467,
+          "judge_in": 0,
+          "judge_out": 0,
+          "judge_latency_ms": 0.0
+        },
+        "api-invoice-amount-type": {
+          "query": 16,
+          "context": 430,
+          "retrieval": 446,
+          "judge_in": 0,
+          "judge_out": 0,
+          "judge_latency_ms": 0.0
+        },
+        "bugfix-stripe-rounding": {
+          "query": 10,
+          "context": 330,
+          "retrieval": 340,
+          "judge_in": 0,
+          "judge_out": 0,
+          "judge_latency_ms": 0.0
+        },
+        "bugfix-webhook-dup": {
+          "query": 10,
+          "context": 324,
+          "retrieval": 334,
+          "judge_in": 0,
+          "judge_out": 0,
+          "judge_latency_ms": 0.0
+        },
+        "bugfix-rounding-direction": {
+          "query": 19,
+          "context": 376,
+          "retrieval": 395,
+          "judge_in": 0,
+          "judge_out": 0,
+          "judge_latency_ms": 0.0
+        },
+        "deprecation-v1": {
+          "query": 11,
+          "context": 423,
+          "retrieval": 434,
+          "judge_in": 0,
+          "judge_out": 0,
+          "judge_latency_ms": 0.0
+        },
+        "deprecation-hs256": {
+          "query": 11,
+          "context": 431,
+          "retrieval": 442,
+          "judge_in": 0,
+          "judge_out": 0,
+          "judge_latency_ms": 0.0
+        },
+        "deprecation-search-tuple": {
+          "query": 13,
+          "context": 412,
+          "retrieval": 425,
+          "judge_in": 0,
+          "judge_out": 0,
+          "judge_latency_ms": 0.0
+        },
+        "config-retry-policy-base": {
+          "query": 15,
+          "context": 410,
+          "retrieval": 425,
+          "judge_in": 0,
+          "judge_out": 0,
+          "judge_latency_ms": 0.0
+        },
+        "config-session-ttl": {
+          "query": 14,
+          "context": 445,
+          "retrieval": 459,
+          "judge_in": 0,
+          "judge_out": 0,
+          "judge_latency_ms": 0.0
+        },
+        "config-search-hybrid-weight": {
+          "query": 11,
+          "context": 451,
+          "retrieval": 462,
+          "judge_in": 0,
+          "judge_out": 0,
+          "judge_latency_ms": 0.0
+        },
+        "pr-rationale-retry": {
+          "query": 19,
+          "context": 409,
+          "retrieval": 428,
+          "judge_in": 0,
+          "judge_out": 0,
+          "judge_latency_ms": 0.0
+        },
+        "pr-rationale-jwt-cleanup": {
+          "query": 13,
+          "context": 350,
+          "retrieval": 363,
+          "judge_in": 0,
+          "judge_out": 0,
+          "judge_latency_ms": 0.0
+        },
+        "cross-ref-invoicing": {
+          "query": 14,
+          "context": 472,
+          "retrieval": 486,
+          "judge_in": 0,
+          "judge_out": 0,
+          "judge_latency_ms": 0.0
+        },
+        "cross-ref-ratelimit": {
+          "query": 15,
+          "context": 404,
+          "retrieval": 419,
+          "judge_in": 0,
+          "judge_out": 0,
+          "judge_latency_ms": 0.0
+        },
+        "stack-trace-webhook": {
+          "query": 21,
+          "context": 394,
+          "retrieval": 415,
+          "judge_in": 0,
+          "judge_out": 0,
+          "judge_latency_ms": 0.0
+        },
+        "temporal-numpy": {
+          "query": 11,
+          "context": 431,
+          "retrieval": 442,
+          "judge_in": 0,
+          "judge_out": 0,
+          "judge_latency_ms": 0.0
+        },
+        "temporal-on-call": {
+          "query": 20,
+          "context": 432,
+          "retrieval": 452,
+          "judge_in": 0,
+          "judge_out": 0,
+          "judge_latency_ms": 0.0
+        },
+        "contradiction-algorithm": {
+          "query": 16,
+          "context": 396,
+          "retrieval": 412,
+          "judge_in": 0,
+          "judge_out": 0,
+          "judge_latency_ms": 0.0
+        },
+        "rubric-security-advice": {
+          "query": 25,
+          "context": 444,
+          "retrieval": 469,
+          "judge_in": 757,
+          "judge_out": 37,
+          "judge_latency_ms": 1715.3592600226402
+        },
+        "rubric-incident-lesson": {
+          "query": 21,
+          "context": 393,
+          "retrieval": 414,
+          "judge_in": 687,
+          "judge_out": 35,
+          "judge_latency_ms": 1701.407715022564
+        }
+      },
+      "judge_tokens_in": 1444,
+      "judge_tokens_out": 72,
+      "judge_calls": 2,
+      "judge_mean_latency_ms": 1708.383487522602
+    },
+    "cost_usd": {
+      "assumed_completion_tokens_per_task": 100,
+      "rates": {
+        "input_per_1k": 0.0025,
+        "output_per_1k": 0.01,
+        "model": "gpt-4o"
+      },
+      "retrieval_usd_in": 0.023504999999999998,
+      "retrieval_usd_out": 0.022,
+      "retrieval_usd_total": 0.045505,
+      "naive_usd_total": 0.3068675,
+      "saved_usd": 0.2613625,
+      "saved_usd_per_1k_tasks": 11.880113636363637
+    }
+  },
+  "task_results": [
+    {
+      "task_id": "api-invoice-signature",
+      "query": "What's the request body shape for POST /v2/invoices on billing-svc?",
+      "answer": "## billing-svc: POST /v2/invoices\n\nCreate an invoice for a customer.\n\n**Request body:**\n```json\n{\"customer_id\": \"cus_abc\", \"amount_cents\": 1299, \"currency\": \"gbp\", \"description\": \"Annual plan\"}\n```\n\n**Responses:**\n- `201` \u2014 invoice created: `{\"id\": \"inv_xyz\", \"status\": \"draft\", \"pdf_url\": \"...\"}`\n- `400` \u2014 validation error\n- `404` \u2014 customer not found\n- `409` \u2014 duplicate idempotency_key\n\n**Headers:**\n- `Idempotency-Key` (optional, required for retries)\n\nAmounts MUST be integers (cents). Passing\n---\n# Cross-reference: invoicing pipeline\n\nEnd-to-end path for an invoice being paid:\n\n1. `billing-svc` creates the invoice (see POST /v2/invoices, api-billing-invoice)\n2. Customer pays via Stripe portal (no internal code path)\n3. Stripe posts a `invoice.payment_succeeded` webhook to billing-svc\n4. `billing/webhooks.py::handle_webhook` verifies signature, enqueues\n   internal `InvoicePaid` event on `order-svc`\n5. `order-svc` marks order as paid, emits `OrderPaid` on pubsub\n6. `reporting-svc` consume\n---\n## search-svc: POST /v3/search\n\nVector + keyword hybrid search.\n\n**Request body:**\n```json\n{\"query\": \"widget\", \"collection\": \"products\", \"limit\": 20, \"filter\": {\"category\": \"electronics\"}}\n```\n\n**Responses:**\n- `200` \u2014 `{\"results\": [{\"id\": \"p_123\", \"score\": 0.81, \"payload\": {...}}, ...], \"total\": 42}`\n- `400` \u2014 malformed filter\n- `429` \u2014 rate-limited\n\n**Notes:**\n- Results are ranked by hybrid (0.6 vector + 0.4 BM25) \u2014 override with\n  `hybrid_weight` in the body (0.0 = pure BM25, 1.0 = pure vecto",
+      "hits": [
+        {
+          "text": "## billing-svc: POST /v2/invoices\n\nCreate an invoice for a customer.\n\n**Request body:**\n```json\n{\"customer_id\": \"cus_abc\", \"amount_cents\": 1299, \"currency\": \"gbp\", \"description\": \"Annual plan\"}\n```\n\n**Responses:**\n- `201` \u2014 invoice created: `{\"id\": \"inv_xyz\", \"status\": \"draft\", \"pdf_url\": \"...\"}`\n- `400` \u2014 validation error\n- `404` \u2014 customer not found\n- `409` \u2014 duplicate idempotency_key\n\n**Headers",
+          "score": 0.7466,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "api-billing-invoice"
+        },
+        {
+          "text": "# Cross-reference: invoicing pipeline\n\nEnd-to-end path for an invoice being paid:\n\n1. `billing-svc` creates the invoice (see POST /v2/invoices, api-billing-invoice)\n2. Customer pays via Stripe portal (no internal code path)\n3. Stripe posts a `invoice.payment_succeeded` webhook to billing-svc\n4. `billing/webhooks.py::handle_webhook` verifies signature, enqueues\n   internal `InvoicePaid` event on `o",
+          "score": 0.6158,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "cross-ref-invoicing"
+        },
+        {
+          "text": "## search-svc: POST /v3/search\n\nVector + keyword hybrid search.\n\n**Request body:**\n```json\n{\"query\": \"widget\", \"collection\": \"products\", \"limit\": 20, \"filter\": {\"category\": \"electronics\"}}\n```\n\n**Responses:**\n- `200` \u2014 `{\"results\": [{\"id\": \"p_123\", \"score\": 0.81, \"payload\": {...}}, ...], \"total\": 42}`\n- `400` \u2014 malformed filter\n- `429` \u2014 rate-limited\n\n**Notes:**\n- Results are ranked by hybrid (0.6",
+          "score": 0.6023,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "api-search-query"
+        }
+      ],
+      "correct": true,
+      "score": 1.0,
+      "grading_notes": "all substrings matched",
+      "search_time_ms": 1254.5839749800507,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 473,
+      "query_tokens": 18,
+      "context_tokens": 455,
+      "judge_tokens_in": 0,
+      "judge_tokens_out": 0,
+      "judge_latency_ms": 0.0
+    },
+    {
+      "task_id": "api-search-shape",
+      "query": "What's the response shape of POST /v3/search on search-svc?",
+      "answer": "## search-svc: POST /v3/search\n\nVector + keyword hybrid search.\n\n**Request body:**\n```json\n{\"query\": \"widget\", \"collection\": \"products\", \"limit\": 20, \"filter\": {\"category\": \"electronics\"}}\n```\n\n**Responses:**\n- `200` \u2014 `{\"results\": [{\"id\": \"p_123\", \"score\": 0.81, \"payload\": {...}}, ...], \"total\": 42}`\n- `400` \u2014 malformed filter\n- `429` \u2014 rate-limited\n\n**Notes:**\n- Results are ranked by hybrid (0.6 vector + 0.4 BM25) \u2014 override with\n  `hybrid_weight` in the body (0.0 = pure BM25, 1.0 = pure vecto\n---\n# search-svc README\n\nOwner: alex. On-call: search-oncall@example.com.\n\n## SDK\n```python\nfrom search_sdk import SearchClient\nclient = SearchClient(url=os.environ['SEARCH_URL'])\nresults = client.search(query='widget', limit=20)\nfor r in results:\n    print(r['id'], r['score'], r['payload']['title'])\n```\nNOTE: pre-c3d4e5f6 the return shape was `list[tuple[int, float]]`. If you\nsee that in an old service, it's using the legacy SDK \u2014 upgrade to\nsearch-sdk >= 3.0.\n\n## Environment\n- `SEARCH_URL` (defaul\n---\n# search_sdk/client.py (excerpt)\n\n```python\nimport httpx\n\nclass SearchClient:\n    def __init__(self, url: str, api_key: str | None = None):\n        self.url = url.rstrip('/')\n        self._http = httpx.Client(headers={'Authorization': f'Bearer {api_key}'} if api_key else {})\n\n    def search(self, query: str, *, collection: str = 'products', limit: int = 20,\n               filter: dict | None = None, hybrid_weight: float = 0.6) -> list[dict]:\n        # Post-c3d4e5f6: returns list[dict] not list[t",
+      "hits": [
+        {
+          "text": "## search-svc: POST /v3/search\n\nVector + keyword hybrid search.\n\n**Request body:**\n```json\n{\"query\": \"widget\", \"collection\": \"products\", \"limit\": 20, \"filter\": {\"category\": \"electronics\"}}\n```\n\n**Responses:**\n- `200` \u2014 `{\"results\": [{\"id\": \"p_123\", \"score\": 0.81, \"payload\": {...}}, ...], \"total\": 42}`\n- `400` \u2014 malformed filter\n- `429` \u2014 rate-limited\n\n**Notes:**\n- Results are ranked by hybrid (0.6",
+          "score": 0.6658,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "api-search-query"
+        },
+        {
+          "text": "# search-svc README\n\nOwner: alex. On-call: search-oncall@example.com.\n\n## SDK\n```python\nfrom search_sdk import SearchClient\nclient = SearchClient(url=os.environ['SEARCH_URL'])\nresults = client.search(query='widget', limit=20)\nfor r in results:\n    print(r['id'], r['score'], r['payload']['title'])\n```\nNOTE: pre-c3d4e5f6 the return shape was `list[tuple[int, float]]`. If you\nsee that in an old servi",
+          "score": 0.6209,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "readme-search"
+        },
+        {
+          "text": "# search_sdk/client.py (excerpt)\n\n```python\nimport httpx\n\nclass SearchClient:\n    def __init__(self, url: str, api_key: str | None = None):\n        self.url = url.rstrip('/')\n        self._http = httpx.Client(headers={'Authorization': f'Bearer {api_key}'} if api_key else {})\n\n    def search(self, query: str, *, collection: str = 'products', limit: int = 20,\n               filter: dict | None = Non",
+          "score": 0.6005,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "code-search-client"
+        }
+      ],
+      "correct": true,
+      "score": 1.0,
+      "grading_notes": "all substrings matched",
+      "search_time_ms": 1246.5275280119386,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 467,
+      "query_tokens": 16,
+      "context_tokens": 451,
+      "judge_tokens_in": 0,
+      "judge_tokens_out": 0,
+      "judge_latency_ms": 0.0
+    },
+    {
+      "task_id": "api-invoice-amount-type",
+      "query": "When calling POST /v2/invoices, must amount be integer or float?",
+      "answer": "## billing-svc: POST /v2/invoices\n\nCreate an invoice for a customer.\n\n**Request body:**\n```json\n{\"customer_id\": \"cus_abc\", \"amount_cents\": 1299, \"currency\": \"gbp\", \"description\": \"Annual plan\"}\n```\n\n**Responses:**\n- `201` \u2014 invoice created: `{\"id\": \"inv_xyz\", \"status\": \"draft\", \"pdf_url\": \"...\"}`\n- `400` \u2014 validation error\n- `404` \u2014 customer not found\n- `409` \u2014 duplicate idempotency_key\n\n**Headers:**\n- `Idempotency-Key` (optional, required for retries)\n\nAmounts MUST be integers (cents). Passing\n---\n# tests/test_payments.py\n\n```python\nimport math\nfrom billing.stripe_client import StripeClient\n\ndef test_rounding():\n    # Reproduction of the float-amount bug fixed in a1b2c3d4\n    client = StripeClient(api_key='sk_test_xxx')\n    # Amount comes in as a float due to upstream division\n    amount_float = 12.99001\n    amount_cents = client._to_cents(amount_float)\n    # Must be an int, rounded DOWN (never overcharge)\n    assert isinstance(amount_cents, int)\n    assert amount_cents == 1299\n\ndef test_\n---\n# billing/stripe_client.py (excerpt)\n\n```python\nimport math\nimport stripe\n\nclass StripeClient:\n    def __init__(self, api_key: str, retry_policy: dict | None = None):\n        stripe.api_key = api_key\n        self.retry_policy = retry_policy or {'base_ms': 200, 'max_ms': 8000, 'max_retries': 5, 'jitter': True}\n\n    @staticmethod\n    def _to_cents(amount: float) -> int:\n        # Floor towards zero so we never overcharge (see a1b2c3d4).\n        sign = 1 if amount >= 0 else -1\n        return sign *",
+      "hits": [
+        {
+          "text": "## billing-svc: POST /v2/invoices\n\nCreate an invoice for a customer.\n\n**Request body:**\n```json\n{\"customer_id\": \"cus_abc\", \"amount_cents\": 1299, \"currency\": \"gbp\", \"description\": \"Annual plan\"}\n```\n\n**Responses:**\n- `201` \u2014 invoice created: `{\"id\": \"inv_xyz\", \"status\": \"draft\", \"pdf_url\": \"...\"}`\n- `400` \u2014 validation error\n- `404` \u2014 customer not found\n- `409` \u2014 duplicate idempotency_key\n\n**Headers",
+          "score": 0.6832,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "api-billing-invoice"
+        },
+        {
+          "text": "# tests/test_payments.py\n\n```python\nimport math\nfrom billing.stripe_client import StripeClient\n\ndef test_rounding():\n    # Reproduction of the float-amount bug fixed in a1b2c3d4\n    client = StripeClient(api_key='sk_test_xxx')\n    # Amount comes in as a float due to upstream division\n    amount_float = 12.99001\n    amount_cents = client._to_cents(amount_float)\n    # Must be an int, rounded DOWN (n",
+          "score": 0.649,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "test-payments-rounding"
+        },
+        {
+          "text": "# billing/stripe_client.py (excerpt)\n\n```python\nimport math\nimport stripe\n\nclass StripeClient:\n    def __init__(self, api_key: str, retry_policy: dict | None = None):\n        stripe.api_key = api_key\n        self.retry_policy = retry_policy or {'base_ms': 200, 'max_ms': 8000, 'max_retries': 5, 'jitter': True}\n\n    @staticmethod\n    def _to_cents(amount: float) -> int:\n        # Floor towards zero ",
+          "score": 0.6122,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "code-stripe-client"
+        }
+      ],
+      "correct": true,
+      "score": 1.0,
+      "grading_notes": "all substrings matched",
+      "search_time_ms": 1750.7977380009834,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 446,
+      "query_tokens": 16,
+      "context_tokens": 430,
+      "judge_tokens_in": 0,
+      "judge_tokens_out": 0,
+      "judge_latency_ms": 0.0
+    },
+    {
+      "task_id": "bugfix-stripe-rounding",
+      "query": "Which commit fixed the stripe float-amount bug?",
+      "answer": "commit a1b2c3d4\nAuthor: jordan@example.com\nDate: 2026-02-10T14:30:00Z\n\nfix(payments): round stripe amounts to cents before submit\n\nStripe.charges.create() was receiving floats like 12.99001, which the API\nrejects with amount_integer_required. Added `math.floor(amount_cents)`\nbefore the call. Unit test added in tests/test_payments.py::test_rounding.\n---\ncommit a1b2c3d4\nAuthor: jordan@example.com\nDate: 2026-02-10T14:30:00Z\n\nfix(payments): round stripe amounts to cents before submit\n\nStripe.charges.create() was receiving floats like 12.99001, which the API\nrejects with amount_integer_required. Added `math.floor(amount_cents)`\nbefore the call. Unit test added in tests/test_payments.py::test_rounding.\n---\n# tests/test_payments.py\n\n```python\nimport math\nfrom billing.stripe_client import StripeClient\n\ndef test_rounding():\n    # Reproduction of the float-amount bug fixed in a1b2c3d4\n    client = StripeClient(api_key='sk_test_xxx')\n    # Amount comes in as a float due to upstream division\n    amount_float = 12.99001\n    amount_cents = client._to_cents(amount_float)\n    # Must be an int, rounded DOWN (never overcharge)\n    assert isinstance(amount_cents, int)\n    assert amount_cents == 1299\n\ndef test_",
+      "hits": [
+        {
+          "text": "commit a1b2c3d4\nAuthor: jordan@example.com\nDate: 2026-02-10T14:30:00Z\n\nfix(payments): round stripe amounts to cents before submit\n\nStripe.charges.create() was receiving floats like 12.99001, which the API\nrejects with amount_integer_required. Added `math.floor(amount_cents)`\nbefore the call. Unit test added in tests/test_payments.py::test_rounding.",
+          "score": 0.7629,
+          "source": "pentatonic-baseline",
+          "doc_id": "commit-a1b2c3"
+        },
+        {
+          "text": "commit a1b2c3d4\nAuthor: jordan@example.com\nDate: 2026-02-10T14:30:00Z\n\nfix(payments): round stripe amounts to cents before submit\n\nStripe.charges.create() was receiving floats like 12.99001, which the API\nrejects with amount_integer_required. Added `math.floor(amount_cents)`\nbefore the call. Unit test added in tests/test_payments.py::test_rounding.",
+          "score": 0.6804,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "commit-a1b2c3"
+        },
+        {
+          "text": "# tests/test_payments.py\n\n```python\nimport math\nfrom billing.stripe_client import StripeClient\n\ndef test_rounding():\n    # Reproduction of the float-amount bug fixed in a1b2c3d4\n    client = StripeClient(api_key='sk_test_xxx')\n    # Amount comes in as a float due to upstream division\n    amount_float = 12.99001\n    amount_cents = client._to_cents(amount_float)\n    # Must be an int, rounded DOWN (n",
+          "score": 0.6632,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "test-payments-rounding"
+        }
+      ],
+      "correct": true,
+      "score": 1.0,
+      "grading_notes": "all substrings matched",
+      "search_time_ms": 1239.0598890196998,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 340,
+      "query_tokens": 10,
+      "context_tokens": 330,
+      "judge_tokens_in": 0,
+      "judge_tokens_out": 0,
+      "judge_latency_ms": 0.0
+    },
+    {
+      "task_id": "bugfix-webhook-dup",
+      "query": "How did we fix the duplicate Stripe webhook handling?",
+      "answer": "Issue #88: Stripe webhook handler processes duplicate events\nReporter: support\nStatus: resolved\n\nProd saw the same event_id land in webhook_events twice when Stripe retried\na 500. Handler was non-idempotent \u2014 it assumed first-delivery. Fix:\nadded unique constraint on webhook_events.stripe_event_id; handler now\nattempts INSERT ON CONFLICT DO NOTHING and treats the duplicate as a no-op.\nMerged PR #76. Closed 2026-02-26.\n---\nIssue #88: Stripe webhook handler processes duplicate events\nReporter: support\nStatus: resolved\n\nProd saw the same event_id land in webhook_events twice when Stripe retried\na 500. Handler was non-idempotent \u2014 it assumed first-delivery. Fix:\nadded unique constraint on webhook_events.stripe_event_id; handler now\nattempts INSERT ON CONFLICT DO NOTHING and treats the duplicate as a no-op.\nMerged PR #76. Closed 2026-02-26.\n---\n# billing-svc runbook: webhook signature errors\n\nSymptom: logs show `stripe.error.SignatureVerificationError`.\n\nDiagnosis:\n1. Confirm the webhook endpoint URL matches Stripe dashboard\n2. Check `STRIPE_WEBHOOK_SECRET` matches the secret in Stripe dashboard\n3. If secret was rotated recently, `kubectl rollout restart deployment/billing-svc`\n\nWe hit this on 2026-03-02 14:15 UTC \u2014 secret rotated but pods didn't reload\nuntil restart. See incident post-mortem in ops/incidents/2026-03-02-webhook.md.",
+      "hits": [
+        {
+          "text": "Issue #88: Stripe webhook handler processes duplicate events\nReporter: support\nStatus: resolved\n\nProd saw the same event_id land in webhook_events twice when Stripe retried\na 500. Handler was non-idempotent \u2014 it assumed first-delivery. Fix:\nadded unique constraint on webhook_events.stripe_event_id; handler now\nattempts INSERT ON CONFLICT DO NOTHING and treats the duplicate as a no-op.\nMerged PR #7",
+          "score": 0.7903,
+          "source": "pentatonic-baseline",
+          "doc_id": "issue-88-webhook-dup"
+        },
+        {
+          "text": "Issue #88: Stripe webhook handler processes duplicate events\nReporter: support\nStatus: resolved\n\nProd saw the same event_id land in webhook_events twice when Stripe retried\na 500. Handler was non-idempotent \u2014 it assumed first-delivery. Fix:\nadded unique constraint on webhook_events.stripe_event_id; handler now\nattempts INSERT ON CONFLICT DO NOTHING and treats the duplicate as a no-op.\nMerged PR #7",
+          "score": 0.7427,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "issue-88-webhook-dup"
+        },
+        {
+          "text": "# billing-svc runbook: webhook signature errors\n\nSymptom: logs show `stripe.error.SignatureVerificationError`.\n\nDiagnosis:\n1. Confirm the webhook endpoint URL matches Stripe dashboard\n2. Check `STRIPE_WEBHOOK_SECRET` matches the secret in Stripe dashboard\n3. If secret was rotated recently, `kubectl rollout restart deployment/billing-svc`\n\nWe hit this on 2026-03-02 14:15 UTC \u2014 secret rotated but po",
+          "score": 0.6326,
+          "source": "pentatonic-baseline",
+          "doc_id": "runbook-billing"
+        }
+      ],
+      "correct": true,
+      "score": 1.0,
+      "grading_notes": "all substrings matched",
+      "search_time_ms": 1166.5845589886885,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 334,
+      "query_tokens": 10,
+      "context_tokens": 324,
+      "judge_tokens_in": 0,
+      "judge_tokens_out": 0,
+      "judge_latency_ms": 0.0
+    },
+    {
+      "task_id": "bugfix-rounding-direction",
+      "query": "When converting a float amount to cents, do we round up, down, or to nearest?",
+      "answer": "# tests/test_payments.py\n\n```python\nimport math\nfrom billing.stripe_client import StripeClient\n\ndef test_rounding():\n    # Reproduction of the float-amount bug fixed in a1b2c3d4\n    client = StripeClient(api_key='sk_test_xxx')\n    # Amount comes in as a float due to upstream division\n    amount_float = 12.99001\n    amount_cents = client._to_cents(amount_float)\n    # Must be an int, rounded DOWN (never overcharge)\n    assert isinstance(amount_cents, int)\n    assert amount_cents == 1299\n\ndef test_\n---\n# billing/stripe_client.py (excerpt)\n\n```python\nimport math\nimport stripe\n\nclass StripeClient:\n    def __init__(self, api_key: str, retry_policy: dict | None = None):\n        stripe.api_key = api_key\n        self.retry_policy = retry_policy or {'base_ms': 200, 'max_ms': 8000, 'max_retries': 5, 'jitter': True}\n\n    @staticmethod\n    def _to_cents(amount: float) -> int:\n        # Floor towards zero so we never overcharge (see a1b2c3d4).\n        sign = 1 if amount >= 0 else -1\n        return sign *\n---\ncommit a1b2c3d4\nAuthor: jordan@example.com\nDate: 2026-02-10T14:30:00Z\n\nfix(payments): round stripe amounts to cents before submit\n\nStripe.charges.create() was receiving floats like 12.99001, which the API\nrejects with amount_integer_required. Added `math.floor(amount_cents)`\nbefore the call. Unit test added in tests/test_payments.py::test_rounding.",
+      "hits": [
+        {
+          "text": "# tests/test_payments.py\n\n```python\nimport math\nfrom billing.stripe_client import StripeClient\n\ndef test_rounding():\n    # Reproduction of the float-amount bug fixed in a1b2c3d4\n    client = StripeClient(api_key='sk_test_xxx')\n    # Amount comes in as a float due to upstream division\n    amount_float = 12.99001\n    amount_cents = client._to_cents(amount_float)\n    # Must be an int, rounded DOWN (n",
+          "score": 0.6878,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "test-payments-rounding"
+        },
+        {
+          "text": "# billing/stripe_client.py (excerpt)\n\n```python\nimport math\nimport stripe\n\nclass StripeClient:\n    def __init__(self, api_key: str, retry_policy: dict | None = None):\n        stripe.api_key = api_key\n        self.retry_policy = retry_policy or {'base_ms': 200, 'max_ms': 8000, 'max_retries': 5, 'jitter': True}\n\n    @staticmethod\n    def _to_cents(amount: float) -> int:\n        # Floor towards zero ",
+          "score": 0.6293,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "code-stripe-client"
+        },
+        {
+          "text": "commit a1b2c3d4\nAuthor: jordan@example.com\nDate: 2026-02-10T14:30:00Z\n\nfix(payments): round stripe amounts to cents before submit\n\nStripe.charges.create() was receiving floats like 12.99001, which the API\nrejects with amount_integer_required. Added `math.floor(amount_cents)`\nbefore the call. Unit test added in tests/test_payments.py::test_rounding.",
+          "score": 0.6111,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "commit-a1b2c3"
+        }
+      ],
+      "correct": true,
+      "score": 1.0,
+      "grading_notes": "all substrings matched",
+      "search_time_ms": 1257.2615180106368,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 395,
+      "query_tokens": 19,
+      "context_tokens": 376,
+      "judge_tokens_in": 0,
+      "judge_tokens_out": 0,
+      "judge_latency_ms": 0.0
+    },
+    {
+      "task_id": "deprecation-v1",
+      "query": "Are the /api/v1/ endpoints still supported?",
+      "answer": "commit d4e5f6a7\nAuthor: jordan@example.com\nDate: 2026-02-17T16:20:00Z\n\ndeprecate: remove legacy /api/v1/ endpoints\n\nAll /api/v1/* routes removed. /api/v2/ has been the default for 9 months.\nCallers still on v1 will get HTTP 410 Gone with a header pointing at the\nv2 equivalent. Full mapping in docs/V1_V2_MIGRATION.md. Sunset announced\n2025-11-01, enforced now.\n---\n[2026-02-15 Slack #platform]\njordan: reminder: /api/v1 is going away in 2 days. anyone still seeing v1 traffic?\nalex: we're clean on search-svc. grep'd all consumers.\npriya: auth is clean, v2-only for 6 months\njordan: cool. flipping the 410 flag on 2026-02-17 09:00 UTC. if your service gets paged, check your callers.\n\n[2026-02-17 Slack #platform]\njordan: v1 is gone. 410s in the last hour: 12. all from an old mobile app build. told them to upgrade. no internal services affected.\n---\n<!-- doc_id: chat-deprecation -->\n[2026-02-15 Slack #platform]\njordan: reminder: /api/v1 is going away in 2 days. anyone still seeing v1 traffic?\nalex: we're clean on search-svc. grep'd all consumers.\npriya: auth is clean, v2-only for 6 months\njordan: cool. flipping the 410 flag on 2026-02-17 09:00 UTC. if your service gets paged, check your callers.\n\n[2026-02-17 Slack #platform]\njordan: v1 is gone. 410s in the last hour: 12. all from an old mobile app build. told them to upgrade. no internal se",
+      "hits": [
+        {
+          "text": "commit d4e5f6a7\nAuthor: jordan@example.com\nDate: 2026-02-17T16:20:00Z\n\ndeprecate: remove legacy /api/v1/ endpoints\n\nAll /api/v1/* routes removed. /api/v2/ has been the default for 9 months.\nCallers still on v1 will get HTTP 410 Gone with a header pointing at the\nv2 equivalent. Full mapping in docs/V1_V2_MIGRATION.md. Sunset announced\n2025-11-01, enforced now.",
+          "score": 0.6597,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "commit-d4e5f6"
+        },
+        {
+          "text": "[2026-02-15 Slack #platform]\njordan: reminder: /api/v1 is going away in 2 days. anyone still seeing v1 traffic?\nalex: we're clean on search-svc. grep'd all consumers.\npriya: auth is clean, v2-only for 6 months\njordan: cool. flipping the 410 flag on 2026-02-17 09:00 UTC. if your service gets paged, check your callers.\n\n[2026-02-17 Slack #platform]\njordan: v1 is gone. 410s in the last hour: 12. all ",
+          "score": 0.6515,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "chat-deprecation"
+        },
+        {
+          "text": "<!-- doc_id: chat-deprecation -->\n[2026-02-15 Slack #platform]\njordan: reminder: /api/v1 is going away in 2 days. anyone still seeing v1 traffic?\nalex: we're clean on search-svc. grep'd all consumers.\npriya: auth is clean, v2-only for 6 months\njordan: cool. flipping the 410 flag on 2026-02-17 09:00 UTC. if your service gets paged, check your callers.\n\n[2026-02-17 Slack #platform]\njordan: v1 is gon",
+          "score": 0.4796,
+          "source": "pentatonic-baseline:L6_documents",
+          "doc_id": "chat-deprecation"
+        }
+      ],
+      "correct": true,
+      "score": 1.0,
+      "grading_notes": "all substrings matched",
+      "search_time_ms": 1249.9748029804323,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 434,
+      "query_tokens": 11,
+      "context_tokens": 423,
+      "judge_tokens_in": 0,
+      "judge_tokens_out": 0,
+      "judge_latency_ms": 0.0
+    },
+    {
+      "task_id": "deprecation-hs256",
+      "query": "Can auth-svc still accept HS256 JWTs?",
+      "answer": "commit b2c3d4e5\nAuthor: priya@example.com\nDate: 2026-02-12T11:15:00Z\n\nfeat(auth): rotate JWT signing key to RS256\n\nWe were using HS256 shared-secret JWTs, which meant any service that verified\ntokens also held the minting secret. Migrated to RS256 asymmetric signing.\nPrivate key lives in auth-svc only; all other services only ever hold the\npublic key. Env var JWT_PRIVATE_KEY_PATH on auth-svc, JWT_PUBLIC_KEY_PATH\neverywhere else. BREAKING: old HS256 tokens rejected after 2026-02-20.\n---\n# auth-svc README\n\nOwner: priya. On-call: auth-oncall@example.com.\n\n## Environment\n- `JWT_PRIVATE_KEY_PATH` (required, RS256 PEM)\n- `JWT_PUBLIC_KEY_PATH` (required, RS256 PEM)\n- `JWT_ISSUER` (default: 'https://auth.example.com')\n- `SESSION_TTL_MINUTES` (default: 60)\n- `REFRESH_TTL_DAYS` (default: 30)\n\n**Do not use HS256**. The migration in b2c3d4e5 removed all HS256 paths.\n\n## Public API\n- `POST /v2/login {email, password}` \u2192 `{access_token, refresh_token}`\n- `POST /v2/refresh {refresh_token}` \u2192\n---\n[2026-02-18 Slack #payments-dev]\npriya: @jordan can you review #47? it's the stripe retry stuff\njordan: on it. Why base=200 and not 500?\npriya: stripe's own SDK uses 250 internally but I went conservative with 200 so we retry faster on transient blips. Max 8s is the ceiling stripe recommends.\njordan: makes sense. one nit \u2014 can we surface jitter=true as configurable? teams with deterministic retries might want to turn it off for tests.\npriya: yep, added. merging.\n\n[2026-02-19 Slack #payments-dev]",
+      "hits": [
+        {
+          "text": "commit b2c3d4e5\nAuthor: priya@example.com\nDate: 2026-02-12T11:15:00Z\n\nfeat(auth): rotate JWT signing key to RS256\n\nWe were using HS256 shared-secret JWTs, which meant any service that verified\ntokens also held the minting secret. Migrated to RS256 asymmetric signing.\nPrivate key lives in auth-svc only; all other services only ever hold the\npublic key. Env var JWT_PRIVATE_KEY_PATH on auth-svc, JWT_",
+          "score": 0.6757,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "commit-b2c3d4"
+        },
+        {
+          "text": "# auth-svc README\n\nOwner: priya. On-call: auth-oncall@example.com.\n\n## Environment\n- `JWT_PRIVATE_KEY_PATH` (required, RS256 PEM)\n- `JWT_PUBLIC_KEY_PATH` (required, RS256 PEM)\n- `JWT_ISSUER` (default: 'https://auth.example.com')\n- `SESSION_TTL_MINUTES` (default: 60)\n- `REFRESH_TTL_DAYS` (default: 30)\n\n**Do not use HS256**. The migration in b2c3d4e5 removed all HS256 paths.\n\n## Public API\n- `POST /",
+          "score": 0.6509,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "readme-auth"
+        },
+        {
+          "text": "[2026-02-18 Slack #payments-dev]\npriya: @jordan can you review #47? it's the stripe retry stuff\njordan: on it. Why base=200 and not 500?\npriya: stripe's own SDK uses 250 internally but I went conservative with 200 so we retry faster on transient blips. Max 8s is the ceiling stripe recommends.\njordan: makes sense. one nit \u2014 can we surface jitter=true as configurable? teams with deterministic retrie",
+          "score": 0.5016,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "chat-pr-47-review"
+        }
+      ],
+      "correct": true,
+      "score": 1.0,
+      "grading_notes": "all substrings matched",
+      "search_time_ms": 1340.946468000766,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 442,
+      "query_tokens": 11,
+      "context_tokens": 431,
+      "judge_tokens_in": 0,
+      "judge_tokens_out": 0,
+      "judge_latency_ms": 0.0
+    },
+    {
+      "task_id": "deprecation-search-tuple",
+      "query": "What does SearchClient.search() return today \u2014 tuples or dicts?",
+      "answer": "commit c3d4e5f6\nAuthor: alex@example.com\nDate: 2026-02-14T09:00:00Z\n\nfeat(search): switch ANN backend from faiss to vector-db\n\nfaiss-cpu was hitting ~4s p99 on 2M vectors. Vector-DB HNSW on the same\ncorpus gives 40ms p99. Breaking: the SearchClient.search() return shape\nchanged \u2014 was list[tuple[int, float]], now list[dict] with keys {id, score,\npayload}. Migration guide in docs/SEARCH_MIGRATION.md.\n---\n# search-svc README\n\nOwner: alex. On-call: search-oncall@example.com.\n\n## SDK\n```python\nfrom search_sdk import SearchClient\nclient = SearchClient(url=os.environ['SEARCH_URL'])\nresults = client.search(query='widget', limit=20)\nfor r in results:\n    print(r['id'], r['score'], r['payload']['title'])\n```\nNOTE: pre-c3d4e5f6 the return shape was `list[tuple[int, float]]`. If you\nsee that in an old service, it's using the legacy SDK \u2014 upgrade to\nsearch-sdk >= 3.0.\n\n## Environment\n- `SEARCH_URL` (defaul\n---\n# search_sdk/client.py (excerpt)\n\n```python\nimport httpx\n\nclass SearchClient:\n    def __init__(self, url: str, api_key: str | None = None):\n        self.url = url.rstrip('/')\n        self._http = httpx.Client(headers={'Authorization': f'Bearer {api_key}'} if api_key else {})\n\n    def search(self, query: str, *, collection: str = 'products', limit: int = 20,\n               filter: dict | None = None, hybrid_weight: float = 0.6) -> list[dict]:\n        # Post-c3d4e5f6: returns list[dict] not list[t",
+      "hits": [
+        {
+          "text": "commit c3d4e5f6\nAuthor: alex@example.com\nDate: 2026-02-14T09:00:00Z\n\nfeat(search): switch ANN backend from faiss to vector-db\n\nfaiss-cpu was hitting ~4s p99 on 2M vectors. Vector-DB HNSW on the same\ncorpus gives 40ms p99. Breaking: the SearchClient.search() return shape\nchanged \u2014 was list[tuple[int, float]], now list[dict] with keys {id, score,\npayload}. Migration guide in docs/SEARCH_MIGRATION.md.",
+          "score": 0.6144,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "commit-c3d4e5"
+        },
+        {
+          "text": "# search-svc README\n\nOwner: alex. On-call: search-oncall@example.com.\n\n## SDK\n```python\nfrom search_sdk import SearchClient\nclient = SearchClient(url=os.environ['SEARCH_URL'])\nresults = client.search(query='widget', limit=20)\nfor r in results:\n    print(r['id'], r['score'], r['payload']['title'])\n```\nNOTE: pre-c3d4e5f6 the return shape was `list[tuple[int, float]]`. If you\nsee that in an old servi",
+          "score": 0.5357,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "readme-search"
+        },
+        {
+          "text": "# search_sdk/client.py (excerpt)\n\n```python\nimport httpx\n\nclass SearchClient:\n    def __init__(self, url: str, api_key: str | None = None):\n        self.url = url.rstrip('/')\n        self._http = httpx.Client(headers={'Authorization': f'Bearer {api_key}'} if api_key else {})\n\n    def search(self, query: str, *, collection: str = 'products', limit: int = 20,\n               filter: dict | None = Non",
+          "score": 0.514,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "code-search-client"
+        }
+      ],
+      "correct": true,
+      "score": 1.0,
+      "grading_notes": "all substrings matched",
+      "search_time_ms": 1271.0949099855497,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 425,
+      "query_tokens": 13,
+      "context_tokens": 412,
+      "judge_tokens_in": 0,
+      "judge_tokens_out": 0,
+      "judge_latency_ms": 0.0
+    },
+    {
+      "task_id": "config-retry-policy-base",
+      "query": "What's the default stripe retry base delay in ms on billing-svc?",
+      "answer": "PR #47: Exponential backoff on StripeClient retries\nAuthor: priya\nStatus: merged\n\nSymptom: StripeClient was retrying every 100ms on 429 rate-limit responses,\nwhich made the rate-limit situation worse. Root cause: retry_delay_ms was\nhardcoded. Fix: exponential backoff with jitter, base=200ms, max=8s, 5 retries.\nTuned per Stripe's own guidance. Config key billing.stripe.retry_policy now\naccepts {base_ms, max_ms, max_retries, jitter}. Merged 2026-02-19.\n---\n# billing-svc README\n\nService owner: jordan. On-call: billing-oncall@example.com.\n\n## Environment\n- `STRIPE_API_KEY` (required)\n- `STRIPE_WEBHOOK_SECRET` (required)\n- `BILLING_DB_URL` (required)\n- `BILLING_STRIPE_RETRY_POLICY_BASE_MS` (default 200)\n- `BILLING_STRIPE_RETRY_POLICY_MAX_MS` (default 8000)\n- `BILLING_STRIPE_RETRY_POLICY_MAX_RETRIES` (default 5)\n\n## Key modules\n- `billing/stripe_client.py` \u2014 wraps stripe-python, adds retry + idempotency\n- `billing/webhooks.py` \u2014 Stripe webhook endpoin\n---\n[2026-02-18 Slack #payments-dev]\npriya: @jordan can you review #47? it's the stripe retry stuff\njordan: on it. Why base=200 and not 500?\npriya: stripe's own SDK uses 250 internally but I went conservative with 200 so we retry faster on transient blips. Max 8s is the ceiling stripe recommends.\njordan: makes sense. one nit \u2014 can we surface jitter=true as configurable? teams with deterministic retries might want to turn it off for tests.\npriya: yep, added. merging.\n\n[2026-02-19 Slack #payments-dev]",
+      "hits": [
+        {
+          "text": "PR #47: Exponential backoff on StripeClient retries\nAuthor: priya\nStatus: merged\n\nSymptom: StripeClient was retrying every 100ms on 429 rate-limit responses,\nwhich made the rate-limit situation worse. Root cause: retry_delay_ms was\nhardcoded. Fix: exponential backoff with jitter, base=200ms, max=8s, 5 retries.\nTuned per Stripe's own guidance. Config key billing.stripe.retry_policy now\naccepts {bas",
+          "score": 0.7033,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "pr-47-retry"
+        },
+        {
+          "text": "# billing-svc README\n\nService owner: jordan. On-call: billing-oncall@example.com.\n\n## Environment\n- `STRIPE_API_KEY` (required)\n- `STRIPE_WEBHOOK_SECRET` (required)\n- `BILLING_DB_URL` (required)\n- `BILLING_STRIPE_RETRY_POLICY_BASE_MS` (default 200)\n- `BILLING_STRIPE_RETRY_POLICY_MAX_MS` (default 8000)\n- `BILLING_STRIPE_RETRY_POLICY_MAX_RETRIES` (default 5)\n\n## Key modules\n- `billing/stripe_client.",
+          "score": 0.672,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "readme-billing"
+        },
+        {
+          "text": "[2026-02-18 Slack #payments-dev]\npriya: @jordan can you review #47? it's the stripe retry stuff\njordan: on it. Why base=200 and not 500?\npriya: stripe's own SDK uses 250 internally but I went conservative with 200 so we retry faster on transient blips. Max 8s is the ceiling stripe recommends.\njordan: makes sense. one nit \u2014 can we surface jitter=true as configurable? teams with deterministic retrie",
+          "score": 0.6684,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "chat-pr-47-review"
+        }
+      ],
+      "correct": true,
+      "score": 1.0,
+      "grading_notes": "all substrings matched",
+      "search_time_ms": 1263.2199359941296,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 425,
+      "query_tokens": 15,
+      "context_tokens": 410,
+      "judge_tokens_in": 0,
+      "judge_tokens_out": 0,
+      "judge_latency_ms": 0.0
+    },
+    {
+      "task_id": "config-session-ttl",
+      "query": "What's the default SESSION_TTL_MINUTES on auth-svc?",
+      "answer": "# auth-svc README\n\nOwner: priya. On-call: auth-oncall@example.com.\n\n## Environment\n- `JWT_PRIVATE_KEY_PATH` (required, RS256 PEM)\n- `JWT_PUBLIC_KEY_PATH` (required, RS256 PEM)\n- `JWT_ISSUER` (default: 'https://auth.example.com')\n- `SESSION_TTL_MINUTES` (default: 60)\n- `REFRESH_TTL_DAYS` (default: 30)\n\n**Do not use HS256**. The migration in b2c3d4e5 removed all HS256 paths.\n\n## Public API\n- `POST /v2/login {email, password}` \u2192 `{access_token, refresh_token}`\n- `POST /v2/refresh {refresh_token}` \u2192\n---\n# config/production.yaml\n\nbilling:\n  stripe:\n    api_key: ${STRIPE_API_KEY}\n    webhook_secret: ${STRIPE_WEBHOOK_SECRET}\n    retry_policy:\n      base_ms: 200\n      max_ms: 8000\n      max_retries: 5\n      jitter: true\n\nauth:\n  jwt:\n    algorithm: RS256\n    private_key_path: /etc/secrets/jwt-private.pem\n    public_key_path: /etc/secrets/jwt-public.pem\n    issuer: https://auth.example.com\n    session_ttl_minutes: 60\n    refresh_ttl_days: 30\n\nsearch:\n  url: http://search-svc.internal:8080\n  api_key:\n---\ncommit b2c3d4e5\nAuthor: priya@example.com\nDate: 2026-02-12T11:15:00Z\n\nfeat(auth): rotate JWT signing key to RS256\n\nWe were using HS256 shared-secret JWTs, which meant any service that verified\ntokens also held the minting secret. Migrated to RS256 asymmetric signing.\nPrivate key lives in auth-svc only; all other services only ever hold the\npublic key. Env var JWT_PRIVATE_KEY_PATH on auth-svc, JWT_PUBLIC_KEY_PATH\neverywhere else. BREAKING: old HS256 tokens rejected after 2026-02-20.",
+      "hits": [
+        {
+          "text": "# auth-svc README\n\nOwner: priya. On-call: auth-oncall@example.com.\n\n## Environment\n- `JWT_PRIVATE_KEY_PATH` (required, RS256 PEM)\n- `JWT_PUBLIC_KEY_PATH` (required, RS256 PEM)\n- `JWT_ISSUER` (default: 'https://auth.example.com')\n- `SESSION_TTL_MINUTES` (default: 60)\n- `REFRESH_TTL_DAYS` (default: 30)\n\n**Do not use HS256**. The migration in b2c3d4e5 removed all HS256 paths.\n\n## Public API\n- `POST /",
+          "score": 0.6372,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "readme-auth"
+        },
+        {
+          "text": "# config/production.yaml\n\nbilling:\n  stripe:\n    api_key: ${STRIPE_API_KEY}\n    webhook_secret: ${STRIPE_WEBHOOK_SECRET}\n    retry_policy:\n      base_ms: 200\n      max_ms: 8000\n      max_retries: 5\n      jitter: true\n\nauth:\n  jwt:\n    algorithm: RS256\n    private_key_path: /etc/secrets/jwt-private.pem\n    public_key_path: /etc/secrets/jwt-public.pem\n    issuer: https://auth.example.com\n    session",
+          "score": 0.5543,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "config-sample"
+        },
+        {
+          "text": "commit b2c3d4e5\nAuthor: priya@example.com\nDate: 2026-02-12T11:15:00Z\n\nfeat(auth): rotate JWT signing key to RS256\n\nWe were using HS256 shared-secret JWTs, which meant any service that verified\ntokens also held the minting secret. Migrated to RS256 asymmetric signing.\nPrivate key lives in auth-svc only; all other services only ever hold the\npublic key. Env var JWT_PRIVATE_KEY_PATH on auth-svc, JWT_",
+          "score": 0.4654,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "commit-b2c3d4"
+        }
+      ],
+      "correct": true,
+      "score": 1.0,
+      "grading_notes": "all substrings matched",
+      "search_time_ms": 1263.2656499918085,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 459,
+      "query_tokens": 14,
+      "context_tokens": 445,
+      "judge_tokens_in": 0,
+      "judge_tokens_out": 0,
+      "judge_latency_ms": 0.0
+    },
+    {
+      "task_id": "config-search-hybrid-weight",
+      "query": "What's the default hybrid_weight for search-svc?",
+      "answer": "## search-svc: POST /v3/search\n\nVector + keyword hybrid search.\n\n**Request body:**\n```json\n{\"query\": \"widget\", \"collection\": \"products\", \"limit\": 20, \"filter\": {\"category\": \"electronics\"}}\n```\n\n**Responses:**\n- `200` \u2014 `{\"results\": [{\"id\": \"p_123\", \"score\": 0.81, \"payload\": {...}}, ...], \"total\": 42}`\n- `400` \u2014 malformed filter\n- `429` \u2014 rate-limited\n\n**Notes:**\n- Results are ranked by hybrid (0.6 vector + 0.4 BM25) \u2014 override with\n  `hybrid_weight` in the body (0.0 = pure BM25, 1.0 = pure vecto\n---\n# search-svc README\n\nOwner: alex. On-call: search-oncall@example.com.\n\n## SDK\n```python\nfrom search_sdk import SearchClient\nclient = SearchClient(url=os.environ['SEARCH_URL'])\nresults = client.search(query='widget', limit=20)\nfor r in results:\n    print(r['id'], r['score'], r['payload']['title'])\n```\nNOTE: pre-c3d4e5f6 the return shape was `list[tuple[int, float]]`. If you\nsee that in an old service, it's using the legacy SDK \u2014 upgrade to\nsearch-sdk >= 3.0.\n\n## Environment\n- `SEARCH_URL` (defaul\n---\n# search_sdk/client.py (excerpt)\n\n```python\nimport httpx\n\nclass SearchClient:\n    def __init__(self, url: str, api_key: str | None = None):\n        self.url = url.rstrip('/')\n        self._http = httpx.Client(headers={'Authorization': f'Bearer {api_key}'} if api_key else {})\n\n    def search(self, query: str, *, collection: str = 'products', limit: int = 20,\n               filter: dict | None = None, hybrid_weight: float = 0.6) -> list[dict]:\n        # Post-c3d4e5f6: returns list[dict] not list[t",
+      "hits": [
+        {
+          "text": "## search-svc: POST /v3/search\n\nVector + keyword hybrid search.\n\n**Request body:**\n```json\n{\"query\": \"widget\", \"collection\": \"products\", \"limit\": 20, \"filter\": {\"category\": \"electronics\"}}\n```\n\n**Responses:**\n- `200` \u2014 `{\"results\": [{\"id\": \"p_123\", \"score\": 0.81, \"payload\": {...}}, ...], \"total\": 42}`\n- `400` \u2014 malformed filter\n- `429` \u2014 rate-limited\n\n**Notes:**\n- Results are ranked by hybrid (0.6",
+          "score": 0.5876,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "api-search-query"
+        },
+        {
+          "text": "# search-svc README\n\nOwner: alex. On-call: search-oncall@example.com.\n\n## SDK\n```python\nfrom search_sdk import SearchClient\nclient = SearchClient(url=os.environ['SEARCH_URL'])\nresults = client.search(query='widget', limit=20)\nfor r in results:\n    print(r['id'], r['score'], r['payload']['title'])\n```\nNOTE: pre-c3d4e5f6 the return shape was `list[tuple[int, float]]`. If you\nsee that in an old servi",
+          "score": 0.5703,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "readme-search"
+        },
+        {
+          "text": "# search_sdk/client.py (excerpt)\n\n```python\nimport httpx\n\nclass SearchClient:\n    def __init__(self, url: str, api_key: str | None = None):\n        self.url = url.rstrip('/')\n        self._http = httpx.Client(headers={'Authorization': f'Bearer {api_key}'} if api_key else {})\n\n    def search(self, query: str, *, collection: str = 'products', limit: int = 20,\n               filter: dict | None = Non",
+          "score": 0.5421,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "code-search-client"
+        }
+      ],
+      "correct": true,
+      "score": 1.0,
+      "grading_notes": "all substrings matched",
+      "search_time_ms": 1285.4226699855644,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 462,
+      "query_tokens": 11,
+      "context_tokens": 451,
+      "judge_tokens_in": 0,
+      "judge_tokens_out": 0,
+      "judge_latency_ms": 0.0
+    },
+    {
+      "task_id": "pr-rationale-retry",
+      "query": "Why did PR #47 choose base=200ms for stripe retries rather than 500ms?",
+      "answer": "[2026-02-18 Slack #payments-dev]\npriya: @jordan can you review #47? it's the stripe retry stuff\njordan: on it. Why base=200 and not 500?\npriya: stripe's own SDK uses 250 internally but I went conservative with 200 so we retry faster on transient blips. Max 8s is the ceiling stripe recommends.\njordan: makes sense. one nit \u2014 can we surface jitter=true as configurable? teams with deterministic retries might want to turn it off for tests.\npriya: yep, added. merging.\n\n[2026-02-19 Slack #payments-dev]\n---\nPR #47: Exponential backoff on StripeClient retries\nAuthor: priya\nStatus: merged\n\nSymptom: StripeClient was retrying every 100ms on 429 rate-limit responses,\nwhich made the rate-limit situation worse. Root cause: retry_delay_ms was\nhardcoded. Fix: exponential backoff with jitter, base=200ms, max=8s, 5 retries.\nTuned per Stripe's own guidance. Config key billing.stripe.retry_policy now\naccepts {base_ms, max_ms, max_retries, jitter}. Merged 2026-02-19.\n---\n# billing-svc README\n\nService owner: jordan. On-call: billing-oncall@example.com.\n\n## Environment\n- `STRIPE_API_KEY` (required)\n- `STRIPE_WEBHOOK_SECRET` (required)\n- `BILLING_DB_URL` (required)\n- `BILLING_STRIPE_RETRY_POLICY_BASE_MS` (default 200)\n- `BILLING_STRIPE_RETRY_POLICY_MAX_MS` (default 8000)\n- `BILLING_STRIPE_RETRY_POLICY_MAX_RETRIES` (default 5)\n\n## Key modules\n- `billing/stripe_client.py` \u2014 wraps stripe-python, adds retry + idempotency\n- `billing/webhooks.py` \u2014 Stripe webhook endpoin",
+      "hits": [
+        {
+          "text": "[2026-02-18 Slack #payments-dev]\npriya: @jordan can you review #47? it's the stripe retry stuff\njordan: on it. Why base=200 and not 500?\npriya: stripe's own SDK uses 250 internally but I went conservative with 200 so we retry faster on transient blips. Max 8s is the ceiling stripe recommends.\njordan: makes sense. one nit \u2014 can we surface jitter=true as configurable? teams with deterministic retrie",
+          "score": 0.7484,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "chat-pr-47-review"
+        },
+        {
+          "text": "PR #47: Exponential backoff on StripeClient retries\nAuthor: priya\nStatus: merged\n\nSymptom: StripeClient was retrying every 100ms on 429 rate-limit responses,\nwhich made the rate-limit situation worse. Root cause: retry_delay_ms was\nhardcoded. Fix: exponential backoff with jitter, base=200ms, max=8s, 5 retries.\nTuned per Stripe's own guidance. Config key billing.stripe.retry_policy now\naccepts {bas",
+          "score": 0.7145,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "pr-47-retry"
+        },
+        {
+          "text": "# billing-svc README\n\nService owner: jordan. On-call: billing-oncall@example.com.\n\n## Environment\n- `STRIPE_API_KEY` (required)\n- `STRIPE_WEBHOOK_SECRET` (required)\n- `BILLING_DB_URL` (required)\n- `BILLING_STRIPE_RETRY_POLICY_BASE_MS` (default 200)\n- `BILLING_STRIPE_RETRY_POLICY_MAX_MS` (default 8000)\n- `BILLING_STRIPE_RETRY_POLICY_MAX_RETRIES` (default 5)\n\n## Key modules\n- `billing/stripe_client.",
+          "score": 0.4761,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "readme-billing"
+        }
+      ],
+      "correct": true,
+      "score": 1.0,
+      "grading_notes": "all substrings matched",
+      "search_time_ms": 1318.2544060109649,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 428,
+      "query_tokens": 19,
+      "context_tokens": 409,
+      "judge_tokens_in": 0,
+      "judge_tokens_out": 0,
+      "judge_latency_ms": 0.0
+    },
+    {
+      "task_id": "pr-rationale-jwt-cleanup",
+      "query": "What did PR #61 clean up and why was it safe?",
+      "answer": "PR #61: Remove JWT_SECRET env var from all services\nAuthor: priya\nStatus: merged\n\nFollowing the RS256 migration (b2c3d4e5), JWT_SECRET is no longer read\nanywhere except auth-svc. Removed it from 11 deployment manifests and\nfrom all .env.example files. Sealed-secret file for staging/prod cleaned\nin platform/secrets/ \u2014 old key destroyed via vault rotate. Merged 2026-02-24.\n---\nPR #61: Remove JWT_SECRET env var from all services\nAuthor: priya\nStatus: merged\n\nFollowing the RS256 migration (b2c3d4e5), JWT_SECRET is no longer read\nanywhere except auth-svc. Removed it from 11 deployment manifests and\nfrom all .env.example files. Sealed-secret file for staging/prod cleaned\nin platform/secrets/ \u2014 old key destroyed via vault rotate. Merged 2026-02-24.\n---\n[2026-02-15 Slack #platform]\njordan: reminder: /api/v1 is going away in 2 days. anyone still seeing v1 traffic?\nalex: we're clean on search-svc. grep'd all consumers.\npriya: auth is clean, v2-only for 6 months\njordan: cool. flipping the 410 flag on 2026-02-17 09:00 UTC. if your service gets paged, check your callers.\n\n[2026-02-17 Slack #platform]\njordan: v1 is gone. 410s in the last hour: 12. all from an old mobile app build. told them to upgrade. no internal services affected.",
+      "hits": [
+        {
+          "text": "PR #61: Remove JWT_SECRET env var from all services\nAuthor: priya\nStatus: merged\n\nFollowing the RS256 migration (b2c3d4e5), JWT_SECRET is no longer read\nanywhere except auth-svc. Removed it from 11 deployment manifests and\nfrom all .env.example files. Sealed-secret file for staging/prod cleaned\nin platform/secrets/ \u2014 old key destroyed via vault rotate. Merged 2026-02-24.",
+          "score": 0.6087,
+          "source": "pentatonic-baseline",
+          "doc_id": "pr-61-jwt-cleanup"
+        },
+        {
+          "text": "PR #61: Remove JWT_SECRET env var from all services\nAuthor: priya\nStatus: merged\n\nFollowing the RS256 migration (b2c3d4e5), JWT_SECRET is no longer read\nanywhere except auth-svc. Removed it from 11 deployment manifests and\nfrom all .env.example files. Sealed-secret file for staging/prod cleaned\nin platform/secrets/ \u2014 old key destroyed via vault rotate. Merged 2026-02-24.",
+          "score": 0.5675,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "pr-61-jwt-cleanup"
+        },
+        {
+          "text": "[2026-02-15 Slack #platform]\njordan: reminder: /api/v1 is going away in 2 days. anyone still seeing v1 traffic?\nalex: we're clean on search-svc. grep'd all consumers.\npriya: auth is clean, v2-only for 6 months\njordan: cool. flipping the 410 flag on 2026-02-17 09:00 UTC. if your service gets paged, check your callers.\n\n[2026-02-17 Slack #platform]\njordan: v1 is gone. 410s in the last hour: 12. all ",
+          "score": 0.5332,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "chat-deprecation"
+        }
+      ],
+      "correct": true,
+      "score": 1.0,
+      "grading_notes": "all substrings matched",
+      "search_time_ms": 1295.8309090172406,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 363,
+      "query_tokens": 13,
+      "context_tokens": 350,
+      "judge_tokens_in": 0,
+      "judge_tokens_out": 0,
+      "judge_latency_ms": 0.0
+    },
+    {
+      "task_id": "cross-ref-invoicing",
+      "query": "After Stripe posts invoice.payment_succeeded, which services consume the event?",
+      "answer": "# Cross-reference: invoicing pipeline\n\nEnd-to-end path for an invoice being paid:\n\n1. `billing-svc` creates the invoice (see POST /v2/invoices, api-billing-invoice)\n2. Customer pays via Stripe portal (no internal code path)\n3. Stripe posts a `invoice.payment_succeeded` webhook to billing-svc\n4. `billing/webhooks.py::handle_webhook` verifies signature, enqueues\n   internal `InvoicePaid` event on `order-svc`\n5. `order-svc` marks order as paid, emits `OrderPaid` on pubsub\n6. `reporting-svc` consumes `OrderPaid` for financial reports\n7. `email-svc` consumes `OrderPaid` for the receipt email\n\nOn we\n---\n# Cross-reference: invoicing pipeline\n\nEnd-to-end path for an invoice being paid:\n\n1. `billing-svc` creates the invoice (see POST /v2/invoices, api-billing-invoice)\n2. Customer pays via Stripe portal (no internal code path)\n3. Stripe posts a `invoice.payment_succeeded` webhook to billing-svc\n4. `billing/webhooks.py::handle_webhook` verifies signature, enqueues\n   internal `InvoicePaid` event on `order-svc`\n5. `order-svc` marks order as paid, emits `OrderPaid` on pubsub\n6. `reporting-svc` consume\n---\n# billing-svc README\n\nService owner: jordan. On-call: billing-oncall@example.com.\n\n## Environment\n- `STRIPE_API_KEY` (required)\n- `STRIPE_WEBHOOK_SECRET` (required)\n- `BILLING_DB_URL` (required)\n- `BILLING_STRIPE_RETRY_POLICY_BASE_MS` (default 200)\n- `BILLING_STRIPE_RETRY_POLICY_MAX_MS` (default 8000)\n- `BILLING_STRIPE_RETRY_POLICY_MAX_RETRIES` (default 5)\n\n## Key modules\n- `billing/stripe_client.py` \u2014 wraps stripe-python, adds retry + idempotency\n- `billing/webhooks.py` \u2014 Stripe webhook endpoint, signed and idempotent\n- `billing/invoicing.py` \u2014 PDF invoice generation (weasyprint)\n\nSee ARCHITE",
+      "hits": [
+        {
+          "text": "# Cross-reference: invoicing pipeline\n\nEnd-to-end path for an invoice being paid:\n\n1. `billing-svc` creates the invoice (see POST /v2/invoices, api-billing-invoice)\n2. Customer pays via Stripe portal (no internal code path)\n3. Stripe posts a `invoice.payment_succeeded` webhook to billing-svc\n4. `billing/webhooks.py::handle_webhook` verifies signature, enqueues\n   internal `InvoicePaid` event on `o",
+          "score": 0.7495,
+          "source": "pentatonic-baseline",
+          "doc_id": "cross-ref-invoicing"
+        },
+        {
+          "text": "# Cross-reference: invoicing pipeline\n\nEnd-to-end path for an invoice being paid:\n\n1. `billing-svc` creates the invoice (see POST /v2/invoices, api-billing-invoice)\n2. Customer pays via Stripe portal (no internal code path)\n3. Stripe posts a `invoice.payment_succeeded` webhook to billing-svc\n4. `billing/webhooks.py::handle_webhook` verifies signature, enqueues\n   internal `InvoicePaid` event on `o",
+          "score": 0.7179,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "cross-ref-invoicing"
+        },
+        {
+          "text": "# billing-svc README\n\nService owner: jordan. On-call: billing-oncall@example.com.\n\n## Environment\n- `STRIPE_API_KEY` (required)\n- `STRIPE_WEBHOOK_SECRET` (required)\n- `BILLING_DB_URL` (required)\n- `BILLING_STRIPE_RETRY_POLICY_BASE_MS` (default 200)\n- `BILLING_STRIPE_RETRY_POLICY_MAX_MS` (default 8000)\n- `BILLING_STRIPE_RETRY_POLICY_MAX_RETRIES` (default 5)\n\n## Key modules\n- `billing/stripe_client.",
+          "score": 0.5969,
+          "source": "pentatonic-baseline",
+          "doc_id": "readme-billing"
+        }
+      ],
+      "correct": true,
+      "score": 1.0,
+      "grading_notes": "all substrings matched",
+      "search_time_ms": 1290.6972740020137,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 486,
+      "query_tokens": 14,
+      "context_tokens": 472,
+      "judge_tokens_in": 0,
+      "judge_tokens_out": 0,
+      "judge_latency_ms": 0.0
+    },
+    {
+      "task_id": "cross-ref-ratelimit",
+      "query": "Where does the shared rate-limit lib live and what's the method signature?",
+      "answer": "Refactor: extract rate-limit middleware into shared lib\nAuthor: alex\nStatus: merged 2026-03-18 (PR #108)\n\nBefore: every service rolled its own token-bucket rate limiter (billing-svc,\nauth-svc, search-svc, gateway all had near-identical code).\nAfter: `shared/rate_limit.py` with the canonical implementation. All 4\nservices now import from there.\nAPI: `RateLimiter(tokens_per_sec: int, burst: int).allow(key: str) -> bool`.\nBackend: Redis (shared across pods). Key format: `ratelimit:{service}:{key}`.\n---\nPR #47: Exponential backoff on StripeClient retries\nAuthor: priya\nStatus: merged\n\nSymptom: StripeClient was retrying every 100ms on 429 rate-limit responses,\nwhich made the rate-limit situation worse. Root cause: retry_delay_ms was\nhardcoded. Fix: exponential backoff with jitter, base=200ms, max=8s, 5 retries.\nTuned per Stripe's own guidance. Config key billing.stripe.retry_policy now\naccepts {base_ms, max_ms, max_retries, jitter}. Merged 2026-02-19.\n---\n# Incident 2026-03-02: billing-svc webhook signature errors\n\n**Severity:** SEV-2\n**Duration:** 2026-03-02 14:15 UTC \u2192 14:22 UTC (7 min)\n**Impact:** Stripe webhooks dropped during the window; 3 payment events\nnot ingested. Replayed via Stripe dashboard at 14:30.\n\n**Root cause:** STRIPE_WEBHOOK_SECRET was rotated in vault at 14:10. Pods\ndon't watch vault live; the new secret only loaded on restart.\n\n**Fix:** kubectl rollout restart deployment/billing-svc at 14:20. Normal\noperation resumed 14:22.",
+      "hits": [
+        {
+          "text": "Refactor: extract rate-limit middleware into shared lib\nAuthor: alex\nStatus: merged 2026-03-18 (PR #108)\n\nBefore: every service rolled its own token-bucket rate limiter (billing-svc,\nauth-svc, search-svc, gateway all had near-identical code).\nAfter: `shared/rate_limit.py` with the canonical implementation. All 4\nservices now import from there.\nAPI: `RateLimiter(tokens_per_sec: int, burst: int).all",
+          "score": 0.6958,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "refactor-rate-limit"
+        },
+        {
+          "text": "PR #47: Exponential backoff on StripeClient retries\nAuthor: priya\nStatus: merged\n\nSymptom: StripeClient was retrying every 100ms on 429 rate-limit responses,\nwhich made the rate-limit situation worse. Root cause: retry_delay_ms was\nhardcoded. Fix: exponential backoff with jitter, base=200ms, max=8s, 5 retries.\nTuned per Stripe's own guidance. Config key billing.stripe.retry_policy now\naccepts {bas",
+          "score": 0.5577,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "pr-47-retry"
+        },
+        {
+          "text": "# Incident 2026-03-02: billing-svc webhook signature errors\n\n**Severity:** SEV-2\n**Duration:** 2026-03-02 14:15 UTC \u2192 14:22 UTC (7 min)\n**Impact:** Stripe webhooks dropped during the window; 3 payment events\nnot ingested. Replayed via Stripe dashboard at 14:30.\n\n**Root cause:** STRIPE_WEBHOOK_SECRET was rotated in vault at 14:10. Pods\ndon't watch vault live; the new secret only loaded on restart.\n",
+          "score": 0.4544,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "incident-2026-03-02-webhook"
+        }
+      ],
+      "correct": true,
+      "score": 1.0,
+      "grading_notes": "all substrings matched",
+      "search_time_ms": 1235.1318040164188,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 419,
+      "query_tokens": 15,
+      "context_tokens": 404,
+      "judge_tokens_in": 0,
+      "judge_tokens_out": 0,
+      "judge_latency_ms": 0.0
+    },
+    {
+      "task_id": "stack-trace-webhook",
+      "query": "If a SignatureVerificationError happens on billing-svc webhook, what's the most likely cause and fix?",
+      "answer": "# billing-svc runbook: webhook signature errors\n\nSymptom: logs show `stripe.error.SignatureVerificationError`.\n\nDiagnosis:\n1. Confirm the webhook endpoint URL matches Stripe dashboard\n2. Check `STRIPE_WEBHOOK_SECRET` matches the secret in Stripe dashboard\n3. If secret was rotated recently, `kubectl rollout restart deployment/billing-svc`\n\nWe hit this on 2026-03-02 14:15 UTC \u2014 secret rotated but pods didn't reload\nuntil restart. See incident post-mortem in ops/incidents/2026-03-02-webhook.md.\n---\nStack trace from prod 2026-03-02 14:15 UTC:\n\nTraceback (most recent call last):\n  File \"billing/webhooks.py\", line 42, in handle_webhook\n    event = StripeClient.parse_event(payload, signature)\n  File \"billing/stripe_client.py\", line 88, in parse_event\n    return stripe.Webhook.construct_event(payload, sig, secret)\n  File \"stripe/webhook.py\", line 56, in construct_event\n    raise SignatureVerificationError(...)\nstripe.error.SignatureVerificationError: No signatures found matching expected signature\n\nCaused by: STRIPE_WEBHOOK_SECRET rotated in prod at 14:10 but pod restart\ndidn't pick up new va\n---\n# billing-svc runbook: webhook signature errors\n\nSymptom: logs show `stripe.error.SignatureVerificationError`.\n\nDiagnosis:\n1. Confirm the webhook endpoint URL matches Stripe dashboard\n2. Check `STRIPE_WEBHOOK_SECRET` matches the secret in Stripe dashboard\n3. If secret was rotated recently, `kubectl rollout restart deployment/billing-svc`\n\nWe hit this on 2026-03-02 14:15 UTC \u2014 secret rotated but pods didn't reload\nuntil restart. See incident post-mortem in ops/incidents/2026-03-02-webhook.md.",
+      "hits": [
+        {
+          "text": "# billing-svc runbook: webhook signature errors\n\nSymptom: logs show `stripe.error.SignatureVerificationError`.\n\nDiagnosis:\n1. Confirm the webhook endpoint URL matches Stripe dashboard\n2. Check `STRIPE_WEBHOOK_SECRET` matches the secret in Stripe dashboard\n3. If secret was rotated recently, `kubectl rollout restart deployment/billing-svc`\n\nWe hit this on 2026-03-02 14:15 UTC \u2014 secret rotated but po",
+          "score": 0.8118,
+          "source": "pentatonic-baseline",
+          "doc_id": "runbook-billing"
+        },
+        {
+          "text": "Stack trace from prod 2026-03-02 14:15 UTC:\n\nTraceback (most recent call last):\n  File \"billing/webhooks.py\", line 42, in handle_webhook\n    event = StripeClient.parse_event(payload, signature)\n  File \"billing/stripe_client.py\", line 88, in parse_event\n    return stripe.Webhook.construct_event(payload, sig, secret)\n  File \"stripe/webhook.py\", line 56, in construct_event\n    raise SignatureVerifica",
+          "score": 0.7632,
+          "source": "pentatonic-baseline",
+          "doc_id": "stack-trace-nullpointer"
+        },
+        {
+          "text": "# billing-svc runbook: webhook signature errors\n\nSymptom: logs show `stripe.error.SignatureVerificationError`.\n\nDiagnosis:\n1. Confirm the webhook endpoint URL matches Stripe dashboard\n2. Check `STRIPE_WEBHOOK_SECRET` matches the secret in Stripe dashboard\n3. If secret was rotated recently, `kubectl rollout restart deployment/billing-svc`\n\nWe hit this on 2026-03-02 14:15 UTC \u2014 secret rotated but po",
+          "score": 0.7203,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "runbook-billing"
+        }
+      ],
+      "correct": true,
+      "score": 1.0,
+      "grading_notes": "all substrings matched",
+      "search_time_ms": 1203.479966003215,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 415,
+      "query_tokens": 21,
+      "context_tokens": 394,
+      "judge_tokens_in": 0,
+      "judge_tokens_out": 0,
+      "judge_latency_ms": 0.0
+    },
+    {
+      "task_id": "temporal-numpy",
+      "query": "Why was numpy upgraded to 1.26 recently?",
+      "answer": "PR #101: Upgrade numpy 1.24.4 \u2192 1.26.2\nAuthor: alex\nStatus: merged\n\nFollow-up to ADR-011 (Python 3.12 minimum). numpy 1.24 doesn't ship 3.12\nwheels, so CI would have to build from source on every install.\nUpgraded to 1.26, which does ship 3.12 wheels. No API surface changes\naffecting us \u2014 only internal dtype-promotion warnings we've already\nsilenced via `warnings.filterwarnings`.\n\nVerified no call-sites depended on the deprecated `np.object_` alias.\nMerged 2026-03-11.\n---\nPR #101: Upgrade numpy 1.24.4 \u2192 1.26.2\nAuthor: alex\nStatus: merged\n\nFollow-up to ADR-011 (Python 3.12 minimum). numpy 1.24 doesn't ship 3.12\nwheels, so CI would have to build from source on every install.\nUpgraded to 1.26, which does ship 3.12 wheels. No API surface changes\naffecting us \u2014 only internal dtype-promotion warnings we've already\nsilenced via `warnings.filterwarnings`.\n\nVerified no call-sites depended on the deprecated `np.object_` alias.\nMerged 2026-03-11.\n---\nADR-011: Python 3.12 minimum across the fleet\nDate: 2026-03-10\nStatus: accepted\n\n**Context.** We've been pinned to 3.10 for 2 years. 3.12's faster interpreter\nand better typing are worth the cutover cost.\n\n**Decision.** Minimum Python = 3.12. All services, CI, dev containers.\nPython 3.11 and 3.10 removed from `.python-version` files.\n\n**Consequences.** One library (numpy==1.24.x) needs upgrade to 1.26 for\n3.12 wheels. PR #101 does the upgrade. No API changes for consumers.",
+      "hits": [
+        {
+          "text": "PR #101: Upgrade numpy 1.24.4 \u2192 1.26.2\nAuthor: alex\nStatus: merged\n\nFollow-up to ADR-011 (Python 3.12 minimum). numpy 1.24 doesn't ship 3.12\nwheels, so CI would have to build from source on every install.\nUpgraded to 1.26, which does ship 3.12 wheels. No API surface changes\naffecting us \u2014 only internal dtype-promotion warnings we've already\nsilenced via `warnings.filterwarnings`.\n\nVerified no call",
+          "score": 0.7771,
+          "source": "pentatonic-baseline",
+          "doc_id": "dep-numpy-upgrade"
+        },
+        {
+          "text": "PR #101: Upgrade numpy 1.24.4 \u2192 1.26.2\nAuthor: alex\nStatus: merged\n\nFollow-up to ADR-011 (Python 3.12 minimum). numpy 1.24 doesn't ship 3.12\nwheels, so CI would have to build from source on every install.\nUpgraded to 1.26, which does ship 3.12 wheels. No API surface changes\naffecting us \u2014 only internal dtype-promotion warnings we've already\nsilenced via `warnings.filterwarnings`.\n\nVerified no call",
+          "score": 0.7281,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "dep-numpy-upgrade"
+        },
+        {
+          "text": "ADR-011: Python 3.12 minimum across the fleet\nDate: 2026-03-10\nStatus: accepted\n\n**Context.** We've been pinned to 3.10 for 2 years. 3.12's faster interpreter\nand better typing are worth the cutover cost.\n\n**Decision.** Minimum Python = 3.12. All services, CI, dev containers.\nPython 3.11 and 3.10 removed from `.python-version` files.\n\n**Consequences.** One library (numpy==1.24.x) needs upgrade to ",
+          "score": 0.6324,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "decision-python"
+        }
+      ],
+      "correct": true,
+      "score": 1.0,
+      "grading_notes": "all substrings matched",
+      "search_time_ms": 1267.1056179970037,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 442,
+      "query_tokens": 11,
+      "context_tokens": 431,
+      "judge_tokens_in": 0,
+      "judge_tokens_out": 0,
+      "judge_latency_ms": 0.0
+    },
+    {
+      "task_id": "temporal-on-call",
+      "query": "Who is the primary on-call for the week of April 15\u201321, 2026?",
+      "answer": "# On-call rotation April 2026\n\n| Week | Primary | Secondary |\n|------|---------|-----------|\n| Apr 1\u20137   | jordan | priya |\n| Apr 8\u201314  | priya  | alex  |\n| Apr 15\u201321 | alex   | jordan |\n| Apr 22\u201328 | jordan | priya |\n| Apr 29\u201330 | priya  | alex  |\n\nHandoff: Mondays 10:00 UTC, #on-call channel. Previous week's primary\nruns through the incident log.\n\nEscalation path: primary \u2192 secondary \u2192 engineering manager (kim) \u2192 CTO.\n---\n# On-call rotation April 2026\n\n| Week | Primary | Secondary |\n|------|---------|-----------|\n| Apr 1\u20137   | jordan | priya |\n| Apr 8\u201314  | priya  | alex  |\n| Apr 15\u201321 | alex   | jordan |\n| Apr 22\u201328 | jordan | priya |\n| Apr 29\u201330 | priya  | alex  |\n\nHandoff: Mondays 10:00 UTC, #on-call channel. Previous week's primary\nruns through the incident log.\n\nEscalation path: primary \u2192 secondary \u2192 engineering manager (kim) \u2192 CTO.\n---\n# auth-svc README\n\nOwner: priya. On-call: auth-oncall@example.com.\n\n## Environment\n- `JWT_PRIVATE_KEY_PATH` (required, RS256 PEM)\n- `JWT_PUBLIC_KEY_PATH` (required, RS256 PEM)\n- `JWT_ISSUER` (default: 'https://auth.example.com')\n- `SESSION_TTL_MINUTES` (default: 60)\n- `REFRESH_TTL_DAYS` (default: 30)\n\n**Do not use HS256**. The migration in b2c3d4e5 removed all HS256 paths.\n\n## Public API\n- `POST /v2/login {email, password}` \u2192 `{access_token, refresh_token}`\n- `POST /v2/refresh {refresh_token}` \u2192",
+      "hits": [
+        {
+          "text": "# On-call rotation April 2026\n\n| Week | Primary | Secondary |\n|------|---------|-----------|\n| Apr 1\u20137   | jordan | priya |\n| Apr 8\u201314  | priya  | alex  |\n| Apr 15\u201321 | alex   | jordan |\n| Apr 22\u201328 | jordan | priya |\n| Apr 29\u201330 | priya  | alex  |\n\nHandoff: Mondays 10:00 UTC, #on-call channel. Previous week's primary\nruns through the incident log.\n\nEscalation path: primary \u2192 secondary \u2192 engineeri",
+          "score": 0.7727,
+          "source": "pentatonic-baseline",
+          "doc_id": "on-call-rotation"
+        },
+        {
+          "text": "# On-call rotation April 2026\n\n| Week | Primary | Secondary |\n|------|---------|-----------|\n| Apr 1\u20137   | jordan | priya |\n| Apr 8\u201314  | priya  | alex  |\n| Apr 15\u201321 | alex   | jordan |\n| Apr 22\u201328 | jordan | priya |\n| Apr 29\u201330 | priya  | alex  |\n\nHandoff: Mondays 10:00 UTC, #on-call channel. Previous week's primary\nruns through the incident log.\n\nEscalation path: primary \u2192 secondary \u2192 engineeri",
+          "score": 0.71,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "on-call-rotation"
+        },
+        {
+          "text": "# auth-svc README\n\nOwner: priya. On-call: auth-oncall@example.com.\n\n## Environment\n- `JWT_PRIVATE_KEY_PATH` (required, RS256 PEM)\n- `JWT_PUBLIC_KEY_PATH` (required, RS256 PEM)\n- `JWT_ISSUER` (default: 'https://auth.example.com')\n- `SESSION_TTL_MINUTES` (default: 60)\n- `REFRESH_TTL_DAYS` (default: 30)\n\n**Do not use HS256**. The migration in b2c3d4e5 removed all HS256 paths.\n\n## Public API\n- `POST /",
+          "score": 0.4314,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "readme-auth"
+        }
+      ],
+      "correct": true,
+      "score": 1.0,
+      "grading_notes": "all substrings matched",
+      "search_time_ms": 1279.5606150175445,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 452,
+      "query_tokens": 20,
+      "context_tokens": 432,
+      "judge_tokens_in": 0,
+      "judge_tokens_out": 0,
+      "judge_latency_ms": 0.0
+    },
+    {
+      "task_id": "contradiction-algorithm",
+      "query": "Which JWT signing algorithm does auth-svc use \u2014 HS256 or RS256?",
+      "answer": "# auth-svc README\n\nOwner: priya. On-call: auth-oncall@example.com.\n\n## Environment\n- `JWT_PRIVATE_KEY_PATH` (required, RS256 PEM)\n- `JWT_PUBLIC_KEY_PATH` (required, RS256 PEM)\n- `JWT_ISSUER` (default: 'https://auth.example.com')\n- `SESSION_TTL_MINUTES` (default: 60)\n- `REFRESH_TTL_DAYS` (default: 30)\n\n**Do not use HS256**. The migration in b2c3d4e5 removed all HS256 paths.\n\n## Public API\n- `POST /v2/login {email, password}` \u2192 `{access_token, refresh_token}`\n- `POST /v2/refresh {refresh_token}` \u2192\n---\ncommit b2c3d4e5\nAuthor: priya@example.com\nDate: 2026-02-12T11:15:00Z\n\nfeat(auth): rotate JWT signing key to RS256\n\nWe were using HS256 shared-secret JWTs, which meant any service that verified\ntokens also held the minting secret. Migrated to RS256 asymmetric signing.\nPrivate key lives in auth-svc only; all other services only ever hold the\npublic key. Env var JWT_PRIVATE_KEY_PATH on auth-svc, JWT_PUBLIC_KEY_PATH\neverywhere else. BREAKING: old HS256 tokens rejected after 2026-02-20.\n---\nPR #61: Remove JWT_SECRET env var from all services\nAuthor: priya\nStatus: merged\n\nFollowing the RS256 migration (b2c3d4e5), JWT_SECRET is no longer read\nanywhere except auth-svc. Removed it from 11 deployment manifests and\nfrom all .env.example files. Sealed-secret file for staging/prod cleaned\nin platform/secrets/ \u2014 old key destroyed via vault rotate. Merged 2026-02-24.",
+      "hits": [
+        {
+          "text": "# auth-svc README\n\nOwner: priya. On-call: auth-oncall@example.com.\n\n## Environment\n- `JWT_PRIVATE_KEY_PATH` (required, RS256 PEM)\n- `JWT_PUBLIC_KEY_PATH` (required, RS256 PEM)\n- `JWT_ISSUER` (default: 'https://auth.example.com')\n- `SESSION_TTL_MINUTES` (default: 60)\n- `REFRESH_TTL_DAYS` (default: 30)\n\n**Do not use HS256**. The migration in b2c3d4e5 removed all HS256 paths.\n\n## Public API\n- `POST /",
+          "score": 0.6996,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "readme-auth"
+        },
+        {
+          "text": "commit b2c3d4e5\nAuthor: priya@example.com\nDate: 2026-02-12T11:15:00Z\n\nfeat(auth): rotate JWT signing key to RS256\n\nWe were using HS256 shared-secret JWTs, which meant any service that verified\ntokens also held the minting secret. Migrated to RS256 asymmetric signing.\nPrivate key lives in auth-svc only; all other services only ever hold the\npublic key. Env var JWT_PRIVATE_KEY_PATH on auth-svc, JWT_",
+          "score": 0.6791,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "commit-b2c3d4"
+        },
+        {
+          "text": "PR #61: Remove JWT_SECRET env var from all services\nAuthor: priya\nStatus: merged\n\nFollowing the RS256 migration (b2c3d4e5), JWT_SECRET is no longer read\nanywhere except auth-svc. Removed it from 11 deployment manifests and\nfrom all .env.example files. Sealed-secret file for staging/prod cleaned\nin platform/secrets/ \u2014 old key destroyed via vault rotate. Merged 2026-02-24.",
+          "score": 0.6419,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "pr-61-jwt-cleanup"
+        }
+      ],
+      "correct": true,
+      "score": 1.0,
+      "grading_notes": "all substrings matched",
+      "search_time_ms": 1353.9299910189584,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 412,
+      "query_tokens": 16,
+      "context_tokens": 396,
+      "judge_tokens_in": 0,
+      "judge_tokens_out": 0,
+      "judge_latency_ms": 0.0
+    },
+    {
+      "task_id": "rubric-security-advice",
+      "query": "A new engineer asks: can I add HS256 as a fallback for older clients in auth-svc? Advise them.",
+      "answer": "commit b2c3d4e5\nAuthor: priya@example.com\nDate: 2026-02-12T11:15:00Z\n\nfeat(auth): rotate JWT signing key to RS256\n\nWe were using HS256 shared-secret JWTs, which meant any service that verified\ntokens also held the minting secret. Migrated to RS256 asymmetric signing.\nPrivate key lives in auth-svc only; all other services only ever hold the\npublic key. Env var JWT_PRIVATE_KEY_PATH on auth-svc, JWT_PUBLIC_KEY_PATH\neverywhere else. BREAKING: old HS256 tokens rejected after 2026-02-20.\n---\n# auth-svc README\n\nOwner: priya. On-call: auth-oncall@example.com.\n\n## Environment\n- `JWT_PRIVATE_KEY_PATH` (required, RS256 PEM)\n- `JWT_PUBLIC_KEY_PATH` (required, RS256 PEM)\n- `JWT_ISSUER` (default: 'https://auth.example.com')\n- `SESSION_TTL_MINUTES` (default: 60)\n- `REFRESH_TTL_DAYS` (default: 30)\n\n**Do not use HS256**. The migration in b2c3d4e5 removed all HS256 paths.\n\n## Public API\n- `POST /v2/login {email, password}` \u2192 `{access_token, refresh_token}`\n- `POST /v2/refresh {refresh_token}` \u2192\n---\n[2026-02-15 Slack #platform]\njordan: reminder: /api/v1 is going away in 2 days. anyone still seeing v1 traffic?\nalex: we're clean on search-svc. grep'd all consumers.\npriya: auth is clean, v2-only for 6 months\njordan: cool. flipping the 410 flag on 2026-02-17 09:00 UTC. if your service gets paged, check your callers.\n\n[2026-02-17 Slack #platform]\njordan: v1 is gone. 410s in the last hour: 12. all from an old mobile app build. told them to upgrade. no internal services affected.",
+      "hits": [
+        {
+          "text": "commit b2c3d4e5\nAuthor: priya@example.com\nDate: 2026-02-12T11:15:00Z\n\nfeat(auth): rotate JWT signing key to RS256\n\nWe were using HS256 shared-secret JWTs, which meant any service that verified\ntokens also held the minting secret. Migrated to RS256 asymmetric signing.\nPrivate key lives in auth-svc only; all other services only ever hold the\npublic key. Env var JWT_PRIVATE_KEY_PATH on auth-svc, JWT_",
+          "score": 0.6146,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "commit-b2c3d4"
+        },
+        {
+          "text": "# auth-svc README\n\nOwner: priya. On-call: auth-oncall@example.com.\n\n## Environment\n- `JWT_PRIVATE_KEY_PATH` (required, RS256 PEM)\n- `JWT_PUBLIC_KEY_PATH` (required, RS256 PEM)\n- `JWT_ISSUER` (default: 'https://auth.example.com')\n- `SESSION_TTL_MINUTES` (default: 60)\n- `REFRESH_TTL_DAYS` (default: 30)\n\n**Do not use HS256**. The migration in b2c3d4e5 removed all HS256 paths.\n\n## Public API\n- `POST /",
+          "score": 0.5987,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "readme-auth"
+        },
+        {
+          "text": "[2026-02-15 Slack #platform]\njordan: reminder: /api/v1 is going away in 2 days. anyone still seeing v1 traffic?\nalex: we're clean on search-svc. grep'd all consumers.\npriya: auth is clean, v2-only for 6 months\njordan: cool. flipping the 410 flag on 2026-02-17 09:00 UTC. if your service gets paged, check your callers.\n\n[2026-02-17 Slack #platform]\njordan: v1 is gone. 410s in the last hour: 12. all ",
+          "score": 0.5356,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "chat-deprecation"
+        }
+      ],
+      "correct": false,
+      "score": 0.5,
+      "grading_notes": "judge=0.50 \u2014 The answer advises against adding HS256 without citing that all services only hold the public key since the RS256 migrat",
+      "search_time_ms": 1301.4432359777857,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 469,
+      "query_tokens": 25,
+      "context_tokens": 444,
+      "judge_tokens_in": 757,
+      "judge_tokens_out": 37,
+      "judge_latency_ms": 1715.3592600226402
+    },
+    {
+      "task_id": "rubric-incident-lesson",
+      "query": "We saw SignatureVerificationError in billing-svc logs right after a secret rotation. What's our SOP?",
+      "answer": "# billing-svc runbook: webhook signature errors\n\nSymptom: logs show `stripe.error.SignatureVerificationError`.\n\nDiagnosis:\n1. Confirm the webhook endpoint URL matches Stripe dashboard\n2. Check `STRIPE_WEBHOOK_SECRET` matches the secret in Stripe dashboard\n3. If secret was rotated recently, `kubectl rollout restart deployment/billing-svc`\n\nWe hit this on 2026-03-02 14:15 UTC \u2014 secret rotated but pods didn't reload\nuntil restart. See incident post-mortem in ops/incidents/2026-03-02-webhook.md.\n---\n# billing-svc runbook: webhook signature errors\n\nSymptom: logs show `stripe.error.SignatureVerificationError`.\n\nDiagnosis:\n1. Confirm the webhook endpoint URL matches Stripe dashboard\n2. Check `STRIPE_WEBHOOK_SECRET` matches the secret in Stripe dashboard\n3. If secret was rotated recently, `kubectl rollout restart deployment/billing-svc`\n\nWe hit this on 2026-03-02 14:15 UTC \u2014 secret rotated but pods didn't reload\nuntil restart. See incident post-mortem in ops/incidents/2026-03-02-webhook.md.\n---\nStack trace from prod 2026-03-02 14:15 UTC:\n\nTraceback (most recent call last):\n  File \"billing/webhooks.py\", line 42, in handle_webhook\n    event = StripeClient.parse_event(payload, signature)\n  File \"billing/stripe_client.py\", line 88, in parse_event\n    return stripe.Webhook.construct_event(payload, sig, secret)\n  File \"stripe/webhook.py\", line 56, in construct_event\n    raise SignatureVerificationError(...)\nstripe.error.SignatureVerificationError: No signatures found matching expected signature\n\nCaused by: STRIPE_WEBHOOK_SECRET rotated in prod at 14:10 but pod restart\ndidn't pick up new va",
+      "hits": [
+        {
+          "text": "# billing-svc runbook: webhook signature errors\n\nSymptom: logs show `stripe.error.SignatureVerificationError`.\n\nDiagnosis:\n1. Confirm the webhook endpoint URL matches Stripe dashboard\n2. Check `STRIPE_WEBHOOK_SECRET` matches the secret in Stripe dashboard\n3. If secret was rotated recently, `kubectl rollout restart deployment/billing-svc`\n\nWe hit this on 2026-03-02 14:15 UTC \u2014 secret rotated but po",
+          "score": 0.7663,
+          "source": "pentatonic-baseline",
+          "doc_id": "runbook-billing"
+        },
+        {
+          "text": "# billing-svc runbook: webhook signature errors\n\nSymptom: logs show `stripe.error.SignatureVerificationError`.\n\nDiagnosis:\n1. Confirm the webhook endpoint URL matches Stripe dashboard\n2. Check `STRIPE_WEBHOOK_SECRET` matches the secret in Stripe dashboard\n3. If secret was rotated recently, `kubectl rollout restart deployment/billing-svc`\n\nWe hit this on 2026-03-02 14:15 UTC \u2014 secret rotated but po",
+          "score": 0.6988,
+          "source": "pentatonic-baseline:L0_workspace_bm25",
+          "doc_id": "runbook-billing"
+        },
+        {
+          "text": "Stack trace from prod 2026-03-02 14:15 UTC:\n\nTraceback (most recent call last):\n  File \"billing/webhooks.py\", line 42, in handle_webhook\n    event = StripeClient.parse_event(payload, signature)\n  File \"billing/stripe_client.py\", line 88, in parse_event\n    return stripe.Webhook.construct_event(payload, sig, secret)\n  File \"stripe/webhook.py\", line 56, in construct_event\n    raise SignatureVerifica",
+          "score": 0.6836,
+          "source": "pentatonic-baseline",
+          "doc_id": "stack-trace-nullpointer"
+        }
+      ],
+      "correct": false,
+      "score": 0.5,
+      "grading_notes": "judge=0.50 \u2014 Mentioned a pod restart (kubectl rollout restart) without referencing that secrets only load at pod startup.",
+      "search_time_ms": 1268.8033459999133,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 414,
+      "query_tokens": 21,
+      "context_tokens": 393,
+      "judge_tokens_in": 687,
+      "judge_tokens_out": 35,
+      "judge_latency_ms": 1701.407715022564
+    }
+  ]
+}