@pentatonic-ai/ai-agent-sdk 0.6.0 → 0.7.0

package/packages/memory-engine/bench/scorecards-pentatonic-baseline/marketplace-ops__pentatonic-memory__20260427-145422.json

@@ -0,0 +1,693 @@
+{
+  "bench": "marketplace-ops",
+  "stack": "pentatonic-memory",
+  "n_tasks": 20,
+  "n_correct": 5,
+  "accuracy": 0.25,
+  "mean_score": 0.25,
+  "p50_search_ms": 22.744177986169234,
+  "p95_search_ms": 29.40429764421424,
+  "total_tokens_in": 0,
+  "total_tokens_out": 0,
+  "total_usd": 0.0,
+  "by_tag": {
+    "factoid": {
+      "n": 12,
+      "mean_score": 0.16666666666666666,
+      "accuracy": 0.16666666666666666
+    },
+    "event-log": {
+      "n": 7,
+      "mean_score": 0.14285714285714285,
+      "accuracy": 0.14285714285714285
+    },
+    "multi-fact": {
+      "n": 1,
+      "mean_score": 0.0,
+      "accuracy": 0.0
+    },
+    "agent-commerce": {
+      "n": 6,
+      "mean_score": 0.16666666666666666,
+      "accuracy": 0.16666666666666666
+    },
+    "math": {
+      "n": 1,
+      "mean_score": 0.0,
+      "accuracy": 0.0
+    },
+    "seller": {
+      "n": 5,
+      "mean_score": 0.4,
+      "accuracy": 0.4
+    },
+    "buyer": {
+      "n": 3,
+      "mean_score": 0.0,
+      "accuracy": 0.0
+    },
+    "multi-doc": {
+      "n": 1,
+      "mean_score": 0.0,
+      "accuracy": 0.0
+    },
+    "policy": {
+      "n": 3,
+      "mean_score": 0.6666666666666666,
+      "accuracy": 0.6666666666666666
+    },
+    "rubric": {
+      "n": 3,
+      "mean_score": 0.0,
+      "accuracy": 0.0
+    },
+    "multi-hop": {
+      "n": 2,
+      "mean_score": 0.0,
+      "accuracy": 0.0
+    },
+    "entity": {
+      "n": 2,
+      "mean_score": 0.5,
+      "accuracy": 0.5
+    },
+    "negative": {
+      "n": 1,
+      "mean_score": 1.0,
+      "accuracy": 1.0
+    }
+  },
+  "extra": {
+    "ingest_ms": 7761.137416004203,
+    "grading": "substring",
+    "limit": 3,
+    "tokens": {
+      "corpus_tokens": 1388,
+      "query_tokens": 240,
+      "context_tokens": 438,
+      "retrieval_tokens": 678,
+      "naive_tokens": 28000,
+      "saved_tokens": 27322,
+      "reduction_pct": 0.9757857142857143,
+      "mean_retrieval_tokens_per_task": 33.9,
+      "tokenizer": "cl100k_base",
+      "per_task": {
+        "thing-lst-9001-sold-price": {
+          "query": 13,
+          "context": 0,
+          "retrieval": 13,
+          "judge_in": 0,
+          "judge_out": 0,
+          "judge_latency_ms": 0.0
+        },
+        "thing-lst-9001-buyer": {
+          "query": 8,
+          "context": 0,
+          "retrieval": 8,
+          "judge_in": 0,
+          "judge_out": 0,
+          "judge_latency_ms": 0.0
+        },
+        "thing-lst-9001-first-offer": {
+          "query": 17,
+          "context": 0,
+          "retrieval": 17,
+          "judge_in": 0,
+          "judge_out": 0,
+          "judge_latency_ms": 0.0
+        },
+        "thing-lst-9014-flagged-reason": {
+          "query": 11,
+          "context": 241,
+          "retrieval": 252,
+          "judge_in": 0,
+          "judge_out": 0,
+          "judge_latency_ms": 0.0
+        },
+        "thing-lst-9014-return-reason": {
+          "query": 11,
+          "context": 43,
+          "retrieval": 54,
+          "judge_in": 0,
+          "judge_out": 0,
+          "judge_latency_ms": 0.0
+        },
+        "thing-lst-9030-agent-offer": {
+          "query": 15,
+          "context": 0,
+          "retrieval": 15,
+          "judge_in": 0,
+          "judge_out": 0,
+          "judge_latency_ms": 0.0
+        },
+        "thing-lst-9030-agent-discount": {
+          "query": 15,
+          "context": 0,
+          "retrieval": 15,
+          "judge_in": 0,
+          "judge_out": 0,
+          "judge_latency_ms": 0.0
+        },
+        "seller-mariposa-rating": {
+          "query": 11,
+          "context": 64,
+          "retrieval": 75,
+          "judge_in": 0,
+          "judge_out": 0,
+          "judge_latency_ms": 0.0
+        },
+        "seller-rix-review-status": {
+          "query": 11,
+          "context": 0,
+          "retrieval": 11,
+          "judge_in": 0,
+          "judge_out": 0,
+          "judge_latency_ms": 0.0
+        },
+        "seller-velocipede-agent-friendly": {
+          "query": 14,
+          "context": 0,
+          "retrieval": 14,
+          "judge_in": 0,
+          "judge_out": 0,
+          "judge_latency_ms": 0.0
+        },
+        "buyer-sera-specialism": {
+          "query": 10,
+          "context": 0,
+          "retrieval": 10,
+          "judge_in": 0,
+          "judge_out": 0,
+          "judge_latency_ms": 0.0
+        },
+        "buyer-ariadne-disputes": {
+          "query": 11,
+          "context": 0,
+          "retrieval": 11,
+          "judge_in": 0,
+          "judge_out": 0,
+          "judge_latency_ms": 0.0
+        },
+        "policy-duplicate-trigger": {
+          "query": 12,
+          "context": 0,
+          "retrieval": 12,
+          "judge_in": 0,
+          "judge_out": 0,
+          "judge_latency_ms": 0.0
+        },
+        "policy-agent-opt-out": {
+          "query": 9,
+          "context": 45,
+          "retrieval": 54,
+          "judge_in": 0,
+          "judge_out": 0,
+          "judge_latency_ms": 0.0
+        },
+        "policy-enhanced-review-lifted": {
+          "query": 12,
+          "context": 45,
+          "retrieval": 57,
+          "judge_in": 0,
+          "judge_out": 0,
+          "judge_latency_ms": 0.0
+        },
+        "rubric-rix-buy-decision": {
+          "query": 17,
+          "context": 0,
+          "retrieval": 17,
+          "judge_in": 326,
+          "judge_out": 39,
+          "judge_latency_ms": 852.8406230211258
+        },
+        "rubric-lst-9014-full-story": {
+          "query": 16,
+          "context": 0,
+          "retrieval": 16,
+          "judge_in": 335,
+          "judge_out": 42,
+          "judge_latency_ms": 864.7506729960442
+        },
+        "rubric-agent-commerce-thora": {
+          "query": 8,
+          "context": 0,
+          "retrieval": 8,
+          "judge_in": 296,
+          "judge_out": 35,
+          "judge_latency_ms": 592.765478014946
+        },
+        "entity-all-sold-things": {
+          "query": 9,
+          "context": 0,
+          "retrieval": 9,
+          "judge_in": 0,
+          "judge_out": 0,
+          "judge_latency_ms": 0.0
+        },
+        "entity-sellers-with-disputes": {
+          "query": 10,
+          "context": 0,
+          "retrieval": 10,
+          "judge_in": 0,
+          "judge_out": 0,
+          "judge_latency_ms": 0.0
+        }
+      },
+      "judge_tokens_in": 957,
+      "judge_tokens_out": 116,
+      "judge_calls": 3,
+      "judge_mean_latency_ms": 770.118924677372
+    },
+    "cost_usd": {
+      "assumed_completion_tokens_per_task": 100,
+      "rates": {
+        "input_per_1k": 0.0025,
+        "output_per_1k": 0.01,
+        "model": "gpt-4o"
+      },
+      "retrieval_usd_in": 0.0016950000000000001,
+      "retrieval_usd_out": 0.02,
+      "retrieval_usd_total": 0.021695,
+      "naive_usd_total": 0.09000000000000001,
+      "saved_usd": 0.068305,
+      "saved_usd_per_1k_tasks": 3.4152500000000003
+    }
+  },
+  "task_results": [
+    {
+      "task_id": "thing-lst-9001-sold-price",
+      "query": "What was the final sale price of LST-9001?",
+      "answer": "",
+      "hits": [],
+      "correct": false,
+      "score": 0.0,
+      "grading_notes": "missing 1/1: ['2,250']",
+      "search_time_ms": 28.186109993839636,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 13,
+      "query_tokens": 13,
+      "context_tokens": 0,
+      "judge_tokens_in": 0,
+      "judge_tokens_out": 0,
+      "judge_latency_ms": 0.0
+    },
+    {
+      "task_id": "thing-lst-9001-buyer",
+      "query": "Who bought LST-9001?",
+      "answer": "",
+      "hits": [],
+      "correct": false,
+      "score": 0.0,
+      "grading_notes": "missing 1/1: ['sera-interiors']",
+      "search_time_ms": 20.723878988064826,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 8,
+      "query_tokens": 8,
+      "context_tokens": 0,
+      "judge_tokens_in": 0,
+      "judge_tokens_out": 0,
+      "judge_latency_ms": 0.0
+    },
+    {
+      "task_id": "thing-lst-9001-first-offer",
+      "query": "What was the first offer received on LST-9001 and was it accepted?",
+      "answer": "",
+      "hits": [],
+      "correct": false,
+      "score": 0.0,
+      "grading_notes": "missing 3/3: ['2,000', 'hendrik', 'declined']",
+      "search_time_ms": 22.80937001341954,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 17,
+      "query_tokens": 17,
+      "context_tokens": 0,
+      "judge_tokens_in": 0,
+      "judge_tokens_out": 0,
+      "judge_latency_ms": 0.0
+    },
+    {
+      "task_id": "thing-lst-9014-flagged-reason",
+      "query": "Why was LST-9014 flagged as duplicate?",
+      "answer": "TES event log for LST-9014:\n2026-02-14T09:20 CREATED by @rix-digital\n2026-02-14T09:25 LISTED (price \u00a3780)\n2026-02-15T12:00 FLAGGED_DUPLICATE \u2014 matches LST-9014-DUP (different seller, same serial number detected via device fingerprint)\n2026-02-15T13:30 LISTING_HELD pending review\n2026-02-16T10:00 REVIEW_CLEARED (original seller confirmed; dup delisted)\n2026-02-16T10:05 RELISTED\n2026-02-19T11:18 SOLD for \u00a3780 to @ariadne\n2026-02-20T14:00 DISPATCHED\n2026-02-22T09:45 DELIVERED\n2026-02-24T16:00 RETURN_REQUEST (buyer claims battery health lower than advertised)\n2026-02-26T10:30 RETURN_APPROVED, refund \u00a3780 issued\n2026-02-26T10:32 THING_RELISTED (accurate battery spec)",
+      "hits": [
+        {
+          "text": "TES event log for LST-9014:\n2026-02-14T09:20 CREATED by @rix-digital\n2026-02-14T09:25 LISTED (price \u00a3780)\n2026-02-15T12:00 FLAGGED_DUPLICATE \u2014 matches LST-9014-DUP (different seller, same serial number detected via device fingerprint)\n2026-02-15T13:30 LISTING_HELD pending review\n2026-02-16T10:00 REVIEW_CLEARED (original seller confirmed; dup delisted)\n2026-02-16T10:05 RELISTED\n2026-02-19T11:18 SOL",
+          "score": 0.735049280295262,
+          "source": "pentatonic-memory",
+          "doc_id": "tes-events-lst-9014"
+        }
+      ],
+      "correct": true,
+      "score": 1.0,
+      "grading_notes": "all substrings matched",
+      "search_time_ms": 19.363785977475345,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 252,
+      "query_tokens": 11,
+      "context_tokens": 241,
+      "judge_tokens_in": 0,
+      "judge_tokens_out": 0,
+      "judge_latency_ms": 0.0
+    },
+    {
+      "task_id": "thing-lst-9014-return-reason",
+      "query": "Why did the buyer return LST-9014?",
+      "answer": "Buyer @ariadne: consumer account. Purchases to date: 14. Average basket: \u00a3410. One open return (LST-9014 battery-health dispute, closed in buyer's favour).",
+      "hits": [
+        {
+          "text": "Buyer @ariadne: consumer account. Purchases to date: 14. Average basket: \u00a3410. One open return (LST-9014 battery-health dispute, closed in buyer's favour).",
+          "score": 0.708891987825674,
+          "source": "pentatonic-memory",
+          "doc_id": "buyer-ariadne-profile"
+        }
+      ],
+      "correct": false,
+      "score": 0.0,
+      "grading_notes": "missing 1/1: ['battery health']",
+      "search_time_ms": 19.234279985539615,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 54,
+      "query_tokens": 11,
+      "context_tokens": 43,
+      "judge_tokens_in": 0,
+      "judge_tokens_out": 0,
+      "judge_latency_ms": 0.0
+    },
+    {
+      "task_id": "thing-lst-9030-agent-offer",
+      "query": "Which agent made the offer on LST-9030 and for whom?",
+      "answer": "",
+      "hits": [],
+      "correct": false,
+      "score": 0.0,
+      "grading_notes": "missing 2/2: ['buyer-agent-7', 'thora']",
+      "search_time_ms": 25.778518989682198,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 15,
+      "query_tokens": 15,
+      "context_tokens": 0,
+      "judge_tokens_in": 0,
+      "judge_tokens_out": 0,
+      "judge_latency_ms": 0.0
+    },
+    {
+      "task_id": "thing-lst-9030-agent-discount",
+      "query": "What percentage discount did the shopping agent offer on LST-9030?",
+      "answer": "",
+      "hits": [],
+      "correct": false,
+      "score": 0.0,
+      "grading_notes": "missing 2/2: ['380', '420']",
+      "search_time_ms": 24.16933499625884,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 15,
+      "query_tokens": 15,
+      "context_tokens": 0,
+      "judge_tokens_in": 0,
+      "judge_tokens_out": 0,
+      "judge_latency_ms": 0.0
+    },
+    {
+      "task_id": "seller-mariposa-rating",
+      "query": "What's @mariposa's average star rating?",
+      "answer": "Seller @mariposa: registered 2024-08. Category: vintage furniture and homeware. Listings to date: 47. Sell-through rate: 89%. Average star rating: 4.8/5 (based on 38 reviews). Disputes in last 12 months: 0.",
+      "hits": [
+        {
+          "text": "Seller @mariposa: registered 2024-08. Category: vintage furniture and homeware. Listings to date: 47. Sell-through rate: 89%. Average star rating: 4.8/5 (based on 38 reviews). Disputes in last 12 months: 0.",
+          "score": 0.7218132427756615,
+          "source": "pentatonic-memory",
+          "doc_id": "seller-mariposa-profile"
+        }
+      ],
+      "correct": true,
+      "score": 1.0,
+      "grading_notes": "all substrings matched",
+      "search_time_ms": 20.875624002655968,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 75,
+      "query_tokens": 11,
+      "context_tokens": 64,
+      "judge_tokens_in": 0,
+      "judge_tokens_out": 0,
+      "judge_latency_ms": 0.0
+    },
+    {
+      "task_id": "seller-rix-review-status",
+      "query": "Is @rix-digital under any special review status?",
+      "answer": "",
+      "hits": [],
+      "correct": false,
+      "score": 0.0,
+      "grading_notes": "missing 1/1: ['enhanced listing review']",
+      "search_time_ms": 22.08369900472462,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 11,
+      "query_tokens": 11,
+      "context_tokens": 0,
+      "judge_tokens_in": 0,
+      "judge_tokens_out": 0,
+      "judge_latency_ms": 0.0
+    },
+    {
+      "task_id": "seller-velocipede-agent-friendly",
+      "query": "Does @velocipede-jo respond well to shopping agents?",
+      "answer": "",
+      "hits": [],
+      "correct": false,
+      "score": 0.0,
+      "grading_notes": "missing 2/2: ['yes', '5 minutes']",
+      "search_time_ms": 25.911798991728574,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 14,
+      "query_tokens": 14,
+      "context_tokens": 0,
+      "judge_tokens_in": 0,
+      "judge_tokens_out": 0,
+      "judge_latency_ms": 0.0
+    },
+    {
+      "task_id": "buyer-sera-specialism",
+      "query": "What does @sera-interiors typically buy?",
+      "answer": "",
+      "hits": [],
+      "correct": false,
+      "score": 0.0,
+      "grading_notes": "missing 2/2: ['mid-century', 'furniture']",
+      "search_time_ms": 18.640466005308554,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 10,
+      "query_tokens": 10,
+      "context_tokens": 0,
+      "judge_tokens_in": 0,
+      "judge_tokens_out": 0,
+      "judge_latency_ms": 0.0
+    },
+    {
+      "task_id": "buyer-ariadne-disputes",
+      "query": "Does @ariadne have any disputes on record?",
+      "answer": "",
+      "hits": [],
+      "correct": false,
+      "score": 0.0,
+      "grading_notes": "missing 2/2: ['LST-9014', 'battery']",
+      "search_time_ms": 20.008318999316543,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 11,
+      "query_tokens": 11,
+      "context_tokens": 0,
+      "judge_tokens_in": 0,
+      "judge_tokens_out": 0,
+      "judge_latency_ms": 0.0
+    },
+    {
+      "task_id": "policy-duplicate-trigger",
+      "query": "What triggers a FLAGGED_DUPLICATE event in TES?",
+      "answer": "",
+      "hits": [],
+      "correct": false,
+      "score": 0.0,
+      "grading_notes": "missing 2/2: ['device fingerprint', 'serial']",
+      "search_time_ms": 20.936009008437395,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 12,
+      "query_tokens": 12,
+      "context_tokens": 0,
+      "judge_tokens_in": 0,
+      "judge_tokens_out": 0,
+      "judge_latency_ms": 0.0
+    },
+    {
+      "task_id": "policy-agent-opt-out",
+      "query": "Can a seller refuse offers from shopping agents?",
+      "answer": "Agent-commerce policy: autonomous shopping agents must register with the marketplace and declare principal human(s). Agent offers follow the same event flow as human offers. Sellers may opt out of agent-commerce at listing creation (AGENT_OPT_OUT flag).",
+      "hits": [
+        {
+          "text": "Agent-commerce policy: autonomous shopping agents must register with the marketplace and declare principal human(s). Agent offers follow the same event flow as human offers. Sellers may opt out of agent-commerce at listing creation (AGENT_OPT_OUT flag).",
+          "score": 0.5153800318362944,
+          "source": "pentatonic-memory",
+          "doc_id": "policy-agent-commerce"
+        }
+      ],
+      "correct": true,
+      "score": 1.0,
+      "grading_notes": "all substrings matched",
+      "search_time_ms": 22.686360985971987,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 54,
+      "query_tokens": 9,
+      "context_tokens": 45,
+      "judge_tokens_in": 0,
+      "judge_tokens_out": 0,
+      "judge_latency_ms": 0.0
+    },
+    {
+      "task_id": "policy-enhanced-review-lifted",
+      "query": "After how long of clean activity is enhanced listing review lifted?",
+      "answer": "Enhanced listing review: triggered when a seller accumulates 3+ spec-discrepancy disputes in a 90-day window. Listings are manually reviewed before going live. Lifted after 30 days of clean activity.",
+      "hits": [
+        {
+          "text": "Enhanced listing review: triggered when a seller accumulates 3+ spec-discrepancy disputes in a 90-day window. Listings are manually reviewed before going live. Lifted after 30 days of clean activity.",
+          "score": 0.5413613700787364,
+          "source": "pentatonic-memory",
+          "doc_id": "policy-seller-enhanced-review"
+        }
+      ],
+      "correct": true,
+      "score": 1.0,
+      "grading_notes": "all substrings matched",
+      "search_time_ms": 22.642649011686444,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 57,
+      "query_tokens": 12,
+      "context_tokens": 45,
+      "judge_tokens_in": 0,
+      "judge_tokens_out": 0,
+      "judge_latency_ms": 0.0
+    },
+    {
+      "task_id": "rubric-rix-buy-decision",
+      "query": "A shopping agent is considering buying from @rix-digital. What's your recommendation?",
+      "answer": "",
+      "hits": [],
+      "correct": false,
+      "score": 0.0,
+      "grading_notes": "judge=0.00 \u2014 The agent's answer does not mention @rix-digital being under enhanced listing review due to spec discrepancies or its di",
+      "search_time_ms": 29.031301994109526,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 17,
+      "query_tokens": 17,
+      "context_tokens": 0,
+      "judge_tokens_in": 326,
+      "judge_tokens_out": 39,
+      "judge_latency_ms": 852.8406230211258
+    },
+    {
+      "task_id": "rubric-lst-9014-full-story",
+      "query": "Tell me the full story of LST-9014 from creation to resolution.",
+      "answer": "",
+      "hits": [],
+      "correct": false,
+      "score": 0.0,
+      "grading_notes": "judge=0.00 \u2014 The answer does not cover stage (1) created 2026-02-14 by @rix-digital, listed at \u00a3780.",
+      "search_time_ms": 27.950554998824373,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 16,
+      "query_tokens": 16,
+      "context_tokens": 0,
+      "judge_tokens_in": 335,
+      "judge_tokens_out": 42,
+      "judge_latency_ms": 864.7506729960442
+    },
+    {
+      "task_id": "rubric-agent-commerce-thora",
+      "query": "Describe how @thora uses the marketplace.",
+      "answer": "",
+      "hits": [],
+      "correct": false,
+      "score": 0.0,
+      "grading_notes": "judge=0.00 \u2014 The answer does not mention @thora using an autonomous shopping agent (@buyer-agent-7) at all.",
+      "search_time_ms": 29.42392899421975,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 8,
+      "query_tokens": 8,
+      "context_tokens": 0,
+      "judge_tokens_in": 296,
+      "judge_tokens_out": 35,
+      "judge_latency_ms": 592.765478014946
+    },
+    {
+      "task_id": "entity-all-sold-things",
+      "query": "List the thing IDs of all completed sales.",
+      "answer": "",
+      "hits": [],
+      "correct": false,
+      "score": 0.0,
+      "grading_notes": "no expected_substrings set",
+      "search_time_ms": 22.80199498636648,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 9,
+      "query_tokens": 9,
+      "context_tokens": 0,
+      "judge_tokens_in": 0,
+      "judge_tokens_out": 0,
+      "judge_latency_ms": 0.0
+    },
+    {
+      "task_id": "entity-sellers-with-disputes",
+      "query": "Which sellers in this corpus have disputes on record?",
+      "answer": "",
+      "hits": [],
+      "correct": true,
+      "score": 1.0,
+      "grading_notes": "no positive criteria (negative-only task)",
+      "search_time_ms": 27.79455398558639,
+      "generation_time_ms": 0.0,
+      "tokens_in": 0,
+      "tokens_out": 0,
+      "retrieval_tokens": 10,
+      "query_tokens": 10,
+      "context_tokens": 0,
+      "judge_tokens_in": 0,
+      "judge_tokens_out": 0,
+      "judge_latency_ms": 0.0
+    }
+  ]
+}