@tangle-network/agent-bench 0.1.0

package/src/search-bench/tasks-fresh.ts

@@ -0,0 +1,688 @@
+/**
+ * Web-search-dependent coding tasks — generated by the fresh-docs-coding-tasks
+ * workflow and adversarially verified against primary docs (correct + current +
+ * deterministic oracle + would-a-frontier-model-fail-from-memory). Each hinges on a
+ * precise recent/niche current-API detail. Do not hand-edit casually — regenerate.
+ */
+import type { SearchTask } from './tasks'
+
+export const freshTasks: SearchTask[] = [
+  {
+    "id": "cf-queues-retryall-delayseconds",
+    "domain": "Cloudflare Queues (Workers)",
+    "prompt": "In a Cloudflare Queues consumer handler (Workers), write the call that marks every message in the current `MessageBatch` to be retried, but delayed by 30 seconds before redelivery. Use the exact batch-level method and the exact option key name from the Queues Workers JavaScript API.",
+    "needsFreshDocs": "Models frequently invent the delay option key (guessing `delay`, `delayMs`, `delayMillis`, or `retryDelay`) and sometimes call a per-message-style method on the batch. The real API is `batch.retryAll({ delaySeconds: 30 })` — `delaySeconds` is an integer 0–86400. Memory often produces a plausible-but-wrong key.",
+    "oracle": {
+      "containsAll": [
+        "retryAll",
+        "delaySeconds"
+      ],
+      "containsAny": [
+        "retryAll({ delaySeconds: 30 })",
+        "retryAll({delaySeconds: 30})",
+        "delaySeconds: 30"
+      ],
+      "notContains": [
+        "delayMs",
+        "delayMillis",
+        "retryDelay",
+        "delaySecs",
+        "delay:",
+        "delayInSeconds"
+      ],
+      "regex": [
+        "retryAll\\s*\\(\\s*\\{[^}]*delaySeconds\\s*:\\s*30"
+      ]
+    },
+    "searchQuery": "Cloudflare Queues retryAll delaySeconds QueueRetryOptions batch consumer",
+    "sourceUrl": "https://developers.cloudflare.com/queues/configuration/javascript-apis/",
+    "expectedUrlIncludes": [
+      "developers.cloudflare.com",
+      "queues",
+      "javascript-apis"
+    ],
+    "correctAnswer": "`batch.retryAll({ delaySeconds: 30 })`. The batch-level retry method is `retryAll(options?: QueueRetryOptions)`, and `QueueRetryOptions` has a single optional key `delaySeconds` (an integer between 0 and 86400)."
+  },
+  {
+    "id": "cf-workflows-step-retry-backoff-enum",
+    "domain": "Cloudflare Workflows (Workers)",
+    "prompt": "In a Cloudflare Workflow, write the `WorkflowStepConfig` second argument to `step.do()` that retries the step at most 5 times with an exponential backoff and a 10-second initial delay. Use the exact field names and the exact backoff string value from the Workflows Workers API.",
+    "needsFreshDocs": "The retry config nests under a `retries` object with fields `limit`, `delay`, `backoff` — models often guess `maxRetries`/`attempts`/`maxAttempts` instead of `limit`, and the backoff string is exactly one of `\"constant\" | \"linear\" | \"exponential\"`. Models commonly hallucinate `\"exponential_backoff\"`, `\"exp\"`, or an `initialDelay`/`backoffType` shape.",
+    "oracle": {
+      "containsAll": [
+        "retries",
+        "limit",
+        "exponential"
+      ],
+      "containsAny": [
+        "backoff: 'exponential'",
+        "backoff: \"exponential\"",
+        "backoff:'exponential'",
+        "limit: 5",
+        "limit:5"
+      ],
+      "notContains": [
+        "maxRetries",
+        "maxAttempts",
+        "exponential_backoff",
+        "backoffType",
+        "initialDelay",
+        "numRetries",
+        "exponentialBackoff"
+      ],
+      "regex": [
+        "retries\\s*:\\s*\\{[^}]*limit\\s*:\\s*5",
+        "backoff\\s*:\\s*['\"]exponential['\"]"
+      ]
+    },
+    "searchQuery": "Cloudflare Workflows WorkflowStepConfig retries limit delay backoff constant linear exponential",
+    "sourceUrl": "https://developers.cloudflare.com/workflows/build/sleeping-and-retrying/",
+    "expectedUrlIncludes": [
+      "developers.cloudflare.com",
+      "workflows",
+      "sleeping-and-retrying"
+    ],
+    "correctAnswer": "Pass `{ retries: { limit: 5, delay: '10 seconds', backoff: 'exponential' } }`. The retry config lives under a `retries` object with fields `limit` (max attempts), `delay` (number ms or duration string), and `backoff`, whose allowed values are exactly `\"constant\" | \"linear\" | \"exponential\"`."
+  },
+  {
+    "id": "cf-vectorize-query-returnmetadata-enum",
+    "domain": "Cloudflare Vectorize (Workers)",
+    "prompt": "Using the Cloudflare Vectorize Workers binding (v2), write the `index.query(vector, options)` call that returns the top 20 matches WITH each match's full metadata but WITHOUT the vector values. Use the exact current option key(s) and the exact string value — do not use the deprecated argument.",
+    "needsFreshDocs": "The modern API uses `returnMetadata` with string values `\"none\" | \"indexed\" | \"all\"` and a separate boolean `returnValues`, which replaced the now-deprecated boolean `returnVectors`. Models trained on older material emit `returnVectors: true` or treat `returnMetadata` as a boolean (`returnMetadata: true`) rather than the `'all'` string.",
+    "oracle": {
+      "containsAll": [
+        "returnMetadata",
+        "all"
+      ],
+      "containsAny": [
+        "returnMetadata: 'all'",
+        "returnMetadata: \"all\"",
+        "returnMetadata:'all'"
+      ],
+      "notContains": [
+        "returnVectors",
+        "returnMetadata: true",
+        "returnMetadata:true",
+        "returnMetadata: false",
+        "includeMetadata",
+        "returnMeta"
+      ],
+      "regex": [
+        "returnMetadata\\s*:\\s*['\"]all['\"]"
+      ]
+    },
+    "searchQuery": "Cloudflare Vectorize query returnMetadata indexed all returnValues deprecated returnVectors",
+    "sourceUrl": "https://developers.cloudflare.com/vectorize/reference/client-api/",
+    "expectedUrlIncludes": [
+      "developers.cloudflare.com",
+      "vectorize",
+      "client-api"
+    ],
+    "correctAnswer": "`await index.query(vector, { topK: 20, returnMetadata: 'all' })`. `returnMetadata` is a string-valued option (`\"none\" | \"indexed\" | \"all\"`) and `returnValues` is a separate boolean; both replaced the deprecated `returnVectors` boolean. Leaving `returnValues` off (default false) omits the vector values."
+  },
+  {
+    "id": "cf-d1-sessions-withsession-constraints",
+    "domain": "Cloudflare D1 (Workers, read replication / Sessions API)",
+    "prompt": "Using the Cloudflare D1 Sessions API for read replication, write the `env.DB.withSession(...)` call that forces the FIRST query of the session to hit the primary database instance (to read your own writes). Use the exact constraint string constant. Separately, state the exact HTTP header name the official example uses to round-trip the session bookmark between requests.",
+    "needsFreshDocs": "The Sessions API constraint constants are the exact strings \"first-primary\" and \"first-unconstrained\" (hyphenated, lowercase), and the canonical example round-trips the bookmark via the header \"x-d1-bookmark\". Models guess \"primary\", \"first_primary\", \"strong\", or header names like \"d1-bookmark\"/\"x-bookmark\". This API shipped April 2025 (read-replication public beta), so it is unlikely to be reliable from memory.",
+    "oracle": {
+      "containsAll": [
+        "withSession",
+        "first-primary",
+        "x-d1-bookmark"
+      ],
+      "containsAny": [
+        "withSession('first-primary')",
+        "withSession(\"first-primary\")",
+        "withSession(`first-primary`)"
+      ],
+      "notContains": [
+        "first_primary",
+        "firstPrimary",
+        "withSession('primary')",
+        "withSession(\"primary\")",
+        "d1-bookmark\"",
+        "x-bookmark",
+        "first-write",
+        "strong-consistency"
+      ],
+      "regex": [
+        "withSession\\s*\\(\\s*['\"`]first-primary['\"`]\\s*\\)"
+      ]
+    },
+    "searchQuery": "Cloudflare D1 withSession first-primary first-unconstrained bookmark x-d1-bookmark read replication",
+    "sourceUrl": "https://developers.cloudflare.com/d1/best-practices/read-replication/",
+    "expectedUrlIncludes": [
+      "developers.cloudflare.com",
+      "d1",
+      "read-replication"
+    ],
+    "correctAnswer": "`const session = env.DB.withSession('first-primary')` (the doc's code uses backticks: withSession(`first-primary`)) forces the first query to the primary instance so you read your own writes; the other constant is `'first-unconstrained'`, which is the default. The official read-replication example round-trips the bookmark via the `x-d1-bookmark` HTTP header — reading it with request.headers.get('x-d1-bookmark') and writing session.getBookmark() back with response.headers.set('x-d1-bookmark', ...)."
+  },
+  {
+    "id": "nextjs15-after-import-next-server",
+    "domain": "Next.js 15 (App Router)",
+    "prompt": "In a Next.js 15.1+ App Router project, I want to run a non-blocking task (analytics logging) after the response has finished streaming, using the stabilized `after()` function. Write the exact `import` statement that brings `after` into scope, then call it inside a Route Handler. Give me the precise module specifier the function is imported from.",
+    "needsFreshDocs": "after() was promoted from experimental `unstable_after` to stable `after` in v15.1.0. The stable export lives in `next/server`, NOT a dedicated `next/after` module (which does not exist), NOT `next/navigation`, and NOT `next/cache`. Models frequently hallucinate `next/after` by analogy to other Next.js function modules, or keep emitting the deprecated `unstable_after` name.",
+    "oracle": {
+      "containsAll": [
+        "import { after } from 'next/server'"
+      ],
+      "containsAny": [
+        "after(",
+        "after(()"
+      ],
+      "notContains": [
+        "next/after",
+        "unstable_after",
+        "import { unstable_after }",
+        "next/navigation",
+        "next/cache",
+        "import after from"
+      ]
+    },
+    "sourceUrl": "https://nextjs.org/docs/app/api-reference/functions/after",
+    "searchQuery": "Next.js 15 after function stable import next/server",
+    "expectedUrlIncludes": [
+      "nextjs.org",
+      "api-reference/functions/after"
+    ],
+    "correctAnswer": "The stabilized function is imported as `import { after } from 'next/server'`. It became stable in v15.1.0 (was `unstable_after` in the v15.0.0-rc). There is no `next/after` module; the wrong/hallucinated forms are `next/after`, `next/cache`, `next/navigation`, and the deprecated `unstable_after`."
+  },
+  {
+    "id": "tanstack-query-v5-infinite-initialpageparam",
+    "domain": "TanStack Query v5 (React)",
+    "prompt": "I'm migrating a `useInfiniteQuery` call to TanStack Query v5. The v4 call had `queryKey`, `queryFn`, and `getNextPageParam` but no explicit first-page parameter. In v5 the query throws/type-errors because a now-required option is missing. Name the exact option key that v5 made mandatory for infinite queries (it supplies the `pageParam` for the very first page), and also name the v5 option that caps how many pages are kept and refetched (it replaced `refetchPage`). Show them inside a `useInfiniteQuery({ ... })` config.",
+    "needsFreshDocs": "v5 made `initialPageParam` a required option on `useInfiniteQuery` (previously the first pageParam defaulted to `undefined`). It also removed `refetchPage` in favor of the new `maxPages` option. Models commonly omit `initialPageParam` entirely (v4 habit), invent `initialPage` / `defaultPageParam`, or still reference the removed `refetchPage`.",
+    "oracle": {
+      "containsAll": [
+        "initialPageParam",
+        "maxPages"
+      ],
+      "containsAny": [
+        "useInfiniteQuery",
+        "getNextPageParam"
+      ],
+      "notContains": [
+        "refetchPage",
+        "defaultPageParam",
+        "initialPage:",
+        "firstPageParam"
+      ]
+    },
+    "sourceUrl": "https://tanstack.com/query/v5/docs/framework/react/guides/migrating-to-v5",
+    "searchQuery": "TanStack Query v5 useInfiniteQuery initialPageParam required maxPages",
+    "expectedUrlIncludes": [
+      "tanstack.com/query/v5",
+      "migrating-to-v5"
+    ],
+    "correctAnswer": "v5 requires an explicit `initialPageParam` (used as the `pageParam` for the first page), and introduces `maxPages` to limit how many pages are stored/refetched (replacing the removed `refetchPage`). Correct config: `useInfiniteQuery({ queryKey, queryFn, initialPageParam: 0, getNextPageParam, maxPages: 3 })`."
+  },
+  {
+    "id": "astro5-content-layer-render-function",
+    "domain": "Astro 5 (Content Layer)",
+    "prompt": "I upgraded to Astro 5's Content Layer API. My old Astro 4 code did `const { Content } = await entry.render()` to render a collection entry, and it no longer works. Write the corrected Astro 5 code: include the exact import (from `astro:content`) and the exact call that produces the `Content` component from an `entry`. Name the function and its module.",
+    "needsFreshDocs": "Astro 5 removed the `entry.render()` method. Rendering is now done with a standalone `render(entry)` function imported from `astro:content` (e.g. `import { render } from 'astro:content'; const { Content } = await render(entry)`). Models trained mostly on Astro 4 will keep calling `await entry.render()`. Separately, the `glob`/`file` loaders import from `astro/loaders`, a distractor models get wrong.",
+    "oracle": {
+      "containsAll": [
+        "render(entry)",
+        "astro:content"
+      ],
+      "containsAny": [
+        "import { render }",
+        "{ Content }",
+        "await render(entry)"
+      ],
+      "notContains": [
+        "entry.render()",
+        "await entry.render",
+        ".render()"
+      ]
+    },
+    "sourceUrl": "https://docs.astro.build/en/guides/content-collections/",
+    "searchQuery": "Astro 5 content layer render function astro:content entry.render removed",
+    "expectedUrlIncludes": [
+      "docs.astro.build",
+      "content-collections"
+    ],
+    "correctAnswer": "In Astro 5 you call the standalone `render()` function imported from `astro:content`: `import { render } from 'astro:content'; const { Content } = await render(entry);`. The old `entry.render()` instance method was removed."
+  },
+  {
+    "id": "svelte5-state-raw-not-frozen",
+    "domain": "Svelte 5 (runes)",
+    "prompt": "In Svelte 5, I have a large array that I do NOT want to be deeply reactive (I'll only ever reassign it, never mutate in place) so I avoid the proxying cost. Write the rune declaration that creates this shallow / non-deeply-reactive state. Give the exact rune name (the `$state.*` variant).",
+    "needsFreshDocs": "The shallow-state rune was renamed from `$state.frozen` to `$state.raw` in v5.0.0-next.218 (PR #12808), well before the stable 5.0 release. Models trained on early Svelte 5 preview material emit `$state.frozen`, which does not exist in shipped Svelte 5. The correct current rune is `$state.raw`.",
+    "oracle": {
+      "containsAll": [
+        "$state.raw"
+      ],
+      "containsAny": [
+        "$state.raw(",
+        "= $state.raw"
+      ],
+      "notContains": [
+        "$state.frozen",
+        "$state.shallow",
+        "$state.snapshot(",
+        "Object.freeze"
+      ]
+    },
+    "sourceUrl": "https://svelte.dev/docs/svelte/$state",
+    "searchQuery": "Svelte 5 $state.raw non-deeply reactive shallow state rune",
+    "expectedUrlIncludes": [
+      "svelte.dev/docs/svelte/$state"
+    ],
+    "correctAnswer": "Use `$state.raw`, e.g. `let items = $state.raw([...]);` — state declared with `$state.raw` is not deeply reactive and can only be reassigned, not mutated. The earlier preview name `$state.frozen` was renamed to `$state.raw` (PR #12808) and no longer exists."
+  },
+  {
+    "id": "drizzle-relations-v2-through",
+    "domain": "ORMs/DBs: Drizzle",
+    "prompt": "Using Drizzle ORM Relational Queries v2 (the new relations API introduced in drizzle-orm v1.0.0-beta), define a many-to-many relation between `users` and `groups` through a junction table `usersToGroups`. Write the relations definition. Show the exact top-level function call, and the exact way each side of the many-to-many maps the junction-table foreign keys.",
+    "needsFreshDocs": "Drizzle's v2 relations API (`defineRelations` + `r.many.<table>({ from, to })` chained through the junction table with `.through()`) shipped in late 2025 and is completely different from the v1 `relations()` / `relationName` + explicit junction-table relation objects. A model relying on memory will almost certainly emit the v1 `relations(usersToGroups, ({ one }) => ...)` boilerplate or invent a `.via()`/`junction:` key rather than the real `from: r.users.id.through(r.usersToGroups.userId)` form.",
+    "oracle": {
+      "containsAll": [
+        "defineRelations",
+        ".through(",
+        "r.many",
+        "from:",
+        "to:"
+      ],
+      "containsAny": [
+        "r.users.id.through",
+        "r.usersToGroups",
+        "usersToGroups.userId"
+      ],
+      "notContains": [
+        "relationName",
+        ".via(",
+        "junction:",
+        "manyToMany("
+      ]
+    },
+    "searchQuery": "Drizzle ORM relations v2 defineRelations through many-to-many",
+    "sourceUrl": "https://orm.drizzle.team/docs/relations-v2",
+    "correctAnswer": "In Drizzle relations v2 you use `defineRelations(schema, (r) => ({ ... }))` and express many-to-many with `r.many.groups({ from: r.users.id.through(r.usersToGroups.userId), to: r.groups.id.through(r.usersToGroups.groupId) })`. The junction table is wired by chaining `.through(...)` onto each side's column reference inside the `from`/`to` keys — there is no separate junction relation object and no `relationName`."
+  },
+  {
+    "id": "prisma-typedsql-param-comment",
+    "domain": "ORMs/DBs: Prisma",
+    "prompt": "You are using Prisma's TypedSQL feature (the `typedSql` preview feature, introduced in Prisma 5.19.0 and supported in Prisma 6). Write a `.sql` file named `getUsersByAge.sql` that takes two integer parameters (min age, max age) as typed, named arguments using TypedSQL's comment-annotation syntax. Then show the TypeScript that imports the generated query function and runs it. Use the exact comment-annotation syntax TypedSQL requires for declaring the typed parameters, the correct generated-import path, and the correct execution method.",
+    "needsFreshDocs": "TypedSQL's parameter-annotation syntax is the literal line comment `-- @param {Int} $1:minAge` (type in braces, positional `$N`, optional `:alias`), and the generated function is run via `prisma.$queryRawTyped(getUsersByAge(...))`, imported from `@prisma/client/sql` (default generator output) or `./generated/prisma/sql` (custom output). This is a 2024-era preview feature: a model tends to invent `$queryRaw` tagged templates, `:minAge` colon named-binds, or an `@arg`/`@input` annotation rather than the literal `-- @param {Type} $N:alias` comment plus the `$queryRawTyped` call.",
+    "oracle": {
+      "containsAll": [
+        "@param",
+        "$queryRawTyped"
+      ],
+      "containsAny": [
+        "@prisma/client/sql",
+        "generated/prisma/sql"
+      ],
+      "regex": [
+        "--\\s*@param\\s*\\{Int\\}\\s*\\$1"
+      ],
+      "notContains": [
+        "$queryRaw`",
+        "$queryRawUnsafe",
+        "@arg",
+        "@input",
+        "createTypedSql",
+        ":minAge,"
+      ]
+    },
+    "searchQuery": "Prisma TypedSQL @param comment syntax queryRawTyped import @prisma/client/sql",
+    "sourceUrl": "https://www.prisma.io/docs/orm/prisma-client/using-raw-sql/typedsql",
+    "correctAnswer": "In getUsersByAge.sql, declare typed params with line comments `-- @param {Int} $1:minAge` and `-- @param {Int} $2:maxAge` above the `SELECT ... WHERE age > $1 AND age < $2` statement. In TS, `import { getUsersByAge } from '@prisma/client/sql'` (default generator output; use `./generated/prisma/sql` if a custom `output` is configured) and run it with `await prisma.$queryRawTyped(getUsersByAge(minAge, maxAge))`. Requires `previewFeatures = [\"typedSql\"]` and generating with `prisma generate --sql`."
+  },
+  {
+    "id": "drizzle-upstash-cache-withcache",
+    "domain": "ORMs/DBs: Drizzle",
+    "prompt": "Using Drizzle ORM's built-in query caching with Upstash Redis, wire up the cache when constructing the `db` instance, then run a single `select` query that reads from the cache, gives that query a custom cache key, and disables automatic invalidation for it. Show the exact import for the Upstash cache helper, the exact option key passed to `drizzle()`, and the exact per-query method and its option keys.",
+    "needsFreshDocs": "Drizzle's caching layer (`upstashCache()` from `drizzle-orm/cache/upstash`, passed as `cache:` to `drizzle()`, opted into per-query via `.$withCache({ tag, autoInvalidate })`) landed in drizzle-orm 0.44 (2025). A model guessing from memory will likely invent `.cache()`, `.withCache()` (no `$`), a `redis:`/`cacheProvider:` config key, or import from `@upstash/...` rather than the real `drizzle-orm/cache/upstash` + `.$withCache({...})`.",
+    "oracle": {
+      "containsAll": [
+        "upstashCache",
+        "drizzle-orm/cache/upstash",
+        ".$withCache",
+        "autoInvalidate"
+      ],
+      "containsAny": [
+        "cache:",
+        "tag:",
+        "tag: 'custom"
+      ],
+      "notContains": [
+        ".withCache(",
+        ".cache(",
+        "cacheProvider",
+        "@upstash/redis/drizzle"
+      ]
+    },
+    "searchQuery": "Drizzle ORM upstashCache $withCache autoInvalidate tag cache config",
+    "sourceUrl": "https://orm.drizzle.team/docs/cache",
+    "correctAnswer": "Import `import { upstashCache } from 'drizzle-orm/cache/upstash'` and pass `cache: upstashCache({ ... })` to `drizzle()`. Per query, call `.$withCache({ tag: 'custom_key', autoInvalidate: false })` (also accepts `config: {}`) to read from cache with a custom key and no auto-invalidation."
+  },
+  {
+    "id": "libsql-client-transaction-modes-concurrency",
+    "domain": "ORMs/DBs: Turso/libSQL",
+    "prompt": "Using the `@libsql/client` TypeScript SDK: (1) what are the exact allowed string values for the transaction mode accepted by `client.transaction(mode)` and `client.batch(stmts, mode)`, and (2) what is the name of the `createClient` config option that caps in-flight requests and what is its default value? Show a `createClient` call setting that option.",
+    "needsFreshDocs": "The libSQL `TransactionMode` union is exactly `'write' | 'read' | 'deferred'` (NOT SQLite's `IMMEDIATE`/`EXCLUSIVE`), and the client's request cap is the `concurrency` option defaulting to 20. A model from memory tends to answer with SQLite-native modes (`deferred`/`immediate`/`exclusive`) and to hallucinate a `maxConcurrency`/`poolSize` option or omit the default.",
+    "oracle": {
+      "containsAll": [
+        "concurrency",
+        "20"
+      ],
+      "containsAny": [
+        "\"write\"",
+        "'write'",
+        "write",
+        "deferred"
+      ],
+      "notContains": [
+        "IMMEDIATE",
+        "EXCLUSIVE",
+        "immediate",
+        "exclusive",
+        "maxConcurrency",
+        "poolSize"
+      ]
+    },
+    "searchQuery": "@libsql/client TransactionMode write read deferred concurrency default 20 createClient",
+    "sourceUrl": "https://github.com/tursodatabase/libsql-client-ts/blob/main/packages/libsql-core/src/api.ts",
+    "correctAnswer": "The `TransactionMode` union is exactly `\"write\" | \"read\" | \"deferred\"` (default `\"deferred\"`); these are the only valid values for `client.transaction(mode)` / `client.batch(stmts, mode)`. The request-cap option is `concurrency`, defaulting to 20, e.g. `createClient({ url, concurrency: 10 })`."
+  },
+  {
+    "id": "biome-v2-organize-imports-assist-source",
+    "domain": "Biome 2.0 configuration (biome.json)",
+    "prompt": "In a Biome 2.0 project, organize-imports was moved out of its old dedicated top-level config field and is now an assist action. Write the exact JSON in biome.json that enables organize-imports in Biome v2. Give the full nested key path and the value used to turn it on.",
+    "needsFreshDocs": "Biome v1 used a top-level `\"organizeImports\": { \"enabled\": true }` field. Biome 2.0 (released 2025) relocated it to the assist subsystem at `assist.actions.source.organizeImports` with a string level value `\"on\"`/`\"off\"`. A model relying on memory will almost certainly emit the deprecated v1 `organizeImports.enabled: true` boolean form, which no longer exists in v2.",
+    "oracle": {
+      "containsAll": [
+        "assist",
+        "actions",
+        "source",
+        "organizeImports",
+        "\"on\""
+      ],
+      "containsAny": [
+        "\"assist\"",
+        "assist:"
+      ],
+      "notContains": [
+        "\"enabled\": true",
+        "\"enabled\":true",
+        "organizeImports\": { \"enabled",
+        "organizeImports\":{\"enabled",
+        "linter.actions",
+        "source.organizeImports.enabled"
+      ]
+    },
+    "searchQuery": "Biome v2 organizeImports assist actions source migrate",
+    "sourceUrl": "https://biomejs.dev/assist/actions/organize-imports/",
+    "correctAnswer": "In Biome 2.0 organize-imports is an assist action enabled via the nested path assist.actions.source.organizeImports set to the string \"on\": { \"assist\": { \"actions\": { \"source\": { \"organizeImports\": \"on\" } } } }. The old v1 form \"organizeImports\": { \"enabled\": true } is removed (handled by `biome migrate`).",
+    "expectedUrlIncludes": [
+      "biomejs.dev",
+      "organize-imports"
+    ]
+  },
+  {
+    "id": "vite6-default-resolve-conditions-exports",
+    "domain": "Vite 6 resolve configuration",
+    "prompt": "I'm migrating a Vite plugin to Vite 6. In v6, internally-added resolve conditions are no longer prepended automatically, so to preserve resolution I must spread Vite's exported default conditions when I set a custom condition. Name the two exact identifiers Vite 6 exports from the 'vite' package for the default client and server resolve conditions, and show how to set resolve.conditions to ['custom'] plus the client defaults.",
+    "needsFreshDocs": "Vite 6 (late 2024) introduced the named exports `defaultClientConditions` and `defaultServerConditions` from 'vite', and stopped auto-adding the built-in conditions to a user-specified `resolve.conditions`. Models often hallucinate names like `defaultConditions`, `DEFAULT_CONDITIONS`, or `defaultResolveConditions`, or claim Vite still merges defaults automatically.",
+    "oracle": {
+      "containsAll": [
+        "defaultClientConditions",
+        "defaultServerConditions"
+      ],
+      "containsAny": [
+        "...defaultClientConditions",
+        "['custom', ...defaultClientConditions]",
+        "[\"custom\", ...defaultClientConditions]"
+      ],
+      "notContains": [
+        "defaultConditions",
+        "DEFAULT_CONDITIONS",
+        "defaultResolveConditions",
+        "defaultClientCondition\"",
+        "getDefaultConditions"
+      ]
+    },
+    "searchQuery": "Vite 6 defaultClientConditions defaultServerConditions resolve.conditions migration",
+    "sourceUrl": "https://v6.vite.dev/guide/migration",
+    "correctAnswer": "Vite 6 exports defaultClientConditions and defaultServerConditions from 'vite'. Because v6 no longer auto-adds built-in conditions, a custom value must spread them, e.g. resolve: { conditions: ['custom', ...defaultClientConditions] } (and ['custom', ...defaultServerConditions] for ssr.resolve.conditions).",
+    "expectedUrlIncludes": [
+      "vite.dev",
+      "migration"
+    ]
+  },
+  {
+    "id": "esbuild-025-serve-cors-origin-option",
+    "domain": "esbuild dev server API (0.25.x CORS)",
+    "prompt": "esbuild 0.25.0 disabled the dev server's CORS by default for security, so cross-origin requests to my server (started with ctx.serve(...)) are now denied. esbuild later added a serve() option to re-enable CORS for a specific origin. What is the exact serve() option name, the exact shape of its value, and which 0.25.x release first added it? Show the ctx.serve call enabling CORS for https://example.com.",
+    "needsFreshDocs": "esbuild 0.25.0 (Feb 2025) was a security release that DISABLED the dev server's CORS by default (no Access-Control-Allow-Origin; fixes GHSA-67mh-4wv8-2f99). It did NOT add a way to re-enable it. The configurable `cors` serve option — an object with an `origin` field, e.g. cors: { origin: 'https://example.com' } (Go API: Origin: []string{...}; CLI: --cors-origin=) — was added later, in 0.25.4 (issue #4125). Models from memory invent flags like `--cors`, `allowCors: true`, `corsOrigin`, or `headers: { 'Access-Control-Allow-Origin': ... }`, miss the nested `cors.origin` object shape, and wrongly attribute the option to 0.25.0 instead of 0.25.4.",
+    "oracle": {
+      "containsAll": [
+        "cors",
+        "origin"
+      ],
+      "containsAny": [
+        "cors: {",
+        "cors:{",
+        "{ origin:",
+        "origin: 'https://example.com'",
+        "origin: \"https://example.com\""
+      ],
+      "notContains": [
+        "corsOrigin",
+        "allowCors",
+        "--cors ",
+        "--cors=",
+        "allowedOrigins",
+        "cors: true",
+        "enableCors",
+        "cors-origin:",
+        "headers: {"
+      ]
+    },
+    "searchQuery": "esbuild 0.25.4 serve cors origin option development server CORS changelog issue 4125",
+    "sourceUrl": "https://github.com/evanw/esbuild/blob/main/CHANGELOG-2025.md",
+    "correctAnswer": "esbuild 0.25.0 disabled dev-server CORS by default. The configurable serve() option is `cors`, an object with an `origin` field (a string in JS, or an array in the Go API), and it was added in esbuild 0.25.4 (issue #4125). Example: ctx.serve({ servedir: '.', cors: { origin: 'https://example.com' } }), which sets Access-Control-Allow-Origin when the request's Origin matches. The CLI equivalent is --cors-origin=https://example.com.",
+    "expectedUrlIncludes": [
+      "github.com/evanw/esbuild",
+      "CHANGELOG-2025"
+    ]
+  },
+  {
+    "id": "bun-test-coverage-threshold-reporter-keys",
+    "domain": "Bun test coverage (bunfig.toml)",
+    "prompt": "In bunfig.toml I want bun test to fail when coverage drops below 90% per metric, and to also write an lcov report. Under the [test] section, give the exact key for per-metric thresholds with its three metric sub-keys, and the exact key for selecting reporters with its value to emit both a console summary and lcov.",
+    "needsFreshDocs": "Bun's bunfig.toml uses `coverageThreshold` with PLURAL per-metric keys `lines`/`functions`/`statements` (as fractions 0-1, not percentages), and `coverageReporter` (SINGULAR key name) whose value is an ARRAY like [\"text\", \"lcov\"]. Models trained on Jest frequently pluralize the reporter key to `coverageReporters`, nest thresholds under a `global` block, use a `branches` key (Bun's documented table omits branches), use percentages like 90 instead of 0.9, or guess singular metric keys (line/function/statement).",
+    "oracle": {
+      "containsAll": [
+        "coverageThreshold",
+        "lines",
+        "functions",
+        "statements",
+        "coverageReporter",
+        "lcov"
+      ],
+      "containsAny": [
+        "[\"text\", \"lcov\"]",
+        "[\"text\",\"lcov\"]",
+        "lines = 0.9",
+        "lines=0.9"
+      ],
+      "notContains": [
+        "coverageReporters",
+        "coverageThresholds",
+        "{ line =",
+        "{ line=",
+        "branches =",
+        "= 90"
+      ]
+    },
+    "searchQuery": "Bun bunfig.toml coverageThreshold lines functions statements coverageReporter lcov",
+    "sourceUrl": "https://bun.com/docs/test/code-coverage",
+    "correctAnswer": "Under [test], use coverageThreshold = { lines = 0.9, functions = 0.9, statements = 0.9 } (plural keys, fractions 0-1) and coverageReporter = [\"text\", \"lcov\"] (singular key, array value) to fail under 90% and write lcov.info.",
+    "expectedUrlIncludes": [
+      "bun.com",
+      "code-coverage"
+    ]
+  },
+  {
+    "id": "anthropic-cache-1h-ttl-usage-fields",
+    "domain": "Anthropic SDK (prompt caching)",
+    "prompt": "Using the Anthropic Messages API, I want to cache a large system prompt with the 1-hour (extended) cache duration instead of the default 5-minute one. (1) Give the exact `cache_control` object I attach to the content block to request the 1-hour TTL. (2) Give the exact `anthropic-beta` header value that enables the extended cache TTL. (3) In the response `usage`, what are the two exact field names inside the `cache_creation` object that break down the 5-minute vs 1-hour cache-write tokens?",
+    "needsFreshDocs": "The 1-hour extended cache (beta header `extended-cache-ttl-2025-04-11`, `ttl: '1h'`) and the `cache_creation.{ephemeral_5m_input_tokens,ephemeral_1h_input_tokens}` usage breakdown were added in 2025. From memory a model is likely to emit the old `prompt-caching-2024-07-31` header, hallucinate `ttl: '3600s'`/`duration`, or invent usage field names like `cache_creation_1h_input_tokens`.",
+    "oracle": {
+      "containsAll": [
+        "ephemeral",
+        "extended-cache-ttl-2025-04-11",
+        "ephemeral_5m_input_tokens",
+        "ephemeral_1h_input_tokens"
+      ],
+      "containsAny": [
+        "\"1h\"",
+        "'1h'",
+        "ttl"
+      ],
+      "notContains": [
+        "prompt-caching-2024-07-31",
+        "3600s",
+        "cache_creation_1h_input_tokens",
+        "\"ttl\": 3600"
+      ],
+      "regex": [
+        "ttl[\"']?\\s*:\\s*[\"']1h[\"']"
+      ]
+    },
+    "searchQuery": "Anthropic prompt caching 1h extended TTL cache_control beta header ephemeral_1h_input_tokens",
+    "sourceUrl": "https://platform.claude.com/docs/en/build-with-claude/prompt-caching",
+    "correctAnswer": "(1) `cache_control: { \"type\": \"ephemeral\", \"ttl\": \"1h\" }`. (2) beta header `anthropic-beta: extended-cache-ttl-2025-04-11`. (3) Inside `usage.cache_creation`, the fields are `ephemeral_5m_input_tokens` and `ephemeral_1h_input_tokens`."
+  },
+  {
+    "id": "openai-agents-handoff-prompt-prefix-and-as-tool",
+    "domain": "OpenAI Agents SDK (Python)",
+    "prompt": "In the OpenAI Agents SDK for Python (the `openai-agents` package): (1) What is the exact name of the recommended prompt-prefix constant you prepend to an agent's `instructions` so it handles handoffs correctly, and exactly which module is it imported from? (2) For an agent named `Refund Agent`, what is the exact default handoff tool name the SDK generates? (3) What is the exact Agent method that turns an agent into a callable tool (agent-as-tool, not a handoff), and name its two key parameters?",
+    "needsFreshDocs": "These are SDK-specific identifiers: `RECOMMENDED_PROMPT_PREFIX` lives in `agents.extensions.handoff_prompt`, the default handoff tool name is `transfer_to_refund_agent`, and agent-as-tool is `Agent.as_tool(tool_name=..., tool_description=...)`. From memory a model is apt to invent `HANDOFF_PROMPT_PREFIX`, put it in `agents.handoffs`, guess `handoff_to_refund_agent`, or call the method `as_function()`/`to_tool()`.",
+    "oracle": {
+      "containsAll": [
+        "RECOMMENDED_PROMPT_PREFIX",
+        "agents.extensions.handoff_prompt",
+        "transfer_to_refund_agent",
+        "as_tool"
+      ],
+      "containsAny": [
+        "tool_name",
+        "tool_description"
+      ],
+      "notContains": [
+        "HANDOFF_PROMPT_PREFIX",
+        "handoff_to_refund_agent",
+        "as_function",
+        "to_tool(",
+        "agents.handoffs.RECOMMENDED"
+      ]
+    },
+    "searchQuery": "OpenAI Agents SDK RECOMMENDED_PROMPT_PREFIX handoff_prompt transfer_to as_tool",
+    "sourceUrl": "https://openai.github.io/openai-agents-python/handoffs/",
+    "correctAnswer": "(1) `RECOMMENDED_PROMPT_PREFIX`, imported from `agents.extensions.handoff_prompt`. (2) `transfer_to_refund_agent`. (3) `Agent.as_tool(tool_name=..., tool_description=...)`."
+  },
+  {
+    "id": "mcp-ts-registertool-annotation-hints",
+    "domain": "Model Context Protocol (TypeScript SDK)",
+    "prompt": "Using the official MCP TypeScript SDK (`@modelcontextprotocol/sdk`), I'm registering a `delete-file` tool with `server.registerTool(name, config, handler)`. In the config object's `annotations` field I want to mark this tool as: not read-only, destructive, idempotent, and not interacting with external systems. List the exact four boolean annotation-hint property names defined by the MCP tool-annotations spec that I set inside `annotations`, and give the correct true/false value for each for this delete-file tool.",
+    "needsFreshDocs": "The tool annotation hints (`readOnlyHint`, `destructiveHint`, `idempotentHint`, `openWorldHint`) were standardized in the MCP 2025-03-26 spec revision and surface via `registerTool`'s `annotations` config. From memory a model tends to drop the `Hint` suffix (`readOnly`, `destructive`) or invent names like `isDestructive`/`mutatesState`/`external`.",
+    "oracle": {
+      "containsAll": [
+        "readOnlyHint",
+        "destructiveHint",
+        "idempotentHint",
+        "openWorldHint",
+        "annotations"
+      ],
+      "containsAny": [
+        "registerTool",
+        "destructiveHint: true",
+        "destructiveHint=true"
+      ],
+      "notContains": [
+        "isDestructive",
+        "mutatesState",
+        "readOnly:",
+        "destructive:",
+        "idempotent:"
+      ]
+    },
+    "searchQuery": "MCP tool annotations readOnlyHint destructiveHint idempotentHint openWorldHint registerTool",
+    "sourceUrl": "https://github.com/modelcontextprotocol/typescript-sdk/blob/main/docs/server.md",
+    "correctAnswer": "The four annotation hints are `readOnlyHint`, `destructiveHint`, `idempotentHint`, and `openWorldHint`, set inside the `annotations` config of `registerTool`. For delete-file: `readOnlyHint: false`, `destructiveHint: true`, `idempotentHint: true`, `openWorldHint: false`."
+  },
+  {
+    "id": "terraform-write-only-arg-wo-suffix",
+    "domain": "Terraform CLI / HCL (managed resources, write-only arguments)",
+    "prompt": "In a Terraform configuration using a provider that supports write-only arguments (introduced in Terraform 1.11, the feature that lets ephemeral values be passed to managed resources), I want to set a database instance's password from an ephemeral value so it is never persisted to state or plan. Show the exact argument name I assign the password to on the resource (the write-only naming convention/suffix), AND the exact companion argument I must increment to force Terraform to re-send the value on the next apply. Give the precise identifiers.",
+    "needsFreshDocs": "Write-only arguments shipped in Terraform 1.11 (March 2025). The convention is the literal suffix `_wo` on the argument name (e.g. `password_wo`) plus a companion `_wo_version` argument (e.g. `password_wo_version`) used as the change trigger. A model relying on memory tends to invent a `write_only = true` meta-argument, an `ephemeral = true` attribute flag, or reuse `sensitive`, none of which is the actual mechanism.",
+    "oracle": {
+      "containsAll": [
+        "_wo",
+        "_wo_version"
+      ],
+      "containsAny": [
+        "password_wo",
+        "password_wo_version",
+        "secret_string_wo"
+      ],
+      "notContains": [
+        "write_only = true",
+        "write_only=true",
+        "ephemeral = true",
+        "ephemeral=true",
+        "sensitive = true",
+        "writeonly",
+        "_writeonly",
+        "wo_version_id"
+      ],
+      "regex": [
+        "\\b\\w+_wo\\b",
+        "\\b\\w+_wo_version\\b"
+      ]
+    },
+    "searchQuery": "terraform 1.11 write-only arguments _wo suffix _wo_version managed resources ephemeral",
+    "sourceUrl": "https://developer.hashicorp.com/terraform/language/manage-sensitive-data/ephemeral",
+    "correctAnswer": "Assign the value to an argument with the `_wo` suffix, e.g. `password_wo = ephemeral.aws_secretsmanager_secret_version.db.secret_string`. To force Terraform to re-send it, increment the companion `password_wo_version` argument. The convention is `<name>_wo` (write-only argument) paired with `<name>_wo_version` (the change trigger). Write-only arguments are never stored in state or plan files."
+  },
+  {
+    "id": "uv-export-pylock-toml-format",
+    "domain": "uv (Python package manager, lockfile export)",
+    "prompt": "Using the uv CLI, I want to export my project's `uv.lock` into the new PEP 751 standardized lockfile format (NOT the legacy pip-style requirements file) and write it to disk. Give the exact `uv` subcommand, the exact flag and the exact format VALUE I pass to select the PEP 751 format, and the exact flag to write the result to a file. Also state the exact format value for the pip-compatible output and for a CycloneDX SBOM.",
+    "needsFreshDocs": "uv's `export` command added a `--format` flag whose PEP 751 value is the literal `pylock.toml` (and `cyclonedx1.5` for SBOM), with `requirements.txt` as the pip-compatible value, and `--output-file` to write to disk. From memory a model tends to guess `--output-format`, a bare `pylock` value without the `.toml`, `uv lock --format`, or `--to pep751`, none of which match the actual CLI.",
+    "oracle": {
+      "containsAll": [
+        "uv export",
+        "--format",
+        "pylock.toml"
+      ],
+      "containsAny": [
+        "--output-file",
+        "requirements.txt",
+        "cyclonedx1.5"
+      ],
+      "notContains": [
+        "--output-format",
+        "--to pylock",
+        "--format pylock\n",
+        "uv lock --format",
+        "pep751",
+        "--format pep-751",
+        "pylock.toml.lock"
+      ],
+      "regex": [
+        "uv export[\\s\\S]*--format\\s+pylock\\.toml"
+      ]
+    },
+    "searchQuery": "uv export --format pylock.toml PEP 751 requirements.txt cyclonedx1.5 --output-file",
+    "sourceUrl": "https://docs.astral.sh/uv/concepts/projects/export/",
+    "correctAnswer": "Run `uv export --format pylock.toml --output-file pylock.toml`. The `--format` flag accepts exactly `requirements.txt` (pip-compatible, the common default), `pylock.toml` (the PEP 751 standardized format), and `cyclonedx1.5` (SBOM). Use `--output-file` to write to disk; otherwise it prints to stdout."
+  }
+]