@jinn-network/client 0.1.8 → 0.1.9-canary.144d87d2

package/dist/solver-types/slates/held-out-slate.swe-rebench-v2.v2.json

@@ -0,0 +1,19 @@
+{
+  "comment": "BASELINE-FAILURE REGRESSION BENCHMARK (issue #986). Screened: gradeable at the current evalSemanticsVersion AND base claude-code/Haiku frozen fails 0/R (R≥3) AND a stronger Codex/GPT-5.5 prover passes ≥1 (proven headroom). Baseline 0% by construction. Held out from the generator train stream via the active-slate-version union. Content-addressed; scores comparable WITHIN this version only.",
+  "schemaVersion": "held-out-slate.v1",
+  "solverType": "swe-rebench-v2.v1",
+  "version": "v2",
+  "generatedAt": "2026-06-04T23:03:42.179Z",
+  "instanceIds": [
+    "agronholm__anyio-1134",
+    "astropy__astropy-19438",
+    "con__nwb2bids-347_interface",
+    "deepset-ai__haystack-9322",
+    "gidgethub__gidgethub-225",
+    "iris-hep__func_adl-199",
+    "MatterMiners__tardis-361",
+    "modelcontextprotocol__python-sdk-222",
+    "twisted__towncrier-687"
+  ],
+  "hash": "sha256:34bb8bfa5e85933e38706551d4afaec3adf4a6ad1c1c0e4d2132f2c47d43d039"
+}

package/dist/solver-types/slates/held-out-slate.swe-rebench-v2.v2.screening-report.json

@@ -0,0 +1,628 @@
+{
+  "generatedAt": "2026-06-04T23:03:42.179Z",
+  "evalSemanticsVersion": "4",
+  "baseCodeDigest": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+  "R": 3,
+  "proverHarness": "claude-code",
+  "proverModel": "opus",
+  "heldOutTotal": 9,
+  "newThisRun": [
+    "deepset-ai__haystack-9322",
+    "gidgethub__gidgethub-225",
+    "iris-hep__func_adl-199",
+    "MatterMiners__tardis-361",
+    "modelcontextprotocol__python-sdk-222",
+    "twisted__towncrier-687",
+    "agronholm__anyio-1134"
+  ],
+  "carriedOver": [
+    "astropy__astropy-19438",
+    "con__nwb2bids-347_interface"
+  ],
+  "screened": [
+    {
+      "instance_id": "aallan__vera-662_interface",
+      "repo": "aallan",
+      "basePasses": 1,
+      "proverPassed": null,
+      "baseRuns": 1,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-passes"
+    },
+    {
+      "instance_id": "agentscope-ai__qwenpaw-3278",
+      "repo": "agentscope-ai",
+      "basePasses": 0,
+      "proverPassed": null,
+      "baseRuns": 1,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-unscorable",
+      "unscorableReason": "grade-error: swe-rebench-v2 eval could not grade the solution (patch_does_not_apply)"
+    },
+    {
+      "instance_id": "agronholm__anyio-1121",
+      "repo": "agronholm",
+      "basePasses": 1,
+      "proverPassed": null,
+      "baseRuns": 1,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-passes"
+    },
+    {
+      "instance_id": "apache__iceberg-python-3295",
+      "repo": "apache",
+      "basePasses": 1,
+      "proverPassed": null,
+      "baseRuns": 1,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-passes"
+    },
+    {
+      "instance_id": "astronomy-commons__hats-648",
+      "repo": "astronomy-commons",
+      "basePasses": 0,
+      "proverPassed": null,
+      "baseRuns": 2,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-unscorable",
+      "unscorableReason": "grade-error: swe-rebench-v2 eval could not grade the solution (patch_merge_conflict)"
+    },
+    {
+      "instance_id": "beever-ai__beever-atlas-102",
+      "repo": "beever-ai",
+      "basePasses": 1,
+      "proverPassed": null,
+      "baseRuns": 1,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-passes"
+    },
+    {
+      "instance_id": "cea-hpc__clustershell-592",
+      "repo": "cea-hpc",
+      "basePasses": 1,
+      "proverPassed": null,
+      "baseRuns": 1,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-passes"
+    },
+    {
+      "instance_id": "cneud__alto-tools-29",
+      "repo": "cneud",
+      "basePasses": 1,
+      "proverPassed": null,
+      "baseRuns": 1,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-passes"
+    },
+    {
+      "instance_id": "copier-org__copier-2646",
+      "repo": "copier-org",
+      "basePasses": 0,
+      "proverPassed": null,
+      "baseRuns": 1,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-unscorable",
+      "unscorableReason": "grade-error: swe-rebench-v2 eval could not grade the solution (patch_does_not_apply)"
+    },
+    {
+      "instance_id": "cpp-linter__cpp-linter-hooks-206_interface",
+      "repo": "cpp-linter",
+      "basePasses": 1,
+      "proverPassed": null,
+      "baseRuns": 1,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-passes"
+    },
+    {
+      "instance_id": "cs-si__eodag-2176",
+      "repo": "cs-si",
+      "basePasses": 0,
+      "proverPassed": false,
+      "baseRuns": 3,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "no-headroom"
+    },
+    {
+      "instance_id": "CWorthy-ocean__C-Star-227",
+      "repo": "CWorthy-ocean",
+      "basePasses": 0,
+      "proverPassed": false,
+      "baseRuns": 3,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "no-headroom"
+    },
+    {
+      "instance_id": "dagster-io__dagster-33605",
+      "repo": "dagster-io",
+      "basePasses": 1,
+      "proverPassed": null,
+      "baseRuns": 1,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-passes"
+    },
+    {
+      "instance_id": "databricks__dbt-databricks-1131",
+      "repo": "databricks",
+      "basePasses": 1,
+      "proverPassed": null,
+      "baseRuns": 1,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-passes"
+    },
+    {
+      "instance_id": "deepset-ai__haystack-9322",
+      "repo": "deepset-ai",
+      "basePasses": 0,
+      "proverPassed": true,
+      "baseRuns": 3,
+      "gradeable": true,
+      "heldOut": true,
+      "reason": "held-out"
+    },
+    {
+      "instance_id": "ets-labs__python-dependency-injector-886",
+      "repo": "ets-labs",
+      "basePasses": 1,
+      "proverPassed": null,
+      "baseRuns": 1,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-passes"
+    },
+    {
+      "instance_id": "facelessuser__pymdown-extensions-2651",
+      "repo": "facelessuser",
+      "basePasses": 1,
+      "proverPassed": null,
+      "baseRuns": 1,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-passes"
+    },
+    {
+      "instance_id": "fsspec__universal_pathlib-395",
+      "repo": "fsspec",
+      "basePasses": 0,
+      "proverPassed": false,
+      "baseRuns": 3,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "no-headroom"
+    },
+    {
+      "instance_id": "gidgethub__gidgethub-225",
+      "repo": "gidgethub",
+      "basePasses": 0,
+      "proverPassed": true,
+      "baseRuns": 3,
+      "gradeable": true,
+      "heldOut": true,
+      "reason": "held-out"
+    },
+    {
+      "instance_id": "googleapis__python-api-core-821",
+      "repo": "googleapis",
+      "basePasses": 1,
+      "proverPassed": null,
+      "baseRuns": 1,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-passes"
+    },
+    {
+      "instance_id": "hgrecco__pint-pandas-287",
+      "repo": "hgrecco",
+      "basePasses": 0,
+      "proverPassed": null,
+      "baseRuns": 1,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-unscorable",
+      "unscorableReason": "grade-error: swe-rebench-v2 eval could not grade the solution (patch_merge_conflict)"
+    },
+    {
+      "instance_id": "hiero-ledger__hiero-sdk-python-1914_interface",
+      "repo": "hiero-ledger",
+      "basePasses": 1,
+      "proverPassed": null,
+      "baseRuns": 1,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-passes"
+    },
+    {
+      "instance_id": "icloud-photos-downloader__icloud_photos_downloader-1060",
+      "repo": "icloud-photos-downloader",
+      "basePasses": 1,
+      "proverPassed": null,
+      "baseRuns": 1,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-passes"
+    },
+    {
+      "instance_id": "iluvcapra__pycmx-12",
+      "repo": "iluvcapra",
+      "basePasses": 1,
+      "proverPassed": null,
+      "baseRuns": 1,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-passes"
+    },
+    {
+      "instance_id": "instructlab__sdg-484",
+      "repo": "instructlab",
+      "basePasses": 0,
+      "proverPassed": null,
+      "baseRuns": 1,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-unscorable",
+      "unscorableReason": "grade-error: insufficient disk for swe-rebench eval: 39.9 GB free, need ≥ 40.0 GB"
+    },
+    {
+      "instance_id": "intel__code-base-investigator-178",
+      "repo": "intel",
+      "basePasses": 1,
+      "proverPassed": null,
+      "baseRuns": 1,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-passes"
+    },
+    {
+      "instance_id": "iris-hep__func_adl-199",
+      "repo": "iris-hep",
+      "basePasses": 0,
+      "proverPassed": true,
+      "baseRuns": 3,
+      "gradeable": true,
+      "heldOut": true,
+      "reason": "held-out"
+    },
+    {
+      "instance_id": "isledecomp__reccmp-142",
+      "repo": "isledecomp",
+      "basePasses": 1,
+      "proverPassed": null,
+      "baseRuns": 1,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-passes"
+    },
+    {
+      "instance_id": "jawah__niquests-277",
+      "repo": "jawah",
+      "basePasses": 1,
+      "proverPassed": null,
+      "baseRuns": 1,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-passes"
+    },
+    {
+      "instance_id": "jlowin__fastmcp-1425",
+      "repo": "jlowin",
+      "basePasses": 0,
+      "proverPassed": null,
+      "baseRuns": 1,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-unscorable",
+      "unscorableReason": "grade-error: swe-rebench-v2 eval could not grade the solution (patch_does_not_apply)"
+    },
+    {
+      "instance_id": "JonathonReinhart__scuba-269",
+      "repo": "JonathonReinhart",
+      "basePasses": 1,
+      "proverPassed": null,
+      "baseRuns": 1,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-passes"
+    },
+    {
+      "instance_id": "MatterMiners__tardis-361",
+      "repo": "MatterMiners",
+      "basePasses": 0,
+      "proverPassed": true,
+      "baseRuns": 3,
+      "gradeable": true,
+      "heldOut": true,
+      "reason": "held-out"
+    },
+    {
+      "instance_id": "mempalace__mempalace-1004",
+      "repo": "mempalace",
+      "basePasses": 0,
+      "proverPassed": false,
+      "baseRuns": 3,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "no-headroom"
+    },
+    {
+      "instance_id": "modelcontextprotocol__python-sdk-222",
+      "repo": "modelcontextprotocol",
+      "basePasses": 0,
+      "proverPassed": true,
+      "baseRuns": 3,
+      "gradeable": true,
+      "heldOut": true,
+      "reason": "held-out"
+    },
+    {
+      "instance_id": "movingpandas__movingpandas-444",
+      "repo": "movingpandas",
+      "basePasses": 0,
+      "proverPassed": false,
+      "baseRuns": 3,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "no-headroom"
+    },
+    {
+      "instance_id": "nesquena__hermes-webui-1818",
+      "repo": "nesquena",
+      "basePasses": 0,
+      "proverPassed": null,
+      "baseRuns": 3,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-unscorable",
+      "unscorableReason": "harness-error: [claude-code-learner] harvestOutput: typed payload /var/folders/2y/y3bmqdw55kx2jhq804chv76m0000gn/T/jinn-eval-work-Fv5BLa/.execute/solution-payload.json failed swe-rebench-v2.v1/solution validation. U"
+    },
+    {
+      "instance_id": "networkx__networkx-7804",
+      "repo": "networkx",
+      "basePasses": 1,
+      "proverPassed": null,
+      "baseRuns": 1,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-passes"
+    },
+    {
+      "instance_id": "oemof__tespy-921",
+      "repo": "oemof",
+      "basePasses": 1,
+      "proverPassed": null,
+      "baseRuns": 1,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-passes"
+    },
+    {
+      "instance_id": "python-scim__scim2-models-139_interface",
+      "repo": "python-scim",
+      "basePasses": 1,
+      "proverPassed": null,
+      "baseRuns": 2,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-passes"
+    },
+    {
+      "instance_id": "python-wheel-build__fromager-1106",
+      "repo": "python-wheel-build",
+      "basePasses": 1,
+      "proverPassed": null,
+      "baseRuns": 1,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-passes"
+    },
+    {
+      "instance_id": "qBraid__pyqasm-120",
+      "repo": "qBraid",
+      "basePasses": 1,
+      "proverPassed": null,
+      "baseRuns": 2,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-passes"
+    },
+    {
+      "instance_id": "raullenchai__rapid-mlx-289",
+      "repo": "raullenchai",
+      "basePasses": 1,
+      "proverPassed": null,
+      "baseRuns": 2,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-passes"
+    },
+    {
+      "instance_id": "RDFLib__pySHACL-285",
+      "repo": "RDFLib",
+      "basePasses": 1,
+      "proverPassed": null,
+      "baseRuns": 1,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-passes"
+    },
+    {
+      "instance_id": "stanfordnlp__dspy-9478_interface",
+      "repo": "stanfordnlp",
+      "basePasses": 1,
+      "proverPassed": null,
+      "baseRuns": 1,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-passes"
+    },
+    {
+      "instance_id": "tobymao__sqlglot-4563",
+      "repo": "tobymao",
+      "basePasses": 1,
+      "proverPassed": null,
+      "baseRuns": 1,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-passes"
+    },
+    {
+      "instance_id": "Toloka__crowd-kit-128",
+      "repo": "Toloka",
+      "basePasses": 1,
+      "proverPassed": null,
+      "baseRuns": 1,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-passes"
+    },
+    {
+      "instance_id": "twisted__towncrier-687",
+      "repo": "twisted",
+      "basePasses": 0,
+      "proverPassed": true,
+      "baseRuns": 3,
+      "gradeable": true,
+      "heldOut": true,
+      "reason": "held-out"
+    },
+    {
+      "instance_id": "ucfopen__canvasapi-716",
+      "repo": "ucfopen",
+      "basePasses": 1,
+      "proverPassed": null,
+      "baseRuns": 1,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-passes"
+    },
+    {
+      "instance_id": "unionai-oss__pandera-1898",
+      "repo": "unionai-oss",
+      "basePasses": 1,
+      "proverPassed": null,
+      "baseRuns": 1,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-passes"
+    },
+    {
+      "instance_id": "vprusso__toqito-1484_interface",
+      "repo": "vprusso",
+      "basePasses": 0,
+      "proverPassed": null,
+      "baseRuns": 1,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-unscorable",
+      "unscorableReason": "grade-error: swe-rebench-v2 eval could not grade the solution (patch_merge_conflict)"
+    },
+    {
+      "instance_id": "zopefoundation__Zope-1243",
+      "repo": "zopefoundation",
+      "basePasses": 1,
+      "proverPassed": null,
+      "baseRuns": 1,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-passes"
+    },
+    {
+      "instance_id": "agronholm__anyio-1134",
+      "repo": "agronholm",
+      "basePasses": 0,
+      "proverPassed": true,
+      "baseRuns": 3,
+      "gradeable": true,
+      "heldOut": true,
+      "reason": "held-out"
+    },
+    {
+      "instance_id": "databricks__dbt-databricks-1428",
+      "repo": "databricks",
+      "basePasses": 0,
+      "proverPassed": null,
+      "baseRuns": 2,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-unscorable",
+      "unscorableReason": "grade-error: swe-rebench-v2 eval could not grade the solution (patch_does_not_apply)"
+    },
+    {
+      "instance_id": "hiero-ledger__hiero-sdk-python-2007_interface",
+      "repo": "hiero-ledger",
+      "basePasses": 1,
+      "proverPassed": null,
+      "baseRuns": 1,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-passes"
+    },
+    {
+      "instance_id": "isledecomp__reccmp-166",
+      "repo": "isledecomp",
+      "basePasses": 0,
+      "proverPassed": null,
+      "baseRuns": 3,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "no-headroom",
+      "unscorableReason": "harness: agent produced no patch"
+    },
+    {
+      "instance_id": "movingpandas__movingpandas-447",
+      "repo": "movingpandas",
+      "basePasses": 0,
+      "proverPassed": null,
+      "baseRuns": 1,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-unscorable",
+      "unscorableReason": "harness: agent produced no patch"
+    },
+    {
+      "instance_id": "tobymao__sqlglot-4618",
+      "repo": "tobymao",
+      "basePasses": 0,
+      "proverPassed": null,
+      "baseRuns": 1,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-unscorable",
+      "unscorableReason": "harness: agent produced no patch"
+    },
+    {
+      "instance_id": "tobymao__sqlglot-4643",
+      "repo": "tobymao",
+      "basePasses": 0,
+      "proverPassed": null,
+      "baseRuns": 1,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-unscorable",
+      "unscorableReason": "harness: agent produced no patch"
+    },
+    {
+      "instance_id": "tobymao__sqlglot-7479",
+      "repo": "tobymao",
+      "basePasses": 0,
+      "proverPassed": null,
+      "baseRuns": 1,
+      "gradeable": true,
+      "heldOut": false,
+      "reason": "base-unscorable",
+      "unscorableReason": "harness: agent produced no patch"
+    }
+  ]
+}

package/dist/solver-types/solver-type.d.ts

@@ -3,6 +3,7 @@
  * See jinn-mono-6q1.1.
  */
 import type { TaskGenerator } from '../tasks/sources.js';
+import type { LoadedHeldOutSlate } from './_swe-rebench-v2-held-out-slate.js';
 /** Overlay fields merged into Task when posting from --spec-file. */
 export type ParsedSpecOverlay = {
     window: unknown;
@@ -46,6 +47,13 @@ export interface SolverTypeDefinition<GenConfig = unknown> {
      * Return `undefined` to skip (default for kinds without auto-gen or not enabled on testnet).
      */
     getTestnetAutoConfig?: (ctx: TestnetAutoContext) => GenConfig | undefined;
+    /**
+     * Load this SolverType's held-out eval slate at `version` (issue #817).
+     * Optional so a future eval orchestrator can compose over SolverTypes that
+     * define a slate; SolverTypes without one omit it. The slate is
+     * content-addressed and throws on hash mismatch or unknown version.
+     */
+    loadHeldOutSlate?: (version: string) => LoadedHeldOutSlate;
     ui?: {
         description: string;
         category: string;

package/dist/solver-types/swe-rebench-v2.d.ts

@@ -40,6 +40,8 @@ export interface SweRebenchV2AutoConfig {
 export interface SweRebenchV2GeneratorStaticConfig {
     stateDir?: string;
     ipfsRegistryUrl?: string;
+    /** IPFS gateway for the fresh-volume pool-recovery fetch (#957). */
+    ipfsGatewayUrl?: string;
     agentEoa?: `0x${string}`;
     safeAddress?: `0x${string}`;
     agentPrivateKey?: `0x${string}`;