@remnic/bench 9.49.0 → 9.50.0

package/fixtures/h6-failure-gate/decision-rule.json

@@ -0,0 +1,146 @@
+{
+  "version": 10,
+  "name": "H6 Failure Gate Decision Rule",
+  "preregistration": {
+    "path": "docs/research/failure-gate/preregistration.md",
+    "sha256": "7c5bb227279b42f5e82aa2a0e7d05b7759736bee9a5e5893e1f126c95906b6aa"
+  },
+  "analysisPopulation": {
+    "datasetVersion": 1,
+    "datasetInventoryHash": "687615b5f7ff46977d268a03e30018070f7d0bec9d01e04da2d0c723e59a5b27",
+    "split": "main",
+    "taskCount": 18,
+    "seedsPerModelProfile": 5,
+    "modelProfileCount": 1,
+    "unit": "task",
+    "pairingKey": [
+      "taskId",
+      "variantId",
+      "seed",
+      "modelProfileId",
+      "modelProfileHash"
+    ],
+    "maximumPrimaryTaskCuts": 0
+  },
+  "analysis": {
+    "bootstrap": {
+      "draws": 10000,
+      "group": "task",
+      "confidenceLevel": 0.95,
+      "method": "percentile"
+    },
+    "shuffle": {
+      "draws": 10000,
+      "group": "task",
+      "alternative": "candidate_benefit",
+      "plusOneCorrection": true
+    },
+    "alpha": 0.05,
+    "multiplicity": {
+      "method": "HOLM",
+      "family": [
+        "H6_TIMING_REPEATED_FAILURE",
+        "H6_CONTENT_COMPOUND"
+      ],
+      "contentCompoundP": "max(repeatedFailureP,taskPassP)"
+    }
+  },
+  "factMatching": {
+    "timing": {
+      "factCount": 1,
+      "requireSameFactIds": true,
+      "requireSameCitationHashes": true,
+      "requireSameRenderedTokenCount": true
+    },
+    "content": {
+      "factCount": 1,
+      "requireSamePathShape": true,
+      "requireSameActionShape": true,
+      "textSimilarity": {
+        "metric": "normalized_token_set_jaccard",
+        "minimumInclusive": 0.8,
+        "maximumInclusive": 1
+      },
+      "tokenCountGap": {
+        "maximumAbsolute": 8,
+        "maximumRelative": 0.05,
+        "requireBoth": true
+      }
+    }
+  },
+  "hypotheses": {
+    "H6-timing": {
+      "baselineArm": "TURN_START_FAILURE",
+      "candidateArm": "PRE_ACTION_FAILURE",
+      "metric": "repeatedFailure",
+      "minimumRelativeRiskReduction": 0.3,
+      "minimumAbsoluteRepeatedFailureBenefit": 0.05,
+      "requireRepeatedFailureBenefitIntervalLowerStrictlyAbove": 0,
+      "requireHolmAdjustedPStrictlyBelow": 0.05,
+      "zeroBaselineDecision": "NOT_ESTIMABLE"
+    },
+    "H6-content": {
+      "baselineArm": "TURN_START_SUCCESS",
+      "candidateArm": "TURN_START_FAILURE",
+      "metrics": [
+        "repeatedFailure",
+        "taskPassed"
+      ],
+      "requireRepeatedFailureBenefitIntervalLowerStrictlyAbove": 0,
+      "requireTaskPassBenefitIntervalLowerStrictlyAbove": 0,
+      "requireHolmAdjustedCompoundPStrictlyBelow": 0.05,
+      "unmatchedPairDecision": "NOT_ESTIMABLE"
+    }
+  },
+  "power": {
+    "simulationDraws": 10000,
+    "minimumTimingPower": 0.8,
+    "minimumContentPower": 0.8,
+    "sourceSplit": "pilot",
+    "increaseIndependentTasksIfBelowThreshold": true
+  },
+  "trapAudit": {
+    "minimumTrappedRate": 0.3,
+    "minimumNonFixedRate": 0.5,
+    "maximumInvalidRows": 0,
+    "requireCompleteRows": true
+  },
+  "timidity": {
+    "baselineArm": "NO_MEMORY",
+    "candidateArm": "PRE_ACTION_FAILURE",
+    "population": "main_no_trap_revisions",
+    "confidenceLevel": 0.9,
+    "passRateMargin": 0.02,
+    "stepsMargin": 2,
+    "requireIntervalsStrictlyInsideMargins": true,
+    "minimumSimulatedPower": 0.8
+  },
+  "completeness": {
+    "expectedRowsFormula": "taskCount*variantsPerTask*seedsPerModelProfile*modelProfileCount*armCount",
+    "primaryArmCount": 5,
+    "hostFaultRetriesAfterFirstTry": 5,
+    "rerunTaskResults": false,
+    "invalidReasons": [
+      "CORPUS_INVALID",
+      "CORE_REPO_DIR_MISMATCH",
+      "START_DRIFT",
+      "TRACE_GAP",
+      "VAGUE_CHECK",
+      "MIXED_ARM_STATE",
+      "UNMATCHED_FACTS",
+      "WAIT_RULE_FAULT",
+      "HOST_RETRIES_EXHAUSTED"
+    ]
+  },
+  "outcomes": {
+    "PASS": "timing=SUPPORTED and content=SUPPORTED",
+    "PARTIAL": "exactly one primary is SUPPORTED and the other is REJECTED",
+    "REJECT": "timing=REJECTED and content=REJECTED",
+    "NOT_ESTIMABLE": "either primary is NOT_ESTIMABLE"
+  },
+  "gateStatuses": [
+    "NO_MATCH",
+    "MATCH_WARN",
+    "ERROR_FAIL_OPEN"
+  ]
+}

package/fixtures/h6-failure-gate/generator/regenerate.ts

@@ -0,0 +1,20 @@
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import {
+  H6_FROZEN_SEED,
+  generateH6BenchmarkDataset,
+  writeH6FixtureBundle,
+} from "../../../src/coding-graph/repo-gen/index.js";
+
+const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url));
+
+export async function regenerateH6Fixtures(outDir?: string): Promise<string> {
+  const dataset = await generateH6BenchmarkDataset(H6_FROZEN_SEED);
+  return writeH6FixtureBundle(outDir ?? join(SCRIPT_DIR, ".."), dataset);
+}
+
+if (process.argv[1] && process.argv[1].endsWith("regenerate.ts")) {
+  void regenerateH6Fixtures().then((path) => {
+    console.log(`Regenerated H6 dataset manifest and task trees at ${path}`);
+  });
+}

package/fixtures/h6-failure-gate/schema/action-intent.schema.json

@@ -0,0 +1,36 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "action-intent.schema.json",
+  "title": "H6ActionIntentV1",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "version",
+    "actionType",
+    "targetSymbol",
+    "filePath",
+    "contextHash"
+  ],
+  "properties": {
+    "version": {
+      "type": "integer",
+      "const": 1
+    },
+    "actionType": {
+      "type": "string",
+      "minLength": 1
+    },
+    "targetSymbol": {
+      "type": "string",
+      "minLength": 1
+    },
+    "filePath": {
+      "type": "string",
+      "minLength": 1
+    },
+    "contextHash": {
+      "type": "string",
+      "minLength": 1
+    }
+  }
+}

package/fixtures/h6-failure-gate/schema/dataset.schema.json

@@ -0,0 +1,166 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "dataset.schema.json",
+  "title": "H6BenchmarkDataset",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "version",
+    "seed",
+    "createdAt",
+    "inventoryHash",
+    "supportArtifactHashes",
+    "taxonomy",
+    "tasks",
+    "splits"
+  ],
+  "properties": {
+    "version": {
+      "type": "integer",
+      "const": 1
+    },
+    "seed": {
+      "type": "integer"
+    },
+    "createdAt": {
+      "type": "string",
+      "format": "date-time"
+    },
+    "inventoryHash": {
+      "type": "string",
+      "pattern": "^[0-9a-f]{64}$"
+    },
+    "supportArtifactHashes": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "trap-taxonomy.json",
+        "arms/arms.json",
+        "schema/action-intent.schema.json",
+        "schema/dataset.schema.json",
+        "schema/task.schema.json",
+        "schema/trap-fingerprint.schema.json"
+      ],
+      "properties": {
+        "trap-taxonomy.json": {
+          "type": "string",
+          "pattern": "^[0-9a-f]{64}$"
+        },
+        "arms/arms.json": {
+          "type": "string",
+          "pattern": "^[0-9a-f]{64}$"
+        },
+        "schema/action-intent.schema.json": {
+          "type": "string",
+          "pattern": "^[0-9a-f]{64}$"
+        },
+        "schema/dataset.schema.json": {
+          "type": "string",
+          "pattern": "^[0-9a-f]{64}$"
+        },
+        "schema/task.schema.json": {
+          "type": "string",
+          "pattern": "^[0-9a-f]{64}$"
+        },
+        "schema/trap-fingerprint.schema.json": {
+          "type": "string",
+          "pattern": "^[0-9a-f]{64}$"
+        }
+      }
+    },
+    "taxonomy": {
+      "type": "array",
+      "minItems": 6,
+      "maxItems": 6,
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": [
+          "trapId",
+          "name",
+          "description",
+          "trapMechanism",
+          "correctFix",
+          "inspiredBy"
+        ],
+        "properties": {
+          "trapId": {
+            "type": "string",
+            "enum": [
+              "flaky-looking-test",
+              "misleading-error-message",
+              "wrong-layer-fix",
+              "hidden-invariant",
+              "stale-cache-illusion",
+              "config-shadowing"
+            ]
+          },
+          "name": {
+            "type": "string",
+            "minLength": 1
+          },
+          "description": {
+            "type": "string"
+          },
+          "trapMechanism": {
+            "type": "string",
+            "minLength": 1
+          },
+          "correctFix": {
+            "type": "string",
+            "minLength": 1
+          },
+          "inspiredBy": {
+            "type": "string",
+            "minLength": 1
+          }
+        }
+      }
+    },
+    "tasks": {
+      "type": "array",
+      "minItems": 30,
+      "maxItems": 30,
+      "items": {
+        "$ref": "./task.schema.json"
+      }
+    },
+    "splits": {
+      "type": "object",
+      "required": [
+        "dev",
+        "pilot",
+        "main"
+      ],
+      "properties": {
+        "dev": {
+          "type": "array",
+          "minItems": 0,
+          "maxItems": 0,
+          "items": {
+            "type": "string",
+            "minLength": 1
+          }
+        },
+        "pilot": {
+          "type": "array",
+          "minItems": 12,
+          "maxItems": 12,
+          "items": {
+            "type": "string",
+            "minLength": 1
+          }
+        },
+        "main": {
+          "type": "array",
+          "minItems": 18,
+          "maxItems": 18,
+          "items": {
+            "type": "string",
+            "minLength": 1
+          }
+        }
+      }
+    }
+  }
+}

package/fixtures/h6-failure-gate/schema/task.schema.json

@@ -0,0 +1,245 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "task.schema.json",
+  "title": "H6BaseTask",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "id",
+    "trapId",
+    "domain",
+    "title",
+    "description",
+    "canonicalBaseFiles",
+    "checkCommand",
+    "fileCount",
+    "lineCount",
+    "fingerprint",
+    "normalizedActionIntent",
+    "offlineCheckMark",
+    "offlineFailureMark",
+    "maxTokenCap",
+    "maxAttemptCap",
+    "split",
+    "variants"
+  ],
+  "properties": {
+    "id": {
+      "type": "string",
+      "minLength": 1
+    },
+    "trapId": {
+      "type": "string",
+      "enum": [
+        "flaky-looking-test",
+        "misleading-error-message",
+        "wrong-layer-fix",
+        "hidden-invariant",
+        "stale-cache-illusion",
+        "config-shadowing"
+      ]
+    },
+    "domain": {
+      "type": "string",
+      "minLength": 1
+    },
+    "title": {
+      "type": "string",
+      "minLength": 1
+    },
+    "description": {
+      "type": "string"
+    },
+    "canonicalBaseFiles": {
+      "type": "array",
+      "minItems": 1,
+      "items": {
+        "$ref": "#/definitions/syntheticFile"
+      }
+    },
+    "checkCommand": {
+      "type": "string",
+      "const": "node test/check.js"
+    },
+    "fileCount": {
+      "type": "integer",
+      "minimum": 8,
+      "maximum": 15
+    },
+    "lineCount": {
+      "type": "integer",
+      "minimum": 300,
+      "maximum": 600
+    },
+    "fingerprint": {
+      "$ref": "./trap-fingerprint.schema.json"
+    },
+    "normalizedActionIntent": {
+      "$ref": "./action-intent.schema.json"
+    },
+    "offlineCheckMark": {
+      "type": "string",
+      "minLength": 1
+    },
+    "offlineFailureMark": {
+      "type": "string",
+      "minLength": 1
+    },
+    "maxTokenCap": {
+      "type": "integer",
+      "minimum": 1
+    },
+    "maxAttemptCap": {
+      "type": "integer",
+      "minimum": 1
+    },
+    "split": {
+      "type": "string",
+      "enum": [
+        "dev",
+        "pilot",
+        "main"
+      ]
+    },
+    "variants": {
+      "type": "array",
+      "minItems": 3,
+      "maxItems": 3,
+      "items": {
+        "$ref": "#/definitions/taskVariant"
+      }
+    }
+  },
+  "definitions": {
+    "syntheticFile": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "path",
+        "content"
+      ],
+      "properties": {
+        "path": {
+          "type": "string",
+          "minLength": 1
+        },
+        "content": {
+          "type": "string"
+        },
+        "isExecutable": {
+          "type": "boolean"
+        }
+      }
+    },
+    "strategyPatch": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "id",
+        "description",
+        "files"
+      ],
+      "properties": {
+        "id": {
+          "type": "string",
+          "minLength": 1
+        },
+        "description": {
+          "type": "string"
+        },
+        "files": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/syntheticFile"
+          }
+        }
+      }
+    },
+    "taskVariant": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "variantId",
+        "baseTaskId",
+        "variantIndex",
+        "distance",
+        "domain",
+        "files",
+        "strategyCandidates",
+        "badStrategyPatch",
+        "goodStrategyPatch",
+        "noTrapControlFiles",
+        "cleanRevisionSha",
+        "trapRevisionSha",
+        "rightRevisionSha",
+        "noTrapRevisionSha"
+      ],
+      "properties": {
+        "variantId": {
+          "type": "string",
+          "minLength": 1
+        },
+        "baseTaskId": {
+          "type": "string",
+          "minLength": 1
+        },
+        "variantIndex": {
+          "type": "integer",
+          "minimum": 1,
+          "maximum": 3
+        },
+        "distance": {
+          "type": "integer",
+          "minimum": 1,
+          "maximum": 3
+        },
+        "domain": {
+          "type": "string",
+          "minLength": 1
+        },
+        "files": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/syntheticFile"
+          }
+        },
+        "strategyCandidates": {
+          "type": "array",
+          "minItems": 2,
+          "maxItems": 2,
+          "items": {
+            "$ref": "#/definitions/strategyPatch"
+          }
+        },
+        "badStrategyPatch": {
+          "$ref": "#/definitions/strategyPatch"
+        },
+        "goodStrategyPatch": {
+          "$ref": "#/definitions/strategyPatch"
+        },
+        "noTrapControlFiles": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/syntheticFile"
+          }
+        },
+        "cleanRevisionSha": {
+          "type": "string",
+          "pattern": "^[0-9a-f]{40}$"
+        },
+        "trapRevisionSha": {
+          "type": "string",
+          "pattern": "^[0-9a-f]{40}$"
+        },
+        "rightRevisionSha": {
+          "type": "string",
+          "pattern": "^[0-9a-f]{40}$"
+        },
+        "noTrapRevisionSha": {
+          "type": "string",
+          "pattern": "^[0-9a-f]{40}$"
+        }
+      }
+    }
+  }
+}

package/fixtures/h6-failure-gate/schema/trap-fingerprint.schema.json

@@ -0,0 +1,48 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "trap-fingerprint.schema.json",
+  "title": "H6TrapFingerprintV1",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "version",
+    "trapId",
+    "symbol",
+    "file",
+    "pattern",
+    "strategyId"
+  ],
+  "properties": {
+    "version": {
+      "type": "integer",
+      "const": 1
+    },
+    "trapId": {
+      "type": "string",
+      "enum": [
+        "flaky-looking-test",
+        "misleading-error-message",
+        "wrong-layer-fix",
+        "hidden-invariant",
+        "stale-cache-illusion",
+        "config-shadowing"
+      ]
+    },
+    "symbol": {
+      "type": "string",
+      "minLength": 1
+    },
+    "file": {
+      "type": "string",
+      "minLength": 1
+    },
+    "pattern": {
+      "type": "string",
+      "minLength": 1
+    },
+    "strategyId": {
+      "type": "string",
+      "minLength": 1
+    }
+  }
+}

package/fixtures/h6-failure-gate/tasks/h6-task-01/files/README.md

@@ -0,0 +1,26 @@
+# Synthetic Repository for quillboard-inventory-sync
+
+This repository contains a local service module for quillboard-inventory-sync.
+Counterfactual SDKs are vendored under `vendor/vellum-quarry-sdk/`.
+Run the project check with `node test/check.js`.
+
+## Environment Architecture
+- Target files live under `src/`.
+- Vendored fakes live under `vendor/`.
+- Run offline validation via `node test/check.js`.
+- All operations operate in pure local mode without network IO.
+
+## Module Breakdown
+- `src/service.mjs`: Executable domain service logic.
+- `src/config.ts`: Configuration defaults and priority loaders.
+- `src/types.ts`: TypeScript interfaces and domain schemas.
+- `src/utils.ts`: Common utility functions.
+- `src/logger.ts`: Logging primitives.
+- `src/helper.ts`: Domain headers and context helpers.
+
+## Testing & Verification
+Execute the offline check runner:
+```bash
+node test/check.js
+```
+The command returns a process status for the current implementation.