orizu 0.5.19 → 0.5.21

package/dist/scorers-verify-parity-bridge.js

@@ -0,0 +1,90 @@
+/**
+ * The customer-side half of `orizu scorers verify-parity` (ALI-1554).
+ *
+ * Embedded as a string constant and written to a temp file at run time on
+ * purpose: no packaging (`files`/`vendor`) change, so there is nothing for the
+ * published tarball to lose. It runs in the CUSTOMER's interpreter and the
+ * customer's unscrubbed environment — it is their metric in their process.
+ *
+ * Contract: reads `{rows: [{row_id, row, model_output}]}` from
+ * ORIZU_PARITY_INPUT_PATH, calls `<module>:<function>(row, model_output)` per
+ * row, and writes `{scores: [{row_id, score} | {row_id, error}]}` to
+ * ORIZU_PARITY_OUTPUT_PATH. Exit 2 (with the reason on stderr) means the check
+ * could not run at all; a per-row exception is that row's error, not an abort.
+ */
+export const PARITY_BRIDGE_SOURCE = `import json, math, os, sys, traceback
+
+
+def fail(message):
+    sys.stderr.write(message + "\\n")
+    sys.exit(2)
+
+
+def last_line(text):
+    lines = [line for line in text.strip().splitlines() if line.strip()]
+    return lines[-1] if lines else "unknown error"
+
+
+def extract(value):
+    # Mirrors orizu_gepa.optimizer._score_from_scorer, minus the clamp: the
+    # CLI clamps both sides so the compared number is what GEPA optimizes.
+    # bool is deliberately NOT rejected: measured, _score_from_scorer(extra=
+    # {'score': True}) -> (1.0, None) and False -> (0.0, None), because bool is an
+    # int subclass and Python's numeric branch accepts it. An exact-match DSPy
+    # metric returning True must compare, not error.
+    if isinstance(value, (int, float)):
+        raw = value
+    elif isinstance(value, dict):
+        raw = value.get("score")
+        if raw is None and isinstance(value.get("model_response"), dict):
+            raw = value["model_response"].get("score")
+    elif hasattr(value, "score"):
+        # official GEPA's EvaluationResult NamedTuple and friends;
+        # objective_scores is deliberately ignored.
+        raw = value.score
+    else:
+        raise TypeError("scorer result must be a number, a dict with a numeric 'score', or an object with a numeric .score attribute")
+    if isinstance(raw, str):
+        raw = float(raw)
+    if not isinstance(raw, (int, float)):
+        raise TypeError("scorer result must include a numeric score")
+    raw = float(raw)
+    if not math.isfinite(raw):
+        raise ValueError("scorer result score must be finite")
+    return raw
+
+
+target = os.environ.get("ORIZU_PARITY_ORIGINAL", "")
+module_name, separator, function_name = target.partition(":")
+if not separator or not module_name or not function_name:
+    fail("--original must be <module>:<function> (got %r)" % target)
+
+# The customer's metric lives beside the command's cwd, not beside this
+# generated bridge file, so cwd must win over the script directory.
+sys.path.insert(0, os.getcwd())
+try:
+    import importlib
+
+    module = importlib.import_module(module_name)
+except Exception:
+    fail("Failed to import original metric module '%s': %s" % (module_name, last_line(traceback.format_exc())))
+
+if not hasattr(module, function_name):
+    fail("Original metric module '%s' has no attribute '%s'" % (module_name, function_name))
+metric = getattr(module, function_name)
+if not callable(metric):
+    fail("Original metric '%s:%s' is not callable" % (module_name, function_name))
+
+with open(os.environ["ORIZU_PARITY_INPUT_PATH"], encoding="utf-8") as handle:
+    payload = json.load(handle)
+
+scores = []
+for item in payload["rows"]:
+    try:
+        scores.append({"row_id": item["row_id"], "score": extract(metric(item["row"], item["model_output"]))})
+    except Exception:
+        scores.append({"row_id": item["row_id"], "error": last_line(traceback.format_exc())})
+
+with open(os.environ["ORIZU_PARITY_OUTPUT_PATH"], "w", encoding="utf-8") as handle:
+    json.dump({"scores": scores}, handle)
+`;

package/dist/scorers-verify-parity.d.ts

@@ -0,0 +1,81 @@
+/**
+ * `orizu scorers verify-parity` (ALI-1554): prove a migrated Orizu scorer
+ * runner scores the same rows the same way the customer's original Python
+ * metric does, UNDER THE PAYLOAD `run-gepa` sends the runner, before the first
+ * optimization spends tokens on a scorer that silently disagrees.
+ *
+ * Honesty note: `run-gepa` executes runners through the vendored Python
+ * `run_file_contract_runner`, not this TypeScript path. What this proves is
+ * "the runner, under the run-gepa flat_row payload, via the CLI's file
+ * contract". The payload is asserted key-for-key by
+ * test/scorers-verify-parity.test.ts (H9) against a literal hand-derived from
+ * `make_scorer_runner`'s flat_row branch — NOT against a fixture shared with
+ * `test_flat_row_payload_shape_mirrors_score_run_exec`, so a rename on the
+ * Python side turns only the Python test red until the shared fixture lands
+ * (ALI-1556). The env allowlist is the TypeScript one.
+ */
+export interface VerifyParityIo {
+    json: boolean;
+    print: (line: string) => void;
+    printErr: (line: string) => void;
+    materializeRunnerVersion: (runnerVersionId: string) => Promise<{
+        runnerDir: string;
+        cleanup: () => void;
+    }>;
+    readRunnerManifest: (runnerDir: string) => {
+        command: string[];
+        supports_body_kind?: string[];
+    };
+    runnerSubprocessEnv: (inputPath: string, outputPath: string) => NodeJS.ProcessEnv;
+    timeoutMs: number;
+    maxOutputBytes: number;
+    fetcher?: (path: string, init?: RequestInit) => Promise<Response>;
+    /** Test seam for the pairing tests (H8); production always uses the embedded bridge. */
+    bridgeSource?: string;
+}
+interface ExtractedScore {
+    raw: number;
+    score: number;
+}
+/**
+ * Python's `float()` grammar, not JavaScript's `Number()`. Measured 2026-08-23:
+ * `float` REJECTS '0x10', '0b1', '0o7', '' and '1,5' (all of which `Number`
+ * happily converts or turns into 0), and ACCEPTS '  0.5  ', '1e3', '1_0', '.5',
+ * '5.', '+3', 'inf', 'Infinity' and 'nan'. Anything `_score_from_scorer` would
+ * refuse during the real optimization must be refused here too, or this gate
+ * certifies a scorer run-gepa cannot use. `inf`/`nan` parse and are then
+ * rejected by the finite check, exactly as Python does.
+ */
+export declare function pythonFloat(text: string): number | null;
+export declare function extractParityScore(value: unknown, label: string): ExtractedScore;
+/**
+ * The Orizu side spawns the runner ONCE PER ROW, so each row gets the full
+ * per-row budget. The original side scores the whole sample in ONE bridge
+ * process, so handing it the per-row budget gives an N-row split 1/N of the
+ * time the Orizu side had: a 40-row split with an LLM-judge metric is SIGTERMed
+ * before it can write output.json and every row's work is lost (round-2 review).
+ * Scale by the sample length, bounded so a huge split cannot ask for forever.
+ */
+export declare function bridgeBudget(perRowTimeoutMs: number, perRowMaxOutputBytes: number, sampleLength: number): {
+    timeoutMs: number;
+    maxOutputBytes: number;
+};
+/**
+ * Resolve the scorer input contract and candidate-output row field with the
+ * SAME precedence `run-gepa` uses (`orizu_gepa.runner.resolve_scorer_input_contract`):
+ * explicit flag > runner manifest (`scorer_input_contract`, `candidate_output_field`)
+ * > defaults (`gepa`, `model_output`). Resolution is by PRESENCE, not truthiness,
+ * so a blank manifest value fails validation instead of silently defaulting.
+ *
+ * Parity may only be claimed for the payload run-gepa will actually send, so a
+ * contract that resolves to anything but `flat_row` is refused here rather than
+ * assumed: proving parity on a shape the optimization never uses is the exact
+ * silent-agreement failure this command exists to prevent.
+ */
+export declare function resolveScorerInputContract(manifest: Record<string, unknown>, flagContract: string | null, flagField: string | null): {
+    contract: string;
+    field: string;
+};
+export declare function verifyScorerParityCommand(args: string[], io: VerifyParityIo): Promise<number>;
+export {};
+//# sourceMappingURL=scorers-verify-parity.d.ts.map

package/dist/scorers-verify-parity.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scorers-verify-parity.d.ts","sourceRoot":"","sources":["../src/scorers-verify-parity.ts"],"names":[],"mappings":"AAaA;;;;;;;;;;;;;;;GAeG;AACH,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,OAAO,CAAA;IACb,KAAK,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAA;IAC7B,QAAQ,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAA;IAChC,wBAAwB,EAAE,CAAC,eAAe,EAAE,MAAM,KAAK,OAAO,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,IAAI,CAAA;KAAE,CAAC,CAAA;IAC1G,kBAAkB,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK;QAAE,OAAO,EAAE,MAAM,EAAE,CAAC;QAAC,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAA;KAAE,CAAA;IAC/F,mBAAmB,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,KAAK,MAAM,CAAC,UAAU,CAAA;IACjF,SAAS,EAAE,MAAM,CAAA;IACjB,cAAc,EAAE,MAAM,CAAA;IACtB,OAAO,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAA;IACjE,wFAAwF;IACxF,YAAY,CAAC,EAAE,MAAM,CAAA;CACtB;AAKD,UAAU,cAAc;IAAG,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE;AAmCvD;;;;;;;;GAQG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAMvD;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,GAAG,cAAc,CAiBhF;AAsDD;;;;;;;GAOG;AACH,wBAAgB,YAAY,CAC1B,eAAe,EAAE,MAAM,EACvB,oBAAoB,EAAE,MAAM,EAC5B,YAAY,EAAE,MAAM,GACnB;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,cAAc,EAAE,MAAM,CAAA;CAAE,CAO/C;AAKD;;;;;;;;;;;GAWG;AACH,wBAAgB,0BAA0B,CACxC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACjC,YAAY,EAAE,MAAM,GAAG,IAAI,EAC3B,SAAS,EAAE,MAAM,GAAG,IAAI,GACvB;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CA0BrC;AA8BD,wBAAsB,yBAAyB,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC,CA0SnG"}