orizu 0.5.19 → 0.5.21

package/scripts/refresh-skilled-proposer-lock.mjs

@@ -0,0 +1,256 @@
+#!/usr/bin/env node
+
+import { createHash, randomUUID } from "node:crypto";
+import { execFileSync } from "node:child_process";
+import {
+  closeSync,
+  fsyncSync,
+  mkdtempSync,
+  openSync,
+  readFileSync,
+  readdirSync,
+  renameSync,
+  rmSync,
+  writeFileSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const scriptDirectory = dirname(fileURLToPath(import.meta.url));
+const cliRoot = resolve(scriptDirectory, "..");
+const repositoryRoot = resolve(cliRoot, "../..");
+const inputPath = join(cliRoot, "requirements", "skilled-proposer.in");
+const lockPath = join(cliRoot, "requirements", "skilled-proposer.lock");
+
+const TARGETS = [
+  { name: "macos-arm64", platforms: ["macosx_15_0_arm64"] },
+  {
+    name: "linux-x86_64",
+    platforms: ["manylinux_2_28_x86_64", "manylinux2014_x86_64"],
+  },
+];
+const PYTHON_MINORS = ["3.10", "3.11", "3.12", "3.13", "3.14"];
+
+function parseArguments(argv) {
+  const options = {
+    check: false,
+    download: false,
+    python: join(repositoryRoot, ".scratch-deps", "venv", "bin", "python"),
+    wheelRoot: join(repositoryRoot, ".scratch-deps", "wheels"),
+  };
+
+  for (let index = 0; index < argv.length; index += 1) {
+    const argument = argv[index];
+    if (argument === "--check") options.check = true;
+    else if (argument === "--download") options.download = true;
+    else if (argument === "--python") options.python = resolve(argv[++index]);
+    else if (argument === "--wheel-root") options.wheelRoot = resolve(argv[++index]);
+    else throw new Error(`Unknown argument: ${argument}`);
+  }
+  if (options.check && options.download) {
+    throw new Error("--check and --download are mutually exclusive");
+  }
+  return options;
+}
+
+function canonicalizeName(name) {
+  return name.toLowerCase().replace(/[._-]+/g, "-");
+}
+
+function parsePins() {
+  return readFileSync(inputPath, "utf8")
+    .split(/\r?\n/u)
+    .map((line) => line.trim())
+    .filter((line) => line && !line.startsWith("#"))
+    .map((line) => {
+      const match = /^([A-Za-z0-9._-]+)==([^ ;]+)(?:\s*;\s*(.+))?$/u.exec(line);
+      if (!match) throw new Error(`Invalid fully pinned requirement: ${line}`);
+      return {
+        name: canonicalizeName(match[1]),
+        version: match[2],
+        marker: match[3] ?? null,
+        source: line,
+      };
+    });
+}
+
+function pinApplies(pin, pythonMinor) {
+  if (!pin.marker) return true;
+  if (pin.marker === 'python_version == "3.10"') return pythonMinor === "3.10";
+  if (pin.marker === 'python_version >= "3.11" and python_version < "3.15"') {
+    return pythonMinor !== "3.10";
+  }
+  throw new Error(`Unsupported environment marker in ${pin.source}`);
+}
+
+function activePins(pins, pythonMinor) {
+  const active = pins.filter((pin) => pinApplies(pin, pythonMinor));
+  const names = new Set(active.map((pin) => pin.name));
+  const expectedCount = pythonMinor === "3.10" ? 60 : 58;
+  if (active.length !== expectedCount || names.size !== expectedCount) {
+    throw new Error(
+      `Expected ${expectedCount} unique active pins for Python ${pythonMinor}; found ${active.length}`,
+    );
+  }
+  return active;
+}
+
+function wheelIdentity(filename) {
+  const match = /^(.+?)-([^-]+)-[^-]+-[^-]+-[^-]+\.whl$/u.exec(filename);
+  if (!match) throw new Error(`Not a wheel filename: ${filename}`);
+  return { name: canonicalizeName(match[1]), version: match[2] };
+}
+
+function downloadWheelhouse(options, target, pythonMinor, pins) {
+  const wheelhouse = join(
+    options.wheelRoot,
+    `${target.name}-py${pythonMinor.replace(".", "")}`,
+  );
+  const temporary = mkdtempSync(join(tmpdir(), "orizu-ali-1505-pins-"));
+  const requirements = join(temporary, "requirements.txt");
+  writeFileSync(
+    requirements,
+    `${activePins(pins, pythonMinor)
+      .map(({ name, version }) => `${name}==${version}`)
+      .join("\n")}\n`,
+  );
+  const pipArguments = [
+    "-m",
+    "pip",
+    "download",
+    "--disable-pip-version-check",
+    "--no-deps",
+    "--only-binary=:all:",
+    "--implementation",
+    "cp",
+    "--python-version",
+    pythonMinor.replace(".", ""),
+    "--abi",
+    `cp${pythonMinor.replace(".", "")}`,
+  ];
+  for (const platform of target.platforms) {
+    pipArguments.push("--platform", platform);
+  }
+  pipArguments.push("--dest", wheelhouse, "--requirement", requirements);
+  try {
+    execFileSync(options.python, pipArguments, { stdio: "inherit", timeout: 10 * 60 * 1000 });
+  } finally {
+    rmSync(temporary, { recursive: true, force: true });
+  }
+}
+
+function digestWheel(path) {
+  return createHash("sha256").update(readFileSync(path)).digest("hex");
+}
+
+function collectWheelHashes(options, pins) {
+  const hashes = new Map(pins.map((pin) => [pin.source, new Set()]));
+
+  for (const target of TARGETS) {
+    for (const pythonMinor of PYTHON_MINORS) {
+      const active = activePins(pins, pythonMinor);
+      const activeByIdentity = new Map(
+        active.map((pin) => [`${pin.name}==${pin.version}`, pin]),
+      );
+      const wheelhouse = join(
+        options.wheelRoot,
+        `${target.name}-py${pythonMinor.replace(".", "")}`,
+      );
+      const files = readdirSync(wheelhouse).filter((file) => file.endsWith(".whl"));
+      const found = new Set();
+      for (const file of files) {
+        const identity = wheelIdentity(file);
+        const key = `${identity.name}==${identity.version}`;
+        const pin = activeByIdentity.get(key);
+        if (!pin) throw new Error(`Unexpected wheel in ${wheelhouse}: ${file}`);
+        if (found.has(pin.name)) {
+          throw new Error(`Multiple wheels for ${pin.name} in ${wheelhouse}`);
+        }
+        found.add(pin.name);
+        hashes.get(pin.source).add(digestWheel(join(wheelhouse, file)));
+      }
+      const missing = active.filter((pin) => !found.has(pin.name));
+      if (missing.length > 0) {
+        throw new Error(
+          `Missing wheels in ${wheelhouse}: ${missing.map((pin) => pin.source).join(", ")}`,
+        );
+      }
+    }
+  }
+  return hashes;
+}
+
+function renderLock(pins, hashes) {
+  const header = [
+    "# Generated by packages/cli/scripts/refresh-skilled-proposer-lock.mjs.",
+    "# Do not edit by hand; see packages/cli/requirements/README.md.",
+    "#",
+    "# Wheel-only coverage: CPython 3.10-3.14 on macOS 15+ arm64 and",
+    "# glibc-based Linux x86_64 compatible with the locked manylinux wheels.",
+    "# Other OS/architecture/libc combinations are intentionally not covered.",
+    "",
+  ];
+  const continuation = ` ${String.fromCharCode(92, 10)}`;
+  const requirements = [...pins]
+    .sort((left, right) => left.source.localeCompare(right.source))
+    .map((pin) => {
+      const pinHashes = [...hashes.get(pin.source)].sort();
+      if (pinHashes.length === 0) throw new Error(`No wheel hash for ${pin.source}`);
+      const hashLines = pinHashes
+        .map((hash) => `    --hash=sha256:${hash}`)
+        .join(continuation);
+      return pin.source + continuation + hashLines;
+    });
+  return `${header.join("\n")}${requirements.join("\n")}\n`;
+}
+
+function main() {
+  const options = parseArguments(process.argv.slice(2));
+  const pins = parsePins();
+  if (options.download) {
+    for (const target of TARGETS) {
+      for (const pythonMinor of PYTHON_MINORS) {
+        downloadWheelhouse(options, target, pythonMinor, pins);
+      }
+    }
+  }
+  const lock = renderLock(pins, collectWheelHashes(options, pins));
+  if (options.check) {
+    if (readFileSync(lockPath, "utf8") !== lock) {
+      throw new Error(`${lockPath} is stale; regenerate it from verified wheelhouses`);
+    }
+    process.stdout.write(`Verified ${lockPath}\n`);
+    return;
+  }
+  const temporaryLock = join(
+    dirname(lockPath),
+    `.skilled-proposer.lock-${process.pid}-${randomUUID()}.tmp`,
+  );
+  try {
+    const temporaryDescriptor = openSync(temporaryLock, "wx");
+    try {
+      writeFileSync(temporaryDescriptor, lock);
+      fsyncSync(temporaryDescriptor);
+    } finally {
+      closeSync(temporaryDescriptor);
+    }
+    renameSync(temporaryLock, lockPath);
+    const requirementsDescriptor = openSync(dirname(lockPath), "r");
+    try {
+      fsyncSync(requirementsDescriptor);
+    } finally {
+      closeSync(requirementsDescriptor);
+    }
+  } finally {
+    rmSync(temporaryLock, { force: true });
+  }
+  process.stdout.write(`Wrote ${lockPath}\n`);
+}
+
+try {
+  main();
+} catch (error) {
+  process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
+  process.exitCode = 1;
+}

package/scripts/vendor-gepa-python.mjs

@@ -1,4 +1,6 @@
-import { cpSync, existsSync, mkdirSync, rmSync } from 'node:fs'
+import { cpSync, existsSync, mkdirSync, readdirSync, rmSync, statSync, writeFileSync, readFileSync } from 'node:fs'
+import { createHash } from 'node:crypto'
+import { spawnSync } from 'node:child_process'
 import { dirname, resolve } from 'node:path'
 import { fileURLToPath } from 'node:url'
 
@@ -7,32 +9,194 @@ const cliRoot = resolve(scriptDir, '..')
 const repoRoot = resolve(cliRoot, '..', '..')
 const sourceRoot = resolve(repoRoot, 'packages', 'orizu-gepa-python')
 const vendorRoot = resolve(cliRoot, 'vendor', 'orizu-gepa-python')
+const connectorSourceRoot = resolve(repoRoot, 'packages', 'orizu-gepa')
+const connectorVendorRoot = resolve(cliRoot, 'vendor', 'orizu-gepa')
+const officialArchive = resolve(cliRoot, 'gepa-python-source.zip')
+const officialVendorRoot = resolve(cliRoot, 'vendor', 'gepa-python')
 const shouldClean = process.argv.includes('--clean')
+const shouldRefresh = process.argv.includes('--refresh')
+const archiveArgumentIndex = process.argv.indexOf('--archive')
+const suppliedArchive = archiveArgumentIndex === -1
+  ? null
+  : process.argv[archiveArgumentIndex + 1]
 const sourceSrc = resolve(sourceRoot, 'src')
 const sourcePyproject = resolve(sourceRoot, 'pyproject.toml')
 const sourceManifest = resolve(sourceRoot, 'manifest.json')
 const vendoredSrc = resolve(vendorRoot, 'src')
 const vendoredPyproject = resolve(vendorRoot, 'pyproject.toml')
 const vendoredManifest = resolve(vendorRoot, 'manifest.json')
+const officialVendoredManifest = resolve(officialVendorRoot, 'manifest.json')
+const officialVendoredGepa = resolve(officialVendorRoot, 'src', 'gepa', '__init__.py')
+const GEPA_VERSION = '0.1.4'
+const GEPA_RELEASE_COMMIT = '8b0ce6cd99a234f6b74daf37558a2ac0ce18f975'
+const GEPA_WHEEL_FILENAME = 'gepa-0.1.4-py3-none-any.whl'
+const GEPA_WHEEL_SHA256 = '12b971039599625c156d2231f6d72a29c31a22e9c237689459b5f1a3c353f532'
 const hasSourcePackage = existsSync(sourceSrc)
   && existsSync(sourcePyproject)
   && existsSync(sourceManifest)
+const hasConnectorSourcePackage = existsSync(resolve(connectorSourceRoot, 'src'))
+  && existsSync(resolve(connectorSourceRoot, 'pyproject.toml'))
 const hasVendoredPackage = existsSync(vendoredSrc)
   && existsSync(vendoredPyproject)
   && existsSync(vendoredManifest)
+const hasVendoredConnectorPackage = existsSync(resolve(connectorVendorRoot, 'src', 'orizu_gepa_connector', '__main__.py'))
+  && existsSync(resolve(connectorVendorRoot, 'pyproject.toml'))
+const hasOfficialVendoredPackage = existsSync(officialVendoredManifest)
+  && existsSync(officialVendoredGepa)
+
+function sha256File(path) {
+  return createHash('sha256').update(readFileSync(path)).digest('hex')
+}
+
+function assertPinnedGepaArchive(path) {
+  if (!existsSync(path)) {
+    throw new Error(`Unable to vendor official GEPA: source archive not found at ${path}`)
+  }
+
+  const actualSha256 = sha256File(path)
+  if (actualSha256 !== GEPA_WHEEL_SHA256) {
+    throw new Error(
+      `Official GEPA source archive sha256 mismatch: expected ${GEPA_WHEEL_SHA256}, got ${actualSha256}`
+    )
+  }
+}
+
+function collectFileHashes(root, prefix = '') {
+  const files = {}
+  for (const entry of readdirSync(root).sort()) {
+    const path = resolve(root, entry)
+    const relativePath = prefix ? `${prefix}/${entry}` : entry
+    if (statSync(path).isDirectory()) {
+      Object.assign(files, collectFileHashes(path, relativePath))
+    } else {
+      files[relativePath] = sha256File(path)
+    }
+  }
+  return files
+}
+
+function assertOfficialVendoredManifest() {
+  let manifest
+  try {
+    manifest = JSON.parse(readFileSync(officialVendoredManifest, 'utf8'))
+  } catch (error) {
+    throw new Error(`Official GEPA vendored manifest is unreadable: ${error.message}`)
+  }
+
+  if (!manifest.files || typeof manifest.files !== 'object' || Array.isArray(manifest.files)) {
+    throw new Error('Official GEPA vendored manifest is missing file hashes')
+  }
+
+  const actualFiles = collectFileHashes(officialVendorRoot)
+  delete actualFiles['manifest.json']
+  const expectedEntries = Object.entries(manifest.files).sort(([left], [right]) => left.localeCompare(right))
+  const actualEntries = Object.entries(actualFiles).sort(([left], [right]) => left.localeCompare(right))
+  if (expectedEntries.length !== actualEntries.length) {
+    throw new Error('Official GEPA vendored manifest file count does not match extracted files')
+  }
+
+  for (let index = 0; index < expectedEntries.length; index += 1) {
+    const [expectedPath, expectedSha256] = expectedEntries[index]
+    const [actualPath, actualSha256] = actualEntries[index]
+    if (expectedPath !== actualPath || expectedSha256 !== actualSha256) {
+      throw new Error(`Official GEPA vendored manifest mismatch at ${expectedPath}`)
+    }
+  }
+}
+
+function extractPinnedGepa(archivePath) {
+  const extractor = String.raw`
+from pathlib import Path, PurePosixPath
+import shutil
+import sys
+import zipfile
+
+archive_path = Path(sys.argv[1])
+destination_root = Path(sys.argv[2])
+license_member = 'gepa-0.1.4.dist-info/licenses/LICENSE'
+
+with zipfile.ZipFile(archive_path) as archive:
+    for member in archive.infolist():
+        if member.is_dir():
+            continue
+        if member.filename.startswith('gepa/'):
+            relative_path = PurePosixPath('src') / PurePosixPath(member.filename)
+        elif member.filename == license_member:
+            relative_path = PurePosixPath('LICENSE')
+        else:
+            continue
+        if relative_path.is_absolute() or '..' in relative_path.parts:
+            raise ValueError(f'unsafe GEPA wheel member: {member.filename}')
+        destination = destination_root.joinpath(*relative_path.parts)
+        destination.parent.mkdir(parents=True, exist_ok=True)
+        with archive.open(member) as source, destination.open('wb') as output:
+            shutil.copyfileobj(source, output)
+`
+  const result = spawnSync('python3', ['-c', extractor, archivePath, officialVendorRoot], {
+    encoding: 'utf8',
+  })
+  if (result.status !== 0) {
+    const details = result.error?.message || result.stderr || result.stdout || 'no subprocess output'
+    throw new Error(`Unable to extract verified official GEPA wheel; python3 is required: ${details}`)
+  }
+}
+
+function vendorOfficialGepa(archivePath) {
+  assertPinnedGepaArchive(archivePath)
+  rmSync(officialVendorRoot, { recursive: true, force: true })
+  mkdirSync(officialVendorRoot, { recursive: true })
+  extractPinnedGepa(archivePath)
+
+  const files = collectFileHashes(officialVendorRoot)
+  writeFileSync(officialVendoredManifest, `${JSON.stringify({
+    schemaVersion: 'gepa-vendor.v1',
+    upstream: {
+      package: 'gepa',
+      version: GEPA_VERSION,
+      repositoryCommit: GEPA_RELEASE_COMMIT,
+      wheel: {
+        filename: GEPA_WHEEL_FILENAME,
+        sha256: GEPA_WHEEL_SHA256,
+      },
+    },
+    files,
+  }, null, 2)}\n`)
+}
+
+if (archiveArgumentIndex !== -1 && (!suppliedArchive || suppliedArchive.startsWith('--'))) {
+  throw new Error('Expected a source archive path after --archive')
+}
+
+if (shouldRefresh) {
+  if (!suppliedArchive) {
+    throw new Error('Refreshing official GEPA requires --archive <verified-wheel-path>')
+  }
+  assertPinnedGepaArchive(suppliedArchive)
+  cpSync(suppliedArchive, officialArchive)
+}
 
 if (shouldClean) {
   if (hasSourcePackage) {
     rmSync(vendorRoot, { recursive: true, force: true })
   }
+  if (hasConnectorSourcePackage) {
+    rmSync(connectorVendorRoot, { recursive: true, force: true })
+  }
+  if (hasSourcePackage || hasConnectorSourcePackage) {
+    rmSync(officialVendorRoot, { recursive: true, force: true })
+  }
   process.exit(0)
 }
 
-if (!hasSourcePackage) {
-  if (hasVendoredPackage) {
+if (!hasSourcePackage || !hasConnectorSourcePackage) {
+  if (hasVendoredPackage && hasVendoredConnectorPackage && hasOfficialVendoredPackage) {
+    assertOfficialVendoredManifest()
     process.exit(0)
   }
-  throw new Error(`Unable to vendor orizu-gepa-python: source package not found at ${sourceRoot}`)
+  if (hasVendoredPackage || hasVendoredConnectorPackage || hasOfficialVendoredPackage) {
+    throw new Error('Unable to vendor GEPA packages: vendored legacy, connector, or official package is incomplete')
+  }
+  throw new Error(`Unable to vendor GEPA packages: source package not found at ${sourceRoot} or ${connectorSourceRoot}`)
 }
 
 rmSync(vendorRoot, { recursive: true, force: true })
@@ -52,3 +216,22 @@ cpSync(sourceSrc, resolve(vendorRoot, 'src'), {
       && !normalizedPath.endsWith('/.DS_Store')
   },
 })
+
+rmSync(connectorVendorRoot, { recursive: true, force: true })
+mkdirSync(connectorVendorRoot, { recursive: true })
+cpSync(resolve(connectorSourceRoot, 'pyproject.toml'), resolve(connectorVendorRoot, 'pyproject.toml'))
+cpSync(resolve(connectorSourceRoot, 'src'), resolve(connectorVendorRoot, 'src'), {
+  recursive: true,
+  filter: sourcePath => {
+    const normalizedPath = sourcePath.replace(/\\/g, '/')
+    return !normalizedPath.endsWith('/__pycache__')
+      && !normalizedPath.includes('/__pycache__/')
+      && !normalizedPath.endsWith('.pyc')
+      && !normalizedPath.endsWith('.pyo')
+      && !normalizedPath.endsWith('/.pytest_cache')
+      && !normalizedPath.includes('/.pytest_cache/')
+      && !normalizedPath.endsWith('/.DS_Store')
+  },
+})
+
+vendorOfficialGepa(suppliedArchive ?? officialArchive)

package/src/diff-comments-cli.ts

@@ -36,6 +36,7 @@ interface DiffCommentPayloadComment {
   body: string
   author: DiffCommentAuthor
   createdAt: string
+  componentKey?: string | null
   anchor: { side: 'old' | 'new'; line: number }
   context: {
     lineText: string
@@ -50,7 +51,11 @@ interface DiffCommentPayloadPairContext {
   bodies: { from: string; to: string } | null
   degraded: { reason: DiffCommentContextUnavailableReason } | null
   lengthStats: CliLengthStatsPair | null
-  lengthDelta: CliLengthDelta | null
+  lengthDelta: (
+    Partial<Record<keyof Omit<CliLengthDelta, 'splitUnavailableReason'>, unknown>> & {
+      splitUnavailableReason?: unknown
+    }
+  ) | null
   lengthDeltaUnavailableReason?: CliLengthMeasurementUnavailableReason
   comments: DiffCommentPayloadComment[]
 }
@@ -76,6 +81,12 @@ interface DiffCommentsPayload {
   pairs: DiffCommentPayloadPair[]
 }
 
+interface RenderableLengthMetricDelta {
+  removed: number | null
+  added: number | null
+  net: number | null
+}
+
 export interface DiffCommentsCliContext {
   getArg: (name: string) => string | null
   rejectDashPrefixedOptionValue: (name: string, value: string | null) => void
@@ -171,6 +182,22 @@ function formatDeltaValue(
   return metric === 'tokens' ? `~${formatted}` : formatted
 }
 
+function isLengthMetricDelta(
+  value: unknown
+): value is RenderableLengthMetricDelta {
+  if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+    return false
+  }
+  const delta = value as Record<string, unknown>
+  const isNullableFiniteNumber = (candidate: unknown) =>
+    candidate === null || (
+      typeof candidate === 'number' && Number.isFinite(candidate)
+    )
+  return isNullableFiniteNumber(delta.removed) &&
+    isNullableFiniteNumber(delta.added) &&
+    isNullableFiniteNumber(delta.net)
+}
+
 function printLengthMeasurements(
   ctx: DiffCommentsCliContext,
   pair: DiffCommentPayloadPair
@@ -210,11 +237,15 @@ function printLengthMeasurements(
     return
   }
 
-  if (pair.lengthDelta.splitUnavailableReason) {
+  const splitUnavailableReason = pair.lengthDelta.splitUnavailableReason
+  if (
+    typeof splitUnavailableReason === 'string' &&
+    splitUnavailableReason.length > 0
+  ) {
     ctx.printLine(
       `  Split unavailable: ${sanitizeHumanInlineText(
         ctx.sanitizeTerminalText,
-        pair.lengthDelta.splitUnavailableReason
+        splitUnavailableReason
       )}`
     )
   }
@@ -225,6 +256,10 @@ function printLengthMeasurements(
     ['words', 'Words'],
   ] as const) {
     const delta = pair.lengthDelta[metric]
+    if (!isLengthMetricDelta(delta)) {
+      ctx.printLine(`  ${label}: unavailable`)
+      continue
+    }
     ctx.printLine(
       `  ${label}: removed ${formatDeltaValue(delta.removed, metric)} · ` +
       `added ${formatDeltaValue(delta.added, metric)} · ` +
@@ -330,6 +365,9 @@ function printDiffComments(ctx: DiffCommentsCliContext, payload: DiffCommentsPay
         `${sanitizeHumanInlineText(ctx.sanitizeTerminalText, String(comment.anchor.line + 1))}`
       )
       printHumanTextBlock(ctx, comment.body)
+      if (typeof comment.componentKey === 'string' && comment.componentKey) {
+        ctx.printLine(`    Component: ${sanitizeHumanInlineText(ctx.sanitizeTerminalText, comment.componentKey)}`)
+      }
 
       if (comment.contextUnavailableReason) {
         ctx.printLine(