orizu 0.5.21 → 0.5.23

package/src/skilled-proposer-launch.ts

@@ -1,34 +1,97 @@
-import { spawnSync } from 'child_process'
+import { spawn, spawnSync } from 'child_process'
+import { existsSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from 'fs'
+import { tmpdir } from 'os'
 import { join } from 'path'
 import { fileURLToPath } from 'url'
 
 import type { GepaEngine } from './gepa-engine-dispatch.js'
-import { bundledOfficialGepaPythonPath } from './gepa-python-paths.js'
+import { bundledOfficialGepaPythonPath, getGepaPythonPathEntries } from './gepa-python-paths.js'
 import { sanitizeTerminalText } from './json-response.js'
+import { SKILLED_PROPOSER_BAKE_REPORT_FIELDS } from './skilled-proposer-bake-report.js'
 
 interface SkilledProposerVenvReport {
   venv: string
+  publishKey: string
   sslCertFile: string | null
   pythonPathEntries: string[]
-  warnings?: Array<{ code: string, message: string }>
+  warnings: Array<{ code: string, message: string }>
+  waitedForPublishLock: boolean
   error?: string
   message?: string
+  publishedVenv?: boolean | null
+  executedPipArgv?: string[] | null
 }
-
 export interface SkilledProposerLaunch {
   python: string
   environment: NodeJS.ProcessEnv
+  report?: SkilledProposerVenvReport
+}
+
+function materializeConfigPayload(engine: GepaEngine, environment: NodeJS.ProcessEnv): (() => void) | undefined {
+  const payload = environment.ORIZU_SKILLED_PROPOSER_CONFIG
+  if (engine !== 'official' || environment.ORIZU_CANDIDATE_PROPOSER !== 'skilled-proposer' || payload === undefined) return undefined
+  const root = mkdtempSync(join(tmpdir(), 'orizu-skilled-proposer-payload-'))
+  try {
+    const path = join(root, 'config.json')
+    writeFileSync(path, payload, { encoding: 'utf8', mode: 0o600 })
+    environment.ORIZU_SKILLED_PROPOSER_CONFIG = `@${path}`
+  } catch (error) {
+    rmSync(root, { recursive: true, force: true })
+    throw error
+  }
+  return () => rmSync(root, { recursive: true, force: true })
+}
+
+export async function spawnSkilledProposerChild(
+  command: string, args: string[], engine: GepaEngine, environment: NodeJS.ProcessEnv,
+) {
+  const childEnvironment = { ...environment }
+  const shouldMaterialize = engine === 'official'
+    && childEnvironment.ORIZU_CANDIDATE_PROPOSER === 'skilled-proposer'
+    && childEnvironment.ORIZU_SKILLED_PROPOSER_CONFIG !== undefined
+  if (!shouldMaterialize) return spawnSync(command, args, { stdio: 'inherit', env: childEnvironment })
+
+  let child: ReturnType<typeof spawn> | undefined
+  let cleanup = () => {}
+  const signals = ['SIGINT', 'SIGTERM', 'SIGHUP', 'SIGQUIT'] as const
+  const handlers = signals.map(signal => ({
+    signal,
+    handle: () => {
+      try { cleanup() }
+      finally {
+        try { child?.kill(signal) }
+        finally {
+          for (const entry of handlers) process.off(entry.signal, entry.handle)
+          process.kill(process.pid, signal)
+        }
+      }
+    },
+  }))
+  for (const entry of handlers) process.once(entry.signal, entry.handle)
+  try {
+    cleanup = materializeConfigPayload(engine, childEnvironment) ?? cleanup
+    await new Promise<void>(resolve => setImmediate(resolve))
+    return await new Promise<{ status: number | null, error?: Error }>(resolve => {
+      let error: Error | undefined
+      child = spawn(command, args, { stdio: 'inherit', env: childEnvironment })
+      child.once('error', cause => { error = cause })
+      child.once('close', status => resolve(error ? { status, error } : { status }))
+    })
+  } finally {
+    for (const entry of handlers) process.off(entry.signal, entry.handle)
+    cleanup()
+  }
 }
 
 function ensureSelectedSkilledProposerVenv(
   python: string,
   vendoredGepaPath: string,
+  cacheRoot = existsSync('/opt/orizu/prebaked.json')
+    ? '/opt/orizu/cache/skilled-proposer'
+    : join(process.cwd(), '.orizu', 'cache', 'skilled-proposer'),
 ): SkilledProposerVenvReport {
   const manager = fileURLToPath(new URL('../scripts/ensure-skilled-proposer-venv.mjs', import.meta.url))
   const lock = fileURLToPath(new URL('../requirements/skilled-proposer.lock', import.meta.url))
-  // This is a regenerable workspace cache. The manager's publish key further
-  // separates interpreters, platforms, and lock revisions inside it.
-  const cacheRoot = join(process.cwd(), '.orizu', 'cache', 'skilled-proposer')
   const result = spawnSync(
     process.execPath,
     [manager, '--python', python, '--cache-root', cacheRoot, '--lock', lock,
@@ -58,6 +121,31 @@ function ensureSelectedSkilledProposerVenv(
   return report
 }
 
+interface BakeCommandIo { print: (value: string) => void; printErr?: (value: string) => void }
+const packageRoot = join(fileURLToPath(new URL('.', import.meta.url)), '..')
+const managerPath = join(packageRoot, 'scripts/ensure-skilled-proposer-venv.mjs')
+const lockPath = join(packageRoot, 'requirements/skilled-proposer.lock')
+const vendoredGepaPath = join(packageRoot, 'vendor/gepa-python/src')
+const packagedPythonPathEntries = ['vendor/orizu-gepa/src', 'vendor/orizu-gepa-python/src', 'vendor/gepa-python/src'].map(path => join(packageRoot, path))
+const bakedCacheRoot = '/opt/orizu/cache/skilled-proposer'
+
+export function skilledProposerBakeCommand(io: BakeCommandIo, verify = false): number {
+  try {
+    const python = process.env.PYTHON || 'python3'
+    if (!verify) { io.print(JSON.stringify(ensureSelectedSkilledProposerVenv(python, vendoredGepaPath, bakedCacheRoot))); return 0 }
+    if (!readdirSync(join(bakedCacheRoot, 'venvs'), { withFileTypes: true }).some(entry => entry.isDirectory() && existsSync(join(bakedCacheRoot, 'venvs', entry.name, '.orizu-skilled-proposer-venv.json')))) throw new Error('ALI_1588_VENV_MARKER_MISSING')
+    const environment = { ...process.env, ORIZU_CANDIDATE_PROPOSER: 'skilled-proposer' }
+    const launch = prepareSkilledProposerLaunch(python, 'official', environment)
+    const report = launch.report!
+    if (report.publishedVenv !== false || report.executedPipArgv !== null) throw new Error('ALI_1588_DEPENDENCY_SETUP_NOT_REUSED')
+    const pythonPathEntries = getGepaPythonPathEntries(undefined)
+    if (JSON.stringify(pythonPathEntries) !== JSON.stringify(packagedPythonPathEntries)) throw new Error('ALI_1588_PYTHON_ROOT_MISSING')
+    const completeReport = { ...report, pythonPathEntries, marker: 'ALI_1588_DEPENDENCY_SETUP_REUSED', python: launch.python, packageRoot, manager: managerPath, lock: lockPath, vendoredGepaPath, launcherVenv: launch.environment.ORIZU_SKILLED_PROPOSER_VENV ?? null, launcherSslCertFile: launch.environment.SSL_CERT_FILE ?? null }
+    io.print(JSON.stringify(Object.fromEntries(SKILLED_PROPOSER_BAKE_REPORT_FIELDS.map(key => [key, completeReport[key]]))))
+    return 0
+  } catch (error) { io.printErr?.(`${error instanceof Error ? error.message : String(error)}\n`); return 1 }
+}
+
 export function prepareSkilledProposerLaunch(
   python: string,
   engine: GepaEngine,
@@ -85,5 +173,6 @@ export function prepareSkilledProposerLaunch(
       ? join(report.venv, 'Scripts', 'python.exe')
       : join(report.venv, 'bin', 'python'),
     environment,
+    report,
   }
 }

package/src/skilled-proposer-venv-manager.mjs

@@ -100,15 +100,14 @@ finally:
 const MANAGER_SCHEMA_VERSION = 1
 const MANIFEST_NAME = '.orizu-skilled-proposer-venv.json'
 const LOCK_WAIT_MILLISECONDS = 25
-// This exceeds the longest repair path while holding the lock: existing-cache
-// validation (2m) + venv (2m) + install (10m) + staged validation (2m), plus
-// cleanup/publication/release margin.
-const LOCK_TIMEOUT_MILLISECONDS = 20 * 60 * 1000
-const PYTHON_PROBE_TIMEOUT_MILLISECONDS = 15_000
+export const LOCK_TIMEOUT_MILLISECONDS = 20 * 60 * 1000
+export const PYTHON_PROBE_TIMEOUT_MILLISECONDS = 15_000
+export const LOCK_RELEASE_TIMEOUT_MILLISECONDS = 15_000
 const VENV_CREATE_TIMEOUT_MILLISECONDS = 2 * 60 * 1000
-const VALIDATION_TIMEOUT_MILLISECONDS = 60_000
+export const VALIDATION_TIMEOUT_MILLISECONDS = 60_000
 const PIP_INSTALL_TIMEOUT_MILLISECONDS = 10 * 60 * 1000
 const TEST_BARRIER_TIMEOUT_MILLISECONDS = 60_000
+export const SKILLED_PROPOSER_MANAGER_MAXIMUM_MILLISECONDS = (2 * PYTHON_PROBE_TIMEOUT_MILLISECONDS) + LOCK_RELEASE_TIMEOUT_MILLISECONDS + LOCK_TIMEOUT_MILLISECONDS + VENV_CREATE_TIMEOUT_MILLISECONDS + PIP_INSTALL_TIMEOUT_MILLISECONDS + (6 * VALIDATION_TIMEOUT_MILLISECONDS) + (2 * 60 * 1000)
 const ABANDONED_STAGE_MAX_AGE_MILLISECONDS = 24 * 60 * 60 * 1000
 const SSL_CERT_FILE_UNRESOLVED_WARNING = {
   code: 'ALI_1505_SSL_CERT_FILE_UNRESOLVED',
@@ -203,11 +202,6 @@ function versionAtLeast(actual, required) {
 }
 
 function assertSupportedWheelTarget(identity) {
-  // `platform.libc_ver()` is not merely cache identity on Linux: it is also a
-  // conservative admission check for the locked glibc wheel matrix. A bad
-  // guess can refuse a compatible host (named, before mutation), but cannot
-  // admit an sdist or make pip install an incompatible wheel because the
-  // customer install remains hash-locked and binary-only.
   const machine = String(identity.machine).toLowerCase()
   const isSupportedMac = identity.system === 'Darwin'
     && machine === 'arm64'
@@ -272,7 +266,7 @@ export function characterizePython(python, probeEnv = isolatedPythonProbeEnv())
   }
 }
 
-function readAndValidateHashLock(lockPath) {
+export function readAndValidateHashLock(lockPath) {
   let contents
   try {
     contents = readFileSync(lockPath, 'utf8')
@@ -305,7 +299,8 @@ function readAndValidateHashLock(lockPath) {
     )
   }
   for (const requirement of requirements) {
-    const isExactPin = /^[A-Za-z0-9_.-]+(?:\[[^\]]+\])?==[^\s;]+/.test(requirement)
+    const requirementSpecifier = requirement.split(/\s+--hash=/, 1)[0].trim()
+    const isExactPin = /^[A-Za-z0-9_.-]+(?:\[[^\]]+\])?==[^\s,;*]+(?:\s*;\s*.+)?$/.test(requirementSpecifier)
     const hashes = [...requirement.matchAll(/(?:^|\s)--hash=([^\s]+)/g)].map(match => match[1])
     const hashTokens = [...requirement.matchAll(/(?:^|\s)(--hash(?:=[^\s]*)?)(?=\s|$)/g)]
       .map(match => match[1])
@@ -343,7 +338,7 @@ function resolveSslCertFile(pythonIdentity, probeEnv) {
   return null
 }
 
-function buildPublishKey(pythonIdentity, lockContents) {
+export function buildPublishKey(pythonIdentity, lockContents) {
   const lockDigest = createHash('sha256').update(lockContents).digest('hex')
   const identity = {
     managerSchemaVersion: MANAGER_SCHEMA_VERSION,
@@ -355,7 +350,6 @@ function buildPublishKey(pythonIdentity, lockContents) {
     soabi: pythonIdentity.soabi,
     system: pythonIdentity.system,
     machine: pythonIdentity.machine,
-    osVersion: pythonIdentity.osVersion,
     libc: pythonIdentity.libc,
     lockDigest,
   }
@@ -523,21 +517,35 @@ function acquirePublishLock(python, lockFile, env) {
   })
 }
 
-async function releasePublishLock(lock) {
+export async function releasePublishLock(lock) {
   if (!lock?.child) return
+  const deadline = Date.now() + LOCK_RELEASE_TIMEOUT_MILLISECONDS
+  const remaining = () => Math.max(0, deadline - Date.now())
+  const waitForExit = timeout => lock.child.exitCode !== null || lock.child.signalCode !== null ? Promise.resolve(true) : new Promise(resolve => {
+    const onExit = () => { clearTimeout(timer); resolve(true) }
+    const timer = setTimeout(() => { lock.child.off('exit', onExit); resolve(false) }, timeout)
+    lock.child.once('exit', onExit)
+  })
+  const guardianAlive = () => { try { process.kill(lock.guardianPid, 0); return true } catch (error) { if (error?.code === 'ESRCH') return false; throw error } }
+  const waitForGuardianExit = async () => { while (guardianAlive() && remaining() > 0) await wait(Math.min(25, remaining())); return !guardianAlive() }
+  const signalGuardian = signal => {
+    if (lock.guardianPid === lock.child.pid) return
+    try { process.kill(lock.guardianPid, signal) } catch (error) { if (error?.code !== 'ESRCH') throw error }
+  }
+  const phase = LOCK_RELEASE_TIMEOUT_MILLISECONDS / 3
+  const finishGuardian = async () => { if (guardianAlive()) signalGuardian('SIGKILL'); return await waitForGuardianExit() }
   if (lock.child.exitCode !== null || lock.child.signalCode !== null) {
-    try {
-      process.kill(lock.guardianPid, 'SIGTERM')
-    } catch (error) {
-      if (error?.code !== 'ESRCH') throw error
-    }
+    if (!await finishGuardian()) throw new SkilledProposerVenvError('ALI_1505_PUBLISH_LOCK_RELEASE_TIMEOUT', 'publish-lock guardian did not exit after SIGKILL')
     return
   }
-  const exited = new Promise(resolve => lock.child.once('exit', resolve))
   lock.child.stdin.end('release\n')
-  const timeout = setTimeout(() => lock.child.kill(), PYTHON_PROBE_TIMEOUT_MILLISECONDS)
-  await exited
-  clearTimeout(timeout)
+  signalGuardian('SIGTERM')
+  if (await waitForExit(Math.min(phase, remaining()))) { if (await finishGuardian()) return }
+  lock.child.kill('SIGTERM')
+  if (await waitForExit(Math.min(phase, remaining()))) { if (await finishGuardian()) return }
+  signalGuardian('SIGKILL'); lock.child.kill('SIGKILL')
+  const [holderExited, guardianExited] = await Promise.all([waitForExit(remaining()), waitForGuardianExit()])
+  if (!holderExited || !guardianExited) throw new SkilledProposerVenvError('ALI_1505_PUBLISH_LOCK_RELEASE_TIMEOUT', 'publish-lock holder or guardian did not exit after SIGKILL')
 }
 
 async function killPublishLockLeaderAtTestHook(lock) {
@@ -603,7 +611,7 @@ function isolatedChildEnv(sslCertFile) {
   return env
 }
 
-export async function ensureSkilledProposerVenv(options) {
+export async function ensureSkilledProposerVenv(options, dependencies = {}) {
   // This must stay first: unsupported interpreters may not create cache state or
   // invoke a mutating `python -m` command.
   const probeEnv = isolatedPythonProbeEnv()
@@ -643,6 +651,7 @@ export async function ensureSkilledProposerVenv(options) {
   const lockedReport = { ...baseReport, waitedForPublishLock: lock.waitedForPublishLock }
 
   const stage = join(options.cacheRoot, 'venvs', `.${publishKey}-${process.pid}-${randomUUID()}.tmp`)
+  let primaryError, hasPrimaryError = false
   try {
     // Holding the per-key publish lock proves that no live builder for this key
     // can own one of these sibling staging directories.
@@ -726,9 +735,17 @@ export async function ensureSkilledProposerVenv(options) {
       )
     }
     return { ...installingReport, publishedVenv: true }
+  } catch (error) {
+    primaryError = error
+    hasPrimaryError = true
+    throw error
   } finally {
     rmSync(stage, { recursive: true, force: true })
-    await releasePublishLock(lock)
+    try { await (dependencies.releasePublishLock || releasePublishLock)(lock) }
+    catch (cleanupError) {
+      if (hasPrimaryError) throw new AggregateError([primaryError, cleanupError], 'ALI_1505_PRIMARY_AND_PUBLISH_LOCK_RELEASE_FAILED')
+      throw cleanupError
+    }
   }
 }
 

package/vendor/orizu-gepa/src/orizu_gepa_connector/engine.py

@@ -219,6 +219,32 @@ def run_official_gepa(
     owned_stop_callbacks = list(stop_callbacks)
     if proposal_budget is not None:
         owned_stop_callbacks.append(ProposalBudgetStopper(proposal_budget))
+    proposer_failures: list[Exception] = []
+    gepa_candidate_proposer = custom_candidate_proposer
+    if custom_candidate_proposer is not None:
+        delegate = custom_candidate_proposer
+
+        def capture_proposer_failure(
+            candidate: dict[str, str], reflective_dataset: Any, components_to_update: list[str],
+        ) -> dict[str, str]:
+            if proposer_failures:
+                raise proposer_failures[0]
+            try:
+                return delegate(candidate, reflective_dataset, components_to_update)
+            except Exception as error:
+                if not proposer_failures:
+                    proposer_failures.append(error)
+                raise
+
+        gepa_candidate_proposer = capture_proposer_failure
+        def raise_proposer_failure(_state: Any) -> bool:
+            if proposer_failures:
+                raise proposer_failures[0]
+            return False
+
+        # Failure must win over every normal stopper in CompositeStopper(any),
+        # otherwise a simultaneous budget stop emits terminal success first.
+        owned_stop_callbacks.insert(0, raise_proposer_failure)
 
     reflection_minibatch_size = config.minibatch_size
     if custom_candidate_proposer is not None and max_metric_calls is not None:
@@ -252,11 +278,18 @@ def run_official_gepa(
         # The connector implements GEPA's component-aware strategy protocol;
         # passing it as a bare callable would discard the selected component.
         kwargs["reflection_strategy"] = reflection_lm
-    if custom_candidate_proposer is not None:
-        kwargs["custom_candidate_proposer"] = custom_candidate_proposer
+    if gepa_candidate_proposer is not None:
+        kwargs["custom_candidate_proposer"] = gepa_candidate_proposer
     kwargs["trainset"] = _with_row_identities(trainset)
     kwargs["valset"] = _with_row_identities(valset)
-    result = optimize(**kwargs)
+    try:
+        result = optimize(**kwargs)
+    except Exception:
+        if proposer_failures:
+            raise proposer_failures[0]
+        raise
+    if proposer_failures:
+        raise proposer_failures[0]
     if custom_candidate_proposer is not None and preflight_evaluation is not None:
         # GEPA's state correctly counts its seed evaluation, but this run
         # served that evaluation from the already-accounted launch preflight.

package/vendor/orizu-gepa/src/orizu_gepa_connector/instruction_set_loader.py

@@ -18,10 +18,38 @@ def _inside(root: Path, candidate: Path) -> bool:
         return False
 
 
-def load_instruction_set(directory: str, name: str, model_config_identity: str) -> dict[str, str]:
-    root = (Path(directory) / name).resolve()
-    if not root.is_dir():
+def _resolve_set_root(directory: str, reference: str) -> Path:
+    directory_root = Path(directory).resolve()
+    if not directory_root.is_dir():
         raise InstructionSetLoaderError("instruction_set_not_synced")
+    direct = (directory_root / reference).resolve()
+    if _inside(directory_root, direct) and direct.is_dir():
+        try:
+            direct_manifest = json.loads((direct / "manifest.json").read_text())
+        except (OSError, json.JSONDecodeError):
+            # Preserve the exact-path loader's specific missing/invalid manifest error.
+            return direct
+        if direct_manifest.get("name") == reference or direct_manifest.get("slug") == reference:
+            return direct
+    matches: list[Path] = []
+    for child in directory_root.iterdir():
+        candidate = child.resolve()
+        if child.name.startswith(".") or not child.is_dir() or not _inside(directory_root, candidate):
+            continue
+        try:
+            manifest = json.loads((candidate / "manifest.json").read_text())
+        except (OSError, json.JSONDecodeError):
+            continue
+        if manifest.get("name") == reference or manifest.get("slug") == reference:
+            matches.append(candidate)
+    if len(matches) != 1:
+        code = "instruction_set_reference_ambiguous" if len(matches) > 1 else "instruction_set_not_synced"
+        raise InstructionSetLoaderError(code)
+    return matches[0]
+
+
+def load_instruction_set(directory: str, name: str, model_config_identity: str) -> dict[str, str]:
+    root = _resolve_set_root(directory, name)
     manifest_path = root / "manifest.json"
     if not manifest_path.is_file():
         raise InstructionSetLoaderError("instruction_set_manifest_missing")

package/vendor/orizu-gepa/src/orizu_gepa_connector/runtime.py

@@ -406,13 +406,17 @@ def run_from_environment() -> dict[str, Any]:
 
     ALI-1502 supplies these variables after verifying runner bytes. Scorer
     contract resolution and seed validation intentionally happen before
-    ``start_run`` so either refusal leaves no stranded running record.
+    ``start_run`` so either refusal leaves no stranded running record. Hosted
+    attach identity is resolved before even those runner-backed preflights.
     """
     env = _required_environment()
     train_split = os.environ.get("ORIZU_TRAIN_SPLIT", "train")
     validation_split = os.environ.get("ORIZU_VALIDATION_SPLIT", "validation")
     config = build_config_from_environment()
     client = OrizuClient.from_env()
+    hosted_run_id = os.environ.get("ORIZU_HOSTED_OPTIMIZATION_RUN_ID", "").strip() or None
+    if hosted_run_id is not None:
+        client.require_hosted_optimization_run(run_id=hosted_run_id, project=env["ORIZU_PROJECT"])
     profile_version_id = os.environ.get("ORIZU_INSTRUCTION_SET_PROFILE_VERSION_ID")
     prompt_context, trainset = client.fetch_exec_context(
         prompt_version_id=os.environ.get("ORIZU_PROMPT_VERSION_ID"), runner_version_id=env["ORIZU_RUNNER_VERSION_ID"],
@@ -492,7 +496,7 @@ def run_from_environment() -> dict[str, Any]:
     is_multi_component_instruction_set = bool(
         profile_version_id and isinstance(shape, list) and len(shape) > 1
     )
-    run_id = client.start_run(
+    run_id = hosted_run_id or client.start_run(
         project=env["ORIZU_PROJECT"], optimizer_version_id=env["ORIZU_OPTIMIZER_VERSION_ID"],
         # A set of one is the legacy prompt subject.  Only a genuine component
         # map is attributed to the profile version instead of a prompt version.
@@ -506,6 +510,7 @@ def run_from_environment() -> dict[str, Any]:
         dataset_version_id=env["ORIZU_DATASET_VERSION_ID"], split_set_id=env["ORIZU_SPLIT_SET_ID"],
         train_split=train_split, validation_split=validation_split, metadata=metadata,
         model_config_settings_version_id=instruction_set.get("model_config_settings_version_id") if isinstance(instruction_set, dict) else None,
+        runner_version_id=prompt_context.runner_version_id,
     )
     print(json.dumps({"optimization_run_id": run_id}), flush=True)
     # The frozen sink treats ``None`` as no local artifact writer.  Keep that