orizu 0.3.7 → 0.4.0

package/src/setup-prompt.ts

@@ -0,0 +1,27 @@
+export interface AgentSetupPromptOptions {
+  workspacePath?: string | null
+}
+
+export function renderAgentSetupPrompt(options?: AgentSetupPromptOptions): string {
+  const workspaceLine = options?.workspacePath
+    ? `- Local Orizu artifacts belong under ${options.workspacePath} (gitignored). Keep secrets out of the repo; credentials live in ~/.config/orizu.`
+    : '- If local Orizu artifacts are needed, suggest creating a gitignored .orizu/ workspace (orizu setup can do this).'
+
+  return `Help me embed Orizu (eval-first LLM optimization) into this repository's workflow.
+
+Start by learning the Orizu workflow:
+1. Run \`orizu skills path --json\` and read the SKILL.md it points to, plus the references it links.
+2. Run \`orizu --version\` and \`orizu whoami\` to confirm the CLI is installed and authenticated. If either fails, pause and tell me how to fix it (npm i -g orizu, orizu login) before continuing.
+
+Then inspect this repository and propose how Orizu should fit it:
+- Identify the LLM application surfaces worth evaluating (prompts, agents, pipelines) and where their inputs/outputs live.
+- Propose Orizu team(s) and project(s) that match how this repo is organized.
+- Propose datasets: existing data that can be uploaded now, or a concrete plan to collect examples if none exists.
+- Propose which prompts to version in Orizu, and which scorers or LLM judges would capture quality for them.
+- Propose runner artifacts where local execution is needed.
+${workspaceLine}
+
+Important boundaries:
+- Present your plan and get my approval before creating anything on the Orizu server or changing repo files.
+- Never duplicate Orizu runtime behavior (scoring, optimization, auth) in repo code; always go through the orizu CLI.`
+}

package/src/skill-installer.ts

@@ -1,14 +1,20 @@
 import {
   cpSync,
   existsSync,
+  lstatSync,
   mkdirSync,
+  readdirSync,
   readFileSync,
+  readlinkSync,
+  realpathSync,
   rmSync,
+  symlinkSync,
   writeFileSync,
 } from 'fs'
-import { dirname, isAbsolute, relative, resolve } from 'path'
+import { dirname, isAbsolute, join, relative, resolve } from 'path'
 import { fileURLToPath } from 'url'
 import { homedir } from 'os'
+import { createHash } from 'crypto'
 
 export const SKILL_NAME = 'orizu-cli'
 export const AGENTS_START_MARKER = '<!-- orizu-cli:start -->'
@@ -16,6 +22,7 @@ export const AGENTS_END_MARKER = '<!-- orizu-cli:end -->'
 
 export const SKILL_INSTALL_TARGETS = [
   'agent-user',
+  'agents-project',
   'codex-project',
   'claude-user',
   'claude-project',
@@ -24,17 +31,46 @@ export const SKILL_INSTALL_TARGETS = [
 
 export type SkillInstallTarget = typeof SKILL_INSTALL_TARGETS[number]
 
+export const SKILL_META_FILENAME = '.orizu-skill-meta.json'
+
+export const SKILL_INSTALL_AGENTS = ['claude', 'codex'] as const
+
+export type SkillInstallAgent = typeof SKILL_INSTALL_AGENTS[number]
+
+export type SkillInstallScope = 'global' | 'local'
+
+export type SkillInstallMode = 'auto' | 'link' | 'copy'
+
+const PROJECT_LEVEL_TARGETS: readonly SkillInstallTarget[] = [
+  'agents-project',
+  'codex-project',
+  'claude-project',
+]
+
 export interface SkillInstallOptions {
   cwd?: string
   homeDir?: string
   overwrite?: boolean
   dryRun?: boolean
+  mode?: SkillInstallMode
+  cliVersion?: string
 }
 
 export interface SkillInstallResult {
   target: SkillInstallTarget
   path: string
   action: 'created' | 'updated' | 'would-create' | 'would-update'
+  mode: 'link' | 'copy' | 'section'
+}
+
+export interface SkillInstallMeta {
+  name: string
+  skillHash: string
+  cliVersion: string | null
+  source: SkillSourceType
+  sourceRoot: string
+  mode: 'copy'
+  installedAt: string
 }
 
 function getCliRoot(): string {
@@ -54,25 +90,49 @@ export function formatMissingSkillSourceError(paths: string[]): string {
     + `Searched:\n- ${paths.join('\n- ')}`
 }
 
-export function resolveSkillSourceDir(): string {
+export type SkillSourceType = 'override' | 'packaged' | 'repo-fallback'
+
+export interface SkillSourceInfo {
+  name: string
+  root: string
+  skillMd: string
+  source: SkillSourceType
+}
+
+export function resolveSkillSource(): SkillSourceInfo {
   const override = process.env.ORIZU_SKILL_SOURCE_DIR
   if (override) {
     const resolvedOverride = resolve(override)
     if (!isSkillDir(resolvedOverride)) {
       throw new Error(`ORIZU_SKILL_SOURCE_DIR is not an Orizu skill directory: ${resolvedOverride}`)
     }
-    return resolvedOverride
+    return {
+      name: SKILL_NAME,
+      root: resolvedOverride,
+      skillMd: resolve(resolvedOverride, 'SKILL.md'),
+      source: 'override',
+    }
   }
 
   const cliRoot = getCliRoot()
   const packagedSkill = resolve(cliRoot, 'vendor', 'skills', SKILL_NAME)
   if (isSkillDir(packagedSkill)) {
-    return packagedSkill
+    return {
+      name: SKILL_NAME,
+      root: packagedSkill,
+      skillMd: resolve(packagedSkill, 'SKILL.md'),
+      source: 'packaged',
+    }
   }
 
   const repoSkill = resolve(cliRoot, '..', '..', 'skills', SKILL_NAME)
   if (isSkillDir(repoSkill)) {
-    return repoSkill
+    return {
+      name: SKILL_NAME,
+      root: repoSkill,
+      skillMd: resolve(repoSkill, 'SKILL.md'),
+      source: 'repo-fallback',
+    }
   }
 
   const searchedPaths = [
@@ -83,6 +143,40 @@ export function resolveSkillSourceDir(): string {
   throw new Error(formatMissingSkillSourceError(searchedPaths))
 }
 
+export function resolveSkillSourceDir(): string {
+  return resolveSkillSource().root
+}
+
+function listSkillFiles(root: string, dir: string, files: string[]): void {
+  for (const entry of readdirSync(dir, { withFileTypes: true })) {
+    const entryPath = join(dir, entry.name)
+    if (!shouldCopySkillPath(root, entryPath)) {
+      continue
+    }
+    if (entry.isDirectory()) {
+      listSkillFiles(root, entryPath, files)
+    } else if (entry.isFile()) {
+      files.push(entryPath)
+    }
+  }
+}
+
+export function computeSkillContentHash(root: string): string {
+  const files: string[] = []
+  listSkillFiles(root, root, files)
+  const hash = createHash('sha256')
+  const sorted = files
+    .map(file => relative(root, file).replace(/\\/g, '/'))
+    .sort()
+  for (const rel of sorted) {
+    hash.update(rel)
+    hash.update('\0')
+    hash.update(readFileSync(join(root, rel)))
+    hash.update('\0')
+  }
+  return `sha256:${hash.digest('hex')}`
+}
+
 function resolveHome(options?: SkillInstallOptions): string {
   return resolve(options?.homeDir || process.env.HOME || homedir())
 }
@@ -102,6 +196,10 @@ export function getSkillInstallPath(
     return resolve(home, '.agents', 'skills', SKILL_NAME)
   }
 
+  if (target === 'agents-project') {
+    return resolve(cwd, '.agents', 'skills', SKILL_NAME)
+  }
+
   if (target === 'codex-project') {
     return resolve(cwd, '.codex', 'skills', SKILL_NAME)
   }
@@ -117,6 +215,69 @@ export function getSkillInstallPath(
   return resolve(cwd, 'AGENTS.md')
 }
 
+export function getTargetForAgent(
+  agent: SkillInstallAgent,
+  scope: SkillInstallScope
+): SkillInstallTarget {
+  if (agent === 'claude') {
+    return scope === 'global' ? 'claude-user' : 'claude-project'
+  }
+  return scope === 'global' ? 'agent-user' : 'agents-project'
+}
+
+export function isSkillInstallAgent(value: string): value is SkillInstallAgent {
+  return (SKILL_INSTALL_AGENTS as readonly string[]).includes(value)
+}
+
+export function isProjectLevelTarget(target: SkillInstallTarget): boolean {
+  return PROJECT_LEVEL_TARGETS.includes(target)
+}
+
+const UNSTABLE_SOURCE_PATTERNS = [
+  /\/_npx\//,
+  /\/\.npm\//,
+  /\/npm-cache\//,
+  /\/\.bun\/install\/cache\//,
+  /\/bunx-/,
+  /\/dlx-/,
+  /\/\.pnpm-store\//,
+  /^\/tmp\//,
+  /^\/(private\/)?var\/folders\//,
+  /\/Temp\//,
+]
+
+export function isStableSkillSourcePath(root: string): boolean {
+  const normalized = root.replace(/\\/g, '/')
+  return !UNSTABLE_SOURCE_PATTERNS.some(pattern => pattern.test(normalized))
+}
+
+export function resolveEffectiveInstallMode(
+  target: SkillInstallTarget,
+  sourceRoot: string,
+  mode: SkillInstallMode = 'auto'
+): 'link' | 'copy' | 'section' {
+  if (target === 'agents-md') {
+    return 'section'
+  }
+
+  if (isProjectLevelTarget(target)) {
+    if (mode === 'link') {
+      throw new Error(
+        `--mode link is not supported for project-level target '${target}': committed project files must not symlink to user-local package paths. Use --mode copy.`
+      )
+    }
+    return 'copy'
+  }
+
+  if (mode === 'link') {
+    return 'link'
+  }
+  if (mode === 'copy') {
+    return 'copy'
+  }
+  return isStableSkillSourcePath(sourceRoot) ? 'link' : 'copy'
+}
+
 function shouldCopySkillPath(sourceRoot: string, sourcePath: string): boolean {
   const rel = relative(sourceRoot, sourcePath).replace(/\\/g, '/')
   if (!rel) {
@@ -130,7 +291,8 @@ function shouldCopySkillPath(sourceRoot: string, sourcePath: string): boolean {
   return !parts.some(part =>
     part === '.DS_Store' ||
     part === '__pycache__' ||
-    part === '.pytest_cache'
+    part === '.pytest_cache' ||
+    part === SKILL_META_FILENAME
   ) && !rel.endsWith('.pyc') && !rel.endsWith('.pyo')
 }
 
@@ -175,6 +337,7 @@ function installAgentsMd(
       target: 'agents-md',
       path: targetPath,
       action: action === 'updated' ? 'would-update' : 'would-create',
+      mode: 'section',
     }
   }
 
@@ -185,7 +348,7 @@ function installAgentsMd(
   mkdirSync(dirname(targetPath), { recursive: true })
   if (!exists) {
     writeFileSync(targetPath, section, 'utf8')
-    return { target: 'agents-md', path: targetPath, action: 'created' }
+    return { target: 'agents-md', path: targetPath, action: 'created', mode: 'section' }
   }
 
   if (hasManagedSection) {
@@ -193,53 +356,101 @@ function installAgentsMd(
       `${escapeRegExp(AGENTS_START_MARKER)}[\\s\\S]*?${escapeRegExp(AGENTS_END_MARKER)}\\n?`
     )
     writeFileSync(targetPath, existing.replace(pattern, section), 'utf8')
-    return { target: 'agents-md', path: targetPath, action: 'updated' }
+    return { target: 'agents-md', path: targetPath, action: 'updated', mode: 'section' }
   }
 
   const existingPrefix = existing.trim().length === 0 ? '' : existing
   const separator = existingPrefix.endsWith('\n') ? '\n' : '\n\n'
   writeFileSync(targetPath, `${existingPrefix}${existingPrefix ? separator : ''}${section}`, 'utf8')
-  return { target: 'agents-md', path: targetPath, action: 'updated' }
+  return { target: 'agents-md', path: targetPath, action: 'updated', mode: 'section' }
 }
 
 function escapeRegExp(value: string): string {
   return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
 }
 
+function pathEntryExists(path: string): boolean {
+  try {
+    lstatSync(path)
+    return true
+  } catch {
+    return false
+  }
+}
+
 export function targetNeedsOverwrite(
   target: SkillInstallTarget,
   options?: SkillInstallOptions
 ): boolean {
   const targetPath = getSkillInstallPath(target, options)
-  if (!existsSync(targetPath)) {
-    return false
+  if (target !== 'agents-md') {
+    return pathEntryExists(targetPath)
   }
 
-  if (target !== 'agents-md') {
-    return true
+  if (!existsSync(targetPath)) {
+    return false
   }
 
   const existing = readFileSync(targetPath, 'utf8')
   return existing.includes(AGENTS_START_MARKER) && existing.includes(AGENTS_END_MARKER)
 }
 
+function writeSkillMeta(
+  targetDir: string,
+  source: SkillSourceInfo,
+  options?: SkillInstallOptions
+): void {
+  const meta: SkillInstallMeta = {
+    name: source.name,
+    skillHash: computeSkillContentHash(source.root),
+    cliVersion: options?.cliVersion || null,
+    source: source.source,
+    sourceRoot: source.root,
+    mode: 'copy',
+    installedAt: new Date().toISOString(),
+  }
+  writeFileSync(join(targetDir, SKILL_META_FILENAME), `${JSON.stringify(meta, null, 2)}\n`, 'utf8')
+}
+
+export function readSkillMeta(targetDir: string): SkillInstallMeta | null {
+  const metaPath = join(targetDir, SKILL_META_FILENAME)
+  if (!existsSync(metaPath)) {
+    return null
+  }
+  try {
+    const parsed = JSON.parse(readFileSync(metaPath, 'utf8')) as SkillInstallMeta
+    return parsed && typeof parsed === 'object' ? parsed : null
+  } catch {
+    return null
+  }
+}
+
+function linkSkillTree(sourceDir: string, targetDir: string) {
+  rmSync(targetDir, { recursive: true, force: true })
+  mkdirSync(dirname(targetDir), { recursive: true })
+  symlinkSync(sourceDir, targetDir, 'dir')
+}
+
 export function installSkillTarget(
   target: SkillInstallTarget,
   options?: SkillInstallOptions
 ): SkillInstallResult {
-  const sourceDir = resolveSkillSourceDir()
+  const source = resolveSkillSource()
   const targetPath = getSkillInstallPath(target, options)
-  const exists = existsSync(targetPath)
 
   if (target === 'agents-md') {
-    return installAgentsMd(sourceDir, targetPath, options)
+    return installAgentsMd(source.root, targetPath, options)
   }
 
+  const exists = pathEntryExists(targetPath)
+  const mode = resolveEffectiveInstallMode(target, source.root, options?.mode)
+
   if (options?.dryRun) {
     return {
       target,
       path: targetPath,
       action: exists ? 'would-update' : 'would-create',
+      mode,
     }
   }
 
@@ -247,11 +458,126 @@ export function installSkillTarget(
     throw new Error(`${targetPath} already exists. Pass --yes to replace it.`)
   }
 
-  copySkillTree(sourceDir, targetPath)
+  if (mode === 'link') {
+    linkSkillTree(source.root, targetPath)
+  } else {
+    copySkillTree(source.root, targetPath)
+    writeSkillMeta(targetPath, source, options)
+  }
+
   return {
     target,
     path: targetPath,
     action: exists ? 'updated' : 'created',
+    mode,
+  }
+}
+
+export type SkillTargetState =
+  | 'current'
+  | 'stale'
+  | 'missing'
+  | 'broken-link'
+  | 'unmanaged'
+
+export interface SkillTargetStatus {
+  target: SkillInstallTarget
+  path: string
+  state: SkillTargetState
+  mode: 'link' | 'copy' | 'section' | null
+  linkTarget: string | null
+  installedHash: string | null
+  sourceHash: string
+  meta: SkillInstallMeta | null
+}
+
+function getAgentsMdStatus(
+  targetPath: string,
+  source: SkillSourceInfo,
+  sourceHash: string
+): SkillTargetStatus {
+  const base = {
+    target: 'agents-md' as const,
+    path: targetPath,
+    mode: 'section' as const,
+    linkTarget: null,
+    installedHash: null,
+    sourceHash,
+    meta: null,
+  }
+
+  if (!existsSync(targetPath)) {
+    return { ...base, state: 'missing' }
+  }
+
+  const existing = readFileSync(targetPath, 'utf8')
+  const start = existing.indexOf(AGENTS_START_MARKER)
+  const end = existing.indexOf(AGENTS_END_MARKER)
+  if (start === -1 || end === -1) {
+    return { ...base, state: 'unmanaged' }
+  }
+
+  const actualSection = existing.slice(start, end + AGENTS_END_MARKER.length)
+  const expectedSection = buildAgentsSection(source.root).trimEnd()
+  return {
+    ...base,
+    state: actualSection.trimEnd() === expectedSection ? 'current' : 'stale',
+  }
+}
+
+export function getSkillTargetStatus(
+  target: SkillInstallTarget,
+  options?: SkillInstallOptions
+): SkillTargetStatus {
+  const source = resolveSkillSource()
+  const sourceHash = computeSkillContentHash(source.root)
+  const targetPath = getSkillInstallPath(target, options)
+
+  if (target === 'agents-md') {
+    return getAgentsMdStatus(targetPath, source, sourceHash)
+  }
+
+  const base = {
+    target,
+    path: targetPath,
+    linkTarget: null as string | null,
+    installedHash: null as string | null,
+    sourceHash,
+    meta: null as SkillInstallMeta | null,
+  }
+
+  if (!pathEntryExists(targetPath)) {
+    return { ...base, state: 'missing', mode: null }
+  }
+
+  const stats = lstatSync(targetPath)
+  if (stats.isSymbolicLink()) {
+    const linkTarget = readlinkSync(targetPath)
+    if (!existsSync(targetPath)) {
+      return { ...base, state: 'broken-link', mode: 'link', linkTarget }
+    }
+
+    const resolvedLink = realpathSync(targetPath)
+    const installedHash = computeSkillContentHash(resolvedLink)
+    const state = resolvedLink === realpathSync(source.root) || installedHash === sourceHash
+      ? 'current'
+      : 'stale'
+    return { ...base, state, mode: 'link', linkTarget, installedHash }
+  }
+
+  const installedHash = computeSkillContentHash(targetPath)
+  const meta = readSkillMeta(targetPath)
+  // A differing directory without our meta file was not installed by Orizu
+  // (or was hand-customized): treat it as unmanaged so `skills update` never
+  // silently overwrites it. Matching content is current regardless of meta.
+  const state =
+    installedHash === sourceHash ? 'current' : meta ? 'stale' : 'unmanaged'
+  return {
+    ...base,
+    state,
+    mode: 'copy',
+    installedHash,
+    meta,
   }
 }
 

package/src/workspace.ts

@@ -0,0 +1,112 @@
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs'
+import { join, resolve } from 'path'
+
+export const WORKSPACE_DIR_NAME = '.orizu'
+
+export interface WorkspaceInitOptions {
+  cwd?: string
+  baseUrl?: string | null
+  cliVersion?: string | null
+  dryRun?: boolean
+}
+
+export interface WorkspaceInitResult {
+  root: string
+  state: 'created' | 'exists' | 'would-create'
+  gitignoreUpdated: boolean
+  actions: string[]
+}
+
+const WORKSPACE_README = `# Orizu workspace
+
+Local Orizu artifacts for this repository. This folder is gitignored by
+default: generated and server-derived content here is not guaranteed to stay
+in sync with the Orizu server, so checking it in would risk drift. Safe,
+user-authored configuration may move to a committed location once a sync model
+exists.
+
+- \`workspace.json\`: safe local metadata (no secrets; credentials stay in
+  \`~/.config/orizu\`).
+- \`generated/\`: runner artifacts, exports, and other generated files.
+`
+
+export function getWorkspaceRoot(cwd?: string): string {
+  return resolve(cwd || process.cwd(), WORKSPACE_DIR_NAME)
+}
+
+export function workspaceExists(cwd?: string): boolean {
+  return existsSync(join(getWorkspaceRoot(cwd), 'workspace.json'))
+}
+
+function ensureGitignoreEntry(cwd: string, dryRun: boolean): { updated: boolean, action: string | null } {
+  const gitignorePath = resolve(cwd, '.gitignore')
+  const entry = `${WORKSPACE_DIR_NAME}/`
+
+  if (existsSync(gitignorePath)) {
+    const existing = readFileSync(gitignorePath, 'utf8')
+    const hasEntry = existing
+      .split('\n')
+      .some(line => line.trim() === entry || line.trim() === WORKSPACE_DIR_NAME)
+    if (hasEntry) {
+      return { updated: false, action: null }
+    }
+    if (!dryRun) {
+      const separator = existing.endsWith('\n') || existing.length === 0 ? '' : '\n'
+      writeFileSync(gitignorePath, `${existing}${separator}\n# Local Orizu workspace (generated; not synced with the server)\n${entry}\n`, 'utf8')
+    }
+    return { updated: true, action: `append ${entry} to .gitignore` }
+  }
+
+  if (!dryRun) {
+    writeFileSync(gitignorePath, `# Local Orizu workspace (generated; not synced with the server)\n${entry}\n`, 'utf8')
+  }
+  return { updated: true, action: `create .gitignore with ${entry}` }
+}
+
+export function initOrizuWorkspace(options?: WorkspaceInitOptions): WorkspaceInitResult {
+  const cwd = resolve(options?.cwd || process.cwd())
+  const root = getWorkspaceRoot(cwd)
+  const dryRun = Boolean(options?.dryRun)
+  const exists = workspaceExists(cwd)
+  const actions: string[] = []
+
+  if (!exists) {
+    actions.push(`create ${WORKSPACE_DIR_NAME}/workspace.json, README.md, and generated/`)
+  }
+
+  const gitignore = ensureGitignoreEntry(cwd, dryRun || exists)
+  if (!exists && gitignore.action) {
+    actions.push(gitignore.action)
+  }
+
+  if (dryRun) {
+    return {
+      root,
+      state: exists ? 'exists' : 'would-create',
+      gitignoreUpdated: false,
+      actions,
+    }
+  }
+
+  if (exists) {
+    return { root, state: 'exists', gitignoreUpdated: false, actions: [] }
+  }
+
+  mkdirSync(join(root, 'generated'), { recursive: true })
+  writeFileSync(join(root, 'README.md'), WORKSPACE_README, 'utf8')
+  writeFileSync(
+    join(root, 'workspace.json'),
+    `${JSON.stringify(
+      {
+        version: 1,
+        server: options?.baseUrl || null,
+        createdByCliVersion: options?.cliVersion || null,
+      },
+      null,
+      2
+    )}\n`,
+    'utf8'
+  )
+
+  return { root, state: 'created', gitignoreUpdated: gitignore.updated, actions }
+}

package/vendor/orizu-gepa-python/src/orizu_gepa/__init__.py

@@ -3,6 +3,7 @@ from .optimizer import (
     DatasetRow,
     NumThreadsPlan,
     PromptContext,
+    RetryableReflectionError,
     RowEvaluation,
     TextGepaConfig,
     TextGepaResult,
@@ -15,6 +16,7 @@ __all__ = [
     "DatasetRow",
     "NumThreadsPlan",
     "PromptContext",
+    "RetryableReflectionError",
     "RowEvaluation",
     "TextGepaConfig",
     "TextGepaResult",