@the-open-engine/zeroshot 6.36.0 → 6.38.0

package/src/agent-cli-provider/pi/protocol-state.ts

@@ -0,0 +1,163 @@
+import { parseTurnEndResult } from '../assistant-stream';
+import { getNumber, getRecord, getString } from '../json';
+import type { ProviderParserState, ResultEvent } from '../types';
+
+const STOP_REASONS = new Set(['stop', 'length', 'toolUse', 'error', 'aborted', 'deferred']);
+const REQUIRED_USAGE_FIELDS = [
+  'input',
+  'output',
+  'cacheRead',
+  'cacheWrite',
+  'totalTokens',
+] as const;
+const OPTIONAL_USAGE_FIELDS = ['cacheWrite1h', 'reasoning'] as const;
+const COST_FIELDS = ['input', 'output', 'cacheRead', 'cacheWrite', 'total'] as const;
+
+interface PiProtocolState {
+  latestAssistant?: Record<string, unknown>;
+  invalid?: string;
+  usage: Record<string, number>;
+  cost: Record<string, number>;
+}
+
+const protocolStates = new WeakMap<ProviderParserState, PiProtocolState>();
+
+function protocolState(state: ProviderParserState): PiProtocolState {
+  const current = protocolStates.get(state);
+  if (current !== undefined) return current;
+  const created = { usage: {}, cost: {} };
+  protocolStates.set(state, created);
+  return created;
+}
+
+export function invalidatePiStream(state: ProviderParserState, error: string): void {
+  protocolState(state).invalid = error;
+}
+
+function validateUsage(usage: Record<string, unknown>, label: string): string | null {
+  for (const field of REQUIRED_USAGE_FIELDS) {
+    if (getNumber(usage, field) === null) return `${label}.${field} is missing.`;
+  }
+  for (const field of OPTIONAL_USAGE_FIELDS) {
+    if (usage[field] !== undefined && getNumber(usage, field) === null) {
+      return `${label}.${field} is invalid.`;
+    }
+  }
+  const cost = getRecord(usage, 'cost');
+  if (cost === null) return `${label}.cost is missing.`;
+  for (const field of COST_FIELDS) {
+    if (getNumber(cost, field) === null) return `${label}.cost.${field} is missing.`;
+  }
+  return null;
+}
+
+function validateAssistant(message: Record<string, unknown>): string | null {
+  if (!Array.isArray(message.content)) return 'Pi assistant message content is missing.';
+  const stopReason = getString(message, 'stopReason');
+  if (stopReason === null || !STOP_REASONS.has(stopReason)) {
+    return 'Pi assistant message stopReason is missing or unsupported.';
+  }
+  const usage = getRecord(message, 'usage');
+  return usage === null
+    ? 'Pi assistant message usage is missing.'
+    : validateUsage(usage, 'Pi assistant usage');
+}
+
+function addNumbers(
+  target: Record<string, number>,
+  source: Record<string, unknown>,
+  fields: readonly string[]
+): void {
+  for (const field of fields) {
+    const value = getNumber(source, field);
+    if (value !== null) target[field] = (target[field] ?? 0) + value;
+  }
+}
+
+export function rememberPiAssistant(
+  message: Record<string, unknown> | null,
+  state: ProviderParserState
+): void {
+  if (message === null) return invalidatePiStream(state, 'Pi message_end was missing its message.');
+  const role = getString(message, 'role');
+  if (role === null) return invalidatePiStream(state, 'Pi message_end was missing its role.');
+  if (role !== 'assistant') return;
+  const error = validateAssistant(message);
+  if (error !== null) return invalidatePiStream(state, error);
+  const current = protocolState(state);
+  current.latestAssistant = message;
+  rememberPiUsage(getRecord(message, 'usage'), state, 'Pi assistant usage');
+}
+
+export function rememberPiUsage(
+  usage: Record<string, unknown> | null,
+  state: ProviderParserState,
+  label: string
+): void {
+  if (usage === null) return invalidatePiStream(state, `${label} is invalid.`);
+  const error = validateUsage(usage, label);
+  if (error !== null) return invalidatePiStream(state, error);
+  const current = protocolState(state);
+  addNumbers(current.usage, usage, [...REQUIRED_USAGE_FIELDS, ...OPTIONAL_USAGE_FIELDS]);
+  const cost = getRecord(usage, 'cost');
+  if (cost !== null) addNumbers(current.cost, cost, COST_FIELDS);
+}
+
+function usageResult(
+  current: PiProtocolState
+): Pick<
+  ResultEvent,
+  | 'inputTokens'
+  | 'outputTokens'
+  | 'cacheReadInputTokens'
+  | 'cacheCreationInputTokens'
+  | 'cost'
+  | 'modelUsage'
+> {
+  const cost = Object.keys(current.cost).length === 0 ? null : current.cost;
+  return {
+    inputTokens: current.usage.input ?? 0,
+    outputTokens: current.usage.output ?? 0,
+    cacheReadInputTokens: current.usage.cacheRead ?? 0,
+    cacheCreationInputTokens: current.usage.cacheWrite ?? 0,
+    cost: cost?.total ?? null,
+    modelUsage: { ...current.usage, ...(cost === null ? {} : { cost }) },
+  };
+}
+
+function failedResult(error: string, current: PiProtocolState): ResultEvent {
+  return {
+    type: 'result',
+    success: false,
+    result: null,
+    error,
+    ...usageResult(current),
+  };
+}
+
+export function settlePiStream(state: ProviderParserState): ResultEvent {
+  const current = protocolState(state);
+  if (current.invalid) {
+    return failedResult(`Invalid Pi JSON stream: ${current.invalid}`, current);
+  }
+  if (!current.latestAssistant) {
+    return failedResult('Pi settled without an assistant message_end.', current);
+  }
+  const parsedResult = parseTurnEndResult(
+    { message: current.latestAssistant },
+    { failureMessage: 'Pi turn failed' }
+  );
+  const result =
+    getString(current.latestAssistant, 'stopReason') === 'deferred'
+      ? {
+          ...parsedResult,
+          success: false,
+          result: null,
+          error: 'Pi turn deferred without completing.',
+        }
+      : parsedResult;
+  return {
+    ...result,
+    ...usageResult(current),
+  };
+}

package/src/agent-cli-provider/pi/release.ts

@@ -0,0 +1,4 @@
+export const PI_SUPPORTED_VERSION = '0.84.1' as const;
+export const PI_PACKAGE_NAME = '@earendil-works/pi-coding-agent' as const;
+export const PI_INSTALL_COMMAND =
+  `npm install -g --ignore-scripts ${PI_PACKAGE_NAME}@${PI_SUPPORTED_VERSION}` as const;

package/src/agent-cli-provider/probe-availability.ts

@@ -0,0 +1,19 @@
+import type { ProviderCliFeatures } from './types';
+
+type AvailabilityProbe = 'command' | 'help-or-version' | 'supported-version' | undefined;
+
+export function runtimeProbeIsAvailable(
+  availabilityProbe: AvailabilityProbe,
+  evidenceAvailable: boolean | undefined,
+  helpText: string,
+  versionText: string,
+  capabilities: ProviderCliFeatures
+): boolean {
+  const probeAvailable =
+    evidenceAvailable ??
+    (availabilityProbe === 'command' ? true : Boolean(helpText || versionText));
+  const versionSupported =
+    availabilityProbe !== 'supported-version' ||
+    ('versionMatches' in capabilities && capabilities.versionMatches === true);
+  return probeAvailable && versionSupported;
+}

package/src/agent-cli-provider/provider-registry.ts

@@ -18,6 +18,7 @@ import {
   OMP_INSTALL_COMMAND,
 } from './omp/release';
 import { OMP_SDK_SETTINGS_DEFAULTS, validateOmpSdkSettings } from './omp/sdk-settings';
+import { PI_INSTALL_COMMAND } from './pi/release';
 import type { OmpSettingsValidationContext } from './omp/sdk-settings';
 import type { ModelLevel, ProviderAdapter, StructuredOutputRecoveryAdapter } from './types';
 
@@ -73,6 +74,9 @@ export interface ProviderDocsMetadata {
 
 export interface ProviderDockerMountPreset {
   readonly host: string;
+  // Optional host-only config-root override. Its value selects the mount source but is never
+  // forwarded into the container, where `container` remains the canonical provider path.
+  readonly hostEnv?: string;
   readonly container: string;
   readonly readonly: boolean;
 }
@@ -130,7 +134,7 @@ interface ProviderRegistryEntryBase {
     settings: Record<string, unknown>,
     context?: OmpSettingsValidationContext
   ) => string | null;
-  readonly availabilityProbe?: 'command' | 'help-or-version';
+  readonly availabilityProbe?: 'command' | 'help-or-version' | 'supported-version';
   readonly docs: ProviderDocsMetadata;
   readonly docker: ProviderDockerMetadata;
   readonly defaultLevels: Readonly<{
@@ -186,6 +190,35 @@ const CLAUDE_DOCKER_ENV_PASSTHROUGH = [
   'CLAUDE_CODE_USE_BEDROCK',
 ] as const;
 
+const PI_EXPLICIT_DOCKER_CREDENTIALS = new Set([
+  'AWS_CONTAINER_CREDENTIALS_FULL_URI',
+  'AWS_CONTAINER_CREDENTIALS_RELATIVE_URI',
+  'AWS_PROFILE',
+  'AWS_WEB_IDENTITY_TOKEN_FILE',
+  'GOOGLE_APPLICATION_CREDENTIALS',
+]);
+
+const PI_DOCKER_CONFIGURATION_ENV = [
+  'AWS_BEDROCK_FORCE_CACHE',
+  'AWS_BEDROCK_FORCE_HTTP1',
+  'AWS_BEDROCK_SKIP_AUTH',
+  'AWS_DEFAULT_REGION',
+  'AWS_ENDPOINT_URL_BEDROCK_RUNTIME',
+  'AWS_REGION',
+  'AZURE_OPENAI_API_VERSION',
+  'AZURE_OPENAI_BASE_URL',
+  'AZURE_OPENAI_DEPLOYMENT_NAME_MAP',
+  'AZURE_OPENAI_RESOURCE_NAME',
+  'CLOUDFLARE_ACCOUNT_ID',
+  'CLOUDFLARE_GATEWAY_ID',
+  'GCLOUD_PROJECT',
+  'GOOGLE_CLOUD_LOCATION',
+  'GOOGLE_CLOUD_PROJECT',
+  'KIMI_CODE_OAUTH_HOST',
+  'KIMI_OAUTH_HOST',
+  'PI_CACHE_RETENTION',
+] as const;
+
 const SPAWN_INVOKE = Object.freeze({ lane: 'spawn' }) as SpawnProviderInvokeSpec;
 const ACP_STDIO_INVOKE = Object.freeze({
   lane: 'acp-stdio',
@@ -444,17 +477,17 @@ export const providerRegistry = [
     binary: 'pi',
     command: { kind: 'fixed', command: 'pi', args: [] },
     invoke: SPAWN_INVOKE,
-    installInstructions: 'npm install -g --ignore-scripts @earendil-works/pi-coding-agent@0.80.3',
+    installInstructions: PI_INSTALL_COMMAND,
     authInstructions: 'pi\n/login',
-    credentialPaths: ['~/.pi'],
+    credentialPaths: ['$PI_CODING_AGENT_DIR/auth.json', '~/.pi/agent/auth.json'],
     credentialEnvKeys: piAdapter.credentialEnvKeys,
     settingsFields: [],
-    availabilityProbe: 'help-or-version',
+    availabilityProbe: 'supported-version',
     capabilities: {
       ...STANDARD_CAPABILITIES,
       mcpServers: false,
       jsonSchema: false,
-      reasoningEffort: false,
+      reasoningEffort: true,
     },
     docs: {
       label: 'Pi',
@@ -462,11 +495,17 @@ export const providerRegistry = [
     },
     docker: {
       mount: {
-        host: '~/.pi',
-        container: '$HOME/.pi',
-        readonly: true,
+        host: '~/.pi/agent',
+        hostEnv: 'PI_CODING_AGENT_DIR',
+        container: '$HOME/.pi/agent',
+        readonly: false,
       },
-      envPassthrough: [],
+      install: PI_INSTALL_COMMAND,
+      configRoots: ['$HOME/.pi/agent'],
+      envPassthrough: [
+        ...piAdapter.credentialEnvKeys.filter((name) => !PI_EXPLICIT_DOCKER_CREDENTIALS.has(name)),
+        ...PI_DOCKER_CONFIGURATION_ENV,
+      ],
     },
     defaultLevels: {
       min: piAdapter.defaultMinLevel,

package/src/agent-cli-provider/single-agent-runtime.ts

@@ -29,6 +29,7 @@ import {
   supportsProviderCapability,
   supportsProviderOutputReformatting,
 } from './provider-registry';
+import { runtimeProbeIsAvailable } from './probe-availability';
 import type {
   BuildProviderCommandOptions,
   CliFeatureOverrides,
@@ -577,6 +578,10 @@ function resolveRuntimeCliFeatures(
       supportsWebSearch: false,
     };
   }
+  if (provider === 'pi') {
+    const detected = detectRuntimeProviderCliFeatures(provider, runtimeSettings);
+    return overrides === undefined ? detected : mergePiFailClosedCliFeatures(detected, overrides);
+  }
   if (getProviderRegistryEntry(provider).invoke.lane === 'acp-stdio') {
     const detected = detectRuntimeProviderCliFeatures(provider, runtimeSettings);
     if (overrides === undefined) return detected;
@@ -595,6 +600,18 @@ function resolveRuntimeCliFeatures(
   };
 }
 
+function mergePiFailClosedCliFeatures(
+  detected: ProviderCliFeatures,
+  overrides: CliFeatureOverrides
+): CliFeatureOverrides {
+  return Object.fromEntries(
+    Object.entries(detected).map(([field, value]) => [
+      field,
+      typeof value === 'boolean' ? value && Reflect.get(overrides, field) !== false : value,
+    ])
+  );
+}
+
 function mergeAcpFailClosedCliFeatures(
   detected: ProviderCliFeatures,
   overrides: CliFeatureOverrides
@@ -679,7 +696,9 @@ export function probeRuntimeProviderCli(
 
   const helpText =
     evidence?.helpText ??
-    stringResult(getHelpOutputFn(helpCommand.command, helpCommand.args)).trim();
+    (adapter.id === 'pi'
+      ? ''
+      : stringResult(getHelpOutputFn(helpCommand.command, helpCommand.args)).trim());
   const versionText =
     evidence?.versionText ??
     stringResult(
@@ -691,11 +710,13 @@ export function probeRuntimeProviderCli(
   const capabilities = attestedCliFeatures(adapter, helpText, versionText);
 
   return {
-    available:
-      evidence?.available ??
-      (registryEntry.availabilityProbe === 'help-or-version'
-        ? Boolean(helpText || versionText)
-        : true),
+    available: runtimeProbeIsAvailable(
+      registryEntry.availabilityProbe,
+      evidence?.available,
+      helpText,
+      versionText,
+      capabilities
+    ),
     helpText,
     versionText,
     capabilities,

package/src/agent-cli-provider/types.ts

@@ -158,8 +158,10 @@ export interface OpencodeCliFeatures extends BaseCliFeatures {
 
 export interface PiCliFeatures extends BaseCliFeatures {
   readonly provider: 'pi';
+  readonly versionMatches: boolean;
   readonly supportsJsonMode: boolean;
   readonly supportsModel: boolean;
+  readonly supportsThinking: boolean;
   readonly supportsNoSession: boolean;
   readonly supportsNoExtensions: boolean;
   readonly supportsNoSkills: boolean;
@@ -573,6 +575,7 @@ export interface ProviderAdapter {
   buildCommand(context: string, options?: BuildProviderCommandOptions): CommandSpec;
   extractSessionId?(line: string): string | null;
   parseEvent(line: string, state: ProviderParserState): ProviderParseResult;
+  finishParsing?(state: ProviderParserState): ProviderParseResult;
   createParserState(): ProviderParserState;
   resolveModelSpec(level: ModelLevel, overrides?: LevelOverrides): ResolvedModelSpec;
   validateModelId(modelId: string | null | undefined): string | null | undefined;

package/src/config-validator.js

@@ -2058,9 +2058,7 @@ function validateProviderSettings(provider, providerSettings) {
       providerModule.resolveModelSpec(level, { [level]: override });
     }
     if (override?.reasoningEffort && !providerSupportsCapability(provider, 'reasoningEffort')) {
-      throw new Error(
-        `reasoningEffort overrides are only supported for Claude, Codex, and Opencode`
-      );
+      throw new Error(`Provider "${provider}" does not support reasoningEffort overrides`);
     }
     if (override?.reasoningEffort && !REASONING_EFFORTS.includes(override.reasoningEffort)) {
       throw new Error(

package/src/foreground-benchmark-files.js

@@ -0,0 +1,81 @@
+const crypto = require('node:crypto');
+const fs = require('node:fs');
+const path = require('node:path');
+
+const { buildTelemetry } = require('./foreground-benchmark-result');
+
+function serialized(value) {
+  return Buffer.from(`${JSON.stringify(value)}\n`, 'utf8');
+}
+
+function atomicWriteNew(targetPath, content) {
+  const directory = path.dirname(targetPath);
+  const temporary = path.join(
+    directory,
+    `.${path.basename(targetPath)}.${process.pid}.${crypto.randomBytes(8).toString('hex')}.tmp`
+  );
+  let descriptor;
+  let published = false;
+  try {
+    descriptor = fs.openSync(temporary, 'wx', 0o600);
+    fs.writeFileSync(descriptor, content);
+    fs.fsyncSync(descriptor);
+    fs.closeSync(descriptor);
+    descriptor = undefined;
+    fs.linkSync(temporary, targetPath);
+    published = true;
+    fs.unlinkSync(temporary);
+    const directoryDescriptor = fs.openSync(directory, fs.constants.O_RDONLY);
+    try {
+      fs.fsyncSync(directoryDescriptor);
+    } finally {
+      fs.closeSync(directoryDescriptor);
+    }
+  } catch (error) {
+    error.atomicTargetPublished = published;
+    throw error;
+  } finally {
+    if (descriptor !== undefined) fs.closeSync(descriptor);
+    try {
+      fs.unlinkSync(temporary);
+    } catch {
+      // Preserve the primary write error; a leftover randomized temp is non-authoritative.
+    }
+  }
+}
+
+function removeOrphanTelemetry(telemetryPath, primaryError) {
+  try {
+    fs.unlinkSync(telemetryPath);
+  } catch (cleanupError) {
+    if (cleanupError.code !== 'ENOENT') primaryError.cleanupError = cleanupError;
+  }
+}
+
+function writeBenchmarkResultBundle(resultPath, result, snapshot) {
+  if (typeof resultPath !== 'string' || resultPath.length === 0) {
+    throw new Error('result path must be non-empty text');
+  }
+  const resolvedResult = path.resolve(resultPath);
+  const telemetryPath = `${resolvedResult}.telemetry.json`;
+  const telemetry = buildTelemetry(result.runId, snapshot);
+  const telemetryBytes = serialized(telemetry);
+  atomicWriteNew(telemetryPath, telemetryBytes);
+  const receipt = {
+    ...result,
+    telemetry: {
+      artifact: path.basename(telemetryPath),
+      byteLength: telemetryBytes.length,
+      sha256: crypto.createHash('sha256').update(telemetryBytes).digest('hex'),
+    },
+  };
+  try {
+    atomicWriteNew(resolvedResult, serialized(receipt));
+  } catch (error) {
+    if (!error.atomicTargetPublished) removeOrphanTelemetry(telemetryPath, error);
+    throw error;
+  }
+  return receipt;
+}
+
+module.exports = { writeBenchmarkResultBundle };