@covibes/zeroshot 5.2.1 → 5.4.0

package/src/agent/output-extraction.js

@@ -0,0 +1,217 @@
+/**
+ * Output Extraction Module - Multi-Provider JSON Extraction
+ *
+ * Clean extraction pipeline for structured JSON from AI provider outputs.
+ * Each provider has different output formats - this module normalizes them.
+ *
+ * Provider formats:
+ * - Claude: {"type":"result","result":{...}} or {"type":"result","structured_output":{...}}
+ * - Codex: Raw text in item.created events, turn.completed has NO result field
+ * - Gemini: Raw text in message events, result event may have NO result field
+ *
+ * Extraction priority (most specific → least specific):
+ * 1. Result wrapper with content (type:result + result/structured_output field)
+ * 2. Accumulated text from provider parser events
+ * 3. Markdown code block extraction
+ * 4. Direct JSON parse of entire output
+ */
+
+const { getProvider, parseChunkWithProvider } = require('../providers');
+
+/**
+ * Strip timestamp prefix from log lines.
+ * Format: [epochMs]content or [epochMs]{json...}
+ *
+ * @param {string} line - Raw log line
+ * @returns {string} Content without timestamp prefix
+ */
+function stripTimestamp(line) {
+  if (!line || typeof line !== 'string') return '';
+  let trimmed = line.trim().replace(/\r$/, '');
+  if (!trimmed) return '';
+
+  const tsMatch = trimmed.match(/^\[(\d{13})\](.*)$/);
+  if (tsMatch) trimmed = (tsMatch[2] || '').trimStart();
+
+  // In cluster logs, lines are often prefixed like:
+  // "validator       | {json...}"
+  // Strip the "<agent> | " prefix so we can JSON.parse the event line.
+  if (!trimmed.startsWith('{') && !trimmed.startsWith('[')) {
+    const pipeMatch = trimmed.match(/^[^|]{1,40}\|\s*(.*)$/);
+    if (pipeMatch) {
+      const afterPipe = (pipeMatch[1] || '').trimStart();
+      if (afterPipe.startsWith('{') || afterPipe.startsWith('[')) return afterPipe;
+    }
+  }
+
+  return trimmed;
+}
+
+/**
+ * Strategy 1: Extract from result wrapper
+ * Handles Claude CLI format: {"type":"result","result":{...}}
+ *
+ * @param {string} output - Raw output
+ * @returns {object|null} Extracted JSON or null
+ */
+function extractFromResultWrapper(output) {
+  const lines = output.split('\n');
+
+  for (const line of lines) {
+    const content = stripTimestamp(line);
+    if (!content.startsWith('{')) continue;
+
+    try {
+      const obj = JSON.parse(content);
+      const extracted = extractResultContent(obj);
+      if (extracted) return extracted;
+    } catch {
+      // Not valid JSON, continue to next line
+    }
+  }
+
+  return null;
+}
+
+function extractResultContent(obj) {
+  // Must be type:result WITH actual content
+  if (obj?.type !== 'result') return null;
+
+  // Check structured_output first (standard CLI format)
+  if (obj.structured_output && typeof obj.structured_output === 'object') {
+    return obj.structured_output;
+  }
+
+  // Check result field - can be object or string
+  if (!obj.result) return null;
+
+  if (typeof obj.result === 'object') {
+    return obj.result;
+  }
+
+  if (typeof obj.result !== 'string') return null;
+
+  // Result is string - might contain markdown-wrapped JSON
+  return extractFromMarkdown(obj.result) || extractDirectJson(obj.result);
+}
+
+/**
+ * Strategy 2: Extract from accumulated text events
+ * Handles non-Claude providers where JSON is in text content
+ *
+ * @param {string} output - Raw output
+ * @param {string} providerName - Provider name for parser selection
+ * @returns {object|null} Extracted JSON or null
+ */
+function extractFromTextEvents(output, providerName) {
+  const provider = getProvider(providerName);
+  const events = parseChunkWithProvider(provider, output);
+
+  // Accumulate all text events
+  const textContent = events
+    .filter((e) => e.type === 'text')
+    .map((e) => e.text)
+    .join('');
+
+  if (!textContent.trim()) return null;
+
+  // Try parsing accumulated text as JSON
+  return extractDirectJson(textContent) || extractFromMarkdown(textContent);
+}
+
+/**
+ * Strategy 3: Extract JSON from markdown code block
+ * Handles: ```json\n{...}\n```
+ *
+ * @param {string} text - Text that may contain markdown
+ * @returns {object|null} Extracted JSON or null
+ */
+function extractFromMarkdown(text) {
+  if (!text) return null;
+
+  // Match ```json ... ``` with any whitespace
+  const match = text.match(/```json\s*([\s\S]*?)```/);
+  if (!match) return null;
+
+  try {
+    const parsed = JSON.parse(match[1].trim());
+    if (typeof parsed === 'object' && parsed !== null) {
+      return parsed;
+    }
+  } catch {
+    // Invalid JSON in markdown block
+  }
+
+  return null;
+}
+
+/**
+ * Strategy 4: Direct JSON parse
+ * Handles raw JSON output (single-line or multi-line)
+ *
+ * @param {string} text - Text to parse
+ * @returns {object|null} Parsed JSON or null
+ */
+function extractDirectJson(text) {
+  if (!text) return null;
+
+  const trimmed = text.trim();
+  if (!trimmed) return null;
+
+  try {
+    const parsed = JSON.parse(trimmed);
+    if (typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed)) {
+      return parsed;
+    }
+  } catch {
+    // Not valid JSON
+  }
+
+  return null;
+}
+
+/**
+ * Main extraction function - tries all strategies in priority order
+ *
+ * @param {string} output - Raw output from AI provider CLI
+ * @param {string} providerName - Provider name ('claude', 'codex', 'gemini')
+ * @returns {object|null} Extracted JSON object or null if extraction failed
+ */
+function extractJsonFromOutput(output, providerName = 'claude') {
+  if (!output || typeof output !== 'string') return null;
+
+  const trimmedOutput = output.trim();
+  if (!trimmedOutput) return null;
+
+  // Check for fatal error indicators
+  if (trimmedOutput.includes('Task not found') || trimmedOutput.includes('Process terminated')) {
+    return null;
+  }
+
+  // Strategy 1: Result wrapper (Claude format)
+  const fromWrapper = extractFromResultWrapper(trimmedOutput);
+  if (fromWrapper) return fromWrapper;
+
+  // Strategy 2: Text events (non-Claude providers)
+  const fromText = extractFromTextEvents(trimmedOutput, providerName);
+  if (fromText) return fromText;
+
+  // Strategy 3: Markdown extraction
+  const fromMarkdown = extractFromMarkdown(trimmedOutput);
+  if (fromMarkdown) return fromMarkdown;
+
+  // Strategy 4: Direct JSON parse (raw output)
+  const fromDirect = extractDirectJson(trimmedOutput);
+  if (fromDirect) return fromDirect;
+
+  return null;
+}
+
+module.exports = {
+  extractJsonFromOutput,
+  extractFromResultWrapper,
+  extractFromTextEvents,
+  extractFromMarkdown,
+  extractDirectJson,
+  stripTimestamp,
+};

package/src/agent/output-reformatter.js

@@ -0,0 +1,175 @@
+/**
+ * Output Reformatter - Convert non-JSON output to valid JSON
+ *
+ * When an LLM outputs markdown/text instead of JSON despite schema instructions,
+ * this module attempts to extract/reformat the content into valid JSON.
+ *
+ * STATUS: SDK NOT IMPLEMENTED - Reformatting is not available.
+ * This module exists for future extension when SDK support is added.
+ *
+ * To enable reformatting:
+ * 1. Implement SDK support in the provider (getSDKEnvVar, callSimple)
+ * 2. The reformatOutput() function will then work automatically
+ */
+
+const DEFAULT_MAX_ATTEMPTS = 3;
+
+/**
+ * Build the reformatting prompt
+ *
+ * @param {string} rawOutput - The non-JSON output to reformat
+ * @param {Object} schema - Target JSON schema
+ * @param {string|null} previousError - Error from previous attempt (for feedback)
+ * @returns {string} The prompt for the reformatting model
+ */
+function buildReformatPrompt(rawOutput, schema, previousError = null) {
+  const schemaStr = JSON.stringify(schema, null, 2);
+  // Truncate long outputs to avoid context limits
+  const truncatedOutput = rawOutput.length > 4000 ? rawOutput.slice(-4000) : rawOutput;
+
+  let prompt = `Convert this text into a JSON object matching the schema.
+
+## SCHEMA
+\`\`\`json
+${schemaStr}
+\`\`\`
+
+## TEXT TO CONVERT
+\`\`\`
+${truncatedOutput}
+\`\`\`
+
+## RULES
+- Output ONLY the JSON object
+- NO markdown code blocks
+- NO explanations
+- Start with { end with }
+- Match ALL required fields from schema`;
+
+  if (previousError) {
+    prompt += `
+
+## PREVIOUS ATTEMPT FAILED
+Error: ${previousError}
+Fix this issue in your response.`;
+  }
+
+  return prompt;
+}
+
+/**
+ * Attempt to reformat non-JSON output into valid JSON
+ *
+ * STATUS: SDK NOT IMPLEMENTED - This function always throws.
+ * When SDK support is added to providers, this will work automatically.
+ *
+ * @param {Object} options
+ * @param {string} options.rawOutput - The non-JSON output to reformat
+ * @param {Object} options.schema - Target JSON schema
+ * @param {string} options.providerName - Provider name (claude, codex, gemini, opencode)
+ * @param {number} [options.maxAttempts=3] - Maximum reformatting attempts
+ * @param {Function} [options.onAttempt] - Callback for each attempt (attempt, error)
+ * @returns {Promise<Object>} The reformatted JSON object
+ * @throws {Error} Always throws - SDK not implemented
+ */
+function reformatOutput({
+  rawOutput,
+  schema: _schema,
+  providerName,
+  maxAttempts: _maxAttempts = DEFAULT_MAX_ATTEMPTS,
+  onAttempt: _onAttempt,
+}) {
+  // SDK not implemented - reformatting not available
+  // When SDK support is added, uncomment the implementation below
+  return Promise.reject(
+    new Error(
+      `Output reformatting not available: SDK not implemented for provider "${providerName}". ` +
+        `Agent output must be valid JSON. Raw output (last 200 chars): ${(rawOutput || '').slice(-200)}`
+    )
+  );
+
+  // FUTURE: When SDK support is added to providers, uncomment this:
+  /*
+  const { getProvider } = require('../providers');
+  const provider = getProvider(providerName);
+
+  let lastError = null;
+
+  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+    if (onAttempt) {
+      onAttempt(attempt, lastError);
+    }
+
+    const prompt = buildReformatPrompt(rawOutput, schema, lastError);
+
+    try {
+      const result = await provider.callSimple(prompt, {
+        level: 'level1',
+        maxTokens: 2000,
+      });
+
+      if (!result?.success) {
+        lastError = result?.error || 'API call failed';
+        continue;
+      }
+
+      if (!result?.text) {
+        lastError = 'Empty response from reformatting model';
+        continue;
+      }
+
+      const parsed = extractJsonFromOutput(result.text, providerName);
+
+      if (!parsed) {
+        lastError = 'Could not extract JSON from reformatted output';
+        continue;
+      }
+
+      const validationError = validateAgainstSchema(parsed, schema);
+      if (validationError) {
+        lastError = validationError;
+        continue;
+      }
+
+      return parsed;
+    } catch (err) {
+      lastError = err.message;
+    }
+  }
+
+  throw new Error(
+    `Failed to reformat output after ${maxAttempts} attempts. Last error: ${lastError}`
+  );
+  */
+}
+
+/**
+ * Validate parsed output against JSON schema
+ *
+ * @param {Object} parsed - Parsed JSON object
+ * @param {Object} schema - JSON schema to validate against
+ * @returns {string|null} Error message if validation failed, null if valid
+ */
+function validateAgainstSchema(parsed, schema) {
+  const Ajv = require('ajv');
+  const ajv = new Ajv({ allErrors: true, strict: false });
+  const validate = ajv.compile(schema);
+  const valid = validate(parsed);
+
+  if (!valid) {
+    const errors = (validate.errors || [])
+      .slice(0, 3)
+      .map((e) => `${e.instancePath || '#'} ${e.message}`)
+      .join('; ');
+    return errors || 'Schema validation failed';
+  }
+
+  return null;
+}
+
+module.exports = {
+  reformatOutput,
+  buildReformatPrompt,
+  validateAgainstSchema,
+  DEFAULT_MAX_ATTEMPTS,
+};

package/src/agent/schema-utils.js

@@ -0,0 +1,146 @@
+/**
+ * Schema utilities for normalizing LLM output before validation.
+ *
+ * PROBLEM: LLMs (Claude, Gemini, Codex) via any interface (CLI, API) may return
+ * enum values that don't exactly match the schema (e.g., "simple" vs "SIMPLE").
+ *
+ * SOLUTION: Normalize enum values BEFORE validation. Provider-agnostic.
+ */
+
+const ENUM_VARIATIONS = {
+  // taskType variations
+  BUG: 'DEBUG',
+  FIX: 'DEBUG',
+  BUGFIX: 'DEBUG',
+  BUG_FIX: 'DEBUG',
+  INVESTIGATE: 'DEBUG',
+  TROUBLESHOOT: 'DEBUG',
+  IMPLEMENT: 'TASK',
+  BUILD: 'TASK',
+  CREATE: 'TASK',
+  ADD: 'TASK',
+  FEATURE: 'TASK',
+  QUESTION: 'INQUIRY',
+  ASK: 'INQUIRY',
+  EXPLORE: 'INQUIRY',
+  RESEARCH: 'INQUIRY',
+  UNDERSTAND: 'INQUIRY',
+  // complexity variations
+  EASY: 'TRIVIAL',
+  BASIC: 'SIMPLE',
+  MINOR: 'SIMPLE',
+  MODERATE: 'STANDARD',
+  MEDIUM: 'STANDARD',
+  NORMAL: 'STANDARD',
+  HARD: 'STANDARD',
+  COMPLEX: 'CRITICAL',
+  RISKY: 'CRITICAL',
+  HIGH_RISK: 'CRITICAL',
+  DANGEROUS: 'CRITICAL',
+};
+
+/**
+ * Normalize enum values in parsed JSON to match schema definitions.
+ *
+ * Handles:
+ * - Case mismatches: "simple" → "SIMPLE"
+ * - Whitespace: " SIMPLE " → "SIMPLE"
+ * - Common variations: "bug" → "DEBUG", "fix" → "DEBUG"
+ *
+ * @param {Object} result - Parsed JSON result from LLM
+ * @param {Object} schema - JSON schema with enum definitions
+ * @returns {Object} Normalized result (mutates and returns same object)
+ */
+function normalizeEnumValues(result, schema) {
+  if (!result || typeof result !== 'object' || !schema?.properties) {
+    return result;
+  }
+
+  for (const [key, propSchema] of Object.entries(schema.properties)) {
+    const matched = normalizeEnumValue({
+      result,
+      key,
+      propSchema,
+    });
+    if (matched) {
+      continue;
+    }
+
+    // Recursively handle nested objects
+    normalizeNestedValues(result, propSchema, key);
+  }
+
+  return result;
+}
+
+function normalizeEnumValue({ result, key, propSchema }) {
+  if (!propSchema.enum || typeof result[key] !== 'string') {
+    return false;
+  }
+
+  const rawValue = result[key];
+  let value = rawValue.trim().toUpperCase();
+  value = normalizeEnumCopyValue(value, propSchema.enum, key, rawValue);
+
+  // Find exact match (case-insensitive)
+  const match = findEnumMatch(propSchema.enum, value);
+  if (match) {
+    result[key] = match;
+    return true;
+  }
+
+  // Common variations mapping
+  const variation = ENUM_VARIATIONS[value];
+  if (variation && propSchema.enum.includes(variation)) {
+    result[key] = variation;
+  }
+
+  return false;
+}
+
+function normalizeEnumCopyValue(value, enumValues, key, rawValue) {
+  // DETECT: Model copied the enum list instead of choosing (e.g., "TRIVIAL|SIMPLE|STANDARD")
+  if (!value.includes('|')) {
+    return value;
+  }
+
+  const parts = value.split('|').map((p) => p.trim());
+  // Check if this looks like the enum list was copied verbatim
+  const matchCount = parts.filter((p) => enumValues.includes(p)).length;
+  if (matchCount < 2) {
+    return value;
+  }
+
+  // Model copied the format - pick the first valid option and warn
+  const firstValid = parts.find((p) => enumValues.includes(p));
+  if (!firstValid) {
+    return value;
+  }
+
+  console.warn(
+    `⚠️  Model copied enum format instead of choosing. Field "${key}" had "${rawValue}", using "${firstValid}"`
+  );
+  return firstValid;
+}
+
+function findEnumMatch(enumValues, value) {
+  return enumValues.find((entry) => entry.toUpperCase() === value);
+}
+
+function normalizeNestedValues(result, propSchema, key) {
+  // Recursively handle nested objects
+  if (propSchema.type === 'object' && propSchema.properties && result[key]) {
+    normalizeEnumValues(result[key], propSchema);
+  }
+
+  // Handle arrays of objects
+  if (propSchema.type === 'array' && propSchema.items?.properties && Array.isArray(result[key])) {
+    for (const item of result[key]) {
+      normalizeEnumValues(item, propSchema.items);
+    }
+  }
+}
+
+module.exports = {
+  normalizeEnumValues,
+};