cognitive-modules-cli 2.2.11 → 2.2.12

package/dist/modules/runner.js

@@ -9,10 +9,37 @@ import * as fs from 'node:fs/promises';
 import * as path from 'node:path';
 import { aggregateRisk, isV22Envelope } from '../types.js';
 import { readModuleProvenance, verifyModuleIntegrity } from '../provenance.js';
+import { extractJsonCandidates } from './json-extract.js';
+import { compactReason, formatPolicySummaryLine } from '../policy-summary.js';
 // =============================================================================
 // Schema Validation
 // =============================================================================
 const ajv = new Ajv({ allErrors: true, strict: false });
+function safeSnippet(s, max = 500) {
+    const raw = String(s ?? '');
+    if (raw.length <= max)
+        return raw;
+    return raw.slice(0, max) + `…(+${raw.length - max} chars)`;
+}
+function parseJsonWithCandidates(raw) {
+    const candidates = extractJsonCandidates(raw);
+    const attempts = [];
+    for (const c of candidates) {
+        try {
+            const parsed = JSON.parse(c.json.trim());
+            return { parsed, extracted: c, attempts };
+        }
+        catch (e) {
+            attempts.push({ strategy: c.strategy, error: e.message });
+        }
+    }
+    const err = new Error(attempts[0]?.error ?? 'Unable to parse JSON');
+    err.details = {
+        parse_attempts: attempts,
+        raw_response_snippet: safeSnippet(raw, 500),
+    };
+    throw err;
+}
 /**
  * Validate data against JSON schema. Returns list of errors.
  */
@@ -1061,7 +1088,7 @@ async function enforcePolicyGates(module, policy) {
                 explain: 'Refused by execution policy.',
                 confidence: 1.0,
                 risk: 'none',
-                suggestion: 'Migrate the module to v2.2, or run with --profile strict/default.',
+                suggestion: 'Migrate the module to v2.2, or rerun with --profile standard.',
             });
         }
     }
@@ -1088,7 +1115,7 @@ async function enforcePolicyGates(module, policy) {
                 explain: 'Refused by execution policy.',
                 confidence: 1.0,
                 risk: 'none',
-                suggestion: 'Install the module via `cog add <name>` (registry tarball) or use --profile strict/default.',
+                suggestion: 'Install the module via `cog add <name>` (registry tarball) or rerun with --profile standard.',
             });
         }
     }
@@ -1120,7 +1147,7 @@ async function enforcePolicyGates(module, policy) {
             explain: 'Refused by execution policy.',
             confidence: 1.0,
             risk: 'none',
-            suggestion: 'Reinstall the module from a registry tarball and retry, or use --profile strict/default.',
+            suggestion: 'Reinstall the module from a registry tarball and retry, or rerun with --profile standard.',
         });
     }
     // Integrity check (tamper detection).
@@ -1142,11 +1169,209 @@ async function enforcePolicyGates(module, policy) {
         return null;
     return null;
 }
+function resolveValidationFlags(module, policy, validateInputOpt, validateOutputOpt) {
+    // Explicit overrides win.
+    if (typeof validateInputOpt === 'boolean' || typeof validateOutputOpt === 'boolean') {
+        const validateInput = typeof validateInputOpt === 'boolean' ? validateInputOpt : true;
+        const validateOutput = typeof validateOutputOpt === 'boolean' ? validateOutputOpt : true;
+        return { validateInput, validateOutput, reason: 'explicit validateInput/validateOutput' };
+    }
+    if (!policy) {
+        return { validateInput: true, validateOutput: true, reason: 'no policy (default on)' };
+    }
+    if (policy.validate === 'off') {
+        return { validateInput: false, validateOutput: false, reason: 'policy.validate=off' };
+    }
+    if (policy.validate === 'on') {
+        return { validateInput: true, validateOutput: true, reason: 'policy.validate=on' };
+    }
+    // auto: decide based on module intent.
+    const tier = module.tier ?? 'decision';
+    const strictness = module.schemaStrictness ?? 'medium';
+    // Certified flows already set policy.validate=on in resolveExecutionPolicy().
+    // Keep auto conservative for exec/decision.
+    // For exploration: do not validate inputs by default, but still validate outputs post-hoc.
+    // This preserves "5-minute path" ergonomics while keeping envelopes structurally reliable.
+    if (strictness === 'high') {
+        return { validateInput: true, validateOutput: true, reason: `auto: schema_strictness=high (tier=${tier})` };
+    }
+    if (tier === 'exploration') {
+        return { validateInput: false, validateOutput: true, reason: 'auto: tier=exploration (post-hoc output only)' };
+    }
+    // exec + decision default to validation on.
+    return { validateInput: true, validateOutput: true, reason: `auto: tier=${tier}` };
+}
+function getProviderCapabilities(provider) {
+    const caps = provider.getCapabilities?.();
+    if (caps)
+        return caps;
+    return {
+        structuredOutput: 'prompt',
+        streaming: provider.supportsStreaming?.() ?? false,
+    };
+}
+function providerSupportsNativeStructuredOutput(caps) {
+    return caps.structuredOutput === 'native';
+}
+function providerSupportsNativeJsonSchema(caps) {
+    if (!providerSupportsNativeStructuredOutput(caps))
+        return false;
+    const dialect = caps.nativeSchemaDialect ?? 'json-schema';
+    return dialect === 'json-schema';
+}
+function schemaByteLength(schema) {
+    try {
+        return Buffer.byteLength(JSON.stringify(schema), 'utf8');
+    }
+    catch {
+        // If something is non-serializable, treat as huge and disable native.
+        return Number.MAX_SAFE_INTEGER;
+    }
+}
+function resolveJsonSchemaParams(module, provider, validateOutput, structured) {
+    if (!validateOutput)
+        return {};
+    if (!module.outputSchema)
+        return {};
+    const pref = structured ?? 'auto';
+    if (pref === 'off')
+        return { policy: { requested: pref, resolved: 'off', reason: 'structured=off' } };
+    if (pref === 'prompt') {
+        return { jsonSchema: module.outputSchema, jsonSchemaMode: 'prompt', allowSchemaFallback: false, policy: { requested: pref, resolved: 'prompt', reason: 'structured=prompt' } };
+    }
+    const caps = getProviderCapabilities(provider);
+    if (caps.structuredOutput === 'none')
+        return {};
+    if (pref === 'native') {
+        // "native" means "prefer native", not "fail hard".
+        // If the provider doesn't support native structured output at all, safely downgrade to prompt guidance.
+        //
+        // Important: "native" only covers JSON Schema-native providers. For providers whose schema dialect is
+        // not JSON Schema (e.g. Gemini responseSchema), we do NOT attempt to translate arbitrary JSON Schema
+        // into the provider dialect (too many subset incompatibilities).
+        //
+        // Instead, we safely downgrade to prompt-only JSON guidance and rely on post-validation. This avoids
+        // user-visible 400s and keeps the protocol contract stable.
+        if (!providerSupportsNativeStructuredOutput(caps)) {
+            const dialect = caps.nativeSchemaDialect ?? 'unknown';
+            return {
+                jsonSchema: module.outputSchema,
+                jsonSchemaMode: 'prompt',
+                allowSchemaFallback: false,
+                policy: {
+                    requested: pref,
+                    resolved: 'prompt',
+                    reason: `provider lacks native structured output (${caps.structuredOutput}); dialect=${dialect}`,
+                },
+            };
+        }
+        const dialect = caps.nativeSchemaDialect ?? 'json-schema';
+        if (dialect !== 'json-schema') {
+            return {
+                jsonSchema: module.outputSchema,
+                jsonSchemaMode: 'prompt',
+                allowSchemaFallback: false,
+                policy: {
+                    requested: pref,
+                    resolved: 'prompt',
+                    reason: `native schema dialect is not JSON Schema (${dialect}); using prompt-only guidance`,
+                },
+            };
+        }
+        const maxBytes = caps.maxNativeSchemaBytes;
+        if (typeof maxBytes === 'number' && maxBytes > 0) {
+            const bytes = schemaByteLength(module.outputSchema);
+            if (bytes > maxBytes) {
+                return {
+                    jsonSchema: module.outputSchema,
+                    jsonSchemaMode: 'prompt',
+                    allowSchemaFallback: false,
+                    policy: { requested: pref, resolved: 'prompt', reason: `native schema too large (${bytes}B > ${maxBytes}B)` },
+                };
+            }
+        }
+        // Some providers accept only a restricted schema subset and can reject otherwise-valid JSON Schema.
+        // Retrying once in prompt mode yields a much better UX while still keeping the initial attempt "native".
+        return {
+            jsonSchema: module.outputSchema,
+            jsonSchemaMode: 'native',
+            allowSchemaFallback: true,
+            policy: {
+                requested: pref,
+                resolved: 'native',
+                reason: 'structured=native',
+            },
+        };
+    }
+    // auto: choose based on provider capabilities.
+    let jsonSchemaMode = providerSupportsNativeJsonSchema(caps) ? 'native' : 'prompt';
+    let sizeReason = null;
+    if (jsonSchemaMode === 'native') {
+        const maxBytes = caps.maxNativeSchemaBytes;
+        if (typeof maxBytes === 'number' && maxBytes > 0) {
+            const bytes = schemaByteLength(module.outputSchema);
+            if (bytes > maxBytes) {
+                jsonSchemaMode = 'prompt';
+                sizeReason = `native schema too large (${bytes}B > ${maxBytes}B)`;
+            }
+        }
+    }
+    return {
+        jsonSchema: module.outputSchema,
+        jsonSchemaMode,
+        allowSchemaFallback: true,
+        policy: {
+            requested: pref,
+            resolved: jsonSchemaMode,
+            reason: sizeReason
+                ? `auto: ${sizeReason}`
+                : providerSupportsNativeJsonSchema(caps)
+                    ? `auto: native JSON Schema supported`
+                    : caps.structuredOutput === 'native'
+                        ? `auto: native schema dialect is not JSON Schema (${caps.nativeSchemaDialect ?? 'unknown'})`
+                        : `auto: provider structuredOutput=${caps.structuredOutput}`,
+        },
+    };
+}
+function isSchemaCompatibilityError(e) {
+    const msg = e instanceof Error ? e.message : String(e ?? '');
+    return (msg.includes('responseSchema') ||
+        msg.includes('response_schema') ||
+        msg.includes('Invalid JSON payload') ||
+        msg.includes('INVALID_ARGUMENT') ||
+        msg.includes('Unknown name') ||
+        msg.includes('should be non-empty for OBJECT type'));
+}
+function resolveStructuredSchemaPlan(module, provider, validateOutput, structuredPref, policy) {
+    // If output validation is disabled, never pass schema hints to providers.
+    if (!validateOutput)
+        return {};
+    if (!module.outputSchema)
+        return {};
+    const requested = structuredPref ?? 'auto';
+    // Progressive Complexity trigger:
+    // When validate is `auto` and tier=exploration, default to "post-hoc validation only":
+    // do not enforce/guide with schemas at the provider layer unless the user explicitly opts in.
+    const tier = module.tier ?? 'decision';
+    const strictness = module.schemaStrictness ?? 'medium';
+    if (requested === 'auto' && policy?.validate === 'auto' && tier === 'exploration' && strictness !== 'high') {
+        return {
+            policy: {
+                requested,
+                resolved: 'off',
+                reason: 'auto: tier=exploration defaults to post-hoc validation (no provider schema hints)',
+            },
+        };
+    }
+    return resolveJsonSchemaParams(module, provider, validateOutput, requested);
+}
 // =============================================================================
 // Main Runner
 // =============================================================================
 export async function runModule(module, provider, options = {}) {
-    const { args, input, verbose = false, validateInput = true, validateOutput = true, useEnvelope, useV22, enableRepair = true, traceId, model: modelOverride, policy } = options;
+    const { args, input, verbose = false, validateInput: validateInputOpt, validateOutput: validateOutputOpt, useEnvelope, useV22, enableRepair: enableRepairOpt, traceId, model: modelOverride, policy, structured: structuredOverride, } = options;
+    const { validateInput, validateOutput, reason: validateReason } = resolveValidationFlags(module, policy, validateInputOpt, validateOutputOpt);
+    const enableRepair = enableRepairOpt ?? policy?.enableRepair ?? true;
     const startTime = Date.now();
     const gate = await enforcePolicyGates(module, policy);
     if (gate) {
@@ -1204,11 +1429,34 @@ export async function runModule(module, provider, options = {}) {
     }
     // Build prompt with clean substitution
     const prompt = buildPrompt(module, inputData);
+    const effectiveStructuredPref = structuredOverride ?? policy?.structured;
+    const structuredPlan = resolveStructuredSchemaPlan(module, provider, validateOutput, effectiveStructuredPref, policy);
     if (verbose) {
         console.error('--- Module ---');
         console.error(`Name: ${module.name} (${module.format})`);
         console.error(`Responsibility: ${module.responsibility}`);
         console.error(`Envelope: ${shouldUseEnvelope}`);
+        if (policy) {
+            console.error('--- Policy ---');
+            const requested = effectiveStructuredPref ?? 'auto';
+            const applied = structuredPlan.jsonSchemaMode ?? 'off';
+            const sReason = structuredPlan.policy?.reason ??
+                (structuredPlan.jsonSchemaMode ? 'auto: schema hints enabled' : 'auto: schema hints disabled');
+            console.error(formatPolicySummaryLine(policy, { validateInput, validateOutput, reason: validateReason }, { requested, applied, reason: sReason }, { enableRepair, requireV22: policy.requireV22 }));
+            console.error(JSON.stringify({
+                profile: policy.profile,
+                validate: policy.validate,
+                validateInput,
+                validateOutput,
+                validate_reason: validateReason,
+                audit: policy.audit,
+                enableRepair,
+                structured: requested,
+                structured_effective: applied,
+                structured_reason: structuredPlan.policy?.reason ?? null,
+                requireV22: policy.requireV22,
+            }, null, 2));
+        }
         console.error('--- Input ---');
         console.error(JSON.stringify(inputData, null, 2));
         console.error('--- Prompt ---');
@@ -1278,11 +1526,58 @@ export async function runModule(module, provider, options = {}) {
     ];
     try {
         // Invoke provider
-        const result = await provider.invoke({
-            messages,
-            jsonSchema: module.outputSchema,
-            temperature: 0.3,
-        });
+        const { allowSchemaFallback, policy: structuredPolicy, ...invokeSchemaParams } = structuredPlan;
+        const invokeParams = { ...invokeSchemaParams };
+        const capsSnapshot = getProviderCapabilities(provider);
+        const plannedSchemaMode = invokeParams.jsonSchema && typeof invokeParams.jsonSchema === 'object'
+            ? (invokeParams.jsonSchemaMode ?? 'prompt')
+            : 'off';
+        let appliedSchemaMode = plannedSchemaMode;
+        let schemaFallbackAttempted = false;
+        let schemaFallbackReason = null;
+        let result;
+        try {
+            result = await provider.invoke({
+                messages,
+                // Progressive Complexity: only enforce schema at the provider layer when validation is enabled.
+                ...invokeParams,
+                temperature: 0.3,
+            });
+        }
+        catch (e) {
+            // If the provider rejects native structured output schemas, retry once in prompt mode.
+            if (allowSchemaFallback &&
+                invokeParams.jsonSchema &&
+                invokeParams.jsonSchemaMode === 'native' &&
+                isSchemaCompatibilityError(e)) {
+                schemaFallbackAttempted = true;
+                schemaFallbackReason = compactReason(e instanceof Error ? e.message : String(e ?? ''), 180);
+                invokeParams.jsonSchemaMode = 'prompt';
+                appliedSchemaMode = 'prompt';
+                result = await provider.invoke({
+                    messages,
+                    ...invokeParams,
+                    temperature: 0.3,
+                });
+            }
+            else {
+                throw e;
+            }
+        }
+        const structuredPolicyMeta = structuredPolicy
+            ? {
+                ...structuredPolicy,
+                planned: structuredPolicy.resolved,
+                applied: appliedSchemaMode,
+                downgraded: appliedSchemaMode !== structuredPolicy.resolved,
+                fallback: schemaFallbackAttempted ? { attempted: true, reason: schemaFallbackReason ?? undefined } : { attempted: false },
+                provider: {
+                    structuredOutput: capsSnapshot.structuredOutput,
+                    nativeSchemaDialect: capsSnapshot.nativeSchemaDialect,
+                    maxNativeSchemaBytes: capsSnapshot.maxNativeSchemaBytes,
+                },
+            }
+            : undefined;
         if (verbose) {
             console.error('--- Response ---');
             console.error(result.content);
@@ -1292,21 +1587,87 @@ export async function runModule(module, provider, options = {}) {
         const latencyMs = Date.now() - startTime;
         // Parse response
         let parsed;
+        let parseExtracted = null;
+        let parseAttempts = [];
+        let parseRetries = 0;
         try {
-            const jsonMatch = result.content.match(/```(?:json)?\s*([\s\S]*?)\s*```/);
-            const jsonStr = jsonMatch ? jsonMatch[1] : result.content;
-            parsed = JSON.parse(jsonStr.trim());
+            const r = parseJsonWithCandidates(result.content);
+            parsed = r.parsed;
+            parseExtracted = r.extracted;
+            parseAttempts = r.attempts;
         }
         catch (e) {
-            const errorResult = makeErrorResponse({
-                code: 'E1000', // PARSE_ERROR
-                message: `Failed to parse JSON response: ${e.message}`,
-                explain: 'Failed to parse LLM response as JSON.',
-                details: { raw_response: result.content.substring(0, 500) },
-                suggestion: 'The LLM response was not valid JSON. Try again or adjust the prompt.',
-            });
-            _invokeErrorHooks(module.name, e, null);
-            return errorResult;
+            const allowParseRetry = policy?.profile !== 'certified';
+            const firstDetails = e?.details;
+            if (firstDetails && typeof firstDetails === 'object' && Array.isArray(firstDetails.parse_attempts)) {
+                parseAttempts = firstDetails.parse_attempts;
+            }
+            if (!allowParseRetry) {
+                const details = typeof firstDetails === 'object' && firstDetails
+                    ? firstDetails
+                    : { raw_response_snippet: safeSnippet(result.content, 500) };
+                const errorResult = makeErrorResponse({
+                    code: 'E1000', // PARSE_ERROR
+                    message: `Failed to parse JSON response: ${e.message}`,
+                    explain: 'Failed to parse LLM response as JSON.',
+                    details: {
+                        ...details,
+                        parse_retry: { attempted: false, reason: 'profile=certified (fail-fast)' },
+                    },
+                    suggestion: 'The LLM response was not valid JSON. Fix the module/provider output or switch provider.',
+                });
+                _invokeErrorHooks(module.name, e, null);
+                return errorResult;
+            }
+            // Retry once with stronger formatting instructions (prompt-only enforcement).
+            parseRetries = 1;
+            const retryMessages = [
+                ...messages,
+                {
+                    role: 'user',
+                    content: 'Your previous response was not valid JSON.\n\nReturn ONLY a single valid JSON value (no markdown, no code fences, no commentary, no trailing text).',
+                },
+            ];
+            try {
+                const retryResult = await provider.invoke({
+                    messages: retryMessages,
+                    ...invokeParams,
+                    temperature: 0.3,
+                });
+                if (verbose) {
+                    console.error('--- Response (retry) ---');
+                    console.error(retryResult.content);
+                    console.error('--- End Response (retry) ---');
+                }
+                const r2 = parseJsonWithCandidates(retryResult.content);
+                parsed = r2.parsed;
+                parseExtracted = r2.extracted;
+                parseAttempts = [...parseAttempts, ...r2.attempts];
+            }
+            catch (e2) {
+                const combinedAttempts = [
+                    ...parseAttempts,
+                    ...((typeof e2?.details === 'object' && e2?.details && Array.isArray(e2.details.parse_attempts))
+                        ? e2.details.parse_attempts
+                        : []),
+                ];
+                const details = typeof e2?.details === 'object' && e2?.details
+                    ? e2.details
+                    : { raw_response_snippet: safeSnippet(result.content, 500) };
+                const errorResult = makeErrorResponse({
+                    code: 'E1000', // PARSE_ERROR
+                    message: `Failed to parse JSON response: ${e2.message}`,
+                    explain: 'Failed to parse LLM response as JSON.',
+                    details: {
+                        ...details,
+                        parse_attempts: combinedAttempts.length ? combinedAttempts : undefined,
+                        parse_retry: { attempted: true, count: 1 },
+                    },
+                    suggestion: 'The LLM response was not valid JSON. Try again or adjust the prompt.',
+                });
+                _invokeErrorHooks(module.name, e2, null);
+                return errorResult;
+            }
         }
         // Convert to v2.2 envelope
         let response;
@@ -1323,6 +1684,37 @@ export async function runModule(module, provider, options = {}) {
         response.version = ENVELOPE_VERSION;
         if (response.meta) {
             response.meta.latency_ms = latencyMs;
+            if (structuredPolicyMeta) {
+                // Publish-grade parity: record how structured output was applied for this run.
+                // This is intentionally small and stable, so users can reason about provider differences.
+                response.meta.policy = {
+                    ...(typeof response.meta.policy === 'object' && response.meta.policy ? response.meta.policy : {}),
+                    structured: structuredPolicyMeta,
+                };
+            }
+            if (policy) {
+                response.meta.policy = {
+                    ...(typeof response.meta.policy === 'object' && response.meta.policy ? response.meta.policy : {}),
+                    validation: {
+                        mode: policy.validate,
+                        input: validateInput,
+                        output: validateOutput,
+                        reason: validateReason,
+                    },
+                    audit: { enabled: policy.audit === true },
+                    repair: { enabled: enableRepair === true },
+                };
+            }
+            // Record parse strategy and retry count for publish-grade diagnostics.
+            const includeParseAttempts = verbose || policy?.profile !== 'core';
+            response.meta.policy = {
+                ...(typeof response.meta.policy === 'object' && response.meta.policy ? response.meta.policy : {}),
+                parse: {
+                    strategy: parseExtracted?.strategy ?? null,
+                    retries: parseRetries,
+                    attempts: includeParseAttempts && parseAttempts.length ? parseAttempts : undefined,
+                },
+            };
             if (traceId) {
                 response.meta.trace_id = traceId;
             }
@@ -1451,7 +1843,9 @@ export async function runModule(module, provider, options = {}) {
  * }
  */
 export async function* runModuleStream(module, provider, options = {}) {
-    const { input, args, validateInput = true, validateOutput = true, useV22 = true, enableRepair = true, traceId, model, policy } = options;
+    const { input, args, validateInput: validateInputOpt, validateOutput: validateOutputOpt, useV22 = true, enableRepair: enableRepairOpt, traceId, model, policy, structured: structuredOverride, } = options;
+    const { validateInput, validateOutput, reason: validateReason } = resolveValidationFlags(module, policy, validateInputOpt, validateOutputOpt);
+    const enableRepair = enableRepairOpt ?? policy?.enableRepair ?? true;
     const startTime = Date.now();
     const moduleName = module.name;
     const providerName = provider?.name;
@@ -1485,6 +1879,13 @@ export async function* runModuleStream(module, provider, options = {}) {
                 inputData.query = args;
             }
         }
+        // Single-file core modules promise "missing fields are empty".
+        if (typeof module.location === 'string' && /\.(md|markdown)$/i.test(module.location)) {
+            if (inputData.query === undefined)
+                inputData.query = '';
+            if (inputData.code === undefined)
+                inputData.code = '';
+        }
         _invokeBeforeHooks(module.name, inputData, module);
         // Validate input if enabled
         if (validateInput && module.inputSchema && Object.keys(module.inputSchema).length > 0) {
@@ -1508,6 +1909,8 @@ export async function* runModuleStream(module, provider, options = {}) {
         const riskRule = module.metaConfig?.risk_rule ?? 'max_changes_risk';
         // Build prompt
         const prompt = buildPrompt(module, inputData);
+        const effectiveStructuredPref = structuredOverride ?? policy?.structured;
+        const structuredPlan = resolveStructuredSchemaPlan(module, provider, validateOutput, effectiveStructuredPref, policy);
         // Build messages
         const systemParts = [
             `You are executing the "${module.name}" Cognitive Module.`,
@@ -1526,52 +1929,188 @@ export async function* runModuleStream(module, provider, options = {}) {
         ];
         // Invoke provider with streaming if supported
         let fullContent;
+        const { allowSchemaFallback, policy: structuredPolicy, ...invokeSchemaParams } = structuredPlan;
+        const invokeParams = { ...invokeSchemaParams };
+        const capsSnapshot = getProviderCapabilities(provider);
+        const plannedSchemaMode = invokeParams.jsonSchema && typeof invokeParams.jsonSchema === 'object'
+            ? (invokeParams.jsonSchemaMode ?? 'prompt')
+            : 'off';
+        let appliedSchemaMode = plannedSchemaMode;
+        let schemaFallbackAttempted = false;
+        let schemaFallbackReason = null;
         if (provider.supportsStreaming?.() && provider.invokeStream) {
             // Use true streaming
             const stream = provider.invokeStream({
                 messages,
-                jsonSchema: module.outputSchema,
+                ...invokeParams,
                 temperature: 0.3,
             });
             // Iterate through the async generator, yielding chunks as they arrive
             let streamResult;
-            while (!(streamResult = await stream.next()).done) {
-                const chunk = streamResult.value;
-                yield makeEvent('delta', { delta: chunk });
+            try {
+                while (!(streamResult = await stream.next()).done) {
+                    const chunk = streamResult.value;
+                    yield makeEvent('delta', { delta: chunk });
+                }
+                // Get the final result (returned from the generator)
+                fullContent = streamResult.value.content;
+            }
+            catch (e) {
+                // If streaming fails (e.g., schema rejected), retry once non-streaming in prompt mode.
+                if (allowSchemaFallback &&
+                    invokeParams.jsonSchema &&
+                    invokeParams.jsonSchemaMode === 'native' &&
+                    isSchemaCompatibilityError(e)) {
+                    schemaFallbackAttempted = true;
+                    schemaFallbackReason = compactReason(e instanceof Error ? e.message : String(e ?? ''), 180);
+                    invokeParams.jsonSchemaMode = 'prompt';
+                    appliedSchemaMode = 'prompt';
+                    const result = await provider.invoke({
+                        messages,
+                        ...invokeParams,
+                        temperature: 0.3,
+                    });
+                    fullContent = result.content;
+                    yield makeEvent('delta', { delta: result.content });
+                }
+                else {
+                    throw e;
+                }
             }
-            // Get the final result (returned from the generator)
-            fullContent = streamResult.value.content;
         }
         else {
             // Fallback to non-streaming invoke
-            const result = await provider.invoke({
-                messages,
-                jsonSchema: module.outputSchema,
-                temperature: 0.3,
-            });
+            let result;
+            try {
+                result = await provider.invoke({
+                    messages,
+                    ...invokeParams,
+                    temperature: 0.3,
+                });
+            }
+            catch (e) {
+                if (allowSchemaFallback &&
+                    invokeParams.jsonSchema &&
+                    invokeParams.jsonSchemaMode === 'native' &&
+                    isSchemaCompatibilityError(e)) {
+                    schemaFallbackAttempted = true;
+                    schemaFallbackReason = compactReason(e instanceof Error ? e.message : String(e ?? ''), 180);
+                    invokeParams.jsonSchemaMode = 'prompt';
+                    appliedSchemaMode = 'prompt';
+                    result = await provider.invoke({
+                        messages,
+                        ...invokeParams,
+                        temperature: 0.3,
+                    });
+                }
+                else {
+                    throw e;
+                }
+            }
             fullContent = result.content;
             // Emit chunk event with full response
             yield makeEvent('delta', { delta: result.content });
         }
+        const structuredPolicyMeta = structuredPolicy
+            ? {
+                ...structuredPolicy,
+                planned: structuredPolicy.resolved,
+                applied: appliedSchemaMode,
+                downgraded: appliedSchemaMode !== structuredPolicy.resolved,
+                fallback: schemaFallbackAttempted ? { attempted: true, reason: schemaFallbackReason ?? undefined } : { attempted: false },
+                provider: {
+                    structuredOutput: capsSnapshot.structuredOutput,
+                    nativeSchemaDialect: capsSnapshot.nativeSchemaDialect,
+                    maxNativeSchemaBytes: capsSnapshot.maxNativeSchemaBytes,
+                },
+            }
+            : undefined;
         // Parse response
         let parsed;
+        let parseExtracted = null;
+        let parseAttempts = [];
+        let parseRetries = 0;
         try {
-            const jsonMatch = fullContent.match(/```(?:json)?\s*([\s\S]*?)\s*```/);
-            const jsonStr = jsonMatch ? jsonMatch[1] : fullContent;
-            parsed = JSON.parse(jsonStr.trim());
+            const r = parseJsonWithCandidates(fullContent);
+            parsed = r.parsed;
+            parseExtracted = r.extracted;
+            parseAttempts = r.attempts;
         }
         catch (e) {
-            const errorResult = makeErrorResponse({
-                code: 'E1000', // PARSE_ERROR
-                message: `Failed to parse JSON: ${e.message}`,
-                suggestion: 'The LLM response was not valid JSON. Try again or adjust the prompt.',
-            });
-            _invokeErrorHooks(module.name, e, null);
-            // errorResult is always an error response from makeErrorResponse
-            const errorObj = errorResult.error;
-            yield makeEvent('error', { error: errorObj });
-            yield makeEvent('end', { result: errorResult });
-            return;
+            const firstDetails = e?.details;
+            if (firstDetails && typeof firstDetails === 'object' && Array.isArray(firstDetails.parse_attempts)) {
+                parseAttempts = firstDetails.parse_attempts;
+            }
+            const allowParseRetry = policy?.profile !== 'certified';
+            if (!allowParseRetry) {
+                const details = typeof firstDetails === 'object' && firstDetails
+                    ? firstDetails
+                    : { raw_response_snippet: safeSnippet(fullContent, 500) };
+                const errorResult = makeErrorResponse({
+                    code: 'E1000', // PARSE_ERROR
+                    message: `Failed to parse JSON: ${e.message}`,
+                    details: {
+                        ...details,
+                        parse_retry: { attempted: false, reason: 'profile=certified (fail-fast)' },
+                    },
+                    suggestion: 'The LLM response was not valid JSON. Fix the module/provider output or switch provider.',
+                });
+                _invokeErrorHooks(module.name, e, null);
+                const errorObj = errorResult.error;
+                yield makeEvent('error', { error: errorObj });
+                yield makeEvent('end', { result: errorResult });
+                return;
+            }
+            // Retry once (non-streaming) with stronger JSON-only instructions.
+            parseRetries = 1;
+            const retryMessages = [
+                ...messages,
+                {
+                    role: 'user',
+                    content: 'Your previous response was not valid JSON.\n\nReturn ONLY a single valid JSON value (no markdown, no code fences, no commentary, no trailing text).',
+                },
+            ];
+            try {
+                const retryResult = await provider.invoke({
+                    messages: retryMessages,
+                    ...invokeParams,
+                    temperature: 0.3,
+                });
+                fullContent = retryResult.content; // do not emit delta for retry; clients should rely on final envelope
+                const r2 = parseJsonWithCandidates(fullContent);
+                parsed = r2.parsed;
+                parseExtracted = r2.extracted;
+                parseAttempts = [...parseAttempts, ...r2.attempts];
+            }
+            catch (e2) {
+                const combinedAttempts = [
+                    ...parseAttempts,
+                    ...((typeof e2?.details === 'object' &&
+                        e2?.details &&
+                        Array.isArray(e2.details.parse_attempts))
+                        ? e2.details.parse_attempts
+                        : []),
+                ];
+                const details = typeof e2?.details === 'object' && e2?.details
+                    ? e2.details
+                    : { raw_response_snippet: safeSnippet(fullContent, 500) };
+                const errorResult = makeErrorResponse({
+                    code: 'E1000', // PARSE_ERROR
+                    message: `Failed to parse JSON: ${e2.message}`,
+                    details: {
+                        ...details,
+                        parse_attempts: combinedAttempts.length ? combinedAttempts : undefined,
+                        parse_retry: { attempted: true, count: 1 },
+                    },
+                    suggestion: 'The LLM response was not valid JSON. Try again or adjust the prompt.',
+                });
+                _invokeErrorHooks(module.name, e2, null);
+                // errorResult is always an error response from makeErrorResponse
+                const errorObj = errorResult.error;
+                yield makeEvent('error', { error: errorObj });
+                yield makeEvent('end', { result: errorResult });
+                return;
+            }
         }
         // Convert to v2.2 envelope
         let response;
@@ -1589,6 +2128,34 @@ export async function* runModuleStream(module, provider, options = {}) {
         const latencyMs = Date.now() - startTime;
         if (response.meta) {
             response.meta.latency_ms = latencyMs;
+            if (structuredPolicyMeta) {
+                response.meta.policy = {
+                    ...(typeof response.meta.policy === 'object' && response.meta.policy ? response.meta.policy : {}),
+                    structured: structuredPolicyMeta,
+                };
+            }
+            if (policy) {
+                response.meta.policy = {
+                    ...(typeof response.meta.policy === 'object' && response.meta.policy ? response.meta.policy : {}),
+                    validation: {
+                        mode: policy.validate,
+                        input: validateInput,
+                        output: validateOutput,
+                        reason: validateReason,
+                    },
+                    audit: { enabled: policy.audit === true },
+                    repair: { enabled: enableRepair === true },
+                };
+            }
+            const includeParseAttempts = policy?.profile !== 'core';
+            response.meta.policy = {
+                ...(typeof response.meta.policy === 'object' && response.meta.policy ? response.meta.policy : {}),
+                parse: {
+                    strategy: parseExtracted?.strategy ?? null,
+                    retries: parseRetries,
+                    attempts: includeParseAttempts && parseAttempts.length ? parseAttempts : undefined,
+                },
+            };
             if (traceId) {
                 response.meta.trace_id = traceId;
             }