@shipfox/api-definitions 11.0.0 → 12.2.0

package/dist/core/workflow-model/normalize-jobs.js

@@ -1,9 +1,9 @@
-import { buildTypedRootsEnvironment } from '@shipfox/expression';
+import { buildTypedRootsEnvironment, classifyShellCodePosition, hoistPlannedRunCommand, UnsafeRunInterpolationError } from '@shipfox/expression';
 import { canonicalizeLabels, findInvalidLabels, MAX_RUNNER_LABEL_LENGTH, MAX_RUNNER_LABELS, RUNNER_LABEL_PATTERN } from '@shipfox/runner-labels';
 import { normalizeAgentIntegrations } from './normalize-agent-integrations.js';
 import { normalizeEnv } from './normalize-env.js';
 import { normalizeIfCondition } from './normalize-if-condition.js';
-import { normalizeJobCheckout } from './normalize-job-checkout.js';
+import { normalizeCheckout, normalizeJobCheckout } from './normalize-job-checkout.js';
 import { normalizeJobListening } from './normalize-job-listening.js';
 import { normalizeJobSuccess } from './normalize-job-success.js';
 import { normalizeNeeds } from './normalize-needs.js';
@@ -12,6 +12,7 @@ import { normalizeStepOutputs } from './normalize-step-outputs.js';
 import { parseDurationMs } from './parse-duration-ms.js';
 import { parseInterpolationField } from './parse-interpolation-field.js';
 import { stableId } from './stable-id.js';
+import { unescapeLiteralName, validateLiteralName } from './validate-literal-name.js';
 import { issue } from './validation-issue.js';
 export function normalizeJobs(document, jobIdBySourceName, issues, stepSourceLocations, context) {
     const entries = Object.entries(document.jobs);
@@ -109,6 +110,8 @@ function normalizeJob(params) {
         sourceName: params.sourceName,
         jobId: id,
         job: params.job,
+        workflowEnvKeys: Object.keys(params.document.env ?? {}),
+        jobEnvKeys: Object.keys(params.job.env ?? {}),
         issues: params.issues,
         stepSourceLocations: params.stepSourceLocations,
         fillSite: stepFillSite,
@@ -125,7 +128,9 @@ function normalizeJob(params) {
         issues: params.issues,
         defaultRunnerLabels: params.context.defaultRunnerLabels
     });
-    const checkout = normalizeJobCheckout(params.job.checkout);
+    const checkout = normalizeJobCheckout({
+        checkout: params.job.checkout
+    });
     const jobEnv = normalizeEnv({
         env: params.job.env,
         path: [
@@ -147,7 +152,6 @@ function normalizeJob(params) {
     const condition = normalizeIfCondition({
         field: 'job.if',
         source: params.job.if,
-        site: 'job-activation',
         path: [
             'jobs',
             params.sourceName,
@@ -185,21 +189,37 @@ function normalizeJob(params) {
         issues: params.issues,
         allowedJobReferences
     });
-    const name = params.job.name === undefined ? undefined : parseInterpolationField({
-        field: 'job.name',
-        source: params.job.name,
+    if (params.job.name !== undefined) {
+        validateLiteralName({
+            field: 'job.name',
+            dynamicField: 'execution_name',
+            source: params.job.name,
+            path: [
+                'jobs',
+                params.sourceName,
+                'name'
+            ],
+            message: 'Job name must be literal. Move runtime interpolation to execution_name.',
+            issues: params.issues
+        });
+    }
+    const name = params.job.name === undefined ? undefined : unescapeLiteralName(params.job.name);
+    const executionName = params.job.execution_name === undefined ? undefined : parseInterpolationField({
+        field: 'job.execution_name',
+        source: params.job.execution_name,
         path: [
             'jobs',
             params.sourceName,
-            'name'
+            'execution_name'
         ],
         issues: params.issues,
+        fillSite: 'execution-creation',
         allowedJobReferences,
         typeOverlay: upstreamJobsTypeOverlay
     }) ?? [
         {
             kind: 'literal',
-            value: params.job.name
+            value: params.job.execution_name
         }
     ];
     return {
@@ -232,6 +252,9 @@ function normalizeJob(params) {
         ...name === undefined ? {} : {
             name
         },
+        ...executionName === undefined ? {} : {
+            executionName
+        },
         ...jobEnv,
         dependencies,
         steps
@@ -326,53 +349,32 @@ function normalizeJobOutputs(params) {
             }
         ];
         templates[key] = template;
-        types[key] = inferJobOutputType({
-            sourceName: params.sourceName,
-            key,
-            source,
-            template,
-            issues: params.issues
-        });
+        types[key] = inferJobOutputType(template);
     }
     return {
         templates,
         types
     };
 }
-function inferJobOutputType(params) {
-    if (params.template.length !== 1) return 'string';
-    const [segment] = params.template;
+function inferJobOutputType(template) {
+    if (template.length !== 1) return 'string';
+    const [segment] = template;
     if (segment?.kind !== 'deferred') return 'string';
     const resultType = segment.expression.resultType;
     if (resultType === undefined) return 'string';
-    if (isScalarExpressionType(resultType)) return resultType;
-    params.issues.push(issue({
-        code: 'invalid-job-output',
-        message: `Job output "${params.key}" must resolve to a scalar value.`,
-        path: [
-            'jobs',
-            params.sourceName,
-            'outputs',
-            params.key
-        ],
-        details: {
-            output: params.key,
-            source: params.source
-        }
-    }));
-    return 'string';
-}
-function isScalarExpressionType(type) {
-    return type === 'string' || type === 'int' || type === 'double' || type === 'bool' || type === 'null' || type === 'timestamp';
+    return resultType;
 }
 function normalizeJobSteps(params) {
     const usedStepIds = new Map();
-    return params.job.steps.map((step, index)=>normalizeStep({
+    return params.job.steps.flatMap((step, index)=>{
+        const normalized = normalizeStep({
             step,
             index,
             sourceName: params.sourceName,
             jobId: params.jobId,
             allSteps: params.job.steps,
+            workflowEnvKeys: params.workflowEnvKeys,
+            jobEnvKeys: params.jobEnvKeys,
             usedStepIds,
             issues: params.issues,
             stepSourceLocations: params.stepSourceLocations,
@@ -382,7 +384,11 @@ function normalizeJobSteps(params) {
             upstreamJobs: params.upstreamJobs,
             directNeedJobs: params.directNeedJobs,
             context: params.context
-        }));
+        });
+        return normalized === undefined ? [] : [
+            normalized
+        ];
+    });
 }
 function normalizeStep(params) {
     const stepKey = params.step.key;
@@ -442,7 +448,6 @@ function normalizeStep(params) {
     const condition = normalizeIfCondition({
         field: 'step.if',
         source: params.step.if,
-        site: 'step-dispatch',
         path: [
             'jobs',
             params.sourceName,
@@ -484,6 +489,21 @@ function normalizeStep(params) {
         allowedJobReferences: params.allowedJobReferences,
         typeOverlay
     });
+    const workingDirectory = params.step.working_directory === undefined ? undefined : parseInterpolationField({
+        field: 'step.working_directory',
+        source: params.step.working_directory,
+        path: [
+            'jobs',
+            params.sourceName,
+            'steps',
+            params.index,
+            'working_directory'
+        ],
+        issues: params.issues,
+        fillSite: params.fillSite,
+        allowedJobReferences: params.allowedJobReferences,
+        typeOverlay
+    });
     const stepBase = {
         id: stepId,
         ...stepKey === undefined ? {} : {
@@ -492,6 +512,9 @@ function normalizeStep(params) {
         ...params.step.name === undefined ? {} : {
             name: params.step.name
         },
+        ...params.step.working_directory === undefined ? {} : {
+            workingDirectory: params.step.working_directory
+        },
         ...outputs === undefined ? {} : {
             outputs
         },
@@ -511,7 +534,10 @@ function normalizeStep(params) {
             stepBase,
             sourceName: params.sourceName,
             stepIndex: params.index,
+            workflowEnvKeys: params.workflowEnvKeys,
+            jobEnvKeys: params.jobEnvKeys,
             name,
+            workingDirectory,
             issues: params.issues,
             fillSite: params.fillSite,
             allowedJobReferences: params.allowedJobReferences,
@@ -525,6 +551,7 @@ function normalizeStep(params) {
             sourceName: params.sourceName,
             stepIndex: params.index,
             name,
+            workingDirectory,
             issues: params.issues,
             fillSite: params.fillSite,
             allowedJobReferences: params.allowedJobReferences,
@@ -532,9 +559,51 @@ function normalizeStep(params) {
             context: params.context
         });
     }
-    // workflowDocumentStepSchema requires either `run` or an agent `prompt`; this
-    // keeps the model-step union honest if callers bypass the document parser.
-    throw new Error(`Workflow step "${stepId}" is neither a run nor an agent step`);
+    if (params.step.checkout !== undefined) {
+        return normalizeCheckoutStep({
+            step: params.step,
+            stepBase,
+            name,
+            sourceName: params.sourceName,
+            stepIndex: params.index,
+            issues: params.issues,
+            fillSite: params.fillSite,
+            allowedJobReferences: params.allowedJobReferences,
+            typeOverlay
+        });
+    }
+    // Keep the model-step union honest if callers bypass the document parser.
+    throw new Error(`Workflow step "${stepId}" is neither a run, agent, nor checkout step`);
+}
+function normalizeCheckoutStep(params) {
+    const checkout = params.step.checkout;
+    if (checkout === undefined) {
+        throw new Error('Checkout step normalization requires checkout settings');
+    }
+    const normalizedCheckout = normalizeCheckout({
+        checkout,
+        issues: params.issues,
+        path: [
+            'jobs',
+            params.sourceName,
+            'steps',
+            params.stepIndex,
+            'checkout'
+        ],
+        fillSite: params.fillSite,
+        allowedJobReferences: params.allowedJobReferences,
+        typeOverlay: params.typeOverlay
+    });
+    return {
+        ...params.stepBase,
+        kind: 'checkout',
+        checkout: normalizedCheckout,
+        ...params.name === undefined ? {} : {
+            templates: {
+                name: params.name
+            }
+        }
+    };
 }
 function normalizeRunStep(params) {
     if (params.step.run === undefined) {
@@ -569,9 +638,25 @@ function normalizeRunStep(params) {
         allowedJobReferences: params.allowedJobReferences,
         typeOverlay: params.typeOverlay
     });
+    addReevaluatingCommandWarnings({
+        command: commandTemplate,
+        source: params.step.run,
+        workflowEnvKeys: params.workflowEnvKeys,
+        jobEnvKeys: params.jobEnvKeys,
+        stepEnvKeys: Object.keys(params.step.env ?? {}),
+        path: [
+            'jobs',
+            params.sourceName,
+            'steps',
+            params.stepIndex,
+            'run'
+        ],
+        issues: params.issues
+    });
     const templates = optionalRunStepTemplates({
         command: commandTemplate,
         name: params.name,
+        workingDirectory: params.workingDirectory,
         env: stepEnv.templates?.env
     });
     return {
@@ -589,6 +674,90 @@ function normalizeRunStep(params) {
         }
     };
 }
+function addReevaluatingCommandWarnings(params) {
+    const envKeys = new Set([
+        ...params.workflowEnvKeys,
+        ...params.jobEnvKeys,
+        ...params.stepEnvKeys
+    ]);
+    let hoisted;
+    try {
+        hoisted = hoistPlannedRunCommand({
+            field: {
+                segments: params.command ?? [
+                    {
+                        kind: 'literal',
+                        value: params.source
+                    }
+                ]
+            },
+            reservedNames: envKeys
+        });
+    } catch (error) {
+        if (!(error instanceof UnsafeRunInterpolationError)) return;
+        for (const occurrence of error.occurrences){
+            params.issues.push(issue({
+                code: 're-evaluating-command',
+                message: unsafeInterpolationWarningMessage(occurrence),
+                path: params.path,
+                severity: 'warning'
+            }));
+        }
+        if (error.partial === undefined) return;
+        hoisted = error.partial;
+    }
+    try {
+        const matches = classifyShellCodePosition({
+            command: hoisted.command,
+            workflowDataNames: [
+                ...envKeys,
+                ...hoisted.bindings.map((binding)=>binding.name)
+            ]
+        }).matches;
+        for (const match of matches){
+            params.issues.push(issue({
+                code: 're-evaluating-command',
+                message: reevaluatingCommandWarningMessage(match.construct, match.name),
+                path: params.path,
+                severity: 'warning'
+            }));
+        }
+    } catch  {
+        return;
+    }
+}
+function unsafeInterpolationWarningMessage(error) {
+    const regionLabel = {
+        single: 'single-quoted shell text',
+        double: 'double-quoted shell text',
+        'dollar-single': 'ANSI-C shell text',
+        'dollar-double': 'dollar-double-quoted shell text',
+        'paren-sub': 'command substitution',
+        arith: 'shell arithmetic',
+        backtick: 'backtick substitution',
+        'param-brace': 'shell parameter expansion',
+        heredoc: 'a heredoc',
+        'line-comment': 'a shell comment',
+        escape: 'a shell escape'
+    }[error.region];
+    return `Value "${error.source}" is inside ${regionLabel} and will be re-executed as code. Hoisting cannot protect it; move the interpolation to a normal quoted argument instead.`;
+}
+function reevaluatingCommandWarningMessage(construct, name) {
+    const constructLabel = {
+        eval: 'eval',
+        'sh-c': 'sh -c',
+        'bash-c': 'bash -c',
+        source: 'source',
+        let: 'let',
+        'declare-i': 'declare -i',
+        arithmetic: 'shell arithmetic',
+        awk: 'awk',
+        jq: 'jq',
+        sed: 'sed',
+        'xargs-sh-c': 'xargs sh -c'
+    }[construct];
+    return `Value "$${name}" is passed to ${constructLabel} and re-executed as code. Hoisting cannot protect it; pass the value to a fixed program instead.`;
+}
 function normalizeAgentStep(params) {
     if (params.step.prompt === undefined) {
         throw new Error('Agent step normalization requires a prompt');
@@ -638,12 +807,28 @@ function normalizeAgentStep(params) {
         allowedJobReferences: params.allowedJobReferences,
         typeOverlay: params.typeOverlay
     });
+    const thinkingTemplate = params.step.thinking === undefined ? undefined : parseInterpolationField({
+        field: 'agent.thinking',
+        source: params.step.thinking,
+        path: [
+            'jobs',
+            params.sourceName,
+            'steps',
+            params.stepIndex,
+            'thinking'
+        ],
+        issues: params.issues,
+        fillSite: params.fillSite,
+        allowedJobReferences: params.allowedJobReferences,
+        typeOverlay: params.typeOverlay
+    });
     validateAgentStep({
         step: params.step,
         sourceName: params.sourceName,
         stepIndex: params.stepIndex,
         issues: params.issues,
-        validateLiteralProvider: providerTemplate === undefined,
+        validateLiteralModel: params.step.model !== undefined && !hasInterpolationSyntax(params.step.model),
+        validateLiteralProvider: params.step.provider !== undefined && !hasInterpolationSyntax(params.step.provider),
         agentValidationCatalog: params.context.agentValidationCatalog
     });
     const integrations = normalizeAgentIntegrations({
@@ -657,7 +842,9 @@ function normalizeAgentStep(params) {
         prompt: promptTemplate,
         model: modelTemplate,
         provider: providerTemplate,
-        name: params.name
+        thinking: thinkingTemplate,
+        name: params.name,
+        workingDirectory: params.workingDirectory
     });
     return {
         ...params.stepBase,
@@ -689,46 +876,88 @@ function normalizeAgentStep(params) {
 function validateAgentStep(params) {
     validateHarnessThinking(params);
     validateHarnessTools(params);
-    if (!params.validateLiteralProvider) return;
     const providerId = params.step.provider;
-    if (providerId === undefined) return;
-    const provider = params.agentValidationCatalog.providers.find((entry)=>entry.id === providerId);
     const harness = params.step.harness;
-    if (provider === undefined && harness === 'pi') return;
-    if (provider === undefined || provider.support_status !== 'supported') {
-        params.issues.push(issue({
-            code: 'invalid-provider',
-            message: `Provider "${providerId}" is not supported.`,
-            path: [
-                'jobs',
-                params.sourceName,
-                'steps',
-                params.stepIndex,
-                'provider'
-            ],
-            details: {
-                provider: providerId
-            }
-        }));
-        return;
+    const provider = providerId === undefined ? undefined : params.agentValidationCatalog.providers.find((entry)=>entry.id === providerId);
+    if (params.validateLiteralProvider && providerId !== undefined) {
+        if (provider === undefined) {
+            if (harness === undefined || harness === 'pi') return;
+            params.issues.push(issue({
+                code: 'invalid-provider',
+                message: `Provider "${providerId}" is not supported.`,
+                path: [
+                    'jobs',
+                    params.sourceName,
+                    'steps',
+                    params.stepIndex,
+                    'provider'
+                ],
+                details: {
+                    provider: providerId
+                }
+            }));
+            return;
+        }
+        if (provider.support_status !== 'supported') {
+            params.issues.push(issue({
+                code: 'invalid-provider',
+                message: `Provider "${providerId}" is not supported.`,
+                path: [
+                    'jobs',
+                    params.sourceName,
+                    'steps',
+                    params.stepIndex,
+                    'provider'
+                ],
+                details: {
+                    provider: providerId
+                }
+            }));
+            return;
+        }
+        if (harness === undefined) return;
+        const descriptor = params.agentValidationCatalog.harnesses.find((entry)=>entry.id === harness);
+        if (!descriptor?.supported_provider_ids.includes(providerId)) {
+            params.issues.push(issue({
+                code: 'harness-provider-incompatible',
+                message: `Harness "${harness}" does not support provider: ${providerId}. Supported providers: ${descriptor?.supported_provider_ids.join(', ') ?? ''}.`,
+                path: [
+                    'jobs',
+                    params.sourceName,
+                    'steps',
+                    params.stepIndex,
+                    'provider'
+                ],
+                details: {
+                    harness,
+                    provider: providerId,
+                    supportedProviders: descriptor?.supported_provider_ids ?? []
+                }
+            }));
+            return;
+        }
     }
-    if (harness === undefined) return;
+    if (!params.validateLiteralModel || providerId === undefined || harness === undefined) return;
+    if (provider === undefined || provider.support_status !== 'supported') return;
+    const model = params.step.model;
+    if (model === undefined) return;
     const descriptor = params.agentValidationCatalog.harnesses.find((entry)=>entry.id === harness);
-    if (descriptor?.supported_provider_ids.includes(providerId)) return;
+    const modelIds = descriptor?.model_ids_by_provider?.[providerId];
+    if (modelIds === undefined || modelIds.includes(model)) return;
     params.issues.push(issue({
-        code: 'harness-provider-incompatible',
-        message: `Harness "${harness}" does not support provider: ${providerId}. Supported providers: ${descriptor?.supported_provider_ids.join(', ') ?? ''}.`,
+        code: 'invalid-model',
+        message: `Agent model "${model}" is not available for harness "${harness}" and provider "${providerId}".`,
         path: [
             'jobs',
             params.sourceName,
             'steps',
             params.stepIndex,
-            'provider'
+            'model'
         ],
         details: {
             harness,
             provider: providerId,
-            supportedProviders: descriptor?.supported_provider_ids ?? []
+            model
         }
     }));
 }
@@ -778,6 +1007,9 @@ function validateHarnessTools(params) {
 function validateHarnessThinking(params) {
     const { harness, thinking } = params.step;
     if (harness === undefined || thinking === undefined) return;
+    // An interpolated level is only known when the step dispatches, so the agent
+    // module checks it against the resolved harness there.
+    if (hasInterpolationSyntax(thinking)) return;
     const supportedLevels = params.agentValidationCatalog.harnesses.find((entry)=>entry.id === harness)?.thinking_levels ?? [];
     if (supportedLevels.includes(thinking)) return;
     params.issues.push(issue({
@@ -895,7 +1127,7 @@ function validateRunnerLabels(params) {
     }
 }
 function optionalRunStepTemplates(params) {
-    if (params.command === undefined && params.name === undefined && params.env === undefined) {
+    if (params.command === undefined && params.name === undefined && params.workingDirectory === undefined && params.env === undefined) {
         return undefined;
     }
     return {
@@ -905,13 +1137,16 @@ function optionalRunStepTemplates(params) {
         ...params.name === undefined ? {} : {
             name: params.name
         },
+        ...params.workingDirectory === undefined ? {} : {
+            workingDirectory: params.workingDirectory
+        },
         ...params.env === undefined ? {} : {
             env: params.env
         }
     };
 }
 function optionalAgentStepTemplates(params) {
-    if (params.prompt === undefined && params.model === undefined && params.provider === undefined && params.name === undefined) {
+    if (params.prompt === undefined && params.model === undefined && params.provider === undefined && params.thinking === undefined && params.name === undefined && params.workingDirectory === undefined) {
         return undefined;
     }
     return {
@@ -924,8 +1159,14 @@ function optionalAgentStepTemplates(params) {
         ...params.provider === undefined ? {} : {
             provider: params.provider
         },
+        ...params.thinking === undefined ? {} : {
+            thinking: params.thinking
+        },
         ...params.name === undefined ? {} : {
             name: params.name
+        },
+        ...params.workingDirectory === undefined ? {} : {
+            workingDirectory: params.workingDirectory
         }
     };
 }