@sequenceholdings/studio-cli 0.1.13 → 0.1.18

package/dist/pipeline/commands.js

@@ -0,0 +1,330 @@
+/**
+ * `seq-studio pipeline <sub>` — the Data Pipelines authoring interface.
+ *
+ * `init` scaffolds a typed stage spec (`<name>.stage.yml` + entrypoint stub);
+ * `validate` runs the full offline SDK gate (envelope + body + schema_ref
+ * resolution + graph validation) over a Pipeline directory. Both are also the
+ * CI gate for `pipelines/**` repos (`.github/workflows/pipeline-verify.yml`).
+ *
+ * All schema logic lives in @sequenceholdings/pipeline-spec (single source of truth) —
+ * this module only handles argv, filesystem scaffolding, and output shaping.
+ * The SDK is lazy-imported like `@sequenceholdings/orm` so public installs of
+ * the CLI without the package get a clear install hint instead of a crash.
+ */
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import { existsSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { authenticatedRequestUrl, validateDeploymentBaseUrl, } from '@sequenceholdings/artifact-studio/deployment-validation';
+import { tryGetAccessToken } from '../auth.js';
+import { readConfig } from '../config.js';
+import { renderEntrypointStub, renderStageTemplate, STAGE_TYPES, } from './templates.js';
+const LOG = '[seq-studio]';
+const PIPELINE_USAGE = `usage:
+  seq-studio pipeline init --type ingestion|transformation|serving <name> [--dir <dir>]
+      scaffold <name>.stage.yml (commented template) + src/ entrypoint stub
+
+  seq-studio pipeline validate [dir] [--assets <file|url>] [--json]
+      run the offline spec gate: envelope + body validation, schema_ref
+      resolution, and repo-level graph validation (references, single-writer,
+      cycles, column subsets). --assets supplies a registry asset export for
+      cross-Pipeline references. Exits 1 on any error-severity finding.
+
+  seq-studio pipeline plan --repo pipelines/<slug> --ref <sha|branch> -e <env> [--json]
+      materialize + SDK/graph + compile + live-diff + provision findings
+      (no Databricks CLI / DAB validate on Atlas); exit 1 on destructive findings
+
+  seq-studio pipeline deploy --repo pipelines/<slug> --ref <sha> -e <env>
+                            [--approved-by <sub>] [--no-wait] [--json]
+      plan then enqueue Trigger deploy (DAB bundle validate hard-gates before
+      bundle deploy); production/banksouth require a pinned 40-hex SHA
+
+  seq-studio pipeline promote --stage <slug> --version <v> -e <env>
+                              [--repo pipelines/<slug>] [--approved-by <you>] [--no-wait]
+      promote a validated version to the next environment (approvals are
+      self-recorded: --approved-by must name the authenticated caller)
+
+  seq-studio pipeline rollback --stage <slug> -e <env> [--repo pipelines/<slug>]
+                               [--approved-by <you>] [--no-wait]
+      redeploy the previously retired deployment's version (production/
+      banksouth require --approved-by — approvals are explicit even for
+      rollbacks)
+
+  seq-studio pipeline run-now --stage <slug> -e <env> [--repo pipelines/<slug>] [--json]
+      fire the stage's active deployment resource (job run-now / DLT
+      start_update) and print the Databricks run URL
+`;
+/**
+ * Same classification as the orm delegate: is the pipeline-spec package
+ * itself absent (installable) or did one of its dependencies fail to load?
+ */
+export function isPipelineSpecMissing(message) {
+    const specifier = message.match(/Cannot find (?:package|module) '([^']+)'/)?.[1];
+    return specifier === '@sequenceholdings/pipeline-spec' || (specifier?.startsWith('@sequenceholdings/pipeline-spec/') ?? false);
+}
+async function importPipelineSpec() {
+    try {
+        return await import('@sequenceholdings/pipeline-spec');
+    }
+    catch (err) {
+        if (err.code === 'ERR_MODULE_NOT_FOUND') {
+            const message = err instanceof Error ? err.message : String(err);
+            if (isPipelineSpecMissing(message)) {
+                console.error('seq-studio pipeline needs the @sequenceholdings/pipeline-spec package. Install it alongside the CLI:\n' +
+                    '  npm install @sequenceholdings/pipeline-spec');
+            }
+            else {
+                console.error('@sequenceholdings/pipeline-spec is installed but a dependency failed to load. Reinstall dependencies, then retry.\n' +
+                    `  ${message}`);
+            }
+            return null;
+        }
+        throw err;
+    }
+}
+const isStageTemplateType = (value) => STAGE_TYPES.includes(value);
+export async function pipelineInitCommand(args) {
+    const name = args.positional[0];
+    const type = typeof args.flags.type === 'string' ? args.flags.type : undefined;
+    if (!name || !type) {
+        console.error('usage: seq-studio pipeline init --type ingestion|transformation|serving <name> [--dir <dir>]');
+        return 1;
+    }
+    if (!isStageTemplateType(type)) {
+        console.error(`${LOG} unknown stage type '${type}' — expected one of: ${STAGE_TYPES.join(', ')}`);
+        return 1;
+    }
+    const spec = await importPipelineSpec();
+    if (!spec)
+        return 1;
+    if (!spec.STAGE_ID_PATTERN.test(name)) {
+        console.error(`${LOG} stage name '${name}' must be a kebab-case slug (a-z, 0-9, -)`);
+        return 1;
+    }
+    const dir = resolve(typeof args.flags.dir === 'string' ? args.flags.dir : '.');
+    const specPath = join(dir, `${name}.stage.yml`);
+    if (existsSync(specPath)) {
+        console.error(`${LOG} ${specPath} already exists — refusing to overwrite`);
+        return 1;
+    }
+    await mkdir(dir, { recursive: true });
+    await writeFile(specPath, renderStageTemplate(name, type), 'utf8');
+    console.log(`${LOG} wrote ${specPath}`);
+    // Serving stages are declarative — no entrypoint stub. The template's
+    // `entrypoint:` and this stub path share the snake-cased name.
+    if (type !== 'serving') {
+        const stubPath = join(dir, 'src', `${name.replace(/-/g, '_')}.py`);
+        if (existsSync(stubPath)) {
+            console.log(`${LOG} ${stubPath} already exists — left untouched`);
+        }
+        else {
+            await mkdir(join(dir, 'src'), { recursive: true });
+            await writeFile(stubPath, renderEntrypointStub(name), 'utf8');
+            console.log(`${LOG} wrote ${stubPath}`);
+        }
+    }
+    const displayDir = typeof args.flags.dir === 'string' ? args.flags.dir : '.';
+    if (type === 'serving') {
+        // A serving stage inherently references its producer — the scaffold's
+        // placeholder cannot resolve until the author points it at a real stage.
+        console.log(`${LOG} next: point source.stage and trigger.after at your producing transformation stage — ` +
+            `\`seq-studio pipeline validate ${displayDir}\` will fail with unresolved references until then`);
+    }
+    else {
+        console.log(`${LOG} next: fill in the TODOs, then run: seq-studio pipeline validate ${displayDir}`);
+    }
+    return 0;
+}
+function graphFindingToReport(finding) {
+    return {
+        severity: finding.severity,
+        code: finding.code,
+        message: finding.message,
+        ...(finding.stage !== undefined ? { stage: finding.stage } : {}),
+        ...(finding.asset !== undefined ? { asset: finding.asset } : {}),
+        ...(finding.consumers !== undefined ? { consumers: finding.consumers } : {}),
+        ...(finding.suggestions !== undefined ? { suggestions: finding.suggestions } : {}),
+        ...(finding.cycle_path !== undefined ? { cycle_path: finding.cycle_path } : {}),
+    };
+}
+export function buildExternalAssetsRequest({ source, knownOrigins, token, }) {
+    const unauthenticated = {
+        url: source,
+        init: {
+            headers: {},
+            redirect: 'manual',
+        },
+    };
+    if (!token)
+        return unauthenticated;
+    let parsed;
+    try {
+        parsed = new URL(source);
+    }
+    catch {
+        return unauthenticated;
+    }
+    if (!knownOrigins.includes(parsed.origin))
+        return unauthenticated;
+    const path = `${parsed.pathname}${parsed.search}`;
+    const url = authenticatedRequestUrl({
+        baseUrl: parsed.origin,
+        path,
+    });
+    return {
+        url,
+        init: {
+            headers: { authorization: `Bearer ${token}` },
+            redirect: 'manual',
+        },
+    };
+}
+/** Trusted origins of every environment the CLI knows about. */
+export function configuredEnvOrigins(envs) {
+    const origins = [];
+    for (const env of Object.values(envs)) {
+        try {
+            origins.push(validateDeploymentBaseUrl(env.url));
+        }
+        catch {
+            // A malformed or untrusted config.toml URL contributes no auth origin.
+        }
+    }
+    return origins;
+}
+async function loadExternalAssets(spec, source) {
+    let raw;
+    if (/^https?:\/\//.test(source)) {
+        const config = await readConfig();
+        const token = await tryGetAccessToken();
+        const request = buildExternalAssetsRequest({
+            source,
+            knownOrigins: configuredEnvOrigins(config.envs),
+            token,
+        });
+        const response = await fetch(request.url, request.init);
+        if (!response.ok) {
+            throw new Error(`fetching assets export failed: ${response.status} ${response.statusText}`);
+        }
+        raw = await response.text();
+    }
+    else {
+        raw = await readFile(resolve(source), 'utf8');
+    }
+    return spec.externalAssetsExportSchema.parse(JSON.parse(raw));
+}
+function printHumanReport(report) {
+    for (const finding of report.findings) {
+        const prefix = finding.severity === 'error' ? 'error' : 'warning';
+        const where = finding.stage ? ` [${finding.stage}]` : '';
+        console.log(`${prefix}(${finding.code})${where}: ${finding.message}`);
+    }
+    const errors = report.findings.filter((finding) => finding.severity === 'error').length;
+    const warnings = report.findings.length - errors;
+    console.log(`${LOG} ${report.stages.length} stage${report.stages.length === 1 ? '' : 's'} in ${report.dir} — ` +
+        `${errors} error${errors === 1 ? '' : 's'}, ${warnings} warning${warnings === 1 ? '' : 's'}`);
+    console.log(report.ok ? `${LOG} validation passed` : `${LOG} validation failed`);
+}
+export async function pipelineValidateCommand(args) {
+    const spec = await importPipelineSpec();
+    if (!spec)
+        return 1;
+    const dirArg = args.positional[0] ?? '.';
+    const dir = resolve(dirArg);
+    const json = args.flags.json === true || args.flags.json === 'true';
+    const emit = (report) => {
+        if (json) {
+            console.log(JSON.stringify(report, null, 2));
+        }
+        else {
+            printHumanReport(report);
+        }
+        return report.ok ? 0 : 1;
+    };
+    let pipeline;
+    try {
+        pipeline = await spec.loadSpecDirectory(dir);
+    }
+    catch (error) {
+        if (error instanceof spec.SpecDefinitionError) {
+            return emit({
+                ok: false,
+                dir: dirArg,
+                stages: [],
+                findings: [
+                    {
+                        severity: 'error',
+                        code: 'spec_definition_error',
+                        message: error.message,
+                        ...(error.file !== undefined ? { file: error.file } : {}),
+                        ...(error.path !== undefined ? { path: error.path } : {}),
+                    },
+                ],
+            });
+        }
+        throw error;
+    }
+    let externalAssets;
+    if (typeof args.flags.assets === 'string') {
+        try {
+            externalAssets = await loadExternalAssets(spec, args.flags.assets);
+        }
+        catch (error) {
+            return emit({
+                ok: false,
+                dir: dirArg,
+                stages: pipeline.stages.map((stage) => stage.stage),
+                findings: [
+                    {
+                        severity: 'error',
+                        code: 'assets_export_unreadable',
+                        message: error instanceof Error ? error.message : String(error),
+                    },
+                ],
+            });
+        }
+    }
+    const result = spec.validateSpecGraph(pipeline.stages, externalAssets);
+    return emit({
+        ok: result.ok,
+        dir: dirArg,
+        stages: pipeline.stages.map((stage) => stage.stage),
+        findings: result.findings.map(graphFindingToReport),
+    });
+}
+export async function runPipelineCommand(sub, args) {
+    switch (sub) {
+        case 'init':
+            return pipelineInitCommand(args);
+        case 'validate':
+            return pipelineValidateCommand(args);
+        case 'plan': {
+            const { pipelinePlanCommand } = await import('./lifecycle.js');
+            return pipelinePlanCommand(args);
+        }
+        case 'deploy': {
+            const { pipelineDeployCommand } = await import('./lifecycle.js');
+            return pipelineDeployCommand(args);
+        }
+        case 'promote': {
+            const { pipelinePromoteCommand } = await import('./lifecycle.js');
+            return pipelinePromoteCommand(args);
+        }
+        case 'rollback': {
+            const { pipelineRollbackCommand } = await import('./lifecycle.js');
+            return pipelineRollbackCommand(args);
+        }
+        case 'run-now': {
+            const { pipelineRunNowCommand } = await import('./lifecycle.js');
+            return pipelineRunNowCommand(args);
+        }
+        case 'help':
+        case '--help':
+        case '-h':
+        case undefined:
+            console.log(PIPELINE_USAGE);
+            return sub ? 0 : 1;
+        default:
+            console.error(`${LOG} unknown pipeline subcommand '${sub}'\n${PIPELINE_USAGE}`);
+            return 1;
+    }
+}

package/dist/pipeline/lifecycle.d.ts

@@ -0,0 +1,58 @@
+/**
+ * Deploy lifecycle verbs for `seq-studio pipeline plan|deploy|promote|rollback`
+ * (SEQ-2450). Wraps the /api/data-pipelines plan/deploy APIs.
+ */
+import { type ResolvedEnv } from '../config.js';
+import type { ParsedArgs } from '../process/commands.js';
+export interface PlanApiResponse {
+    plan: {
+        hasDestructive: boolean;
+        findings: Array<{
+            resourceKey: string;
+            classification: 'create' | 'update' | 'destructive';
+            destructiveReason?: string;
+            message: string;
+            managedTables?: string[];
+        }>;
+        unmodeled: Array<{
+            resourceKey: string;
+            reason: string;
+            sourcePath: string;
+        }>;
+        planHash: string;
+        commit: string;
+        environment: string;
+        bundleName: string;
+    };
+    deploymentIds: string[];
+    stageIds: string[];
+    text: string;
+}
+export interface EnqueueResponse {
+    deploymentId: string;
+    triggerRunId: string | null;
+    status: string;
+}
+export interface DeploymentDetail {
+    id: string;
+    status: string;
+    statusDetail: string | null;
+}
+/**
+ * The CLI's local dev-loop env is named `local` — chosen so `-e local` lines
+ * up with `seqapi -e local` / `artifact-studio --env local` (see `config.ts`
+ * `BUILT_IN_ENV_URLS`). There is no `local` in the server's deploy-lifecycle
+ * enum (`DEPLOY_ENVIRONMENTS` in `atlas/src/server/services/data-pipelines/schema.ts`
+ * is `dev | staging | production | banksouth`); the server's name for that
+ * same developer-loop target is `dev` (`targetFactsForEnvironment` treats
+ * `dev` as the one target exempt from a `lakebase_branch` binding). Map at
+ * this one boundary — every lifecycle request body funnels through here —
+ * so the CLI never sends the wire-invalid `local` and every other env name
+ * passes through unchanged.
+ */
+export declare function deployEnvironmentForEnv(env: ResolvedEnv): string;
+export declare function pipelinePlanCommand(args: ParsedArgs): Promise<number>;
+export declare function pipelineDeployCommand(args: ParsedArgs): Promise<number>;
+export declare function pipelinePromoteCommand(args: ParsedArgs): Promise<number>;
+export declare function pipelineRunNowCommand(args: ParsedArgs): Promise<number>;
+export declare function pipelineRollbackCommand(args: ParsedArgs): Promise<number>;