@sequenceholdings/studio-cli 0.1.13 → 0.1.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +258 -38
- package/dist/agents/apply-chunks.d.ts +13 -0
- package/dist/agents/apply-chunks.js +43 -0
- package/dist/agents/commands.d.ts +10 -0
- package/dist/agents/commands.js +218 -0
- package/dist/agents/scaffold.d.ts +2 -0
- package/dist/agents/scaffold.js +77 -0
- package/dist/agents/source.d.ts +18 -0
- package/dist/agents/source.js +121 -0
- package/dist/artifact/delegate.d.ts +2 -2
- package/dist/artifact/delegate.js +31 -73
- package/dist/atlas-client.js +52 -37
- package/dist/auth-cmds/commands.d.ts +1 -1
- package/dist/auth-cmds/commands.js +12 -7
- package/dist/auth.d.ts +104 -24
- package/dist/auth.js +456 -94
- package/dist/config.d.ts +3 -3
- package/dist/config.js +18 -13
- package/dist/env-catalog.js +13 -3
- package/dist/env-flags.d.ts +2 -0
- package/dist/env-flags.js +2 -0
- package/dist/env-registry.d.ts +27 -0
- package/dist/env-registry.js +204 -0
- package/dist/envs/commands.d.ts +1 -1
- package/dist/envs/commands.js +41 -3
- package/dist/file-lock.d.ts +5 -0
- package/dist/file-lock.js +187 -0
- package/dist/functions/commands.d.ts +10 -10
- package/dist/functions/commands.js +87 -53
- package/dist/functions/manifest.d.ts +1 -0
- package/dist/functions/manifest.js +36 -0
- package/dist/functions/source-selection.d.ts +24 -0
- package/dist/functions/source-selection.js +67 -0
- package/dist/login.d.ts +8 -3
- package/dist/login.js +46 -34
- package/dist/main.d.ts +3 -1
- package/dist/main.js +41 -12
- package/dist/orm/delegate.js +25 -7
- package/dist/pat-hints.js +2 -2
- package/dist/pipeline/commands.d.ts +58 -0
- package/dist/pipeline/commands.js +330 -0
- package/dist/pipeline/lifecycle.d.ts +58 -0
- package/dist/pipeline/lifecycle.js +348 -0
- package/dist/pipeline/pinning.d.ts +5 -0
- package/dist/pipeline/pinning.js +9 -0
- package/dist/pipeline/templates.d.ts +11 -0
- package/dist/pipeline/templates.js +166 -0
- package/dist/process/build.d.ts +4 -0
- package/dist/process/build.js +33 -2
- package/dist/process/codegen.js +19 -1
- package/dist/process/commands.js +97 -47
- package/dist/process/compiler-subprocess.d.ts +29 -0
- package/dist/process/compiler-subprocess.js +99 -0
- package/dist/process/compiler-worker.d.ts +1 -0
- package/dist/process/compiler-worker.js +38 -0
- package/dist/process/lint.d.ts +8 -0
- package/dist/process/lint.js +84 -29
- package/dist/process/repo-install.js +18 -2
- package/dist/repos/commands.d.ts +1 -1
- package/dist/repos/commands.js +17 -12
- package/dist/secrets/commands.d.ts +1 -1
- package/dist/secrets/commands.js +18 -18
- package/package.json +12 -5
package/dist/main.js
CHANGED
|
@@ -5,10 +5,12 @@
|
|
|
5
5
|
* seq-studio process <sub> manage Lattice processes
|
|
6
6
|
* seq-studio artifact <sub> manage Artifact Studio apps
|
|
7
7
|
* seq-studio functions <sub> manage Managed Functions
|
|
8
|
+
* seq-studio agents <sub> manage typed agent definitions
|
|
8
9
|
* seq-studio secrets <sub> manage org-owned Managed Secrets
|
|
9
10
|
* seq-studio repos <sub> manage platform git-service repos
|
|
11
|
+
* seq-studio pipeline <sub> author + validate Data Pipelines stage specs
|
|
10
12
|
* seq-studio auth <sub> manage git-service PATs
|
|
11
|
-
* seq-studio envs <sub> list/refresh
|
|
13
|
+
* seq-studio envs <sub> add/list/refresh environments
|
|
12
14
|
* seq-studio login authenticate interactively with Auth0
|
|
13
15
|
* seq-studio logout remove cached user tokens
|
|
14
16
|
* seq-studio doctor check token + env + writer gate
|
|
@@ -22,13 +24,15 @@ const TOP_LEVEL_USAGE = `usage:
|
|
|
22
24
|
seq-studio process <sub> [args] lint | plan | apply | test | simulate | bundle | init
|
|
23
25
|
seq-studio artifact <sub> [args] init | build | plan | deploy | dev | list | show | pull | promote | rollback
|
|
24
26
|
seq-studio functions <sub> [args] init | build | deploy | list | show | logs | promote | rollback | delete
|
|
27
|
+
seq-studio agents <sub> [args] init | validate | plan | apply | list | show
|
|
25
28
|
seq-studio secrets <sub> [args] create | set | list | attach | detach | apply
|
|
26
29
|
seq-studio repos <sub> [args] list | namespaces | show | create | clone | pull | delete
|
|
30
|
+
seq-studio pipeline <sub> [args] init | validate — Data Pipelines stage specs
|
|
27
31
|
seq-studio auth <sub> [args] pat create | pat list | pat revoke
|
|
28
32
|
seq-studio orm <sub> [args] init | validate | plan | apply
|
|
29
|
-
seq-studio envs <sub> list | refresh
|
|
30
|
-
seq-studio login
|
|
31
|
-
seq-studio logout
|
|
33
|
+
seq-studio envs <sub> add | list | refresh environments
|
|
34
|
+
seq-studio login [--env <name>] authenticate in the browser
|
|
35
|
+
seq-studio logout [--env <name>] remove one realm's cached user tokens
|
|
32
36
|
seq-studio doctor [-e <env>] diagnose config, auth, and writer gate
|
|
33
37
|
seq-studio version print the installed version
|
|
34
38
|
seq-studio help show this message
|
|
@@ -71,6 +75,10 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
71
75
|
const { runFunctionsCommand } = await import('./functions/commands.js');
|
|
72
76
|
return runFunctionsCommand(sub, parseArgs(rest));
|
|
73
77
|
}
|
|
78
|
+
case 'agents': {
|
|
79
|
+
const { runAgentsCommand } = await import('./agents/commands.js');
|
|
80
|
+
return runAgentsCommand(sub, parseArgs(rest));
|
|
81
|
+
}
|
|
74
82
|
case 'secrets': {
|
|
75
83
|
const { runSecretsCommand } = await import('./secrets/commands.js');
|
|
76
84
|
return runSecretsCommand(sub, parseArgs(rest));
|
|
@@ -87,13 +95,19 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
87
95
|
const { runOrmCommand } = await import('./orm/delegate.js');
|
|
88
96
|
return runOrmCommand(sub, rest);
|
|
89
97
|
}
|
|
98
|
+
case 'pipeline': {
|
|
99
|
+
// Lazy import keeps doctor/process from pulling in @sequenceholdings/pipeline-spec
|
|
100
|
+
// (an optional peer, same posture as @sequenceholdings/orm).
|
|
101
|
+
const { runPipelineCommand } = await import('./pipeline/commands.js');
|
|
102
|
+
return runPipelineCommand(sub, parseArgs(rest));
|
|
103
|
+
}
|
|
90
104
|
case 'envs': {
|
|
91
105
|
const { runEnvsCommand } = await import('./envs/commands.js');
|
|
92
106
|
return runEnvsCommand(sub, rest);
|
|
93
107
|
}
|
|
94
108
|
case 'login':
|
|
95
109
|
case 'logout':
|
|
96
|
-
return runSessionCommand({
|
|
110
|
+
return runSessionCommand({ args: [sub, ...rest].filter((a) => Boolean(a)), command: namespace });
|
|
97
111
|
case 'version':
|
|
98
112
|
case '--version':
|
|
99
113
|
case '-v': {
|
|
@@ -109,17 +123,32 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
109
123
|
return 1;
|
|
110
124
|
}
|
|
111
125
|
}
|
|
112
|
-
async function runSessionCommand({
|
|
113
|
-
|
|
114
|
-
|
|
126
|
+
async function runSessionCommand({ args, command, }) {
|
|
127
|
+
const usage = `usage: seq-studio ${command} [--env <name>]`;
|
|
128
|
+
if (args[0] === 'help' || args[0] === '--help' || args[0] === '-h') {
|
|
129
|
+
console.log(usage);
|
|
115
130
|
return 0;
|
|
116
131
|
}
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
132
|
+
let envName;
|
|
133
|
+
if (args.length > 0) {
|
|
134
|
+
const flags = parseArgs(args).flags;
|
|
135
|
+
envName =
|
|
136
|
+
(typeof flags.env === 'string' ? flags.env : undefined) ??
|
|
137
|
+
(typeof flags.e === 'string' ? flags.e : undefined);
|
|
138
|
+
if (!envName) {
|
|
139
|
+
console.error(usage);
|
|
140
|
+
return 1;
|
|
141
|
+
}
|
|
120
142
|
}
|
|
121
143
|
const auth = await import('./login.js');
|
|
122
|
-
|
|
144
|
+
if (command === 'logout') {
|
|
145
|
+
await auth.logout(envName);
|
|
146
|
+
}
|
|
147
|
+
else {
|
|
148
|
+
// An OpCo env name (registered via `seq-studio envs add` or `seqapi env add`)
|
|
149
|
+
// logs into that environment's Auth0 realm; omitted or a built-in name uses Sequence.
|
|
150
|
+
await auth.login(envName);
|
|
151
|
+
}
|
|
123
152
|
return 0;
|
|
124
153
|
}
|
|
125
154
|
async function runProcessNamespace(sub, rest) {
|
package/dist/orm/delegate.js
CHANGED
|
@@ -8,12 +8,20 @@
|
|
|
8
8
|
import { getAccessToken, tryGetAccessToken } from '../auth.js';
|
|
9
9
|
import { resolveEnvWithDiscovery } from '../config.js';
|
|
10
10
|
import { normalizeShortEnvFlag, readEnvFromArgv } from '../env-flags.js';
|
|
11
|
+
import { REQUIRE_EXPLICIT_ENV_MESSAGE } from '../env-flags.js';
|
|
12
|
+
/**
|
|
13
|
+
* ORM subcommands that talk to Atlas — must not silently target `local`.
|
|
14
|
+
* `init`, `generate`, `validate`, `diff`, and `migrate-from-yaml` are offline.
|
|
15
|
+
*/
|
|
16
|
+
const ORM_NETWORK_SUBS = new Set(['plan', 'apply']);
|
|
11
17
|
const ORM_USAGE = `usage:
|
|
12
|
-
seq-studio orm init <dir>
|
|
13
|
-
seq-studio orm
|
|
14
|
-
seq-studio orm
|
|
15
|
-
seq-studio orm
|
|
16
|
-
seq-studio orm
|
|
18
|
+
seq-studio orm init <dir> scaffold a namespace package (v2: TypeScript schema + GraphQL documents)
|
|
19
|
+
seq-studio orm generate [dir] write schema.graphql + operations.manifest.json + typePolicies.gen.ts (offline; runs consumer codegen when codegen.ts is present)
|
|
20
|
+
seq-studio orm validate [dir] parse + validate definitions and verify committed migrations are in sync
|
|
21
|
+
seq-studio orm plan [dir] -e <env> compile definitions and diff against the registry
|
|
22
|
+
seq-studio orm apply [dir] -e <env> author the migration, register, apply, and refresh the generated outputs (the everyday command)
|
|
23
|
+
seq-studio orm diff [dir] write/verify the committed migration standalone (--check is the offline CI gate; --allow-destructive consents; --rebase records a v1 -> v2 conversion)
|
|
24
|
+
seq-studio orm migrate-from-yaml <dir> convert a v1 YAML namespace to the v2 TypeScript format (offline; migrations/ preserved)
|
|
17
25
|
|
|
18
26
|
Environments: see \`seq-studio envs list\` (built-in: local; more are
|
|
19
27
|
discovered after you authenticate).
|
|
@@ -38,11 +46,21 @@ export async function runOrmCommand(sub, rest) {
|
|
|
38
46
|
}
|
|
39
47
|
const normalized = normalizeShortEnvFlag(rest);
|
|
40
48
|
const requested = readEnvFromArgv(normalized);
|
|
49
|
+
if (ORM_NETWORK_SUBS.has(sub) && !requested) {
|
|
50
|
+
console.error(`[seq-studio] ${REQUIRE_EXPLICIT_ENV_MESSAGE}`);
|
|
51
|
+
return 1;
|
|
52
|
+
}
|
|
53
|
+
let targetUrl;
|
|
41
54
|
if (requested) {
|
|
42
55
|
const resolved = await resolveEnvWithDiscovery({ requested });
|
|
56
|
+
targetUrl = resolved.url;
|
|
43
57
|
process.env['SEQUENCE_ORM_BASE_URL'] = resolved.url;
|
|
44
58
|
}
|
|
45
|
-
const token = await tryGetAccessToken({
|
|
59
|
+
const token = await tryGetAccessToken({
|
|
60
|
+
failClosedForM2m: true,
|
|
61
|
+
env: requested,
|
|
62
|
+
targetUrl,
|
|
63
|
+
});
|
|
46
64
|
if (token)
|
|
47
65
|
process.env['SEQUENCE_ORM_TOKEN'] = token;
|
|
48
66
|
// Lazy import keeps `process`/`doctor` from pulling in the orm package.
|
|
@@ -70,7 +88,7 @@ export async function runOrmCommand(sub, rest) {
|
|
|
70
88
|
const { runCli, setTokenProvider } = ormCli;
|
|
71
89
|
setTokenProvider(async () => {
|
|
72
90
|
try {
|
|
73
|
-
return await getAccessToken();
|
|
91
|
+
return await getAccessToken({ env: requested, targetUrl });
|
|
74
92
|
}
|
|
75
93
|
catch {
|
|
76
94
|
return null;
|
package/dist/pat-hints.js
CHANGED
|
@@ -21,7 +21,7 @@ export function formatPatSetupHint({ envUrl, envName, indent = ' ', }) {
|
|
|
21
21
|
`${indent} 2. New token → scopes repo:read (add repo:write for push) → copy once`,
|
|
22
22
|
`${indent} 3. export ATLAS_GIT_PAT=<token>`,
|
|
23
23
|
`${indent} 4. Copy the clone URL from Repositories → Clone, then:`,
|
|
24
|
-
`${indent} ATLAS_GIT_PAT=<token> seq-studio repos clone --url <https://…/repos/<id>/git
|
|
25
|
-
`${indent} (or: seq-studio repos clone <ns>/<name>)`,
|
|
24
|
+
`${indent} ATLAS_GIT_PAT=<token> seq-studio repos clone --url <https://…/repos/<id>/git> -e ${envName}`,
|
|
25
|
+
`${indent} (or: seq-studio repos clone <ns>/<name> -e ${envName})`,
|
|
26
26
|
];
|
|
27
27
|
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `seq-studio pipeline <sub>` — the Data Pipelines authoring interface.
|
|
3
|
+
*
|
|
4
|
+
* `init` scaffolds a typed stage spec (`<name>.stage.yml` + entrypoint stub);
|
|
5
|
+
* `validate` runs the full offline SDK gate (envelope + body + schema_ref
|
|
6
|
+
* resolution + graph validation) over a Pipeline directory. Both are also the
|
|
7
|
+
* CI gate for `pipelines/**` repos (`.github/workflows/pipeline-verify.yml`).
|
|
8
|
+
*
|
|
9
|
+
* All schema logic lives in @sequenceholdings/pipeline-spec (single source of truth) —
|
|
10
|
+
* this module only handles argv, filesystem scaffolding, and output shaping.
|
|
11
|
+
* The SDK is lazy-imported like `@sequenceholdings/orm` so public installs of
|
|
12
|
+
* the CLI without the package get a clear install hint instead of a crash.
|
|
13
|
+
*/
|
|
14
|
+
import type { ParsedArgs } from '../process/commands.js';
|
|
15
|
+
/**
|
|
16
|
+
* Same classification as the orm delegate: is the pipeline-spec package
|
|
17
|
+
* itself absent (installable) or did one of its dependencies fail to load?
|
|
18
|
+
*/
|
|
19
|
+
export declare function isPipelineSpecMissing(message: string): boolean;
|
|
20
|
+
export declare function pipelineInitCommand(args: ParsedArgs): Promise<number>;
|
|
21
|
+
interface ValidateReportFinding {
|
|
22
|
+
readonly severity: 'error' | 'warning';
|
|
23
|
+
readonly code: string;
|
|
24
|
+
readonly message: string;
|
|
25
|
+
readonly stage?: string;
|
|
26
|
+
readonly asset?: string;
|
|
27
|
+
readonly consumers?: readonly string[];
|
|
28
|
+
readonly suggestions?: readonly string[];
|
|
29
|
+
readonly cycle_path?: readonly string[];
|
|
30
|
+
readonly file?: string;
|
|
31
|
+
readonly path?: string;
|
|
32
|
+
}
|
|
33
|
+
/** The stable machine shape `--json` emits (consumed by CI annotations). */
|
|
34
|
+
export interface ValidateReport {
|
|
35
|
+
readonly ok: boolean;
|
|
36
|
+
readonly dir: string;
|
|
37
|
+
readonly stages: readonly string[];
|
|
38
|
+
readonly findings: readonly ValidateReportFinding[];
|
|
39
|
+
}
|
|
40
|
+
interface ExternalAssetsRequest {
|
|
41
|
+
readonly url: string;
|
|
42
|
+
readonly init: {
|
|
43
|
+
readonly headers: Record<string, string>;
|
|
44
|
+
readonly redirect: 'manual';
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
export declare function buildExternalAssetsRequest({ source, knownOrigins, token, }: {
|
|
48
|
+
source: string;
|
|
49
|
+
knownOrigins: readonly string[];
|
|
50
|
+
token: string | null;
|
|
51
|
+
}): ExternalAssetsRequest;
|
|
52
|
+
/** Trusted origins of every environment the CLI knows about. */
|
|
53
|
+
export declare function configuredEnvOrigins(envs: Readonly<Record<string, {
|
|
54
|
+
url: string;
|
|
55
|
+
}>>): string[];
|
|
56
|
+
export declare function pipelineValidateCommand(args: ParsedArgs): Promise<number>;
|
|
57
|
+
export declare function runPipelineCommand(sub: string | undefined, args: ParsedArgs): Promise<number>;
|
|
58
|
+
export {};
|
|
@@ -0,0 +1,330 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `seq-studio pipeline <sub>` — the Data Pipelines authoring interface.
|
|
3
|
+
*
|
|
4
|
+
* `init` scaffolds a typed stage spec (`<name>.stage.yml` + entrypoint stub);
|
|
5
|
+
* `validate` runs the full offline SDK gate (envelope + body + schema_ref
|
|
6
|
+
* resolution + graph validation) over a Pipeline directory. Both are also the
|
|
7
|
+
* CI gate for `pipelines/**` repos (`.github/workflows/pipeline-verify.yml`).
|
|
8
|
+
*
|
|
9
|
+
* All schema logic lives in @sequenceholdings/pipeline-spec (single source of truth) —
|
|
10
|
+
* this module only handles argv, filesystem scaffolding, and output shaping.
|
|
11
|
+
* The SDK is lazy-imported like `@sequenceholdings/orm` so public installs of
|
|
12
|
+
* the CLI without the package get a clear install hint instead of a crash.
|
|
13
|
+
*/
|
|
14
|
+
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
|
15
|
+
import { existsSync } from 'node:fs';
|
|
16
|
+
import { join, resolve } from 'node:path';
|
|
17
|
+
import { authenticatedRequestUrl, validateDeploymentBaseUrl, } from '@sequenceholdings/artifact-studio/deployment-validation';
|
|
18
|
+
import { tryGetAccessToken } from '../auth.js';
|
|
19
|
+
import { readConfig } from '../config.js';
|
|
20
|
+
import { renderEntrypointStub, renderStageTemplate, STAGE_TYPES, } from './templates.js';
|
|
21
|
+
const LOG = '[seq-studio]';
|
|
22
|
+
const PIPELINE_USAGE = `usage:
|
|
23
|
+
seq-studio pipeline init --type ingestion|transformation|serving <name> [--dir <dir>]
|
|
24
|
+
scaffold <name>.stage.yml (commented template) + src/ entrypoint stub
|
|
25
|
+
|
|
26
|
+
seq-studio pipeline validate [dir] [--assets <file|url>] [--json]
|
|
27
|
+
run the offline spec gate: envelope + body validation, schema_ref
|
|
28
|
+
resolution, and repo-level graph validation (references, single-writer,
|
|
29
|
+
cycles, column subsets). --assets supplies a registry asset export for
|
|
30
|
+
cross-Pipeline references. Exits 1 on any error-severity finding.
|
|
31
|
+
|
|
32
|
+
seq-studio pipeline plan --repo pipelines/<slug> --ref <sha|branch> -e <env> [--json]
|
|
33
|
+
materialize + SDK/graph + compile + live-diff + provision findings
|
|
34
|
+
(no Databricks CLI / DAB validate on Atlas); exit 1 on destructive findings
|
|
35
|
+
|
|
36
|
+
seq-studio pipeline deploy --repo pipelines/<slug> --ref <sha> -e <env>
|
|
37
|
+
[--approved-by <sub>] [--no-wait] [--json]
|
|
38
|
+
plan then enqueue Trigger deploy (DAB bundle validate hard-gates before
|
|
39
|
+
bundle deploy); production/banksouth require a pinned 40-hex SHA
|
|
40
|
+
|
|
41
|
+
seq-studio pipeline promote --stage <slug> --version <v> -e <env>
|
|
42
|
+
[--repo pipelines/<slug>] [--approved-by <you>] [--no-wait]
|
|
43
|
+
promote a validated version to the next environment (approvals are
|
|
44
|
+
self-recorded: --approved-by must name the authenticated caller)
|
|
45
|
+
|
|
46
|
+
seq-studio pipeline rollback --stage <slug> -e <env> [--repo pipelines/<slug>]
|
|
47
|
+
[--approved-by <you>] [--no-wait]
|
|
48
|
+
redeploy the previously retired deployment's version (production/
|
|
49
|
+
banksouth require --approved-by — approvals are explicit even for
|
|
50
|
+
rollbacks)
|
|
51
|
+
|
|
52
|
+
seq-studio pipeline run-now --stage <slug> -e <env> [--repo pipelines/<slug>] [--json]
|
|
53
|
+
fire the stage's active deployment resource (job run-now / DLT
|
|
54
|
+
start_update) and print the Databricks run URL
|
|
55
|
+
`;
|
|
56
|
+
/**
|
|
57
|
+
* Same classification as the orm delegate: is the pipeline-spec package
|
|
58
|
+
* itself absent (installable) or did one of its dependencies fail to load?
|
|
59
|
+
*/
|
|
60
|
+
export function isPipelineSpecMissing(message) {
|
|
61
|
+
const specifier = message.match(/Cannot find (?:package|module) '([^']+)'/)?.[1];
|
|
62
|
+
return specifier === '@sequenceholdings/pipeline-spec' || (specifier?.startsWith('@sequenceholdings/pipeline-spec/') ?? false);
|
|
63
|
+
}
|
|
64
|
+
async function importPipelineSpec() {
|
|
65
|
+
try {
|
|
66
|
+
return await import('@sequenceholdings/pipeline-spec');
|
|
67
|
+
}
|
|
68
|
+
catch (err) {
|
|
69
|
+
if (err.code === 'ERR_MODULE_NOT_FOUND') {
|
|
70
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
71
|
+
if (isPipelineSpecMissing(message)) {
|
|
72
|
+
console.error('seq-studio pipeline needs the @sequenceholdings/pipeline-spec package. Install it alongside the CLI:\n' +
|
|
73
|
+
' npm install @sequenceholdings/pipeline-spec');
|
|
74
|
+
}
|
|
75
|
+
else {
|
|
76
|
+
console.error('@sequenceholdings/pipeline-spec is installed but a dependency failed to load. Reinstall dependencies, then retry.\n' +
|
|
77
|
+
` ${message}`);
|
|
78
|
+
}
|
|
79
|
+
return null;
|
|
80
|
+
}
|
|
81
|
+
throw err;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
const isStageTemplateType = (value) => STAGE_TYPES.includes(value);
|
|
85
|
+
export async function pipelineInitCommand(args) {
|
|
86
|
+
const name = args.positional[0];
|
|
87
|
+
const type = typeof args.flags.type === 'string' ? args.flags.type : undefined;
|
|
88
|
+
if (!name || !type) {
|
|
89
|
+
console.error('usage: seq-studio pipeline init --type ingestion|transformation|serving <name> [--dir <dir>]');
|
|
90
|
+
return 1;
|
|
91
|
+
}
|
|
92
|
+
if (!isStageTemplateType(type)) {
|
|
93
|
+
console.error(`${LOG} unknown stage type '${type}' — expected one of: ${STAGE_TYPES.join(', ')}`);
|
|
94
|
+
return 1;
|
|
95
|
+
}
|
|
96
|
+
const spec = await importPipelineSpec();
|
|
97
|
+
if (!spec)
|
|
98
|
+
return 1;
|
|
99
|
+
if (!spec.STAGE_ID_PATTERN.test(name)) {
|
|
100
|
+
console.error(`${LOG} stage name '${name}' must be a kebab-case slug (a-z, 0-9, -)`);
|
|
101
|
+
return 1;
|
|
102
|
+
}
|
|
103
|
+
const dir = resolve(typeof args.flags.dir === 'string' ? args.flags.dir : '.');
|
|
104
|
+
const specPath = join(dir, `${name}.stage.yml`);
|
|
105
|
+
if (existsSync(specPath)) {
|
|
106
|
+
console.error(`${LOG} ${specPath} already exists — refusing to overwrite`);
|
|
107
|
+
return 1;
|
|
108
|
+
}
|
|
109
|
+
await mkdir(dir, { recursive: true });
|
|
110
|
+
await writeFile(specPath, renderStageTemplate(name, type), 'utf8');
|
|
111
|
+
console.log(`${LOG} wrote ${specPath}`);
|
|
112
|
+
// Serving stages are declarative — no entrypoint stub. The template's
|
|
113
|
+
// `entrypoint:` and this stub path share the snake-cased name.
|
|
114
|
+
if (type !== 'serving') {
|
|
115
|
+
const stubPath = join(dir, 'src', `${name.replace(/-/g, '_')}.py`);
|
|
116
|
+
if (existsSync(stubPath)) {
|
|
117
|
+
console.log(`${LOG} ${stubPath} already exists — left untouched`);
|
|
118
|
+
}
|
|
119
|
+
else {
|
|
120
|
+
await mkdir(join(dir, 'src'), { recursive: true });
|
|
121
|
+
await writeFile(stubPath, renderEntrypointStub(name), 'utf8');
|
|
122
|
+
console.log(`${LOG} wrote ${stubPath}`);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
const displayDir = typeof args.flags.dir === 'string' ? args.flags.dir : '.';
|
|
126
|
+
if (type === 'serving') {
|
|
127
|
+
// A serving stage inherently references its producer — the scaffold's
|
|
128
|
+
// placeholder cannot resolve until the author points it at a real stage.
|
|
129
|
+
console.log(`${LOG} next: point source.stage and trigger.after at your producing transformation stage — ` +
|
|
130
|
+
`\`seq-studio pipeline validate ${displayDir}\` will fail with unresolved references until then`);
|
|
131
|
+
}
|
|
132
|
+
else {
|
|
133
|
+
console.log(`${LOG} next: fill in the TODOs, then run: seq-studio pipeline validate ${displayDir}`);
|
|
134
|
+
}
|
|
135
|
+
return 0;
|
|
136
|
+
}
|
|
137
|
+
function graphFindingToReport(finding) {
|
|
138
|
+
return {
|
|
139
|
+
severity: finding.severity,
|
|
140
|
+
code: finding.code,
|
|
141
|
+
message: finding.message,
|
|
142
|
+
...(finding.stage !== undefined ? { stage: finding.stage } : {}),
|
|
143
|
+
...(finding.asset !== undefined ? { asset: finding.asset } : {}),
|
|
144
|
+
...(finding.consumers !== undefined ? { consumers: finding.consumers } : {}),
|
|
145
|
+
...(finding.suggestions !== undefined ? { suggestions: finding.suggestions } : {}),
|
|
146
|
+
...(finding.cycle_path !== undefined ? { cycle_path: finding.cycle_path } : {}),
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
export function buildExternalAssetsRequest({ source, knownOrigins, token, }) {
|
|
150
|
+
const unauthenticated = {
|
|
151
|
+
url: source,
|
|
152
|
+
init: {
|
|
153
|
+
headers: {},
|
|
154
|
+
redirect: 'manual',
|
|
155
|
+
},
|
|
156
|
+
};
|
|
157
|
+
if (!token)
|
|
158
|
+
return unauthenticated;
|
|
159
|
+
let parsed;
|
|
160
|
+
try {
|
|
161
|
+
parsed = new URL(source);
|
|
162
|
+
}
|
|
163
|
+
catch {
|
|
164
|
+
return unauthenticated;
|
|
165
|
+
}
|
|
166
|
+
if (!knownOrigins.includes(parsed.origin))
|
|
167
|
+
return unauthenticated;
|
|
168
|
+
const path = `${parsed.pathname}${parsed.search}`;
|
|
169
|
+
const url = authenticatedRequestUrl({
|
|
170
|
+
baseUrl: parsed.origin,
|
|
171
|
+
path,
|
|
172
|
+
});
|
|
173
|
+
return {
|
|
174
|
+
url,
|
|
175
|
+
init: {
|
|
176
|
+
headers: { authorization: `Bearer ${token}` },
|
|
177
|
+
redirect: 'manual',
|
|
178
|
+
},
|
|
179
|
+
};
|
|
180
|
+
}
|
|
181
|
+
/** Trusted origins of every environment the CLI knows about. */
|
|
182
|
+
export function configuredEnvOrigins(envs) {
|
|
183
|
+
const origins = [];
|
|
184
|
+
for (const env of Object.values(envs)) {
|
|
185
|
+
try {
|
|
186
|
+
origins.push(validateDeploymentBaseUrl(env.url));
|
|
187
|
+
}
|
|
188
|
+
catch {
|
|
189
|
+
// A malformed or untrusted config.toml URL contributes no auth origin.
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
return origins;
|
|
193
|
+
}
|
|
194
|
+
async function loadExternalAssets(spec, source) {
|
|
195
|
+
let raw;
|
|
196
|
+
if (/^https?:\/\//.test(source)) {
|
|
197
|
+
const config = await readConfig();
|
|
198
|
+
const token = await tryGetAccessToken();
|
|
199
|
+
const request = buildExternalAssetsRequest({
|
|
200
|
+
source,
|
|
201
|
+
knownOrigins: configuredEnvOrigins(config.envs),
|
|
202
|
+
token,
|
|
203
|
+
});
|
|
204
|
+
const response = await fetch(request.url, request.init);
|
|
205
|
+
if (!response.ok) {
|
|
206
|
+
throw new Error(`fetching assets export failed: ${response.status} ${response.statusText}`);
|
|
207
|
+
}
|
|
208
|
+
raw = await response.text();
|
|
209
|
+
}
|
|
210
|
+
else {
|
|
211
|
+
raw = await readFile(resolve(source), 'utf8');
|
|
212
|
+
}
|
|
213
|
+
return spec.externalAssetsExportSchema.parse(JSON.parse(raw));
|
|
214
|
+
}
|
|
215
|
+
function printHumanReport(report) {
|
|
216
|
+
for (const finding of report.findings) {
|
|
217
|
+
const prefix = finding.severity === 'error' ? 'error' : 'warning';
|
|
218
|
+
const where = finding.stage ? ` [${finding.stage}]` : '';
|
|
219
|
+
console.log(`${prefix}(${finding.code})${where}: ${finding.message}`);
|
|
220
|
+
}
|
|
221
|
+
const errors = report.findings.filter((finding) => finding.severity === 'error').length;
|
|
222
|
+
const warnings = report.findings.length - errors;
|
|
223
|
+
console.log(`${LOG} ${report.stages.length} stage${report.stages.length === 1 ? '' : 's'} in ${report.dir} — ` +
|
|
224
|
+
`${errors} error${errors === 1 ? '' : 's'}, ${warnings} warning${warnings === 1 ? '' : 's'}`);
|
|
225
|
+
console.log(report.ok ? `${LOG} validation passed` : `${LOG} validation failed`);
|
|
226
|
+
}
|
|
227
|
+
export async function pipelineValidateCommand(args) {
|
|
228
|
+
const spec = await importPipelineSpec();
|
|
229
|
+
if (!spec)
|
|
230
|
+
return 1;
|
|
231
|
+
const dirArg = args.positional[0] ?? '.';
|
|
232
|
+
const dir = resolve(dirArg);
|
|
233
|
+
const json = args.flags.json === true || args.flags.json === 'true';
|
|
234
|
+
const emit = (report) => {
|
|
235
|
+
if (json) {
|
|
236
|
+
console.log(JSON.stringify(report, null, 2));
|
|
237
|
+
}
|
|
238
|
+
else {
|
|
239
|
+
printHumanReport(report);
|
|
240
|
+
}
|
|
241
|
+
return report.ok ? 0 : 1;
|
|
242
|
+
};
|
|
243
|
+
let pipeline;
|
|
244
|
+
try {
|
|
245
|
+
pipeline = await spec.loadSpecDirectory(dir);
|
|
246
|
+
}
|
|
247
|
+
catch (error) {
|
|
248
|
+
if (error instanceof spec.SpecDefinitionError) {
|
|
249
|
+
return emit({
|
|
250
|
+
ok: false,
|
|
251
|
+
dir: dirArg,
|
|
252
|
+
stages: [],
|
|
253
|
+
findings: [
|
|
254
|
+
{
|
|
255
|
+
severity: 'error',
|
|
256
|
+
code: 'spec_definition_error',
|
|
257
|
+
message: error.message,
|
|
258
|
+
...(error.file !== undefined ? { file: error.file } : {}),
|
|
259
|
+
...(error.path !== undefined ? { path: error.path } : {}),
|
|
260
|
+
},
|
|
261
|
+
],
|
|
262
|
+
});
|
|
263
|
+
}
|
|
264
|
+
throw error;
|
|
265
|
+
}
|
|
266
|
+
let externalAssets;
|
|
267
|
+
if (typeof args.flags.assets === 'string') {
|
|
268
|
+
try {
|
|
269
|
+
externalAssets = await loadExternalAssets(spec, args.flags.assets);
|
|
270
|
+
}
|
|
271
|
+
catch (error) {
|
|
272
|
+
return emit({
|
|
273
|
+
ok: false,
|
|
274
|
+
dir: dirArg,
|
|
275
|
+
stages: pipeline.stages.map((stage) => stage.stage),
|
|
276
|
+
findings: [
|
|
277
|
+
{
|
|
278
|
+
severity: 'error',
|
|
279
|
+
code: 'assets_export_unreadable',
|
|
280
|
+
message: error instanceof Error ? error.message : String(error),
|
|
281
|
+
},
|
|
282
|
+
],
|
|
283
|
+
});
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
const result = spec.validateSpecGraph(pipeline.stages, externalAssets);
|
|
287
|
+
return emit({
|
|
288
|
+
ok: result.ok,
|
|
289
|
+
dir: dirArg,
|
|
290
|
+
stages: pipeline.stages.map((stage) => stage.stage),
|
|
291
|
+
findings: result.findings.map(graphFindingToReport),
|
|
292
|
+
});
|
|
293
|
+
}
|
|
294
|
+
export async function runPipelineCommand(sub, args) {
|
|
295
|
+
switch (sub) {
|
|
296
|
+
case 'init':
|
|
297
|
+
return pipelineInitCommand(args);
|
|
298
|
+
case 'validate':
|
|
299
|
+
return pipelineValidateCommand(args);
|
|
300
|
+
case 'plan': {
|
|
301
|
+
const { pipelinePlanCommand } = await import('./lifecycle.js');
|
|
302
|
+
return pipelinePlanCommand(args);
|
|
303
|
+
}
|
|
304
|
+
case 'deploy': {
|
|
305
|
+
const { pipelineDeployCommand } = await import('./lifecycle.js');
|
|
306
|
+
return pipelineDeployCommand(args);
|
|
307
|
+
}
|
|
308
|
+
case 'promote': {
|
|
309
|
+
const { pipelinePromoteCommand } = await import('./lifecycle.js');
|
|
310
|
+
return pipelinePromoteCommand(args);
|
|
311
|
+
}
|
|
312
|
+
case 'rollback': {
|
|
313
|
+
const { pipelineRollbackCommand } = await import('./lifecycle.js');
|
|
314
|
+
return pipelineRollbackCommand(args);
|
|
315
|
+
}
|
|
316
|
+
case 'run-now': {
|
|
317
|
+
const { pipelineRunNowCommand } = await import('./lifecycle.js');
|
|
318
|
+
return pipelineRunNowCommand(args);
|
|
319
|
+
}
|
|
320
|
+
case 'help':
|
|
321
|
+
case '--help':
|
|
322
|
+
case '-h':
|
|
323
|
+
case undefined:
|
|
324
|
+
console.log(PIPELINE_USAGE);
|
|
325
|
+
return sub ? 0 : 1;
|
|
326
|
+
default:
|
|
327
|
+
console.error(`${LOG} unknown pipeline subcommand '${sub}'\n${PIPELINE_USAGE}`);
|
|
328
|
+
return 1;
|
|
329
|
+
}
|
|
330
|
+
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deploy lifecycle verbs for `seq-studio pipeline plan|deploy|promote|rollback`
|
|
3
|
+
* (SEQ-2450). Wraps the /api/data-pipelines plan/deploy APIs.
|
|
4
|
+
*/
|
|
5
|
+
import { type ResolvedEnv } from '../config.js';
|
|
6
|
+
import type { ParsedArgs } from '../process/commands.js';
|
|
7
|
+
export interface PlanApiResponse {
|
|
8
|
+
plan: {
|
|
9
|
+
hasDestructive: boolean;
|
|
10
|
+
findings: Array<{
|
|
11
|
+
resourceKey: string;
|
|
12
|
+
classification: 'create' | 'update' | 'destructive';
|
|
13
|
+
destructiveReason?: string;
|
|
14
|
+
message: string;
|
|
15
|
+
managedTables?: string[];
|
|
16
|
+
}>;
|
|
17
|
+
unmodeled: Array<{
|
|
18
|
+
resourceKey: string;
|
|
19
|
+
reason: string;
|
|
20
|
+
sourcePath: string;
|
|
21
|
+
}>;
|
|
22
|
+
planHash: string;
|
|
23
|
+
commit: string;
|
|
24
|
+
environment: string;
|
|
25
|
+
bundleName: string;
|
|
26
|
+
};
|
|
27
|
+
deploymentIds: string[];
|
|
28
|
+
stageIds: string[];
|
|
29
|
+
text: string;
|
|
30
|
+
}
|
|
31
|
+
export interface EnqueueResponse {
|
|
32
|
+
deploymentId: string;
|
|
33
|
+
triggerRunId: string | null;
|
|
34
|
+
status: string;
|
|
35
|
+
}
|
|
36
|
+
export interface DeploymentDetail {
|
|
37
|
+
id: string;
|
|
38
|
+
status: string;
|
|
39
|
+
statusDetail: string | null;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* The CLI's local dev-loop env is named `local` — chosen so `-e local` lines
|
|
43
|
+
* up with `seqapi -e local` / `artifact-studio --env local` (see `config.ts`
|
|
44
|
+
* `BUILT_IN_ENV_URLS`). There is no `local` in the server's deploy-lifecycle
|
|
45
|
+
* enum (`DEPLOY_ENVIRONMENTS` in `atlas/src/server/services/data-pipelines/schema.ts`
|
|
46
|
+
* is `dev | staging | production | banksouth`); the server's name for that
|
|
47
|
+
* same developer-loop target is `dev` (`targetFactsForEnvironment` treats
|
|
48
|
+
* `dev` as the one target exempt from a `lakebase_branch` binding). Map at
|
|
49
|
+
* this one boundary — every lifecycle request body funnels through here —
|
|
50
|
+
* so the CLI never sends the wire-invalid `local` and every other env name
|
|
51
|
+
* passes through unchanged.
|
|
52
|
+
*/
|
|
53
|
+
export declare function deployEnvironmentForEnv(env: ResolvedEnv): string;
|
|
54
|
+
export declare function pipelinePlanCommand(args: ParsedArgs): Promise<number>;
|
|
55
|
+
export declare function pipelineDeployCommand(args: ParsedArgs): Promise<number>;
|
|
56
|
+
export declare function pipelinePromoteCommand(args: ParsedArgs): Promise<number>;
|
|
57
|
+
export declare function pipelineRunNowCommand(args: ParsedArgs): Promise<number>;
|
|
58
|
+
export declare function pipelineRollbackCommand(args: ParsedArgs): Promise<number>;
|