@sequenceholdings/studio-cli 0.1.13 → 0.1.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +258 -38
- package/dist/agents/apply-chunks.d.ts +13 -0
- package/dist/agents/apply-chunks.js +43 -0
- package/dist/agents/commands.d.ts +10 -0
- package/dist/agents/commands.js +218 -0
- package/dist/agents/scaffold.d.ts +2 -0
- package/dist/agents/scaffold.js +77 -0
- package/dist/agents/source.d.ts +18 -0
- package/dist/agents/source.js +121 -0
- package/dist/artifact/delegate.d.ts +2 -2
- package/dist/artifact/delegate.js +31 -73
- package/dist/atlas-client.js +52 -37
- package/dist/auth-cmds/commands.d.ts +1 -1
- package/dist/auth-cmds/commands.js +12 -7
- package/dist/auth.d.ts +104 -24
- package/dist/auth.js +456 -94
- package/dist/config.d.ts +3 -3
- package/dist/config.js +18 -13
- package/dist/env-catalog.js +13 -3
- package/dist/env-flags.d.ts +2 -0
- package/dist/env-flags.js +2 -0
- package/dist/env-registry.d.ts +27 -0
- package/dist/env-registry.js +204 -0
- package/dist/envs/commands.d.ts +1 -1
- package/dist/envs/commands.js +41 -3
- package/dist/file-lock.d.ts +5 -0
- package/dist/file-lock.js +187 -0
- package/dist/functions/commands.d.ts +10 -10
- package/dist/functions/commands.js +87 -53
- package/dist/functions/manifest.d.ts +1 -0
- package/dist/functions/manifest.js +36 -0
- package/dist/functions/source-selection.d.ts +24 -0
- package/dist/functions/source-selection.js +67 -0
- package/dist/login.d.ts +8 -3
- package/dist/login.js +46 -34
- package/dist/main.d.ts +3 -1
- package/dist/main.js +41 -12
- package/dist/orm/delegate.js +25 -7
- package/dist/pat-hints.js +2 -2
- package/dist/pipeline/commands.d.ts +58 -0
- package/dist/pipeline/commands.js +330 -0
- package/dist/pipeline/lifecycle.d.ts +58 -0
- package/dist/pipeline/lifecycle.js +348 -0
- package/dist/pipeline/pinning.d.ts +5 -0
- package/dist/pipeline/pinning.js +9 -0
- package/dist/pipeline/templates.d.ts +11 -0
- package/dist/pipeline/templates.js +166 -0
- package/dist/process/build.d.ts +4 -0
- package/dist/process/build.js +33 -2
- package/dist/process/codegen.js +19 -1
- package/dist/process/commands.js +97 -47
- package/dist/process/compiler-subprocess.d.ts +29 -0
- package/dist/process/compiler-subprocess.js +99 -0
- package/dist/process/compiler-worker.d.ts +1 -0
- package/dist/process/compiler-worker.js +38 -0
- package/dist/process/lint.d.ts +8 -0
- package/dist/process/lint.js +84 -29
- package/dist/process/repo-install.js +18 -2
- package/dist/repos/commands.d.ts +1 -1
- package/dist/repos/commands.js +17 -12
- package/dist/secrets/commands.d.ts +1 -1
- package/dist/secrets/commands.js +18 -18
- package/package.json +12 -5
|
@@ -0,0 +1,348 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deploy lifecycle verbs for `seq-studio pipeline plan|deploy|promote|rollback`
|
|
3
|
+
* (SEQ-2450). Wraps the /api/data-pipelines plan/deploy APIs.
|
|
4
|
+
*/
|
|
5
|
+
import { getAccessToken, NotLoggedInError } from '../auth.js';
|
|
6
|
+
import { AtlasApiError, getJson, postJson } from '../atlas-client.js';
|
|
7
|
+
import { resolveEnvWithDiscovery } from '../config.js';
|
|
8
|
+
import { isFullSha, requiresPinnedSha } from './pinning.js';
|
|
9
|
+
const LOG = '[seq-studio]';
|
|
10
|
+
const TERMINAL = new Set(['active', 'failed', 'retired']);
|
|
11
|
+
async function envAndToken(args) {
|
|
12
|
+
const requested = typeof args.flags.e === 'string' ? args.flags.e : typeof args.flags.env === 'string' ? args.flags.env : undefined;
|
|
13
|
+
if (!requested) {
|
|
14
|
+
throw new Error('missing -e/--env <environment>');
|
|
15
|
+
}
|
|
16
|
+
const env = await resolveEnvWithDiscovery({ requested });
|
|
17
|
+
let token;
|
|
18
|
+
try {
|
|
19
|
+
token = await getAccessToken();
|
|
20
|
+
}
|
|
21
|
+
catch (error) {
|
|
22
|
+
if (error instanceof NotLoggedInError)
|
|
23
|
+
throw error;
|
|
24
|
+
throw error;
|
|
25
|
+
}
|
|
26
|
+
return { env, token };
|
|
27
|
+
}
|
|
28
|
+
function flagString(flags, key) {
|
|
29
|
+
const value = flags[key];
|
|
30
|
+
if (value === true)
|
|
31
|
+
throw new Error(`--${key} requires a value`);
|
|
32
|
+
return value;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* The CLI's local dev-loop env is named `local` — chosen so `-e local` lines
|
|
36
|
+
* up with `seqapi -e local` / `artifact-studio --env local` (see `config.ts`
|
|
37
|
+
* `BUILT_IN_ENV_URLS`). There is no `local` in the server's deploy-lifecycle
|
|
38
|
+
* enum (`DEPLOY_ENVIRONMENTS` in `atlas/src/server/services/data-pipelines/schema.ts`
|
|
39
|
+
* is `dev | staging | production | banksouth`); the server's name for that
|
|
40
|
+
* same developer-loop target is `dev` (`targetFactsForEnvironment` treats
|
|
41
|
+
* `dev` as the one target exempt from a `lakebase_branch` binding). Map at
|
|
42
|
+
* this one boundary — every lifecycle request body funnels through here —
|
|
43
|
+
* so the CLI never sends the wire-invalid `local` and every other env name
|
|
44
|
+
* passes through unchanged.
|
|
45
|
+
*/
|
|
46
|
+
export function deployEnvironmentForEnv(env) {
|
|
47
|
+
return env.name === 'local' ? 'dev' : env.name;
|
|
48
|
+
}
|
|
49
|
+
export async function pipelinePlanCommand(args) {
|
|
50
|
+
const repo = flagString(args.flags, 'repo');
|
|
51
|
+
const ref = flagString(args.flags, 'ref');
|
|
52
|
+
if (!repo || !ref) {
|
|
53
|
+
console.error('usage: seq-studio pipeline plan --repo pipelines/<slug> --ref <sha|branch> -e <env> [--json]');
|
|
54
|
+
return 1;
|
|
55
|
+
}
|
|
56
|
+
const { env, token } = await envAndToken(args);
|
|
57
|
+
const json = args.flags.json === true || args.flags.json === 'true';
|
|
58
|
+
let response;
|
|
59
|
+
try {
|
|
60
|
+
response = await postJson({
|
|
61
|
+
baseUrl: env.url,
|
|
62
|
+
token,
|
|
63
|
+
path: '/api/data-pipelines/pipelines/plan',
|
|
64
|
+
body: { repo, ref, environment: deployEnvironmentForEnv(env) },
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
catch (error) {
|
|
68
|
+
if (error instanceof AtlasApiError) {
|
|
69
|
+
console.error(`${LOG} plan failed: ${error.message}`);
|
|
70
|
+
return 1;
|
|
71
|
+
}
|
|
72
|
+
throw error;
|
|
73
|
+
}
|
|
74
|
+
if (json) {
|
|
75
|
+
console.log(JSON.stringify(response, null, 2));
|
|
76
|
+
}
|
|
77
|
+
else {
|
|
78
|
+
console.log(response.text);
|
|
79
|
+
printFindingsTable(response.plan.findings);
|
|
80
|
+
}
|
|
81
|
+
return response.plan.hasDestructive ? 1 : 0;
|
|
82
|
+
}
|
|
83
|
+
export async function pipelineDeployCommand(args) {
|
|
84
|
+
const repo = flagString(args.flags, 'repo');
|
|
85
|
+
const ref = flagString(args.flags, 'ref');
|
|
86
|
+
const deploymentId = flagString(args.flags, 'deployment-id');
|
|
87
|
+
const approvedBy = flagString(args.flags, 'approved-by');
|
|
88
|
+
const { env, token } = await envAndToken(args);
|
|
89
|
+
if (requiresPinnedSha(env.name) && ref && !isFullSha(ref)) {
|
|
90
|
+
console.error(`${LOG} deploy to ${env.name} requires a pinned 40-hex commit SHA (got '${ref}')`);
|
|
91
|
+
return 1;
|
|
92
|
+
}
|
|
93
|
+
let enqueue;
|
|
94
|
+
try {
|
|
95
|
+
if (deploymentId) {
|
|
96
|
+
// Direct execute of a persisted plan.
|
|
97
|
+
const stageId = flagString(args.flags, 'stage-id');
|
|
98
|
+
if (!stageId) {
|
|
99
|
+
console.error('usage: seq-studio pipeline deploy --deployment-id <id> --stage-id <id> -e <env> [--approved-by <sub>] [--no-wait]');
|
|
100
|
+
return 1;
|
|
101
|
+
}
|
|
102
|
+
enqueue = await postJson({
|
|
103
|
+
baseUrl: env.url,
|
|
104
|
+
token,
|
|
105
|
+
path: `/api/data-pipelines/stages/${stageId}/deploy`,
|
|
106
|
+
body: { deploymentId, ...(approvedBy ? { approvedBy } : {}) },
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
else {
|
|
110
|
+
if (!repo || !ref) {
|
|
111
|
+
console.error('usage: seq-studio pipeline deploy --repo pipelines/<slug> --ref <sha> -e <env> [--approved-by <sub>] [--no-wait]');
|
|
112
|
+
return 1;
|
|
113
|
+
}
|
|
114
|
+
// Plan then deploy the first planning deployment.
|
|
115
|
+
const plan = await postJson({
|
|
116
|
+
baseUrl: env.url,
|
|
117
|
+
token,
|
|
118
|
+
path: '/api/data-pipelines/pipelines/plan',
|
|
119
|
+
body: { repo, ref, environment: deployEnvironmentForEnv(env) },
|
|
120
|
+
});
|
|
121
|
+
if (plan.plan.hasDestructive) {
|
|
122
|
+
console.error(`${LOG} plan has destructive findings — refusing to deploy`);
|
|
123
|
+
if (args.flags.json === true || args.flags.json === 'true') {
|
|
124
|
+
console.log(JSON.stringify(plan, null, 2));
|
|
125
|
+
}
|
|
126
|
+
else {
|
|
127
|
+
console.error(plan.text);
|
|
128
|
+
}
|
|
129
|
+
return 1;
|
|
130
|
+
}
|
|
131
|
+
const firstId = plan.deploymentIds[0];
|
|
132
|
+
const stageId = plan.stageIds[0];
|
|
133
|
+
if (!firstId || !stageId) {
|
|
134
|
+
console.error(`${LOG} plan produced no deployment rows`);
|
|
135
|
+
return 1;
|
|
136
|
+
}
|
|
137
|
+
enqueue = await postJson({
|
|
138
|
+
baseUrl: env.url,
|
|
139
|
+
token,
|
|
140
|
+
path: `/api/data-pipelines/stages/${stageId}/deploy`,
|
|
141
|
+
body: {
|
|
142
|
+
deploymentId: firstId,
|
|
143
|
+
...(approvedBy ? { approvedBy } : {}),
|
|
144
|
+
},
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
catch (error) {
|
|
149
|
+
if (error instanceof AtlasApiError) {
|
|
150
|
+
console.error(`${LOG} deploy failed: ${error.message}`);
|
|
151
|
+
return 1;
|
|
152
|
+
}
|
|
153
|
+
throw error;
|
|
154
|
+
}
|
|
155
|
+
console.log(`${LOG} enqueued deploy ${enqueue.deploymentId} (trigger=${enqueue.triggerRunId ?? 'n/a'})`);
|
|
156
|
+
if (args.flags['no-wait'] === true || args.flags['no-wait'] === 'true') {
|
|
157
|
+
return 0;
|
|
158
|
+
}
|
|
159
|
+
return pollDeployment({ baseUrl: env.url, token, deploymentId: enqueue.deploymentId });
|
|
160
|
+
}
|
|
161
|
+
export async function pipelinePromoteCommand(args) {
|
|
162
|
+
const stage = flagString(args.flags, 'stage');
|
|
163
|
+
const version = flagString(args.flags, 'version');
|
|
164
|
+
const repo = flagString(args.flags, 'repo');
|
|
165
|
+
const approvedBy = flagString(args.flags, 'approved-by');
|
|
166
|
+
if (!stage || !version) {
|
|
167
|
+
console.error('usage: seq-studio pipeline promote --stage <slug> --version <v> -e <env> [--repo pipelines/<slug>] [--approved-by <you>] [--no-wait]');
|
|
168
|
+
return 1;
|
|
169
|
+
}
|
|
170
|
+
const { env, token } = await envAndToken(args);
|
|
171
|
+
if (requiresPinnedSha(env.name) && !approvedBy) {
|
|
172
|
+
console.error(`${LOG} promote to ${env.name} requires --approved-by <your sub/email> (approvals are self-recorded)`);
|
|
173
|
+
return 1;
|
|
174
|
+
}
|
|
175
|
+
const stageId = await resolveStageIdBySlug({ baseUrl: env.url, token, slug: stage, repo });
|
|
176
|
+
let enqueue;
|
|
177
|
+
try {
|
|
178
|
+
enqueue = await postJson({
|
|
179
|
+
baseUrl: env.url,
|
|
180
|
+
token,
|
|
181
|
+
path: `/api/data-pipelines/stages/${stageId}/promote`,
|
|
182
|
+
body: {
|
|
183
|
+
version,
|
|
184
|
+
environment: deployEnvironmentForEnv(env),
|
|
185
|
+
...(approvedBy ? { approvedBy } : {}),
|
|
186
|
+
},
|
|
187
|
+
});
|
|
188
|
+
}
|
|
189
|
+
catch (error) {
|
|
190
|
+
if (error instanceof AtlasApiError) {
|
|
191
|
+
console.error(`${LOG} promote failed: ${error.message}`);
|
|
192
|
+
return 1;
|
|
193
|
+
}
|
|
194
|
+
throw error;
|
|
195
|
+
}
|
|
196
|
+
console.log(`${LOG} enqueued promote ${enqueue.deploymentId}`);
|
|
197
|
+
if (args.flags['no-wait'] === true || args.flags['no-wait'] === 'true')
|
|
198
|
+
return 0;
|
|
199
|
+
return pollDeployment({ baseUrl: env.url, token, deploymentId: enqueue.deploymentId });
|
|
200
|
+
}
|
|
201
|
+
export async function pipelineRunNowCommand(args) {
|
|
202
|
+
const stage = flagString(args.flags, 'stage');
|
|
203
|
+
const repo = flagString(args.flags, 'repo');
|
|
204
|
+
if (!stage) {
|
|
205
|
+
console.error('usage: seq-studio pipeline run-now --stage <slug> -e <env> [--repo pipelines/<slug>] [--json]');
|
|
206
|
+
return 1;
|
|
207
|
+
}
|
|
208
|
+
const { env, token } = await envAndToken(args);
|
|
209
|
+
const json = args.flags.json === true || args.flags.json === 'true';
|
|
210
|
+
let stageId;
|
|
211
|
+
try {
|
|
212
|
+
stageId = await resolveStageIdBySlug({ baseUrl: env.url, token, slug: stage, repo });
|
|
213
|
+
}
|
|
214
|
+
catch (error) {
|
|
215
|
+
console.error(`${LOG} ${error instanceof Error ? error.message : String(error)}`);
|
|
216
|
+
return 1;
|
|
217
|
+
}
|
|
218
|
+
let response;
|
|
219
|
+
try {
|
|
220
|
+
response = await postJson({
|
|
221
|
+
baseUrl: env.url,
|
|
222
|
+
token,
|
|
223
|
+
path: `/api/data-pipelines/stages/${stageId}/run`,
|
|
224
|
+
body: { environment: deployEnvironmentForEnv(env) },
|
|
225
|
+
});
|
|
226
|
+
}
|
|
227
|
+
catch (error) {
|
|
228
|
+
if (error instanceof AtlasApiError) {
|
|
229
|
+
console.error(`${LOG} run-now failed: ${error.message}`);
|
|
230
|
+
return 1;
|
|
231
|
+
}
|
|
232
|
+
throw error;
|
|
233
|
+
}
|
|
234
|
+
if (json) {
|
|
235
|
+
console.log(JSON.stringify(response, null, 2));
|
|
236
|
+
}
|
|
237
|
+
else {
|
|
238
|
+
console.log(`${LOG} run-now ${stage} → ${deployEnvironmentForEnv(env)}`);
|
|
239
|
+
console.log(`${LOG} invocation ${response.invocationId} (${response.status})`);
|
|
240
|
+
console.log(`${LOG} ${response.databricks_url}`);
|
|
241
|
+
}
|
|
242
|
+
return 0;
|
|
243
|
+
}
|
|
244
|
+
export async function pipelineRollbackCommand(args) {
|
|
245
|
+
const stage = flagString(args.flags, 'stage');
|
|
246
|
+
const repo = flagString(args.flags, 'repo');
|
|
247
|
+
const approvedBy = flagString(args.flags, 'approved-by');
|
|
248
|
+
if (!stage) {
|
|
249
|
+
console.error('usage: seq-studio pipeline rollback --stage <slug> -e <env> [--repo pipelines/<slug>] [--approved-by <you>] [--no-wait]');
|
|
250
|
+
return 1;
|
|
251
|
+
}
|
|
252
|
+
const { env, token } = await envAndToken(args);
|
|
253
|
+
if (requiresPinnedSha(env.name) && !approvedBy) {
|
|
254
|
+
console.error(`${LOG} rollback in ${env.name} requires --approved-by <your sub/email> (approvals are explicit even for rollbacks)`);
|
|
255
|
+
return 1;
|
|
256
|
+
}
|
|
257
|
+
const stageId = await resolveStageIdBySlug({ baseUrl: env.url, token, slug: stage, repo });
|
|
258
|
+
let enqueue;
|
|
259
|
+
try {
|
|
260
|
+
enqueue = await postJson({
|
|
261
|
+
baseUrl: env.url,
|
|
262
|
+
token,
|
|
263
|
+
path: `/api/data-pipelines/stages/${stageId}/rollback`,
|
|
264
|
+
body: {
|
|
265
|
+
environment: deployEnvironmentForEnv(env),
|
|
266
|
+
...(approvedBy ? { approvedBy } : {}),
|
|
267
|
+
},
|
|
268
|
+
});
|
|
269
|
+
}
|
|
270
|
+
catch (error) {
|
|
271
|
+
if (error instanceof AtlasApiError) {
|
|
272
|
+
console.error(`${LOG} rollback failed: ${error.message}`);
|
|
273
|
+
return 1;
|
|
274
|
+
}
|
|
275
|
+
throw error;
|
|
276
|
+
}
|
|
277
|
+
console.log(`${LOG} enqueued rollback ${enqueue.deploymentId}`);
|
|
278
|
+
if (args.flags['no-wait'] === true || args.flags['no-wait'] === 'true')
|
|
279
|
+
return 0;
|
|
280
|
+
return pollDeployment({ baseUrl: env.url, token, deploymentId: enqueue.deploymentId });
|
|
281
|
+
}
|
|
282
|
+
function printFindingsTable(findings) {
|
|
283
|
+
if (findings.length === 0) {
|
|
284
|
+
console.log(`${LOG} no resource changes`);
|
|
285
|
+
return;
|
|
286
|
+
}
|
|
287
|
+
console.log(`${LOG} ${'CLASS'.padEnd(12)} ${'RESOURCE'.padEnd(32)} MESSAGE`);
|
|
288
|
+
for (const f of findings) {
|
|
289
|
+
const klass = f.destructiveReason
|
|
290
|
+
? `${f.classification}/${f.destructiveReason}`
|
|
291
|
+
: f.classification;
|
|
292
|
+
console.log(`${LOG} ${klass.padEnd(12)} ${f.resourceKey.padEnd(32)} ${f.message}`);
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
async function pollDeployment({ baseUrl, token, deploymentId, }) {
|
|
296
|
+
const started = Date.now();
|
|
297
|
+
const timeoutMs = 15 * 60 * 1000;
|
|
298
|
+
while (Date.now() - started < timeoutMs) {
|
|
299
|
+
const detail = await getJson({
|
|
300
|
+
baseUrl,
|
|
301
|
+
token,
|
|
302
|
+
path: `/api/data-pipelines/deployments/${deploymentId}`,
|
|
303
|
+
});
|
|
304
|
+
console.log(`${LOG} deployment ${deploymentId}: ${detail.status}`);
|
|
305
|
+
if (TERMINAL.has(detail.status)) {
|
|
306
|
+
if (detail.status === 'active') {
|
|
307
|
+
console.log(`${LOG} deploy succeeded`);
|
|
308
|
+
return 0;
|
|
309
|
+
}
|
|
310
|
+
console.error(`${LOG} deploy ended '${detail.status}': ${detail.statusDetail ?? ''}`);
|
|
311
|
+
return 1;
|
|
312
|
+
}
|
|
313
|
+
await new Promise((r) => setTimeout(r, 2000));
|
|
314
|
+
}
|
|
315
|
+
console.error(`${LOG} timed out waiting for deployment ${deploymentId}`);
|
|
316
|
+
return 1;
|
|
317
|
+
}
|
|
318
|
+
/**
|
|
319
|
+
* Stage identity is `(repo, slug)` — a bare slug can be ambiguous across
|
|
320
|
+
* Pipelines. `--repo pipelines/<domain>` scopes the lookup server-side; an
|
|
321
|
+
* ambiguous bare slug fails with the candidate repos rather than guessing.
|
|
322
|
+
*
|
|
323
|
+
* The `slug` query param is an EXACT server-side filter (contracts.ts
|
|
324
|
+
* `listStagesQuerySchema`) — the list route's default sort is
|
|
325
|
+
* `updated_at DESC`, so scanning only the first page (as this used to do)
|
|
326
|
+
* silently missed a matching stage buried on a later page, and would have
|
|
327
|
+
* missed a same-slug duplicate on a later page too, falsely reporting the
|
|
328
|
+
* bare-slug lookup as unique (SEQ-2447 review). Filtering server-side by the
|
|
329
|
+
* exact slug means every match is guaranteed to fit on one page regardless
|
|
330
|
+
* of how many other stages exist.
|
|
331
|
+
*/
|
|
332
|
+
async function resolveStageIdBySlug({ baseUrl, token, slug, repo, }) {
|
|
333
|
+
const repoFilter = repo ? `&repo=${encodeURIComponent(repo)}` : '';
|
|
334
|
+
const listed = await getJson({
|
|
335
|
+
baseUrl,
|
|
336
|
+
token,
|
|
337
|
+
path: `/api/data-pipelines/stages?limit=200&slug=${encodeURIComponent(slug)}${repoFilter}`,
|
|
338
|
+
});
|
|
339
|
+
const first = listed.stages[0];
|
|
340
|
+
if (!first) {
|
|
341
|
+
throw new Error(`stage '${slug}' not found${repo ? ` in ${repo}` : ''}`);
|
|
342
|
+
}
|
|
343
|
+
if (listed.stages.length > 1) {
|
|
344
|
+
throw new Error(`stage '${slug}' exists in multiple Pipelines (${listed.stages.map((m) => m.repo).join(', ')}) — ` +
|
|
345
|
+
'disambiguate with --repo pipelines/<domain>');
|
|
346
|
+
}
|
|
347
|
+
return first.id;
|
|
348
|
+
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
/** Client-side pin rules mirrored from the server deploy adapter (SEQ-2450). */
|
|
2
|
+
export declare const FULL_SHA_PATTERN: RegExp;
|
|
3
|
+
export declare const PINNED_ENVIRONMENTS: Set<string>;
|
|
4
|
+
export declare function isFullSha(ref: string): boolean;
|
|
5
|
+
export declare function requiresPinnedSha(environment: string): boolean;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/** Client-side pin rules mirrored from the server deploy adapter (SEQ-2450). */
|
|
2
|
+
export const FULL_SHA_PATTERN = /^[0-9a-f]{40}$/i;
|
|
3
|
+
export const PINNED_ENVIRONMENTS = new Set(['production', 'banksouth']);
|
|
4
|
+
export function isFullSha(ref) {
|
|
5
|
+
return FULL_SHA_PATTERN.test(ref);
|
|
6
|
+
}
|
|
7
|
+
export function requiresPinnedSha(environment) {
|
|
8
|
+
return PINNED_ENVIRONMENTS.has(environment);
|
|
9
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `seq-studio pipeline init` scaffolds — commented per-kind `<name>.stage.yml`
|
|
3
|
+
* templates plus an entrypoint stub. Kept as string constants (not a
|
|
4
|
+
* templates/ directory) so the pipeline family needs no copy step and works
|
|
5
|
+
* from any install of the CLI.
|
|
6
|
+
*/
|
|
7
|
+
export declare const STAGE_TYPES: readonly ["ingestion", "transformation", "serving"];
|
|
8
|
+
export type StageTemplateType = (typeof STAGE_TYPES)[number];
|
|
9
|
+
export declare function renderStageTemplate(name: string, type: StageTemplateType): string;
|
|
10
|
+
/** Serving stages are declarative — no entrypoint stub. */
|
|
11
|
+
export declare function renderEntrypointStub(name: string): string;
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `seq-studio pipeline init` scaffolds — commented per-kind `<name>.stage.yml`
|
|
3
|
+
* templates plus an entrypoint stub. Kept as string constants (not a
|
|
4
|
+
* templates/ directory) so the pipeline family needs no copy step and works
|
|
5
|
+
* from any install of the CLI.
|
|
6
|
+
*/
|
|
7
|
+
const envelopeTemplate = (name, type) => `# ${name}.stage.yml — one deployable ${type} stage of this Pipeline.
|
|
8
|
+
# Docs: @sequenceholdings/pipeline-spec (the schema is the single source of truth).
|
|
9
|
+
schema_version: 1
|
|
10
|
+
stage: ${name}
|
|
11
|
+
title: TODO — one-line human title
|
|
12
|
+
description: >-
|
|
13
|
+
TODO — what this stage does and why it exists.
|
|
14
|
+
type: ${type}
|
|
15
|
+
owners:
|
|
16
|
+
- you@seqholdings.com
|
|
17
|
+
# critical | standard | experimental
|
|
18
|
+
criticality: experimental
|
|
19
|
+
# Optional reliability posture:
|
|
20
|
+
# slos:
|
|
21
|
+
# freshness: PT4H # ISO-8601 durations
|
|
22
|
+
# max_duration: PT1H
|
|
23
|
+
# alerts:
|
|
24
|
+
# channels: [eng-alerts] # symbolic channel refs — never webhook URLs
|
|
25
|
+
# runbook: https://notion.so/your-runbook
|
|
26
|
+
environments:
|
|
27
|
+
- staging
|
|
28
|
+
`;
|
|
29
|
+
const INGESTION_BODY = `
|
|
30
|
+
# Where the ingestion code runs: databricks (API pulls) | edge-worker (on-prem VM).
|
|
31
|
+
# ('upload' is reserved — not yet supported.)
|
|
32
|
+
runtime: databricks
|
|
33
|
+
# databricks runtime pulls from an external system (symbolic refs, never secrets).
|
|
34
|
+
# Secure-by-default: declare a credential family for authenticated APIs.
|
|
35
|
+
# For genuinely keyless / public sources, omit credential entirely (do not
|
|
36
|
+
# invent a sentinel binding) — e.g. source: { system: espn }.
|
|
37
|
+
source:
|
|
38
|
+
system: my_source_system
|
|
39
|
+
credential: my_source_credential
|
|
40
|
+
# edge-worker runtime instead binds a registered worker:
|
|
41
|
+
# worker:
|
|
42
|
+
# machine: my_vm
|
|
43
|
+
# data_source: my_library
|
|
44
|
+
# credential: my_cred_family
|
|
45
|
+
feeds:
|
|
46
|
+
# One feed = one stream of raw files into the volume landing zone.
|
|
47
|
+
# The platform derives (per-source catalog, never the deployment catalog):
|
|
48
|
+
# /Volumes/src_<source>/bronze/<feed>/batch_id=<ts>/, the bronze table
|
|
49
|
+
# src_<source>.bronze.<feed>, envelope columns (batch_id, ingested_at,
|
|
50
|
+
# source_system, content_hash), and a presence manifest for full_snapshot.
|
|
51
|
+
my_feed:
|
|
52
|
+
# full_snapshot | incremental | event_stream
|
|
53
|
+
snapshot_mode: full_snapshot
|
|
54
|
+
natural_key:
|
|
55
|
+
- id
|
|
56
|
+
# incremental feeds declare their cursor:
|
|
57
|
+
# cursor:
|
|
58
|
+
# column: updated_at
|
|
59
|
+
# encoding: yyyyddd # extensible string — provider-neutral
|
|
60
|
+
# none | contains_pii
|
|
61
|
+
pii: none
|
|
62
|
+
# parquet | json | csv | custom ('xlsx' is reserved — not yet supported)
|
|
63
|
+
format: json
|
|
64
|
+
# format: custom requires a pure file→rows decoder (never writes tables):
|
|
65
|
+
# decoder: src/decoders/my_decoder.py
|
|
66
|
+
schedule:
|
|
67
|
+
cron: "0 6 * * *"
|
|
68
|
+
tz: America/New_York
|
|
69
|
+
# Max staleness before the freshness SLO breaches:
|
|
70
|
+
freshness_slo: P1D
|
|
71
|
+
# Raw-file replay buffer (not an archive):
|
|
72
|
+
retention: P30D
|
|
73
|
+
entrypoint: src/{{snake}}.py
|
|
74
|
+
`;
|
|
75
|
+
const TRANSFORMATION_BODY = `
|
|
76
|
+
# dlt | spark-job | polars
|
|
77
|
+
engine: dlt
|
|
78
|
+
inputs:
|
|
79
|
+
# Reference upstream assets by stage + feed (ingestion) or output:
|
|
80
|
+
# - stage: my-ingest
|
|
81
|
+
# feed: my_feed
|
|
82
|
+
# - stage: upstream-transform
|
|
83
|
+
# output: silver.some_table
|
|
84
|
+
# columns: [id, amount] # the subset you consume (validated)
|
|
85
|
+
# Lakebase CDF reverse-sync input (columns REQUIRED, never inferred):
|
|
86
|
+
# - kind: lakebase_cdf
|
|
87
|
+
# table: manual.my_overrides
|
|
88
|
+
# fold: latest_state # latest_state | events
|
|
89
|
+
# columns:
|
|
90
|
+
# id: { type: string, nullable: false, key: true }
|
|
91
|
+
[]
|
|
92
|
+
trigger:
|
|
93
|
+
# At least one of cron / after:
|
|
94
|
+
cron: "0 7 * * *"
|
|
95
|
+
# after: [my-ingest]
|
|
96
|
+
outputs:
|
|
97
|
+
# Typed contracts — names must be silver.* or gold.*:
|
|
98
|
+
silver.{{snake}}:
|
|
99
|
+
columns:
|
|
100
|
+
id:
|
|
101
|
+
type: string
|
|
102
|
+
nullable: false
|
|
103
|
+
key: true
|
|
104
|
+
value:
|
|
105
|
+
type: int
|
|
106
|
+
# fail | quarantine (→ silver.{{snake}}__quarantine) | warn
|
|
107
|
+
on_type_violation: fail
|
|
108
|
+
# additive-only (default: breaking changes fail CI) | versioned
|
|
109
|
+
compatibility: additive-only
|
|
110
|
+
# overwrite | append | auto_cdc
|
|
111
|
+
write_pattern: overwrite
|
|
112
|
+
entrypoint: src/{{snake}}.py
|
|
113
|
+
`;
|
|
114
|
+
const SERVING_BODY = `
|
|
115
|
+
# The transformation output this stage serves:
|
|
116
|
+
source:
|
|
117
|
+
stage: my-gold-stage
|
|
118
|
+
output: gold.my_table
|
|
119
|
+
projection:
|
|
120
|
+
# The served contract (inline columns or schema_ref):
|
|
121
|
+
columns:
|
|
122
|
+
id:
|
|
123
|
+
type: uuid
|
|
124
|
+
nullable: false
|
|
125
|
+
key: true
|
|
126
|
+
name:
|
|
127
|
+
type: string
|
|
128
|
+
primary_key:
|
|
129
|
+
- id
|
|
130
|
+
# SNAPSHOT | TRIGGERED | CONTINUOUS (overwrite-written sources: SNAPSHOT only)
|
|
131
|
+
sync_mode: SNAPSHOT
|
|
132
|
+
trigger:
|
|
133
|
+
after: my-gold-stage
|
|
134
|
+
expose:
|
|
135
|
+
view: public.my_table
|
|
136
|
+
# Optional ORM coupling ('managed' mode is reserved — not yet supported):
|
|
137
|
+
# orm_binding:
|
|
138
|
+
# namespace: my_namespace
|
|
139
|
+
# table: my_table
|
|
140
|
+
# mode: referenced
|
|
141
|
+
`;
|
|
142
|
+
const ENTRYPOINT_STUB = `"""Entrypoint stub for the {{name}} stage.
|
|
143
|
+
|
|
144
|
+
The platform invokes this per the stage spec ({{name}}.stage.yml).
|
|
145
|
+
Decoders and transforms stay pure — bronze materialization, envelope
|
|
146
|
+
stamping, and table writes are platform-owned.
|
|
147
|
+
"""
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def main() -> None:
|
|
151
|
+
raise NotImplementedError("implement the {{name}} stage")
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
if __name__ == "__main__":
|
|
155
|
+
main()
|
|
156
|
+
`;
|
|
157
|
+
export const STAGE_TYPES = ['ingestion', 'transformation', 'serving'];
|
|
158
|
+
export function renderStageTemplate(name, type) {
|
|
159
|
+
const body = type === 'ingestion' ? INGESTION_BODY : type === 'transformation' ? TRANSFORMATION_BODY : SERVING_BODY;
|
|
160
|
+
return (envelopeTemplate(name, type) +
|
|
161
|
+
body.replace(/\{\{name\}\}/g, name).replace(/\{\{snake\}\}/g, name.replace(/-/g, '_')));
|
|
162
|
+
}
|
|
163
|
+
/** Serving stages are declarative — no entrypoint stub. */
|
|
164
|
+
export function renderEntrypointStub(name) {
|
|
165
|
+
return ENTRYPOINT_STUB.replace(/\{\{name\}\}/g, name);
|
|
166
|
+
}
|
package/dist/process/build.d.ts
CHANGED
|
@@ -18,4 +18,8 @@ export interface BuildBundleOptions {
|
|
|
18
18
|
};
|
|
19
19
|
}
|
|
20
20
|
export declare function buildBundleFromProcesses(defs: readonly LoadedProcess[], options?: BuildBundleOptions): Promise<LatticeBundle>;
|
|
21
|
+
export declare function finalizeCompiledBundle({ bundle, resolveProcessPin, }: {
|
|
22
|
+
bundle: LatticeBundle;
|
|
23
|
+
resolveProcessPin: ResolveProcessPin;
|
|
24
|
+
}): Promise<LatticeBundle>;
|
|
21
25
|
export declare function summarizeBundle(bundle: LatticeBundle): LatticeBundleSummary;
|
package/dist/process/build.js
CHANGED
|
@@ -27,6 +27,16 @@ export async function buildBundleFromProcesses(defs, options) {
|
|
|
27
27
|
// the SDK so a CLI-built bundle is byte-identical to a server-applied one.
|
|
28
28
|
return finalizeBundleIdentity({ processes, metadata });
|
|
29
29
|
}
|
|
30
|
+
export async function finalizeCompiledBundle({ bundle, resolveProcessPin, }) {
|
|
31
|
+
const processes = bundle.processes.map((process) => structuredClone(process));
|
|
32
|
+
const localProcessIds = new Set(processes.map((process) => process.id));
|
|
33
|
+
await resolveSubprocessPinsInProcesses(processes, async (processId, version) => {
|
|
34
|
+
if (!version && localProcessIds.has(processId))
|
|
35
|
+
return null;
|
|
36
|
+
return resolveProcessPin(processId, version);
|
|
37
|
+
});
|
|
38
|
+
return finalizeBundleIdentity({ processes, metadata: bundle.metadata });
|
|
39
|
+
}
|
|
30
40
|
async function resolveSubprocessPinsInProcesses(processes, resolve) {
|
|
31
41
|
const pins = [];
|
|
32
42
|
for (const proc of processes) {
|
|
@@ -86,8 +96,20 @@ function serializeProcess(input) {
|
|
|
86
96
|
...(input.process.supervisor_retry !== undefined
|
|
87
97
|
? { supervisor_retry: serializeRetryConfig(input.process.supervisor_retry) }
|
|
88
98
|
: {}),
|
|
99
|
+
// Both run_as keys are omitted when unset so existing hashes stay stable
|
|
100
|
+
// (an email-only run_as serializes byte-identically to before the
|
|
101
|
+
// service_account field existed).
|
|
89
102
|
...(input.process.run_as !== undefined
|
|
90
|
-
? {
|
|
103
|
+
? {
|
|
104
|
+
run_as: {
|
|
105
|
+
...(input.process.run_as.email !== undefined
|
|
106
|
+
? { email: input.process.run_as.email }
|
|
107
|
+
: {}),
|
|
108
|
+
...(input.process.run_as.service_account !== undefined
|
|
109
|
+
? { service_account: input.process.run_as.service_account }
|
|
110
|
+
: {}),
|
|
111
|
+
},
|
|
112
|
+
}
|
|
91
113
|
: {}),
|
|
92
114
|
};
|
|
93
115
|
}
|
|
@@ -135,6 +157,10 @@ function serializeNodeMetadata(node) {
|
|
|
135
157
|
meta.limits = native.limits;
|
|
136
158
|
if (native.bindings)
|
|
137
159
|
meta.bindings = native.bindings;
|
|
160
|
+
// Per-node service-account override — omitted when unset for hash stability.
|
|
161
|
+
if (native.run_as?.service_account) {
|
|
162
|
+
meta.run_as = { service_account: native.run_as.service_account };
|
|
163
|
+
}
|
|
138
164
|
const nativeInputMapper = native.input;
|
|
139
165
|
if (typeof nativeInputMapper === 'function') {
|
|
140
166
|
meta.input_mapper_source = nativeInputMapper.toString();
|
|
@@ -164,8 +190,9 @@ function serializeNodeMetadata(node) {
|
|
|
164
190
|
const human = node;
|
|
165
191
|
const meta = {
|
|
166
192
|
metadata: human.metadata ?? {},
|
|
167
|
-
timeout: human.timeout ?? '7d',
|
|
168
193
|
};
|
|
194
|
+
if (human.timeout !== undefined)
|
|
195
|
+
meta.timeout = human.timeout;
|
|
169
196
|
if (human.on_timeout_edge_id)
|
|
170
197
|
meta.on_timeout_edge_id = human.on_timeout_edge_id;
|
|
171
198
|
if (human.completeLabel)
|
|
@@ -327,6 +354,10 @@ function serializeNodeMetadata(node) {
|
|
|
327
354
|
meta.limits = mf.limits;
|
|
328
355
|
if (mf.bindings)
|
|
329
356
|
meta.bindings = mf.bindings;
|
|
357
|
+
// Per-node service-account override — omitted when unset for hash stability.
|
|
358
|
+
if (mf.run_as?.service_account) {
|
|
359
|
+
meta.run_as = { service_account: mf.run_as.service_account };
|
|
360
|
+
}
|
|
330
361
|
// NOTE: `middleware` (e.g. withRetry) is serialized generically for every
|
|
331
362
|
// node kind in `serializeNode` above — don't duplicate it here.
|
|
332
363
|
const inputMapper = mf.input;
|
package/dist/process/codegen.js
CHANGED
|
@@ -50,7 +50,16 @@ export function generateProcessFiles(process, bundle) {
|
|
|
50
50
|
? ` max_concurrent_runs: ${process.max_concurrent_runs},`
|
|
51
51
|
: null,
|
|
52
52
|
process.supervisor_retry !== undefined ? ` supervisor_retry: ${pretty(process.supervisor_retry, 1)},` : null,
|
|
53
|
-
process.run_as !== undefined
|
|
53
|
+
process.run_as !== undefined
|
|
54
|
+
? ` run_as: { ${[
|
|
55
|
+
process.run_as.email !== undefined ? `email: ${lit(process.run_as.email)}` : null,
|
|
56
|
+
process.run_as.service_account !== undefined
|
|
57
|
+
? `service_account: ${lit(process.run_as.service_account)}`
|
|
58
|
+
: null,
|
|
59
|
+
]
|
|
60
|
+
.filter((f) => f !== null)
|
|
61
|
+
.join(', ')} },`
|
|
62
|
+
: null,
|
|
54
63
|
Array.isArray(process.tags) && process.tags.length > 0
|
|
55
64
|
? ` tags: ${pretty(process.tags, 1)},`
|
|
56
65
|
: null,
|
|
@@ -125,6 +134,7 @@ function emitNode(node, ctx) {
|
|
|
125
134
|
` function: ${lit(String(m.registered_function_id))},`,
|
|
126
135
|
m.config !== undefined ? ` config: ${pretty(m.config, 1)},` : null,
|
|
127
136
|
m.limits !== undefined ? ` limits: ${pretty(m.limits, 1)},` : null,
|
|
137
|
+
runAsField(m.run_as),
|
|
128
138
|
bindings(m.bindings),
|
|
129
139
|
middleware(m.middleware),
|
|
130
140
|
mapper('input', m.input_mapper_source),
|
|
@@ -189,6 +199,7 @@ function emitNode(node, ctx) {
|
|
|
189
199
|
` function: ${lit(String(m.function_id))},`,
|
|
190
200
|
m.version ? ` version: ${lit(String(m.version))},` : null,
|
|
191
201
|
m.limits !== undefined ? ` limits: ${pretty(m.limits, 1)},` : null,
|
|
202
|
+
runAsField(m.run_as),
|
|
192
203
|
bindings(m.bindings),
|
|
193
204
|
middleware(m.middleware),
|
|
194
205
|
mapper('input', m.input_mapper_source),
|
|
@@ -209,6 +220,13 @@ function emitEdges(edges) {
|
|
|
209
220
|
const items = edges.map((e) => `{ id: ${lit(e.id)}, to: ${lit(e.to)} }`).join(', ');
|
|
210
221
|
return `[${items}] as const`;
|
|
211
222
|
}
|
|
223
|
+
/** Per-node service-account override — `metadata.run_as.service_account`. */
|
|
224
|
+
function runAsField(runAs) {
|
|
225
|
+
const sa = runAs?.service_account;
|
|
226
|
+
if (typeof sa !== 'string' || sa.length === 0)
|
|
227
|
+
return null;
|
|
228
|
+
return ` run_as: { service_account: ${lit(sa)} },`;
|
|
229
|
+
}
|
|
212
230
|
/** A mapper field whose value is a serialized `Function.toString()` string. */
|
|
213
231
|
function mapper(field, source) {
|
|
214
232
|
if (typeof source !== 'string' || source.length === 0)
|