@sequenceholdings/studio-cli 0.1.12 → 0.1.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +116 -30
- package/dist/artifact/delegate.d.ts +2 -2
- package/dist/artifact/delegate.js +31 -73
- package/dist/atlas-client.js +52 -37
- package/dist/auth-cmds/commands.d.ts +1 -1
- package/dist/auth-cmds/commands.js +12 -7
- package/dist/auth.d.ts +97 -19
- package/dist/auth.js +376 -81
- package/dist/config.d.ts +13 -3
- package/dist/config.js +41 -14
- package/dist/env-catalog.js +13 -3
- package/dist/env-flags.d.ts +2 -0
- package/dist/env-flags.js +2 -0
- package/dist/env-registry.d.ts +27 -0
- package/dist/env-registry.js +204 -0
- package/dist/envs/commands.d.ts +1 -1
- package/dist/envs/commands.js +65 -10
- package/dist/file-lock.d.ts +5 -0
- package/dist/file-lock.js +187 -0
- package/dist/functions/commands.d.ts +9 -1
- package/dist/functions/commands.js +71 -29
- package/dist/functions/manifest.d.ts +1 -0
- package/dist/functions/manifest.js +36 -0
- package/dist/login.d.ts +14 -3
- package/dist/login.js +60 -40
- package/dist/main.d.ts +2 -1
- package/dist/main.js +36 -12
- package/dist/orm/delegate.d.ts +9 -0
- package/dist/orm/delegate.js +36 -4
- package/dist/pat-hints.js +2 -2
- package/dist/pipeline/commands.d.ts +58 -0
- package/dist/pipeline/commands.js +330 -0
- package/dist/pipeline/lifecycle.d.ts +58 -0
- package/dist/pipeline/lifecycle.js +348 -0
- package/dist/pipeline/pinning.d.ts +5 -0
- package/dist/pipeline/pinning.js +9 -0
- package/dist/pipeline/templates.d.ts +11 -0
- package/dist/pipeline/templates.js +166 -0
- package/dist/process/build.d.ts +12 -1
- package/dist/process/build.js +49 -2
- package/dist/process/codegen.js +21 -1
- package/dist/process/commands.d.ts +19 -0
- package/dist/process/commands.js +153 -40
- package/dist/process/compiler-subprocess.d.ts +29 -0
- package/dist/process/compiler-subprocess.js +99 -0
- package/dist/process/compiler-worker.d.ts +1 -0
- package/dist/process/compiler-worker.js +38 -0
- package/dist/process/discover.d.ts +4 -1
- package/dist/process/discover.js +5 -2
- package/dist/process/lint.d.ts +8 -0
- package/dist/process/lint.js +125 -29
- package/dist/process/repo-install.d.ts +21 -0
- package/dist/process/repo-install.js +99 -0
- package/dist/process/simulate.js +14 -1
- package/dist/repos/commands.d.ts +1 -1
- package/dist/repos/commands.js +17 -12
- package/dist/secrets/commands.d.ts +1 -1
- package/dist/secrets/commands.js +18 -18
- package/package.json +9 -4
|
@@ -0,0 +1,348 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deploy lifecycle verbs for `seq-studio pipeline plan|deploy|promote|rollback`
|
|
3
|
+
* (SEQ-2450). Wraps the /api/data-pipelines plan/deploy APIs.
|
|
4
|
+
*/
|
|
5
|
+
import { getAccessToken, NotLoggedInError } from '../auth.js';
|
|
6
|
+
import { AtlasApiError, getJson, postJson } from '../atlas-client.js';
|
|
7
|
+
import { resolveEnvWithDiscovery } from '../config.js';
|
|
8
|
+
import { isFullSha, requiresPinnedSha } from './pinning.js';
|
|
9
|
+
const LOG = '[seq-studio]';
|
|
10
|
+
const TERMINAL = new Set(['active', 'failed', 'retired']);
|
|
11
|
+
async function envAndToken(args) {
|
|
12
|
+
const requested = typeof args.flags.e === 'string' ? args.flags.e : typeof args.flags.env === 'string' ? args.flags.env : undefined;
|
|
13
|
+
if (!requested) {
|
|
14
|
+
throw new Error('missing -e/--env <environment>');
|
|
15
|
+
}
|
|
16
|
+
const env = await resolveEnvWithDiscovery({ requested });
|
|
17
|
+
let token;
|
|
18
|
+
try {
|
|
19
|
+
token = await getAccessToken();
|
|
20
|
+
}
|
|
21
|
+
catch (error) {
|
|
22
|
+
if (error instanceof NotLoggedInError)
|
|
23
|
+
throw error;
|
|
24
|
+
throw error;
|
|
25
|
+
}
|
|
26
|
+
return { env, token };
|
|
27
|
+
}
|
|
28
|
+
function flagString(flags, key) {
|
|
29
|
+
const value = flags[key];
|
|
30
|
+
if (value === true)
|
|
31
|
+
throw new Error(`--${key} requires a value`);
|
|
32
|
+
return value;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* The CLI's local dev-loop env is named `local` — chosen so `-e local` lines
|
|
36
|
+
* up with `seqapi -e local` / `artifact-studio --env local` (see `config.ts`
|
|
37
|
+
* `BUILT_IN_ENV_URLS`). There is no `local` in the server's deploy-lifecycle
|
|
38
|
+
* enum (`DEPLOY_ENVIRONMENTS` in `atlas/src/server/services/data-pipelines/schema.ts`
|
|
39
|
+
* is `dev | staging | production | banksouth`); the server's name for that
|
|
40
|
+
* same developer-loop target is `dev` (`targetFactsForEnvironment` treats
|
|
41
|
+
* `dev` as the one target exempt from a `lakebase_branch` binding). Map at
|
|
42
|
+
* this one boundary — every lifecycle request body funnels through here —
|
|
43
|
+
* so the CLI never sends the wire-invalid `local` and every other env name
|
|
44
|
+
* passes through unchanged.
|
|
45
|
+
*/
|
|
46
|
+
export function deployEnvironmentForEnv(env) {
|
|
47
|
+
return env.name === 'local' ? 'dev' : env.name;
|
|
48
|
+
}
|
|
49
|
+
export async function pipelinePlanCommand(args) {
|
|
50
|
+
const repo = flagString(args.flags, 'repo');
|
|
51
|
+
const ref = flagString(args.flags, 'ref');
|
|
52
|
+
if (!repo || !ref) {
|
|
53
|
+
console.error('usage: seq-studio pipeline plan --repo pipelines/<slug> --ref <sha|branch> -e <env> [--json]');
|
|
54
|
+
return 1;
|
|
55
|
+
}
|
|
56
|
+
const { env, token } = await envAndToken(args);
|
|
57
|
+
const json = args.flags.json === true || args.flags.json === 'true';
|
|
58
|
+
let response;
|
|
59
|
+
try {
|
|
60
|
+
response = await postJson({
|
|
61
|
+
baseUrl: env.url,
|
|
62
|
+
token,
|
|
63
|
+
path: '/api/data-pipelines/pipelines/plan',
|
|
64
|
+
body: { repo, ref, environment: deployEnvironmentForEnv(env) },
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
catch (error) {
|
|
68
|
+
if (error instanceof AtlasApiError) {
|
|
69
|
+
console.error(`${LOG} plan failed: ${error.message}`);
|
|
70
|
+
return 1;
|
|
71
|
+
}
|
|
72
|
+
throw error;
|
|
73
|
+
}
|
|
74
|
+
if (json) {
|
|
75
|
+
console.log(JSON.stringify(response, null, 2));
|
|
76
|
+
}
|
|
77
|
+
else {
|
|
78
|
+
console.log(response.text);
|
|
79
|
+
printFindingsTable(response.plan.findings);
|
|
80
|
+
}
|
|
81
|
+
return response.plan.hasDestructive ? 1 : 0;
|
|
82
|
+
}
|
|
83
|
+
export async function pipelineDeployCommand(args) {
|
|
84
|
+
const repo = flagString(args.flags, 'repo');
|
|
85
|
+
const ref = flagString(args.flags, 'ref');
|
|
86
|
+
const deploymentId = flagString(args.flags, 'deployment-id');
|
|
87
|
+
const approvedBy = flagString(args.flags, 'approved-by');
|
|
88
|
+
const { env, token } = await envAndToken(args);
|
|
89
|
+
if (requiresPinnedSha(env.name) && ref && !isFullSha(ref)) {
|
|
90
|
+
console.error(`${LOG} deploy to ${env.name} requires a pinned 40-hex commit SHA (got '${ref}')`);
|
|
91
|
+
return 1;
|
|
92
|
+
}
|
|
93
|
+
let enqueue;
|
|
94
|
+
try {
|
|
95
|
+
if (deploymentId) {
|
|
96
|
+
// Direct execute of a persisted plan.
|
|
97
|
+
const stageId = flagString(args.flags, 'stage-id');
|
|
98
|
+
if (!stageId) {
|
|
99
|
+
console.error('usage: seq-studio pipeline deploy --deployment-id <id> --stage-id <id> -e <env> [--approved-by <sub>] [--no-wait]');
|
|
100
|
+
return 1;
|
|
101
|
+
}
|
|
102
|
+
enqueue = await postJson({
|
|
103
|
+
baseUrl: env.url,
|
|
104
|
+
token,
|
|
105
|
+
path: `/api/data-pipelines/stages/${stageId}/deploy`,
|
|
106
|
+
body: { deploymentId, ...(approvedBy ? { approvedBy } : {}) },
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
else {
|
|
110
|
+
if (!repo || !ref) {
|
|
111
|
+
console.error('usage: seq-studio pipeline deploy --repo pipelines/<slug> --ref <sha> -e <env> [--approved-by <sub>] [--no-wait]');
|
|
112
|
+
return 1;
|
|
113
|
+
}
|
|
114
|
+
// Plan then deploy the first planning deployment.
|
|
115
|
+
const plan = await postJson({
|
|
116
|
+
baseUrl: env.url,
|
|
117
|
+
token,
|
|
118
|
+
path: '/api/data-pipelines/pipelines/plan',
|
|
119
|
+
body: { repo, ref, environment: deployEnvironmentForEnv(env) },
|
|
120
|
+
});
|
|
121
|
+
if (plan.plan.hasDestructive) {
|
|
122
|
+
console.error(`${LOG} plan has destructive findings — refusing to deploy`);
|
|
123
|
+
if (args.flags.json === true || args.flags.json === 'true') {
|
|
124
|
+
console.log(JSON.stringify(plan, null, 2));
|
|
125
|
+
}
|
|
126
|
+
else {
|
|
127
|
+
console.error(plan.text);
|
|
128
|
+
}
|
|
129
|
+
return 1;
|
|
130
|
+
}
|
|
131
|
+
const firstId = plan.deploymentIds[0];
|
|
132
|
+
const stageId = plan.stageIds[0];
|
|
133
|
+
if (!firstId || !stageId) {
|
|
134
|
+
console.error(`${LOG} plan produced no deployment rows`);
|
|
135
|
+
return 1;
|
|
136
|
+
}
|
|
137
|
+
enqueue = await postJson({
|
|
138
|
+
baseUrl: env.url,
|
|
139
|
+
token,
|
|
140
|
+
path: `/api/data-pipelines/stages/${stageId}/deploy`,
|
|
141
|
+
body: {
|
|
142
|
+
deploymentId: firstId,
|
|
143
|
+
...(approvedBy ? { approvedBy } : {}),
|
|
144
|
+
},
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
catch (error) {
|
|
149
|
+
if (error instanceof AtlasApiError) {
|
|
150
|
+
console.error(`${LOG} deploy failed: ${error.message}`);
|
|
151
|
+
return 1;
|
|
152
|
+
}
|
|
153
|
+
throw error;
|
|
154
|
+
}
|
|
155
|
+
console.log(`${LOG} enqueued deploy ${enqueue.deploymentId} (trigger=${enqueue.triggerRunId ?? 'n/a'})`);
|
|
156
|
+
if (args.flags['no-wait'] === true || args.flags['no-wait'] === 'true') {
|
|
157
|
+
return 0;
|
|
158
|
+
}
|
|
159
|
+
return pollDeployment({ baseUrl: env.url, token, deploymentId: enqueue.deploymentId });
|
|
160
|
+
}
|
|
161
|
+
export async function pipelinePromoteCommand(args) {
|
|
162
|
+
const stage = flagString(args.flags, 'stage');
|
|
163
|
+
const version = flagString(args.flags, 'version');
|
|
164
|
+
const repo = flagString(args.flags, 'repo');
|
|
165
|
+
const approvedBy = flagString(args.flags, 'approved-by');
|
|
166
|
+
if (!stage || !version) {
|
|
167
|
+
console.error('usage: seq-studio pipeline promote --stage <slug> --version <v> -e <env> [--repo pipelines/<slug>] [--approved-by <you>] [--no-wait]');
|
|
168
|
+
return 1;
|
|
169
|
+
}
|
|
170
|
+
const { env, token } = await envAndToken(args);
|
|
171
|
+
if (requiresPinnedSha(env.name) && !approvedBy) {
|
|
172
|
+
console.error(`${LOG} promote to ${env.name} requires --approved-by <your sub/email> (approvals are self-recorded)`);
|
|
173
|
+
return 1;
|
|
174
|
+
}
|
|
175
|
+
const stageId = await resolveStageIdBySlug({ baseUrl: env.url, token, slug: stage, repo });
|
|
176
|
+
let enqueue;
|
|
177
|
+
try {
|
|
178
|
+
enqueue = await postJson({
|
|
179
|
+
baseUrl: env.url,
|
|
180
|
+
token,
|
|
181
|
+
path: `/api/data-pipelines/stages/${stageId}/promote`,
|
|
182
|
+
body: {
|
|
183
|
+
version,
|
|
184
|
+
environment: deployEnvironmentForEnv(env),
|
|
185
|
+
...(approvedBy ? { approvedBy } : {}),
|
|
186
|
+
},
|
|
187
|
+
});
|
|
188
|
+
}
|
|
189
|
+
catch (error) {
|
|
190
|
+
if (error instanceof AtlasApiError) {
|
|
191
|
+
console.error(`${LOG} promote failed: ${error.message}`);
|
|
192
|
+
return 1;
|
|
193
|
+
}
|
|
194
|
+
throw error;
|
|
195
|
+
}
|
|
196
|
+
console.log(`${LOG} enqueued promote ${enqueue.deploymentId}`);
|
|
197
|
+
if (args.flags['no-wait'] === true || args.flags['no-wait'] === 'true')
|
|
198
|
+
return 0;
|
|
199
|
+
return pollDeployment({ baseUrl: env.url, token, deploymentId: enqueue.deploymentId });
|
|
200
|
+
}
|
|
201
|
+
export async function pipelineRunNowCommand(args) {
|
|
202
|
+
const stage = flagString(args.flags, 'stage');
|
|
203
|
+
const repo = flagString(args.flags, 'repo');
|
|
204
|
+
if (!stage) {
|
|
205
|
+
console.error('usage: seq-studio pipeline run-now --stage <slug> -e <env> [--repo pipelines/<slug>] [--json]');
|
|
206
|
+
return 1;
|
|
207
|
+
}
|
|
208
|
+
const { env, token } = await envAndToken(args);
|
|
209
|
+
const json = args.flags.json === true || args.flags.json === 'true';
|
|
210
|
+
let stageId;
|
|
211
|
+
try {
|
|
212
|
+
stageId = await resolveStageIdBySlug({ baseUrl: env.url, token, slug: stage, repo });
|
|
213
|
+
}
|
|
214
|
+
catch (error) {
|
|
215
|
+
console.error(`${LOG} ${error instanceof Error ? error.message : String(error)}`);
|
|
216
|
+
return 1;
|
|
217
|
+
}
|
|
218
|
+
let response;
|
|
219
|
+
try {
|
|
220
|
+
response = await postJson({
|
|
221
|
+
baseUrl: env.url,
|
|
222
|
+
token,
|
|
223
|
+
path: `/api/data-pipelines/stages/${stageId}/run`,
|
|
224
|
+
body: { environment: deployEnvironmentForEnv(env) },
|
|
225
|
+
});
|
|
226
|
+
}
|
|
227
|
+
catch (error) {
|
|
228
|
+
if (error instanceof AtlasApiError) {
|
|
229
|
+
console.error(`${LOG} run-now failed: ${error.message}`);
|
|
230
|
+
return 1;
|
|
231
|
+
}
|
|
232
|
+
throw error;
|
|
233
|
+
}
|
|
234
|
+
if (json) {
|
|
235
|
+
console.log(JSON.stringify(response, null, 2));
|
|
236
|
+
}
|
|
237
|
+
else {
|
|
238
|
+
console.log(`${LOG} run-now ${stage} → ${deployEnvironmentForEnv(env)}`);
|
|
239
|
+
console.log(`${LOG} invocation ${response.invocationId} (${response.status})`);
|
|
240
|
+
console.log(`${LOG} ${response.databricks_url}`);
|
|
241
|
+
}
|
|
242
|
+
return 0;
|
|
243
|
+
}
|
|
244
|
+
export async function pipelineRollbackCommand(args) {
|
|
245
|
+
const stage = flagString(args.flags, 'stage');
|
|
246
|
+
const repo = flagString(args.flags, 'repo');
|
|
247
|
+
const approvedBy = flagString(args.flags, 'approved-by');
|
|
248
|
+
if (!stage) {
|
|
249
|
+
console.error('usage: seq-studio pipeline rollback --stage <slug> -e <env> [--repo pipelines/<slug>] [--approved-by <you>] [--no-wait]');
|
|
250
|
+
return 1;
|
|
251
|
+
}
|
|
252
|
+
const { env, token } = await envAndToken(args);
|
|
253
|
+
if (requiresPinnedSha(env.name) && !approvedBy) {
|
|
254
|
+
console.error(`${LOG} rollback in ${env.name} requires --approved-by <your sub/email> (approvals are explicit even for rollbacks)`);
|
|
255
|
+
return 1;
|
|
256
|
+
}
|
|
257
|
+
const stageId = await resolveStageIdBySlug({ baseUrl: env.url, token, slug: stage, repo });
|
|
258
|
+
let enqueue;
|
|
259
|
+
try {
|
|
260
|
+
enqueue = await postJson({
|
|
261
|
+
baseUrl: env.url,
|
|
262
|
+
token,
|
|
263
|
+
path: `/api/data-pipelines/stages/${stageId}/rollback`,
|
|
264
|
+
body: {
|
|
265
|
+
environment: deployEnvironmentForEnv(env),
|
|
266
|
+
...(approvedBy ? { approvedBy } : {}),
|
|
267
|
+
},
|
|
268
|
+
});
|
|
269
|
+
}
|
|
270
|
+
catch (error) {
|
|
271
|
+
if (error instanceof AtlasApiError) {
|
|
272
|
+
console.error(`${LOG} rollback failed: ${error.message}`);
|
|
273
|
+
return 1;
|
|
274
|
+
}
|
|
275
|
+
throw error;
|
|
276
|
+
}
|
|
277
|
+
console.log(`${LOG} enqueued rollback ${enqueue.deploymentId}`);
|
|
278
|
+
if (args.flags['no-wait'] === true || args.flags['no-wait'] === 'true')
|
|
279
|
+
return 0;
|
|
280
|
+
return pollDeployment({ baseUrl: env.url, token, deploymentId: enqueue.deploymentId });
|
|
281
|
+
}
|
|
282
|
+
function printFindingsTable(findings) {
|
|
283
|
+
if (findings.length === 0) {
|
|
284
|
+
console.log(`${LOG} no resource changes`);
|
|
285
|
+
return;
|
|
286
|
+
}
|
|
287
|
+
console.log(`${LOG} ${'CLASS'.padEnd(12)} ${'RESOURCE'.padEnd(32)} MESSAGE`);
|
|
288
|
+
for (const f of findings) {
|
|
289
|
+
const klass = f.destructiveReason
|
|
290
|
+
? `${f.classification}/${f.destructiveReason}`
|
|
291
|
+
: f.classification;
|
|
292
|
+
console.log(`${LOG} ${klass.padEnd(12)} ${f.resourceKey.padEnd(32)} ${f.message}`);
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
async function pollDeployment({ baseUrl, token, deploymentId, }) {
|
|
296
|
+
const started = Date.now();
|
|
297
|
+
const timeoutMs = 15 * 60 * 1000;
|
|
298
|
+
while (Date.now() - started < timeoutMs) {
|
|
299
|
+
const detail = await getJson({
|
|
300
|
+
baseUrl,
|
|
301
|
+
token,
|
|
302
|
+
path: `/api/data-pipelines/deployments/${deploymentId}`,
|
|
303
|
+
});
|
|
304
|
+
console.log(`${LOG} deployment ${deploymentId}: ${detail.status}`);
|
|
305
|
+
if (TERMINAL.has(detail.status)) {
|
|
306
|
+
if (detail.status === 'active') {
|
|
307
|
+
console.log(`${LOG} deploy succeeded`);
|
|
308
|
+
return 0;
|
|
309
|
+
}
|
|
310
|
+
console.error(`${LOG} deploy ended '${detail.status}': ${detail.statusDetail ?? ''}`);
|
|
311
|
+
return 1;
|
|
312
|
+
}
|
|
313
|
+
await new Promise((r) => setTimeout(r, 2000));
|
|
314
|
+
}
|
|
315
|
+
console.error(`${LOG} timed out waiting for deployment ${deploymentId}`);
|
|
316
|
+
return 1;
|
|
317
|
+
}
|
|
318
|
+
/**
|
|
319
|
+
* Stage identity is `(repo, slug)` — a bare slug can be ambiguous across
|
|
320
|
+
* Pipelines. `--repo pipelines/<domain>` scopes the lookup server-side; an
|
|
321
|
+
* ambiguous bare slug fails with the candidate repos rather than guessing.
|
|
322
|
+
*
|
|
323
|
+
* The `slug` query param is an EXACT server-side filter (contracts.ts
|
|
324
|
+
* `listStagesQuerySchema`) — the list route's default sort is
|
|
325
|
+
* `updated_at DESC`, so scanning only the first page (as this used to do)
|
|
326
|
+
* silently missed a matching stage buried on a later page, and would have
|
|
327
|
+
* missed a same-slug duplicate on a later page too, falsely reporting the
|
|
328
|
+
* bare-slug lookup as unique (SEQ-2447 review). Filtering server-side by the
|
|
329
|
+
* exact slug means every match is guaranteed to fit on one page regardless
|
|
330
|
+
* of how many other stages exist.
|
|
331
|
+
*/
|
|
332
|
+
async function resolveStageIdBySlug({ baseUrl, token, slug, repo, }) {
|
|
333
|
+
const repoFilter = repo ? `&repo=${encodeURIComponent(repo)}` : '';
|
|
334
|
+
const listed = await getJson({
|
|
335
|
+
baseUrl,
|
|
336
|
+
token,
|
|
337
|
+
path: `/api/data-pipelines/stages?limit=200&slug=${encodeURIComponent(slug)}${repoFilter}`,
|
|
338
|
+
});
|
|
339
|
+
const first = listed.stages[0];
|
|
340
|
+
if (!first) {
|
|
341
|
+
throw new Error(`stage '${slug}' not found${repo ? ` in ${repo}` : ''}`);
|
|
342
|
+
}
|
|
343
|
+
if (listed.stages.length > 1) {
|
|
344
|
+
throw new Error(`stage '${slug}' exists in multiple Pipelines (${listed.stages.map((m) => m.repo).join(', ')}) — ` +
|
|
345
|
+
'disambiguate with --repo pipelines/<domain>');
|
|
346
|
+
}
|
|
347
|
+
return first.id;
|
|
348
|
+
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
/** Client-side pin rules mirrored from the server deploy adapter (SEQ-2450). */
|
|
2
|
+
export declare const FULL_SHA_PATTERN: RegExp;
|
|
3
|
+
export declare const PINNED_ENVIRONMENTS: Set<string>;
|
|
4
|
+
export declare function isFullSha(ref: string): boolean;
|
|
5
|
+
export declare function requiresPinnedSha(environment: string): boolean;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/** Client-side pin rules mirrored from the server deploy adapter (SEQ-2450). */
|
|
2
|
+
export const FULL_SHA_PATTERN = /^[0-9a-f]{40}$/i;
|
|
3
|
+
export const PINNED_ENVIRONMENTS = new Set(['production', 'banksouth']);
|
|
4
|
+
export function isFullSha(ref) {
|
|
5
|
+
return FULL_SHA_PATTERN.test(ref);
|
|
6
|
+
}
|
|
7
|
+
export function requiresPinnedSha(environment) {
|
|
8
|
+
return PINNED_ENVIRONMENTS.has(environment);
|
|
9
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `seq-studio pipeline init` scaffolds — commented per-kind `<name>.stage.yml`
|
|
3
|
+
* templates plus an entrypoint stub. Kept as string constants (not a
|
|
4
|
+
* templates/ directory) so the pipeline family needs no copy step and works
|
|
5
|
+
* from any install of the CLI.
|
|
6
|
+
*/
|
|
7
|
+
export declare const STAGE_TYPES: readonly ["ingestion", "transformation", "serving"];
|
|
8
|
+
export type StageTemplateType = (typeof STAGE_TYPES)[number];
|
|
9
|
+
export declare function renderStageTemplate(name: string, type: StageTemplateType): string;
|
|
10
|
+
/** Serving stages are declarative — no entrypoint stub. */
|
|
11
|
+
export declare function renderEntrypointStub(name: string): string;
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `seq-studio pipeline init` scaffolds — commented per-kind `<name>.stage.yml`
|
|
3
|
+
* templates plus an entrypoint stub. Kept as string constants (not a
|
|
4
|
+
* templates/ directory) so the pipeline family needs no copy step and works
|
|
5
|
+
* from any install of the CLI.
|
|
6
|
+
*/
|
|
7
|
+
const envelopeTemplate = (name, type) => `# ${name}.stage.yml — one deployable ${type} stage of this Pipeline.
|
|
8
|
+
# Docs: @sequenceholdings/pipeline-spec (the schema is the single source of truth).
|
|
9
|
+
schema_version: 1
|
|
10
|
+
stage: ${name}
|
|
11
|
+
title: TODO — one-line human title
|
|
12
|
+
description: >-
|
|
13
|
+
TODO — what this stage does and why it exists.
|
|
14
|
+
type: ${type}
|
|
15
|
+
owners:
|
|
16
|
+
- you@seqholdings.com
|
|
17
|
+
# critical | standard | experimental
|
|
18
|
+
criticality: experimental
|
|
19
|
+
# Optional reliability posture:
|
|
20
|
+
# slos:
|
|
21
|
+
# freshness: PT4H # ISO-8601 durations
|
|
22
|
+
# max_duration: PT1H
|
|
23
|
+
# alerts:
|
|
24
|
+
# channels: [eng-alerts] # symbolic channel refs — never webhook URLs
|
|
25
|
+
# runbook: https://notion.so/your-runbook
|
|
26
|
+
environments:
|
|
27
|
+
- staging
|
|
28
|
+
`;
|
|
29
|
+
const INGESTION_BODY = `
|
|
30
|
+
# Where the ingestion code runs: databricks (API pulls) | edge-worker (on-prem VM).
|
|
31
|
+
# ('upload' is reserved — not yet supported.)
|
|
32
|
+
runtime: databricks
|
|
33
|
+
# databricks runtime pulls from an external system (symbolic refs, never secrets).
|
|
34
|
+
# Secure-by-default: declare a credential family for authenticated APIs.
|
|
35
|
+
# For genuinely keyless / public sources, omit credential entirely (do not
|
|
36
|
+
# invent a sentinel binding) — e.g. source: { system: espn }.
|
|
37
|
+
source:
|
|
38
|
+
system: my_source_system
|
|
39
|
+
credential: my_source_credential
|
|
40
|
+
# edge-worker runtime instead binds a registered worker:
|
|
41
|
+
# worker:
|
|
42
|
+
# machine: my_vm
|
|
43
|
+
# data_source: my_library
|
|
44
|
+
# credential: my_cred_family
|
|
45
|
+
feeds:
|
|
46
|
+
# One feed = one stream of raw files into the volume landing zone.
|
|
47
|
+
# The platform derives (per-source catalog, never the deployment catalog):
|
|
48
|
+
# /Volumes/src_<source>/bronze/<feed>/batch_id=<ts>/, the bronze table
|
|
49
|
+
# src_<source>.bronze.<feed>, envelope columns (batch_id, ingested_at,
|
|
50
|
+
# source_system, content_hash), and a presence manifest for full_snapshot.
|
|
51
|
+
my_feed:
|
|
52
|
+
# full_snapshot | incremental | event_stream
|
|
53
|
+
snapshot_mode: full_snapshot
|
|
54
|
+
natural_key:
|
|
55
|
+
- id
|
|
56
|
+
# incremental feeds declare their cursor:
|
|
57
|
+
# cursor:
|
|
58
|
+
# column: updated_at
|
|
59
|
+
# encoding: yyyyddd # extensible string — provider-neutral
|
|
60
|
+
# none | contains_pii
|
|
61
|
+
pii: none
|
|
62
|
+
# parquet | json | csv | custom ('xlsx' is reserved — not yet supported)
|
|
63
|
+
format: json
|
|
64
|
+
# format: custom requires a pure file→rows decoder (never writes tables):
|
|
65
|
+
# decoder: src/decoders/my_decoder.py
|
|
66
|
+
schedule:
|
|
67
|
+
cron: "0 6 * * *"
|
|
68
|
+
tz: America/New_York
|
|
69
|
+
# Max staleness before the freshness SLO breaches:
|
|
70
|
+
freshness_slo: P1D
|
|
71
|
+
# Raw-file replay buffer (not an archive):
|
|
72
|
+
retention: P30D
|
|
73
|
+
entrypoint: src/{{snake}}.py
|
|
74
|
+
`;
|
|
75
|
+
const TRANSFORMATION_BODY = `
|
|
76
|
+
# dlt | spark-job | polars
|
|
77
|
+
engine: dlt
|
|
78
|
+
inputs:
|
|
79
|
+
# Reference upstream assets by stage + feed (ingestion) or output:
|
|
80
|
+
# - stage: my-ingest
|
|
81
|
+
# feed: my_feed
|
|
82
|
+
# - stage: upstream-transform
|
|
83
|
+
# output: silver.some_table
|
|
84
|
+
# columns: [id, amount] # the subset you consume (validated)
|
|
85
|
+
# Lakebase CDF reverse-sync input (columns REQUIRED, never inferred):
|
|
86
|
+
# - kind: lakebase_cdf
|
|
87
|
+
# table: manual.my_overrides
|
|
88
|
+
# fold: latest_state # latest_state | events
|
|
89
|
+
# columns:
|
|
90
|
+
# id: { type: string, nullable: false, key: true }
|
|
91
|
+
[]
|
|
92
|
+
trigger:
|
|
93
|
+
# At least one of cron / after:
|
|
94
|
+
cron: "0 7 * * *"
|
|
95
|
+
# after: [my-ingest]
|
|
96
|
+
outputs:
|
|
97
|
+
# Typed contracts — names must be silver.* or gold.*:
|
|
98
|
+
silver.{{snake}}:
|
|
99
|
+
columns:
|
|
100
|
+
id:
|
|
101
|
+
type: string
|
|
102
|
+
nullable: false
|
|
103
|
+
key: true
|
|
104
|
+
value:
|
|
105
|
+
type: int
|
|
106
|
+
# fail | quarantine (→ silver.{{snake}}__quarantine) | warn
|
|
107
|
+
on_type_violation: fail
|
|
108
|
+
# additive-only (default: breaking changes fail CI) | versioned
|
|
109
|
+
compatibility: additive-only
|
|
110
|
+
# overwrite | append | auto_cdc
|
|
111
|
+
write_pattern: overwrite
|
|
112
|
+
entrypoint: src/{{snake}}.py
|
|
113
|
+
`;
|
|
114
|
+
const SERVING_BODY = `
|
|
115
|
+
# The transformation output this stage serves:
|
|
116
|
+
source:
|
|
117
|
+
stage: my-gold-stage
|
|
118
|
+
output: gold.my_table
|
|
119
|
+
projection:
|
|
120
|
+
# The served contract (inline columns or schema_ref):
|
|
121
|
+
columns:
|
|
122
|
+
id:
|
|
123
|
+
type: uuid
|
|
124
|
+
nullable: false
|
|
125
|
+
key: true
|
|
126
|
+
name:
|
|
127
|
+
type: string
|
|
128
|
+
primary_key:
|
|
129
|
+
- id
|
|
130
|
+
# SNAPSHOT | TRIGGERED | CONTINUOUS (overwrite-written sources: SNAPSHOT only)
|
|
131
|
+
sync_mode: SNAPSHOT
|
|
132
|
+
trigger:
|
|
133
|
+
after: my-gold-stage
|
|
134
|
+
expose:
|
|
135
|
+
view: public.my_table
|
|
136
|
+
# Optional ORM coupling ('managed' mode is reserved — not yet supported):
|
|
137
|
+
# orm_binding:
|
|
138
|
+
# namespace: my_namespace
|
|
139
|
+
# table: my_table
|
|
140
|
+
# mode: referenced
|
|
141
|
+
`;
|
|
142
|
+
const ENTRYPOINT_STUB = `"""Entrypoint stub for the {{name}} stage.
|
|
143
|
+
|
|
144
|
+
The platform invokes this per the stage spec ({{name}}.stage.yml).
|
|
145
|
+
Decoders and transforms stay pure — bronze materialization, envelope
|
|
146
|
+
stamping, and table writes are platform-owned.
|
|
147
|
+
"""
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def main() -> None:
|
|
151
|
+
raise NotImplementedError("implement the {{name}} stage")
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
if __name__ == "__main__":
|
|
155
|
+
main()
|
|
156
|
+
`;
|
|
157
|
+
export const STAGE_TYPES = ['ingestion', 'transformation', 'serving'];
|
|
158
|
+
export function renderStageTemplate(name, type) {
|
|
159
|
+
const body = type === 'ingestion' ? INGESTION_BODY : type === 'transformation' ? TRANSFORMATION_BODY : SERVING_BODY;
|
|
160
|
+
return (envelopeTemplate(name, type) +
|
|
161
|
+
body.replace(/\{\{name\}\}/g, name).replace(/\{\{snake\}\}/g, name.replace(/-/g, '_')));
|
|
162
|
+
}
|
|
163
|
+
/** Serving stages are declarative — no entrypoint stub. */
|
|
164
|
+
export function renderEntrypointStub(name) {
|
|
165
|
+
return ENTRYPOINT_STUB.replace(/\{\{name\}\}/g, name);
|
|
166
|
+
}
|
package/dist/process/build.d.ts
CHANGED
|
@@ -8,7 +8,18 @@ export type ResolveProcessPin = (processId: string, version?: string) => Promise
|
|
|
8
8
|
version: string;
|
|
9
9
|
bundleHash: string;
|
|
10
10
|
} | null>;
|
|
11
|
-
export
|
|
11
|
+
export interface BuildBundleOptions {
|
|
12
12
|
resolveProcessPin?: ResolveProcessPin;
|
|
13
|
+
/** Override git provenance instead of reading from the local working tree. */
|
|
14
|
+
provenance?: {
|
|
15
|
+
gitCommit: string | null;
|
|
16
|
+
gitBranch: string | null;
|
|
17
|
+
gitDirty: boolean | null;
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
export declare function buildBundleFromProcesses(defs: readonly LoadedProcess[], options?: BuildBundleOptions): Promise<LatticeBundle>;
|
|
21
|
+
export declare function finalizeCompiledBundle({ bundle, resolveProcessPin, }: {
|
|
22
|
+
bundle: LatticeBundle;
|
|
23
|
+
resolveProcessPin: ResolveProcessPin;
|
|
13
24
|
}): Promise<LatticeBundle>;
|
|
14
25
|
export declare function summarizeBundle(bundle: LatticeBundle): LatticeBundleSummary;
|
package/dist/process/build.js
CHANGED
|
@@ -20,9 +20,22 @@ export async function buildBundleFromProcesses(defs, options) {
|
|
|
20
20
|
return options.resolveProcessPin(processId, version);
|
|
21
21
|
});
|
|
22
22
|
}
|
|
23
|
+
const metadata = options?.provenance
|
|
24
|
+
? collectGitMetadataWithOverrides(options.provenance)
|
|
25
|
+
: collectGitMetadata();
|
|
23
26
|
// Manifest, hash, version, and co-located subprocess pins are all derived in
|
|
24
27
|
// the SDK so a CLI-built bundle is byte-identical to a server-applied one.
|
|
25
|
-
return finalizeBundleIdentity({ processes, metadata
|
|
28
|
+
return finalizeBundleIdentity({ processes, metadata });
|
|
29
|
+
}
|
|
30
|
+
export async function finalizeCompiledBundle({ bundle, resolveProcessPin, }) {
|
|
31
|
+
const processes = bundle.processes.map((process) => structuredClone(process));
|
|
32
|
+
const localProcessIds = new Set(processes.map((process) => process.id));
|
|
33
|
+
await resolveSubprocessPinsInProcesses(processes, async (processId, version) => {
|
|
34
|
+
if (!version && localProcessIds.has(processId))
|
|
35
|
+
return null;
|
|
36
|
+
return resolveProcessPin(processId, version);
|
|
37
|
+
});
|
|
38
|
+
return finalizeBundleIdentity({ processes, metadata: bundle.metadata });
|
|
26
39
|
}
|
|
27
40
|
async function resolveSubprocessPinsInProcesses(processes, resolve) {
|
|
28
41
|
const pins = [];
|
|
@@ -83,8 +96,20 @@ function serializeProcess(input) {
|
|
|
83
96
|
...(input.process.supervisor_retry !== undefined
|
|
84
97
|
? { supervisor_retry: serializeRetryConfig(input.process.supervisor_retry) }
|
|
85
98
|
: {}),
|
|
99
|
+
// Both run_as keys are omitted when unset so existing hashes stay stable
|
|
100
|
+
// (an email-only run_as serializes byte-identically to before the
|
|
101
|
+
// service_account field existed).
|
|
86
102
|
...(input.process.run_as !== undefined
|
|
87
|
-
? {
|
|
103
|
+
? {
|
|
104
|
+
run_as: {
|
|
105
|
+
...(input.process.run_as.email !== undefined
|
|
106
|
+
? { email: input.process.run_as.email }
|
|
107
|
+
: {}),
|
|
108
|
+
...(input.process.run_as.service_account !== undefined
|
|
109
|
+
? { service_account: input.process.run_as.service_account }
|
|
110
|
+
: {}),
|
|
111
|
+
},
|
|
112
|
+
}
|
|
88
113
|
: {}),
|
|
89
114
|
};
|
|
90
115
|
}
|
|
@@ -132,6 +157,10 @@ function serializeNodeMetadata(node) {
|
|
|
132
157
|
meta.limits = native.limits;
|
|
133
158
|
if (native.bindings)
|
|
134
159
|
meta.bindings = native.bindings;
|
|
160
|
+
// Per-node service-account override — omitted when unset for hash stability.
|
|
161
|
+
if (native.run_as?.service_account) {
|
|
162
|
+
meta.run_as = { service_account: native.run_as.service_account };
|
|
163
|
+
}
|
|
135
164
|
const nativeInputMapper = native.input;
|
|
136
165
|
if (typeof nativeInputMapper === 'function') {
|
|
137
166
|
meta.input_mapper_source = nativeInputMapper.toString();
|
|
@@ -266,6 +295,11 @@ function serializeNodeMetadata(node) {
|
|
|
266
295
|
on_branch_error: parallel.on_branch_error,
|
|
267
296
|
join_source: toSource(parallel.join),
|
|
268
297
|
};
|
|
298
|
+
// Serialized only when non-default ('eager') so existing barrier
|
|
299
|
+
// bundles keep their hashes; a missing field reads as 'barrier'.
|
|
300
|
+
if (parallel.join_mode === 'eager') {
|
|
301
|
+
pmeta.join_mode = 'eager';
|
|
302
|
+
}
|
|
269
303
|
if (parallel.max_concurrency !== undefined) {
|
|
270
304
|
pmeta.max_concurrency = parallel.max_concurrency;
|
|
271
305
|
}
|
|
@@ -319,6 +353,10 @@ function serializeNodeMetadata(node) {
|
|
|
319
353
|
meta.limits = mf.limits;
|
|
320
354
|
if (mf.bindings)
|
|
321
355
|
meta.bindings = mf.bindings;
|
|
356
|
+
// Per-node service-account override — omitted when unset for hash stability.
|
|
357
|
+
if (mf.run_as?.service_account) {
|
|
358
|
+
meta.run_as = { service_account: mf.run_as.service_account };
|
|
359
|
+
}
|
|
322
360
|
// NOTE: `middleware` (e.g. withRetry) is serialized generically for every
|
|
323
361
|
// node kind in `serializeNode` above — don't duplicate it here.
|
|
324
362
|
const inputMapper = mf.input;
|
|
@@ -348,6 +386,15 @@ function collectGitMetadata() {
|
|
|
348
386
|
git_dirty: get('git status --porcelain') !== '',
|
|
349
387
|
};
|
|
350
388
|
}
|
|
389
|
+
function collectGitMetadataWithOverrides(provenance) {
|
|
390
|
+
return {
|
|
391
|
+
created_at: new Date().toISOString(),
|
|
392
|
+
created_by: process.env['USER'] ?? null,
|
|
393
|
+
git_commit: provenance.gitCommit,
|
|
394
|
+
git_branch: provenance.gitBranch,
|
|
395
|
+
git_dirty: provenance.gitDirty ?? false,
|
|
396
|
+
};
|
|
397
|
+
}
|
|
351
398
|
export function summarizeBundle(bundle) {
|
|
352
399
|
return {
|
|
353
400
|
name: bundle.manifest.bundle.name,
|