@sequenceholdings/studio-cli 0.1.13 → 0.1.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/README.md +258 -38
  2. package/dist/agents/apply-chunks.d.ts +13 -0
  3. package/dist/agents/apply-chunks.js +43 -0
  4. package/dist/agents/commands.d.ts +10 -0
  5. package/dist/agents/commands.js +218 -0
  6. package/dist/agents/scaffold.d.ts +2 -0
  7. package/dist/agents/scaffold.js +77 -0
  8. package/dist/agents/source.d.ts +18 -0
  9. package/dist/agents/source.js +121 -0
  10. package/dist/artifact/delegate.d.ts +2 -2
  11. package/dist/artifact/delegate.js +31 -73
  12. package/dist/atlas-client.js +52 -37
  13. package/dist/auth-cmds/commands.d.ts +1 -1
  14. package/dist/auth-cmds/commands.js +12 -7
  15. package/dist/auth.d.ts +104 -24
  16. package/dist/auth.js +456 -94
  17. package/dist/config.d.ts +3 -3
  18. package/dist/config.js +18 -13
  19. package/dist/env-catalog.js +13 -3
  20. package/dist/env-flags.d.ts +2 -0
  21. package/dist/env-flags.js +2 -0
  22. package/dist/env-registry.d.ts +27 -0
  23. package/dist/env-registry.js +204 -0
  24. package/dist/envs/commands.d.ts +1 -1
  25. package/dist/envs/commands.js +41 -3
  26. package/dist/file-lock.d.ts +5 -0
  27. package/dist/file-lock.js +187 -0
  28. package/dist/functions/commands.d.ts +10 -10
  29. package/dist/functions/commands.js +87 -53
  30. package/dist/functions/manifest.d.ts +1 -0
  31. package/dist/functions/manifest.js +36 -0
  32. package/dist/functions/source-selection.d.ts +24 -0
  33. package/dist/functions/source-selection.js +67 -0
  34. package/dist/login.d.ts +8 -3
  35. package/dist/login.js +46 -34
  36. package/dist/main.d.ts +3 -1
  37. package/dist/main.js +41 -12
  38. package/dist/orm/delegate.js +25 -7
  39. package/dist/pat-hints.js +2 -2
  40. package/dist/pipeline/commands.d.ts +58 -0
  41. package/dist/pipeline/commands.js +330 -0
  42. package/dist/pipeline/lifecycle.d.ts +58 -0
  43. package/dist/pipeline/lifecycle.js +348 -0
  44. package/dist/pipeline/pinning.d.ts +5 -0
  45. package/dist/pipeline/pinning.js +9 -0
  46. package/dist/pipeline/templates.d.ts +11 -0
  47. package/dist/pipeline/templates.js +166 -0
  48. package/dist/process/build.d.ts +4 -0
  49. package/dist/process/build.js +33 -2
  50. package/dist/process/codegen.js +19 -1
  51. package/dist/process/commands.js +97 -47
  52. package/dist/process/compiler-subprocess.d.ts +29 -0
  53. package/dist/process/compiler-subprocess.js +99 -0
  54. package/dist/process/compiler-worker.d.ts +1 -0
  55. package/dist/process/compiler-worker.js +38 -0
  56. package/dist/process/lint.d.ts +8 -0
  57. package/dist/process/lint.js +84 -29
  58. package/dist/process/repo-install.js +18 -2
  59. package/dist/repos/commands.d.ts +1 -1
  60. package/dist/repos/commands.js +17 -12
  61. package/dist/secrets/commands.d.ts +1 -1
  62. package/dist/secrets/commands.js +18 -18
  63. package/package.json +12 -5
@@ -0,0 +1,348 @@
1
+ /**
2
+ * Deploy lifecycle verbs for `seq-studio pipeline plan|deploy|promote|rollback`
3
+ * (SEQ-2450). Wraps the /api/data-pipelines plan/deploy APIs.
4
+ */
5
+ import { getAccessToken, NotLoggedInError } from '../auth.js';
6
+ import { AtlasApiError, getJson, postJson } from '../atlas-client.js';
7
+ import { resolveEnvWithDiscovery } from '../config.js';
8
+ import { isFullSha, requiresPinnedSha } from './pinning.js';
9
+ const LOG = '[seq-studio]';
10
+ const TERMINAL = new Set(['active', 'failed', 'retired']);
11
+ async function envAndToken(args) {
12
+ const requested = typeof args.flags.e === 'string' ? args.flags.e : typeof args.flags.env === 'string' ? args.flags.env : undefined;
13
+ if (!requested) {
14
+ throw new Error('missing -e/--env <environment>');
15
+ }
16
+ const env = await resolveEnvWithDiscovery({ requested });
17
+ let token;
18
+ try {
19
+ token = await getAccessToken();
20
+ }
21
+ catch (error) {
22
+ if (error instanceof NotLoggedInError)
23
+ throw error;
24
+ throw error;
25
+ }
26
+ return { env, token };
27
+ }
28
+ function flagString(flags, key) {
29
+ const value = flags[key];
30
+ if (value === true)
31
+ throw new Error(`--${key} requires a value`);
32
+ return value;
33
+ }
34
+ /**
35
+ * The CLI's local dev-loop env is named `local` — chosen so `-e local` lines
36
+ * up with `seqapi -e local` / `artifact-studio --env local` (see `config.ts`
37
+ * `BUILT_IN_ENV_URLS`). There is no `local` in the server's deploy-lifecycle
38
+ * enum (`DEPLOY_ENVIRONMENTS` in `atlas/src/server/services/data-pipelines/schema.ts`
39
+ * is `dev | staging | production | banksouth`); the server's name for that
40
+ * same developer-loop target is `dev` (`targetFactsForEnvironment` treats
41
+ * `dev` as the one target exempt from a `lakebase_branch` binding). Map at
42
+ * this one boundary — every lifecycle request body funnels through here —
43
+ * so the CLI never sends the wire-invalid `local` and every other env name
44
+ * passes through unchanged.
45
+ */
46
+ export function deployEnvironmentForEnv(env) {
47
+ return env.name === 'local' ? 'dev' : env.name;
48
+ }
49
+ export async function pipelinePlanCommand(args) {
50
+ const repo = flagString(args.flags, 'repo');
51
+ const ref = flagString(args.flags, 'ref');
52
+ if (!repo || !ref) {
53
+ console.error('usage: seq-studio pipeline plan --repo pipelines/<slug> --ref <sha|branch> -e <env> [--json]');
54
+ return 1;
55
+ }
56
+ const { env, token } = await envAndToken(args);
57
+ const json = args.flags.json === true || args.flags.json === 'true';
58
+ let response;
59
+ try {
60
+ response = await postJson({
61
+ baseUrl: env.url,
62
+ token,
63
+ path: '/api/data-pipelines/pipelines/plan',
64
+ body: { repo, ref, environment: deployEnvironmentForEnv(env) },
65
+ });
66
+ }
67
+ catch (error) {
68
+ if (error instanceof AtlasApiError) {
69
+ console.error(`${LOG} plan failed: ${error.message}`);
70
+ return 1;
71
+ }
72
+ throw error;
73
+ }
74
+ if (json) {
75
+ console.log(JSON.stringify(response, null, 2));
76
+ }
77
+ else {
78
+ console.log(response.text);
79
+ printFindingsTable(response.plan.findings);
80
+ }
81
+ return response.plan.hasDestructive ? 1 : 0;
82
+ }
83
+ export async function pipelineDeployCommand(args) {
84
+ const repo = flagString(args.flags, 'repo');
85
+ const ref = flagString(args.flags, 'ref');
86
+ const deploymentId = flagString(args.flags, 'deployment-id');
87
+ const approvedBy = flagString(args.flags, 'approved-by');
88
+ const { env, token } = await envAndToken(args);
89
+ if (requiresPinnedSha(env.name) && ref && !isFullSha(ref)) {
90
+ console.error(`${LOG} deploy to ${env.name} requires a pinned 40-hex commit SHA (got '${ref}')`);
91
+ return 1;
92
+ }
93
+ let enqueue;
94
+ try {
95
+ if (deploymentId) {
96
+ // Direct execute of a persisted plan.
97
+ const stageId = flagString(args.flags, 'stage-id');
98
+ if (!stageId) {
99
+ console.error('usage: seq-studio pipeline deploy --deployment-id <id> --stage-id <id> -e <env> [--approved-by <sub>] [--no-wait]');
100
+ return 1;
101
+ }
102
+ enqueue = await postJson({
103
+ baseUrl: env.url,
104
+ token,
105
+ path: `/api/data-pipelines/stages/${stageId}/deploy`,
106
+ body: { deploymentId, ...(approvedBy ? { approvedBy } : {}) },
107
+ });
108
+ }
109
+ else {
110
+ if (!repo || !ref) {
111
+ console.error('usage: seq-studio pipeline deploy --repo pipelines/<slug> --ref <sha> -e <env> [--approved-by <sub>] [--no-wait]');
112
+ return 1;
113
+ }
114
+ // Plan then deploy the first planning deployment.
115
+ const plan = await postJson({
116
+ baseUrl: env.url,
117
+ token,
118
+ path: '/api/data-pipelines/pipelines/plan',
119
+ body: { repo, ref, environment: deployEnvironmentForEnv(env) },
120
+ });
121
+ if (plan.plan.hasDestructive) {
122
+ console.error(`${LOG} plan has destructive findings — refusing to deploy`);
123
+ if (args.flags.json === true || args.flags.json === 'true') {
124
+ console.log(JSON.stringify(plan, null, 2));
125
+ }
126
+ else {
127
+ console.error(plan.text);
128
+ }
129
+ return 1;
130
+ }
131
+ const firstId = plan.deploymentIds[0];
132
+ const stageId = plan.stageIds[0];
133
+ if (!firstId || !stageId) {
134
+ console.error(`${LOG} plan produced no deployment rows`);
135
+ return 1;
136
+ }
137
+ enqueue = await postJson({
138
+ baseUrl: env.url,
139
+ token,
140
+ path: `/api/data-pipelines/stages/${stageId}/deploy`,
141
+ body: {
142
+ deploymentId: firstId,
143
+ ...(approvedBy ? { approvedBy } : {}),
144
+ },
145
+ });
146
+ }
147
+ }
148
+ catch (error) {
149
+ if (error instanceof AtlasApiError) {
150
+ console.error(`${LOG} deploy failed: ${error.message}`);
151
+ return 1;
152
+ }
153
+ throw error;
154
+ }
155
+ console.log(`${LOG} enqueued deploy ${enqueue.deploymentId} (trigger=${enqueue.triggerRunId ?? 'n/a'})`);
156
+ if (args.flags['no-wait'] === true || args.flags['no-wait'] === 'true') {
157
+ return 0;
158
+ }
159
+ return pollDeployment({ baseUrl: env.url, token, deploymentId: enqueue.deploymentId });
160
+ }
161
+ export async function pipelinePromoteCommand(args) {
162
+ const stage = flagString(args.flags, 'stage');
163
+ const version = flagString(args.flags, 'version');
164
+ const repo = flagString(args.flags, 'repo');
165
+ const approvedBy = flagString(args.flags, 'approved-by');
166
+ if (!stage || !version) {
167
+ console.error('usage: seq-studio pipeline promote --stage <slug> --version <v> -e <env> [--repo pipelines/<slug>] [--approved-by <you>] [--no-wait]');
168
+ return 1;
169
+ }
170
+ const { env, token } = await envAndToken(args);
171
+ if (requiresPinnedSha(env.name) && !approvedBy) {
172
+ console.error(`${LOG} promote to ${env.name} requires --approved-by <your sub/email> (approvals are self-recorded)`);
173
+ return 1;
174
+ }
175
+ const stageId = await resolveStageIdBySlug({ baseUrl: env.url, token, slug: stage, repo });
176
+ let enqueue;
177
+ try {
178
+ enqueue = await postJson({
179
+ baseUrl: env.url,
180
+ token,
181
+ path: `/api/data-pipelines/stages/${stageId}/promote`,
182
+ body: {
183
+ version,
184
+ environment: deployEnvironmentForEnv(env),
185
+ ...(approvedBy ? { approvedBy } : {}),
186
+ },
187
+ });
188
+ }
189
+ catch (error) {
190
+ if (error instanceof AtlasApiError) {
191
+ console.error(`${LOG} promote failed: ${error.message}`);
192
+ return 1;
193
+ }
194
+ throw error;
195
+ }
196
+ console.log(`${LOG} enqueued promote ${enqueue.deploymentId}`);
197
+ if (args.flags['no-wait'] === true || args.flags['no-wait'] === 'true')
198
+ return 0;
199
+ return pollDeployment({ baseUrl: env.url, token, deploymentId: enqueue.deploymentId });
200
+ }
201
+ export async function pipelineRunNowCommand(args) {
202
+ const stage = flagString(args.flags, 'stage');
203
+ const repo = flagString(args.flags, 'repo');
204
+ if (!stage) {
205
+ console.error('usage: seq-studio pipeline run-now --stage <slug> -e <env> [--repo pipelines/<slug>] [--json]');
206
+ return 1;
207
+ }
208
+ const { env, token } = await envAndToken(args);
209
+ const json = args.flags.json === true || args.flags.json === 'true';
210
+ let stageId;
211
+ try {
212
+ stageId = await resolveStageIdBySlug({ baseUrl: env.url, token, slug: stage, repo });
213
+ }
214
+ catch (error) {
215
+ console.error(`${LOG} ${error instanceof Error ? error.message : String(error)}`);
216
+ return 1;
217
+ }
218
+ let response;
219
+ try {
220
+ response = await postJson({
221
+ baseUrl: env.url,
222
+ token,
223
+ path: `/api/data-pipelines/stages/${stageId}/run`,
224
+ body: { environment: deployEnvironmentForEnv(env) },
225
+ });
226
+ }
227
+ catch (error) {
228
+ if (error instanceof AtlasApiError) {
229
+ console.error(`${LOG} run-now failed: ${error.message}`);
230
+ return 1;
231
+ }
232
+ throw error;
233
+ }
234
+ if (json) {
235
+ console.log(JSON.stringify(response, null, 2));
236
+ }
237
+ else {
238
+ console.log(`${LOG} run-now ${stage} → ${deployEnvironmentForEnv(env)}`);
239
+ console.log(`${LOG} invocation ${response.invocationId} (${response.status})`);
240
+ console.log(`${LOG} ${response.databricks_url}`);
241
+ }
242
+ return 0;
243
+ }
244
+ export async function pipelineRollbackCommand(args) {
245
+ const stage = flagString(args.flags, 'stage');
246
+ const repo = flagString(args.flags, 'repo');
247
+ const approvedBy = flagString(args.flags, 'approved-by');
248
+ if (!stage) {
249
+ console.error('usage: seq-studio pipeline rollback --stage <slug> -e <env> [--repo pipelines/<slug>] [--approved-by <you>] [--no-wait]');
250
+ return 1;
251
+ }
252
+ const { env, token } = await envAndToken(args);
253
+ if (requiresPinnedSha(env.name) && !approvedBy) {
254
+ console.error(`${LOG} rollback in ${env.name} requires --approved-by <your sub/email> (approvals are explicit even for rollbacks)`);
255
+ return 1;
256
+ }
257
+ const stageId = await resolveStageIdBySlug({ baseUrl: env.url, token, slug: stage, repo });
258
+ let enqueue;
259
+ try {
260
+ enqueue = await postJson({
261
+ baseUrl: env.url,
262
+ token,
263
+ path: `/api/data-pipelines/stages/${stageId}/rollback`,
264
+ body: {
265
+ environment: deployEnvironmentForEnv(env),
266
+ ...(approvedBy ? { approvedBy } : {}),
267
+ },
268
+ });
269
+ }
270
+ catch (error) {
271
+ if (error instanceof AtlasApiError) {
272
+ console.error(`${LOG} rollback failed: ${error.message}`);
273
+ return 1;
274
+ }
275
+ throw error;
276
+ }
277
+ console.log(`${LOG} enqueued rollback ${enqueue.deploymentId}`);
278
+ if (args.flags['no-wait'] === true || args.flags['no-wait'] === 'true')
279
+ return 0;
280
+ return pollDeployment({ baseUrl: env.url, token, deploymentId: enqueue.deploymentId });
281
+ }
282
+ function printFindingsTable(findings) {
283
+ if (findings.length === 0) {
284
+ console.log(`${LOG} no resource changes`);
285
+ return;
286
+ }
287
+ console.log(`${LOG} ${'CLASS'.padEnd(12)} ${'RESOURCE'.padEnd(32)} MESSAGE`);
288
+ for (const f of findings) {
289
+ const klass = f.destructiveReason
290
+ ? `${f.classification}/${f.destructiveReason}`
291
+ : f.classification;
292
+ console.log(`${LOG} ${klass.padEnd(12)} ${f.resourceKey.padEnd(32)} ${f.message}`);
293
+ }
294
+ }
295
+ async function pollDeployment({ baseUrl, token, deploymentId, }) {
296
+ const started = Date.now();
297
+ const timeoutMs = 15 * 60 * 1000;
298
+ while (Date.now() - started < timeoutMs) {
299
+ const detail = await getJson({
300
+ baseUrl,
301
+ token,
302
+ path: `/api/data-pipelines/deployments/${deploymentId}`,
303
+ });
304
+ console.log(`${LOG} deployment ${deploymentId}: ${detail.status}`);
305
+ if (TERMINAL.has(detail.status)) {
306
+ if (detail.status === 'active') {
307
+ console.log(`${LOG} deploy succeeded`);
308
+ return 0;
309
+ }
310
+ console.error(`${LOG} deploy ended '${detail.status}': ${detail.statusDetail ?? ''}`);
311
+ return 1;
312
+ }
313
+ await new Promise((r) => setTimeout(r, 2000));
314
+ }
315
+ console.error(`${LOG} timed out waiting for deployment ${deploymentId}`);
316
+ return 1;
317
+ }
318
+ /**
319
+ * Stage identity is `(repo, slug)` — a bare slug can be ambiguous across
320
+ * Pipelines. `--repo pipelines/<domain>` scopes the lookup server-side; an
321
+ * ambiguous bare slug fails with the candidate repos rather than guessing.
322
+ *
323
+ * The `slug` query param is an EXACT server-side filter (contracts.ts
324
+ * `listStagesQuerySchema`) — the list route's default sort is
325
+ * `updated_at DESC`, so scanning only the first page (as this used to do)
326
+ * silently missed a matching stage buried on a later page, and would have
327
+ * missed a same-slug duplicate on a later page too, falsely reporting the
328
+ * bare-slug lookup as unique (SEQ-2447 review). Filtering server-side by the
329
+ * exact slug means every match is guaranteed to fit on one page regardless
330
+ * of how many other stages exist.
331
+ */
332
+ async function resolveStageIdBySlug({ baseUrl, token, slug, repo, }) {
333
+ const repoFilter = repo ? `&repo=${encodeURIComponent(repo)}` : '';
334
+ const listed = await getJson({
335
+ baseUrl,
336
+ token,
337
+ path: `/api/data-pipelines/stages?limit=200&slug=${encodeURIComponent(slug)}${repoFilter}`,
338
+ });
339
+ const first = listed.stages[0];
340
+ if (!first) {
341
+ throw new Error(`stage '${slug}' not found${repo ? ` in ${repo}` : ''}`);
342
+ }
343
+ if (listed.stages.length > 1) {
344
+ throw new Error(`stage '${slug}' exists in multiple Pipelines (${listed.stages.map((m) => m.repo).join(', ')}) — ` +
345
+ 'disambiguate with --repo pipelines/<domain>');
346
+ }
347
+ return first.id;
348
+ }
@@ -0,0 +1,5 @@
1
+ /** Client-side pin rules mirrored from the server deploy adapter (SEQ-2450). */
2
+ export declare const FULL_SHA_PATTERN: RegExp;
3
+ export declare const PINNED_ENVIRONMENTS: Set<string>;
4
+ export declare function isFullSha(ref: string): boolean;
5
+ export declare function requiresPinnedSha(environment: string): boolean;
@@ -0,0 +1,9 @@
1
+ /** Client-side pin rules mirrored from the server deploy adapter (SEQ-2450). */
2
+ export const FULL_SHA_PATTERN = /^[0-9a-f]{40}$/i;
3
+ export const PINNED_ENVIRONMENTS = new Set(['production', 'banksouth']);
4
+ export function isFullSha(ref) {
5
+ return FULL_SHA_PATTERN.test(ref);
6
+ }
7
+ export function requiresPinnedSha(environment) {
8
+ return PINNED_ENVIRONMENTS.has(environment);
9
+ }
@@ -0,0 +1,11 @@
1
+ /**
2
+ * `seq-studio pipeline init` scaffolds — commented per-kind `<name>.stage.yml`
3
+ * templates plus an entrypoint stub. Kept as string constants (not a
4
+ * templates/ directory) so the pipeline family needs no copy step and works
5
+ * from any install of the CLI.
6
+ */
7
+ export declare const STAGE_TYPES: readonly ["ingestion", "transformation", "serving"];
8
+ export type StageTemplateType = (typeof STAGE_TYPES)[number];
9
+ export declare function renderStageTemplate(name: string, type: StageTemplateType): string;
10
+ /** Serving stages are declarative — no entrypoint stub. */
11
+ export declare function renderEntrypointStub(name: string): string;
@@ -0,0 +1,166 @@
1
+ /**
2
+ * `seq-studio pipeline init` scaffolds — commented per-kind `<name>.stage.yml`
3
+ * templates plus an entrypoint stub. Kept as string constants (not a
4
+ * templates/ directory) so the pipeline family needs no copy step and works
5
+ * from any install of the CLI.
6
+ */
7
+ const envelopeTemplate = (name, type) => `# ${name}.stage.yml — one deployable ${type} stage of this Pipeline.
8
+ # Docs: @sequenceholdings/pipeline-spec (the schema is the single source of truth).
9
+ schema_version: 1
10
+ stage: ${name}
11
+ title: TODO — one-line human title
12
+ description: >-
13
+ TODO — what this stage does and why it exists.
14
+ type: ${type}
15
+ owners:
16
+ - you@seqholdings.com
17
+ # critical | standard | experimental
18
+ criticality: experimental
19
+ # Optional reliability posture:
20
+ # slos:
21
+ # freshness: PT4H # ISO-8601 durations
22
+ # max_duration: PT1H
23
+ # alerts:
24
+ # channels: [eng-alerts] # symbolic channel refs — never webhook URLs
25
+ # runbook: https://notion.so/your-runbook
26
+ environments:
27
+ - staging
28
+ `;
29
+ const INGESTION_BODY = `
30
+ # Where the ingestion code runs: databricks (API pulls) | edge-worker (on-prem VM).
31
+ # ('upload' is reserved — not yet supported.)
32
+ runtime: databricks
33
+ # databricks runtime pulls from an external system (symbolic refs, never secrets).
34
+ # Secure-by-default: declare a credential family for authenticated APIs.
35
+ # For genuinely keyless / public sources, omit credential entirely (do not
36
+ # invent a sentinel binding) — e.g. source: { system: espn }.
37
+ source:
38
+ system: my_source_system
39
+ credential: my_source_credential
40
+ # edge-worker runtime instead binds a registered worker:
41
+ # worker:
42
+ # machine: my_vm
43
+ # data_source: my_library
44
+ # credential: my_cred_family
45
+ feeds:
46
+ # One feed = one stream of raw files into the volume landing zone.
47
+ # The platform derives (per-source catalog, never the deployment catalog):
48
+ # /Volumes/src_<source>/bronze/<feed>/batch_id=<ts>/, the bronze table
49
+ # src_<source>.bronze.<feed>, envelope columns (batch_id, ingested_at,
50
+ # source_system, content_hash), and a presence manifest for full_snapshot.
51
+ my_feed:
52
+ # full_snapshot | incremental | event_stream
53
+ snapshot_mode: full_snapshot
54
+ natural_key:
55
+ - id
56
+ # incremental feeds declare their cursor:
57
+ # cursor:
58
+ # column: updated_at
59
+ # encoding: yyyyddd # extensible string — provider-neutral
60
+ # none | contains_pii
61
+ pii: none
62
+ # parquet | json | csv | custom ('xlsx' is reserved — not yet supported)
63
+ format: json
64
+ # format: custom requires a pure file→rows decoder (never writes tables):
65
+ # decoder: src/decoders/my_decoder.py
66
+ schedule:
67
+ cron: "0 6 * * *"
68
+ tz: America/New_York
69
+ # Max staleness before the freshness SLO breaches:
70
+ freshness_slo: P1D
71
+ # Raw-file replay buffer (not an archive):
72
+ retention: P30D
73
+ entrypoint: src/{{snake}}.py
74
+ `;
75
+ const TRANSFORMATION_BODY = `
76
+ # dlt | spark-job | polars
77
+ engine: dlt
78
+ inputs:
79
+ # Reference upstream assets by stage + feed (ingestion) or output:
80
+ # - stage: my-ingest
81
+ # feed: my_feed
82
+ # - stage: upstream-transform
83
+ # output: silver.some_table
84
+ # columns: [id, amount] # the subset you consume (validated)
85
+ # Lakebase CDF reverse-sync input (columns REQUIRED, never inferred):
86
+ # - kind: lakebase_cdf
87
+ # table: manual.my_overrides
88
+ # fold: latest_state # latest_state | events
89
+ # columns:
90
+ # id: { type: string, nullable: false, key: true }
91
+ []
92
+ trigger:
93
+ # At least one of cron / after:
94
+ cron: "0 7 * * *"
95
+ # after: [my-ingest]
96
+ outputs:
97
+ # Typed contracts — names must be silver.* or gold.*:
98
+ silver.{{snake}}:
99
+ columns:
100
+ id:
101
+ type: string
102
+ nullable: false
103
+ key: true
104
+ value:
105
+ type: int
106
+ # fail | quarantine (→ silver.{{snake}}__quarantine) | warn
107
+ on_type_violation: fail
108
+ # additive-only (default: breaking changes fail CI) | versioned
109
+ compatibility: additive-only
110
+ # overwrite | append | auto_cdc
111
+ write_pattern: overwrite
112
+ entrypoint: src/{{snake}}.py
113
+ `;
114
+ const SERVING_BODY = `
115
+ # The transformation output this stage serves:
116
+ source:
117
+ stage: my-gold-stage
118
+ output: gold.my_table
119
+ projection:
120
+ # The served contract (inline columns or schema_ref):
121
+ columns:
122
+ id:
123
+ type: uuid
124
+ nullable: false
125
+ key: true
126
+ name:
127
+ type: string
128
+ primary_key:
129
+ - id
130
+ # SNAPSHOT | TRIGGERED | CONTINUOUS (overwrite-written sources: SNAPSHOT only)
131
+ sync_mode: SNAPSHOT
132
+ trigger:
133
+ after: my-gold-stage
134
+ expose:
135
+ view: public.my_table
136
+ # Optional ORM coupling ('managed' mode is reserved — not yet supported):
137
+ # orm_binding:
138
+ # namespace: my_namespace
139
+ # table: my_table
140
+ # mode: referenced
141
+ `;
142
+ const ENTRYPOINT_STUB = `"""Entrypoint stub for the {{name}} stage.
143
+
144
+ The platform invokes this per the stage spec ({{name}}.stage.yml).
145
+ Decoders and transforms stay pure — bronze materialization, envelope
146
+ stamping, and table writes are platform-owned.
147
+ """
148
+
149
+
150
+ def main() -> None:
151
+ raise NotImplementedError("implement the {{name}} stage")
152
+
153
+
154
+ if __name__ == "__main__":
155
+ main()
156
+ `;
157
+ export const STAGE_TYPES = ['ingestion', 'transformation', 'serving'];
158
+ export function renderStageTemplate(name, type) {
159
+ const body = type === 'ingestion' ? INGESTION_BODY : type === 'transformation' ? TRANSFORMATION_BODY : SERVING_BODY;
160
+ return (envelopeTemplate(name, type) +
161
+ body.replace(/\{\{name\}\}/g, name).replace(/\{\{snake\}\}/g, name.replace(/-/g, '_')));
162
+ }
163
+ /** Serving stages are declarative — no entrypoint stub. */
164
+ export function renderEntrypointStub(name) {
165
+ return ENTRYPOINT_STUB.replace(/\{\{name\}\}/g, name);
166
+ }
@@ -18,4 +18,8 @@ export interface BuildBundleOptions {
18
18
  };
19
19
  }
20
20
  export declare function buildBundleFromProcesses(defs: readonly LoadedProcess[], options?: BuildBundleOptions): Promise<LatticeBundle>;
21
+ export declare function finalizeCompiledBundle({ bundle, resolveProcessPin, }: {
22
+ bundle: LatticeBundle;
23
+ resolveProcessPin: ResolveProcessPin;
24
+ }): Promise<LatticeBundle>;
21
25
  export declare function summarizeBundle(bundle: LatticeBundle): LatticeBundleSummary;
@@ -27,6 +27,16 @@ export async function buildBundleFromProcesses(defs, options) {
27
27
  // the SDK so a CLI-built bundle is byte-identical to a server-applied one.
28
28
  return finalizeBundleIdentity({ processes, metadata });
29
29
  }
30
+ export async function finalizeCompiledBundle({ bundle, resolveProcessPin, }) {
31
+ const processes = bundle.processes.map((process) => structuredClone(process));
32
+ const localProcessIds = new Set(processes.map((process) => process.id));
33
+ await resolveSubprocessPinsInProcesses(processes, async (processId, version) => {
34
+ if (!version && localProcessIds.has(processId))
35
+ return null;
36
+ return resolveProcessPin(processId, version);
37
+ });
38
+ return finalizeBundleIdentity({ processes, metadata: bundle.metadata });
39
+ }
30
40
  async function resolveSubprocessPinsInProcesses(processes, resolve) {
31
41
  const pins = [];
32
42
  for (const proc of processes) {
@@ -86,8 +96,20 @@ function serializeProcess(input) {
86
96
  ...(input.process.supervisor_retry !== undefined
87
97
  ? { supervisor_retry: serializeRetryConfig(input.process.supervisor_retry) }
88
98
  : {}),
99
+ // Both run_as keys are omitted when unset so existing hashes stay stable
100
+ // (an email-only run_as serializes byte-identically to before the
101
+ // service_account field existed).
89
102
  ...(input.process.run_as !== undefined
90
- ? { run_as: { email: input.process.run_as.email } }
103
+ ? {
104
+ run_as: {
105
+ ...(input.process.run_as.email !== undefined
106
+ ? { email: input.process.run_as.email }
107
+ : {}),
108
+ ...(input.process.run_as.service_account !== undefined
109
+ ? { service_account: input.process.run_as.service_account }
110
+ : {}),
111
+ },
112
+ }
91
113
  : {}),
92
114
  };
93
115
  }
@@ -135,6 +157,10 @@ function serializeNodeMetadata(node) {
135
157
  meta.limits = native.limits;
136
158
  if (native.bindings)
137
159
  meta.bindings = native.bindings;
160
+ // Per-node service-account override — omitted when unset for hash stability.
161
+ if (native.run_as?.service_account) {
162
+ meta.run_as = { service_account: native.run_as.service_account };
163
+ }
138
164
  const nativeInputMapper = native.input;
139
165
  if (typeof nativeInputMapper === 'function') {
140
166
  meta.input_mapper_source = nativeInputMapper.toString();
@@ -164,8 +190,9 @@ function serializeNodeMetadata(node) {
164
190
  const human = node;
165
191
  const meta = {
166
192
  metadata: human.metadata ?? {},
167
- timeout: human.timeout ?? '7d',
168
193
  };
194
+ if (human.timeout !== undefined)
195
+ meta.timeout = human.timeout;
169
196
  if (human.on_timeout_edge_id)
170
197
  meta.on_timeout_edge_id = human.on_timeout_edge_id;
171
198
  if (human.completeLabel)
@@ -327,6 +354,10 @@ function serializeNodeMetadata(node) {
327
354
  meta.limits = mf.limits;
328
355
  if (mf.bindings)
329
356
  meta.bindings = mf.bindings;
357
+ // Per-node service-account override — omitted when unset for hash stability.
358
+ if (mf.run_as?.service_account) {
359
+ meta.run_as = { service_account: mf.run_as.service_account };
360
+ }
330
361
  // NOTE: `middleware` (e.g. withRetry) is serialized generically for every
331
362
  // node kind in `serializeNode` above — don't duplicate it here.
332
363
  const inputMapper = mf.input;
@@ -50,7 +50,16 @@ export function generateProcessFiles(process, bundle) {
50
50
  ? ` max_concurrent_runs: ${process.max_concurrent_runs},`
51
51
  : null,
52
52
  process.supervisor_retry !== undefined ? ` supervisor_retry: ${pretty(process.supervisor_retry, 1)},` : null,
53
- process.run_as !== undefined ? ` run_as: { email: ${lit(process.run_as.email)} },` : null,
53
+ process.run_as !== undefined
54
+ ? ` run_as: { ${[
55
+ process.run_as.email !== undefined ? `email: ${lit(process.run_as.email)}` : null,
56
+ process.run_as.service_account !== undefined
57
+ ? `service_account: ${lit(process.run_as.service_account)}`
58
+ : null,
59
+ ]
60
+ .filter((f) => f !== null)
61
+ .join(', ')} },`
62
+ : null,
54
63
  Array.isArray(process.tags) && process.tags.length > 0
55
64
  ? ` tags: ${pretty(process.tags, 1)},`
56
65
  : null,
@@ -125,6 +134,7 @@ function emitNode(node, ctx) {
125
134
  ` function: ${lit(String(m.registered_function_id))},`,
126
135
  m.config !== undefined ? ` config: ${pretty(m.config, 1)},` : null,
127
136
  m.limits !== undefined ? ` limits: ${pretty(m.limits, 1)},` : null,
137
+ runAsField(m.run_as),
128
138
  bindings(m.bindings),
129
139
  middleware(m.middleware),
130
140
  mapper('input', m.input_mapper_source),
@@ -189,6 +199,7 @@ function emitNode(node, ctx) {
189
199
  ` function: ${lit(String(m.function_id))},`,
190
200
  m.version ? ` version: ${lit(String(m.version))},` : null,
191
201
  m.limits !== undefined ? ` limits: ${pretty(m.limits, 1)},` : null,
202
+ runAsField(m.run_as),
192
203
  bindings(m.bindings),
193
204
  middleware(m.middleware),
194
205
  mapper('input', m.input_mapper_source),
@@ -209,6 +220,13 @@ function emitEdges(edges) {
209
220
  const items = edges.map((e) => `{ id: ${lit(e.id)}, to: ${lit(e.to)} }`).join(', ');
210
221
  return `[${items}] as const`;
211
222
  }
223
+ /** Per-node service-account override — `metadata.run_as.service_account`. */
224
+ function runAsField(runAs) {
225
+ const sa = runAs?.service_account;
226
+ if (typeof sa !== 'string' || sa.length === 0)
227
+ return null;
228
+ return ` run_as: { service_account: ${lit(sa)} },`;
229
+ }
212
230
  /** A mapper field whose value is a serialized `Function.toString()` string. */
213
231
  function mapper(field, source) {
214
232
  if (typeof source !== 'string' || source.length === 0)