@sequenceholdings/studio-cli 0.1.12 → 0.1.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/README.md +116 -30
  2. package/dist/artifact/delegate.d.ts +2 -2
  3. package/dist/artifact/delegate.js +31 -73
  4. package/dist/atlas-client.js +52 -37
  5. package/dist/auth-cmds/commands.d.ts +1 -1
  6. package/dist/auth-cmds/commands.js +12 -7
  7. package/dist/auth.d.ts +97 -19
  8. package/dist/auth.js +376 -81
  9. package/dist/config.d.ts +13 -3
  10. package/dist/config.js +41 -14
  11. package/dist/env-catalog.js +13 -3
  12. package/dist/env-flags.d.ts +2 -0
  13. package/dist/env-flags.js +2 -0
  14. package/dist/env-registry.d.ts +27 -0
  15. package/dist/env-registry.js +204 -0
  16. package/dist/envs/commands.d.ts +1 -1
  17. package/dist/envs/commands.js +65 -10
  18. package/dist/file-lock.d.ts +5 -0
  19. package/dist/file-lock.js +187 -0
  20. package/dist/functions/commands.d.ts +9 -1
  21. package/dist/functions/commands.js +71 -29
  22. package/dist/functions/manifest.d.ts +1 -0
  23. package/dist/functions/manifest.js +36 -0
  24. package/dist/login.d.ts +14 -3
  25. package/dist/login.js +60 -40
  26. package/dist/main.d.ts +2 -1
  27. package/dist/main.js +36 -12
  28. package/dist/orm/delegate.d.ts +9 -0
  29. package/dist/orm/delegate.js +36 -4
  30. package/dist/pat-hints.js +2 -2
  31. package/dist/pipeline/commands.d.ts +58 -0
  32. package/dist/pipeline/commands.js +330 -0
  33. package/dist/pipeline/lifecycle.d.ts +58 -0
  34. package/dist/pipeline/lifecycle.js +348 -0
  35. package/dist/pipeline/pinning.d.ts +5 -0
  36. package/dist/pipeline/pinning.js +9 -0
  37. package/dist/pipeline/templates.d.ts +11 -0
  38. package/dist/pipeline/templates.js +166 -0
  39. package/dist/process/build.d.ts +12 -1
  40. package/dist/process/build.js +49 -2
  41. package/dist/process/codegen.js +21 -1
  42. package/dist/process/commands.d.ts +19 -0
  43. package/dist/process/commands.js +153 -40
  44. package/dist/process/compiler-subprocess.d.ts +29 -0
  45. package/dist/process/compiler-subprocess.js +99 -0
  46. package/dist/process/compiler-worker.d.ts +1 -0
  47. package/dist/process/compiler-worker.js +38 -0
  48. package/dist/process/discover.d.ts +4 -1
  49. package/dist/process/discover.js +5 -2
  50. package/dist/process/lint.d.ts +8 -0
  51. package/dist/process/lint.js +125 -29
  52. package/dist/process/repo-install.d.ts +21 -0
  53. package/dist/process/repo-install.js +99 -0
  54. package/dist/process/simulate.js +14 -1
  55. package/dist/repos/commands.d.ts +1 -1
  56. package/dist/repos/commands.js +17 -12
  57. package/dist/secrets/commands.d.ts +1 -1
  58. package/dist/secrets/commands.js +18 -18
  59. package/package.json +9 -4
@@ -0,0 +1,348 @@
1
+ /**
2
+ * Deploy lifecycle verbs for `seq-studio pipeline plan|deploy|promote|rollback`
3
+ * (SEQ-2450). Wraps the /api/data-pipelines plan/deploy APIs.
4
+ */
5
+ import { getAccessToken, NotLoggedInError } from '../auth.js';
6
+ import { AtlasApiError, getJson, postJson } from '../atlas-client.js';
7
+ import { resolveEnvWithDiscovery } from '../config.js';
8
+ import { isFullSha, requiresPinnedSha } from './pinning.js';
9
+ const LOG = '[seq-studio]';
10
+ const TERMINAL = new Set(['active', 'failed', 'retired']);
11
+ async function envAndToken(args) {
12
+ const requested = typeof args.flags.e === 'string' ? args.flags.e : typeof args.flags.env === 'string' ? args.flags.env : undefined;
13
+ if (!requested) {
14
+ throw new Error('missing -e/--env <environment>');
15
+ }
16
+ const env = await resolveEnvWithDiscovery({ requested });
17
+ let token;
18
+ try {
19
+ token = await getAccessToken();
20
+ }
21
+ catch (error) {
22
+ if (error instanceof NotLoggedInError)
23
+ throw error;
24
+ throw error;
25
+ }
26
+ return { env, token };
27
+ }
28
+ function flagString(flags, key) {
29
+ const value = flags[key];
30
+ if (value === true)
31
+ throw new Error(`--${key} requires a value`);
32
+ return value;
33
+ }
34
+ /**
35
+ * The CLI's local dev-loop env is named `local` — chosen so `-e local` lines
36
+ * up with `seqapi -e local` / `artifact-studio --env local` (see `config.ts`
37
+ * `BUILT_IN_ENV_URLS`). There is no `local` in the server's deploy-lifecycle
38
+ * enum (`DEPLOY_ENVIRONMENTS` in `atlas/src/server/services/data-pipelines/schema.ts`
39
+ * is `dev | staging | production | banksouth`); the server's name for that
40
+ * same developer-loop target is `dev` (`targetFactsForEnvironment` treats
41
+ * `dev` as the one target exempt from a `lakebase_branch` binding). Map at
42
+ * this one boundary — every lifecycle request body funnels through here —
43
+ * so the CLI never sends the wire-invalid `local` and every other env name
44
+ * passes through unchanged.
45
+ */
46
+ export function deployEnvironmentForEnv(env) {
47
+ return env.name === 'local' ? 'dev' : env.name;
48
+ }
49
+ export async function pipelinePlanCommand(args) {
50
+ const repo = flagString(args.flags, 'repo');
51
+ const ref = flagString(args.flags, 'ref');
52
+ if (!repo || !ref) {
53
+ console.error('usage: seq-studio pipeline plan --repo pipelines/<slug> --ref <sha|branch> -e <env> [--json]');
54
+ return 1;
55
+ }
56
+ const { env, token } = await envAndToken(args);
57
+ const json = args.flags.json === true || args.flags.json === 'true';
58
+ let response;
59
+ try {
60
+ response = await postJson({
61
+ baseUrl: env.url,
62
+ token,
63
+ path: '/api/data-pipelines/pipelines/plan',
64
+ body: { repo, ref, environment: deployEnvironmentForEnv(env) },
65
+ });
66
+ }
67
+ catch (error) {
68
+ if (error instanceof AtlasApiError) {
69
+ console.error(`${LOG} plan failed: ${error.message}`);
70
+ return 1;
71
+ }
72
+ throw error;
73
+ }
74
+ if (json) {
75
+ console.log(JSON.stringify(response, null, 2));
76
+ }
77
+ else {
78
+ console.log(response.text);
79
+ printFindingsTable(response.plan.findings);
80
+ }
81
+ return response.plan.hasDestructive ? 1 : 0;
82
+ }
83
+ export async function pipelineDeployCommand(args) {
84
+ const repo = flagString(args.flags, 'repo');
85
+ const ref = flagString(args.flags, 'ref');
86
+ const deploymentId = flagString(args.flags, 'deployment-id');
87
+ const approvedBy = flagString(args.flags, 'approved-by');
88
+ const { env, token } = await envAndToken(args);
89
+ if (requiresPinnedSha(env.name) && ref && !isFullSha(ref)) {
90
+ console.error(`${LOG} deploy to ${env.name} requires a pinned 40-hex commit SHA (got '${ref}')`);
91
+ return 1;
92
+ }
93
+ let enqueue;
94
+ try {
95
+ if (deploymentId) {
96
+ // Direct execute of a persisted plan.
97
+ const stageId = flagString(args.flags, 'stage-id');
98
+ if (!stageId) {
99
+ console.error('usage: seq-studio pipeline deploy --deployment-id <id> --stage-id <id> -e <env> [--approved-by <sub>] [--no-wait]');
100
+ return 1;
101
+ }
102
+ enqueue = await postJson({
103
+ baseUrl: env.url,
104
+ token,
105
+ path: `/api/data-pipelines/stages/${stageId}/deploy`,
106
+ body: { deploymentId, ...(approvedBy ? { approvedBy } : {}) },
107
+ });
108
+ }
109
+ else {
110
+ if (!repo || !ref) {
111
+ console.error('usage: seq-studio pipeline deploy --repo pipelines/<slug> --ref <sha> -e <env> [--approved-by <sub>] [--no-wait]');
112
+ return 1;
113
+ }
114
+ // Plan then deploy the first planning deployment.
115
+ const plan = await postJson({
116
+ baseUrl: env.url,
117
+ token,
118
+ path: '/api/data-pipelines/pipelines/plan',
119
+ body: { repo, ref, environment: deployEnvironmentForEnv(env) },
120
+ });
121
+ if (plan.plan.hasDestructive) {
122
+ console.error(`${LOG} plan has destructive findings — refusing to deploy`);
123
+ if (args.flags.json === true || args.flags.json === 'true') {
124
+ console.log(JSON.stringify(plan, null, 2));
125
+ }
126
+ else {
127
+ console.error(plan.text);
128
+ }
129
+ return 1;
130
+ }
131
+ const firstId = plan.deploymentIds[0];
132
+ const stageId = plan.stageIds[0];
133
+ if (!firstId || !stageId) {
134
+ console.error(`${LOG} plan produced no deployment rows`);
135
+ return 1;
136
+ }
137
+ enqueue = await postJson({
138
+ baseUrl: env.url,
139
+ token,
140
+ path: `/api/data-pipelines/stages/${stageId}/deploy`,
141
+ body: {
142
+ deploymentId: firstId,
143
+ ...(approvedBy ? { approvedBy } : {}),
144
+ },
145
+ });
146
+ }
147
+ }
148
+ catch (error) {
149
+ if (error instanceof AtlasApiError) {
150
+ console.error(`${LOG} deploy failed: ${error.message}`);
151
+ return 1;
152
+ }
153
+ throw error;
154
+ }
155
+ console.log(`${LOG} enqueued deploy ${enqueue.deploymentId} (trigger=${enqueue.triggerRunId ?? 'n/a'})`);
156
+ if (args.flags['no-wait'] === true || args.flags['no-wait'] === 'true') {
157
+ return 0;
158
+ }
159
+ return pollDeployment({ baseUrl: env.url, token, deploymentId: enqueue.deploymentId });
160
+ }
161
+ export async function pipelinePromoteCommand(args) {
162
+ const stage = flagString(args.flags, 'stage');
163
+ const version = flagString(args.flags, 'version');
164
+ const repo = flagString(args.flags, 'repo');
165
+ const approvedBy = flagString(args.flags, 'approved-by');
166
+ if (!stage || !version) {
167
+ console.error('usage: seq-studio pipeline promote --stage <slug> --version <v> -e <env> [--repo pipelines/<slug>] [--approved-by <you>] [--no-wait]');
168
+ return 1;
169
+ }
170
+ const { env, token } = await envAndToken(args);
171
+ if (requiresPinnedSha(env.name) && !approvedBy) {
172
+ console.error(`${LOG} promote to ${env.name} requires --approved-by <your sub/email> (approvals are self-recorded)`);
173
+ return 1;
174
+ }
175
+ const stageId = await resolveStageIdBySlug({ baseUrl: env.url, token, slug: stage, repo });
176
+ let enqueue;
177
+ try {
178
+ enqueue = await postJson({
179
+ baseUrl: env.url,
180
+ token,
181
+ path: `/api/data-pipelines/stages/${stageId}/promote`,
182
+ body: {
183
+ version,
184
+ environment: deployEnvironmentForEnv(env),
185
+ ...(approvedBy ? { approvedBy } : {}),
186
+ },
187
+ });
188
+ }
189
+ catch (error) {
190
+ if (error instanceof AtlasApiError) {
191
+ console.error(`${LOG} promote failed: ${error.message}`);
192
+ return 1;
193
+ }
194
+ throw error;
195
+ }
196
+ console.log(`${LOG} enqueued promote ${enqueue.deploymentId}`);
197
+ if (args.flags['no-wait'] === true || args.flags['no-wait'] === 'true')
198
+ return 0;
199
+ return pollDeployment({ baseUrl: env.url, token, deploymentId: enqueue.deploymentId });
200
+ }
201
+ export async function pipelineRunNowCommand(args) {
202
+ const stage = flagString(args.flags, 'stage');
203
+ const repo = flagString(args.flags, 'repo');
204
+ if (!stage) {
205
+ console.error('usage: seq-studio pipeline run-now --stage <slug> -e <env> [--repo pipelines/<slug>] [--json]');
206
+ return 1;
207
+ }
208
+ const { env, token } = await envAndToken(args);
209
+ const json = args.flags.json === true || args.flags.json === 'true';
210
+ let stageId;
211
+ try {
212
+ stageId = await resolveStageIdBySlug({ baseUrl: env.url, token, slug: stage, repo });
213
+ }
214
+ catch (error) {
215
+ console.error(`${LOG} ${error instanceof Error ? error.message : String(error)}`);
216
+ return 1;
217
+ }
218
+ let response;
219
+ try {
220
+ response = await postJson({
221
+ baseUrl: env.url,
222
+ token,
223
+ path: `/api/data-pipelines/stages/${stageId}/run`,
224
+ body: { environment: deployEnvironmentForEnv(env) },
225
+ });
226
+ }
227
+ catch (error) {
228
+ if (error instanceof AtlasApiError) {
229
+ console.error(`${LOG} run-now failed: ${error.message}`);
230
+ return 1;
231
+ }
232
+ throw error;
233
+ }
234
+ if (json) {
235
+ console.log(JSON.stringify(response, null, 2));
236
+ }
237
+ else {
238
+ console.log(`${LOG} run-now ${stage} → ${deployEnvironmentForEnv(env)}`);
239
+ console.log(`${LOG} invocation ${response.invocationId} (${response.status})`);
240
+ console.log(`${LOG} ${response.databricks_url}`);
241
+ }
242
+ return 0;
243
+ }
244
+ export async function pipelineRollbackCommand(args) {
245
+ const stage = flagString(args.flags, 'stage');
246
+ const repo = flagString(args.flags, 'repo');
247
+ const approvedBy = flagString(args.flags, 'approved-by');
248
+ if (!stage) {
249
+ console.error('usage: seq-studio pipeline rollback --stage <slug> -e <env> [--repo pipelines/<slug>] [--approved-by <you>] [--no-wait]');
250
+ return 1;
251
+ }
252
+ const { env, token } = await envAndToken(args);
253
+ if (requiresPinnedSha(env.name) && !approvedBy) {
254
+ console.error(`${LOG} rollback in ${env.name} requires --approved-by <your sub/email> (approvals are explicit even for rollbacks)`);
255
+ return 1;
256
+ }
257
+ const stageId = await resolveStageIdBySlug({ baseUrl: env.url, token, slug: stage, repo });
258
+ let enqueue;
259
+ try {
260
+ enqueue = await postJson({
261
+ baseUrl: env.url,
262
+ token,
263
+ path: `/api/data-pipelines/stages/${stageId}/rollback`,
264
+ body: {
265
+ environment: deployEnvironmentForEnv(env),
266
+ ...(approvedBy ? { approvedBy } : {}),
267
+ },
268
+ });
269
+ }
270
+ catch (error) {
271
+ if (error instanceof AtlasApiError) {
272
+ console.error(`${LOG} rollback failed: ${error.message}`);
273
+ return 1;
274
+ }
275
+ throw error;
276
+ }
277
+ console.log(`${LOG} enqueued rollback ${enqueue.deploymentId}`);
278
+ if (args.flags['no-wait'] === true || args.flags['no-wait'] === 'true')
279
+ return 0;
280
+ return pollDeployment({ baseUrl: env.url, token, deploymentId: enqueue.deploymentId });
281
+ }
282
+ function printFindingsTable(findings) {
283
+ if (findings.length === 0) {
284
+ console.log(`${LOG} no resource changes`);
285
+ return;
286
+ }
287
+ console.log(`${LOG} ${'CLASS'.padEnd(12)} ${'RESOURCE'.padEnd(32)} MESSAGE`);
288
+ for (const f of findings) {
289
+ const klass = f.destructiveReason
290
+ ? `${f.classification}/${f.destructiveReason}`
291
+ : f.classification;
292
+ console.log(`${LOG} ${klass.padEnd(12)} ${f.resourceKey.padEnd(32)} ${f.message}`);
293
+ }
294
+ }
295
+ async function pollDeployment({ baseUrl, token, deploymentId, }) {
296
+ const started = Date.now();
297
+ const timeoutMs = 15 * 60 * 1000;
298
+ while (Date.now() - started < timeoutMs) {
299
+ const detail = await getJson({
300
+ baseUrl,
301
+ token,
302
+ path: `/api/data-pipelines/deployments/${deploymentId}`,
303
+ });
304
+ console.log(`${LOG} deployment ${deploymentId}: ${detail.status}`);
305
+ if (TERMINAL.has(detail.status)) {
306
+ if (detail.status === 'active') {
307
+ console.log(`${LOG} deploy succeeded`);
308
+ return 0;
309
+ }
310
+ console.error(`${LOG} deploy ended '${detail.status}': ${detail.statusDetail ?? ''}`);
311
+ return 1;
312
+ }
313
+ await new Promise((r) => setTimeout(r, 2000));
314
+ }
315
+ console.error(`${LOG} timed out waiting for deployment ${deploymentId}`);
316
+ return 1;
317
+ }
318
+ /**
319
+ * Stage identity is `(repo, slug)` — a bare slug can be ambiguous across
320
+ * Pipelines. `--repo pipelines/<domain>` scopes the lookup server-side; an
321
+ * ambiguous bare slug fails with the candidate repos rather than guessing.
322
+ *
323
+ * The `slug` query param is an EXACT server-side filter (contracts.ts
324
+ * `listStagesQuerySchema`) — the list route's default sort is
325
+ * `updated_at DESC`, so scanning only the first page (as this used to do)
326
+ * silently missed a matching stage buried on a later page, and would have
327
+ * missed a same-slug duplicate on a later page too, falsely reporting the
328
+ * bare-slug lookup as unique (SEQ-2447 review). Filtering server-side by the
329
+ * exact slug means every match is guaranteed to fit on one page regardless
330
+ * of how many other stages exist.
331
+ */
332
+ async function resolveStageIdBySlug({ baseUrl, token, slug, repo, }) {
333
+ const repoFilter = repo ? `&repo=${encodeURIComponent(repo)}` : '';
334
+ const listed = await getJson({
335
+ baseUrl,
336
+ token,
337
+ path: `/api/data-pipelines/stages?limit=200&slug=${encodeURIComponent(slug)}${repoFilter}`,
338
+ });
339
+ const first = listed.stages[0];
340
+ if (!first) {
341
+ throw new Error(`stage '${slug}' not found${repo ? ` in ${repo}` : ''}`);
342
+ }
343
+ if (listed.stages.length > 1) {
344
+ throw new Error(`stage '${slug}' exists in multiple Pipelines (${listed.stages.map((m) => m.repo).join(', ')}) — ` +
345
+ 'disambiguate with --repo pipelines/<domain>');
346
+ }
347
+ return first.id;
348
+ }
@@ -0,0 +1,5 @@
1
+ /** Client-side pin rules mirrored from the server deploy adapter (SEQ-2450). */
2
+ export declare const FULL_SHA_PATTERN: RegExp;
3
+ export declare const PINNED_ENVIRONMENTS: Set<string>;
4
+ export declare function isFullSha(ref: string): boolean;
5
+ export declare function requiresPinnedSha(environment: string): boolean;
@@ -0,0 +1,9 @@
1
+ /** Client-side pin rules mirrored from the server deploy adapter (SEQ-2450). */
2
+ export const FULL_SHA_PATTERN = /^[0-9a-f]{40}$/i;
3
+ export const PINNED_ENVIRONMENTS = new Set(['production', 'banksouth']);
4
+ export function isFullSha(ref) {
5
+ return FULL_SHA_PATTERN.test(ref);
6
+ }
7
+ export function requiresPinnedSha(environment) {
8
+ return PINNED_ENVIRONMENTS.has(environment);
9
+ }
@@ -0,0 +1,11 @@
1
+ /**
2
+ * `seq-studio pipeline init` scaffolds — commented per-kind `<name>.stage.yml`
3
+ * templates plus an entrypoint stub. Kept as string constants (not a
4
+ * templates/ directory) so the pipeline family needs no copy step and works
5
+ * from any install of the CLI.
6
+ */
7
+ export declare const STAGE_TYPES: readonly ["ingestion", "transformation", "serving"];
8
+ export type StageTemplateType = (typeof STAGE_TYPES)[number];
9
+ export declare function renderStageTemplate(name: string, type: StageTemplateType): string;
10
+ /** Serving stages are declarative — no entrypoint stub. */
11
+ export declare function renderEntrypointStub(name: string): string;
@@ -0,0 +1,166 @@
1
+ /**
2
+ * `seq-studio pipeline init` scaffolds — commented per-kind `<name>.stage.yml`
3
+ * templates plus an entrypoint stub. Kept as string constants (not a
4
+ * templates/ directory) so the pipeline family needs no copy step and works
5
+ * from any install of the CLI.
6
+ */
7
+ const envelopeTemplate = (name, type) => `# ${name}.stage.yml — one deployable ${type} stage of this Pipeline.
8
+ # Docs: @sequenceholdings/pipeline-spec (the schema is the single source of truth).
9
+ schema_version: 1
10
+ stage: ${name}
11
+ title: TODO — one-line human title
12
+ description: >-
13
+ TODO — what this stage does and why it exists.
14
+ type: ${type}
15
+ owners:
16
+ - you@seqholdings.com
17
+ # critical | standard | experimental
18
+ criticality: experimental
19
+ # Optional reliability posture:
20
+ # slos:
21
+ # freshness: PT4H # ISO-8601 durations
22
+ # max_duration: PT1H
23
+ # alerts:
24
+ # channels: [eng-alerts] # symbolic channel refs — never webhook URLs
25
+ # runbook: https://notion.so/your-runbook
26
+ environments:
27
+ - staging
28
+ `;
29
+ const INGESTION_BODY = `
30
+ # Where the ingestion code runs: databricks (API pulls) | edge-worker (on-prem VM).
31
+ # ('upload' is reserved — not yet supported.)
32
+ runtime: databricks
33
+ # databricks runtime pulls from an external system (symbolic refs, never secrets).
34
+ # Secure-by-default: declare a credential family for authenticated APIs.
35
+ # For genuinely keyless / public sources, omit credential entirely (do not
36
+ # invent a sentinel binding) — e.g. source: { system: espn }.
37
+ source:
38
+ system: my_source_system
39
+ credential: my_source_credential
40
+ # edge-worker runtime instead binds a registered worker:
41
+ # worker:
42
+ # machine: my_vm
43
+ # data_source: my_library
44
+ # credential: my_cred_family
45
+ feeds:
46
+ # One feed = one stream of raw files into the volume landing zone.
47
+ # The platform derives (per-source catalog, never the deployment catalog):
48
+ # /Volumes/src_<source>/bronze/<feed>/batch_id=<ts>/, the bronze table
49
+ # src_<source>.bronze.<feed>, envelope columns (batch_id, ingested_at,
50
+ # source_system, content_hash), and a presence manifest for full_snapshot.
51
+ my_feed:
52
+ # full_snapshot | incremental | event_stream
53
+ snapshot_mode: full_snapshot
54
+ natural_key:
55
+ - id
56
+ # incremental feeds declare their cursor:
57
+ # cursor:
58
+ # column: updated_at
59
+ # encoding: yyyyddd # extensible string — provider-neutral
60
+ # none | contains_pii
61
+ pii: none
62
+ # parquet | json | csv | custom ('xlsx' is reserved — not yet supported)
63
+ format: json
64
+ # format: custom requires a pure file→rows decoder (never writes tables):
65
+ # decoder: src/decoders/my_decoder.py
66
+ schedule:
67
+ cron: "0 6 * * *"
68
+ tz: America/New_York
69
+ # Max staleness before the freshness SLO breaches:
70
+ freshness_slo: P1D
71
+ # Raw-file replay buffer (not an archive):
72
+ retention: P30D
73
+ entrypoint: src/{{snake}}.py
74
+ `;
75
+ const TRANSFORMATION_BODY = `
76
+ # dlt | spark-job | polars
77
+ engine: dlt
78
+ inputs:
79
+ # Reference upstream assets by stage + feed (ingestion) or output:
80
+ # - stage: my-ingest
81
+ # feed: my_feed
82
+ # - stage: upstream-transform
83
+ # output: silver.some_table
84
+ # columns: [id, amount] # the subset you consume (validated)
85
+ # Lakebase CDF reverse-sync input (columns REQUIRED, never inferred):
86
+ # - kind: lakebase_cdf
87
+ # table: manual.my_overrides
88
+ # fold: latest_state # latest_state | events
89
+ # columns:
90
+ # id: { type: string, nullable: false, key: true }
91
+ []
92
+ trigger:
93
+ # At least one of cron / after:
94
+ cron: "0 7 * * *"
95
+ # after: [my-ingest]
96
+ outputs:
97
+ # Typed contracts — names must be silver.* or gold.*:
98
+ silver.{{snake}}:
99
+ columns:
100
+ id:
101
+ type: string
102
+ nullable: false
103
+ key: true
104
+ value:
105
+ type: int
106
+ # fail | quarantine (→ silver.{{snake}}__quarantine) | warn
107
+ on_type_violation: fail
108
+ # additive-only (default: breaking changes fail CI) | versioned
109
+ compatibility: additive-only
110
+ # overwrite | append | auto_cdc
111
+ write_pattern: overwrite
112
+ entrypoint: src/{{snake}}.py
113
+ `;
114
+ const SERVING_BODY = `
115
+ # The transformation output this stage serves:
116
+ source:
117
+ stage: my-gold-stage
118
+ output: gold.my_table
119
+ projection:
120
+ # The served contract (inline columns or schema_ref):
121
+ columns:
122
+ id:
123
+ type: uuid
124
+ nullable: false
125
+ key: true
126
+ name:
127
+ type: string
128
+ primary_key:
129
+ - id
130
+ # SNAPSHOT | TRIGGERED | CONTINUOUS (overwrite-written sources: SNAPSHOT only)
131
+ sync_mode: SNAPSHOT
132
+ trigger:
133
+ after: my-gold-stage
134
+ expose:
135
+ view: public.my_table
136
+ # Optional ORM coupling ('managed' mode is reserved — not yet supported):
137
+ # orm_binding:
138
+ # namespace: my_namespace
139
+ # table: my_table
140
+ # mode: referenced
141
+ `;
142
+ const ENTRYPOINT_STUB = `"""Entrypoint stub for the {{name}} stage.
143
+
144
+ The platform invokes this per the stage spec ({{name}}.stage.yml).
145
+ Decoders and transforms stay pure — bronze materialization, envelope
146
+ stamping, and table writes are platform-owned.
147
+ """
148
+
149
+
150
+ def main() -> None:
151
+ raise NotImplementedError("implement the {{name}} stage")
152
+
153
+
154
+ if __name__ == "__main__":
155
+ main()
156
+ `;
157
+ export const STAGE_TYPES = ['ingestion', 'transformation', 'serving'];
158
+ export function renderStageTemplate(name, type) {
159
+ const body = type === 'ingestion' ? INGESTION_BODY : type === 'transformation' ? TRANSFORMATION_BODY : SERVING_BODY;
160
+ return (envelopeTemplate(name, type) +
161
+ body.replace(/\{\{name\}\}/g, name).replace(/\{\{snake\}\}/g, name.replace(/-/g, '_')));
162
+ }
163
+ /** Serving stages are declarative — no entrypoint stub. */
164
+ export function renderEntrypointStub(name) {
165
+ return ENTRYPOINT_STUB.replace(/\{\{name\}\}/g, name);
166
+ }
@@ -8,7 +8,18 @@ export type ResolveProcessPin = (processId: string, version?: string) => Promise
8
8
  version: string;
9
9
  bundleHash: string;
10
10
  } | null>;
11
- export declare function buildBundleFromProcesses(defs: readonly LoadedProcess[], options?: {
11
+ export interface BuildBundleOptions {
12
12
  resolveProcessPin?: ResolveProcessPin;
13
+ /** Override git provenance instead of reading from the local working tree. */
14
+ provenance?: {
15
+ gitCommit: string | null;
16
+ gitBranch: string | null;
17
+ gitDirty: boolean | null;
18
+ };
19
+ }
20
+ export declare function buildBundleFromProcesses(defs: readonly LoadedProcess[], options?: BuildBundleOptions): Promise<LatticeBundle>;
21
+ export declare function finalizeCompiledBundle({ bundle, resolveProcessPin, }: {
22
+ bundle: LatticeBundle;
23
+ resolveProcessPin: ResolveProcessPin;
13
24
  }): Promise<LatticeBundle>;
14
25
  export declare function summarizeBundle(bundle: LatticeBundle): LatticeBundleSummary;
@@ -20,9 +20,22 @@ export async function buildBundleFromProcesses(defs, options) {
20
20
  return options.resolveProcessPin(processId, version);
21
21
  });
22
22
  }
23
+ const metadata = options?.provenance
24
+ ? collectGitMetadataWithOverrides(options.provenance)
25
+ : collectGitMetadata();
23
26
  // Manifest, hash, version, and co-located subprocess pins are all derived in
24
27
  // the SDK so a CLI-built bundle is byte-identical to a server-applied one.
25
- return finalizeBundleIdentity({ processes, metadata: collectGitMetadata() });
28
+ return finalizeBundleIdentity({ processes, metadata });
29
+ }
30
+ export async function finalizeCompiledBundle({ bundle, resolveProcessPin, }) {
31
+ const processes = bundle.processes.map((process) => structuredClone(process));
32
+ const localProcessIds = new Set(processes.map((process) => process.id));
33
+ await resolveSubprocessPinsInProcesses(processes, async (processId, version) => {
34
+ if (!version && localProcessIds.has(processId))
35
+ return null;
36
+ return resolveProcessPin(processId, version);
37
+ });
38
+ return finalizeBundleIdentity({ processes, metadata: bundle.metadata });
26
39
  }
27
40
  async function resolveSubprocessPinsInProcesses(processes, resolve) {
28
41
  const pins = [];
@@ -83,8 +96,20 @@ function serializeProcess(input) {
83
96
  ...(input.process.supervisor_retry !== undefined
84
97
  ? { supervisor_retry: serializeRetryConfig(input.process.supervisor_retry) }
85
98
  : {}),
99
+ // Both run_as keys are omitted when unset so existing hashes stay stable
100
+ // (an email-only run_as serializes byte-identically to before the
101
+ // service_account field existed).
86
102
  ...(input.process.run_as !== undefined
87
- ? { run_as: { email: input.process.run_as.email } }
103
+ ? {
104
+ run_as: {
105
+ ...(input.process.run_as.email !== undefined
106
+ ? { email: input.process.run_as.email }
107
+ : {}),
108
+ ...(input.process.run_as.service_account !== undefined
109
+ ? { service_account: input.process.run_as.service_account }
110
+ : {}),
111
+ },
112
+ }
88
113
  : {}),
89
114
  };
90
115
  }
@@ -132,6 +157,10 @@ function serializeNodeMetadata(node) {
132
157
  meta.limits = native.limits;
133
158
  if (native.bindings)
134
159
  meta.bindings = native.bindings;
160
+ // Per-node service-account override — omitted when unset for hash stability.
161
+ if (native.run_as?.service_account) {
162
+ meta.run_as = { service_account: native.run_as.service_account };
163
+ }
135
164
  const nativeInputMapper = native.input;
136
165
  if (typeof nativeInputMapper === 'function') {
137
166
  meta.input_mapper_source = nativeInputMapper.toString();
@@ -266,6 +295,11 @@ function serializeNodeMetadata(node) {
266
295
  on_branch_error: parallel.on_branch_error,
267
296
  join_source: toSource(parallel.join),
268
297
  };
298
+ // Serialized only when non-default ('eager') so existing barrier
299
+ // bundles keep their hashes; a missing field reads as 'barrier'.
300
+ if (parallel.join_mode === 'eager') {
301
+ pmeta.join_mode = 'eager';
302
+ }
269
303
  if (parallel.max_concurrency !== undefined) {
270
304
  pmeta.max_concurrency = parallel.max_concurrency;
271
305
  }
@@ -319,6 +353,10 @@ function serializeNodeMetadata(node) {
319
353
  meta.limits = mf.limits;
320
354
  if (mf.bindings)
321
355
  meta.bindings = mf.bindings;
356
+ // Per-node service-account override — omitted when unset for hash stability.
357
+ if (mf.run_as?.service_account) {
358
+ meta.run_as = { service_account: mf.run_as.service_account };
359
+ }
322
360
  // NOTE: `middleware` (e.g. withRetry) is serialized generically for every
323
361
  // node kind in `serializeNode` above — don't duplicate it here.
324
362
  const inputMapper = mf.input;
@@ -348,6 +386,15 @@ function collectGitMetadata() {
348
386
  git_dirty: get('git status --porcelain') !== '',
349
387
  };
350
388
  }
389
+ function collectGitMetadataWithOverrides(provenance) {
390
+ return {
391
+ created_at: new Date().toISOString(),
392
+ created_by: process.env['USER'] ?? null,
393
+ git_commit: provenance.gitCommit,
394
+ git_branch: provenance.gitBranch,
395
+ git_dirty: provenance.gitDirty ?? false,
396
+ };
397
+ }
351
398
  export function summarizeBundle(bundle) {
352
399
  return {
353
400
  name: bundle.manifest.bundle.name,