@sequenceholdings/pipeline-spec 0.2.6 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +175 -0
- package/dist/bodies/serving.d.ts +29 -0
- package/dist/bodies/serving.d.ts.map +1 -1
- package/dist/bodies/serving.js +4 -0
- package/dist/bodies/serving.js.map +1 -1
- package/dist/diff.d.ts +1 -1
- package/dist/diff.d.ts.map +1 -1
- package/dist/diff.js +16 -0
- package/dist/diff.js.map +1 -1
- package/dist/envelope.d.ts +52 -0
- package/dist/envelope.d.ts.map +1 -1
- package/dist/graph.d.ts.map +1 -1
- package/dist/graph.js +3 -2
- package/dist/graph.js.map +1 -1
- package/dist/hash.d.ts.map +1 -1
- package/dist/hash.js +1 -0
- package/dist/hash.js.map +1 -1
- package/dist/index.d.ts +4 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -1
- package/dist/index.js.map +1 -1
- package/dist/loader.d.ts +3 -0
- package/dist/loader.d.ts.map +1 -1
- package/dist/loader.js +19 -2
- package/dist/loader.js.map +1 -1
- package/dist/orm-aliases.d.ts +33 -0
- package/dist/orm-aliases.d.ts.map +1 -0
- package/dist/orm-aliases.js +64 -0
- package/dist/orm-aliases.js.map +1 -0
- package/dist/resolve.d.ts.map +1 -1
- package/dist/resolve.js +7 -1
- package/dist/resolve.js.map +1 -1
- package/dist/source-package.d.ts +27 -0
- package/dist/source-package.d.ts.map +1 -0
- package/dist/source-package.js +140 -0
- package/dist/source-package.js.map +1 -0
- package/dist/workflow.d.ts +95 -0
- package/dist/workflow.d.ts.map +1 -0
- package/dist/workflow.js +115 -0
- package/dist/workflow.js.map +1 -0
- package/package.json +2 -2
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
import { dirname } from 'node:path/posix';
|
|
2
|
+
import { SpecDefinitionError } from './errors.js';
|
|
3
|
+
import { PIPELINE_CONFIG_FILE, STAGE_FILE_SUFFIXES } from './constants.js';
|
|
4
|
+
const metadataFiles = new Set([PIPELINE_CONFIG_FILE, 'pipeline.yml', 'external-assets.json', 'orm-contracts.json']);
|
|
5
|
+
const metadataDirectories = ['schemas', 'documentation', 'migrations', 'extra-resources'];
|
|
6
|
+
/** Selection only: transport separately verifies regular files, object hashes and bounded sizes. */
|
|
7
|
+
export function selectPipelineSourcePaths({ config, paths }) {
|
|
8
|
+
if (!config.source_paths)
|
|
9
|
+
return undefined;
|
|
10
|
+
const selected = new Set();
|
|
11
|
+
for (const requested of config.source_paths) {
|
|
12
|
+
const matches = paths.filter(path => path === requested || path.startsWith(`${requested}/`));
|
|
13
|
+
if (!matches.length)
|
|
14
|
+
throw new SpecDefinitionError(`source path '${requested}' is missing or empty`, PIPELINE_CONFIG_FILE, 'source_paths');
|
|
15
|
+
for (const path of matches)
|
|
16
|
+
selected.add(path);
|
|
17
|
+
}
|
|
18
|
+
for (const path of paths) {
|
|
19
|
+
// Omitting metadata must never silently remove stages, contracts, workflow edges, or extra jobs.
|
|
20
|
+
if (metadataFiles.has(path) || metadataDirectories.some(dir => path === dir || path.startsWith(`${dir}/`)) ||
|
|
21
|
+
(config.stage_directories.includes(dirname(path)) && STAGE_FILE_SUFFIXES.some(suffix => path.endsWith(suffix)))) {
|
|
22
|
+
selected.add(path);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
return selected;
|
|
26
|
+
}
|
|
27
|
+
/** Dynamic imports/data dependencies belong in the reviewed directory selection, not guessed imports. */
|
|
28
|
+
export function assertPipelineSourceDependencies({ stages, selectedPaths }) {
|
|
29
|
+
for (const stage of stages) {
|
|
30
|
+
const required = [stage.entrypoint, ...Object.values(stage.feeds ?? {}).map(feed => feed.decoder),
|
|
31
|
+
...Object.values(stage.operations ?? {}).map(operation => operation.entrypoint)];
|
|
32
|
+
for (const path of required) {
|
|
33
|
+
if (path !== undefined && !selectedPaths.has(path)) {
|
|
34
|
+
throw new SpecDefinitionError(`stage '${stage.stage}' requires '${path}'; include its runtime directory in source_paths`, PIPELINE_CONFIG_FILE, 'source_paths');
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
/** Validate the same selection locally and after snapshot materialization. Never follow symlinks. */
|
|
40
|
+
export async function validatePipelineSourcePackage({ dir, config, stages }) {
|
|
41
|
+
if (!config.source_paths)
|
|
42
|
+
return;
|
|
43
|
+
const { readdir, readFile, lstat } = await import('node:fs/promises');
|
|
44
|
+
const { join, relative, sep, resolve } = await import('node:path');
|
|
45
|
+
const { load } = await import('js-yaml');
|
|
46
|
+
const inventory = new Map();
|
|
47
|
+
const visit = async (path) => {
|
|
48
|
+
if (inventory.has(path))
|
|
49
|
+
return;
|
|
50
|
+
let info;
|
|
51
|
+
try {
|
|
52
|
+
info = await lstat(join(dir, path));
|
|
53
|
+
}
|
|
54
|
+
catch (error) {
|
|
55
|
+
if (error instanceof Error && 'code' in error && error.code === 'ENOENT')
|
|
56
|
+
return;
|
|
57
|
+
throw error;
|
|
58
|
+
}
|
|
59
|
+
inventory.set(path, info);
|
|
60
|
+
if (info.isDirectory())
|
|
61
|
+
for (const name of await readdir(join(dir, path)))
|
|
62
|
+
await visit(`${path}/${name}`);
|
|
63
|
+
};
|
|
64
|
+
for (const path of [...config.source_paths, ...metadataFiles, ...metadataDirectories]) {
|
|
65
|
+
// A literal file selection must not tunnel through a symlinked ancestor.
|
|
66
|
+
const parts = path.split('/');
|
|
67
|
+
for (let end = 1; end < parts.length; end++) {
|
|
68
|
+
try {
|
|
69
|
+
if ((await lstat(join(dir, ...parts.slice(0, end)))).isSymbolicLink()) {
|
|
70
|
+
throw new SpecDefinitionError(`source path '${path}' has a symlinked ancestor`, PIPELINE_CONFIG_FILE);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
catch (error) {
|
|
74
|
+
if (!(error instanceof Error && 'code' in error && error.code === 'ENOENT'))
|
|
75
|
+
throw error;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
await visit(path);
|
|
79
|
+
}
|
|
80
|
+
for (const directory of config.stage_directories) {
|
|
81
|
+
for (const name of await readdir(join(dir, directory))) {
|
|
82
|
+
if (STAGE_FILE_SUFFIXES.some(suffix => name.endsWith(suffix)))
|
|
83
|
+
await visit(directory === '.' ? name : `${directory}/${name}`);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
const paths = [...inventory].filter(([, info]) => !info.isDirectory()).map(([path]) => path);
|
|
87
|
+
const selectedPaths = selectPipelineSourcePaths({ config, paths });
|
|
88
|
+
for (const [path, info] of inventory) {
|
|
89
|
+
if (selectedPaths.has(path) && !info.isFile())
|
|
90
|
+
throw new SpecDefinitionError(`source path '${path}' must be a regular file`, PIPELINE_CONFIG_FILE);
|
|
91
|
+
}
|
|
92
|
+
assertPipelineSourceDependencies({ stages, selectedPaths });
|
|
93
|
+
const localFileKeys = new Set(['notebook_path', 'python_file', 'jar', 'whl', 'requirements']);
|
|
94
|
+
const assertLocalFile = (reference, file) => {
|
|
95
|
+
const child = reference.replace(/^file:/, '');
|
|
96
|
+
if (child.startsWith('/') || /^[a-z][a-z0-9+.-]*:/i.test(child))
|
|
97
|
+
return;
|
|
98
|
+
// Extra resource files are relocated to resources/ by the bundle compiler.
|
|
99
|
+
const local = child.replace(/^\$\{workspace\.file_path\}\//, '../').replace(/^\$\{workspace\.root_path\}\/files\//, '../');
|
|
100
|
+
if (local.includes('${'))
|
|
101
|
+
throw new SpecDefinitionError(`extra resource '${file}' has an unresolved local file reference '${child}'; use a literal path for a selected source package`, PIPELINE_CONFIG_FILE, 'source_paths');
|
|
102
|
+
const path = relative(resolve(dir), resolve(dir, 'resources', local)).split(sep).join('/');
|
|
103
|
+
if (!selectedPaths.has(path))
|
|
104
|
+
throw new SpecDefinitionError(`extra resource '${file}' requires local source '${child}'; include it in source_paths`, PIPELINE_CONFIG_FILE, 'source_paths');
|
|
105
|
+
};
|
|
106
|
+
const visiting = new Set();
|
|
107
|
+
const inspect = (value, file, ancestry = []) => {
|
|
108
|
+
if (!value || typeof value !== 'object')
|
|
109
|
+
return;
|
|
110
|
+
if (visiting.has(value))
|
|
111
|
+
throw new SpecDefinitionError(`cyclic extra resource '${file}'`, PIPELINE_CONFIG_FILE);
|
|
112
|
+
visiting.add(value);
|
|
113
|
+
if (Array.isArray(value)) {
|
|
114
|
+
for (const child of value) {
|
|
115
|
+
if (ancestry.at(-1) === 'dependencies' && ancestry.at(-2) === 'spec' && ancestry.includes('environments') && typeof child === 'string') {
|
|
116
|
+
const requirementFile = /^(?:-r\s*|--requirement(?:\s+|=))/.test(child);
|
|
117
|
+
const dependency = child.replace(/^.*?\s+@\s+/, '').replace(/^(?:-r\s*|--requirement(?:\s+|=))/, '');
|
|
118
|
+
// Named PyPI requirements have no local file; preserve them and remote URIs.
|
|
119
|
+
if (requirementFile || dependency.includes('/') || dependency.includes('${') || /\.(whl|zip|tar\.gz)$/.test(dependency))
|
|
120
|
+
assertLocalFile(dependency, file);
|
|
121
|
+
}
|
|
122
|
+
inspect(child, file, ancestry);
|
|
123
|
+
}
|
|
124
|
+
visiting.delete(value);
|
|
125
|
+
return;
|
|
126
|
+
}
|
|
127
|
+
for (const [key, child] of Object.entries(value)) {
|
|
128
|
+
const remoteGitTask = 'source' in value && value.source === 'GIT' && ['notebook_path', 'python_file'].includes(key);
|
|
129
|
+
if (!remoteGitTask && (localFileKeys.has(key) || (key === 'path' && ['file', 'notebook'].includes(ancestry.at(-1) ?? ''))) && typeof child === 'string')
|
|
130
|
+
assertLocalFile(child, file);
|
|
131
|
+
inspect(child, file, [...ancestry, key]);
|
|
132
|
+
}
|
|
133
|
+
visiting.delete(value);
|
|
134
|
+
};
|
|
135
|
+
for (const path of selectedPaths) {
|
|
136
|
+
if (path.startsWith('extra-resources/') && /\.ya?ml$/.test(path))
|
|
137
|
+
inspect(load(await readFile(join(dir, path), 'utf8')), path);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
//# sourceMappingURL=source-package.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"source-package.js","sourceRoot":"","sources":["../src/source-package.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAA;AAEzC,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAA;AACjD,OAAO,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAA;AAE1E,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,CAAC,oBAAoB,EAAE,cAAc,EAAE,sBAAsB,EAAE,oBAAoB,CAAC,CAAC,CAAA;AACnH,MAAM,mBAAmB,GAAG,CAAC,SAAS,EAAE,eAAe,EAAE,YAAY,EAAE,iBAAiB,CAAC,CAAA;AAEzF,oGAAoG;AACpG,MAAM,UAAU,yBAAyB,CAAC,EAAE,MAAM,EAAE,KAAK,EAGxD;IACC,IAAI,CAAC,MAAM,CAAC,YAAY;QAAE,OAAO,SAAS,CAAA;IAC1C,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAA;IAClC,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;QAC5C,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,SAAS,GAAG,CAAC,CAAC,CAAA;QAC5F,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,MAAM,IAAI,mBAAmB,CAAC,gBAAgB,SAAS,uBAAuB,EAAE,oBAAoB,EAAE,cAAc,CAAC,CAAA;QAC1I,KAAK,MAAM,IAAI,IAAI,OAAO;YAAE,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAChD,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,iGAAiG;QACjG,IAAI,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,mBAAmB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,KAAK,GAAG,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC;YACxG,CAAC,MAAM,CAAC,iBAAiB,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,mBAAmB,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;YAClH,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACpB,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAA;AACjB,CAAC;AAED,yGAAyG;AACzG,MAAM,UAAU,gCAAgC,CAAC,EAAE,MAAM,EAAE,aAAa,EAQvE;IACC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,QAAQ,GAAG,CAAC,KAAK,CAAC,UAAU,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC;YAC/F,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAA;QAClF,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC5B,IAAI,IAAI,KAAK,SAAS,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnD,MAAM,IAAI,mBAAmB,CAAC,UAAU,KAAK,CAAC,KAAK,eAAe,IAAI,kDAAkD,EAAE,oBAAoB,EAAE,cAAc,CAAC,CAAA;YACjK,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAED,qGAAqG;AACrG,MAAM,CAAC,KAAK,UAAU,6BAA6B,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAIxE;IACC,IAAI,CAAC,MAAM,CAAC,YAAY;QAAE,OAAM;IAChC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAA;IACrE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,CAAA;IAClE,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAA;IACxC,MAAM,SAAS,GAAG,IAAI,GAAG,EAA6C,CAAA;IACtE,MAAM,KAAK,GAAG,KAAK,EAAE,IAAY,EAAiB,EAAE;QAClD,IAAI,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,OAAM;QAC/B,IAAI,IAAuC,CAAA;QAC3C,IAAI,CAAC;YAAC,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAA;QAAC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACzD,IAAI,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ;gBAAE,OAAM;YAChF,MAAM,KAAK,CAAA;QACb,CAAC;QACD,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;QACzB,IAAI,IAAI,CAAC,WAAW,EAAE;YAAE,KAAK,MAAM,IAAI,IAAI,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;gBAAE,MAAM,KAAK,CAAC,GAAG,IAAI,IAAI,IAAI,EAAE,CAAC,CAAA;IAC3G,CAAC,CAAA;IACD,KAAK,MAAM,IAAI,IAAI,CAAC,GAAG,MAAM,CAAC,YAAY,EAAE,GAAG,aAAa,EAAE,GAAG,mBAAmB,CAAC,EAAE,CAAC;QACtF,yEAAyE;QACzE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;QAC7B,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,KAAK,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC;YAC5C,IAAI,CAAC;gBACH,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc,EAAE,EAAE,CAAC;oBACtE,MAAM,IAAI,mBAAmB,CAAC,gBAAgB,IAAI,4BAA4B,EAAE,oBAAoB,CAAC,CAAA;gBACvG,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,CAAC,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC;oBAAE,MAAM,KAAK,CAAA;YAC1F,CAAC;QACH,CAAC;QACD,MAAM,KAAK,CAAC,IAAI,CAAC,CAAA;IACnB,CAAC;IACD,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,iBAAiB,EAAE,CAAC;QACjD,KAAK,MAAM,IAAI,IAAI,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC,EAAE,CAAC;YACvD,IAAI,mBAAmB,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;gBAAE,MAAM,KAAK,CAAC,SAAS,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,SAAS,IAAI,IAAI,EAAE,CAAC,CAAA;QAC/H,CAAC;IACH,CAAC;IACD,MAAM,KAAK,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAA;IAC5F,MAAM,aAAa,GAAG,yBAAyB,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAE,CAAA;IACnE,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,SAAS,EAAE,CAAC;QACrC,IAAI,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAAE,MAAM,IAAI,mBAAmB,CAAC,gBAAgB,IAAI,0BAA0B,EAAE,oBAAoB,CAAC,CAAA;IACpJ,CAAC;IACD,gCAAgC,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAA;IAC3D,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,CAAC,eAAe,EAAE,aAAa,EAAE,KAAK,EAAE,KAAK,EAAE,cAAc,CAAC,CAAC,CAAA;IAC7F,MAAM,eAAe,GAAG,CAAC,SAAiB,EAAE,IAAY,EAAQ,EAAE;QAChE,MAAM,KAAK,GAAG,SAAS,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAA;QAC7C,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,sBAAsB,CAAC,IAAI,CAAC,KAAK,CAAC;YAAE,OAAM;QACvE,2EAA2E;QAC3E,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,+BAA+B,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,sCAAsC,EAAE,KAAK,CAAC,CAAA;QAC1H,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,mBAAmB,CAAC,mBAAmB,IAAI,6CAA6C,KAAK,qDAAqD,EAAE,oBAAoB,EAAE,cAAc,CAAC,CAAA;QAC7N,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC,GAAG,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAC1F,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,mBAAmB,CAAC,mBAAmB,IAAI,4BAA4B,KAAK,+BAA+B,EAAE,oBAAoB,EAAE,cAAc,CAAC,CAAA;IAC5L,CAAC,CAAA;IACD,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAA;IAClC,MAAM,OAAO,GAAG,CAAC,KAAc,EAAE,IAAY,EAAE,WAA8B,EAAE,EAAQ,EAAE;QACvF,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;YAAE,OAAM;QAC/C,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC;YAAE,MAAM,IAAI,mBAAmB,CAAC,0BAA0B,IAAI,GAAG,EAAE,oBAAoB,CAAC,CAAA;QAC/G,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QACnB,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACzB,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;gBAC1B,IAAI,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,cAAc,IAAI,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,MAAM,IAAI,QAAQ,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;oBACvI,MAAM,eAAe,GAAG,mCAAmC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;oBACvE,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,mCAAmC,EAAE,EAAE,CAAC,CAAA;oBACpG,6EAA6E;oBAC7E,IAAI,eAAe,IAAI,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,sBAAsB,CAAC,IAAI,CAAC,UAAU,CAAC;wBAAE,eAAe,CAAC,UAAU,EAAE,IAAI,CAAC,CAAA;gBAC5J,CAAC;gBACD,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAA;YAChC,CAAC;YACD,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACtB,OAAM;QACR,CAAC;QACD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACjD,MAAM,aAAa,GAAG,QAAQ,IAAI,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,KAAK,IAAI,CAAC,eAAe,EAAE,aAAa,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAA;YACnH,IAAI,CAAC,aAAa,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,MAAM,IAAI,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,OAAO,KAAK,KAAK,QAAQ;gBAAE,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;YACrL,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,GAAG,QAAQ,EAAE,GAAG,CAAC,CAAC,CAAA;QAC1C,CAAC;QACD,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;IACxB,CAAC,CAAA;IACD,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;QACjC,IAAI,IAAI,CAAC,UAAU,CAAC,kBAAkB,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,CAAA;IAChI,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import type { StageSpec } from './envelope.js';
|
|
3
|
+
/** A native Databricks DAG. Stage inputs remain lineage; these are execution edges. */
|
|
4
|
+
export declare const pipelineWorkflowSchema: z.ZodPipe<z.ZodObject<{
|
|
5
|
+
schema_version: z.ZodLiteral<1>;
|
|
6
|
+
resource_key: z.ZodDefault<z.ZodString>;
|
|
7
|
+
tasks: z.ZodArray<z.ZodObject<{
|
|
8
|
+
stage: z.ZodString;
|
|
9
|
+
depends_on: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
10
|
+
parameters: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
11
|
+
}, z.core.$strict>>;
|
|
12
|
+
trigger: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
13
|
+
kind: z.ZodLiteral<"manual">;
|
|
14
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
15
|
+
kind: z.ZodLiteral<"completion">;
|
|
16
|
+
source_repo: z.ZodString;
|
|
17
|
+
source_stage: z.ZodString;
|
|
18
|
+
source_commit: z.ZodString;
|
|
19
|
+
source_system: z.ZodString;
|
|
20
|
+
operation: z.ZodString;
|
|
21
|
+
directory: z.ZodString;
|
|
22
|
+
receipts: z.ZodString;
|
|
23
|
+
after: z.ZodString;
|
|
24
|
+
enabled: z.ZodDefault<z.ZodBoolean>;
|
|
25
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
26
|
+
cron: z.ZodString;
|
|
27
|
+
tz: z.ZodString;
|
|
28
|
+
kind: z.ZodLiteral<"schedule">;
|
|
29
|
+
enabled: z.ZodDefault<z.ZodBoolean>;
|
|
30
|
+
}, z.core.$strict>], "kind">;
|
|
31
|
+
}, z.core.$strict>, z.ZodTransform<{
|
|
32
|
+
tasks: {
|
|
33
|
+
depends_on: string[];
|
|
34
|
+
parameters: {
|
|
35
|
+
[k: string]: string;
|
|
36
|
+
};
|
|
37
|
+
stage: string;
|
|
38
|
+
}[];
|
|
39
|
+
schema_version: 1;
|
|
40
|
+
resource_key: string;
|
|
41
|
+
trigger: {
|
|
42
|
+
kind: "manual";
|
|
43
|
+
} | {
|
|
44
|
+
kind: "completion";
|
|
45
|
+
source_repo: string;
|
|
46
|
+
source_stage: string;
|
|
47
|
+
source_commit: string;
|
|
48
|
+
source_system: string;
|
|
49
|
+
operation: string;
|
|
50
|
+
directory: string;
|
|
51
|
+
receipts: string;
|
|
52
|
+
after: string;
|
|
53
|
+
enabled: boolean;
|
|
54
|
+
} | {
|
|
55
|
+
cron: string;
|
|
56
|
+
tz: string;
|
|
57
|
+
kind: "schedule";
|
|
58
|
+
enabled: boolean;
|
|
59
|
+
};
|
|
60
|
+
}, {
|
|
61
|
+
schema_version: 1;
|
|
62
|
+
resource_key: string;
|
|
63
|
+
tasks: {
|
|
64
|
+
stage: string;
|
|
65
|
+
depends_on: string[];
|
|
66
|
+
parameters: Record<string, string>;
|
|
67
|
+
}[];
|
|
68
|
+
trigger: {
|
|
69
|
+
kind: "manual";
|
|
70
|
+
} | {
|
|
71
|
+
kind: "completion";
|
|
72
|
+
source_repo: string;
|
|
73
|
+
source_stage: string;
|
|
74
|
+
source_commit: string;
|
|
75
|
+
source_system: string;
|
|
76
|
+
operation: string;
|
|
77
|
+
directory: string;
|
|
78
|
+
receipts: string;
|
|
79
|
+
after: string;
|
|
80
|
+
enabled: boolean;
|
|
81
|
+
} | {
|
|
82
|
+
cron: string;
|
|
83
|
+
tz: string;
|
|
84
|
+
kind: "schedule";
|
|
85
|
+
enabled: boolean;
|
|
86
|
+
};
|
|
87
|
+
}>>;
|
|
88
|
+
export type PipelineWorkflow = z.output<typeof pipelineWorkflowSchema>;
|
|
89
|
+
/** Execution edges are explicit and cannot compete with stage-owned triggers. */
|
|
90
|
+
export declare function validateWorkflowStages(workflow: PipelineWorkflow, stages: readonly StageSpec[]): void;
|
|
91
|
+
/** Missing pipeline.yml preserves existing stage-only release units. */
|
|
92
|
+
export declare function loadPipelineWorkflow({ directory }: {
|
|
93
|
+
directory: string;
|
|
94
|
+
}): Promise<PipelineWorkflow | undefined>;
|
|
95
|
+
//# sourceMappingURL=workflow.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"workflow.d.ts","sourceRoot":"","sources":["../src/workflow.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAGvB,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,eAAe,CAAA;AAkB9C,uFAAuF;AACvF,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0ChC,CAAA;AAEH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,sBAAsB,CAAC,CAAA;AAEtE,iFAAiF;AACjF,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,gBAAgB,EAAE,MAAM,EAAE,SAAS,SAAS,EAAE,GAAG,IAAI,CAiBrG;AAED,wEAAwE;AACxE,wBAAsB,oBAAoB,CAAC,EAAE,SAAS,EAAE,EAAE;IAAE,SAAS,EAAE,MAAM,CAAA;CAAE,GAAG,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAAC,CAUtH"}
|
package/dist/workflow.js
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { readFile } from 'node:fs/promises';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { load } from 'js-yaml';
|
|
4
|
+
import { z } from 'zod';
|
|
5
|
+
import { scheduleSchema } from './bodies/ingestion.js';
|
|
6
|
+
import { SpecDefinitionError } from './errors.js';
|
|
7
|
+
const stage = z.string().regex(/^[a-z][a-z0-9-]{0,62}$/);
|
|
8
|
+
const volumeDirectory = z.string().regex(/^\/Volumes\/[A-Za-z0-9_]+\/[A-Za-z0-9_]+\/[A-Za-z0-9_]+(?:\/[A-Za-z0-9_-]+)*$/);
|
|
9
|
+
const completion = z.strictObject({
|
|
10
|
+
kind: z.literal('completion'),
|
|
11
|
+
source_repo: z.string().regex(/^pipelines\/[a-z0-9][a-z0-9-]*$/),
|
|
12
|
+
source_stage: stage,
|
|
13
|
+
source_commit: z.string().regex(/^[0-9a-f]{40}$/),
|
|
14
|
+
source_system: z.string().regex(/^[a-z][a-z0-9_]*$/),
|
|
15
|
+
operation: z.string().regex(/^[a-z][a-z0-9_]*$/),
|
|
16
|
+
directory: volumeDirectory,
|
|
17
|
+
receipts: volumeDirectory,
|
|
18
|
+
// An explicit inclusive lower boundary avoids replaying a historical archive on deploy.
|
|
19
|
+
after: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9_-]{0,95}$/),
|
|
20
|
+
enabled: z.boolean().default(false),
|
|
21
|
+
});
|
|
22
|
+
/** A native Databricks DAG. Stage inputs remain lineage; these are execution edges. */
|
|
23
|
+
export const pipelineWorkflowSchema = z.strictObject({
|
|
24
|
+
schema_version: z.literal(1),
|
|
25
|
+
resource_key: z.string().regex(/^[a-z][a-z0-9_]{0,80}$/).default('mp_workflow'),
|
|
26
|
+
tasks: z.array(z.strictObject({
|
|
27
|
+
stage,
|
|
28
|
+
depends_on: z.array(stage).default([]),
|
|
29
|
+
parameters: z.record(z.string().regex(/^[a-z][a-z0-9_]*$/), z.string().max(2048)).default({}),
|
|
30
|
+
})).min(1).max(100),
|
|
31
|
+
trigger: z.discriminatedUnion('kind', [
|
|
32
|
+
z.strictObject({ kind: z.literal('manual') }), completion,
|
|
33
|
+
scheduleSchema.omit({ policy: true }).extend({ kind: z.literal('schedule'), enabled: z.boolean().default(false) }),
|
|
34
|
+
]),
|
|
35
|
+
}).superRefine((workflow, ctx) => {
|
|
36
|
+
const tasks = new Map(workflow.tasks.map(task => [task.stage, task]));
|
|
37
|
+
const issue = (message) => ctx.addIssue({ code: 'custom', message });
|
|
38
|
+
if (tasks.size !== workflow.tasks.length)
|
|
39
|
+
issue('A stage may appear only once in a workflow');
|
|
40
|
+
for (const task of workflow.tasks) {
|
|
41
|
+
if (new Set(task.depends_on).size !== task.depends_on.length)
|
|
42
|
+
issue(`${task.stage}: duplicate dependency`);
|
|
43
|
+
for (const parent of task.depends_on)
|
|
44
|
+
if (!tasks.has(parent))
|
|
45
|
+
issue(`${task.stage}: unknown dependency ${parent}`);
|
|
46
|
+
}
|
|
47
|
+
const visiting = new Set();
|
|
48
|
+
const visited = new Set();
|
|
49
|
+
const visit = (name) => {
|
|
50
|
+
if (visiting.has(name)) {
|
|
51
|
+
issue(`Workflow contains a cycle at ${name}`);
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
54
|
+
if (visited.has(name))
|
|
55
|
+
return;
|
|
56
|
+
visiting.add(name);
|
|
57
|
+
for (const parent of tasks.get(name)?.depends_on ?? [])
|
|
58
|
+
visit(parent);
|
|
59
|
+
visiting.delete(name);
|
|
60
|
+
visited.add(name);
|
|
61
|
+
};
|
|
62
|
+
for (const name of tasks.keys())
|
|
63
|
+
visit(name);
|
|
64
|
+
const trigger = workflow.trigger;
|
|
65
|
+
if (trigger.kind === 'completion' && (trigger.receipts === trigger.directory || trigger.receipts.startsWith(`${trigger.directory}/`) ||
|
|
66
|
+
trigger.directory.startsWith(`${trigger.receipts}/`)))
|
|
67
|
+
issue('Completion and delivery receipt directories must be disjoint');
|
|
68
|
+
}).transform(workflow => ({
|
|
69
|
+
...workflow,
|
|
70
|
+
tasks: [...workflow.tasks].sort((a, b) => a.stage.localeCompare(b.stage)).map(task => ({
|
|
71
|
+
...task, depends_on: [...task.depends_on].sort(),
|
|
72
|
+
parameters: Object.fromEntries(Object.entries(task.parameters).sort(([a], [b]) => a.localeCompare(b))),
|
|
73
|
+
})),
|
|
74
|
+
}));
|
|
75
|
+
/** Execution edges are explicit and cannot compete with stage-owned triggers. */
|
|
76
|
+
export function validateWorkflowStages(workflow, stages) {
|
|
77
|
+
const selected = new Set(workflow.tasks.map(task => task.stage));
|
|
78
|
+
for (const task of workflow.tasks) {
|
|
79
|
+
const matches = stages.filter(spec => spec.stage === task.stage);
|
|
80
|
+
const spec = matches.length === 1 ? matches[0] : undefined;
|
|
81
|
+
if (!spec)
|
|
82
|
+
throw new SpecDefinitionError(`Workflow stage '${task.stage}' is not uniquely defined in this release`, 'pipeline.yml');
|
|
83
|
+
// Native job parameters automatically reach notebook widgets. Python argv
|
|
84
|
+
// tasks need a separate adapter; accepting them here would drop the batch.
|
|
85
|
+
if (spec.type !== 'transformation' || !spec.trigger.manual || spec.engine !== 'spark-job') {
|
|
86
|
+
throw new SpecDefinitionError(`Workflow stage '${task.stage}' must be a manual Spark notebook job`, 'pipeline.yml');
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
for (const spec of stages) {
|
|
90
|
+
if (spec.type === 'transformation' && spec.trigger.after?.some(parent => selected.has(parent))) {
|
|
91
|
+
throw new SpecDefinitionError(`Stage '${spec.stage}' automatically follows a workflow member; make it manual and declare its dependency in pipeline.yml`, 'pipeline.yml');
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
/** Missing pipeline.yml preserves existing stage-only release units. */
|
|
96
|
+
export async function loadPipelineWorkflow({ directory }) {
|
|
97
|
+
let content;
|
|
98
|
+
try {
|
|
99
|
+
content = await readFile(join(directory, 'pipeline.yml'), 'utf8');
|
|
100
|
+
}
|
|
101
|
+
catch (error) {
|
|
102
|
+
if (error instanceof Error && 'code' in error && error.code === 'ENOENT')
|
|
103
|
+
return undefined;
|
|
104
|
+
throw error;
|
|
105
|
+
}
|
|
106
|
+
if (Buffer.byteLength(content) > 128 * 1024)
|
|
107
|
+
throw new SpecDefinitionError('Exceeds 128 KiB', 'pipeline.yml');
|
|
108
|
+
try {
|
|
109
|
+
return pipelineWorkflowSchema.parse(load(content));
|
|
110
|
+
}
|
|
111
|
+
catch (error) {
|
|
112
|
+
throw new SpecDefinitionError(error instanceof Error ? error.message : String(error), 'pipeline.yml');
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
//# sourceMappingURL=workflow.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"workflow.js","sourceRoot":"","sources":["../src/workflow.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAC3C,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAC9B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AACvB,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAA;AACtD,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAA;AAGjD,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAA;AACxD,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,+EAA+E,CAAC,CAAA;AACzH,MAAM,UAAU,GAAG,CAAC,CAAC,YAAY,CAAC;IAChC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC;IAC7B,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,iCAAiC,CAAC;IAChE,YAAY,EAAE,KAAK;IACnB,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,gBAAgB,CAAC;IACjD,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,mBAAmB,CAAC;IACpD,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,mBAAmB,CAAC;IAChD,SAAS,EAAE,eAAe;IAC1B,QAAQ,EAAE,eAAe;IACzB,wFAAwF;IACxF,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,kCAAkC,CAAC;IAC3D,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;CACpC,CAAC,CAAA;AAEF,uFAAuF;AACvF,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAC,YAAY,CAAC;IACnD,cAAc,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC5B,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC;IAC/E,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC;QAC5B,KAAK;QACL,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;QACtC,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,mBAAmB,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;KAC9F,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC;IACnB,OAAO,EAAE,CAAC,CAAC,kBAAkB,CAAC,MAAM,EAAE;QACpC,CAAC,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,UAAU;QACzD,cAAc,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;KACnH,CAAC;CACH,CAAC,CAAC,WAAW,CAAC,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE;IAC/B,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAA;IACrE,MAAM,KAAK,GAAG,CAAC,OAAe,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAA;IAC5E,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,KAAK,CAAC,MAAM;QAAE,KAAK,CAAC,4CAA4C,CAAC,CAAA;IAC7F,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QAClC,IAAI,IAAI,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,UAAU,CAAC,MAAM;YAAE,KAAK,CAAC,GAAG,IAAI,CAAC,KAAK,wBAAwB,CAAC,CAAA;QAC1G,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,UAAU;YAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC;gBAAE,KAAK,CAAC,GAAG,IAAI,CAAC,KAAK,wBAAwB,MAAM,EAAE,CAAC,CAAA;IACpH,CAAC;IACD,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAA;IAClC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAA;IACjC,MAAM,KAAK,GAAG,CAAC,IAAY,EAAQ,EAAE;QACnC,IAAI,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAAC,KAAK,CAAC,gCAAgC,IAAI,EAAE,CAAC,CAAC;YAAC,OAAM;QAAC,CAAC;QACjF,IAAI,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,OAAM;QAC7B,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAClB,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,UAAU,IAAI,EAAE;YAAE,KAAK,CAAC,MAAM,CAAC,CAAA;QACrE,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;QACrB,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IACnB,CAAC,CAAA;IACD,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,IAAI,EAAE;QAAE,KAAK,CAAC,IAAI,CAAC,CAAA;IAC5C,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAA;IAChC,IAAI,OAAO,CAAC,IAAI,KAAK,YAAY,IAAI,CACnC,OAAO,CAAC,QAAQ,KAAK,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,OAAO,CAAC,SAAS,GAAG,CAAC;QAC9F,OAAO,CAAC,SAAS,CAAC,UAAU,CAAC,GAAG,OAAO,CAAC,QAAQ,GAAG,CAAC,CACrD;QAAE,KAAK,CAAC,8DAA8D,CAAC,CAAA;AAC1E,CAAC,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IACxB,GAAG,QAAQ;IACX,KAAK,EAAE,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACrF,GAAG,IAAI,EAAE,UAAU,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE;QAChD,UAAU,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;KACvG,CAAC,CAAC;CACJ,CAAC,CAAC,CAAA;AAIH,iFAAiF;AACjF,MAAM,UAAU,sBAAsB,CAAC,QAA0B,EAAE,MAA4B;IAC7F,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAA;IAChE,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QAClC,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,CAAC,CAAA;QAChE,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;QAC1D,IAAI,CAAC,IAAI;YAAE,MAAM,IAAI,mBAAmB,CAAC,mBAAmB,IAAI,CAAC,KAAK,2CAA2C,EAAE,cAAc,CAAC,CAAA;QAClI,0EAA0E;QAC1E,2EAA2E;QAC3E,IAAI,IAAI,CAAC,IAAI,KAAK,gBAAgB,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;YAC1F,MAAM,IAAI,mBAAmB,CAAC,mBAAmB,IAAI,CAAC,KAAK,uCAAuC,EAAE,cAAc,CAAC,CAAA;QACrH,CAAC;IACH,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,IAAI,IAAI,CAAC,IAAI,KAAK,gBAAgB,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;YAC/F,MAAM,IAAI,mBAAmB,CAAC,UAAU,IAAI,CAAC,KAAK,sGAAsG,EAAE,cAAc,CAAC,CAAA;QAC3K,CAAC;IACH,CAAC;AACH,CAAC;AAED,wEAAwE;AACxE,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,EAAE,SAAS,EAAyB;IAC7E,IAAI,OAAe,CAAA;IACnB,IAAI,CAAC;QAAC,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,EAAE,MAAM,CAAC,CAAA;IAAC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACvF,IAAI,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,SAAS,CAAA;QAC1F,MAAM,KAAK,CAAA;IACb,CAAC;IACD,IAAI,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,GAAG,GAAG,IAAI;QAAE,MAAM,IAAI,mBAAmB,CAAC,iBAAiB,EAAE,cAAc,CAAC,CAAA;IAC7G,IAAI,CAAC;QAAC,OAAO,sBAAsB,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAA;IAAC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACxE,MAAM,IAAI,mBAAmB,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,cAAc,CAAC,CAAA;IACvG,CAAC;AACH,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sequenceholdings/pipeline-spec",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.1",
|
|
4
4
|
"description": "Data Pipelines spec SDK — typed stage specs (ingestion/transformation/serving), directory loader, target resolution with deterministic hashing, and offline graph validation",
|
|
5
5
|
"license": "UNLICENSED",
|
|
6
6
|
"publishConfig": {
|
|
@@ -27,7 +27,7 @@
|
|
|
27
27
|
"dependencies": {
|
|
28
28
|
"js-yaml": "^4.3.2",
|
|
29
29
|
"zod": "^4.1.13",
|
|
30
|
-
"@sequenceholdings/orm": "2.
|
|
30
|
+
"@sequenceholdings/orm": "2.8.1"
|
|
31
31
|
},
|
|
32
32
|
"devDependencies": {
|
|
33
33
|
"@types/js-yaml": "^4.0.9",
|