@zhuoyuezs/ml-platform 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CI-ARTIFACT.md +21 -0
- package/DEVELOPMENT.md +7 -5
- package/checksums.json +38 -33
- package/package.json +6 -3
- package/release.json +7 -7
- package/runtime/business-client/README.md +17 -0
- package/runtime/business-client/package-lock.json +2 -2
- package/runtime/business-client/package.json +4 -2
- package/runtime/business-client/src/catalog.js +3 -1
- package/runtime/business-client/src/cli.js +132 -24
- package/runtime/business-client/src/config.js +6 -2
- package/runtime/business-client/src/http.js +130 -13
- package/skills/feature-management/SKILL.md +178 -10
- package/skills/feature-management/assets/catalog-template/datasets/example_temperature_training.v1.json +16 -0
- package/skills/feature-management/assets/catalog-template/feature_sets/example_temperature_core.v1.json +1 -0
- package/skills/feature-management/assets/catalog-template/features/example_temperature_mean_5m.v1.json +9 -2
- package/skills/feature-management/assets/catalog-template/operator_package/tests/test_operator.py +51 -24
- package/skills/feature-management/assets/catalog-template/operators/example_temperature_features.v1.json +16 -1
- package/skills/feature-management/assets/catalog-template/parameters/example_temperature.v1.json +1 -0
- package/skills/feature-management/references/commands.md +29 -0
- package/skills/feature-management/references/contracts.md +43 -6
- package/skills/feature-management/references/operator-authoring.md +15 -7
- package/skills/feature-management/references/platform-capability-guide.md +73 -0
package/CI-ARTIFACT.md
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# CI npm artifact
|
|
2
|
+
|
|
3
|
+
每次 `dev`/`main` 分支更新、合并请求或从 GitLab UI 手动运行的 Pipeline 都会生成完整的
|
|
4
|
+
`@zhuoyuezs/ml-platform` tarball。
|
|
5
|
+
GitLab pipeline 的 `npm:ml-platform-release` job 会将 `business-client-js` 放入正式包的
|
|
6
|
+
`runtime/business-client`,并上传已验收的 `.tgz` 与 `.sha256` 文件。
|
|
7
|
+
|
|
8
|
+
创建与 `npm/ml-platform/package.json` 版本一致的 SemVer tag(例如 `v0.1.5`)后,
|
|
9
|
+
同一个已验收归档会自动发布到 npmjs。发布 job 需要在 GitLab CI/CD Variables 中配置
|
|
10
|
+
受保护且 masked 的 `NPM_TOKEN`;普通分支、手动 Pipeline 和 Merge Request 不会发布 npm。
|
|
11
|
+
|
|
12
|
+
下载 artifact 后只需要 Node.js 18+:
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
sha256sum -c zhuoyuezs-ml-platform-*.tgz.sha256
|
|
16
|
+
npm install --global ./zhuoyuezs-ml-platform-*.tgz
|
|
17
|
+
ml-platform version
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
运行已安装的 CLI 不需要 Python、`uv`、Git 仓库或平台源码。`uv` 只在 CI 构建阶段用于
|
|
21
|
+
执行现有的 release staging 脚本。
|
package/DEVELOPMENT.md
CHANGED
|
@@ -146,8 +146,9 @@ Host 副本、日志、会话数据、attic、凭证和实验文件。
|
|
|
146
146
|
- Codex project/user scope 和显式 Skills 目录;
|
|
147
147
|
- 最终归档敏感文件和重复 client 检查。
|
|
148
148
|
|
|
149
|
-
完成标准:同一个不可变 `.tgz`
|
|
150
|
-
|
|
149
|
+
完成标准:同一个不可变 `.tgz` 通过全部验收,并作为唯一允许发布的输入。GitLab CI 仅在
|
|
150
|
+
SemVer tag(`vX.Y.Z` 或 `X.Y.Z`)与 `package.json.version` 一致、且配置了 `NPM_TOKEN` 时
|
|
151
|
+
自动执行 npmjs 发布;dev、main 和 Merge Request pipeline 只生成并保存 artifact。
|
|
151
152
|
|
|
152
153
|
## 4. package.json
|
|
153
154
|
|
|
@@ -260,9 +261,10 @@ npx @zhuoyuezs/ml-platform@<version> install
|
|
|
260
261
|
- 不执行 GitHub Release 或其他未授权的外部发布;
|
|
261
262
|
- candidate 验证后不得用相同版本重新构建不同内容。
|
|
262
263
|
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
264
|
+
受控 GitLab CI 使用 masked/protected 的 `NPM_TOKEN` 临时配置 npm auth,并发布 builder 生成的
|
|
265
|
+
同一个 `.tgz`;token 不写入仓库或归档。本地登录不能替代 CI provenance,当前 token 发布也不
|
|
266
|
+
宣称包具有 provenance 来源证明。后续可迁移到 npm Trusted Publishing 和 OIDC,并在受支持的
|
|
267
|
+
公开 CI 环境执行 `npm publish --provenance`。
|
|
266
268
|
|
|
267
269
|
## 9. 参考与取舍
|
|
268
270
|
|
package/checksums.json
CHANGED
|
@@ -2,43 +2,43 @@
|
|
|
2
2
|
"files": [
|
|
3
3
|
{
|
|
4
4
|
"path": "runtime/business-client/README.md",
|
|
5
|
-
"sha256": "sha256:
|
|
6
|
-
"size_bytes":
|
|
5
|
+
"sha256": "sha256:72b0cbbb18a7d0f1bca5d97706588fbebcd2781833ae023bae0317be13342601",
|
|
6
|
+
"size_bytes": 1880
|
|
7
7
|
},
|
|
8
8
|
{
|
|
9
9
|
"path": "runtime/business-client/package-lock.json",
|
|
10
|
-
"sha256": "sha256:
|
|
10
|
+
"sha256": "sha256:d5e7102ec51df3192ae901908f36cf0380aba5a0421b9824877403ab034d6740",
|
|
11
11
|
"size_bytes": 381
|
|
12
12
|
},
|
|
13
13
|
{
|
|
14
14
|
"path": "runtime/business-client/package.json",
|
|
15
|
-
"sha256": "sha256:
|
|
16
|
-
"size_bytes":
|
|
15
|
+
"sha256": "sha256:b784fa27db6e40f72da11b9bca007155146174bc33edc4511c61d4ea4af93cf1",
|
|
16
|
+
"size_bytes": 508
|
|
17
17
|
},
|
|
18
18
|
{
|
|
19
19
|
"path": "runtime/business-client/src/catalog.js",
|
|
20
|
-
"sha256": "sha256:
|
|
21
|
-
"size_bytes":
|
|
20
|
+
"sha256": "sha256:afc2ba28eee2d1dcd833a2e5c4ac923421dc60302b31bf985aa638849bbf3d36",
|
|
21
|
+
"size_bytes": 54246
|
|
22
22
|
},
|
|
23
23
|
{
|
|
24
24
|
"path": "runtime/business-client/src/cli.js",
|
|
25
|
-
"sha256": "sha256:
|
|
26
|
-
"size_bytes":
|
|
25
|
+
"sha256": "sha256:f2634e346a4c5bdc42c81e316791906f809b768a77672e36d29cc51410c010ed",
|
|
26
|
+
"size_bytes": 29818
|
|
27
27
|
},
|
|
28
28
|
{
|
|
29
29
|
"path": "runtime/business-client/src/config.js",
|
|
30
|
-
"sha256": "sha256:
|
|
31
|
-
"size_bytes":
|
|
30
|
+
"sha256": "sha256:4190e09c339a3f0b2861ca7dbc96fe2aa3bdd7927a5e1c4a4c00103a53236bd8",
|
|
31
|
+
"size_bytes": 2186
|
|
32
32
|
},
|
|
33
33
|
{
|
|
34
34
|
"path": "runtime/business-client/src/http.js",
|
|
35
|
-
"sha256": "sha256:
|
|
36
|
-
"size_bytes":
|
|
35
|
+
"sha256": "sha256:e515b39f19e7b5e7d6c2388e9cf2d8e2c9fd18b585bb53bd4f8eeeb7412af81c",
|
|
36
|
+
"size_bytes": 15192
|
|
37
37
|
},
|
|
38
38
|
{
|
|
39
39
|
"path": "skills/feature-management/SKILL.md",
|
|
40
|
-
"sha256": "sha256:
|
|
41
|
-
"size_bytes":
|
|
40
|
+
"sha256": "sha256:7b58cc4ff2a7b83a29a6dbf9ad54e9e4a02efd332cd01c5d8fb83020b51ea506",
|
|
41
|
+
"size_bytes": 29676
|
|
42
42
|
},
|
|
43
43
|
{
|
|
44
44
|
"path": "skills/feature-management/agents/openai.yaml",
|
|
@@ -52,18 +52,18 @@
|
|
|
52
52
|
},
|
|
53
53
|
{
|
|
54
54
|
"path": "skills/feature-management/assets/catalog-template/datasets/example_temperature_training.v1.json",
|
|
55
|
-
"sha256": "sha256:
|
|
56
|
-
"size_bytes":
|
|
55
|
+
"sha256": "sha256:4dbe304631d6ecbf60b19d7c5a377178e0593525362f5fe509c60622b99e8a46",
|
|
56
|
+
"size_bytes": 900
|
|
57
57
|
},
|
|
58
58
|
{
|
|
59
59
|
"path": "skills/feature-management/assets/catalog-template/feature_sets/example_temperature_core.v1.json",
|
|
60
|
-
"sha256": "sha256:
|
|
61
|
-
"size_bytes":
|
|
60
|
+
"sha256": "sha256:22f37b954200864c19ed5593ab8455f562cc3dee7e05a0b01d5856a163dc0660",
|
|
61
|
+
"size_bytes": 352
|
|
62
62
|
},
|
|
63
63
|
{
|
|
64
64
|
"path": "skills/feature-management/assets/catalog-template/features/example_temperature_mean_5m.v1.json",
|
|
65
|
-
"sha256": "sha256:
|
|
66
|
-
"size_bytes":
|
|
65
|
+
"sha256": "sha256:7e58fa9c6a5ef581866d5cae598682de73e5ef3ce49fdd630432bada3a8a41a1",
|
|
66
|
+
"size_bytes": 807
|
|
67
67
|
},
|
|
68
68
|
{
|
|
69
69
|
"path": "skills/feature-management/assets/catalog-template/operator_package/pyproject.toml",
|
|
@@ -77,33 +77,38 @@
|
|
|
77
77
|
},
|
|
78
78
|
{
|
|
79
79
|
"path": "skills/feature-management/assets/catalog-template/operator_package/tests/test_operator.py",
|
|
80
|
-
"sha256": "sha256:
|
|
81
|
-
"size_bytes":
|
|
80
|
+
"sha256": "sha256:6fe073e07ed57d0fb4e782150153556b97c5b873b9eb2dad45df0d7f522d8650",
|
|
81
|
+
"size_bytes": 3678
|
|
82
82
|
},
|
|
83
83
|
{
|
|
84
84
|
"path": "skills/feature-management/assets/catalog-template/operators/example_temperature_features.v1.json",
|
|
85
|
-
"sha256": "sha256:
|
|
86
|
-
"size_bytes":
|
|
85
|
+
"sha256": "sha256:646965a05b982ae291960b23f359403c69bf8a173d6cc65d706fb2dc53ab030b",
|
|
86
|
+
"size_bytes": 1277
|
|
87
87
|
},
|
|
88
88
|
{
|
|
89
89
|
"path": "skills/feature-management/assets/catalog-template/parameters/example_temperature.v1.json",
|
|
90
|
-
"sha256": "sha256:
|
|
91
|
-
"size_bytes":
|
|
90
|
+
"sha256": "sha256:28c7d3576c4b7e94c1e9db7f95f26d83ee2e10dc9f81ec6223f4fbd9b2f5eeda",
|
|
91
|
+
"size_bytes": 1488
|
|
92
92
|
},
|
|
93
93
|
{
|
|
94
94
|
"path": "skills/feature-management/references/commands.md",
|
|
95
|
-
"sha256": "sha256:
|
|
96
|
-
"size_bytes":
|
|
95
|
+
"sha256": "sha256:c20ccf9ad8d505a1880a7e35773edb5040bbbcfeba3a64316e3148a6e337bdc2",
|
|
96
|
+
"size_bytes": 11546
|
|
97
97
|
},
|
|
98
98
|
{
|
|
99
99
|
"path": "skills/feature-management/references/contracts.md",
|
|
100
|
-
"sha256": "sha256:
|
|
101
|
-
"size_bytes":
|
|
100
|
+
"sha256": "sha256:913d5f0ad52b5183aa3a4bdeef2b6974f7757842300f547679fee38552792892",
|
|
101
|
+
"size_bytes": 27738
|
|
102
102
|
},
|
|
103
103
|
{
|
|
104
104
|
"path": "skills/feature-management/references/operator-authoring.md",
|
|
105
|
-
"sha256": "sha256:
|
|
106
|
-
"size_bytes":
|
|
105
|
+
"sha256": "sha256:2ecfccbc9fe771bc1103db679d67af78847307eeb440f680f4055c35405ac923",
|
|
106
|
+
"size_bytes": 6089
|
|
107
|
+
},
|
|
108
|
+
{
|
|
109
|
+
"path": "skills/feature-management/references/platform-capability-guide.md",
|
|
110
|
+
"sha256": "sha256:40d0f43a30688c5852ef09203ec35b4dc2c6b9db4bc7dd26dfae52e12d10e9ee",
|
|
111
|
+
"size_bytes": 4257
|
|
107
112
|
}
|
|
108
113
|
],
|
|
109
114
|
"schema_version": "data_platform.ml_platform_checksums/v1"
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zhuoyuezs/ml-platform",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.5",
|
|
4
4
|
"description": "ML 数据平台 business CLI 与 Agent Skills 管理工具",
|
|
5
5
|
"bin": {
|
|
6
6
|
"ml-platform": "scripts/main.js"
|
|
@@ -13,10 +13,13 @@
|
|
|
13
13
|
"release.json",
|
|
14
14
|
"checksums.json",
|
|
15
15
|
"README.md",
|
|
16
|
-
"DEVELOPMENT.md"
|
|
16
|
+
"DEVELOPMENT.md",
|
|
17
|
+
"CI-ARTIFACT.md"
|
|
17
18
|
],
|
|
18
19
|
"scripts": {
|
|
19
|
-
"test": "node --test test/*.test.js"
|
|
20
|
+
"test": "node --test test/*.test.js",
|
|
21
|
+
"api:check": "node ../../scripts/check_api_contract.js",
|
|
22
|
+
"pack": "npm pack"
|
|
20
23
|
},
|
|
21
24
|
"engines": {
|
|
22
25
|
"node": ">=18"
|
package/release.json
CHANGED
|
@@ -13,8 +13,8 @@
|
|
|
13
13
|
"entrypoint": "src/cli.js",
|
|
14
14
|
"name": "ml-platform",
|
|
15
15
|
"path": "runtime/business-client",
|
|
16
|
-
"sha256": "sha256:
|
|
17
|
-
"version": "0.
|
|
16
|
+
"sha256": "sha256:b509f9ccdedc4aa5d4d01c77609e4a16020ac6f74c62efc3af861877d897fe94",
|
|
17
|
+
"version": "0.7.0"
|
|
18
18
|
},
|
|
19
19
|
"policy_sha256": "sha256:02fbde0134696b78c0c365d6e0b76c9814227208da5b2b3d4e5dbd82dc4a3606",
|
|
20
20
|
"public_schema_versions": [
|
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
"ml_data_platform.feature_set/v1",
|
|
26
26
|
"ml_data_platform.dataset_manifest/v1"
|
|
27
27
|
],
|
|
28
|
-
"release_version": "0.1.
|
|
28
|
+
"release_version": "0.1.5",
|
|
29
29
|
"runtime_requirements": {
|
|
30
30
|
"node": ">=18",
|
|
31
31
|
"os": [
|
|
@@ -37,10 +37,10 @@
|
|
|
37
37
|
"skills": {
|
|
38
38
|
"feature-management": {
|
|
39
39
|
"path": "skills/feature-management",
|
|
40
|
-
"requires_cli": ">=0.
|
|
41
|
-
"revision": "0.1.
|
|
42
|
-
"sha256": "sha256:
|
|
40
|
+
"requires_cli": ">=0.7.0 <0.8.0",
|
|
41
|
+
"revision": "0.1.5",
|
|
42
|
+
"sha256": "sha256:ea7ef622f10300a3b5ee200adcf10070d18fadfb899d1bf2cc031ef42ffc5c03"
|
|
43
43
|
}
|
|
44
44
|
},
|
|
45
|
-
"source_commit": "
|
|
45
|
+
"source_commit": "d18a5b535a2a6c41fe4732e8e73739f2728c1e58"
|
|
46
46
|
}
|
|
@@ -25,3 +25,20 @@ ml-platform build-registered-dataset <dataset_id> <dataset_version> \
|
|
|
25
25
|
|
|
26
26
|
`resolve-manifest` 和 `build-dataset <manifest.json>` 仍用于尚未注册的本地契约。Catalog
|
|
27
27
|
normalizer 保留并校验非默认 `project`;`apply --project` 与 Catalog 内项目不一致时拒绝执行。
|
|
28
|
+
|
|
29
|
+
Registry 资产详情和版本发现使用与 `dev` OpenAPI 一致的命令:
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
ml-platform get-parameter <name> <version> --project <project>
|
|
33
|
+
ml-platform get-feature <name> <version> --project <project>
|
|
34
|
+
ml-platform list-feature-versions <name> --project <project> --all
|
|
35
|
+
ml-platform list-jobs --project <project> --status succeeded --all
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
仓库中的 `api-spec/latest.json` 从 `origin/dev` 生成。同步或验证契约时运行:
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
uv run python scripts/generate_openapi_schema.py --ref origin/dev
|
|
42
|
+
uv run python scripts/generate_openapi_schema.py --ref origin/dev --check
|
|
43
|
+
npm run api:check --prefix business-client-js
|
|
44
|
+
```
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zhuoyuezs/ml-platform-business-client",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.7.0",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "@zhuoyuezs/ml-platform-business-client",
|
|
9
|
-
"version": "0.
|
|
9
|
+
"version": "0.7.0",
|
|
10
10
|
"license": "UNLICENSED",
|
|
11
11
|
"bin": {
|
|
12
12
|
"ml-platform": "src/cli.js"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zhuoyuezs/ml-platform-business-client",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.7.0",
|
|
4
4
|
"private": true,
|
|
5
5
|
"description": "JavaScript parity implementation of the ML Platform business client",
|
|
6
6
|
"bin": {
|
|
@@ -11,7 +11,9 @@
|
|
|
11
11
|
"README.md"
|
|
12
12
|
],
|
|
13
13
|
"scripts": {
|
|
14
|
-
"test": "node --test test/*.test.js"
|
|
14
|
+
"test": "node --test test/*.test.js",
|
|
15
|
+
"api:check": "node ../scripts/check_api_contract.js",
|
|
16
|
+
"pack": "npm pack"
|
|
15
17
|
},
|
|
16
18
|
"engines": {
|
|
17
19
|
"node": ">=18"
|
|
@@ -60,6 +60,8 @@ function normalizeCodeArtifact(value) { object(value, "operator code_artifact");
|
|
|
60
60
|
function normalizeParameter(payload) {
|
|
61
61
|
exactKeys(payload, ["schema_version", "project", "name", "display_name", "version", "data_type", "unit", "expected_frequency", "source", "time_semantics", "availability_sla", "value_field", "quality_rules", "rounding", "owner"], ["name", "display_name", "version", "source"], "parameter");
|
|
62
62
|
const source = object(payload.source, "parameter source");
|
|
63
|
+
if (Object.prototype.hasOwnProperty.call(source, "parameters")) throw new Error("parameter source uses unsupported field 'parameters'; use 'params'");
|
|
64
|
+
if (source.mode === "sql" && typeof source.sql === "string" && source.sql.trimEnd().endsWith(";")) throw new Error("SQL Parameter source.sql must not end with a semicolon");
|
|
63
65
|
const time = object(defaultIfAbsent(payload, "time_semantics", {}), "parameter time_semantics");
|
|
64
66
|
return { schema_version: pythonString(defaultIfAbsent(payload, "schema_version", "ml_data_platform.parameter/v1")), project: identifier(defaultIfAbsent(payload, "project", "default"), "parameter project"), name: identifier(payload.name, "parameter name"), display_name: requiredString(payload.display_name, "parameter display_name"), version: identifier(payload.version, "parameter version"), data_type: pythonString(defaultIfAbsent(payload, "data_type", "time_series")), unit: pythonString(defaultIfAbsent(payload, "unit", "")), expected_frequency: pythonString(defaultIfAbsent(payload, "expected_frequency", "10min")), source: { adapter: pythonString(defaultIfAbsent(source, "adapter", "synthetic")), mode: choice(defaultIfAbsent(source, "mode", "direct_column"), ["direct_column", "sql", "derived"], "parameter source mode"), source_metric: optionalString(source.source_metric, "source_metric"), source_group: optionalString(source.source_group, "source_group"), measurement: optionalString(source.measurement, "measurement"), field: optionalString(source.field, "field"), tags: stringMap(defaultIfAbsent(source, "tags", {}), "source tags"), schema: optionalString(aliasIfAbsent(source, "schema", "schema_name"), "schema"), table: optionalString(source.table, "table"), time_column: optionalString(source.time_column, "time_column"), value_column: optionalString(source.value_column, "value_column"), metric_name_column: optionalString(source.metric_name_column, "metric_name_column"), metric_name: optionalString(source.metric_name, "metric_name"), unit_column: optionalString(source.unit_column, "unit_column"), unit: optionalString(aliasIfAbsent(source, "unit", "source_unit"), "source unit"), filters: stringMap(defaultIfAbsent(source, "filters", {}), "source filters"), sql: optionalString(source.sql, "source sql"), params: object(defaultIfAbsent(source, "params", {}), "source params") }, time_semantics: { event_time_field: pythonString(defaultIfAbsent(time, "event_time_field", "event_time")), ingested_at_field: pythonString(defaultIfAbsent(time, "ingested_at_field", "ingested_at")), timezone: pythonString(defaultIfAbsent(time, "timezone", "Asia/Shanghai")), availability: normalizeAvailability(time.availability) }, availability_sla: normalizeAvailabilitySla(payload.availability_sla), value_field: pythonString(defaultIfAbsent(payload, "value_field", "value")), quality_rules: normalizeQualityRules(object(defaultIfAbsent(payload, "quality_rules", {}), "parameter quality_rules")), rounding: normalizeRounding(payload.rounding), owner: pythonString(defaultIfAbsent(payload, "owner", "demo")) };
|
|
65
67
|
}
|
|
@@ -165,7 +167,7 @@ function requiresPackage(spec) { return ["custom_python", "feature", "spark_udf"
|
|
|
165
167
|
function validateCatalog(catalog, current) {
|
|
166
168
|
const parameters = new Set([...current.parameters.keys(), ...catalog.parameters.map(key)]); const operators = new Map([...current.operators, ...catalog.operators.map((x) => [key(x.spec), x.spec])]); const features = new Map([...current.features, ...catalog.features.map((x) => [key(x), x])]); const featureSets = new Map([...current.feature_sets, ...catalog.feature_sets.map((x) => [key(x), x])]);
|
|
167
169
|
for (const item of catalog.operators) if (!item.package_path && requiresPackage(item.spec)) throw new Error(`catalog operator ${key(item.spec)} requires a wheel package path`);
|
|
168
|
-
for (const feature of catalog.features) { for (const input of feature.inputs) if (!parameters.has(parameterReferenceKey(input))) throw new Error(`catalog feature ${key(feature)} references missing parameter ${parameterReferenceKey(input)}`); const operatorKey = `${feature.operator}:${feature.operator_version}`; const operator = operators.get(operatorKey); if (!operator) throw new Error(`catalog feature ${key(feature)} references missing operator ${operatorKey}`); if (operator.type !== "feature") throw new Error(`catalog feature ${key(feature)} requires OperatorSpec.type='feature'`); const declaredInputs = new Set(operator.input_schema.parameters ?? []); if (declaredInputs.size) for (const input of feature.inputs) if (!declaredInputs.has(
|
|
170
|
+
for (const feature of catalog.features) { for (const input of feature.inputs) if (!parameters.has(parameterReferenceKey(input))) throw new Error(`catalog feature ${key(feature)} references missing parameter ${parameterReferenceKey(input)}`); const operatorKey = `${feature.operator}:${feature.operator_version}`; const operator = operators.get(operatorKey); if (!operator) throw new Error(`catalog feature ${key(feature)} references missing operator ${operatorKey}`); if (operator.type !== "feature") throw new Error(`catalog feature ${key(feature)} requires OperatorSpec.type='feature'`); const declaredInputs = new Set(operator.input_schema.parameters ?? []); if (declaredInputs.size) for (const input of feature.inputs) { const inputKey = parameterReferenceKey(input); const qualifiedInputKey = `${operator.project ?? feature.project ?? "default"}/${inputKey}`; if (!declaredInputs.has(inputKey) && !declaredInputs.has(qualifiedInputKey)) throw new Error(`catalog feature ${key(feature)} input is not declared by ${operatorKey}`); } const columns = operator.output_schema.columns ?? []; if (columns.length && !columns.includes(feature.output_column)) throw new Error(`catalog feature ${key(feature)} output_column is not declared by ${operatorKey}`); }
|
|
169
171
|
for (const set of catalog.feature_sets) { const names = []; for (const item of set.features) { const feature = features.get(referenceKey(item)); if (!feature) throw new Error(`catalog feature set ${key(set)} references missing feature ${referenceKey(item)}`); names.push(feature.name); } unique(names, `catalog feature set ${key(set)} final output names`); }
|
|
170
172
|
for (const dataset of catalog.datasets) {
|
|
171
173
|
const datasetId = datasetKey(dataset); const declared = new Set();
|
|
@@ -14,7 +14,12 @@ const BUSINESS_COMMANDS = new Set([
|
|
|
14
14
|
"configure", "show-config", "health", "list-parameters", "list-operators", "list-features",
|
|
15
15
|
"list-feature-sets", "list-datasets", "get-dataset", "list-jobs", "resolve-manifest",
|
|
16
16
|
"resolve-dataset", "build-dataset", "build-registered-dataset",
|
|
17
|
-
"get-
|
|
17
|
+
"get-parameter", "get-feature", "get-feature-set", "get-operator",
|
|
18
|
+
"list-parameter-versions", "list-feature-versions", "list-feature-set-versions", "list-operator-versions",
|
|
19
|
+
"add-parameter", "add-feature", "add-feature-set", "add-operator", "publish-operator",
|
|
20
|
+
"add-dataset", "update-dataset", "delete-dataset", "list-dataset-artifacts",
|
|
21
|
+
"register-operator", "show-operator-specs", "delete-operator", "run-operator",
|
|
22
|
+
"get-job", "wait-job", "retry-job", "cancel-job", "get-dataset-artifact", "download-dataset-artifact", "fetch-inference-data", "apply",
|
|
18
23
|
]);
|
|
19
24
|
|
|
20
25
|
const COMMAND_USAGE = {
|
|
@@ -26,21 +31,44 @@ const COMMAND_USAGE = {
|
|
|
26
31
|
"list-projects": "list-projects",
|
|
27
32
|
"get-project": "get-project NAME",
|
|
28
33
|
"delete-project": "delete-project NAME",
|
|
29
|
-
"list-parameters": "list-parameters [--project PROJECT] [-q QUERY] [--limit N] [--offset N]",
|
|
30
|
-
"list-operators": "list-operators [--project PROJECT] [-q QUERY] [--limit N] [--offset N]",
|
|
31
|
-
"list-features": "list-features [--project PROJECT] [-q QUERY] [--limit N] [--offset N]",
|
|
32
|
-
"list-feature-sets": "list-feature-sets [--project PROJECT] [-q QUERY] [--limit N] [--offset N]",
|
|
33
|
-
"list-datasets": "list-datasets [--project PROJECT] [-q QUERY] [--limit N] [--offset N]",
|
|
34
|
+
"list-parameters": "list-parameters [--project PROJECT] [--name NAME] [--version VERSION] [-q QUERY] [--limit N] [--offset N] [--all]",
|
|
35
|
+
"list-operators": "list-operators [--project PROJECT] [--name NAME] [--version VERSION] [-q QUERY] [--limit N] [--offset N] [--all]",
|
|
36
|
+
"list-features": "list-features [--project PROJECT] [--name NAME] [--version VERSION] [-q QUERY] [--limit N] [--offset N] [--all]",
|
|
37
|
+
"list-feature-sets": "list-feature-sets [--project PROJECT] [--name NAME] [--version VERSION] [-q QUERY] [--limit N] [--offset N] [--all]",
|
|
38
|
+
"list-datasets": "list-datasets [--project PROJECT] [--dataset-id ID] [--dataset-version VERSION] [-q QUERY] [--limit N] [--offset N] [--include-deleted] [--all]",
|
|
39
|
+
"get-parameter": "get-parameter NAME VERSION [--project PROJECT]",
|
|
40
|
+
"get-feature": "get-feature NAME VERSION [--project PROJECT]",
|
|
41
|
+
"get-feature-set": "get-feature-set NAME VERSION [--project PROJECT]",
|
|
42
|
+
"get-operator": "get-operator NAME VERSION [--project PROJECT]",
|
|
43
|
+
"list-parameter-versions": "list-parameter-versions NAME [--project PROJECT] [--limit N] [--offset N] [--all]",
|
|
44
|
+
"list-feature-versions": "list-feature-versions NAME [--project PROJECT] [--limit N] [--offset N] [--all]",
|
|
45
|
+
"list-feature-set-versions": "list-feature-set-versions NAME [--project PROJECT] [--limit N] [--offset N] [--all]",
|
|
46
|
+
"list-operator-versions": "list-operator-versions NAME [--project PROJECT] [--limit N] [--offset N] [--all]",
|
|
47
|
+
"add-parameter": "add-parameter SPEC_JSON",
|
|
48
|
+
"add-feature": "add-feature SPEC_JSON",
|
|
49
|
+
"add-feature-set": "add-feature-set SPEC_JSON",
|
|
50
|
+
"add-operator": "add-operator SPEC_JSON",
|
|
51
|
+
"publish-operator": "publish-operator SPEC_JSON --package PATH",
|
|
52
|
+
"add-dataset": "add-dataset MANIFEST_JSON [--force]",
|
|
53
|
+
"update-dataset": "update-dataset MANIFEST_JSON [--force]",
|
|
54
|
+
"delete-dataset": "delete-dataset DATASET_ID DATASET_VERSION [--project PROJECT]",
|
|
55
|
+
"list-dataset-artifacts": "list-dataset-artifacts [--project PROJECT] [--dataset-id ID] [--dataset-version VERSION] [--include-deleted] [--limit N] [--offset N]",
|
|
34
56
|
"get-dataset": "get-dataset DATASET_ID DATASET_VERSION [--project PROJECT] [--include-deleted]",
|
|
35
|
-
"list-jobs": "list-jobs [-q QUERY] [--limit N] [--offset N]",
|
|
57
|
+
"list-jobs": "list-jobs [filters] [-q QUERY] [--limit N] [--offset N] [--all]",
|
|
36
58
|
"resolve-manifest": "resolve-manifest MANIFEST [--out PATH]",
|
|
37
59
|
"resolve-dataset": "resolve-dataset DATASET_ID DATASET_VERSION [--project PROJECT] [--out PATH]",
|
|
38
|
-
"build-dataset": "build-dataset MANIFEST [--source-mode direct] [--upload-chronon-metadata] [--partition-duration DURATION] [--max-parallelism N] [--no-resume] [--wait] [--poll-interval SECONDS] [--wait-timeout-seconds SECONDS]",
|
|
39
|
-
"build-registered-dataset": "build-registered-dataset DATASET_ID DATASET_VERSION [--project PROJECT] [--source-mode direct] [--upload-chronon-metadata] [--partition-duration DURATION] [--max-parallelism N] [--no-resume] [--wait] [--poll-interval SECONDS] [--wait-timeout-seconds SECONDS]",
|
|
60
|
+
"build-dataset": "build-dataset MANIFEST [--engine chronon] [--source-mode direct] [--upload-chronon-metadata] [--partition-duration DURATION] [--max-parallelism N] [--no-resume] [--wait] [--poll-interval SECONDS] [--wait-timeout-seconds SECONDS]",
|
|
61
|
+
"build-registered-dataset": "build-registered-dataset DATASET_ID DATASET_VERSION [--project PROJECT] [--engine chronon] [--source-mode direct] [--upload-chronon-metadata] [--partition-duration DURATION] [--max-parallelism N] [--no-resume] [--wait] [--poll-interval SECONDS] [--wait-timeout-seconds SECONDS]",
|
|
40
62
|
"get-job": "get-job JOB_ID",
|
|
41
63
|
"wait-job": "wait-job JOB_ID [--poll-interval SECONDS] [--wait-timeout-seconds SECONDS]",
|
|
64
|
+
"retry-job": "retry-job JOB_ID",
|
|
65
|
+
"cancel-job": "cancel-job JOB_ID",
|
|
66
|
+
"register-operator": "register-operator SPEC_JSON",
|
|
67
|
+
"show-operator-specs": "show-operator-specs [--project PROJECT] [-q QUERY] [--limit N] [--offset N]",
|
|
68
|
+
"delete-operator": "delete-operator NAME VERSION [--project PROJECT]",
|
|
69
|
+
"run-operator": "run-operator NAME VERSION [--project PROJECT] [--config JSON]",
|
|
42
70
|
"get-dataset-artifact": "get-dataset-artifact DATASET_ID MANIFEST_HASH [--project PROJECT]",
|
|
43
|
-
"download-dataset-artifact": "download-dataset-artifact DATASET_ID MANIFEST_HASH [--project PROJECT] --out-dir PATH [--force]",
|
|
71
|
+
"download-dataset-artifact": "download-dataset-artifact DATASET_ID MANIFEST_HASH [--project PROJECT] --out-dir PATH [--file PATH] [--force]",
|
|
44
72
|
"fetch-inference-data": "fetch-inference-data MANIFEST --cutoff-time TIMESTAMP [--max-workers N] [--max-source-lag-hours HOURS] [--no-validate-freshness] [--allow-missing]",
|
|
45
73
|
apply: "apply CATALOG_DIRECTORY [--dry-run] [--project PROJECT]",
|
|
46
74
|
};
|
|
@@ -52,7 +80,10 @@ function usage(command) {
|
|
|
52
80
|
}
|
|
53
81
|
|
|
54
82
|
function parse(args) {
|
|
55
|
-
const
|
|
83
|
+
const timeoutValue = process.env.ML_PLATFORM_API_TIMEOUT_SECONDS
|
|
84
|
+
|| process.env.DATA_PLATFORM_DEMO_API_TIMEOUT_SECONDS
|
|
85
|
+
|| 30;
|
|
86
|
+
const options = { profile: "server", timeout: finiteNumber(timeoutValue, "--request-timeout-seconds") };
|
|
56
87
|
let index = 0;
|
|
57
88
|
while (index < args.length && args[index].startsWith("--")) {
|
|
58
89
|
const flag = args[index++];
|
|
@@ -66,7 +97,7 @@ function parse(args) {
|
|
|
66
97
|
if (!(options.timeout > 0)) throw new Error("--request-timeout-seconds must be positive");
|
|
67
98
|
options.command = args[index++];
|
|
68
99
|
options.rest = args.slice(index);
|
|
69
|
-
options.apiUrl ??= process.env.ML_PLATFORM_API_URL || loadApiUrl();
|
|
100
|
+
options.apiUrl ??= process.env.ML_PLATFORM_API_URL || process.env.DATA_PLATFORM_DEMO_API_URL || loadApiUrl();
|
|
70
101
|
return options;
|
|
71
102
|
}
|
|
72
103
|
|
|
@@ -105,7 +136,7 @@ function positional(rest, label) { const value = rest.shift(); if (!value || val
|
|
|
105
136
|
|
|
106
137
|
function client(options) {
|
|
107
138
|
if (options.profile !== "server") throw new Error("the business client supports only --profile server");
|
|
108
|
-
if (!options.apiUrl) throw new Error("server profile requires --api-url, ML_PLATFORM_API_URL, or a saved configure target");
|
|
139
|
+
if (!options.apiUrl) throw new Error("server profile requires --api-url, ML_PLATFORM_API_URL, DATA_PLATFORM_DEMO_API_URL, or a saved configure target");
|
|
109
140
|
return new PlatformApiClient(options.apiUrl, options.timeout);
|
|
110
141
|
}
|
|
111
142
|
|
|
@@ -116,6 +147,7 @@ function readObject(file) {
|
|
|
116
147
|
}
|
|
117
148
|
|
|
118
149
|
function readDataset(file) { return normalizeDataset(readObject(file)); }
|
|
150
|
+
function readSpec(file) { return readObject(file); }
|
|
119
151
|
|
|
120
152
|
function datasetEndpoint(project, datasetId, datasetVersion) {
|
|
121
153
|
return `/datasets/${encodeURIComponent(project)}/${encodeURIComponent(datasetId)}/${encodeURIComponent(datasetVersion)}`;
|
|
@@ -128,6 +160,8 @@ async function registeredDataset(api, project, datasetId, datasetVersion, includ
|
|
|
128
160
|
}
|
|
129
161
|
|
|
130
162
|
function buildOptions(rest) {
|
|
163
|
+
const engine = take(rest, "--engine", "chronon");
|
|
164
|
+
if (engine !== "chronon") throw new Error("--engine must be chronon");
|
|
131
165
|
const sourceMode = take(rest, "--source-mode");
|
|
132
166
|
if (sourceMode !== undefined && sourceMode !== "direct") throw new Error("--source-mode must be direct");
|
|
133
167
|
const maxParallelism = integer(take(rest, "--max-parallelism", 1), "--max-parallelism");
|
|
@@ -139,6 +173,7 @@ function buildOptions(rest) {
|
|
|
139
173
|
const waitTimeout = rawTimeout === undefined ? undefined : finiteNumber(rawTimeout, "--wait-timeout-seconds");
|
|
140
174
|
if (waitTimeout !== undefined && !(waitTimeout > 0)) throw new Error("--wait-timeout-seconds must be positive");
|
|
141
175
|
return {
|
|
176
|
+
engine,
|
|
142
177
|
sourceMode,
|
|
143
178
|
upload: boolean(rest, "--upload-chronon-metadata"),
|
|
144
179
|
partitionDuration: take(rest, "--partition-duration"),
|
|
@@ -151,7 +186,7 @@ function buildOptions(rest) {
|
|
|
151
186
|
}
|
|
152
187
|
|
|
153
188
|
async function submitBuild(api, manifest, build) {
|
|
154
|
-
let result = await api.post("/datasets/build", manifest, { engine:
|
|
189
|
+
let result = await api.post("/datasets/build", manifest, { engine: build.engine, source_mode: build.sourceMode, upload_chronon_metadata: build.upload, partition_duration: build.partitionDuration, max_parallelism: build.maxParallelism, resume: build.resume });
|
|
155
190
|
let exitCode = 0;
|
|
156
191
|
if (build.wait) {
|
|
157
192
|
if (!result || typeof result !== "object" || !result.job_id) throw new Error("server build did not return a job_id to wait for");
|
|
@@ -205,20 +240,54 @@ async function runBusinessCli(argv) {
|
|
|
205
240
|
} else if (options.command === "list-projects") result = await client(options).get("/projects");
|
|
206
241
|
else if (options.command === "get-project") result = await client(options).get(`/projects/${encodeURIComponent(positional(rest, "project name"))}`);
|
|
207
242
|
else if (options.command === "delete-project") result = await client(options).delete(`/projects/${encodeURIComponent(positional(rest, "project name"))}`);
|
|
208
|
-
else if (
|
|
243
|
+
else if (["add-parameter", "add-feature", "add-feature-set", "add-operator", "register-operator"].includes(options.command)) {
|
|
244
|
+
const spec = readSpec(positional(rest, "spec JSON"));
|
|
245
|
+
const endpoint = { "add-parameter": "/parameters", "add-feature": "/features", "add-feature-set": "/feature-sets", "add-operator": "/operators", "register-operator": "/operators" }[options.command];
|
|
246
|
+
result = await client(options).post(endpoint, spec);
|
|
247
|
+
} else if (options.command === "publish-operator") {
|
|
248
|
+
const specPath = positional(rest, "OperatorSpec JSON"); const packagePath = take(rest, "--package");
|
|
249
|
+
if (!packagePath) throw new Error("--package is required");
|
|
250
|
+
const spec = readSpec(specPath); const api = client(options); const project = spec.project || "default";
|
|
251
|
+
const upload = await api.uploadOperatorPackage(project, spec.name, spec.version, packagePath);
|
|
252
|
+
if (!upload || typeof upload.code_artifact !== "object") throw new Error("operator package upload response has no code_artifact");
|
|
253
|
+
const published = { ...spec, code_hash: upload.code_artifact.sha256, package_uri: upload.code_artifact.uri, code_artifact: upload.code_artifact };
|
|
254
|
+
result = { status: "published", profile: "server", path: null, operator: published, registration: await api.post("/operators", published) };
|
|
255
|
+
} else if (options.command === "get-dataset") {
|
|
209
256
|
const datasetId = positional(rest, "dataset id"); const datasetVersion = positional(rest, "dataset version"); const project = take(rest, "--project", "default");
|
|
210
257
|
result = await registeredDataset(client(options), project, datasetId, datasetVersion, boolean(rest, "--include-deleted"));
|
|
211
|
-
}
|
|
212
|
-
|
|
258
|
+
} else if (["get-parameter", "get-feature", "get-feature-set", "get-operator"].includes(options.command)) {
|
|
259
|
+
const resource = options.command.slice(4);
|
|
260
|
+
const endpoint = resource === "feature-set" ? "feature-sets" : `${resource}s`;
|
|
261
|
+
const name = positional(rest, `${resource} name`);
|
|
262
|
+
const version = positional(rest, `${resource} version`);
|
|
263
|
+
const project = take(rest, "--project", "default");
|
|
264
|
+
result = await client(options).get(`/${endpoint}/${encodeURIComponent(project)}/${encodeURIComponent(name)}/${encodeURIComponent(version)}`);
|
|
265
|
+
} else if (["list-parameter-versions", "list-feature-versions", "list-feature-set-versions", "list-operator-versions"].includes(options.command)) {
|
|
266
|
+
const resource = options.command.replace(/^list-/, "").replace(/-versions$/, "");
|
|
267
|
+
const endpoint = resource === "feature-set" ? "feature-sets" : `${resource}s`;
|
|
268
|
+
const name = positional(rest, `${resource} name`);
|
|
269
|
+
const project = take(rest, "--project", "default");
|
|
270
|
+
const allPages = boolean(rest, "--all");
|
|
271
|
+
const rawLimit = take(rest, "--limit"); const rawOffset = take(rest, "--offset");
|
|
272
|
+
if (allPages && (rawLimit !== undefined || rawOffset !== undefined)) throw new Error("--all cannot be combined with --limit or --offset");
|
|
273
|
+
if (allPages) result = await fetchAll(client(options), `/${endpoint}/${encodeURIComponent(project)}/${encodeURIComponent(name)}`);
|
|
274
|
+
else { const limit = Number(rawLimit ?? 50); const offset = Number(rawOffset ?? 0); if (!Number.isInteger(limit) || limit < 1 || limit > 500) throw new Error("limit must be between 1 and 500"); if (!Number.isInteger(offset) || offset < 0) throw new Error("offset must be >= 0"); result = await client(options).get(`/${endpoint}/${encodeURIComponent(project)}/${encodeURIComponent(name)}`, { limit, offset }); }
|
|
275
|
+
} else if (options.command.startsWith("list-") && !["list-jobs", "list-dataset-artifacts"].includes(options.command)) {
|
|
213
276
|
const endpoints = { "list-parameters": "/parameters", "list-operators": "/operators", "list-features": "/features", "list-feature-sets": "/feature-sets", "list-datasets": "/datasets" };
|
|
214
277
|
const project = take(rest, "--project");
|
|
215
|
-
const
|
|
216
|
-
|
|
217
|
-
|
|
278
|
+
const name = take(rest, "--name"); const version = take(rest, "--version");
|
|
279
|
+
const query = take(rest, "--q", take(rest, "-q")); const includeDeleted = options.command === "list-datasets" && boolean(rest, "--include-deleted"); const rawLimit = take(rest, "--limit"); const rawOffset = take(rest, "--offset");
|
|
280
|
+
const allPages = boolean(rest, "--all");
|
|
281
|
+
if (allPages && (rawLimit !== undefined || rawOffset !== undefined)) throw new Error("--all cannot be combined with --limit or --offset");
|
|
282
|
+
if (allPages) result = await fetchAll(client(options), endpoints[options.command], { q: query, project, name, version, include_deleted: includeDeleted });
|
|
283
|
+
else { const limit = Number(rawLimit ?? 50); const offset = Number(rawOffset ?? 0); if (!Number.isInteger(limit) || limit < 1 || limit > 500) throw new Error("limit must be between 1 and 500"); if (!Number.isInteger(offset) || offset < 0) throw new Error("offset must be >= 0"); result = await client(options).get(endpoints[options.command], { q: query, limit, offset, project, name, version, include_deleted: includeDeleted }); }
|
|
218
284
|
} else if (options.command === "list-jobs") {
|
|
219
285
|
const query = take(rest, "--q", take(rest, "-q")); const rawLimit = take(rest, "--limit"); const rawOffset = take(rest, "--offset");
|
|
220
|
-
|
|
221
|
-
|
|
286
|
+
const filters = {}; for (const field of ["project", "dataset-id", "dataset-version", "manifest-hash", "status", "parent-job-id", "created-from", "created-to", "updated-from", "updated-to"]) filters[field.replaceAll("-", "_")] = take(rest, `--${field}`);
|
|
287
|
+
const allPages = boolean(rest, "--all");
|
|
288
|
+
if (allPages && (rawLimit !== undefined || rawOffset !== undefined)) throw new Error("--all cannot be combined with --limit or --offset");
|
|
289
|
+
if (allPages) result = await fetchAll(client(options), "/jobs", { q: query, ...filters });
|
|
290
|
+
else { const limit = Number(rawLimit ?? 50); const offset = Number(rawOffset ?? 0); if (!Number.isInteger(limit) || limit < 1 || limit > 500) throw new Error("limit must be between 1 and 500"); if (!Number.isInteger(offset) || offset < 0) throw new Error("offset must be >= 0"); result = await client(options).get("/jobs", { q: query, limit, offset, ...filters }); }
|
|
222
291
|
}
|
|
223
292
|
else if (options.command === "resolve-manifest") {
|
|
224
293
|
const manifest = positional(rest, "manifest");
|
|
@@ -239,19 +308,49 @@ async function runBusinessCli(argv) {
|
|
|
239
308
|
const datasetId = positional(rest, "dataset id"); const datasetVersion = positional(rest, "dataset version"); const project = take(rest, "--project", "default"); const build = buildOptions(rest); const api = client(options);
|
|
240
309
|
const manifest = normalizeRegistryDataset(await registeredDataset(api, project, datasetId, datasetVersion));
|
|
241
310
|
({ result, exitCode } = await submitBuild(api, manifest, build));
|
|
311
|
+
} else if (["add-dataset", "update-dataset"].includes(options.command)) {
|
|
312
|
+
const manifest = readDataset(positional(rest, "manifest JSON")); const force = boolean(rest, "--force"); const project = manifest.project || "default";
|
|
313
|
+
if (options.command === "add-dataset") result = await client(options).post("/datasets", manifest, { force });
|
|
314
|
+
else result = await client(options).put(`/datasets/${encodeURIComponent(project)}/${encodeURIComponent(manifest.dataset_id)}/${encodeURIComponent(manifest.dataset_version)}`, manifest, { force });
|
|
315
|
+
} else if (options.command === "delete-dataset") {
|
|
316
|
+
const id = positional(rest, "dataset id"); const version = positional(rest, "dataset version"); const project = take(rest, "--project", "default");
|
|
317
|
+
result = await client(options).delete(`/datasets/${encodeURIComponent(project)}/${encodeURIComponent(id)}/${encodeURIComponent(version)}`);
|
|
318
|
+
} else if (options.command === "list-dataset-artifacts") {
|
|
319
|
+
const project = take(rest, "--project"); const datasetId = take(rest, "--dataset-id"); const datasetVersion = take(rest, "--dataset-version"); const includeDeleted = boolean(rest, "--include-deleted");
|
|
320
|
+
result = await listCollection(client(options), "/dataset-artifacts", { project, dataset_id: datasetId, dataset_version: datasetVersion, include_deleted: includeDeleted }, rest);
|
|
321
|
+
} else if (options.command === "show-operator-specs") {
|
|
322
|
+
const project = take(rest, "--project"); const query = take(rest, "--q", take(rest, "-q"));
|
|
323
|
+
result = await listCollection(client(options), "/operators", { project, q: query }, rest);
|
|
324
|
+
} else if (options.command === "delete-operator") {
|
|
325
|
+
const name = positional(rest, "operator name"); const version = positional(rest, "operator version"); const project = take(rest, "--project", "default");
|
|
326
|
+
result = await client(options).delete(`/operators/${encodeURIComponent(project)}/${encodeURIComponent(name)}/${encodeURIComponent(version)}`);
|
|
327
|
+
} else if (options.command === "run-operator") {
|
|
328
|
+
const name = positional(rest, "operator name"); const version = positional(rest, "operator version"); const project = take(rest, "--project", "default"); const rawConfig = take(rest, "--config", "{}");
|
|
329
|
+
let config; try { config = JSON.parse(rawConfig); } catch (_) { throw new Error("--config must be valid JSON"); }
|
|
330
|
+
result = await client(options).post(`/operators/${encodeURIComponent(project)}/${encodeURIComponent(name)}/${encodeURIComponent(version)}/run`, config);
|
|
242
331
|
} else if (options.command === "get-job") result = await client(options).get(`/jobs/${encodeURIComponent(positional(rest, "job id"))}`);
|
|
243
332
|
else if (options.command === "wait-job") {
|
|
244
333
|
const jobId = positional(rest, "job id"); const poll = finiteNumber(take(rest, "--poll-interval", 2), "--poll-interval"); const rawTimeout = take(rest, "--wait-timeout-seconds");
|
|
245
334
|
result = await waitForJob(client(options), jobId, poll, rawTimeout === undefined ? undefined : finiteNumber(rawTimeout, "--wait-timeout-seconds"));
|
|
246
335
|
if (["failed", "cancelled"].includes(result.status)) exitCode = 1;
|
|
247
336
|
}
|
|
337
|
+
else if (options.command === "cancel-job") {
|
|
338
|
+
result = await client(options).post(`/jobs/${encodeURIComponent(positional(rest, "job id"))}/cancel`);
|
|
339
|
+
}
|
|
340
|
+
else if (options.command === "retry-job") {
|
|
341
|
+
result = await client(options).post(`/jobs/${encodeURIComponent(positional(rest, "job id"))}/retry`);
|
|
342
|
+
}
|
|
248
343
|
else if (options.command === "get-dataset-artifact") {
|
|
249
344
|
const dataset = positional(rest, "dataset id"); const hash = positional(rest, "manifest hash"); const project = take(rest, "--project", "default");
|
|
250
345
|
result = await client(options).get(`/dataset-artifacts/${encodeURIComponent(project)}/${encodeURIComponent(dataset)}/${encodeURIComponent(hash)}`);
|
|
251
346
|
} else if (options.command === "download-dataset-artifact") {
|
|
252
347
|
const dataset = positional(rest, "dataset id"); const hash = positional(rest, "manifest hash"); const project = take(rest, "--project", "default"); const outDir = take(rest, "--out-dir");
|
|
253
348
|
if (!outDir) throw new Error("--out-dir is required");
|
|
254
|
-
|
|
349
|
+
const files = [];
|
|
350
|
+
while (rest.includes("--file")) files.push(take(rest, "--file"));
|
|
351
|
+
result = files.length
|
|
352
|
+
? await client(options).downloadDatasetArtifactFiles(project, dataset, hash, files, resolvedPath(outDir), boolean(rest, "--force"))
|
|
353
|
+
: await client(options).downloadDatasetArtifact(project, dataset, hash, resolvedPath(outDir), boolean(rest, "--force"));
|
|
255
354
|
} else if (options.command === "fetch-inference-data") {
|
|
256
355
|
const manifest = positional(rest, "manifest"); const cutoffTime = take(rest, "--cutoff-time"); if (!cutoffTime) throw new Error("--cutoff-time is required");
|
|
257
356
|
const realtime = {}; const maxWorkers = take(rest, "--max-workers"); const maxLag = take(rest, "--max-source-lag-hours"); if (maxWorkers !== undefined) { realtime.max_workers = integer(maxWorkers, "--max-workers"); if (realtime.max_workers < 1) throw new Error("--max-workers must be at least 1"); } if (maxLag !== undefined) { realtime.max_source_lag_hours = finiteNumber(maxLag, "--max-source-lag-hours"); if (realtime.max_source_lag_hours < 0) throw new Error("--max-source-lag-hours must be non-negative"); } if (boolean(rest, "--no-validate-freshness")) realtime.validate_freshness = false; if (boolean(rest, "--allow-missing")) realtime.allow_missing = true;
|
|
@@ -267,7 +366,16 @@ async function runBusinessCli(argv) {
|
|
|
267
366
|
|
|
268
367
|
async function fetchAll(api, endpoint, params = {}) { const items = []; for (let offset = 0; ; offset += 500) { const page = await api.get(endpoint, { ...params, limit: 500, offset }); if (Array.isArray(page)) return page; if (!page || !Array.isArray(page.items)) throw new Error(`${endpoint} returned an invalid paginated response`); items.push(...page.items); if (page.total == null || items.length >= page.total || page.items.length < 500) return items; } }
|
|
269
368
|
|
|
270
|
-
|
|
369
|
+
async function listCollection(api, endpoint, params, rest) {
|
|
370
|
+
const rawLimit = take(rest, "--limit"); const rawOffset = take(rest, "--offset");
|
|
371
|
+
if (rawLimit === undefined && rawOffset === undefined) return fetchAll(api, endpoint, params);
|
|
372
|
+
const limit = Number(rawLimit ?? 50); const offset = Number(rawOffset ?? 0);
|
|
373
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > 500) throw new Error("limit must be between 1 and 500");
|
|
374
|
+
if (!Number.isInteger(offset) || offset < 0) throw new Error("offset must be >= 0");
|
|
375
|
+
return api.get(endpoint, { ...params, limit, offset });
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
module.exports = { BUSINESS_COMMANDS, fetchAll, listCollection, parse, runBusinessCli, usage, waitForJob };
|
|
271
379
|
|
|
272
380
|
if (require.main === module) {
|
|
273
381
|
runBusinessCli(process.argv.slice(2)).then((code) => { process.exitCode = code; }).catch((error) => {
|