@zhuoyuezs/ml-platform 0.1.3 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/checksums.json +33 -28
- package/package.json +1 -1
- package/release.json +5 -5
- package/runtime/business-client/src/catalog.js +3 -1
- package/runtime/business-client/src/cli.js +86 -14
- package/runtime/business-client/src/config.js +6 -2
- package/runtime/business-client/src/http.js +88 -17
- package/skills/feature-management/SKILL.md +150 -10
- package/skills/feature-management/assets/catalog-template/datasets/example_temperature_training.v1.json +16 -0
- package/skills/feature-management/assets/catalog-template/feature_sets/example_temperature_core.v1.json +1 -0
- package/skills/feature-management/assets/catalog-template/features/example_temperature_mean_5m.v1.json +9 -2
- package/skills/feature-management/assets/catalog-template/operator_package/tests/test_operator.py +51 -24
- package/skills/feature-management/assets/catalog-template/operators/example_temperature_features.v1.json +16 -1
- package/skills/feature-management/assets/catalog-template/parameters/example_temperature.v1.json +1 -0
- package/skills/feature-management/references/commands.md +29 -0
- package/skills/feature-management/references/contracts.md +43 -6
- package/skills/feature-management/references/operator-authoring.md +15 -7
- package/skills/feature-management/references/platform-capability-guide.md +44 -0
package/checksums.json
CHANGED
|
@@ -17,28 +17,28 @@
|
|
|
17
17
|
},
|
|
18
18
|
{
|
|
19
19
|
"path": "runtime/business-client/src/catalog.js",
|
|
20
|
-
"sha256": "sha256:
|
|
21
|
-
"size_bytes":
|
|
20
|
+
"sha256": "sha256:afc2ba28eee2d1dcd833a2e5c4ac923421dc60302b31bf985aa638849bbf3d36",
|
|
21
|
+
"size_bytes": 54246
|
|
22
22
|
},
|
|
23
23
|
{
|
|
24
24
|
"path": "runtime/business-client/src/cli.js",
|
|
25
|
-
"sha256": "sha256:
|
|
26
|
-
"size_bytes":
|
|
25
|
+
"sha256": "sha256:db25cecb760a2b94ac1433c1d04ab70b429825e78dd1168e72688d8b9a8556be",
|
|
26
|
+
"size_bytes": 25911
|
|
27
27
|
},
|
|
28
28
|
{
|
|
29
29
|
"path": "runtime/business-client/src/config.js",
|
|
30
|
-
"sha256": "sha256:
|
|
31
|
-
"size_bytes":
|
|
30
|
+
"sha256": "sha256:4190e09c339a3f0b2861ca7dbc96fe2aa3bdd7927a5e1c4a4c00103a53236bd8",
|
|
31
|
+
"size_bytes": 2186
|
|
32
32
|
},
|
|
33
33
|
{
|
|
34
34
|
"path": "runtime/business-client/src/http.js",
|
|
35
|
-
"sha256": "sha256:
|
|
36
|
-
"size_bytes":
|
|
35
|
+
"sha256": "sha256:95ac9342c09605966f055dde2ac5cd67fdf28e8bc877e2233e3c48077ffdd729",
|
|
36
|
+
"size_bytes": 12429
|
|
37
37
|
},
|
|
38
38
|
{
|
|
39
39
|
"path": "skills/feature-management/SKILL.md",
|
|
40
|
-
"sha256": "sha256:
|
|
41
|
-
"size_bytes":
|
|
40
|
+
"sha256": "sha256:69842537b8df63e58630ca0581575f29f6ec9e97d22c9c2c30b1575270161f6e",
|
|
41
|
+
"size_bytes": 28100
|
|
42
42
|
},
|
|
43
43
|
{
|
|
44
44
|
"path": "skills/feature-management/agents/openai.yaml",
|
|
@@ -52,18 +52,18 @@
|
|
|
52
52
|
},
|
|
53
53
|
{
|
|
54
54
|
"path": "skills/feature-management/assets/catalog-template/datasets/example_temperature_training.v1.json",
|
|
55
|
-
"sha256": "sha256:
|
|
56
|
-
"size_bytes":
|
|
55
|
+
"sha256": "sha256:4dbe304631d6ecbf60b19d7c5a377178e0593525362f5fe509c60622b99e8a46",
|
|
56
|
+
"size_bytes": 900
|
|
57
57
|
},
|
|
58
58
|
{
|
|
59
59
|
"path": "skills/feature-management/assets/catalog-template/feature_sets/example_temperature_core.v1.json",
|
|
60
|
-
"sha256": "sha256:
|
|
61
|
-
"size_bytes":
|
|
60
|
+
"sha256": "sha256:22f37b954200864c19ed5593ab8455f562cc3dee7e05a0b01d5856a163dc0660",
|
|
61
|
+
"size_bytes": 352
|
|
62
62
|
},
|
|
63
63
|
{
|
|
64
64
|
"path": "skills/feature-management/assets/catalog-template/features/example_temperature_mean_5m.v1.json",
|
|
65
|
-
"sha256": "sha256:
|
|
66
|
-
"size_bytes":
|
|
65
|
+
"sha256": "sha256:7e58fa9c6a5ef581866d5cae598682de73e5ef3ce49fdd630432bada3a8a41a1",
|
|
66
|
+
"size_bytes": 807
|
|
67
67
|
},
|
|
68
68
|
{
|
|
69
69
|
"path": "skills/feature-management/assets/catalog-template/operator_package/pyproject.toml",
|
|
@@ -77,33 +77,38 @@
|
|
|
77
77
|
},
|
|
78
78
|
{
|
|
79
79
|
"path": "skills/feature-management/assets/catalog-template/operator_package/tests/test_operator.py",
|
|
80
|
-
"sha256": "sha256:
|
|
81
|
-
"size_bytes":
|
|
80
|
+
"sha256": "sha256:6fe073e07ed57d0fb4e782150153556b97c5b873b9eb2dad45df0d7f522d8650",
|
|
81
|
+
"size_bytes": 3678
|
|
82
82
|
},
|
|
83
83
|
{
|
|
84
84
|
"path": "skills/feature-management/assets/catalog-template/operators/example_temperature_features.v1.json",
|
|
85
|
-
"sha256": "sha256:
|
|
86
|
-
"size_bytes":
|
|
85
|
+
"sha256": "sha256:646965a05b982ae291960b23f359403c69bf8a173d6cc65d706fb2dc53ab030b",
|
|
86
|
+
"size_bytes": 1277
|
|
87
87
|
},
|
|
88
88
|
{
|
|
89
89
|
"path": "skills/feature-management/assets/catalog-template/parameters/example_temperature.v1.json",
|
|
90
|
-
"sha256": "sha256:
|
|
91
|
-
"size_bytes":
|
|
90
|
+
"sha256": "sha256:28c7d3576c4b7e94c1e9db7f95f26d83ee2e10dc9f81ec6223f4fbd9b2f5eeda",
|
|
91
|
+
"size_bytes": 1488
|
|
92
92
|
},
|
|
93
93
|
{
|
|
94
94
|
"path": "skills/feature-management/references/commands.md",
|
|
95
|
-
"sha256": "sha256:
|
|
96
|
-
"size_bytes":
|
|
95
|
+
"sha256": "sha256:c20ccf9ad8d505a1880a7e35773edb5040bbbcfeba3a64316e3148a6e337bdc2",
|
|
96
|
+
"size_bytes": 11546
|
|
97
97
|
},
|
|
98
98
|
{
|
|
99
99
|
"path": "skills/feature-management/references/contracts.md",
|
|
100
|
-
"sha256": "sha256:
|
|
101
|
-
"size_bytes":
|
|
100
|
+
"sha256": "sha256:913d5f0ad52b5183aa3a4bdeef2b6974f7757842300f547679fee38552792892",
|
|
101
|
+
"size_bytes": 27738
|
|
102
102
|
},
|
|
103
103
|
{
|
|
104
104
|
"path": "skills/feature-management/references/operator-authoring.md",
|
|
105
|
-
"sha256": "sha256:
|
|
106
|
-
"size_bytes":
|
|
105
|
+
"sha256": "sha256:2ecfccbc9fe771bc1103db679d67af78847307eeb440f680f4055c35405ac923",
|
|
106
|
+
"size_bytes": 6089
|
|
107
|
+
},
|
|
108
|
+
{
|
|
109
|
+
"path": "skills/feature-management/references/platform-capability-guide.md",
|
|
110
|
+
"sha256": "sha256:fb606c3637891dab462d5b94004b60a3b8f02eb178aef589716f968c3b87cd5e",
|
|
111
|
+
"size_bytes": 2627
|
|
107
112
|
}
|
|
108
113
|
],
|
|
109
114
|
"schema_version": "data_platform.ml_platform_checksums/v1"
|
package/package.json
CHANGED
package/release.json
CHANGED
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
"entrypoint": "src/cli.js",
|
|
14
14
|
"name": "ml-platform",
|
|
15
15
|
"path": "runtime/business-client",
|
|
16
|
-
"sha256": "sha256:
|
|
16
|
+
"sha256": "sha256:bf935c8c82aa508c26afa4f76dc94dd3d9e23c6e2bb33d797190d548a0c98249",
|
|
17
17
|
"version": "0.6.0"
|
|
18
18
|
},
|
|
19
19
|
"policy_sha256": "sha256:02fbde0134696b78c0c365d6e0b76c9814227208da5b2b3d4e5dbd82dc4a3606",
|
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
"ml_data_platform.feature_set/v1",
|
|
26
26
|
"ml_data_platform.dataset_manifest/v1"
|
|
27
27
|
],
|
|
28
|
-
"release_version": "0.1.
|
|
28
|
+
"release_version": "0.1.4",
|
|
29
29
|
"runtime_requirements": {
|
|
30
30
|
"node": ">=18",
|
|
31
31
|
"os": [
|
|
@@ -38,9 +38,9 @@
|
|
|
38
38
|
"feature-management": {
|
|
39
39
|
"path": "skills/feature-management",
|
|
40
40
|
"requires_cli": ">=0.6.0 <0.7.0",
|
|
41
|
-
"revision": "0.1.
|
|
42
|
-
"sha256": "sha256:
|
|
41
|
+
"revision": "0.1.4",
|
|
42
|
+
"sha256": "sha256:0ae088bcfbb4b9a0fcfa22752bd978b2a34ab033d63af4ebb4f641d3c2ac65fd"
|
|
43
43
|
}
|
|
44
44
|
},
|
|
45
|
-
"source_commit": "
|
|
45
|
+
"source_commit": "2f92b97d658f16ebc6f2081b07a56365aeced649"
|
|
46
46
|
}
|
|
@@ -60,6 +60,8 @@ function normalizeCodeArtifact(value) { object(value, "operator code_artifact");
|
|
|
60
60
|
function normalizeParameter(payload) {
|
|
61
61
|
exactKeys(payload, ["schema_version", "project", "name", "display_name", "version", "data_type", "unit", "expected_frequency", "source", "time_semantics", "availability_sla", "value_field", "quality_rules", "rounding", "owner"], ["name", "display_name", "version", "source"], "parameter");
|
|
62
62
|
const source = object(payload.source, "parameter source");
|
|
63
|
+
if (Object.prototype.hasOwnProperty.call(source, "parameters")) throw new Error("parameter source uses unsupported field 'parameters'; use 'params'");
|
|
64
|
+
if (source.mode === "sql" && typeof source.sql === "string" && source.sql.trimEnd().endsWith(";")) throw new Error("SQL Parameter source.sql must not end with a semicolon");
|
|
63
65
|
const time = object(defaultIfAbsent(payload, "time_semantics", {}), "parameter time_semantics");
|
|
64
66
|
return { schema_version: pythonString(defaultIfAbsent(payload, "schema_version", "ml_data_platform.parameter/v1")), project: identifier(defaultIfAbsent(payload, "project", "default"), "parameter project"), name: identifier(payload.name, "parameter name"), display_name: requiredString(payload.display_name, "parameter display_name"), version: identifier(payload.version, "parameter version"), data_type: pythonString(defaultIfAbsent(payload, "data_type", "time_series")), unit: pythonString(defaultIfAbsent(payload, "unit", "")), expected_frequency: pythonString(defaultIfAbsent(payload, "expected_frequency", "10min")), source: { adapter: pythonString(defaultIfAbsent(source, "adapter", "synthetic")), mode: choice(defaultIfAbsent(source, "mode", "direct_column"), ["direct_column", "sql", "derived"], "parameter source mode"), source_metric: optionalString(source.source_metric, "source_metric"), source_group: optionalString(source.source_group, "source_group"), measurement: optionalString(source.measurement, "measurement"), field: optionalString(source.field, "field"), tags: stringMap(defaultIfAbsent(source, "tags", {}), "source tags"), schema: optionalString(aliasIfAbsent(source, "schema", "schema_name"), "schema"), table: optionalString(source.table, "table"), time_column: optionalString(source.time_column, "time_column"), value_column: optionalString(source.value_column, "value_column"), metric_name_column: optionalString(source.metric_name_column, "metric_name_column"), metric_name: optionalString(source.metric_name, "metric_name"), unit_column: optionalString(source.unit_column, "unit_column"), unit: optionalString(aliasIfAbsent(source, "unit", "source_unit"), "source unit"), filters: stringMap(defaultIfAbsent(source, "filters", {}), "source filters"), sql: optionalString(source.sql, "source sql"), params: object(defaultIfAbsent(source, "params", {}), "source params") }, time_semantics: { event_time_field: pythonString(defaultIfAbsent(time, "event_time_field", "event_time")), ingested_at_field: pythonString(defaultIfAbsent(time, "ingested_at_field", "ingested_at")), timezone: pythonString(defaultIfAbsent(time, "timezone", "Asia/Shanghai")), availability: normalizeAvailability(time.availability) }, availability_sla: normalizeAvailabilitySla(payload.availability_sla), value_field: pythonString(defaultIfAbsent(payload, "value_field", "value")), quality_rules: normalizeQualityRules(object(defaultIfAbsent(payload, "quality_rules", {}), "parameter quality_rules")), rounding: normalizeRounding(payload.rounding), owner: pythonString(defaultIfAbsent(payload, "owner", "demo")) };
|
|
65
67
|
}
|
|
@@ -165,7 +167,7 @@ function requiresPackage(spec) { return ["custom_python", "feature", "spark_udf"
|
|
|
165
167
|
function validateCatalog(catalog, current) {
|
|
166
168
|
const parameters = new Set([...current.parameters.keys(), ...catalog.parameters.map(key)]); const operators = new Map([...current.operators, ...catalog.operators.map((x) => [key(x.spec), x.spec])]); const features = new Map([...current.features, ...catalog.features.map((x) => [key(x), x])]); const featureSets = new Map([...current.feature_sets, ...catalog.feature_sets.map((x) => [key(x), x])]);
|
|
167
169
|
for (const item of catalog.operators) if (!item.package_path && requiresPackage(item.spec)) throw new Error(`catalog operator ${key(item.spec)} requires a wheel package path`);
|
|
168
|
-
for (const feature of catalog.features) { for (const input of feature.inputs) if (!parameters.has(parameterReferenceKey(input))) throw new Error(`catalog feature ${key(feature)} references missing parameter ${parameterReferenceKey(input)}`); const operatorKey = `${feature.operator}:${feature.operator_version}`; const operator = operators.get(operatorKey); if (!operator) throw new Error(`catalog feature ${key(feature)} references missing operator ${operatorKey}`); if (operator.type !== "feature") throw new Error(`catalog feature ${key(feature)} requires OperatorSpec.type='feature'`); const declaredInputs = new Set(operator.input_schema.parameters ?? []); if (declaredInputs.size) for (const input of feature.inputs) if (!declaredInputs.has(
|
|
170
|
+
for (const feature of catalog.features) { for (const input of feature.inputs) if (!parameters.has(parameterReferenceKey(input))) throw new Error(`catalog feature ${key(feature)} references missing parameter ${parameterReferenceKey(input)}`); const operatorKey = `${feature.operator}:${feature.operator_version}`; const operator = operators.get(operatorKey); if (!operator) throw new Error(`catalog feature ${key(feature)} references missing operator ${operatorKey}`); if (operator.type !== "feature") throw new Error(`catalog feature ${key(feature)} requires OperatorSpec.type='feature'`); const declaredInputs = new Set(operator.input_schema.parameters ?? []); if (declaredInputs.size) for (const input of feature.inputs) { const inputKey = parameterReferenceKey(input); const qualifiedInputKey = `${operator.project ?? feature.project ?? "default"}/${inputKey}`; if (!declaredInputs.has(inputKey) && !declaredInputs.has(qualifiedInputKey)) throw new Error(`catalog feature ${key(feature)} input is not declared by ${operatorKey}`); } const columns = operator.output_schema.columns ?? []; if (columns.length && !columns.includes(feature.output_column)) throw new Error(`catalog feature ${key(feature)} output_column is not declared by ${operatorKey}`); }
|
|
169
171
|
for (const set of catalog.feature_sets) { const names = []; for (const item of set.features) { const feature = features.get(referenceKey(item)); if (!feature) throw new Error(`catalog feature set ${key(set)} references missing feature ${referenceKey(item)}`); names.push(feature.name); } unique(names, `catalog feature set ${key(set)} final output names`); }
|
|
170
172
|
for (const dataset of catalog.datasets) {
|
|
171
173
|
const datasetId = datasetKey(dataset); const declared = new Set();
|
|
@@ -14,7 +14,10 @@ const BUSINESS_COMMANDS = new Set([
|
|
|
14
14
|
"configure", "show-config", "health", "list-parameters", "list-operators", "list-features",
|
|
15
15
|
"list-feature-sets", "list-datasets", "get-dataset", "list-jobs", "resolve-manifest",
|
|
16
16
|
"resolve-dataset", "build-dataset", "build-registered-dataset",
|
|
17
|
-
"
|
|
17
|
+
"add-parameter", "add-feature", "add-feature-set", "add-operator", "publish-operator",
|
|
18
|
+
"add-dataset", "update-dataset", "delete-dataset", "list-dataset-artifacts",
|
|
19
|
+
"register-operator", "show-operator-specs", "delete-operator", "run-operator",
|
|
20
|
+
"get-job", "wait-job", "retry-job", "cancel-job", "get-dataset-artifact", "download-dataset-artifact", "fetch-inference-data", "apply",
|
|
18
21
|
]);
|
|
19
22
|
|
|
20
23
|
const COMMAND_USAGE = {
|
|
@@ -30,15 +33,30 @@ const COMMAND_USAGE = {
|
|
|
30
33
|
"list-operators": "list-operators [--project PROJECT] [-q QUERY] [--limit N] [--offset N]",
|
|
31
34
|
"list-features": "list-features [--project PROJECT] [-q QUERY] [--limit N] [--offset N]",
|
|
32
35
|
"list-feature-sets": "list-feature-sets [--project PROJECT] [-q QUERY] [--limit N] [--offset N]",
|
|
33
|
-
"list-datasets": "list-datasets [--project PROJECT] [-q QUERY] [--limit N] [--offset N]",
|
|
36
|
+
"list-datasets": "list-datasets [--project PROJECT] [-q QUERY] [--limit N] [--offset N] [--include-deleted]",
|
|
37
|
+
"add-parameter": "add-parameter SPEC_JSON",
|
|
38
|
+
"add-feature": "add-feature SPEC_JSON",
|
|
39
|
+
"add-feature-set": "add-feature-set SPEC_JSON",
|
|
40
|
+
"add-operator": "add-operator SPEC_JSON",
|
|
41
|
+
"publish-operator": "publish-operator SPEC_JSON --package PATH",
|
|
42
|
+
"add-dataset": "add-dataset MANIFEST_JSON [--force]",
|
|
43
|
+
"update-dataset": "update-dataset MANIFEST_JSON [--force]",
|
|
44
|
+
"delete-dataset": "delete-dataset DATASET_ID DATASET_VERSION [--project PROJECT]",
|
|
45
|
+
"list-dataset-artifacts": "list-dataset-artifacts [--project PROJECT] [--dataset-id ID] [--dataset-version VERSION] [--include-deleted] [--limit N] [--offset N]",
|
|
34
46
|
"get-dataset": "get-dataset DATASET_ID DATASET_VERSION [--project PROJECT] [--include-deleted]",
|
|
35
47
|
"list-jobs": "list-jobs [-q QUERY] [--limit N] [--offset N]",
|
|
36
48
|
"resolve-manifest": "resolve-manifest MANIFEST [--out PATH]",
|
|
37
49
|
"resolve-dataset": "resolve-dataset DATASET_ID DATASET_VERSION [--project PROJECT] [--out PATH]",
|
|
38
|
-
"build-dataset": "build-dataset MANIFEST [--source-mode direct] [--upload-chronon-metadata] [--partition-duration DURATION] [--max-parallelism N] [--no-resume] [--wait] [--poll-interval SECONDS] [--wait-timeout-seconds SECONDS]",
|
|
39
|
-
"build-registered-dataset": "build-registered-dataset DATASET_ID DATASET_VERSION [--project PROJECT] [--source-mode direct] [--upload-chronon-metadata] [--partition-duration DURATION] [--max-parallelism N] [--no-resume] [--wait] [--poll-interval SECONDS] [--wait-timeout-seconds SECONDS]",
|
|
50
|
+
"build-dataset": "build-dataset MANIFEST [--engine chronon] [--source-mode direct] [--upload-chronon-metadata] [--partition-duration DURATION] [--max-parallelism N] [--no-resume] [--wait] [--poll-interval SECONDS] [--wait-timeout-seconds SECONDS]",
|
|
51
|
+
"build-registered-dataset": "build-registered-dataset DATASET_ID DATASET_VERSION [--project PROJECT] [--engine chronon] [--source-mode direct] [--upload-chronon-metadata] [--partition-duration DURATION] [--max-parallelism N] [--no-resume] [--wait] [--poll-interval SECONDS] [--wait-timeout-seconds SECONDS]",
|
|
40
52
|
"get-job": "get-job JOB_ID",
|
|
41
53
|
"wait-job": "wait-job JOB_ID [--poll-interval SECONDS] [--wait-timeout-seconds SECONDS]",
|
|
54
|
+
"retry-job": "retry-job JOB_ID",
|
|
55
|
+
"cancel-job": "cancel-job JOB_ID",
|
|
56
|
+
"register-operator": "register-operator SPEC_JSON",
|
|
57
|
+
"show-operator-specs": "show-operator-specs [--project PROJECT] [-q QUERY] [--limit N] [--offset N]",
|
|
58
|
+
"delete-operator": "delete-operator NAME VERSION [--project PROJECT]",
|
|
59
|
+
"run-operator": "run-operator NAME VERSION [--project PROJECT] [--config JSON]",
|
|
42
60
|
"get-dataset-artifact": "get-dataset-artifact DATASET_ID MANIFEST_HASH [--project PROJECT]",
|
|
43
61
|
"download-dataset-artifact": "download-dataset-artifact DATASET_ID MANIFEST_HASH [--project PROJECT] --out-dir PATH [--force]",
|
|
44
62
|
"fetch-inference-data": "fetch-inference-data MANIFEST --cutoff-time TIMESTAMP [--max-workers N] [--max-source-lag-hours HOURS] [--no-validate-freshness] [--allow-missing]",
|
|
@@ -52,7 +70,10 @@ function usage(command) {
|
|
|
52
70
|
}
|
|
53
71
|
|
|
54
72
|
function parse(args) {
|
|
55
|
-
const
|
|
73
|
+
const timeoutValue = process.env.ML_PLATFORM_API_TIMEOUT_SECONDS
|
|
74
|
+
|| process.env.DATA_PLATFORM_DEMO_API_TIMEOUT_SECONDS
|
|
75
|
+
|| 30;
|
|
76
|
+
const options = { profile: "server", timeout: finiteNumber(timeoutValue, "--request-timeout-seconds") };
|
|
56
77
|
let index = 0;
|
|
57
78
|
while (index < args.length && args[index].startsWith("--")) {
|
|
58
79
|
const flag = args[index++];
|
|
@@ -66,7 +87,7 @@ function parse(args) {
|
|
|
66
87
|
if (!(options.timeout > 0)) throw new Error("--request-timeout-seconds must be positive");
|
|
67
88
|
options.command = args[index++];
|
|
68
89
|
options.rest = args.slice(index);
|
|
69
|
-
options.apiUrl ??= process.env.ML_PLATFORM_API_URL || loadApiUrl();
|
|
90
|
+
options.apiUrl ??= process.env.ML_PLATFORM_API_URL || process.env.DATA_PLATFORM_DEMO_API_URL || loadApiUrl();
|
|
70
91
|
return options;
|
|
71
92
|
}
|
|
72
93
|
|
|
@@ -105,7 +126,7 @@ function positional(rest, label) { const value = rest.shift(); if (!value || val
|
|
|
105
126
|
|
|
106
127
|
function client(options) {
|
|
107
128
|
if (options.profile !== "server") throw new Error("the business client supports only --profile server");
|
|
108
|
-
if (!options.apiUrl) throw new Error("server profile requires --api-url, ML_PLATFORM_API_URL, or a saved configure target");
|
|
129
|
+
if (!options.apiUrl) throw new Error("server profile requires --api-url, ML_PLATFORM_API_URL, DATA_PLATFORM_DEMO_API_URL, or a saved configure target");
|
|
109
130
|
return new PlatformApiClient(options.apiUrl, options.timeout);
|
|
110
131
|
}
|
|
111
132
|
|
|
@@ -116,6 +137,7 @@ function readObject(file) {
|
|
|
116
137
|
}
|
|
117
138
|
|
|
118
139
|
function readDataset(file) { return normalizeDataset(readObject(file)); }
|
|
140
|
+
function readSpec(file) { return readObject(file); }
|
|
119
141
|
|
|
120
142
|
function datasetEndpoint(project, datasetId, datasetVersion) {
|
|
121
143
|
return `/datasets/${encodeURIComponent(project)}/${encodeURIComponent(datasetId)}/${encodeURIComponent(datasetVersion)}`;
|
|
@@ -128,6 +150,8 @@ async function registeredDataset(api, project, datasetId, datasetVersion, includ
|
|
|
128
150
|
}
|
|
129
151
|
|
|
130
152
|
function buildOptions(rest) {
|
|
153
|
+
const engine = take(rest, "--engine", "chronon");
|
|
154
|
+
if (engine !== "chronon") throw new Error("--engine must be chronon");
|
|
131
155
|
const sourceMode = take(rest, "--source-mode");
|
|
132
156
|
if (sourceMode !== undefined && sourceMode !== "direct") throw new Error("--source-mode must be direct");
|
|
133
157
|
const maxParallelism = integer(take(rest, "--max-parallelism", 1), "--max-parallelism");
|
|
@@ -139,6 +163,7 @@ function buildOptions(rest) {
|
|
|
139
163
|
const waitTimeout = rawTimeout === undefined ? undefined : finiteNumber(rawTimeout, "--wait-timeout-seconds");
|
|
140
164
|
if (waitTimeout !== undefined && !(waitTimeout > 0)) throw new Error("--wait-timeout-seconds must be positive");
|
|
141
165
|
return {
|
|
166
|
+
engine,
|
|
142
167
|
sourceMode,
|
|
143
168
|
upload: boolean(rest, "--upload-chronon-metadata"),
|
|
144
169
|
partitionDuration: take(rest, "--partition-duration"),
|
|
@@ -151,7 +176,7 @@ function buildOptions(rest) {
|
|
|
151
176
|
}
|
|
152
177
|
|
|
153
178
|
async function submitBuild(api, manifest, build) {
|
|
154
|
-
let result = await api.post("/datasets/build", manifest, { engine:
|
|
179
|
+
let result = await api.post("/datasets/build", manifest, { engine: build.engine, source_mode: build.sourceMode, upload_chronon_metadata: build.upload, partition_duration: build.partitionDuration, max_parallelism: build.maxParallelism, resume: build.resume });
|
|
155
180
|
let exitCode = 0;
|
|
156
181
|
if (build.wait) {
|
|
157
182
|
if (!result || typeof result !== "object" || !result.job_id) throw new Error("server build did not return a job_id to wait for");
|
|
@@ -205,16 +230,28 @@ async function runBusinessCli(argv) {
|
|
|
205
230
|
} else if (options.command === "list-projects") result = await client(options).get("/projects");
|
|
206
231
|
else if (options.command === "get-project") result = await client(options).get(`/projects/${encodeURIComponent(positional(rest, "project name"))}`);
|
|
207
232
|
else if (options.command === "delete-project") result = await client(options).delete(`/projects/${encodeURIComponent(positional(rest, "project name"))}`);
|
|
208
|
-
else if (
|
|
233
|
+
else if (["add-parameter", "add-feature", "add-feature-set", "add-operator", "register-operator"].includes(options.command)) {
|
|
234
|
+
const spec = readSpec(positional(rest, "spec JSON"));
|
|
235
|
+
const endpoint = { "add-parameter": "/parameters", "add-feature": "/features", "add-feature-set": "/feature-sets", "add-operator": "/operators", "register-operator": "/operators" }[options.command];
|
|
236
|
+
result = await client(options).post(endpoint, spec);
|
|
237
|
+
} else if (options.command === "publish-operator") {
|
|
238
|
+
const specPath = positional(rest, "OperatorSpec JSON"); const packagePath = take(rest, "--package");
|
|
239
|
+
if (!packagePath) throw new Error("--package is required");
|
|
240
|
+
const spec = readSpec(specPath); const api = client(options); const project = spec.project || "default";
|
|
241
|
+
const upload = await api.uploadOperatorPackage(project, spec.name, spec.version, packagePath);
|
|
242
|
+
if (!upload || typeof upload.code_artifact !== "object") throw new Error("operator package upload response has no code_artifact");
|
|
243
|
+
const published = { ...spec, code_hash: upload.code_artifact.sha256, package_uri: upload.code_artifact.uri, code_artifact: upload.code_artifact };
|
|
244
|
+
result = { status: "published", profile: "server", path: null, operator: published, registration: await api.post("/operators", published) };
|
|
245
|
+
} else if (options.command === "get-dataset") {
|
|
209
246
|
const datasetId = positional(rest, "dataset id"); const datasetVersion = positional(rest, "dataset version"); const project = take(rest, "--project", "default");
|
|
210
247
|
result = await registeredDataset(client(options), project, datasetId, datasetVersion, boolean(rest, "--include-deleted"));
|
|
211
248
|
}
|
|
212
|
-
else if (options.command.startsWith("list-") &&
|
|
249
|
+
else if (options.command.startsWith("list-") && !["list-jobs", "list-dataset-artifacts"].includes(options.command)) {
|
|
213
250
|
const endpoints = { "list-parameters": "/parameters", "list-operators": "/operators", "list-features": "/features", "list-feature-sets": "/feature-sets", "list-datasets": "/datasets" };
|
|
214
251
|
const project = take(rest, "--project");
|
|
215
|
-
const query = take(rest, "--q", take(rest, "-q")); const rawLimit = take(rest, "--limit"); const rawOffset = take(rest, "--offset");
|
|
216
|
-
if (rawLimit !== undefined || rawOffset !== undefined) { const limit = Number(rawLimit ?? 50); const offset = Number(rawOffset ?? 0); if (!Number.isInteger(limit) || limit < 1 || limit > 500) throw new Error("limit must be between 1 and 500"); if (!Number.isInteger(offset) || offset < 0) throw new Error("offset must be >= 0"); result = await client(options).get(endpoints[options.command], { q: query, limit, offset, project }); }
|
|
217
|
-
else result = await fetchAll(client(options), endpoints[options.command], { q: query, project });
|
|
252
|
+
const query = take(rest, "--q", take(rest, "-q")); const includeDeleted = options.command === "list-datasets" && boolean(rest, "--include-deleted"); const rawLimit = take(rest, "--limit"); const rawOffset = take(rest, "--offset");
|
|
253
|
+
if (rawLimit !== undefined || rawOffset !== undefined) { const limit = Number(rawLimit ?? 50); const offset = Number(rawOffset ?? 0); if (!Number.isInteger(limit) || limit < 1 || limit > 500) throw new Error("limit must be between 1 and 500"); if (!Number.isInteger(offset) || offset < 0) throw new Error("offset must be >= 0"); result = await client(options).get(endpoints[options.command], { q: query, limit, offset, project, include_deleted: includeDeleted }); }
|
|
254
|
+
else result = await fetchAll(client(options), endpoints[options.command], { q: query, project, include_deleted: includeDeleted });
|
|
218
255
|
} else if (options.command === "list-jobs") {
|
|
219
256
|
const query = take(rest, "--q", take(rest, "-q")); const rawLimit = take(rest, "--limit"); const rawOffset = take(rest, "--offset");
|
|
220
257
|
if (rawLimit !== undefined || rawOffset !== undefined) { const limit = Number(rawLimit ?? 50); const offset = Number(rawOffset ?? 0); if (!Number.isInteger(limit) || limit < 1 || limit > 500) throw new Error("limit must be between 1 and 500"); if (!Number.isInteger(offset) || offset < 0) throw new Error("offset must be >= 0"); result = await client(options).get("/jobs", { q: query, limit, offset }); }
|
|
@@ -239,12 +276,38 @@ async function runBusinessCli(argv) {
|
|
|
239
276
|
const datasetId = positional(rest, "dataset id"); const datasetVersion = positional(rest, "dataset version"); const project = take(rest, "--project", "default"); const build = buildOptions(rest); const api = client(options);
|
|
240
277
|
const manifest = normalizeRegistryDataset(await registeredDataset(api, project, datasetId, datasetVersion));
|
|
241
278
|
({ result, exitCode } = await submitBuild(api, manifest, build));
|
|
279
|
+
} else if (["add-dataset", "update-dataset"].includes(options.command)) {
|
|
280
|
+
const manifest = readDataset(positional(rest, "manifest JSON")); const force = boolean(rest, "--force"); const project = manifest.project || "default";
|
|
281
|
+
if (options.command === "add-dataset") result = await client(options).post("/datasets", manifest, { force });
|
|
282
|
+
else result = await client(options).put(`/datasets/${encodeURIComponent(project)}/${encodeURIComponent(manifest.dataset_id)}/${encodeURIComponent(manifest.dataset_version)}`, manifest, { force });
|
|
283
|
+
} else if (options.command === "delete-dataset") {
|
|
284
|
+
const id = positional(rest, "dataset id"); const version = positional(rest, "dataset version"); const project = take(rest, "--project", "default");
|
|
285
|
+
result = await client(options).delete(`/datasets/${encodeURIComponent(project)}/${encodeURIComponent(id)}/${encodeURIComponent(version)}`);
|
|
286
|
+
} else if (options.command === "list-dataset-artifacts") {
|
|
287
|
+
const project = take(rest, "--project"); const datasetId = take(rest, "--dataset-id"); const datasetVersion = take(rest, "--dataset-version"); const includeDeleted = boolean(rest, "--include-deleted");
|
|
288
|
+
result = await listCollection(client(options), "/dataset-artifacts", { project, dataset_id: datasetId, dataset_version: datasetVersion, include_deleted: includeDeleted }, rest);
|
|
289
|
+
} else if (options.command === "show-operator-specs") {
|
|
290
|
+
const project = take(rest, "--project"); const query = take(rest, "--q", take(rest, "-q"));
|
|
291
|
+
result = await listCollection(client(options), "/operators", { project, q: query }, rest);
|
|
292
|
+
} else if (options.command === "delete-operator") {
|
|
293
|
+
const name = positional(rest, "operator name"); const version = positional(rest, "operator version"); const project = take(rest, "--project", "default");
|
|
294
|
+
result = await client(options).delete(`/operators/${encodeURIComponent(project)}/${encodeURIComponent(name)}/${encodeURIComponent(version)}`);
|
|
295
|
+
} else if (options.command === "run-operator") {
|
|
296
|
+
const name = positional(rest, "operator name"); const version = positional(rest, "operator version"); const project = take(rest, "--project", "default"); const rawConfig = take(rest, "--config", "{}");
|
|
297
|
+
let config; try { config = JSON.parse(rawConfig); } catch (_) { throw new Error("--config must be valid JSON"); }
|
|
298
|
+
result = await client(options).post(`/operators/${encodeURIComponent(project)}/${encodeURIComponent(name)}/${encodeURIComponent(version)}/run`, config);
|
|
242
299
|
} else if (options.command === "get-job") result = await client(options).get(`/jobs/${encodeURIComponent(positional(rest, "job id"))}`);
|
|
243
300
|
else if (options.command === "wait-job") {
|
|
244
301
|
const jobId = positional(rest, "job id"); const poll = finiteNumber(take(rest, "--poll-interval", 2), "--poll-interval"); const rawTimeout = take(rest, "--wait-timeout-seconds");
|
|
245
302
|
result = await waitForJob(client(options), jobId, poll, rawTimeout === undefined ? undefined : finiteNumber(rawTimeout, "--wait-timeout-seconds"));
|
|
246
303
|
if (["failed", "cancelled"].includes(result.status)) exitCode = 1;
|
|
247
304
|
}
|
|
305
|
+
else if (options.command === "cancel-job") {
|
|
306
|
+
result = await client(options).post(`/jobs/${encodeURIComponent(positional(rest, "job id"))}/cancel`);
|
|
307
|
+
}
|
|
308
|
+
else if (options.command === "retry-job") {
|
|
309
|
+
result = await client(options).post(`/jobs/${encodeURIComponent(positional(rest, "job id"))}/retry`);
|
|
310
|
+
}
|
|
248
311
|
else if (options.command === "get-dataset-artifact") {
|
|
249
312
|
const dataset = positional(rest, "dataset id"); const hash = positional(rest, "manifest hash"); const project = take(rest, "--project", "default");
|
|
250
313
|
result = await client(options).get(`/dataset-artifacts/${encodeURIComponent(project)}/${encodeURIComponent(dataset)}/${encodeURIComponent(hash)}`);
|
|
@@ -267,7 +330,16 @@ async function runBusinessCli(argv) {
|
|
|
267
330
|
|
|
268
331
|
async function fetchAll(api, endpoint, params = {}) { const items = []; for (let offset = 0; ; offset += 500) { const page = await api.get(endpoint, { ...params, limit: 500, offset }); if (Array.isArray(page)) return page; if (!page || !Array.isArray(page.items)) throw new Error(`${endpoint} returned an invalid paginated response`); items.push(...page.items); if (page.total == null || items.length >= page.total || page.items.length < 500) return items; } }
|
|
269
332
|
|
|
270
|
-
|
|
333
|
+
async function listCollection(api, endpoint, params, rest) {
|
|
334
|
+
const rawLimit = take(rest, "--limit"); const rawOffset = take(rest, "--offset");
|
|
335
|
+
if (rawLimit === undefined && rawOffset === undefined) return fetchAll(api, endpoint, params);
|
|
336
|
+
const limit = Number(rawLimit ?? 50); const offset = Number(rawOffset ?? 0);
|
|
337
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > 500) throw new Error("limit must be between 1 and 500");
|
|
338
|
+
if (!Number.isInteger(offset) || offset < 0) throw new Error("offset must be >= 0");
|
|
339
|
+
return api.get(endpoint, { ...params, limit, offset });
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
module.exports = { BUSINESS_COMMANDS, fetchAll, listCollection, parse, runBusinessCli, usage, waitForJob };
|
|
271
343
|
|
|
272
344
|
if (require.main === module) {
|
|
273
345
|
runBusinessCli(process.argv.slice(2)).then((code) => { process.exitCode = code; }).catch((error) => {
|
|
@@ -11,11 +11,15 @@ function expandHome(value) {
|
|
|
11
11
|
}
|
|
12
12
|
|
|
13
13
|
function configPath() {
|
|
14
|
-
|
|
14
|
+
for (const variable of ["ML_PLATFORM_CONFIG_PATH", "DATA_PLATFORM_DEMO_CONFIG_PATH"]) {
|
|
15
|
+
if (process.env[variable]?.trim()) return path.resolve(expandHome(process.env[variable]));
|
|
16
|
+
}
|
|
15
17
|
const base = process.env.XDG_CONFIG_HOME
|
|
16
18
|
? path.resolve(process.env.XDG_CONFIG_HOME)
|
|
17
19
|
: path.join(os.homedir(), ".config");
|
|
18
|
-
|
|
20
|
+
const primary = path.join(base, "ml-platform", "config.json");
|
|
21
|
+
const legacy = path.join(base, "data-platform-demo", "config.json");
|
|
22
|
+
return !fs.existsSync(primary) && fs.existsSync(legacy) ? legacy : primary;
|
|
19
23
|
}
|
|
20
24
|
|
|
21
25
|
function normalizeApiUrl(value) {
|
|
@@ -45,6 +45,7 @@ class PlatformApiClient {
|
|
|
45
45
|
|
|
46
46
|
get(endpoint, params) { return this.request("GET", endpoint, { params }); }
|
|
47
47
|
post(endpoint, payload, params) { return this.request("POST", endpoint, { payload, params }); }
|
|
48
|
+
put(endpoint, payload, params) { return this.request("PUT", endpoint, { payload, params }); }
|
|
48
49
|
delete(endpoint, params) { return this.request("DELETE", endpoint, { params }); }
|
|
49
50
|
uploadOperatorPackage(project, name, version, packagePath) {
|
|
50
51
|
const resolved = path.resolve(expandHome(packagePath));
|
|
@@ -63,33 +64,95 @@ class PlatformApiClient {
|
|
|
63
64
|
fs.mkdirSync(target, { recursive: true });
|
|
64
65
|
const controller = new AbortController();
|
|
65
66
|
const timer = setTimeout(() => controller.abort(), this.timeoutMs);
|
|
67
|
+
const temporaryDir = fs.mkdtempSync(path.join(os.tmpdir(), "data-platform-artifact-download-"));
|
|
68
|
+
const temporary = path.join(temporaryDir, "artifact.zip");
|
|
66
69
|
let response;
|
|
67
70
|
try {
|
|
68
71
|
response = await fetch(this.url(`/dataset-artifacts/${encodeURIComponent(project)}/${encodeURIComponent(datasetId)}/${encodeURIComponent(manifestHash)}/download`), { headers: { Accept: "application/zip" }, signal: controller.signal });
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
let
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
72
|
+
if (!response.ok) {
|
|
73
|
+
let detail = response.statusText;
|
|
74
|
+
try { detail = JSON.parse(await abortable(response.text(), controller.signal))?.detail ?? detail; } catch (_) { /* use status text */ }
|
|
75
|
+
throw new Error(`artifact download failed: HTTP ${response.status}: ${detail}`);
|
|
76
|
+
}
|
|
77
|
+
const total = contentLength(response);
|
|
78
|
+
let lastReported = -1;
|
|
79
|
+
const report = (received, complete = false) => {
|
|
80
|
+
if ((complete && received !== lastReported) || received === 0 || received - lastReported >= 1024 * 1024) {
|
|
81
|
+
reportDownloadProgress(received, total);
|
|
82
|
+
lastReported = received;
|
|
83
|
+
}
|
|
84
|
+
};
|
|
85
|
+
report(0);
|
|
86
|
+
await writeResponseBody(response, temporary, controller.signal, report);
|
|
82
87
|
const files = await extractArchive(temporary, target);
|
|
83
88
|
return { dataset_id: datasetId, manifest_hash: manifestHash, output_dir: target, file_count: files.length, files };
|
|
84
89
|
} catch (error) {
|
|
85
90
|
fs.rmSync(target, { recursive: true, force: true });
|
|
86
|
-
throw error;
|
|
91
|
+
if (error instanceof Error && error.message.startsWith("artifact download failed:")) throw error;
|
|
92
|
+
throw new Error(`artifact download failed: ${error.message}`);
|
|
87
93
|
} finally {
|
|
88
|
-
|
|
94
|
+
clearTimeout(timer);
|
|
95
|
+
fs.rmSync(temporaryDir, { recursive: true, force: true });
|
|
89
96
|
}
|
|
90
97
|
}
|
|
91
98
|
}
|
|
92
99
|
|
|
100
|
+
function contentLength(response) {
|
|
101
|
+
const raw = response.headers?.get?.("content-length") ?? response.headers?.["content-length"];
|
|
102
|
+
const value = Number(raw);
|
|
103
|
+
return Number.isFinite(value) && value >= 0 ? value : null;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function reportDownloadProgress(received, total) {
|
|
107
|
+
const suffix = total == null ? "" : `/${total}`;
|
|
108
|
+
process.stderr.write(`artifact download: received ${received}${suffix} bytes\n`);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
function abortable(value, signal) {
|
|
112
|
+
if (signal.aborted) return Promise.reject(new Error("This operation was aborted"));
|
|
113
|
+
return new Promise((resolve, reject) => {
|
|
114
|
+
const onAbort = () => { cleanup(); reject(new Error("This operation was aborted")); };
|
|
115
|
+
const cleanup = () => signal.removeEventListener("abort", onAbort);
|
|
116
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
117
|
+
Promise.resolve(value).then((result) => { cleanup(); resolve(result); }, (error) => { cleanup(); reject(error); });
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
async function writeResponseBody(response, outputPath, signal, onProgress) {
|
|
122
|
+
const handle = fs.openSync(outputPath, "w");
|
|
123
|
+
let received = 0;
|
|
124
|
+
try {
|
|
125
|
+
const write = (value) => {
|
|
126
|
+
const chunk = Buffer.from(value);
|
|
127
|
+
fs.writeSync(handle, chunk);
|
|
128
|
+
received += chunk.length;
|
|
129
|
+
onProgress(received);
|
|
130
|
+
};
|
|
131
|
+
if (response.body?.getReader) {
|
|
132
|
+
const reader = response.body.getReader();
|
|
133
|
+
while (true) {
|
|
134
|
+
const item = await abortable(reader.read(), signal);
|
|
135
|
+
if (item.done) break;
|
|
136
|
+
write(item.value);
|
|
137
|
+
}
|
|
138
|
+
} else if (response.body?.[Symbol.asyncIterator]) {
|
|
139
|
+
const iterator = response.body[Symbol.asyncIterator]();
|
|
140
|
+
while (true) {
|
|
141
|
+
const item = await abortable(iterator.next(), signal);
|
|
142
|
+
if (item.done) break;
|
|
143
|
+
write(item.value);
|
|
144
|
+
}
|
|
145
|
+
} else if (typeof response.arrayBuffer === "function") {
|
|
146
|
+
write(await abortable(response.arrayBuffer(), signal));
|
|
147
|
+
} else {
|
|
148
|
+
throw new Error("artifact download response has no readable body");
|
|
149
|
+
}
|
|
150
|
+
onProgress(received, true);
|
|
151
|
+
} finally {
|
|
152
|
+
fs.closeSync(handle);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
|
|
93
156
|
async function extractArchive(archivePath, outputDir) {
|
|
94
157
|
const archive = fs.readFileSync(archivePath);
|
|
95
158
|
const end = findEndOfCentralDirectory(archive);
|
|
@@ -111,8 +174,8 @@ async function extractArchive(archivePath, outputDir) {
|
|
|
111
174
|
if (next > archive.length || compressedSize === 0xffffffff || size === 0xffffffff || localOffset === 0xffffffff) throw new Error("artifact archive entry metadata is invalid");
|
|
112
175
|
if (flags & 1) throw new Error("artifact archive contains an encrypted entry");
|
|
113
176
|
if (![0, 8].includes(method)) throw new Error(`artifact archive compression method is unsupported: ${method}`);
|
|
114
|
-
|
|
115
|
-
const rawName =
|
|
177
|
+
const rawNameBytes = archive.subarray(cursor + 46, cursor + 46 + nameLength);
|
|
178
|
+
const rawName = decodeZipFilename(rawNameBytes, flags);
|
|
116
179
|
const name = rawName.replace(/\\/g, "/"); const parts = name.split("/"); const mode = (attributes >>> 16) & 0xffff;
|
|
117
180
|
if (!name || name.startsWith("/") || /^[A-Za-z]:/.test(name) || parts.includes("..")) throw new Error(`artifact archive contains unsafe path: ${JSON.stringify(rawName)}`);
|
|
118
181
|
if ((mode & 0o170000) === 0o120000) throw new Error(`artifact archive contains a symbolic link: ${JSON.stringify(rawName)}`);
|
|
@@ -131,6 +194,14 @@ async function extractArchive(archivePath, outputDir) {
|
|
|
131
194
|
return files.sort();
|
|
132
195
|
}
|
|
133
196
|
|
|
197
|
+
function decodeZipFilename(bytes, flags) {
|
|
198
|
+
if (flags & 0x800) return bytes.toString("utf8");
|
|
199
|
+
// ZIP producers commonly omit the UTF-8 flag for ASCII entry names. Keep
|
|
200
|
+
// those names interoperable while refusing ambiguous non-ASCII encodings.
|
|
201
|
+
if (bytes.some((value) => value > 0x7f)) throw new Error("artifact archive contains an unsupported non-UTF-8 filename");
|
|
202
|
+
return bytes.toString("ascii");
|
|
203
|
+
}
|
|
204
|
+
|
|
134
205
|
function findEndOfCentralDirectory(archive) { const minimum = Math.max(0, archive.length - 65557); for (let offset = archive.length - 22; offset >= minimum; offset -= 1) if (archive.readUInt32LE(offset) === 0x06054b50 && offset + 22 + archive.readUInt16LE(offset + 20) === archive.length) return offset; throw new Error("artifact download is not a valid zip archive"); }
|
|
135
206
|
function crc32(buffer) { let crc = 0xffffffff; for (const byte of buffer) { crc ^= byte; for (let bit = 0; bit < 8; bit += 1) crc = (crc >>> 1) ^ (0xedb88320 & -(crc & 1)); } return (crc ^ 0xffffffff) >>> 0; }
|
|
136
207
|
|
|
@@ -19,8 +19,9 @@ Treat `Recipe` and public `Feature.compute` as removed. Treat `computation_hash`
|
|
|
19
19
|
1. Resolve the Skill root as the directory containing this `SKILL.md`.
|
|
20
20
|
2. Read [references/contracts.md](references/contracts.md) before creating or changing JSON assets.
|
|
21
21
|
3. Read [references/operator-authoring.md](references/operator-authoring.md) whenever creating or changing Operator code or a wheel.
|
|
22
|
-
4. Read [references/commands.md](references/commands.md) before
|
|
22
|
+
4. Read [references/commands.md](references/commands.md) immediately before a platform CLI action: discovery, validation, publication, build, realtime fetch, or artifact download. A local-only draft that is forbidden to call the CLI does not need this reference.
|
|
23
23
|
5. Use [assets/catalog-template](assets/catalog-template) as a copyable starting point for a new end-to-end catalog. Rename every `example_*` identifier and update every referenced path before validation.
|
|
24
|
+
6. Read [references/platform-capability-guide.md](references/platform-capability-guide.md) only when selecting an asset type or explaining a platform capability. Do not load it merely to write a confirmed local draft; it is never evidence for unresolved business semantics.
|
|
24
25
|
|
|
25
26
|
## Initialize The Client
|
|
26
27
|
|
|
@@ -46,6 +47,19 @@ ml-platform show-config
|
|
|
46
47
|
ml-platform --profile server health
|
|
47
48
|
```
|
|
48
49
|
|
|
50
|
+
Use the global `--profile server` form shown above, before the business
|
|
51
|
+
command. Do not append `--profile` after a business command. Before discovery,
|
|
52
|
+
check the installed release with `ml-platform --help` and the relevant
|
|
53
|
+
subcommand's `--help`; CLI releases may differ in supported flags. In the
|
|
54
|
+
deployed 0.6.x CLI, registry `list-*` commands emit JSON by default and do not
|
|
55
|
+
accept a `--json` flag.
|
|
56
|
+
|
|
57
|
+
If the installed release rejects the global `--profile server` form shown above,
|
|
58
|
+
record the CLI syntax error and inspect `show-config`/subcommand help before
|
|
59
|
+
retrying the same read-only command without that flag when supported. Do not
|
|
60
|
+
switch clients or classify a syntax error as an API outage; keep each command
|
|
61
|
+
separate so CLI compatibility remains distinguishable from network health.
|
|
62
|
+
|
|
49
63
|
The effective API target is resolved in this order: explicit `--api-url`,
|
|
50
64
|
`ML_PLATFORM_API_URL`, then the saved value written by `ml-platform configure`.
|
|
51
65
|
If no target is configured, ask the user for an approved API URL before making
|
|
@@ -61,7 +75,7 @@ Collect only missing information. Do not invent a source table, source field, fo
|
|
|
61
75
|
|
|
62
76
|
Confirm:
|
|
63
77
|
|
|
64
|
-
- the target project (namespace) for the assets; use the built-in `default` only when the user has no dedicated project.
|
|
78
|
+
- the target project (namespace) for the assets; use the built-in `default` only when the user has no dedicated project. After the target is confirmed, write its explicit `project` field in every Parameter, Operator, Feature, FeatureSet and DatasetManifest draft; never rely on the schema default. Reads and filters use the `--project` command option;
|
|
65
79
|
- business meaning, stable asset names, owner, and intended consumers;
|
|
66
80
|
- Parameter source adapter and credential-free source mapping;
|
|
67
81
|
- feature formula, exact Parameter dependencies, windows, inclusion rules, rounding, null behavior, and output dtype;
|
|
@@ -75,9 +89,64 @@ Confirm:
|
|
|
75
89
|
|
|
76
90
|
Separate preparation from mutation. Creating files and running local tests or `apply --dry-run` does not authorize publishing a wheel, changing the Registry, or submitting a build Job.
|
|
77
91
|
|
|
92
|
+
A label such as "10-minute mean" confirms neither its input Parameter nor a
|
|
93
|
+
complete formula. Before drafting an Operator or Feature, ask for the exact
|
|
94
|
+
versioned input Parameter(s), or for an approved source contract from which
|
|
95
|
+
they will be created. Do not infer an input from the Feature name or a similar
|
|
96
|
+
existing asset.
|
|
97
|
+
|
|
98
|
+
### Semantic Asset Review Gate
|
|
99
|
+
|
|
100
|
+
Treat resource correctness as a separate gate from workflow correctness. Before
|
|
101
|
+
creating Catalog JSON, write a local semantic asset review with one row per
|
|
102
|
+
Parameter, Operator, Feature and Dataset field:
|
|
103
|
+
|
|
104
|
+
```text
|
|
105
|
+
asset_key | business_meaning | source/evidence | confirmed_by | unresolved | proposed_value
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
The review must distinguish three sources: confirmed business facts, facts
|
|
109
|
+
observed from the selected Project Registry/CLI, and agent proposals. A Skill
|
|
110
|
+
reference, schema default, similarly named old asset, or platform capability is
|
|
111
|
+
not business evidence. Do not convert a proposal into `proposed_value` without
|
|
112
|
+
explicit user confirmation or an exact match in the selected Project.
|
|
113
|
+
|
|
114
|
+
Stop with `need_business_confirmation` when any required source mapping, field,
|
|
115
|
+
unit, formula, time conversion, missing/quality policy, FeatureSet order,
|
|
116
|
+
dataset read policy, rowset/endpoint policy, owner, or approval boundary is
|
|
117
|
+
unresolved. Do not create placeholder Catalog files to make the workflow look
|
|
118
|
+
complete.
|
|
119
|
+
|
|
120
|
+
For a requested Feature, `exact Parameter dependencies` is an independent
|
|
121
|
+
required contract item. If the user names a transform or window but not its
|
|
122
|
+
input Parameter(s), include `input_parameter` in the unresolved questions and
|
|
123
|
+
stop. Do this before treating an Operator formula as complete: a time window,
|
|
124
|
+
cutoff rule, or Feature name never identifies the input.
|
|
125
|
+
|
|
126
|
+
After confirmation, show the proposed dependency closure and obtain approval of
|
|
127
|
+
the semantic asset plan before generating Catalog files. A successful local
|
|
128
|
+
schema check or remote dry-run is not semantic approval.
|
|
129
|
+
|
|
130
|
+
Before emitting or writing a draft, mechanically check that every asset carries
|
|
131
|
+
the confirmed `project` field, that every Feature input carries that same
|
|
132
|
+
`project`, and that every Operator `input_schema.parameters` reference is
|
|
133
|
+
qualified as `project/parameter:version`. The schema's `default` values are not
|
|
134
|
+
valid substitutes for an explicit non-default target. When a confirmed Feature
|
|
135
|
+
config defines window closure, cutoff inclusion, duplicate handling, or empty
|
|
136
|
+
window behavior, copy all of those fields into the Feature config rather than
|
|
137
|
+
reducing it to the window string.
|
|
138
|
+
|
|
78
139
|
## Discover Existing Assets
|
|
79
140
|
|
|
80
|
-
Check the selected profile and list existing Parameter, Operator, Feature, FeatureSet, and Dataset versions before choosing names. Scope discovery to the target project with `--project`, because the true registry key is `project/name:version` and the same `name:version` may exist independently under another project. Registry list commands are paginated; search by stable identifier or follow every page until `offset + len(items) >= total`. Reuse an exact immutable version only when its full content matches. Never reference an asset in another project; cross-project references are rejected.
|
|
141
|
+
Check the selected profile and list existing Parameter, Operator, Feature, FeatureSet, and Dataset versions before choosing names. Scope discovery to the target project with `--project`, because the true registry key is `project/name:version` and the same `name:version` may exist independently under another project. Run each `list-*` command as a separate command so one failed check cannot hide the status of the others. Do not add `--json` unless the installed subcommand help explicitly advertises it; supported releases emit machine-readable JSON by default. Registry list commands are paginated; search by stable identifier or follow every page until `offset + len(items) >= total`. Reuse an exact immutable version only when its full content matches. Never reference an asset in another project; cross-project references are rejected.
|
|
142
|
+
|
|
143
|
+
CLI readiness is a hard gate for every Registry conclusion. If `command -v`,
|
|
144
|
+
`version`, `show-config`, or server `health` fails, returns a nonzero status, or
|
|
145
|
+
does not identify the intended target, stop with `platform_not_ready`. Do not
|
|
146
|
+
list assets, select reuse versus authoring, dry-run, publish, build, fetch, or
|
|
147
|
+
report a Registry fact from prompt text, a remembered response, or a similar
|
|
148
|
+
local asset. Report the failed command and its configured target; resume only
|
|
149
|
+
after the user/environment restores the same target.
|
|
81
150
|
|
|
82
151
|
Create a new version when source semantics, formula code, config meaning, inputs, output dtype, time behavior, quality rules, or column order change. Never overwrite an immutable version or use suffixes such as `new`, `final`, or `test2`.
|
|
83
152
|
|
|
@@ -140,8 +209,29 @@ Keep platform demos and V95 built-ins unchanged. Do not put unrelated business a
|
|
|
140
209
|
|
|
141
210
|
### Define Parameters
|
|
142
211
|
|
|
212
|
+
Use the platform-capability guide's asset table before choosing the resource
|
|
213
|
+
type. For each Parameter record the business signal, exact source evidence,
|
|
214
|
+
unit, time/availability semantics and missing policy in the semantic review.
|
|
215
|
+
Never infer a source table, column, measurement, tag, unit or timezone from a
|
|
216
|
+
Parameter name.
|
|
217
|
+
|
|
143
218
|
Create one stable, versioned source contract for each independently readable value. Keep credentials out of JSON. Use only supported direct adapters and explicit source identity. Put source data validity in `quality_rules`; put dataset-specific transformations in preprocess Operators.
|
|
144
219
|
|
|
220
|
+
When the approved source contract supplies SQL, copy its query text and named
|
|
221
|
+
parameters exactly into `source.sql`/`source.params`. Do not retype, beautify,
|
|
222
|
+
rename an identifier, change a join key, or "simplify" an approved query while
|
|
223
|
+
drafting. SQL whitespace may be formatted only when a reviewed formatter proves
|
|
224
|
+
the token stream is unchanged; every source identifier and `<= %(end)s` boundary
|
|
225
|
+
remains part of the Parameter contract.
|
|
226
|
+
|
|
227
|
+
For SQL Parameters, the runtime contract is a single statement: use the exact
|
|
228
|
+
`source.sql` field and a mapping under `source.params` for every named
|
|
229
|
+
placeholder other than the reserved `start` and `end` window parameters. Do not
|
|
230
|
+
use `source.parameters` (it is not a supported alias and may be silently
|
|
231
|
+
dropped by older CLI/server releases). Do not leave a trailing semicolon; the
|
|
232
|
+
deployed Worker rejects SQL containing multiple statements. Verify the resolved
|
|
233
|
+
Parameter still contains the named params before authorizing a build.
|
|
234
|
+
|
|
145
235
|
Use Parameter `rounding` when every consumer must receive the same fixed-point value. The platform applies `mode` (`half_up` or `half_even`) and `decimals` after source normalization and before quality checks, source caching, replay, and Operator execution. Prefer an unrounded source expression, do not repeat the same rounding in an Operator, and publish a new Parameter plus its affected reverse-dependency closure when the rule changes.
|
|
146
236
|
|
|
147
237
|
Do not represent a rolling mean, lag, ratio, trend, or model input as a Parameter. Those are Features.
|
|
@@ -166,7 +256,11 @@ Use a meaningful immutable `function_hash`. Change the Operator version whenever
|
|
|
166
256
|
|
|
167
257
|
### Define Features
|
|
168
258
|
|
|
169
|
-
Create one JSON file per single output column.
|
|
259
|
+
Create one JSON file per single output column. Set the confirmed `project` on the
|
|
260
|
+
Feature and on every nested Feature input; a FeatureInput without `project`
|
|
261
|
+
silently targets `default`. Bind exactly one Operator version and physical
|
|
262
|
+
`output_column`. List only the Parameter versions actually required by that
|
|
263
|
+
column's formula.
|
|
170
264
|
|
|
171
265
|
Features may share the same Operator and config. The planner will merge their exact inputs and call the Operator once per computation group. Do not duplicate every Operator input into every Feature merely to make schemas look uniform.
|
|
172
266
|
|
|
@@ -176,8 +270,41 @@ Reference immutable Feature versions in the exact consumer column order. Keep th
|
|
|
176
270
|
|
|
177
271
|
### Define The DatasetManifest
|
|
178
272
|
|
|
273
|
+
Do not use DatasetManifest as a catch-all for unresolved semantics. Confirm the
|
|
274
|
+
dataset purpose and consumer, then separately confirm the target grid, horizon,
|
|
275
|
+
read policy, explicit Parameter outputs, rowset or rowset splits, abnormal windows and
|
|
276
|
+
endpoint eligibility. Keep the semantic asset review beside the draft Catalog
|
|
277
|
+
so a reviewer can compare business decisions with generated fields.
|
|
278
|
+
|
|
179
279
|
Reference exactly one FeatureSet. Use `parameters` only for Parameter columns that must also appear explicitly in `parameter_dataset.parquet`; Feature dependencies are resolved automatically.
|
|
180
280
|
|
|
281
|
+
Map the current Manifest schema exactly rather than carrying descriptive fields
|
|
282
|
+
from Parameter, Feature, or FeatureSet into it. A target Parameter reference is
|
|
283
|
+
**nested**; never flatten its `version` or `project` onto `target`:
|
|
284
|
+
|
|
285
|
+
```json
|
|
286
|
+
"target": {
|
|
287
|
+
"parameter": {"parameter": "hot_metal_si", "version": "v1", "project": "<project>"},
|
|
288
|
+
"si_time_source": "dispatch_time",
|
|
289
|
+
"offset_minutes": 20,
|
|
290
|
+
"interpolation": {"method": "linear"}
|
|
291
|
+
}
|
|
292
|
+
```
|
|
293
|
+
|
|
294
|
+
`target.interpolation` is `{"method": "linear"}`, `{"method": "none"}` or
|
|
295
|
+
`{"method": "forward_fill"}` -- neither it nor `target.parameter` accepts a
|
|
296
|
+
bare string. The current DatasetManifest has no top-level `owner`,
|
|
297
|
+
`description`, `si_null_policy`, or `preserve_all_candidate_rows`. Map a
|
|
298
|
+
business statement only to an existing schema field (for example `mark_only`,
|
|
299
|
+
`endpoint_policy`, a Feature config, or a Parameter missing policy); otherwise
|
|
300
|
+
keep it in the semantic review and ask for a platform/schema change. Validate
|
|
301
|
+
the generated object against the current schema before running `apply --dry-run`.
|
|
302
|
+
|
|
303
|
+
When a confirmed missing-data policy applies to a Feature-only Parameter, put it
|
|
304
|
+
in `parameter_missing_policies`, using `parameter`, `version`, `project`, and
|
|
305
|
+
`policy`. Do not copy the `ParameterRequest` names `name` and `missing_policy`
|
|
306
|
+
into that list. Omit the section when no such policy is confirmed.
|
|
307
|
+
|
|
181
308
|
Declare one dataset-wide `prediction` contract when Features use a forecast cutoff. Operators consume `context.cutoff_times`; do not copy the same prediction horizon into every Feature config. Check the selected Operator's `input_schema.prediction` limits and satisfy an explicit-horizon requirement before publishing or building.
|
|
182
309
|
|
|
183
310
|
Use `snapshot` for reproducible training data, `as_of` for historical visibility replay, and `latest` for current inference-style reads. Use a fresh `dataset_version` when validating a new release or intentionally requesting a new immutable dataset contract.
|
|
@@ -202,11 +329,16 @@ freshness, tail-edge, gap-fill, and rowset contracts.
|
|
|
202
329
|
|
|
203
330
|
Run, in order:
|
|
204
331
|
|
|
205
|
-
1.
|
|
206
|
-
2.
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
332
|
+
1. Semantic asset review: no required unresolved field and every proposed value has a source or user confirmation.
|
|
333
|
+
2. Operator unit tests. Before building, turn every applicable item in
|
|
334
|
+
`operator-authoring.md` into a separately named test; do not let one broad
|
|
335
|
+
happy-path assertion substitute for cutoff-before/at/after, duplicate and
|
|
336
|
+
missing input, empty history, requested-output, event-time order, or
|
|
337
|
+
dtype/rounding coverage. The template is only a starting point.
|
|
338
|
+
3. Wheel build and wheel filename verification.
|
|
339
|
+
4. Catalog path, schema, dependency, immutability, and package validation with `apply --dry-run` against the same target profile intended for publication.
|
|
340
|
+
5. Compare the catalog's exact dependency closure with the intended old-to-new version mapping. Reject any affected downstream reference that still points to an old version unless its retention is explicit and justified.
|
|
341
|
+
6. A human-readable summary of planned new, unchanged, retained, and conflicting assets.
|
|
210
342
|
|
|
211
343
|
Stop on any error. Do not weaken schema validation, fabricate a missing dependency, change an existing version in place, or switch profiles to make validation pass.
|
|
212
344
|
|
|
@@ -214,7 +346,7 @@ Stop on any error. Do not weaken schema validation, fabricate a missing dependen
|
|
|
214
346
|
|
|
215
347
|
Publish only after the user explicitly approves Registry and wheel changes. Use catalog `apply` so publication follows Parameter -> Operator -> Feature -> FeatureSet -> Dataset order.
|
|
216
348
|
|
|
217
|
-
After publication, resolve every new DatasetManifest and compare its exact Parameter, Operator, Feature, and FeatureSet versions with the pre-publication mapping. Stop if an affected old key or any unexpected version remains. Resolve the manifest before building. Submit a build only when requested. For server builds, wait for the terminal Job state and download the exact artifact by `dataset_id + manifest_hash`.
|
|
349
|
+
After publication, resolve every new DatasetManifest and compare its exact Parameter, Operator, Feature, and FeatureSet versions with the pre-publication mapping. Stop if an affected old key or any unexpected version remains. Resolve the manifest before building. Submit a build only when requested. For the current partitioned materialization runtime, use `--max-parallelism 1` unless a separately verified platform capability says otherwise; do not retry or resubmit a failed Job until its error and already-completed partitions are understood. For server builds, wait for the terminal Job state and download the exact artifact by `dataset_id + manifest_hash`.
|
|
218
350
|
|
|
219
351
|
Do not delete versioned Registry assets, cancel Jobs, rebuild images, modify Kubernetes, or change service configuration as part of this workflow unless the user separately and explicitly requests that action. An empty Project may be soft-deleted only on an explicit request; rely on the server to reject deletion when resources still exist.
|
|
220
352
|
|
|
@@ -232,6 +364,14 @@ gap-filled inputs as degraded freshness, not as a normal read.
|
|
|
232
364
|
|
|
233
365
|
## Verify The Artifact
|
|
234
366
|
|
|
367
|
+
Treat Registry metadata inspection and downloaded-file inspection as separate
|
|
368
|
+
checkpoints. A successful `get-dataset-artifact` can establish the immutable
|
|
369
|
+
identity, object inventory, row count, validation summary, and lineage metadata;
|
|
370
|
+
it cannot establish Parquet schema, column order, or file-content hashes. If a
|
|
371
|
+
download times out, report whether no response/progress was observed or whether
|
|
372
|
+
some body bytes were received, preserve the metadata-only evidence, and leave
|
|
373
|
+
downloaded-file validation explicitly incomplete.
|
|
374
|
+
|
|
235
375
|
Require all of the following before reporting success:
|
|
236
376
|
|
|
237
377
|
- Job status is `succeeded`;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"schema_version": "ml_data_platform.dataset_manifest/v1",
|
|
3
|
+
"project": "replace_with_project",
|
|
3
4
|
"dataset_id": "example_temperature_training",
|
|
4
5
|
"dataset_version": "v1",
|
|
5
6
|
"mode": "training",
|
|
@@ -9,7 +10,22 @@
|
|
|
9
10
|
"end": "2026-07-02T00:00:00+08:00",
|
|
10
11
|
"grid": "10min"
|
|
11
12
|
},
|
|
13
|
+
"prediction": {
|
|
14
|
+
"horizon": "0min"
|
|
15
|
+
},
|
|
16
|
+
"rowset": {
|
|
17
|
+
"strategy": "fixed_grid",
|
|
18
|
+
"grid": "10min"
|
|
19
|
+
},
|
|
12
20
|
"parameters": [],
|
|
21
|
+
"parameter_missing_policies": [
|
|
22
|
+
{
|
|
23
|
+
"parameter": "example_temperature",
|
|
24
|
+
"version": "v1",
|
|
25
|
+
"project": "replace_with_project",
|
|
26
|
+
"policy": "report_only"
|
|
27
|
+
}
|
|
28
|
+
],
|
|
13
29
|
"feature_set": {
|
|
14
30
|
"name": "example_temperature_core",
|
|
15
31
|
"version": "v1"
|
|
@@ -1,17 +1,24 @@
|
|
|
1
1
|
{
|
|
2
2
|
"schema_version": "ml_data_platform.feature/v1",
|
|
3
|
+
"project": "replace_with_project",
|
|
3
4
|
"name": "example_temperature_mean_5m",
|
|
4
5
|
"version": "v1",
|
|
5
6
|
"inputs": [
|
|
6
7
|
{
|
|
7
8
|
"parameter": "example_temperature",
|
|
8
|
-
"version": "v1"
|
|
9
|
+
"version": "v1",
|
|
10
|
+
"project": "replace_with_project"
|
|
9
11
|
}
|
|
10
12
|
],
|
|
11
13
|
"operator": "example_temperature_features",
|
|
12
14
|
"operator_version": "v1",
|
|
13
15
|
"config": {
|
|
14
|
-
"window": "5min"
|
|
16
|
+
"window": "5min",
|
|
17
|
+
"window_closed": "right",
|
|
18
|
+
"cutoff_included": true,
|
|
19
|
+
"post_cutoff_allowed": false,
|
|
20
|
+
"duplicate_event_time": "last",
|
|
21
|
+
"empty_window": "null"
|
|
15
22
|
},
|
|
16
23
|
"output_column": "example_temperature_mean_5m",
|
|
17
24
|
"output_dtype": "float64",
|
package/skills/feature-management/assets/catalog-template/operator_package/tests/test_operator.py
CHANGED
|
@@ -9,20 +9,21 @@ from business_feature_operator_template import compute_features
|
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
class ComputeFeaturesTest(unittest.TestCase):
|
|
12
|
-
def context(
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
12
|
+
def context(
|
|
13
|
+
self,
|
|
14
|
+
timestamps: list[str] | None = None,
|
|
15
|
+
values: list[float | None] | None = None,
|
|
16
|
+
targets: list[str] | None = None,
|
|
17
|
+
requested: list[str] | None = None,
|
|
18
|
+
) -> SimpleNamespace:
|
|
19
|
+
timestamps = timestamps or [
|
|
20
|
+
"2026-07-01 09:54:00",
|
|
21
|
+
"2026-07-01 09:56:00",
|
|
22
|
+
"2026-07-01 10:00:00",
|
|
23
|
+
"2026-07-01 10:01:00",
|
|
24
|
+
]
|
|
25
|
+
values = values or [10.0, 20.0, 30.0, 40.0]
|
|
26
|
+
source = pd.DataFrame({"timestamp": pd.DatetimeIndex(timestamps), "value": values})
|
|
26
27
|
return SimpleNamespace(
|
|
27
28
|
requested_output_columns=requested or ["example_temperature_mean_5m"],
|
|
28
29
|
inputs=[
|
|
@@ -32,24 +33,50 @@ class ComputeFeaturesTest(unittest.TestCase):
|
|
|
32
33
|
)
|
|
33
34
|
],
|
|
34
35
|
metric_frames={"example_temperature:v1": source},
|
|
35
|
-
target_times=pd.DatetimeIndex(
|
|
36
|
-
["2026-07-01 10:00:00", "2026-07-01 10:02:00"]
|
|
37
|
-
),
|
|
36
|
+
target_times=pd.DatetimeIndex(targets or ["2026-07-01 10:00:00", "2026-07-01 10:02:00"]),
|
|
38
37
|
config={"window": "5min"},
|
|
39
38
|
)
|
|
40
39
|
|
|
41
|
-
def
|
|
40
|
+
def test_formula_uses_the_confirmed_mean(self) -> None:
|
|
42
41
|
result = compute_features(self.context())
|
|
43
|
-
|
|
44
|
-
self.assertEqual(
|
|
45
|
-
list(result.columns),
|
|
46
|
-
["event_time", "example_temperature_mean_5m"],
|
|
47
|
-
)
|
|
48
42
|
self.assertEqual(result["example_temperature_mean_5m"].tolist(), [25.0, 35.0])
|
|
49
43
|
|
|
44
|
+
def test_excludes_event_at_open_left_window_boundary(self) -> None:
|
|
45
|
+
result = compute_features(self.context(["2026-07-01 09:55:00", "2026-07-01 09:56:00"], [10.0, 20.0]))
|
|
46
|
+
self.assertEqual(result["example_temperature_mean_5m"].iloc[0], 20.0)
|
|
47
|
+
|
|
48
|
+
def test_includes_event_exactly_at_cutoff(self) -> None:
|
|
49
|
+
result = compute_features(self.context(["2026-07-01 10:00:00"], [30.0]))
|
|
50
|
+
self.assertEqual(result["example_temperature_mean_5m"].iloc[0], 30.0)
|
|
51
|
+
|
|
52
|
+
def test_excludes_event_after_cutoff(self) -> None:
|
|
53
|
+
result = compute_features(self.context(["2026-07-01 10:00:01"], [40.0]))
|
|
54
|
+
self.assertTrue(pd.isna(result["example_temperature_mean_5m"].iloc[0]))
|
|
55
|
+
|
|
56
|
+
def test_uses_last_value_for_duplicate_event_time(self) -> None:
|
|
57
|
+
result = compute_features(self.context(["2026-07-01 10:00:00", "2026-07-01 10:00:00"], [10.0, 30.0]))
|
|
58
|
+
self.assertEqual(result["example_temperature_mean_5m"].iloc[0], 30.0)
|
|
59
|
+
|
|
60
|
+
def test_empty_history_returns_null(self) -> None:
|
|
61
|
+
result = compute_features(self.context(["2026-07-01 09:00:00"], [10.0]))
|
|
62
|
+
self.assertTrue(pd.isna(result["example_temperature_mean_5m"].iloc[0]))
|
|
63
|
+
|
|
64
|
+
def test_requested_output_is_exactly_the_supported_subset(self) -> None:
|
|
65
|
+
result = compute_features(self.context(requested=["example_temperature_mean_5m"]))
|
|
66
|
+
self.assertEqual(list(result.columns), ["event_time", "example_temperature_mean_5m"])
|
|
67
|
+
|
|
68
|
+
def test_preserves_requested_event_time_order(self) -> None:
|
|
69
|
+
targets = ["2026-07-01 10:02:00", "2026-07-01 10:00:00"]
|
|
70
|
+
result = compute_features(self.context(targets=targets))
|
|
71
|
+
self.assertEqual(list(result["event_time"]), list(pd.DatetimeIndex(targets)))
|
|
72
|
+
|
|
73
|
+
def test_output_dtype_is_float64(self) -> None:
|
|
74
|
+
result = compute_features(self.context())
|
|
75
|
+
self.assertEqual(str(result["example_temperature_mean_5m"].dtype), "float64")
|
|
76
|
+
|
|
50
77
|
def test_rejects_unknown_requested_output(self) -> None:
|
|
51
78
|
with self.assertRaisesRegex(ValueError, "unsupported output columns"):
|
|
52
|
-
compute_features(self.context(["unknown_feature"]))
|
|
79
|
+
compute_features(self.context(requested=["unknown_feature"]))
|
|
53
80
|
|
|
54
81
|
|
|
55
82
|
if __name__ == "__main__":
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"schema_version": "ml_data_platform.operator/v1",
|
|
3
|
+
"project": "replace_with_project",
|
|
3
4
|
"name": "example_temperature_features",
|
|
4
5
|
"version": "v1",
|
|
5
6
|
"type": "feature",
|
|
@@ -10,7 +11,21 @@
|
|
|
10
11
|
"code_artifact": null,
|
|
11
12
|
"input_schema": {
|
|
12
13
|
"parameters": [
|
|
13
|
-
"example_temperature:v1"
|
|
14
|
+
"replace_with_project/example_temperature:v1"
|
|
15
|
+
],
|
|
16
|
+
"prediction": {
|
|
17
|
+
"required": true,
|
|
18
|
+
"minimum_horizon": "0min",
|
|
19
|
+
"maximum_horizon": "0min"
|
|
20
|
+
},
|
|
21
|
+
"history_requirements": [
|
|
22
|
+
{
|
|
23
|
+
"anchor": "cutoff",
|
|
24
|
+
"lookback": "5min",
|
|
25
|
+
"output_columns": [
|
|
26
|
+
"example_temperature_mean_5m"
|
|
27
|
+
]
|
|
28
|
+
}
|
|
14
29
|
]
|
|
15
30
|
},
|
|
16
31
|
"output_schema": {
|
|
@@ -49,6 +49,15 @@ ml-platform --profile server list-feature-sets -q <stable-name>
|
|
|
49
49
|
ml-platform --profile server list-datasets -q <stable-dataset-id>
|
|
50
50
|
```
|
|
51
51
|
|
|
52
|
+
Run each command separately. Check `ml-platform --help` and the specific
|
|
53
|
+
subcommand help before adding flags. In the deployed 0.6.x CLI, `list-*`
|
|
54
|
+
commands print JSON by default and do not accept a `--json` flag.
|
|
55
|
+
If a deployed release rejects the global `--profile server` prefix, treat that
|
|
56
|
+
as CLI compatibility drift: record the syntax error, inspect `show-config` and
|
|
57
|
+
the subcommand help, then retry the same read-only command without the rejected
|
|
58
|
+
flag only if the installed help permits it. Do not report this as API/network
|
|
59
|
+
unavailability or switch to a direct HTTP client.
|
|
60
|
+
|
|
52
61
|
By default each `list-*` command auto-pages through the whole result set and
|
|
53
62
|
prints a **bare JSON array** of every matching item, so it never silently stops
|
|
54
63
|
at the server's default page size. Use `-q` for a case-insensitive identifier
|
|
@@ -217,6 +226,17 @@ ml-platform --profile server wait-job <job_id>
|
|
|
217
226
|
ml-platform --profile server get-job <job_id>
|
|
218
227
|
```
|
|
219
228
|
|
|
229
|
+
With explicit user authorization, stop a running build through the server CLI:
|
|
230
|
+
|
|
231
|
+
```bash
|
|
232
|
+
ml-platform --profile server cancel-job <job_id>
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
Cancellation is terminal for that Job; it does not change the immutable
|
|
236
|
+
DatasetManifest and it does not automatically resubmit with different
|
|
237
|
+
partition or parallelism settings. Check `get-job` after the request and
|
|
238
|
+
record the returned terminal status.
|
|
239
|
+
|
|
220
240
|
## Fetch Realtime Inference Data
|
|
221
241
|
|
|
222
242
|
Fetch one causal-cutoff row in memory:
|
|
@@ -252,6 +272,15 @@ ml-platform --profile server \
|
|
|
252
272
|
`--project` defaults to `default`; pass the owning project when the dataset lives
|
|
253
273
|
elsewhere, or the lookup returns `artifact_not_found`.
|
|
254
274
|
|
|
275
|
+
The download command writes progress to stderr after response headers arrive.
|
|
276
|
+
Interpret a timeout with no progress line as a request/header or first-byte
|
|
277
|
+
failure. Interpret a timeout after one or more `artifact download: received ...`
|
|
278
|
+
lines as an incomplete response body. In either case, report the timeout and
|
|
279
|
+
keep downloaded-file validation incomplete; do not replace the supported CLI
|
|
280
|
+
with direct HTTP or object-store access. A successful metadata query remains
|
|
281
|
+
valid metadata-only evidence, but it does not prove Parquet schema, column
|
|
282
|
+
order, or file-content hashes.
|
|
283
|
+
|
|
255
284
|
Inspect at least:
|
|
256
285
|
|
|
257
286
|
```text
|
|
@@ -35,6 +35,9 @@ Use this reference for the implemented V1 schemas. Prefer the running CLI and `s
|
|
|
35
35
|
`project` field in its own JSON spec (it defaults to `default` when omitted).
|
|
36
36
|
So to place an asset in a non-default project, set `"project": "<name>"` in the
|
|
37
37
|
spec before `apply`.
|
|
38
|
+
- For a confirmed non-default target, include that field in every draft
|
|
39
|
+
Parameter, Operator, Feature, Feature input, FeatureSet, and DatasetManifest;
|
|
40
|
+
do not rely on Pydantic's `default` fallback during authoring.
|
|
38
41
|
- Reads and filters, by contrast, take the project out-of-band: `list-* --project`
|
|
39
42
|
and the leading project argument of the artifact commands. `apply --project`
|
|
40
43
|
scopes the comparison snapshot that publication is validated against.
|
|
@@ -91,6 +94,11 @@ Required shape:
|
|
|
91
94
|
|
|
92
95
|
Supported source modes are `direct_column`, `sql`, and `derived`. Prefer `direct_column` for ordinary business parameters.
|
|
93
96
|
|
|
97
|
+
For `sql`, `source.sql` must contain one statement without a trailing semicolon,
|
|
98
|
+
and named placeholders must be supplied through `source.params`; `start` and
|
|
99
|
+
`end` are reserved runtime window parameters. `source.parameters` is not a
|
|
100
|
+
supported field.
|
|
101
|
+
|
|
94
102
|
`availability.strategy` 支持 `source_field`、`fixed_delay` 和 `unsupported`。只有一个
|
|
95
103
|
事件时间列时，可以使用经过数据 owner 确认的固定延迟上界，例如
|
|
96
104
|
`{"strategy":"fixed_delay","delay":"PT10M","accuracy":"upper_bound"}`。
|
|
@@ -282,6 +290,14 @@ The list must be nonempty, unique, and ordered exactly as consumers expect.
|
|
|
282
290
|
"horizon": "10min"
|
|
283
291
|
},
|
|
284
292
|
"parameters": [],
|
|
293
|
+
"parameter_missing_policies": [
|
|
294
|
+
{
|
|
295
|
+
"parameter": "hot_blast_pressure",
|
|
296
|
+
"version": "v1",
|
|
297
|
+
"project": "default",
|
|
298
|
+
"policy": "report_only"
|
|
299
|
+
}
|
|
300
|
+
],
|
|
285
301
|
"feature_set": {"name": "pressure_core", "version": "v1"},
|
|
286
302
|
"preprocess": [],
|
|
287
303
|
"output": {
|
|
@@ -321,9 +337,13 @@ declared `realtime_fetch` policy contains meaningful defaults and changes the
|
|
|
321
337
|
manifest hash. Declaring `endpoint_policy` also changes the manifest hash.
|
|
322
338
|
|
|
323
339
|
Each explicit `parameters` request may set `missing_policy`. A
|
|
324
|
-
`parameter_missing_policies` entry
|
|
325
|
-
|
|
326
|
-
|
|
340
|
+
`parameter_missing_policies` entry is reserved for a Parameter used only through
|
|
341
|
+
the selected FeatureSet's input closure; it must not also appear in
|
|
342
|
+
`parameters`. Its exact V1 shape is `{"parameter": "<name>", "version":
|
|
343
|
+
"<version>", "project": "<project>", "policy": "<policy>"}`. Do not use
|
|
344
|
+
the `ParameterRequest` field names `name` or `missing_policy` in this list. For
|
|
345
|
+
a confirmed non-default target, set `project` explicitly instead of relying on
|
|
346
|
+
its `default` fallback. The policies are:
|
|
327
347
|
|
|
328
348
|
- `report_only`: retain all target rows and report source gaps.
|
|
329
349
|
- `fail_on_requested_range_gap`: fail when a detected gap affects the requested
|
|
@@ -424,13 +444,30 @@ near the end of the target range can have its lookback clipped by the range end,
|
|
|
424
444
|
and a window close to the end can remove most remaining rows.
|
|
425
445
|
|
|
426
446
|
`target` states which instant the label is aligned to, so reviewing a manifest
|
|
427
|
-
shows the target definition rather than only a Parameter version string.
|
|
428
|
-
|
|
447
|
+
shows the target definition rather than only a Parameter version string. When
|
|
448
|
+
the target is a Parameter, keep its complete versioned reference **inside**
|
|
449
|
+
`target.parameter`; do not flatten `version` or `project` onto `target`.
|
|
450
|
+
`si_time_source` is `weight_time` or `dispatch_time`. `interpolation` is an
|
|
451
|
+
object with a `method`, never a bare string.
|
|
429
452
|
|
|
430
453
|
```json
|
|
431
|
-
"target": {
|
|
454
|
+
"target": {
|
|
455
|
+
"parameter": {
|
|
456
|
+
"parameter": "hot_metal_si",
|
|
457
|
+
"version": "v1",
|
|
458
|
+
"project": "replace_with_project"
|
|
459
|
+
},
|
|
460
|
+
"si_time_source": "weight_time",
|
|
461
|
+
"offset_minutes": 0,
|
|
462
|
+
"interpolation": {"method": "linear"}
|
|
463
|
+
}
|
|
432
464
|
```
|
|
433
465
|
|
|
466
|
+
Allowed interpolation methods are `none`, `linear`, and `forward_fill`. A
|
|
467
|
+
business phrase such as "allow and report target nulls" belongs in an existing
|
|
468
|
+
Parameter missing policy, Feature configuration, or the semantic review; it is
|
|
469
|
+
not a free-form `target` or DatasetManifest field.
|
|
470
|
+
|
|
434
471
|
The declaration is checked against the selected Si Parameter, but only for
|
|
435
472
|
`direct_column` reads, which name their time column in the contract. A `sql` or
|
|
436
473
|
`derived` Parameter can apply an offset inside its query text, so those are
|
|
@@ -67,12 +67,17 @@ Use `context.cutoff_times` for causal feature windows. It is derived once from
|
|
|
67
67
|
`context.target_times - context.prediction_horizon` using the DatasetManifest
|
|
68
68
|
prediction contract; do not parse a duplicated Feature-level horizon.
|
|
69
69
|
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
70
|
+
An Operator requires an explicit forecast contract whenever its formula or code
|
|
71
|
+
uses `context.cutoff_times`, or any `input_schema.history_requirements` entry uses
|
|
72
|
+
`anchor: cutoff`. In that case always declare `input_schema.prediction` with
|
|
73
|
+
`required`, `minimum_horizon`, and `maximum_horizon`. This includes a strict
|
|
74
|
+
nowcast: declare `required=true` and both bounds as `0min`; do not omit the
|
|
75
|
+
section and silently accept the runtime's `default_nowcast` fallback. Declare
|
|
76
|
+
source history as `input_schema.history_requirements`, using `anchor: cutoff`
|
|
77
|
+
for causal windows and `anchor: target_time` only for intentional target-aligned
|
|
78
|
+
formulas. Scope a requirement with `output_columns` when only some outputs need
|
|
79
|
+
it. Include alignment slack such as hourly floor boundaries in the declared
|
|
80
|
+
lookback.
|
|
76
81
|
|
|
77
82
|
For realtime reads, `history_requirements` is also the per-Parameter fetch
|
|
78
83
|
contract. Do not rely on a package-local lookback map or a single fixed window
|
|
@@ -110,7 +115,10 @@ Never infer a rule from a Feature name alone. Require a business decision when a
|
|
|
110
115
|
|
|
111
116
|
## Required Tests
|
|
112
117
|
|
|
113
|
-
Use deterministic fixtures and cover
|
|
118
|
+
Use deterministic fixtures and cover each applicable item with a separately
|
|
119
|
+
named test. A broad formula or happy-path test does not substitute for the
|
|
120
|
+
cutoff-before, cutoff-at, cutoff-after, duplicate/missing, empty-history,
|
|
121
|
+
requested-output, event-time-order, or dtype/rounding cases below:
|
|
114
122
|
|
|
115
123
|
1. expected formula values;
|
|
116
124
|
2. the event immediately before cutoff;
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# Platform Capability Guide
|
|
2
|
+
|
|
3
|
+
Use this guide to explain what the platform can execute. It is not business
|
|
4
|
+
evidence. A platform default never answers an unresolved business question.
|
|
5
|
+
|
|
6
|
+
## Public Assets
|
|
7
|
+
|
|
8
|
+
| Asset | Stores | Use when | Do not use for |
|
|
9
|
+
|---|---|---|---|
|
|
10
|
+
| Project | Namespace and isolation boundary | Every independent business release | A dataset version or a source table |
|
|
11
|
+
| Parameter | One readable source contract and its quality/time semantics | Raw or independently readable source values/events | Rolling means, ratios, trends, model columns |
|
|
12
|
+
| Operator | Versioned deterministic executable code | A formula needs computation over declared inputs | A business definition without confirmed formula |
|
|
13
|
+
| Feature | One immutable output-column contract | One model/input column produced by an Operator | A group of columns or an unnamed formula |
|
|
14
|
+
| FeatureSet | Ordered Feature references | Consumer column order is part of the contract | Copying formulas or creating multiple datasets |
|
|
15
|
+
| DatasetManifest | Dataset mode, time grid, rowsets, policy and FeatureSet reference | A reproducible dataset contract is ready | Filling unknown source or business semantics |
|
|
16
|
+
| DatasetArtifact | Output of an authorized build | Build was explicitly approved and completed | Proving a Catalog is semantically correct |
|
|
17
|
+
|
|
18
|
+
## Decision Rules
|
|
19
|
+
|
|
20
|
+
- Parameter answers **what source value is exposed**; Operator/Feature answers
|
|
21
|
+
**how a confirmed business formula is computed**.
|
|
22
|
+
- FeatureSet order is consumer-facing and must be confirmed; it is not inferred
|
|
23
|
+
from filesystem order or JSON discovery order.
|
|
24
|
+
- Dataset `time_range`, `prediction`, `rowset_splits`, `abnormal_windows` and
|
|
25
|
+
`endpoint_policy` are separate contracts. A platform default does not choose
|
|
26
|
+
a business policy.
|
|
27
|
+
- `apply --dry-run` checks structure, references, package and immutability. It
|
|
28
|
+
does not prove source correctness, formula correctness or artifact parity.
|
|
29
|
+
- `publish` creates immutable Registry resources. `build` creates an execution
|
|
30
|
+
Job and must have a separate explicit approval.
|
|
31
|
+
|
|
32
|
+
## Required Semantic Asset Review
|
|
33
|
+
|
|
34
|
+
Before writing Catalog JSON, produce a local review table with one row per
|
|
35
|
+
Parameter, Operator, Feature and Dataset field:
|
|
36
|
+
|
|
37
|
+
```text
|
|
38
|
+
asset_key | business_meaning | source/evidence | confirmed_by | unresolved | proposed_value
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Stop before Catalog generation if any required `source/evidence`, formula,
|
|
42
|
+
time boundary, unit, null policy, output dtype, FeatureSet order, read policy,
|
|
43
|
+
or approval field is unresolved. The review table is a proposal for the user;
|
|
44
|
+
it is not a Registry asset and must not contain guessed values.
|