@zhuoyuezs/ml-platform 0.1.11 → 0.1.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/checksums.json
CHANGED
|
@@ -7,18 +7,18 @@
|
|
|
7
7
|
},
|
|
8
8
|
{
|
|
9
9
|
"path": "runtime/business-client/package-lock.json",
|
|
10
|
-
"sha256": "sha256:
|
|
10
|
+
"sha256": "sha256:cfd3633238713a21444bd4d1810e7ee2c48557157f9817be9b0e7c59a4dc46b3",
|
|
11
11
|
"size_bytes": 381
|
|
12
12
|
},
|
|
13
13
|
{
|
|
14
14
|
"path": "runtime/business-client/package.json",
|
|
15
|
-
"sha256": "sha256:
|
|
15
|
+
"sha256": "sha256:e6bd1fd8a9cf49366a97f10e03b368c448d33f48df38f036a96c2979f1a2ec1b",
|
|
16
16
|
"size_bytes": 501
|
|
17
17
|
},
|
|
18
18
|
{
|
|
19
19
|
"path": "runtime/business-client/src/catalog.js",
|
|
20
|
-
"sha256": "sha256:
|
|
21
|
-
"size_bytes":
|
|
20
|
+
"sha256": "sha256:5625abc155779d827850de475db104995feadea6b64d56419e3665005669225f",
|
|
21
|
+
"size_bytes": 54291
|
|
22
22
|
},
|
|
23
23
|
{
|
|
24
24
|
"path": "runtime/business-client/src/cli.js",
|
package/package.json
CHANGED
package/release.json
CHANGED
|
@@ -13,8 +13,8 @@
|
|
|
13
13
|
"entrypoint": "src/cli.js",
|
|
14
14
|
"name": "ml-platform",
|
|
15
15
|
"path": "runtime/business-client",
|
|
16
|
-
"sha256": "sha256:
|
|
17
|
-
"version": "0.7.
|
|
16
|
+
"sha256": "sha256:0c928cad92c70c10b00d58a8bd9bd95f0bf88ac2f876c7967c67c5b17bfbfa2c",
|
|
17
|
+
"version": "0.7.2"
|
|
18
18
|
},
|
|
19
19
|
"policy_sha256": "sha256:02fbde0134696b78c0c365d6e0b76c9814227208da5b2b3d4e5dbd82dc4a3606",
|
|
20
20
|
"public_schema_versions": [
|
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
"ml_data_platform.feature_set/v1",
|
|
26
26
|
"ml_data_platform.dataset_manifest/v1"
|
|
27
27
|
],
|
|
28
|
-
"release_version": "0.1.
|
|
28
|
+
"release_version": "0.1.12",
|
|
29
29
|
"runtime_requirements": {
|
|
30
30
|
"node": ">=18",
|
|
31
31
|
"os": [
|
|
@@ -37,16 +37,16 @@
|
|
|
37
37
|
"skills": {
|
|
38
38
|
"feature-management": {
|
|
39
39
|
"path": "skills/feature-management",
|
|
40
|
-
"requires_cli": ">=0.7.
|
|
41
|
-
"revision": "0.1.
|
|
40
|
+
"requires_cli": ">=0.7.2 <0.8.0",
|
|
41
|
+
"revision": "0.1.12",
|
|
42
42
|
"sha256": "sha256:182b6f2df4fe58c946e482dfaf416c7d5b78c6fde9c8c8d7293103246618cb4e"
|
|
43
43
|
},
|
|
44
44
|
"model-lifecycle-management": {
|
|
45
45
|
"path": "skills/model-lifecycle-management",
|
|
46
|
-
"requires_cli": ">=0.7.
|
|
47
|
-
"revision": "0.1.
|
|
46
|
+
"requires_cli": ">=0.7.2 <0.8.0",
|
|
47
|
+
"revision": "0.1.12",
|
|
48
48
|
"sha256": "sha256:61ba17f955b36d1c2e8329a10454d040ee7eca5afdb90ff0914d3f85b6e2d203"
|
|
49
49
|
}
|
|
50
50
|
},
|
|
51
|
-
"source_commit": "
|
|
51
|
+
"source_commit": "0246c26b396ce551fed3a9c8d4341a2230765025"
|
|
52
52
|
}
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zhuoyuezs/ml-platform-business-client",
|
|
3
|
-
"version": "0.7.
|
|
3
|
+
"version": "0.7.2",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "@zhuoyuezs/ml-platform-business-client",
|
|
9
|
-
"version": "0.7.
|
|
9
|
+
"version": "0.7.2",
|
|
10
10
|
"license": "UNLICENSED",
|
|
11
11
|
"bin": {
|
|
12
12
|
"ml-platform": "src/cli.js"
|
|
@@ -57,12 +57,18 @@ function normalizeConstraintRule(value) { const label = "parameter quality rule"
|
|
|
57
57
|
function normalizeQualityRules(value) { exactKeys(value, ["valid_range", "allow_missing", "missing_detection", "rules"], [], "parameter quality_rules"); const normalizedRange = validRange(value.valid_range); const allowMissing = bool(defaultIfAbsent(value, "allow_missing", true), "allow_missing"); const rules = objectList(defaultIfAbsent(value, "rules", []), "parameter quality rules").map(normalizeConstraintRule); unique(rules.map((item) => item.id), "parameter quality_rules rule ids"); const reserved = new Set(); if (normalizedRange != null) reserved.add("contract_valid_range"); if (!allowMissing) reserved.add("contract_allow_missing"); if (rules.some((item) => reserved.has(item.id))) throw new Error("parameter quality_rules uses reserved compatibility rule ids"); const result = { valid_range: normalizedRange, allow_missing: allowMissing, rules }; const missing = normalizeMissingDetection(value.missing_detection); if (missing != null) result.missing_detection = missing; return result; }
|
|
58
58
|
function normalizeCodeArtifact(value) { object(value, "operator code_artifact"); const sha256 = requiredString(value.sha256, "code_artifact sha256"); if (!wheelDigestPattern.test(sha256)) throw new Error("code_artifact sha256 is invalid"); const filename = requiredString(value.filename, "code_artifact filename"); if (path.basename(filename) !== filename || !wheelFilenamePattern.test(filename)) throw new Error("code_artifact filename is invalid"); if (!Number.isInteger(value.size_bytes) || value.size_bytes <= 0) throw new Error("code_artifact size_bytes must be a positive integer"); if (defaultIfAbsent(value, "format", "wheel") !== "wheel") throw new Error("code_artifact format must be wheel"); return { uri: requiredString(value.uri, "code_artifact uri"), sha256, format: "wheel", filename, size_bytes: value.size_bytes, python_version: optionalString(value.python_version, "code_artifact python_version"), dependency_lock_hash: optionalString(value.dependency_lock_hash, "code_artifact dependency_lock_hash") }; }
|
|
59
59
|
|
|
60
|
+
function normalizeSource(value, label) {
|
|
61
|
+
const source = object(value, label);
|
|
62
|
+
if (Object.prototype.hasOwnProperty.call(source, "parameters")) throw new Error(`${label} uses unsupported field 'parameters'; use 'params'`);
|
|
63
|
+
if (source.mode === "sql" && typeof source.sql === "string" && source.sql.trimEnd().endsWith(";")) throw new Error(`SQL ${label}.sql must not end with a semicolon`);
|
|
64
|
+
return { adapter: pythonString(defaultIfAbsent(source, "adapter", "synthetic")), mode: choice(defaultIfAbsent(source, "mode", "direct_column"), ["direct_column", "sql", "derived"], `${label} mode`), source_metric: optionalString(source.source_metric, "source_metric"), source_group: optionalString(source.source_group, "source_group"), measurement: optionalString(source.measurement, "measurement"), field: optionalString(source.field, "field"), tags: stringMap(defaultIfAbsent(source, "tags", {}), `${label} tags`), schema: optionalString(aliasIfAbsent(source, "schema", "schema_name"), "schema"), table: optionalString(source.table, "table"), time_column: optionalString(source.time_column, "time_column"), value_column: optionalString(source.value_column, "value_column"), metric_name_column: optionalString(source.metric_name_column, "metric_name_column"), metric_name: optionalString(source.metric_name, "metric_name"), unit_column: optionalString(source.unit_column, "unit_column"), unit: optionalString(aliasIfAbsent(source, "unit", "source_unit"), "source unit"), filters: stringMap(defaultIfAbsent(source, "filters", {}), `${label} filters`), sql: optionalString(source.sql, "source sql"), params: object(defaultIfAbsent(source, "params", {}), `${label} params`) };
|
|
65
|
+
}
|
|
66
|
+
|
|
60
67
|
function normalizeParameter(payload) {
|
|
61
|
-
const source =
|
|
62
|
-
|
|
63
|
-
if (source.mode === "sql" && typeof source.sql === "string" && source.sql.trimEnd().endsWith(";")) throw new Error("SQL Parameter source.sql must not end with a semicolon");
|
|
68
|
+
const source = normalizeSource(payload.source, "parameter source");
|
|
69
|
+
const online_source = payload.online_source == null ? null : normalizeSource(payload.online_source, "parameter online_source");
|
|
64
70
|
const time = object(defaultIfAbsent(payload, "time_semantics", {}), "parameter time_semantics");
|
|
65
|
-
return { schema_version: pythonString(defaultIfAbsent(payload, "schema_version", "ml_data_platform.parameter/v1")), project: identifier(defaultIfAbsent(payload, "project", "default"), "parameter project"), name: identifier(payload.name, "parameter name"), display_name: requiredString(payload.display_name, "parameter display_name"), version: identifier(payload.version, "parameter version"), data_type: pythonString(defaultIfAbsent(payload, "data_type", "time_series")), unit: pythonString(defaultIfAbsent(payload, "unit", "")), expected_frequency: pythonString(defaultIfAbsent(payload, "expected_frequency", "10min")), source
|
|
71
|
+
return { schema_version: pythonString(defaultIfAbsent(payload, "schema_version", "ml_data_platform.parameter/v1")), project: identifier(defaultIfAbsent(payload, "project", "default"), "parameter project"), name: identifier(payload.name, "parameter name"), display_name: requiredString(payload.display_name, "parameter display_name"), version: identifier(payload.version, "parameter version"), data_type: pythonString(defaultIfAbsent(payload, "data_type", "time_series")), unit: pythonString(defaultIfAbsent(payload, "unit", "")), expected_frequency: pythonString(defaultIfAbsent(payload, "expected_frequency", "10min")), source, online_source, time_semantics: { event_time_field: pythonString(defaultIfAbsent(time, "event_time_field", "event_time")), ingested_at_field: pythonString(defaultIfAbsent(time, "ingested_at_field", "ingested_at")), timezone: pythonString(defaultIfAbsent(time, "timezone", "Asia/Shanghai")), availability: normalizeAvailability(time.availability) }, availability_sla: normalizeAvailabilitySla(payload.availability_sla), value_field: pythonString(defaultIfAbsent(payload, "value_field", "value")), quality_rules: normalizeQualityRules(object(defaultIfAbsent(payload, "quality_rules", {}), "parameter quality_rules")), rounding: normalizeRounding(payload.rounding), owner: pythonString(defaultIfAbsent(payload, "owner", "demo")) };
|
|
66
72
|
}
|
|
67
73
|
|
|
68
74
|
function normalizeOperator(payload) { object(payload, "operator"); const artifact = payload.code_artifact == null ? null : normalizeCodeArtifact(payload.code_artifact); const normalized = { schema_version: pythonString(defaultIfAbsent(payload, "schema_version", "ml_data_platform.operator/v1")), project: identifier(defaultIfAbsent(payload, "project", "default"), "operator project"), name: identifier(payload.name, "operator name"), version: identifier(payload.version, "operator version"), type: choice(defaultIfAbsent(payload, "type", "standard"), ["standard", "custom_python", "spark_udf", "feature"], "operator type"), function_hash: pythonString(defaultIfAbsent(payload, "function_hash", "demo")), entrypoint: optionalString(payload.entrypoint, "operator entrypoint"), code_hash: optionalString(payload.code_hash, "operator code_hash"), package_uri: optionalString(payload.package_uri, "operator package_uri"), code_artifact: artifact, input_schema: object(defaultIfAbsent(payload, "input_schema", {}), "operator input_schema"), output_schema: object(defaultIfAbsent(payload, "output_schema", {}), "operator output_schema"), config_schema: object(defaultIfAbsent(payload, "config_schema", {}), "operator config_schema"), runtime: object(defaultIfAbsent(payload, "runtime", {}), "operator runtime"), resources: object(defaultIfAbsent(payload, "resources", {}), "operator resources"), deterministic: bool(defaultIfAbsent(payload, "deterministic", true), "operator deterministic"), supports_batch: bool(defaultIfAbsent(payload, "supports_batch", true), "operator supports_batch"), supports_online: bool(defaultIfAbsent(payload, "supports_online", false), "operator supports_online"), owner: pythonString(defaultIfAbsent(payload, "owner", "demo")) }; if (artifact && normalized.code_hash !== artifact.sha256) throw new Error("operator code_hash must equal code_artifact.sha256"); if (artifact && normalized.package_uri !== artifact.uri) throw new Error("operator package_uri must equal code_artifact.uri"); return normalized; }
|
|
@@ -161,7 +167,7 @@ function loadCatalog(directory) {
|
|
|
161
167
|
}
|
|
162
168
|
|
|
163
169
|
async function fetchAll(client, endpoint, params = {}) { const rows = []; for (let offset = 0; ; offset += 500) { const response = await client.get(endpoint, { ...params, limit: 500, offset }); if (Array.isArray(response)) return response; const page = response.items; if (!Array.isArray(page)) throw new Error(`${endpoint} returned an invalid list response`); rows.push(...page); if (response.total == null || rows.length >= response.total || page.length < 500) return rows; } }
|
|
164
|
-
async function snapshot(client, targetProject) { const scope = { project: targetProject ?? "default" }; const [parameters, operators, features, featureSets, datasets] = await Promise.all([fetchAll(client, "/parameters", scope), fetchAll(client, "/operators", scope), fetchAll(client, "/features", scope), fetchAll(client, "/feature-sets", scope), fetchAll(client, "/datasets", { ...scope, include_deleted: true })]); return { parameters: new Map(parameters.map((x) => [key(
|
|
170
|
+
async function snapshot(client, targetProject) { const scope = { project: targetProject ?? "default" }; const [parameters, operators, features, featureSets, datasets] = await Promise.all([fetchAll(client, "/parameters", scope), fetchAll(client, "/operators", scope), fetchAll(client, "/features", scope), fetchAll(client, "/feature-sets", scope), fetchAll(client, "/datasets", { ...scope, include_deleted: true })]); return { parameters: new Map(parameters.map((x) => [key(x), x])), operators: new Map(operators.map((x) => [key(normalizeRegistryOperator(x)), normalizeRegistryOperator(x)])), features: new Map(features.map((x) => [key(normalizeFeature(x)), normalizeFeature(x)])), feature_sets: new Map(featureSets.map((x) => [key(normalizeFeatureSet(x)), normalizeFeatureSet(x)])), datasets: new Map(datasets.filter((x) => !x.deleted_at).map((x) => [datasetKey(normalizeRegistryDataset(x)), normalizeRegistryDataset(x)])) }; }
|
|
165
171
|
|
|
166
172
|
function requiresPackage(spec) { return ["custom_python", "feature", "spark_udf"].includes(spec.type) && spec.code_artifact == null && !String(spec.package_uri ?? "").startsWith("pythonpath://"); }
|
|
167
173
|
function validateCatalog(catalog, current) {
|
|
@@ -193,6 +199,7 @@ function sha256File(file) { return `sha256:${crypto.createHash("sha256").update(
|
|
|
193
199
|
|
|
194
200
|
async function applyCatalog(catalog, client, dryRun, targetProject) {
|
|
195
201
|
validateCatalogProject(catalog, targetProject); const current = await snapshot(client, targetProject); validateCatalog(catalog, current); const groups = { parameters: catalog.parameters.length, operators: catalog.operators.length, features: catalog.features.length, feature_sets: catalog.feature_sets.length, datasets: catalog.datasets.length }; const summary = Object.fromEntries(Object.entries(groups).map(([name, count]) => [name, { declared: count, planned: 0, applied: 0, unchanged: 0 }]));
|
|
202
|
+
for (const asset of catalog.parameters) { const assetKey = key(asset); if (current.parameters.has(assetKey)) current.parameters.set(assetKey, normalizeParameter(await client.get(`/parameters/${encodeURIComponent(asset.project)}/${encodeURIComponent(asset.name)}/${encodeURIComponent(asset.version)}`))); }
|
|
196
203
|
for (const asset of catalog.parameters) assertImmutable("parameter", key(asset), asset, current.parameters.get(key(asset)));
|
|
197
204
|
for (const item of catalog.operators) { const existing = current.operators.get(key(item.spec)); if (!existing) continue; let candidate = item.spec; if (item.package_path) { if (existing.code_hash !== sha256File(item.package_path) || !existing.code_artifact) throw new Error(`operator version is immutable: ${key(item.spec)}; publish a new version`); candidate = normalizeOperator({ ...item.spec, code_hash: existing.code_artifact.sha256, package_uri: existing.code_artifact.uri, code_artifact: existing.code_artifact }); } assertImmutable("operator", key(item.spec), candidate, existing); }
|
|
198
205
|
const plain = async (assets, existing, label, endpoint, keyFn = key, params) => { for (const asset of assets) { const assetKey = keyFn(asset); assertImmutable(label, assetKey, asset, existing.get(assetKey)); if (existing.has(assetKey)) summary[label].unchanged += 1; else if (dryRun) summary[label].planned += 1; else { await client.post(endpoint, asset, params); summary[label].applied += 1; } } };
|