@zhuoyuezs/ml-platform 0.1.13 → 0.1.15

package/checksums.json

@@ -7,23 +7,23 @@
     },
     {
       "path": "runtime/business-client/package-lock.json",
-      "sha256": "sha256:667e3849ddf32e18495b60e2a064fa7488dc70c48c3cc433472c3882be936bda",
+      "sha256": "sha256:a966c6f4dd7a74efaec7ca14c41b0a72c42ab53d2360bc45cb680874e1669720",
       "size_bytes": 381
     },
     {
       "path": "runtime/business-client/package.json",
-      "sha256": "sha256:60df549a9afa300d1bd890ad12785bcdf2d7aa03975ad2251c8045268c07ffd6",
+      "sha256": "sha256:f2277bf940755b1237fb923e27596483341b2c3a93c16cd0113ce61c5496280a",
       "size_bytes": 501
     },
     {
       "path": "runtime/business-client/src/catalog.js",
-      "sha256": "sha256:e286c76dc954a11f5d005ed2626fe9d08681015c3a19c95d427cd44999e79169",
-      "size_bytes": 54553
+      "sha256": "sha256:8c636034f22f7fc70c018f7714923c23a641dd7f1b38f709e6feb3a0e773f105",
+      "size_bytes": 55999
     },
     {
       "path": "runtime/business-client/src/cli.js",
-      "sha256": "sha256:5a31e407dba79205e56d2ed1e74260213d0b89c6bb5a665ec0f52e92927ebcaf",
-      "size_bytes": 44897
+      "sha256": "sha256:4f135325b57725c614d266285013df799cdd9a2096d8fbe0a9e9608dae2bcdb7",
+      "size_bytes": 44947
     },
     {
       "path": "runtime/business-client/src/config.js",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zhuoyuezs/ml-platform",
-  "version": "0.1.13",
+  "version": "0.1.15",
   "description": "ML 数据平台 business CLI 与 Agent Skills 管理工具",
   "bin": {
     "ml-platform": "scripts/main.js"

package/release.json

@@ -13,8 +13,8 @@
     "entrypoint": "src/cli.js",
     "name": "ml-platform",
     "path": "runtime/business-client",
-    "sha256": "sha256:373996d6b3a6ca50b810f745637c0b2c8999bd87ce3a38055973a845ddaa6f29",
-    "version": "0.7.3"
+    "sha256": "sha256:02dd0494eccb72dc5da160b65c7c17b266b4da9ce09a31122f434b553f8afbd0",
+    "version": "0.7.5"
   },
   "policy_sha256": "sha256:02fbde0134696b78c0c365d6e0b76c9814227208da5b2b3d4e5dbd82dc4a3606",
   "public_schema_versions": [
@@ -25,7 +25,7 @@
     "ml_data_platform.feature_set/v1",
     "ml_data_platform.dataset_manifest/v1"
   ],
-  "release_version": "0.1.13",
+  "release_version": "0.1.15",
   "runtime_requirements": {
     "node": ">=18",
     "os": [
@@ -37,16 +37,16 @@
   "skills": {
     "feature-management": {
       "path": "skills/feature-management",
-      "requires_cli": ">=0.7.3 <0.8.0",
-      "revision": "0.1.13",
+      "requires_cli": ">=0.7.5 <0.8.0",
+      "revision": "0.1.15",
       "sha256": "sha256:182b6f2df4fe58c946e482dfaf416c7d5b78c6fde9c8c8d7293103246618cb4e"
     },
     "model-lifecycle-management": {
       "path": "skills/model-lifecycle-management",
-      "requires_cli": ">=0.7.3 <0.8.0",
-      "revision": "0.1.13",
+      "requires_cli": ">=0.7.5 <0.8.0",
+      "revision": "0.1.15",
       "sha256": "sha256:61ba17f955b36d1c2e8329a10454d040ee7eca5afdb90ff0914d3f85b6e2d203"
     }
   },
-  "source_commit": "4d04a0ecf38e1272883d7f00b11e1b4193174a43"
+  "source_commit": "b91c349695595e23b0295ec48342b61cd5f80c8e"
 }

package/runtime/business-client/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@zhuoyuezs/ml-platform-business-client",
-  "version": "0.7.3",
+  "version": "0.7.5",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@zhuoyuezs/ml-platform-business-client",
-      "version": "0.7.3",
+      "version": "0.7.5",
       "license": "UNLICENSED",
       "bin": {
         "ml-platform": "src/cli.js"

package/runtime/business-client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zhuoyuezs/ml-platform-business-client",
-  "version": "0.7.3",
+  "version": "0.7.5",
   "private": true,
   "description": "JavaScript implementation of the ML Platform business client",
   "bin": {

package/runtime/business-client/src/catalog.js

@@ -167,7 +167,22 @@ function loadCatalog(directory) {
 }
 
 async function fetchAll(client, endpoint, params = {}) { const rows = []; for (let offset = 0; ; offset += 500) { const response = await client.get(endpoint, { ...params, limit: 500, offset }); if (Array.isArray(response)) return response; const page = response.items; if (!Array.isArray(page)) throw new Error(`${endpoint} returned an invalid list response`); rows.push(...page); if (response.total == null || rows.length >= response.total || page.length < 500) return rows; } }
-async function snapshot(client, targetProject) { const scope = { project: targetProject ?? "default" }; const [parameters, operators, features, featureSets, datasets] = await Promise.all([fetchAll(client, "/parameters", scope), fetchAll(client, "/operators", scope), fetchAll(client, "/features", scope), fetchAll(client, "/feature-sets", scope), fetchAll(client, "/datasets", { ...scope, include_deleted: true })]); return { parameters: new Map(parameters.map((x) => [key(x), x])), operators: new Map(operators.map((x) => [key(normalizeRegistryOperator(x)), normalizeRegistryOperator(x)])), features: new Map(features.map((x) => [key(x), x])), feature_sets: new Map(featureSets.map((x) => [key(normalizeFeatureSet(x)), normalizeFeatureSet(x)])), datasets: new Map(datasets.filter((x) => !x.deleted_at).map((x) => [datasetKey(normalizeRegistryDataset(x)), normalizeRegistryDataset(x)])) }; }
+async function snapshot(client, targetProject) { const scope = { project: targetProject ?? "default" }; const [parameters, operators, features, featureSets, datasets] = await Promise.all([fetchAll(client, "/parameters", scope), fetchAll(client, "/operators", scope), fetchAll(client, "/features", scope), fetchAll(client, "/feature-sets", scope), fetchAll(client, "/datasets", { ...scope, include_deleted: true })]); return { parameters: new Map(parameters.map((x) => [key(x), x])), operators: new Map(operators.map((x) => [key(x), x])), features: new Map(features.map((x) => [key(x), x])), feature_sets: new Map(featureSets.map((x) => [key(x), x])), datasets: new Map(datasets.filter((x) => !x.deleted_at).map((x) => [datasetKey(x), x])) }; }
+
+async function hydrate(client, current, assets, endpoint, normalizer, keyFn = key, seen = new Set()) { for (const asset of assets) { const assetKey = keyFn(asset); if (current.has(assetKey) && !seen.has(assetKey)) { current.set(assetKey, normalizer(await client.get(endpoint(asset)))); seen.add(assetKey); } } }
+async function hydrateCatalog(client, catalog, current) {
+  const hydrated = { parameters: new Set(), operators: new Set(), features: new Set(), feature_sets: new Set(), datasets: new Set() };
+  const resource = (prefix) => (asset) => `/${prefix}/${encodeURIComponent(asset.project)}/${encodeURIComponent(asset.name)}/${encodeURIComponent(asset.version)}`;
+  await hydrate(client, current.parameters, catalog.parameters, resource("parameters"), normalizeParameter, key, hydrated.parameters);
+  const operatorRefs = catalog.features.map((asset) => ({ project: asset.project, name: asset.operator, version: asset.operator_version }));
+  await hydrate(client, current.operators, [...catalog.operators.map((item) => item.spec), ...operatorRefs], resource("operators"), normalizeOperator, key, hydrated.operators);
+  const featureSetRefs = catalog.datasets.map((asset) => ({ project: asset.project, ...asset.feature_set }));
+  await hydrate(client, current.feature_sets, [...catalog.feature_sets, ...featureSetRefs], resource("feature-sets"), normalizeFeatureSet, key, hydrated.feature_sets);
+  await hydrate(client, current.datasets, catalog.datasets, (asset) => `/datasets/${encodeURIComponent(asset.project)}/${encodeURIComponent(asset.dataset_id)}/${encodeURIComponent(asset.dataset_version)}`, normalizeDataset, datasetKey, hydrated.datasets);
+  const declaredFeatureSets = new Map(catalog.feature_sets.map((asset) => [key(asset), asset]));
+  const featureRefs = catalog.datasets.flatMap((asset) => (declaredFeatureSets.get(referenceKey(asset.feature_set)) ?? current.feature_sets.get(referenceKey(asset.feature_set)) ?? { features: [] }).features.map((feature) => ({ project: asset.project, ...feature })));
+  await hydrate(client, current.features, [...catalog.features, ...featureRefs], resource("features"), normalizeFeature, key, hydrated.features);
+}
 
 function requiresPackage(spec) { return ["custom_python", "feature", "spark_udf"].includes(spec.type) && spec.code_artifact == null && !String(spec.package_uri ?? "").startsWith("pythonpath://"); }
 function validateCatalog(catalog, current) {
@@ -198,16 +213,14 @@ function assertImmutable(label, assetKey, candidate, existing) { if (existing &&
 function sha256File(file) { return `sha256:${crypto.createHash("sha256").update(fs.readFileSync(file)).digest("hex")}`; }
 
 async function applyCatalog(catalog, client, dryRun, targetProject) {
-  validateCatalogProject(catalog, targetProject); const current = await snapshot(client, targetProject); validateCatalog(catalog, current); const groups = { parameters: catalog.parameters.length, operators: catalog.operators.length, features: catalog.features.length, feature_sets: catalog.feature_sets.length, datasets: catalog.datasets.length }; const summary = Object.fromEntries(Object.entries(groups).map(([name, count]) => [name, { declared: count, planned: 0, applied: 0, unchanged: 0 }]));
-  for (const asset of catalog.parameters) { const assetKey = key(asset); if (current.parameters.has(assetKey)) current.parameters.set(assetKey, normalizeParameter(await client.get(`/parameters/${encodeURIComponent(asset.project)}/${encodeURIComponent(asset.name)}/${encodeURIComponent(asset.version)}`))); }
-  for (const asset of catalog.features) { const assetKey = key(asset); if (current.features.has(assetKey)) current.features.set(assetKey, normalizeFeature(await client.get(`/features/${encodeURIComponent(asset.project)}/${encodeURIComponent(asset.name)}/${encodeURIComponent(asset.version)}`))); }
+  validateCatalogProject(catalog, targetProject); const current = await snapshot(client, targetProject); await hydrateCatalog(client, catalog, current); validateCatalog(catalog, current); const groups = { parameters: catalog.parameters.length, operators: catalog.operators.length, features: catalog.features.length, feature_sets: catalog.feature_sets.length, datasets: catalog.datasets.length }; const summary = Object.fromEntries(Object.entries(groups).map(([name, count]) => [name, { declared: count, planned: 0, applied: 0, unchanged: 0 }]));
   for (const asset of catalog.parameters) assertImmutable("parameter", key(asset), asset, current.parameters.get(key(asset)));
   for (const item of catalog.operators) { const existing = current.operators.get(key(item.spec)); if (!existing) continue; let candidate = item.spec; if (item.package_path) { if (existing.code_hash !== sha256File(item.package_path) || !existing.code_artifact) throw new Error(`operator version is immutable: ${key(item.spec)}; publish a new version`); candidate = normalizeOperator({ ...item.spec, code_hash: existing.code_artifact.sha256, package_uri: existing.code_artifact.uri, code_artifact: existing.code_artifact }); } assertImmutable("operator", key(item.spec), candidate, existing); }
   const plain = async (assets, existing, label, endpoint, keyFn = key, params) => { for (const asset of assets) { const assetKey = keyFn(asset); assertImmutable(label, assetKey, asset, existing.get(assetKey)); if (existing.has(assetKey)) summary[label].unchanged += 1; else if (dryRun) summary[label].planned += 1; else { await client.post(endpoint, asset, params); summary[label].applied += 1; } } };
   try {
     await plain(catalog.parameters, current.parameters, "parameters", "/parameters");
     for (const item of catalog.operators) { const assetKey = key(item.spec); if (current.operators.has(assetKey)) summary.operators.unchanged += 1; else if (dryRun) summary.operators.planned += 1; else { let candidate = item.spec; if (item.package_path) { const uploaded = await client.uploadOperatorPackage(item.spec.project, item.spec.name, item.spec.version, item.package_path); candidate = normalizeOperator({ ...item.spec, code_hash: uploaded.code_artifact.sha256, package_uri: uploaded.code_artifact.uri, code_artifact: uploaded.code_artifact }); } await client.post("/operators", candidate); summary.operators.applied += 1; } }
-    await plain(catalog.features, current.features, "features", "/features"); await plain(catalog.feature_sets, current.feature_sets, "feature_sets", "/feature-sets"); await plain(catalog.datasets, current.datasets, "datasets", "/datasets", datasetKey, { force: false });
+    await plain(catalog.features, current.features, "features", "/features"); await plain(catalog.feature_sets, current.feature_sets, "feature_sets", "/feature-sets"); await plain(catalog.datasets, current.datasets, "datasets", "/datasets", datasetKey);
   } catch (error) { throw new Error(`catalog apply failed after progress=${JSON.stringify(canonical(summary))}: ${error.message}`); }
   return { status: dryRun ? "validated" : "applied", profile: "server", catalog: { name: catalog.manifest.name, version: catalog.manifest.version, path: catalog.path }, summary };
 }

package/runtime/business-client/src/cli.js

@@ -405,8 +405,9 @@ async function runBusinessCli(argv) {
     ({ result, exitCode } = await submitBuild(api, manifest, build));
   } else if (["add-dataset", "update-dataset"].includes(options.command)) {
     const manifest = readDataset(positional(rest, "manifest JSON")); const force = boolean(rest, "--force"); const project = manifest.project || "default";
-    if (options.command === "add-dataset") result = await client(options).post("/datasets", manifest, { force });
-    else result = await client(options).put(`/datasets/${encodeURIComponent(project)}/${encodeURIComponent(manifest.dataset_id)}/${encodeURIComponent(manifest.dataset_version)}`, manifest, { force });
+    const params = force ? { force: true } : undefined;
+    if (options.command === "add-dataset") result = await client(options).post("/datasets", manifest, params);
+    else result = await client(options).put(`/datasets/${encodeURIComponent(project)}/${encodeURIComponent(manifest.dataset_id)}/${encodeURIComponent(manifest.dataset_version)}`, manifest, params);
   } else if (options.command === "delete-dataset") {
     const id = positional(rest, "dataset id"); const version = positional(rest, "dataset version"); const project = take(rest, "--project", "default");
     result = await client(options).delete(`/datasets/${encodeURIComponent(project)}/${encodeURIComponent(id)}/${encodeURIComponent(version)}`);