@zhuoyuezs/ml-platform 0.1.1 → 0.1.4

package/runtime/business-client/src/catalog.js

@@ -34,7 +34,7 @@ function objectList(value, label) { if (!Array.isArray(value) || value.some((ite
 function stringMap(value, label) { object(value, label); if (Object.entries(value).some(([name, item]) => typeof name !== "string" || typeof item !== "string")) throw new Error(`${label} must contain only string keys and values`); return value; }
 function unique(values, label) { if (new Set(values).size !== values.length) throw new Error(`${label} must be unique`); }
 function ref(value, label) { exactKeys(value, ["name", "version", "project"], ["name", "version"], label); return { name: identifier(value.name, `${label} name`), version: identifier(value.version, `${label} version`) }; }
-function parameterRef(value) { exactKeys(value, ["parameter", "version", "project"], ["parameter", "version"], "feature input"); return { parameter: identifier(value.parameter, "feature input parameter"), version: identifier(value.version, "feature input version") }; }
+function parameterRef(value, parentProject) { exactKeys(value, ["parameter", "version", "project"], ["parameter", "version"], "feature input"); const project = identifier(defaultIfAbsent(value, "project", "default"), "feature input project"); if (parentProject && project !== parentProject) throw new Error("feature input cross-project reference is not allowed"); return { parameter: identifier(value.parameter, "feature input parameter"), version: identifier(value.version, "feature input version"), project }; }
 function key(value) { return `${value.name}:${value.version}`; }
 function datasetKey(value) { return `${value.dataset_id}:${value.dataset_version}`; }
 function referenceKey(value) { return `${value.name}:${value.version}`; }
@@ -60,13 +60,16 @@ function normalizeCodeArtifact(value) { object(value, "operator code_artifact");
 function normalizeParameter(payload) {
   exactKeys(payload, ["schema_version", "project", "name", "display_name", "version", "data_type", "unit", "expected_frequency", "source", "time_semantics", "availability_sla", "value_field", "quality_rules", "rounding", "owner"], ["name", "display_name", "version", "source"], "parameter");
   const source = object(payload.source, "parameter source");
+  if (Object.prototype.hasOwnProperty.call(source, "parameters")) throw new Error("parameter source uses unsupported field 'parameters'; use 'params'");
+  if (source.mode === "sql" && typeof source.sql === "string" && source.sql.trimEnd().endsWith(";")) throw new Error("SQL Parameter source.sql must not end with a semicolon");
   const time = object(defaultIfAbsent(payload, "time_semantics", {}), "parameter time_semantics");
-  return { schema_version: pythonString(defaultIfAbsent(payload, "schema_version", "ml_data_platform.parameter/v1")), name: identifier(payload.name, "parameter name"), display_name: requiredString(payload.display_name, "parameter display_name"), version: identifier(payload.version, "parameter version"), data_type: pythonString(defaultIfAbsent(payload, "data_type", "time_series")), unit: pythonString(defaultIfAbsent(payload, "unit", "")), expected_frequency: pythonString(defaultIfAbsent(payload, "expected_frequency", "10min")), source: { adapter: pythonString(defaultIfAbsent(source, "adapter", "synthetic")), mode: choice(defaultIfAbsent(source, "mode", "direct_column"), ["direct_column", "sql", "derived"], "parameter source mode"), source_metric: optionalString(source.source_metric, "source_metric"), source_group: optionalString(source.source_group, "source_group"), measurement: optionalString(source.measurement, "measurement"), field: optionalString(source.field, "field"), tags: stringMap(defaultIfAbsent(source, "tags", {}), "source tags"), schema: optionalString(aliasIfAbsent(source, "schema", "schema_name"), "schema"), table: optionalString(source.table, "table"), time_column: optionalString(source.time_column, "time_column"), value_column: optionalString(source.value_column, "value_column"), metric_name_column: optionalString(source.metric_name_column, "metric_name_column"), metric_name: optionalString(source.metric_name, "metric_name"), unit_column: optionalString(source.unit_column, "unit_column"), unit: optionalString(aliasIfAbsent(source, "unit", "source_unit"), "source unit"), filters: stringMap(defaultIfAbsent(source, "filters", {}), "source filters"), sql: optionalString(source.sql, "source sql"), params: object(defaultIfAbsent(source, "params", {}), "source params") }, time_semantics: { event_time_field: pythonString(defaultIfAbsent(time, "event_time_field", "event_time")), ingested_at_field: pythonString(defaultIfAbsent(time, "ingested_at_field", "ingested_at")), timezone: pythonString(defaultIfAbsent(time, "timezone", "Asia/Shanghai")), availability: normalizeAvailability(time.availability) }, availability_sla: normalizeAvailabilitySla(payload.availability_sla), value_field: pythonString(defaultIfAbsent(payload, "value_field", "value")), quality_rules: normalizeQualityRules(object(defaultIfAbsent(payload, "quality_rules", {}), "parameter quality_rules")), rounding: normalizeRounding(payload.rounding), owner: pythonString(defaultIfAbsent(payload, "owner", "demo")) };
+  return { schema_version: pythonString(defaultIfAbsent(payload, "schema_version", "ml_data_platform.parameter/v1")), project: identifier(defaultIfAbsent(payload, "project", "default"), "parameter project"), name: identifier(payload.name, "parameter name"), display_name: requiredString(payload.display_name, "parameter display_name"), version: identifier(payload.version, "parameter version"), data_type: pythonString(defaultIfAbsent(payload, "data_type", "time_series")), unit: pythonString(defaultIfAbsent(payload, "unit", "")), expected_frequency: pythonString(defaultIfAbsent(payload, "expected_frequency", "10min")), source: { adapter: pythonString(defaultIfAbsent(source, "adapter", "synthetic")), mode: choice(defaultIfAbsent(source, "mode", "direct_column"), ["direct_column", "sql", "derived"], "parameter source mode"), source_metric: optionalString(source.source_metric, "source_metric"), source_group: optionalString(source.source_group, "source_group"), measurement: optionalString(source.measurement, "measurement"), field: optionalString(source.field, "field"), tags: stringMap(defaultIfAbsent(source, "tags", {}), "source tags"), schema: optionalString(aliasIfAbsent(source, "schema", "schema_name"), "schema"), table: optionalString(source.table, "table"), time_column: optionalString(source.time_column, "time_column"), value_column: optionalString(source.value_column, "value_column"), metric_name_column: optionalString(source.metric_name_column, "metric_name_column"), metric_name: optionalString(source.metric_name, "metric_name"), unit_column: optionalString(source.unit_column, "unit_column"), unit: optionalString(aliasIfAbsent(source, "unit", "source_unit"), "source unit"), filters: stringMap(defaultIfAbsent(source, "filters", {}), "source filters"), sql: optionalString(source.sql, "source sql"), params: object(defaultIfAbsent(source, "params", {}), "source params") }, time_semantics: { event_time_field: pythonString(defaultIfAbsent(time, "event_time_field", "event_time")), ingested_at_field: pythonString(defaultIfAbsent(time, "ingested_at_field", "ingested_at")), timezone: pythonString(defaultIfAbsent(time, "timezone", "Asia/Shanghai")), availability: normalizeAvailability(time.availability) }, availability_sla: normalizeAvailabilitySla(payload.availability_sla), value_field: pythonString(defaultIfAbsent(payload, "value_field", "value")), quality_rules: normalizeQualityRules(object(defaultIfAbsent(payload, "quality_rules", {}), "parameter quality_rules")), rounding: normalizeRounding(payload.rounding), owner: pythonString(defaultIfAbsent(payload, "owner", "demo")) };
 }
 
-function normalizeOperator(payload) { exactKeys(payload, ["schema_version", "project", "name", "version", "type", "function_hash", "entrypoint", "code_hash", "package_uri", "code_artifact", "input_schema", "output_schema", "config_schema", "runtime", "resources", "deterministic", "supports_batch", "supports_online", "owner"], ["name", "version"], "operator"); const artifact = payload.code_artifact == null ? null : normalizeCodeArtifact(payload.code_artifact); const normalized = { schema_version: pythonString(defaultIfAbsent(payload, "schema_version", "ml_data_platform.operator/v1")), name: identifier(payload.name, "operator name"), version: identifier(payload.version, "operator version"), type: choice(defaultIfAbsent(payload, "type", "standard"), ["standard", "custom_python", "spark_udf", "feature"], "operator type"), function_hash: pythonString(defaultIfAbsent(payload, "function_hash", "demo")), entrypoint: optionalString(payload.entrypoint, "operator entrypoint"), code_hash: optionalString(payload.code_hash, "operator code_hash"), package_uri: optionalString(payload.package_uri, "operator package_uri"), code_artifact: artifact, input_schema: object(defaultIfAbsent(payload, "input_schema", {}), "operator input_schema"), output_schema: object(defaultIfAbsent(payload, "output_schema", {}), "operator output_schema"), config_schema: object(defaultIfAbsent(payload, "config_schema", {}), "operator config_schema"), runtime: object(defaultIfAbsent(payload, "runtime", {}), "operator runtime"), resources: object(defaultIfAbsent(payload, "resources", {}), "operator resources"), deterministic: bool(defaultIfAbsent(payload, "deterministic", true), "operator deterministic"), supports_batch: bool(defaultIfAbsent(payload, "supports_batch", true), "operator supports_batch"), supports_online: bool(defaultIfAbsent(payload, "supports_online", false), "operator supports_online"), owner: pythonString(defaultIfAbsent(payload, "owner", "demo")) }; if (artifact && normalized.code_hash !== artifact.sha256) throw new Error("operator code_hash must equal code_artifact.sha256"); if (artifact && normalized.package_uri !== artifact.uri) throw new Error("operator package_uri must equal code_artifact.uri"); return normalized; }
-function normalizeFeature(payload) { exactKeys(payload, ["schema_version", "project", "name", "version", "inputs", "operator", "operator_version", "config", "output_column", "output_dtype", "offline_online_supported", "owner", "description"], ["name", "version", "inputs", "operator", "operator_version", "output_column"], "feature"); const inputs = objectList(payload.inputs, "feature inputs").map(parameterRef); if (!inputs.length) throw new Error("feature inputs must not be empty"); unique(inputs.map(parameterReferenceKey), "feature inputs"); if (typeof payload.output_column !== "string" || !outputColumnPattern.test(payload.output_column)) throw new Error("feature output_column is invalid"); return { schema_version: pythonString(defaultIfAbsent(payload, "schema_version", "ml_data_platform.feature/v1")), name: identifier(payload.name, "feature name"), version: identifier(payload.version, "feature version"), inputs, operator: identifier(payload.operator, "feature operator"), operator_version: identifier(payload.operator_version, "feature operator_version"), config: object(defaultIfAbsent(payload, "config", {}), "feature config"), output_column: payload.output_column, output_dtype: pythonString(defaultIfAbsent(payload, "output_dtype", "float64")), offline_online_supported: bool(defaultIfAbsent(payload, "offline_online_supported", true), "offline_online_supported"), owner: pythonString(defaultIfAbsent(payload, "owner", "demo")), description: pythonString(defaultIfAbsent(payload, "description", "")) }; }
-function normalizeFeatureSet(payload) { exactKeys(payload, ["schema_version", "project", "name", "version", "features", "owner", "description"], ["name", "version", "features"], "feature set"); const features = objectList(payload.features, "feature set features").map((item) => ref(item, "feature reference")); if (!features.length) throw new Error("feature_set features must not be empty"); unique(features.map(referenceKey), "feature_set features"); return { schema_version: pythonString(defaultIfAbsent(payload, "schema_version", "ml_data_platform.feature_set/v1")), name: identifier(payload.name, "feature set name"), version: identifier(payload.version, "feature set version"), features, owner: pythonString(defaultIfAbsent(payload, "owner", "demo")), description: pythonString(defaultIfAbsent(payload, "description", "")) }; }
+function normalizeOperator(payload) { exactKeys(payload, ["schema_version", "project", "name", "version", "type", "function_hash", "entrypoint", "code_hash", "package_uri", "code_artifact", "input_schema", "output_schema", "config_schema", "runtime", "resources", "deterministic", "supports_batch", "supports_online", "owner"], ["name", "version"], "operator"); const artifact = payload.code_artifact == null ? null : normalizeCodeArtifact(payload.code_artifact); const normalized = { schema_version: pythonString(defaultIfAbsent(payload, "schema_version", "ml_data_platform.operator/v1")), project: identifier(defaultIfAbsent(payload, "project", "default"), "operator project"), name: identifier(payload.name, "operator name"), version: identifier(payload.version, "operator version"), type: choice(defaultIfAbsent(payload, "type", "standard"), ["standard", "custom_python", "spark_udf", "feature"], "operator type"), function_hash: pythonString(defaultIfAbsent(payload, "function_hash", "demo")), entrypoint: optionalString(payload.entrypoint, "operator entrypoint"), code_hash: optionalString(payload.code_hash, "operator code_hash"), package_uri: optionalString(payload.package_uri, "operator package_uri"), code_artifact: artifact, input_schema: object(defaultIfAbsent(payload, "input_schema", {}), "operator input_schema"), output_schema: object(defaultIfAbsent(payload, "output_schema", {}), "operator output_schema"), config_schema: object(defaultIfAbsent(payload, "config_schema", {}), "operator config_schema"), runtime: object(defaultIfAbsent(payload, "runtime", {}), "operator runtime"), resources: object(defaultIfAbsent(payload, "resources", {}), "operator resources"), deterministic: bool(defaultIfAbsent(payload, "deterministic", true), "operator deterministic"), supports_batch: bool(defaultIfAbsent(payload, "supports_batch", true), "operator supports_batch"), supports_online: bool(defaultIfAbsent(payload, "supports_online", false), "operator supports_online"), owner: pythonString(defaultIfAbsent(payload, "owner", "demo")) }; if (artifact && normalized.code_hash !== artifact.sha256) throw new Error("operator code_hash must equal code_artifact.sha256"); if (artifact && normalized.package_uri !== artifact.uri) throw new Error("operator package_uri must equal code_artifact.uri"); return normalized; }
+function normalizeRegistryOperator(payload) { const value = object(payload, "operator"); if (!hasOwn(value, "deleted_at")) return normalizeOperator(value); const { deleted_at, ...contract } = value; void deleted_at; return normalizeOperator(contract); }
+function normalizeFeature(payload) { exactKeys(payload, ["schema_version", "project", "name", "version", "inputs", "operator", "operator_version", "config", "output_column", "output_dtype", "offline_online_supported", "owner", "description"], ["name", "version", "inputs", "operator", "operator_version", "output_column"], "feature"); const project = identifier(defaultIfAbsent(payload, "project", "default"), "feature project"); const inputs = objectList(payload.inputs, "feature inputs").map((item) => parameterRef(item, project)); if (!inputs.length) throw new Error("feature inputs must not be empty"); unique(inputs.map(parameterReferenceKey), "feature inputs"); if (typeof payload.output_column !== "string" || !outputColumnPattern.test(payload.output_column)) throw new Error("feature output_column is invalid"); return { schema_version: pythonString(defaultIfAbsent(payload, "schema_version", "ml_data_platform.feature/v1")), project, name: identifier(payload.name, "feature name"), version: identifier(payload.version, "feature version"), inputs, operator: identifier(payload.operator, "feature operator"), operator_version: identifier(payload.operator_version, "feature operator_version"), config: object(defaultIfAbsent(payload, "config", {}), "feature config"), output_column: payload.output_column, output_dtype: pythonString(defaultIfAbsent(payload, "output_dtype", "float64")), offline_online_supported: bool(defaultIfAbsent(payload, "offline_online_supported", true), "offline_online_supported"), owner: pythonString(defaultIfAbsent(payload, "owner", "demo")), description: pythonString(defaultIfAbsent(payload, "description", "")) }; }
+function normalizeFeatureSet(payload) { exactKeys(payload, ["schema_version", "project", "name", "version", "features", "owner", "description"], ["name", "version", "features"], "feature set"); const features = objectList(payload.features, "feature set features").map((item) => ref(item, "feature reference")); if (!features.length) throw new Error("feature_set features must not be empty"); unique(features.map(referenceKey), "feature_set features"); return { schema_version: pythonString(defaultIfAbsent(payload, "schema_version", "ml_data_platform.feature_set/v1")), project: identifier(defaultIfAbsent(payload, "project", "default"), "feature set project"), name: identifier(payload.name, "feature set name"), version: identifier(payload.version, "feature set version"), features, owner: pythonString(defaultIfAbsent(payload, "owner", "demo")), description: pythonString(defaultIfAbsent(payload, "description", "")) }; }
 function timestamp(value, label) {
   requiredString(value, label);
   const match = timestampPattern.exec(value);
@@ -90,15 +93,15 @@ function timestamp(value, label) {
   return value;
 }
 function validateReferenceProject(value, parentProject, label) { if (value.project == null) return; const project = identifier(value.project, `${label} project`); if (project !== parentProject) throw new Error(`${label} cross-project reference is not allowed`); }
-function normalizeParameterRequest(value, parentProject) { object(value, "dataset parameter request"); validateReferenceProject(value, parentProject, "dataset parameter request"); let alignment = null; if (value.alignment != null) { object(value.alignment, "parameter alignment"); alignment = { grid: pythonString(defaultIfAbsent(value.alignment, "grid", "10min")), method: choice(defaultIfAbsent(value.alignment, "method", "last_before_or_at"), ["last_before_or_at", "nearest", "none"], "parameter alignment method") }; } return { name: identifier(value.name, "dataset parameter name"), version: identifier(value.version, "dataset parameter version"), alias: optionalString(value.alias, "dataset parameter alias"), alignment, required: bool(defaultIfAbsent(value, "required", true), "dataset parameter required"), missing_policy: choice(defaultIfAbsent(value, "missing_policy", "report_only"), ["report_only", "fail_on_requested_range_gap", "drop_target_and_lookback"], "dataset parameter missing_policy") }; }
-function normalizeMissingPolicy(value, parentProject) { exactKeys(value, ["parameter", "version", "project", "policy"], ["parameter", "version"], "dataset parameter missing policy"); validateReferenceProject(value, parentProject, "dataset parameter missing policy"); return { parameter: identifier(value.parameter, "dataset parameter missing policy parameter"), version: identifier(value.version, "dataset parameter missing policy version"), policy: choice(defaultIfAbsent(value, "policy", "report_only"), ["report_only", "fail_on_requested_range_gap", "drop_target_and_lookback"], "dataset parameter missing policy policy") }; }
+function normalizeParameterRequest(value, parentProject) { object(value, "dataset parameter request"); validateReferenceProject(value, parentProject, "dataset parameter request"); let alignment = null; if (value.alignment != null) { object(value.alignment, "parameter alignment"); alignment = { grid: pythonString(defaultIfAbsent(value.alignment, "grid", "10min")), method: choice(defaultIfAbsent(value.alignment, "method", "last_before_or_at"), ["last_before_or_at", "nearest", "none"], "parameter alignment method") }; } return { name: identifier(value.name, "dataset parameter name"), version: identifier(value.version, "dataset parameter version"), project: parentProject, alias: optionalString(value.alias, "dataset parameter alias"), alignment, required: bool(defaultIfAbsent(value, "required", true), "dataset parameter required"), missing_policy: choice(defaultIfAbsent(value, "missing_policy", "report_only"), ["report_only", "fail_on_requested_range_gap", "drop_target_and_lookback"], "dataset parameter missing_policy") }; }
+function normalizeMissingPolicy(value, parentProject) { exactKeys(value, ["parameter", "version", "project", "policy"], ["parameter", "version"], "dataset parameter missing policy"); validateReferenceProject(value, parentProject, "dataset parameter missing policy"); return { parameter: identifier(value.parameter, "dataset parameter missing policy parameter"), version: identifier(value.version, "dataset parameter missing policy version"), project: parentProject, policy: choice(defaultIfAbsent(value, "policy", "report_only"), ["report_only", "fail_on_requested_range_gap", "drop_target_and_lookback"], "dataset parameter missing policy policy") }; }
 function normalizePreprocess(value) { object(value, "dataset preprocess"); return { name: identifier(value.name, "preprocess name"), version: identifier(value.version, "preprocess version"), function_hash: pythonString(defaultIfAbsent(value, "function_hash", "demo")), config: object(defaultIfAbsent(value, "config", {}), "preprocess config") }; }
-function normalizeTarget(value) { object(value, "dataset target"); const result = structuredClone(value); if (result.parameter && typeof result.parameter === "object" && !Array.isArray(result.parameter)) delete result.parameter.project; return result; }
+function normalizeTarget(value, parentProject) { object(value, "dataset target"); const result = structuredClone(value); if (result.parameter && typeof result.parameter === "object" && !Array.isArray(result.parameter)) result.parameter = parameterRef(result.parameter, parentProject); return result; }
 function distinct(left, right, label) { if (left === right) throw new Error(`${label} inputs must be different`); }
 function normalizeRelationshipCheck(value, label) { object(value, label); const type = choice(value.type, ["compare", "absolute_difference", "ratio_range", "sum_equals", "co_presence"], `${label} type`); if (type === "co_presence") { exactKeys(value, ["type"], ["type"], label); return { type }; } if (type === "compare") { exactKeys(value, ["type", "left", "operator", "right", "tolerance"], ["type", "left", "operator", "right"], label); const left = identifier(value.left, `${label} left`); const right = identifier(value.right, `${label} right`); distinct(left, right, label); return { type, left, operator: choice(value.operator, ["lt", "lte", "gt", "gte", "eq", "ne"], `${label} operator`), right, tolerance: nonNegativeNumber(defaultIfAbsent(value, "tolerance", 0), `${label} tolerance`) }; } if (type === "absolute_difference") { exactKeys(value, ["type", "left", "right", "max_difference"], ["type", "left", "right", "max_difference"], label); const left = identifier(value.left, `${label} left`); const right = identifier(value.right, `${label} right`); distinct(left, right, label); return { type, left, right, max_difference: nonNegativeNumber(value.max_difference, `${label} max_difference`) }; } if (type === "ratio_range") { exactKeys(value, ["type", "numerator", "denominator", "min", "max", "inclusive_min", "inclusive_max"], ["type", "numerator", "denominator"], label); const numerator = identifier(value.numerator, `${label} numerator`); const denominator = identifier(value.denominator, `${label} denominator`); distinct(numerator, denominator, label); const range = validRange([defaultIfAbsent(value, "min", null), defaultIfAbsent(value, "max", null)]); return { type, numerator, denominator, min: range[0], max: range[1], inclusive_min: bool(defaultIfAbsent(value, "inclusive_min", true), `${label} inclusive_min`), inclusive_max: bool(defaultIfAbsent(value, "inclusive_max", true), `${label} inclusive_max`) }; } exactKeys(value, ["type", "terms", "target", "tolerance"], ["type", "terms", "target"], label); if (!Array.isArray(value.terms) || value.terms.some((item) => typeof item !== "string" || !item)) throw new Error(`${label} terms must be a list of non-empty strings`); const terms = value.terms.map((item) => identifier(item, `${label} term`)); if (!terms.length) throw new Error(`${label} terms must not be empty`); unique(terms, `${label} terms`); const target = identifier(value.target, `${label} target`); if (terms.includes(target)) throw new Error(`${label} target must not also be a term`); return { type, terms, target, tolerance: nonNegativeNumber(defaultIfAbsent(value, "tolerance", 0), `${label} tolerance`) }; }
 function relationshipAliases(check) { if (["compare", "absolute_difference"].includes(check.type)) return [check.left, check.right]; if (check.type === "ratio_range") return [check.numerator, check.denominator]; if (check.type === "sum_equals") return [...check.terms, check.target]; return []; }
 function normalizeRelationshipRule(value, relationshipId) { const label = `parameter relationship ${relationshipId} rule`; exactKeys(value, ["id", "stage", "check", "acceptance", "enforcement", "description"], ["id", "check"], label); return { id: identifier(value.id, `${label} id`), stage: choice(defaultIfAbsent(value, "stage", "normalized_source"), ["normalized_source", "post_preprocess", "aligned_grid"], `${label} stage`), check: normalizeRelationshipCheck(value.check, `${label} check`), acceptance: normalizeAcceptance(object(defaultIfAbsent(value, "acceptance", {}), `${label} acceptance`), label), enforcement: choice(defaultIfAbsent(value, "enforcement", "warn"), ["observe", "warn", "fail"], `${label} enforcement`), description: pythonString(defaultIfAbsent(value, "description", "")) }; }
-function normalizeParameterRelationship(value) { exactKeys(value, ["id", "inputs", "alignment", "rules", "description"], ["id", "inputs", "rules"], "parameter relationship"); const id = identifier(value.id, "parameter relationship id"); const inputs = objectList(value.inputs, "parameter relationship inputs").map((item) => { exactKeys(item, ["alias", "parameter", "version"], ["alias", "parameter", "version"], "parameter relationship input"); return { alias: identifier(item.alias, "parameter relationship input alias"), parameter: identifier(item.parameter, "parameter relationship input parameter"), version: identifier(item.version, "parameter relationship input version") }; }); if (inputs.length < 2) throw new Error("parameter relationship requires at least two inputs"); unique(inputs.map((item) => item.alias), "parameter relationship input aliases"); unique(inputs.map(parameterReferenceKey), "parameter relationship inputs"); const alignmentValue = object(defaultIfAbsent(value, "alignment", {}), "parameter relationship alignment"); exactKeys(alignmentValue, ["method", "tolerance"], [], "parameter relationship alignment"); const tolerance = optionalDuration(alignmentValue.tolerance, "parameter relationship alignment tolerance"); if (tolerance != null && durationSeconds(tolerance) === 0) throw new Error("parameter relationship alignment tolerance must be positive"); const alignment = { method: choice(defaultIfAbsent(alignmentValue, "method", "last_before_or_at"), ["exact", "last_before_or_at"], "parameter relationship alignment method"), tolerance }; const rules = objectList(value.rules, "parameter relationship rules").map((item) => normalizeRelationshipRule(item, id)); if (!rules.length) throw new Error("parameter relationship rules must not be empty"); unique(rules.map((item) => item.id), `parameter relationship ${id} rule ids`); const aliases = new Set(inputs.map((item) => item.alias)); for (const rule of rules) { if (rule.stage !== "aligned_grid") throw new Error(`parameter relationship ${id} rule ${rule.id} must use stage=aligned_grid`); const missing = relationshipAliases(rule.check).filter((item) => !aliases.has(item)); if (missing.length) throw new Error(`parameter relationship ${id} rule ${rule.id} references undeclared aliases: ${missing.join(", ")}`); } return { id, inputs, alignment, rules, description: pythonString(defaultIfAbsent(value, "description", "")) }; }
+function normalizeParameterRelationship(value, parentProject) { exactKeys(value, ["id", "inputs", "alignment", "rules", "description"], ["id", "inputs", "rules"], "parameter relationship"); const id = identifier(value.id, "parameter relationship id"); const inputs = objectList(value.inputs, "parameter relationship inputs").map((item) => { exactKeys(item, ["alias", "parameter", "version", "project"], ["alias", "parameter", "version"], "parameter relationship input"); validateReferenceProject(item, parentProject, "parameter relationship input"); return { alias: identifier(item.alias, "parameter relationship input alias"), parameter: identifier(item.parameter, "parameter relationship input parameter"), version: identifier(item.version, "parameter relationship input version"), project: parentProject }; }); if (inputs.length < 2) throw new Error("parameter relationship requires at least two inputs"); unique(inputs.map((item) => item.alias), "parameter relationship input aliases"); unique(inputs.map(parameterReferenceKey), "parameter relationship inputs"); const alignmentValue = object(defaultIfAbsent(value, "alignment", {}), "parameter relationship alignment"); exactKeys(alignmentValue, ["method", "tolerance"], [], "parameter relationship alignment"); const tolerance = optionalDuration(alignmentValue.tolerance, "parameter relationship alignment tolerance"); if (tolerance != null && durationSeconds(tolerance) === 0) throw new Error("parameter relationship alignment tolerance must be positive"); const alignment = { method: choice(defaultIfAbsent(alignmentValue, "method", "last_before_or_at"), ["exact", "last_before_or_at"], "parameter relationship alignment method"), tolerance }; const rules = objectList(value.rules, "parameter relationship rules").map((item) => normalizeRelationshipRule(item, id)); if (!rules.length) throw new Error("parameter relationship rules must not be empty"); unique(rules.map((item) => item.id), `parameter relationship ${id} rule ids`); const aliases = new Set(inputs.map((item) => item.alias)); for (const rule of rules) { if (rule.stage !== "aligned_grid") throw new Error(`parameter relationship ${id} rule ${rule.id} must use stage=aligned_grid`); const missing = relationshipAliases(rule.check).filter((item) => !aliases.has(item)); if (missing.length) throw new Error(`parameter relationship ${id} rule ${rule.id} references undeclared aliases: ${missing.join(", ")}`); } return { id, inputs, alignment, rules, description: pythonString(defaultIfAbsent(value, "description", "")) }; }
 function normalizeDataset(payload) {
   object(payload, "dataset"); payload = Object.fromEntries(Object.entries(payload).filter(([name]) => name !== "deleted_at"));
   exactKeys(payload, ["schema_version", "project", "dataset_id", "dataset_version", "mode", "read_policy", "snapshot_id", "as_of", "time_range", "parameters", "feature_set", "preprocess", "output", "source_read", "realtime_fetch", "abnormal_windows", "prediction", "target", "parameter_relationships", "parameter_missing_policies", "rowset", "rowset_splits", "endpoint_policy"], ["dataset_id", "time_range", "feature_set"], "dataset");
@@ -116,14 +119,27 @@ function normalizeDataset(payload) {
   unique(missingPolicies.map(parameterReferenceKey), "dataset parameter_missing_policies Parameter references");
   const overlap = parameters.map(referenceKey).filter((item) => new Set(missingPolicies.map(parameterReferenceKey)).has(item)); if (overlap.length) throw new Error("dataset parameter_missing_policies is only for Feature-only Parameters");
   if (payload.endpoint_policy != null) { const drops = [...parameters.filter((item) => item.missing_policy === "drop_target_and_lookback"), ...missingPolicies.filter((item) => item.policy === "drop_target_and_lookback")]; if (drops.length) throw new Error("dataset endpoint_policy is incompatible with drop_target_and_lookback"); }
-  const output = { schema_version: pythonString(defaultIfAbsent(payload, "schema_version", "ml_data_platform.dataset_manifest/v1")), dataset_id: identifier(payload.dataset_id, "dataset id"), dataset_version: identifier(defaultIfAbsent(payload, "dataset_version", "v1"), "dataset version"), mode: choice(defaultIfAbsent(payload, "mode", "training"), ["training", "inference"], "dataset mode"), read_policy: readPolicy, snapshot_id: snapshotId, as_of: asOf, time_range: { start: timestamp(time.start, "dataset time_range start"), end: timestamp(time.end, "dataset time_range end"), grid: pythonString(defaultIfAbsent(time, "grid", "10min")) }, parameters, feature_set: ref(object(payload.feature_set, "dataset feature set"), "dataset feature set"), preprocess: objectList(defaultIfAbsent(payload, "preprocess", []), "dataset preprocess").map(normalizePreprocess), output: object(defaultIfAbsent(payload, "output", {}), "dataset output") };
+  const output = { schema_version: pythonString(defaultIfAbsent(payload, "schema_version", "ml_data_platform.dataset_manifest/v1")), project: parentProject, dataset_id: identifier(payload.dataset_id, "dataset id"), dataset_version: identifier(defaultIfAbsent(payload, "dataset_version", "v1"), "dataset version"), mode: choice(defaultIfAbsent(payload, "mode", "training"), ["training", "inference"], "dataset mode"), read_policy: readPolicy, snapshot_id: snapshotId, as_of: asOf, time_range: { start: timestamp(time.start, "dataset time_range start"), end: timestamp(time.end, "dataset time_range end"), grid: pythonString(defaultIfAbsent(time, "grid", "10min")) }, parameters, feature_set: ref(object(payload.feature_set, "dataset feature set"), "dataset feature set"), preprocess: objectList(defaultIfAbsent(payload, "preprocess", []), "dataset preprocess").map(normalizePreprocess), output: object(defaultIfAbsent(payload, "output", {}), "dataset output") };
   if (missingPolicies.length) output.parameter_missing_policies = missingPolicies;
   for (const name of ["source_read", "realtime_fetch", "abnormal_windows", "prediction", "rowset", "rowset_splits", "endpoint_policy"]) if (payload[name] != null) output[name] = payload[name];
-  if (payload.target != null) output.target = normalizeTarget(payload.target);
-  const relationships = objectList(defaultIfAbsent(payload, "parameter_relationships", []), "dataset parameter_relationships").map(normalizeParameterRelationship); unique(relationships.map((item) => item.id), "dataset parameter_relationship ids"); if (relationships.length) output.parameter_relationships = relationships;
+  if (payload.target != null) output.target = normalizeTarget(payload.target, parentProject);
+  const relationships = objectList(defaultIfAbsent(payload, "parameter_relationships", []), "dataset parameter_relationships").map((item) => normalizeParameterRelationship(item, parentProject)); unique(relationships.map((item) => item.id), "dataset parameter_relationship ids"); if (relationships.length) output.parameter_relationships = relationships;
   return output;
 }
 
+function normalizeRegistryDataset(payload) {
+  const value = object(payload, "registered dataset");
+  const responseOnly = ["entity_keys", "label_materializations"];
+  const contract = Object.fromEntries(Object.entries(value).filter(([name]) => !responseOnly.includes(name)));
+  const normalized = normalizeDataset(contract);
+  for (const name of responseOnly) {
+    if (!hasOwn(value, name)) continue;
+    if (!Array.isArray(value[name])) throw new Error(`registered dataset ${name} must be a list`);
+    normalized[name] = structuredClone(value[name]);
+  }
+  return normalized;
+}
+
 function assetPath(root, relative, suffix) { if (typeof relative !== "string" || !relative) throw new Error("catalog asset path must be a non-empty string"); if (path.isAbsolute(relative)) throw new Error(`catalog asset path must be relative: ${relative}`); const target = path.resolve(root, relative); if (path.extname(target).toLowerCase() !== suffix) throw new Error(`catalog asset must use ${suffix}: ${relative}`); if (!target.startsWith(`${root}${path.sep}`)) throw new Error(`catalog asset path escapes catalog directory: ${relative}`); if (!fs.existsSync(target) || !fs.statSync(target).isFile()) throw new Error(`catalog asset not found: ${target}`); return target; }
 function readJson(file) { try { return JSON.parse(fs.readFileSync(file, "utf8")); } catch (error) { throw new Error(`cannot read JSON asset ${file}: ${error.message}`); } }
 function readObject(file) { return object(readJson(file), `JSON file ${file}`); }
@@ -145,13 +161,13 @@ function loadCatalog(directory) {
 }
 
 async function fetchAll(client, endpoint, params = {}) { const rows = []; for (let offset = 0; ; offset += 500) { const response = await client.get(endpoint, { ...params, limit: 500, offset }); if (Array.isArray(response)) return response; const page = response.items; if (!Array.isArray(page)) throw new Error(`${endpoint} returned an invalid list response`); rows.push(...page); if (response.total == null || rows.length >= response.total || page.length < 500) return rows; } }
-async function snapshot(client, targetProject) { const scope = targetProject ? { project: targetProject } : {}; const [parameters, operators, features, featureSets, datasets] = await Promise.all([fetchAll(client, "/parameters", scope), fetchAll(client, "/operators", scope), fetchAll(client, "/features", scope), fetchAll(client, "/feature-sets", scope), fetchAll(client, "/datasets", { ...scope, include_deleted: true })]); return { parameters: new Map(parameters.map((x) => [key(normalizeParameter(x)), normalizeParameter(x)])), operators: new Map(operators.map((x) => [key(normalizeOperator(x)), normalizeOperator(x)])), features: new Map(features.map((x) => [key(normalizeFeature(x)), normalizeFeature(x)])), feature_sets: new Map(featureSets.map((x) => [key(normalizeFeatureSet(x)), normalizeFeatureSet(x)])), datasets: new Map(datasets.filter((x) => !x.deleted_at).map((x) => [datasetKey(normalizeDataset(x)), normalizeDataset(x)])) }; }
+async function snapshot(client, targetProject) { const scope = { project: targetProject ?? "default" }; const [parameters, operators, features, featureSets, datasets] = await Promise.all([fetchAll(client, "/parameters", scope), fetchAll(client, "/operators", scope), fetchAll(client, "/features", scope), fetchAll(client, "/feature-sets", scope), fetchAll(client, "/datasets", { ...scope, include_deleted: true })]); return { parameters: new Map(parameters.map((x) => [key(normalizeParameter(x)), normalizeParameter(x)])), operators: new Map(operators.map((x) => [key(normalizeRegistryOperator(x)), normalizeRegistryOperator(x)])), features: new Map(features.map((x) => [key(normalizeFeature(x)), normalizeFeature(x)])), feature_sets: new Map(featureSets.map((x) => [key(normalizeFeatureSet(x)), normalizeFeatureSet(x)])), datasets: new Map(datasets.filter((x) => !x.deleted_at).map((x) => [datasetKey(normalizeRegistryDataset(x)), normalizeRegistryDataset(x)])) }; }
 
 function requiresPackage(spec) { return ["custom_python", "feature", "spark_udf"].includes(spec.type) && spec.code_artifact == null && !String(spec.package_uri ?? "").startsWith("pythonpath://"); }
 function validateCatalog(catalog, current) {
   const parameters = new Set([...current.parameters.keys(), ...catalog.parameters.map(key)]); const operators = new Map([...current.operators, ...catalog.operators.map((x) => [key(x.spec), x.spec])]); const features = new Map([...current.features, ...catalog.features.map((x) => [key(x), x])]); const featureSets = new Map([...current.feature_sets, ...catalog.feature_sets.map((x) => [key(x), x])]);
   for (const item of catalog.operators) if (!item.package_path && requiresPackage(item.spec)) throw new Error(`catalog operator ${key(item.spec)} requires a wheel package path`);
-  for (const feature of catalog.features) { for (const input of feature.inputs) if (!parameters.has(parameterReferenceKey(input))) throw new Error(`catalog feature ${key(feature)} references missing parameter ${parameterReferenceKey(input)}`); const operatorKey = `${feature.operator}:${feature.operator_version}`; const operator = operators.get(operatorKey); if (!operator) throw new Error(`catalog feature ${key(feature)} references missing operator ${operatorKey}`); if (operator.type !== "feature") throw new Error(`catalog feature ${key(feature)} requires OperatorSpec.type='feature'`); const declaredInputs = new Set(operator.input_schema.parameters ?? []); if (declaredInputs.size) for (const input of feature.inputs) if (!declaredInputs.has(parameterReferenceKey(input))) throw new Error(`catalog feature ${key(feature)} input is not declared by ${operatorKey}`); const columns = operator.output_schema.columns ?? []; if (columns.length && !columns.includes(feature.output_column)) throw new Error(`catalog feature ${key(feature)} output_column is not declared by ${operatorKey}`); }
+  for (const feature of catalog.features) { for (const input of feature.inputs) if (!parameters.has(parameterReferenceKey(input))) throw new Error(`catalog feature ${key(feature)} references missing parameter ${parameterReferenceKey(input)}`); const operatorKey = `${feature.operator}:${feature.operator_version}`; const operator = operators.get(operatorKey); if (!operator) throw new Error(`catalog feature ${key(feature)} references missing operator ${operatorKey}`); if (operator.type !== "feature") throw new Error(`catalog feature ${key(feature)} requires OperatorSpec.type='feature'`); const declaredInputs = new Set(operator.input_schema.parameters ?? []); if (declaredInputs.size) for (const input of feature.inputs) { const inputKey = parameterReferenceKey(input); const qualifiedInputKey = `${operator.project ?? feature.project ?? "default"}/${inputKey}`; if (!declaredInputs.has(inputKey) && !declaredInputs.has(qualifiedInputKey)) throw new Error(`catalog feature ${key(feature)} input is not declared by ${operatorKey}`); } const columns = operator.output_schema.columns ?? []; if (columns.length && !columns.includes(feature.output_column)) throw new Error(`catalog feature ${key(feature)} output_column is not declared by ${operatorKey}`); }
   for (const set of catalog.feature_sets) { const names = []; for (const item of set.features) { const feature = features.get(referenceKey(item)); if (!feature) throw new Error(`catalog feature set ${key(set)} references missing feature ${referenceKey(item)}`); names.push(feature.name); } unique(names, `catalog feature set ${key(set)} final output names`); }
   for (const dataset of catalog.datasets) {
     const datasetId = datasetKey(dataset); const declared = new Set();
@@ -163,22 +179,29 @@ function validateCatalog(catalog, current) {
     for (const item of dataset.preprocess) if (!operators.has(referenceKey(item))) throw new Error(`catalog dataset ${datasetId} references missing preprocess operator ${referenceKey(item)}`);
   }
 }
+function validateCatalogProject(catalog, targetProject) {
+  const projects = [...catalog.parameters, ...catalog.operators.map((item) => item.spec), ...catalog.features, ...catalog.feature_sets, ...catalog.datasets].map((item) => item.project ?? "default");
+  const uniqueProjects = [...new Set(projects)];
+  if (uniqueProjects.length > 1) throw new Error(`catalog contains multiple projects: ${uniqueProjects.sort().join(", ")}`);
+  const expected = targetProject ?? "default";
+  if (uniqueProjects.length && uniqueProjects[0] !== expected) throw new Error(`catalog project ${uniqueProjects[0]} does not match target project ${expected}`);
+}
 function canonical(value) { if (Array.isArray(value)) return value.map(canonical); if (value && typeof value === "object") return Object.fromEntries(Object.keys(value).sort().map((name) => [name, canonical(value[name])])); return value; }
 function equal(left, right) { return JSON.stringify(canonical(left)) === JSON.stringify(canonical(right)); }
 function assertImmutable(label, assetKey, candidate, existing) { if (existing && !equal(candidate, existing)) { const error = new Error(`${label} version is immutable: ${assetKey}`); error.code = "EEXIST"; throw error; } }
 function sha256File(file) { return `sha256:${crypto.createHash("sha256").update(fs.readFileSync(file)).digest("hex")}`; }
 
 async function applyCatalog(catalog, client, dryRun, targetProject) {
-  const current = await snapshot(client, targetProject); validateCatalog(catalog, current); const groups = { parameters: catalog.parameters.length, operators: catalog.operators.length, features: catalog.features.length, feature_sets: catalog.feature_sets.length, datasets: catalog.datasets.length }; const summary = Object.fromEntries(Object.entries(groups).map(([name, count]) => [name, { declared: count, planned: 0, applied: 0, unchanged: 0 }]));
+  validateCatalogProject(catalog, targetProject); const current = await snapshot(client, targetProject); validateCatalog(catalog, current); const groups = { parameters: catalog.parameters.length, operators: catalog.operators.length, features: catalog.features.length, feature_sets: catalog.feature_sets.length, datasets: catalog.datasets.length }; const summary = Object.fromEntries(Object.entries(groups).map(([name, count]) => [name, { declared: count, planned: 0, applied: 0, unchanged: 0 }]));
   for (const asset of catalog.parameters) assertImmutable("parameter", key(asset), asset, current.parameters.get(key(asset)));
   for (const item of catalog.operators) { const existing = current.operators.get(key(item.spec)); if (!existing) continue; let candidate = item.spec; if (item.package_path) { if (existing.code_hash !== sha256File(item.package_path) || !existing.code_artifact) throw new Error(`operator version is immutable: ${key(item.spec)}; publish a new version`); candidate = normalizeOperator({ ...item.spec, code_hash: existing.code_artifact.sha256, package_uri: existing.code_artifact.uri, code_artifact: existing.code_artifact }); } assertImmutable("operator", key(item.spec), candidate, existing); }
   const plain = async (assets, existing, label, endpoint, keyFn = key, params) => { for (const asset of assets) { const assetKey = keyFn(asset); assertImmutable(label, assetKey, asset, existing.get(assetKey)); if (existing.has(assetKey)) summary[label].unchanged += 1; else if (dryRun) summary[label].planned += 1; else { await client.post(endpoint, asset, params); summary[label].applied += 1; } } };
   try {
     await plain(catalog.parameters, current.parameters, "parameters", "/parameters");
-    for (const item of catalog.operators) { const assetKey = key(item.spec); if (current.operators.has(assetKey)) summary.operators.unchanged += 1; else if (dryRun) summary.operators.planned += 1; else { let candidate = item.spec; if (item.package_path) { const uploaded = await client.uploadOperatorPackage(targetProject ?? "default", item.spec.name, item.spec.version, item.package_path); candidate = normalizeOperator({ ...item.spec, code_hash: uploaded.code_artifact.sha256, package_uri: uploaded.code_artifact.uri, code_artifact: uploaded.code_artifact }); } await client.post("/operators", candidate); summary.operators.applied += 1; } }
+    for (const item of catalog.operators) { const assetKey = key(item.spec); if (current.operators.has(assetKey)) summary.operators.unchanged += 1; else if (dryRun) summary.operators.planned += 1; else { let candidate = item.spec; if (item.package_path) { const uploaded = await client.uploadOperatorPackage(item.spec.project, item.spec.name, item.spec.version, item.package_path); candidate = normalizeOperator({ ...item.spec, code_hash: uploaded.code_artifact.sha256, package_uri: uploaded.code_artifact.uri, code_artifact: uploaded.code_artifact }); } await client.post("/operators", candidate); summary.operators.applied += 1; } }
     await plain(catalog.features, current.features, "features", "/features"); await plain(catalog.feature_sets, current.feature_sets, "feature_sets", "/feature-sets"); await plain(catalog.datasets, current.datasets, "datasets", "/datasets", datasetKey, { force: false });
   } catch (error) { throw new Error(`catalog apply failed after progress=${JSON.stringify(canonical(summary))}: ${error.message}`); }
   return { status: dryRun ? "validated" : "applied", profile: "server", catalog: { name: catalog.manifest.name, version: catalog.manifest.version, path: catalog.path }, summary };
 }
 
-module.exports = { applyCatalog, fetchAll, loadCatalog, normalizeDataset, normalizeFeature, normalizeFeatureSet, normalizeOperator, normalizeParameter, validateCatalog };
+module.exports = { applyCatalog, fetchAll, loadCatalog, normalizeDataset, normalizeFeature, normalizeFeatureSet, normalizeOperator, normalizeRegistryDataset, normalizeRegistryOperator, normalizeParameter, validateCatalog, validateCatalogProject };

package/runtime/business-client/src/cli.js

@@ -6,14 +6,18 @@ const fs = require("fs");
 const path = require("path");
 const { configPath, expandHome, loadApiUrl, normalizeApiUrl, saveApiUrl } = require("./config");
 const { PlatformApiClient } = require("./http");
-const { applyCatalog, loadCatalog, normalizeDataset } = require("./catalog");
+const { applyCatalog, loadCatalog, normalizeDataset, normalizeRegistryDataset } = require("./catalog");
 const { version: CLIENT_VERSION } = require("../package.json");
 
 const BUSINESS_COMMANDS = new Set([
   "version", "create-project", "list-projects", "get-project", "delete-project",
   "configure", "show-config", "health", "list-parameters", "list-operators", "list-features",
-  "list-feature-sets", "list-datasets", "list-jobs", "resolve-manifest", "build-dataset",
-  "get-job", "wait-job", "get-dataset-artifact", "download-dataset-artifact", "fetch-inference-data", "apply",
+  "list-feature-sets", "list-datasets", "get-dataset", "list-jobs", "resolve-manifest",
+  "resolve-dataset", "build-dataset", "build-registered-dataset",
+  "add-parameter", "add-feature", "add-feature-set", "add-operator", "publish-operator",
+  "add-dataset", "update-dataset", "delete-dataset", "list-dataset-artifacts",
+  "register-operator", "show-operator-specs", "delete-operator", "run-operator",
+  "get-job", "wait-job", "retry-job", "cancel-job", "get-dataset-artifact", "download-dataset-artifact", "fetch-inference-data", "apply",
 ]);
 
 const COMMAND_USAGE = {
@@ -29,12 +33,30 @@ const COMMAND_USAGE = {
   "list-operators": "list-operators [--project PROJECT] [-q QUERY] [--limit N] [--offset N]",
   "list-features": "list-features [--project PROJECT] [-q QUERY] [--limit N] [--offset N]",
   "list-feature-sets": "list-feature-sets [--project PROJECT] [-q QUERY] [--limit N] [--offset N]",
-  "list-datasets": "list-datasets [--project PROJECT] [-q QUERY] [--limit N] [--offset N]",
+  "list-datasets": "list-datasets [--project PROJECT] [-q QUERY] [--limit N] [--offset N] [--include-deleted]",
+  "add-parameter": "add-parameter SPEC_JSON",
+  "add-feature": "add-feature SPEC_JSON",
+  "add-feature-set": "add-feature-set SPEC_JSON",
+  "add-operator": "add-operator SPEC_JSON",
+  "publish-operator": "publish-operator SPEC_JSON --package PATH",
+  "add-dataset": "add-dataset MANIFEST_JSON [--force]",
+  "update-dataset": "update-dataset MANIFEST_JSON [--force]",
+  "delete-dataset": "delete-dataset DATASET_ID DATASET_VERSION [--project PROJECT]",
+  "list-dataset-artifacts": "list-dataset-artifacts [--project PROJECT] [--dataset-id ID] [--dataset-version VERSION] [--include-deleted] [--limit N] [--offset N]",
+  "get-dataset": "get-dataset DATASET_ID DATASET_VERSION [--project PROJECT] [--include-deleted]",
   "list-jobs": "list-jobs [-q QUERY] [--limit N] [--offset N]",
   "resolve-manifest": "resolve-manifest MANIFEST [--out PATH]",
-  "build-dataset": "build-dataset MANIFEST [--source-mode direct] [--upload-chronon-metadata] [--partition-duration DURATION] [--max-parallelism N] [--no-resume] [--wait] [--poll-interval SECONDS] [--wait-timeout-seconds SECONDS]",
+  "resolve-dataset": "resolve-dataset DATASET_ID DATASET_VERSION [--project PROJECT] [--out PATH]",
+  "build-dataset": "build-dataset MANIFEST [--engine chronon] [--source-mode direct] [--upload-chronon-metadata] [--partition-duration DURATION] [--max-parallelism N] [--no-resume] [--wait] [--poll-interval SECONDS] [--wait-timeout-seconds SECONDS]",
+  "build-registered-dataset": "build-registered-dataset DATASET_ID DATASET_VERSION [--project PROJECT] [--engine chronon] [--source-mode direct] [--upload-chronon-metadata] [--partition-duration DURATION] [--max-parallelism N] [--no-resume] [--wait] [--poll-interval SECONDS] [--wait-timeout-seconds SECONDS]",
   "get-job": "get-job JOB_ID",
   "wait-job": "wait-job JOB_ID [--poll-interval SECONDS] [--wait-timeout-seconds SECONDS]",
+  "retry-job": "retry-job JOB_ID",
+  "cancel-job": "cancel-job JOB_ID",
+  "register-operator": "register-operator SPEC_JSON",
+  "show-operator-specs": "show-operator-specs [--project PROJECT] [-q QUERY] [--limit N] [--offset N]",
+  "delete-operator": "delete-operator NAME VERSION [--project PROJECT]",
+  "run-operator": "run-operator NAME VERSION [--project PROJECT] [--config JSON]",
   "get-dataset-artifact": "get-dataset-artifact DATASET_ID MANIFEST_HASH [--project PROJECT]",
   "download-dataset-artifact": "download-dataset-artifact DATASET_ID MANIFEST_HASH [--project PROJECT] --out-dir PATH [--force]",
   "fetch-inference-data": "fetch-inference-data MANIFEST --cutoff-time TIMESTAMP [--max-workers N] [--max-source-lag-hours HOURS] [--no-validate-freshness] [--allow-missing]",
@@ -48,7 +70,10 @@ function usage(command) {
 }
 
 function parse(args) {
-  const options = { profile: "server", timeout: finiteNumber(process.env.ML_PLATFORM_API_TIMEOUT_SECONDS ?? 30, "--request-timeout-seconds") };
+  const timeoutValue = process.env.ML_PLATFORM_API_TIMEOUT_SECONDS
+    || process.env.DATA_PLATFORM_DEMO_API_TIMEOUT_SECONDS
+    || 30;
+  const options = { profile: "server", timeout: finiteNumber(timeoutValue, "--request-timeout-seconds") };
   let index = 0;
   while (index < args.length && args[index].startsWith("--")) {
     const flag = args[index++];
@@ -62,7 +87,7 @@ function parse(args) {
   if (!(options.timeout > 0)) throw new Error("--request-timeout-seconds must be positive");
   options.command = args[index++];
   options.rest = args.slice(index);
-  options.apiUrl ??= process.env.ML_PLATFORM_API_URL || loadApiUrl();
+  options.apiUrl ??= process.env.ML_PLATFORM_API_URL || process.env.DATA_PLATFORM_DEMO_API_URL || loadApiUrl();
   return options;
 }
 
@@ -101,7 +126,7 @@ function positional(rest, label) { const value = rest.shift(); if (!value || val
 
 function client(options) {
   if (options.profile !== "server") throw new Error("the business client supports only --profile server");
-  if (!options.apiUrl) throw new Error("server profile requires --api-url, ML_PLATFORM_API_URL, or a saved configure target");
+  if (!options.apiUrl) throw new Error("server profile requires --api-url, ML_PLATFORM_API_URL, DATA_PLATFORM_DEMO_API_URL, or a saved configure target");
   return new PlatformApiClient(options.apiUrl, options.timeout);
 }
 
@@ -112,6 +137,54 @@ function readObject(file) {
 }
 
 function readDataset(file) { return normalizeDataset(readObject(file)); }
+function readSpec(file) { return readObject(file); }
+
+function datasetEndpoint(project, datasetId, datasetVersion) {
+  return `/datasets/${encodeURIComponent(project)}/${encodeURIComponent(datasetId)}/${encodeURIComponent(datasetVersion)}`;
+}
+
+async function registeredDataset(api, project, datasetId, datasetVersion, includeDeleted = false) {
+  const payload = await api.get(datasetEndpoint(project, datasetId, datasetVersion), { include_deleted: includeDeleted });
+  if (!payload || typeof payload !== "object" || Array.isArray(payload)) throw new Error("registered dataset response must be a JSON object");
+  return payload;
+}
+
+function buildOptions(rest) {
+  const engine = take(rest, "--engine", "chronon");
+  if (engine !== "chronon") throw new Error("--engine must be chronon");
+  const sourceMode = take(rest, "--source-mode");
+  if (sourceMode !== undefined && sourceMode !== "direct") throw new Error("--source-mode must be direct");
+  const maxParallelism = integer(take(rest, "--max-parallelism", 1), "--max-parallelism");
+  if (maxParallelism < 1) throw new Error("--max-parallelism must be at least 1");
+  const wait = boolean(rest, "--wait");
+  const pollInterval = finiteNumber(take(rest, "--poll-interval", 2), "--poll-interval");
+  if (!(pollInterval > 0)) throw new Error("--poll-interval must be positive");
+  const rawTimeout = take(rest, "--wait-timeout-seconds");
+  const waitTimeout = rawTimeout === undefined ? undefined : finiteNumber(rawTimeout, "--wait-timeout-seconds");
+  if (waitTimeout !== undefined && !(waitTimeout > 0)) throw new Error("--wait-timeout-seconds must be positive");
+  return {
+    engine,
+    sourceMode,
+    upload: boolean(rest, "--upload-chronon-metadata"),
+    partitionDuration: take(rest, "--partition-duration"),
+    maxParallelism,
+    resume: !boolean(rest, "--no-resume"),
+    wait,
+    pollInterval,
+    waitTimeout,
+  };
+}
+
+async function submitBuild(api, manifest, build) {
+  let result = await api.post("/datasets/build", manifest, { engine: build.engine, source_mode: build.sourceMode, upload_chronon_metadata: build.upload, partition_duration: build.partitionDuration, max_parallelism: build.maxParallelism, resume: build.resume });
+  let exitCode = 0;
+  if (build.wait) {
+    if (!result || typeof result !== "object" || !result.job_id) throw new Error("server build did not return a job_id to wait for");
+    result = await waitForJob(api, String(result.job_id), build.pollInterval, build.waitTimeout);
+    if (["failed", "cancelled"].includes(result.status)) exitCode = 1;
+  }
+  return { result, exitCode };
+}
 
 async function waitForJob(api, jobId, pollInterval, timeoutSeconds) {
   if (!(pollInterval > 0)) throw new Error("job poll interval must be positive");
@@ -157,12 +230,28 @@ async function runBusinessCli(argv) {
   } else if (options.command === "list-projects") result = await client(options).get("/projects");
   else if (options.command === "get-project") result = await client(options).get(`/projects/${encodeURIComponent(positional(rest, "project name"))}`);
   else if (options.command === "delete-project") result = await client(options).delete(`/projects/${encodeURIComponent(positional(rest, "project name"))}`);
-  else if (options.command.startsWith("list-") && options.command !== "list-jobs") {
+  else if (["add-parameter", "add-feature", "add-feature-set", "add-operator", "register-operator"].includes(options.command)) {
+    const spec = readSpec(positional(rest, "spec JSON"));
+    const endpoint = { "add-parameter": "/parameters", "add-feature": "/features", "add-feature-set": "/feature-sets", "add-operator": "/operators", "register-operator": "/operators" }[options.command];
+    result = await client(options).post(endpoint, spec);
+  } else if (options.command === "publish-operator") {
+    const specPath = positional(rest, "OperatorSpec JSON"); const packagePath = take(rest, "--package");
+    if (!packagePath) throw new Error("--package is required");
+    const spec = readSpec(specPath); const api = client(options); const project = spec.project || "default";
+    const upload = await api.uploadOperatorPackage(project, spec.name, spec.version, packagePath);
+    if (!upload || typeof upload.code_artifact !== "object") throw new Error("operator package upload response has no code_artifact");
+    const published = { ...spec, code_hash: upload.code_artifact.sha256, package_uri: upload.code_artifact.uri, code_artifact: upload.code_artifact };
+    result = { status: "published", profile: "server", path: null, operator: published, registration: await api.post("/operators", published) };
+  } else if (options.command === "get-dataset") {
+    const datasetId = positional(rest, "dataset id"); const datasetVersion = positional(rest, "dataset version"); const project = take(rest, "--project", "default");
+    result = await registeredDataset(client(options), project, datasetId, datasetVersion, boolean(rest, "--include-deleted"));
+  }
+  else if (options.command.startsWith("list-") && !["list-jobs", "list-dataset-artifacts"].includes(options.command)) {
     const endpoints = { "list-parameters": "/parameters", "list-operators": "/operators", "list-features": "/features", "list-feature-sets": "/feature-sets", "list-datasets": "/datasets" };
     const project = take(rest, "--project");
-    const query = take(rest, "--q", take(rest, "-q")); const rawLimit = take(rest, "--limit"); const rawOffset = take(rest, "--offset");
-    if (rawLimit !== undefined || rawOffset !== undefined) { const limit = Number(rawLimit ?? 50); const offset = Number(rawOffset ?? 0); if (!Number.isInteger(limit) || limit < 1 || limit > 500) throw new Error("limit must be between 1 and 500"); if (!Number.isInteger(offset) || offset < 0) throw new Error("offset must be >= 0"); result = await client(options).get(endpoints[options.command], { q: query, limit, offset, project }); }
-    else result = await fetchAll(client(options), endpoints[options.command], { q: query, project });
+    const query = take(rest, "--q", take(rest, "-q")); const includeDeleted = options.command === "list-datasets" && boolean(rest, "--include-deleted"); const rawLimit = take(rest, "--limit"); const rawOffset = take(rest, "--offset");
+    if (rawLimit !== undefined || rawOffset !== undefined) { const limit = Number(rawLimit ?? 50); const offset = Number(rawOffset ?? 0); if (!Number.isInteger(limit) || limit < 1 || limit > 500) throw new Error("limit must be between 1 and 500"); if (!Number.isInteger(offset) || offset < 0) throw new Error("offset must be >= 0"); result = await client(options).get(endpoints[options.command], { q: query, limit, offset, project, include_deleted: includeDeleted }); }
+    else result = await fetchAll(client(options), endpoints[options.command], { q: query, project, include_deleted: includeDeleted });
   } else if (options.command === "list-jobs") {
     const query = take(rest, "--q", take(rest, "-q")); const rawLimit = take(rest, "--limit"); const rawOffset = take(rest, "--offset");
     if (rawLimit !== undefined || rawOffset !== undefined) { const limit = Number(rawLimit ?? 50); const offset = Number(rawOffset ?? 0); if (!Number.isInteger(limit) || limit < 1 || limit > 500) throw new Error("limit must be between 1 and 500"); if (!Number.isInteger(offset) || offset < 0) throw new Error("offset must be >= 0"); result = await client(options).get("/jobs", { q: query, limit, offset }); }
@@ -173,26 +262,52 @@ async function runBusinessCli(argv) {
     result = await client(options).post("/datasets/resolve", readDataset(manifest));
     const out = take(rest, "--out");
     if (out) { const target = resolvedPath(out); fs.mkdirSync(path.dirname(target), { recursive: true }); fs.writeFileSync(target, `${JSON.stringify(result, null, 2)}\n`); }
+  } else if (options.command === "resolve-dataset") {
+    const datasetId = positional(rest, "dataset id"); const datasetVersion = positional(rest, "dataset version"); const project = take(rest, "--project", "default"); const api = client(options);
+    const manifest = normalizeRegistryDataset(await registeredDataset(api, project, datasetId, datasetVersion));
+    result = await api.post("/datasets/resolve", manifest);
+    const out = take(rest, "--out");
+    if (out) { const target = resolvedPath(out); fs.mkdirSync(path.dirname(target), { recursive: true }); fs.writeFileSync(target, `${JSON.stringify(result, null, 2)}\n`); }
   } else if (options.command === "build-dataset") {
     const manifest = positional(rest, "manifest");
-    const sourceMode = take(rest, "--source-mode");
-    if (sourceMode !== undefined && sourceMode !== "direct") throw new Error("--source-mode must be direct");
-    const upload = boolean(rest, "--upload-chronon-metadata");
-    const partitionDuration = take(rest, "--partition-duration"); const maxParallelism = integer(take(rest, "--max-parallelism", 1), "--max-parallelism"); const resume = !boolean(rest, "--no-resume");
-    const api = client(options);
-    result = await api.post("/datasets/build", readDataset(manifest), { engine: "chronon", source_mode: sourceMode, upload_chronon_metadata: upload, partition_duration: partitionDuration, max_parallelism: maxParallelism, resume });
-    if (boolean(rest, "--wait")) {
-      if (!result || typeof result !== "object" || !result.job_id) throw new Error("server build did not return a job_id to wait for");
-      const rawTimeout = take(rest, "--wait-timeout-seconds");
-      result = await waitForJob(api, String(result.job_id), finiteNumber(take(rest, "--poll-interval", 2), "--poll-interval"), rawTimeout === undefined ? undefined : finiteNumber(rawTimeout, "--wait-timeout-seconds"));
-      if (["failed", "cancelled"].includes(result.status)) exitCode = 1;
-    }
+    const build = buildOptions(rest); const api = client(options);
+    ({ result, exitCode } = await submitBuild(api, readDataset(manifest), build));
+  } else if (options.command === "build-registered-dataset") {
+    const datasetId = positional(rest, "dataset id"); const datasetVersion = positional(rest, "dataset version"); const project = take(rest, "--project", "default"); const build = buildOptions(rest); const api = client(options);
+    const manifest = normalizeRegistryDataset(await registeredDataset(api, project, datasetId, datasetVersion));
+    ({ result, exitCode } = await submitBuild(api, manifest, build));
+  } else if (["add-dataset", "update-dataset"].includes(options.command)) {
+    const manifest = readDataset(positional(rest, "manifest JSON")); const force = boolean(rest, "--force"); const project = manifest.project || "default";
+    if (options.command === "add-dataset") result = await client(options).post("/datasets", manifest, { force });
+    else result = await client(options).put(`/datasets/${encodeURIComponent(project)}/${encodeURIComponent(manifest.dataset_id)}/${encodeURIComponent(manifest.dataset_version)}`, manifest, { force });
+  } else if (options.command === "delete-dataset") {
+    const id = positional(rest, "dataset id"); const version = positional(rest, "dataset version"); const project = take(rest, "--project", "default");
+    result = await client(options).delete(`/datasets/${encodeURIComponent(project)}/${encodeURIComponent(id)}/${encodeURIComponent(version)}`);
+  } else if (options.command === "list-dataset-artifacts") {
+    const project = take(rest, "--project"); const datasetId = take(rest, "--dataset-id"); const datasetVersion = take(rest, "--dataset-version"); const includeDeleted = boolean(rest, "--include-deleted");
+    result = await listCollection(client(options), "/dataset-artifacts", { project, dataset_id: datasetId, dataset_version: datasetVersion, include_deleted: includeDeleted }, rest);
+  } else if (options.command === "show-operator-specs") {
+    const project = take(rest, "--project"); const query = take(rest, "--q", take(rest, "-q"));
+    result = await listCollection(client(options), "/operators", { project, q: query }, rest);
+  } else if (options.command === "delete-operator") {
+    const name = positional(rest, "operator name"); const version = positional(rest, "operator version"); const project = take(rest, "--project", "default");
+    result = await client(options).delete(`/operators/${encodeURIComponent(project)}/${encodeURIComponent(name)}/${encodeURIComponent(version)}`);
+  } else if (options.command === "run-operator") {
+    const name = positional(rest, "operator name"); const version = positional(rest, "operator version"); const project = take(rest, "--project", "default"); const rawConfig = take(rest, "--config", "{}");
+    let config; try { config = JSON.parse(rawConfig); } catch (_) { throw new Error("--config must be valid JSON"); }
+    result = await client(options).post(`/operators/${encodeURIComponent(project)}/${encodeURIComponent(name)}/${encodeURIComponent(version)}/run`, config);
   } else if (options.command === "get-job") result = await client(options).get(`/jobs/${encodeURIComponent(positional(rest, "job id"))}`);
   else if (options.command === "wait-job") {
     const jobId = positional(rest, "job id"); const poll = finiteNumber(take(rest, "--poll-interval", 2), "--poll-interval"); const rawTimeout = take(rest, "--wait-timeout-seconds");
     result = await waitForJob(client(options), jobId, poll, rawTimeout === undefined ? undefined : finiteNumber(rawTimeout, "--wait-timeout-seconds"));
     if (["failed", "cancelled"].includes(result.status)) exitCode = 1;
   }
+  else if (options.command === "cancel-job") {
+    result = await client(options).post(`/jobs/${encodeURIComponent(positional(rest, "job id"))}/cancel`);
+  }
+  else if (options.command === "retry-job") {
+    result = await client(options).post(`/jobs/${encodeURIComponent(positional(rest, "job id"))}/retry`);
+  }
   else if (options.command === "get-dataset-artifact") {
     const dataset = positional(rest, "dataset id"); const hash = positional(rest, "manifest hash"); const project = take(rest, "--project", "default");
     result = await client(options).get(`/dataset-artifacts/${encodeURIComponent(project)}/${encodeURIComponent(dataset)}/${encodeURIComponent(hash)}`);
@@ -202,7 +317,7 @@ async function runBusinessCli(argv) {
     result = await client(options).downloadDatasetArtifact(project, dataset, hash, resolvedPath(outDir), boolean(rest, "--force"));
   } else if (options.command === "fetch-inference-data") {
     const manifest = positional(rest, "manifest"); const cutoffTime = take(rest, "--cutoff-time"); if (!cutoffTime) throw new Error("--cutoff-time is required");
-    const realtime = {}; const maxWorkers = take(rest, "--max-workers"); const maxLag = take(rest, "--max-source-lag-hours"); if (maxWorkers !== undefined) realtime.max_workers = integer(maxWorkers, "--max-workers"); if (maxLag !== undefined) realtime.max_source_lag_hours = finiteNumber(maxLag, "--max-source-lag-hours"); if (boolean(rest, "--no-validate-freshness")) realtime.validate_freshness = false; if (boolean(rest, "--allow-missing")) realtime.allow_missing = true;
+    const realtime = {}; const maxWorkers = take(rest, "--max-workers"); const maxLag = take(rest, "--max-source-lag-hours"); if (maxWorkers !== undefined) { realtime.max_workers = integer(maxWorkers, "--max-workers"); if (realtime.max_workers < 1) throw new Error("--max-workers must be at least 1"); } if (maxLag !== undefined) { realtime.max_source_lag_hours = finiteNumber(maxLag, "--max-source-lag-hours"); if (realtime.max_source_lag_hours < 0) throw new Error("--max-source-lag-hours must be non-negative"); } if (boolean(rest, "--no-validate-freshness")) realtime.validate_freshness = false; if (boolean(rest, "--allow-missing")) realtime.allow_missing = true;
     const request = { manifest: readDataset(manifest), cutoff_time: cutoffTime }; if (Object.keys(realtime).length) request.realtime_fetch = realtime; result = await client(options).post("/inference-data/fetch", request);
   } else if (options.command === "apply") {
     const directory = positional(rest, "catalog directory"); const dryRun = boolean(rest, "--dry-run"); const targetProject = take(rest, "--project");
@@ -215,7 +330,16 @@ async function runBusinessCli(argv) {
 
 async function fetchAll(api, endpoint, params = {}) { const items = []; for (let offset = 0; ; offset += 500) { const page = await api.get(endpoint, { ...params, limit: 500, offset }); if (Array.isArray(page)) return page; if (!page || !Array.isArray(page.items)) throw new Error(`${endpoint} returned an invalid paginated response`); items.push(...page.items); if (page.total == null || items.length >= page.total || page.items.length < 500) return items; } }
 
-module.exports = { BUSINESS_COMMANDS, fetchAll, parse, runBusinessCli, usage, waitForJob };
+async function listCollection(api, endpoint, params, rest) {
+  const rawLimit = take(rest, "--limit"); const rawOffset = take(rest, "--offset");
+  if (rawLimit === undefined && rawOffset === undefined) return fetchAll(api, endpoint, params);
+  const limit = Number(rawLimit ?? 50); const offset = Number(rawOffset ?? 0);
+  if (!Number.isInteger(limit) || limit < 1 || limit > 500) throw new Error("limit must be between 1 and 500");
+  if (!Number.isInteger(offset) || offset < 0) throw new Error("offset must be >= 0");
+  return api.get(endpoint, { ...params, limit, offset });
+}
+
+module.exports = { BUSINESS_COMMANDS, fetchAll, listCollection, parse, runBusinessCli, usage, waitForJob };
 
 if (require.main === module) {
   runBusinessCli(process.argv.slice(2)).then((code) => { process.exitCode = code; }).catch((error) => {

package/runtime/business-client/src/config.js

@@ -11,11 +11,15 @@ function expandHome(value) {
 }
 
 function configPath() {
-  if (process.env.ML_PLATFORM_CONFIG_PATH?.trim()) return path.resolve(expandHome(process.env.ML_PLATFORM_CONFIG_PATH));
+  for (const variable of ["ML_PLATFORM_CONFIG_PATH", "DATA_PLATFORM_DEMO_CONFIG_PATH"]) {
+    if (process.env[variable]?.trim()) return path.resolve(expandHome(process.env[variable]));
+  }
   const base = process.env.XDG_CONFIG_HOME
     ? path.resolve(process.env.XDG_CONFIG_HOME)
     : path.join(os.homedir(), ".config");
-  return path.join(base, "ml-platform", "config.json");
+  const primary = path.join(base, "ml-platform", "config.json");
+  const legacy = path.join(base, "data-platform-demo", "config.json");
+  return !fs.existsSync(primary) && fs.existsSync(legacy) ? legacy : primary;
 }
 
 function normalizeApiUrl(value) {