@zhuoyuezs/ml-platform 0.1.8 → 0.1.10

package/skills/feature-management/assets/catalog-template/operator_package/tests/test_operator.py

@@ -1,83 +0,0 @@
-from __future__ import annotations
-
-import unittest
-from types import SimpleNamespace
-
-import pandas as pd
-
-from business_feature_operator_template import compute_features
-
-
-class ComputeFeaturesTest(unittest.TestCase):
-    def context(
-        self,
-        timestamps: list[str] | None = None,
-        values: list[float | None] | None = None,
-        targets: list[str] | None = None,
-        requested: list[str] | None = None,
-    ) -> SimpleNamespace:
-        timestamps = timestamps or [
-            "2026-07-01 09:54:00",
-            "2026-07-01 09:56:00",
-            "2026-07-01 10:00:00",
-            "2026-07-01 10:01:00",
-        ]
-        values = values or [10.0, 20.0, 30.0, 40.0]
-        source = pd.DataFrame({"timestamp": pd.DatetimeIndex(timestamps), "value": values})
-        return SimpleNamespace(
-            requested_output_columns=requested or ["example_temperature_mean_5m"],
-            inputs=[
-                SimpleNamespace(
-                    parameter="example_temperature",
-                    key="example_temperature:v1",
-                )
-            ],
-            metric_frames={"example_temperature:v1": source},
-            target_times=pd.DatetimeIndex(targets or ["2026-07-01 10:00:00", "2026-07-01 10:02:00"]),
-            config={"window": "5min"},
-        )
-
-    def test_formula_uses_the_confirmed_mean(self) -> None:
-        result = compute_features(self.context())
-        self.assertEqual(result["example_temperature_mean_5m"].tolist(), [25.0, 35.0])
-
-    def test_excludes_event_at_open_left_window_boundary(self) -> None:
-        result = compute_features(self.context(["2026-07-01 09:55:00", "2026-07-01 09:56:00"], [10.0, 20.0]))
-        self.assertEqual(result["example_temperature_mean_5m"].iloc[0], 20.0)
-
-    def test_includes_event_exactly_at_cutoff(self) -> None:
-        result = compute_features(self.context(["2026-07-01 10:00:00"], [30.0]))
-        self.assertEqual(result["example_temperature_mean_5m"].iloc[0], 30.0)
-
-    def test_excludes_event_after_cutoff(self) -> None:
-        result = compute_features(self.context(["2026-07-01 10:00:01"], [40.0]))
-        self.assertTrue(pd.isna(result["example_temperature_mean_5m"].iloc[0]))
-
-    def test_uses_last_value_for_duplicate_event_time(self) -> None:
-        result = compute_features(self.context(["2026-07-01 10:00:00", "2026-07-01 10:00:00"], [10.0, 30.0]))
-        self.assertEqual(result["example_temperature_mean_5m"].iloc[0], 30.0)
-
-    def test_empty_history_returns_null(self) -> None:
-        result = compute_features(self.context(["2026-07-01 09:00:00"], [10.0]))
-        self.assertTrue(pd.isna(result["example_temperature_mean_5m"].iloc[0]))
-
-    def test_requested_output_is_exactly_the_supported_subset(self) -> None:
-        result = compute_features(self.context(requested=["example_temperature_mean_5m"]))
-        self.assertEqual(list(result.columns), ["event_time", "example_temperature_mean_5m"])
-
-    def test_preserves_requested_event_time_order(self) -> None:
-        targets = ["2026-07-01 10:02:00", "2026-07-01 10:00:00"]
-        result = compute_features(self.context(targets=targets))
-        self.assertEqual(list(result["event_time"]), list(pd.DatetimeIndex(targets)))
-
-    def test_output_dtype_is_float64(self) -> None:
-        result = compute_features(self.context())
-        self.assertEqual(str(result["example_temperature_mean_5m"].dtype), "float64")
-
-    def test_rejects_unknown_requested_output(self) -> None:
-        with self.assertRaisesRegex(ValueError, "unsupported output columns"):
-            compute_features(self.context(requested=["unknown_feature"]))
-
-
-if __name__ == "__main__":
-    unittest.main()

package/skills/feature-management/assets/catalog-template/operators/example_temperature_features.v1.json

@@ -1,58 +0,0 @@
-{
-  "schema_version": "ml_data_platform.operator/v1",
-  "project": "replace_with_project",
-  "name": "example_temperature_features",
-  "version": "v1",
-  "type": "feature",
-  "function_hash": "example_temperature_features.formulas.v1",
-  "entrypoint": "business_feature_operator_template:compute_features",
-  "code_hash": null,
-  "package_uri": null,
-  "code_artifact": null,
-  "input_schema": {
-    "parameters": [
-      "replace_with_project/example_temperature:v1"
-    ],
-    "prediction": {
-      "required": true,
-      "minimum_horizon": "0min",
-      "maximum_horizon": "0min"
-    },
-    "history_requirements": [
-      {
-        "anchor": "cutoff",
-        "lookback": "5min",
-        "output_columns": [
-          "example_temperature_mean_5m"
-        ]
-      }
-    ]
-  },
-  "output_schema": {
-    "columns": [
-      "event_time",
-      "example_temperature_mean_5m"
-    ]
-  },
-  "config_schema": {
-    "properties": {
-      "window": {
-        "type": "string"
-      }
-    },
-    "required": [
-      "window"
-    ]
-  },
-  "runtime": {
-    "engine": "python_entrypoint",
-    "network": "none"
-  },
-  "resources": {
-    "timeout_seconds": 60
-  },
-  "deterministic": true,
-  "supports_batch": true,
-  "supports_online": false,
-  "owner": "replace_with_business_owner"
-}

package/skills/feature-management/assets/catalog-template/parameters/example_temperature.v1.json

@@ -1,58 +0,0 @@
-{
-  "schema_version": "ml_data_platform.parameter/v1",
-  "project": "replace_with_project",
-  "name": "example_temperature",
-  "display_name": "Example temperature",
-  "version": "v1",
-  "data_type": "time_series",
-  "unit": "degC",
-  "expected_frequency": "1min",
-  "source": {
-    "adapter": "postgresql_direct",
-    "mode": "direct_column",
-    "schema": "process_data",
-    "table": "sensor_readings",
-    "time_column": "event_time",
-    "value_column": "temperature",
-    "metric_name": "example_temperature",
-    "unit_column": "unit",
-    "filters": {
-      "furnace_id": "BF12"
-    }
-  },
-  "time_semantics": {
-    "event_time_field": "event_time",
-    "ingested_at_field": "ingested_at",
-    "timezone": "Asia/Shanghai",
-    "availability": {
-      "strategy": "source_field",
-      "field": "ingested_at",
-      "accuracy": "exact"
-    }
-  },
-  "availability_sla": {
-    "max_delay": "PT10M"
-  },
-  "value_field": "value",
-  "quality_rules": {
-    "valid_range": [0.0, 2000.0],
-    "allow_missing": true,
-    "rules": [
-      {
-        "id": "finite_source_values",
-        "stage": "normalized_source",
-        "check": {
-          "type": "finite"
-        },
-        "acceptance": {
-          "max_violation_rate": 0.0,
-          "min_evaluated_rows": 1,
-          "min_comparable_rate": 0.0
-        },
-        "enforcement": "fail",
-        "description": "Reject non-finite source values before preprocessing."
-      }
-    ]
-  },
-  "owner": "replace_with_business_owner"
-}

package/skills/feature-management/references/commands.md

@@ -1,358 +0,0 @@
-# Platform Commands
-
-Use the `ml-platform` executable installed on `PATH` by the same ML Platform release as this Skill. The Skill contains no client runtime or launcher. Do not require a platform source checkout. Use the same API target for discovery, dry-run, publication, resolve, and build.
-
-## Contents
-
-1. Select a target
-2. Discover Registry state
-3. Test and build an Operator
-4. Validate and publish a catalog
-5. Resolve and build a dataset
-6. Fetch realtime inference data
-7. Inspect and change Registry lifecycle state
-8. Inspect and download an artifact
-9. Prohibited shortcuts
-
-## Select A Target
-
-Verify the installed CLI before using the Skill:
-
-```bash
-command -v ml-platform
-ml-platform version
-ml-platform --help
-```
-
-Configure the server API for business users:
-
-```bash
-ml-platform configure --api-url http://<platform-api-host>:8060
-ml-platform --profile server show-config
-ml-platform --profile server health
-```
-
-The URL is environment-specific and must be supplied by the user, an approved
-`ML_PLATFORM_API_URL`, or the saved value from `ml-platform configure`. Do not
-hardcode an internal address in this Skill or in a catalog. `show-config` displays
-the effective target before discovery, dry-run, publication, build, or artifact
-commands.
-
-Do not place database, MinIO, or source credentials in commands when the server profile can use managed Secrets.
-
-## Discover Registry State
-
-```bash
-ml-platform --profile server list-parameters -q <stable-name>
-ml-platform --profile server list-operators -q <stable-name>
-ml-platform --profile server list-features -q <stable-name>
-ml-platform --profile server list-feature-sets -q <stable-name>
-ml-platform --profile server list-datasets -q <stable-dataset-id>
-```
-
-Run each command separately. Check `ml-platform --help` and the specific
-subcommand help before adding flags. In the deployed 0.6.x CLI, `list-*`
-commands print JSON by default and do not accept a `--json` flag.
-If a deployed release rejects the global `--profile server` prefix, treat that
-as CLI compatibility drift: record the syntax error, inspect `show-config` and
-the subcommand help, then retry the same read-only command without the rejected
-flag only if the installed help permits it. Do not report this as API/network
-unavailability or switch to a direct HTTP client.
-
-By default each `list-*` command auto-pages through the whole result set and
-prints a **bare JSON array** of every matching item, so it never silently stops
-at the server's default page size. Use `-q` for a case-insensitive identifier
-substring search. To fetch one explicit page instead, pass `--limit` (1-500)
-and/or `--offset`; in that single-page mode the command prints the
-`{"items": [...], "total": ..., "limit": ..., "offset": ...}` envelope so you can
-resume with the next `--offset`. Note that `--offset` paging is not stable
-under concurrent writes: if items are added or removed between page requests,
-boundary rows can be skipped or repeated. A stable cursor is planned as a
-follow-up. Catalog `apply` and `apply --dry-run` page
-through the entire Registry automatically. Filter structured JSON with a JSON
-parser when the output is large. Do not scrape it with fragile text replacement.
-
-Every `list-*` registry command (except `list-jobs`) accepts `--project <name>`
-to restrict results to a single project. Omit it to list across all projects:
-
-```bash
-ml-platform --profile server list-parameters --project <project>
-ml-platform --profile server list-feature-sets --project <project>
-```
-
-## Projects
-
-A project is an isolated namespace over the whole registry. The real identity of
-every asset is `project/name:version`, so the same `name:version` can exist
-independently under different projects and never collide. The platform ships a
-built-in `default` project used whenever a project is not specified.
-
-Project is supplied differently for writes vs reads:
-
-Manage projects through the business client as well:
-
-```bash
-ml-platform create-project <project> --display-name "<display name>" --description "<description>"
-ml-platform list-projects
-ml-platform get-project <project>
-ml-platform delete-project <project>
-```
-
-`delete-project` is subject to the server's protection and only succeeds for an
-empty project. Project commands are server-only and do not modify local storage.
-
-- Creation: the project an asset is published under comes from the optional
-  `project` field in its own JSON spec (defaults to `default` when omitted). Set
-  `"project": "<name>"` in the spec to publish into a non-default project.
-- `apply --project <project>` scopes the comparison snapshot the catalog is
-  validated against (omit to compare against `default`);
-- `list-* --project <project>` filters discovery to that project;
-- `get-dataset-artifact` / `download-dataset-artifact` take the project as their
-  first positional argument (defaults to `default`).
-
-Before choosing names, discover within the target project: identical names in a
-different project are unrelated and must not be reused across projects to mean the
-same thing. Cross-project references are rejected by the platform.
-
-## Test And Build An Operator
-
-From the Operator package directory:
-
-```bash
-uv run python -m unittest discover -s tests -p 'test_*.py'
-```
-
-Build the package:
-
-```bash
-uv build <catalog>/operator_package \
-  --wheel \
-  --out-dir <catalog>/operator_package/dist
-```
-
-Confirm that `catalog.json` references the exact generated filename.
-
-## Validate And Publish A Catalog
-
-Validate against the intended server Registry without mutation:
-
-```bash
-ml-platform --profile server apply <catalog-directory> --dry-run
-```
-
-Review the returned counts and conflicts. After explicit user approval, publish:
-
-```bash
-ml-platform --profile server apply <catalog-directory>
-```
-
-Add `--project <project>` to publish into a specific project and compare against
-that project's snapshot only. Without it, the catalog is applied to the built-in
-`default` project:
-
-```bash
-ml-platform --profile server apply <catalog-directory> --project <project> --dry-run
-ml-platform --profile server apply <catalog-directory> --project <project>
-```
-
-`apply` processes dependencies in this order:
-
-```text
-Parameter -> Operator package/Operator -> Feature -> FeatureSet -> Dataset
-```
-
-The cross-store publication is not transactional. If an error occurs, report the returned progress summary and rerun only after understanding which immutable assets were already published.
-
-## Resolve And Build A Dataset
-
-For an existing registered DatasetManifest, use the Registry identity only; no
-local Catalog or Manifest path is needed:
-
-```bash
-ml-platform --profile server list-projects
-ml-platform --profile server list-datasets --project <project>
-ml-platform --profile server \
-  get-dataset <dataset_id> <dataset_version> --project <project>
-ml-platform --profile server \
-  resolve-dataset <dataset_id> <dataset_version> --project <project>
-```
-
-After the user confirms the exact triple and authorizes one build:
-
-```bash
-ml-platform --profile server \
-  build-registered-dataset <dataset_id> <dataset_version> \
-  --project <project> \
-  --partition-duration 1d \
-  --max-parallelism 1 \
-  --wait
-```
-
-The command first fetches that immutable DatasetManifest from the API and sends
-the validated response to the resolve/build endpoint. It never searches the
-current repository for a similarly named file. Record the API source,
-`project/dataset_id:dataset_version`, manifest hash, Job ID, terminal status,
-and artifact reference.
-
-When authoring a new or changed contract that has not been registered yet, use
-the local-file commands below.
-
-Resolve without fetching source data:
-
-```bash
-ml-platform --profile server \
-  resolve-manifest <catalog>/datasets/<dataset>.json \
-  --out /tmp/<dataset>.resolved.json
-```
-
-Submit and wait for a server build:
-
-```bash
-ml-platform --profile server \
-  build-dataset <catalog>/datasets/<dataset>.json \
-  --partition-duration 1d \
-  --max-parallelism 1 \
-  --wait
-```
-
-`--partition-duration` is an execution control and does not change the
-DatasetManifest hash. The initial implementation is sequential, so keep
-`--max-parallelism 1`. Completed partitions are checkpointed and resumed by
-default; use `--no-resume` only to force a clean execution.
-
-Record the returned `job_id`, `dataset_id`, and `manifest_hash`. If a separate wait is needed:
-
-```bash
-ml-platform --profile server wait-job <job_id>
-ml-platform --profile server get-job <job_id>
-```
-
-With explicit user authorization, stop a running build through the server CLI:
-
-```bash
-ml-platform --profile server cancel-job <job_id>
-```
-
-Cancellation is terminal for that Job; it does not change the immutable
-DatasetManifest and it does not automatically resubmit with different
-partition or parallelism settings. Check `get-job` after the request and
-record the returned terminal status.
-
-## Fetch Realtime Inference Data
-
-Fetch one causal-cutoff row in memory:
-
-```bash
-ml-platform --profile server \
-  fetch-inference-data <catalog>/datasets/<dataset>.json \
-  --cutoff-time 2026-07-31T10:00:00+08:00
-```
-
-Optional flags `--max-workers`, `--max-source-lag-hours`,
-`--no-validate-freshness`, and `--allow-missing` override individual
-`realtime_fetch` fields for this request. Prefer the versioned manifest policy;
-use overrides only when the user explicitly requests different runtime behavior.
-
-This command calls `POST /inference-data/fetch`. It does not submit a build Job,
-write parquet, or publish a DatasetArtifact. Treat HTTP `503` as retryable source
-freshness/missing-data failure and HTTP `400` as a contract error that requires a
-manifest or request change.
-
-For a deployment-bound row or full sequence context, prepare one JSON object
-containing `feature_retrieval`, `cutoff_time`, and `entity_keys`, then run:
-
-```bash
-ml-platform --profile server fetch-inference-context /tmp/inference-context.json
-```
-
-The `FeatureRetrievalSpec` must bind an existing inference-mode DatasetManifest
-by project, dataset id, version, and manifest hash; its FeatureSet and contract
-signature must match the resolved online contract. Context mode also fixes
-`context_length`, `frequency`, `context_end`, `stride`, input-adapter hash, and
-temporal-binding hash. Do not replace these immutable deployment bindings with
-ad-hoc CLI flags. As with row fetch, this operation writes no artifact.
-
-## Inspect And Change Registry Lifecycle State
-
-Lifecycle commands apply only to Parameter, Feature, and FeatureSet versions.
-Before a mutation, inspect the exact resource and its references:
-
-```bash
-ml-platform --profile server get-parameter-lifecycle <name> <version> --project <project>
-ml-platform --profile server list-parameter-references <name> <version> --project <project> --direct
-ml-platform --profile server list-parameter-references <name> <version> --project <project>
-```
-
-Equivalent `get-*-lifecycle` and `list-*-references` commands exist for
-`feature` and `feature-set`. The default reference query returns the transitive
-closure; `--direct` returns only immediate dependents.
-
-After explicit authorization for the exact resource, soft-delete it with
-auditable metadata:
-
-```bash
-ml-platform --profile server delete-parameter <name> <version> \
-  --project <project> --actor <actor> --reason '<reason>' \
-  --idempotency-key <stable-key>
-```
-
-Restore uses the same audit fields and is rejected if the resource's own
-dependencies are not active:
-
-```bash
-ml-platform --profile server restore-parameter <name> <version> \
-  --project <project> --actor <actor> --reason '<reason>' \
-  --idempotency-key <stable-key>
-```
-
-These operations never erase immutable specs, historical references, or
-contract hashes. Do not use the Operator or Dataset deletion commands as a
-substitute; they have separate lifecycle rules.
-
-## Inspect And Download An Artifact
-
-```bash
-ml-platform --profile server \
-  get-dataset-artifact <dataset_id> <manifest_hash> --project <project>
-
-ml-platform --profile server \
-  download-dataset-artifact <dataset_id> <manifest_hash> \
-  --project <project> \
-  --out-dir /tmp/<dataset_id>-artifact
-```
-
-`--project` defaults to `default`; pass the owning project when the dataset lives
-elsewhere, or the lookup returns `artifact_not_found`.
-
-The download command writes progress to stderr after response headers arrive.
-Interpret a timeout with no progress line as a request/header or first-byte
-failure. Interpret a timeout after one or more `artifact download: received ...`
-lines as an incomplete response body. In either case, report the timeout and
-keep downloaded-file validation incomplete; do not replace the supported CLI
-with direct HTTP or object-store access. A successful metadata query remains
-valid metadata-only evidence, but it does not prove Parquet schema, column
-order, or file-content hashes.
-
-Inspect at least:
-
-```text
-dataset_manifest.json
-resolved_manifest.json
-validation.json
-lineage.json
-feature_computations/computation_manifest.json
-feature_computations/executions/*.json
-feature_dataset.parquet
-```
-
-Use a structured Parquet reader for schema, row count, column order, and missing-rate inspection.
-
-## Prohibited Shortcuts
-
-- Do not use `seed-demo v95` for business catalogs.
-- Do not call individual `add-*` commands when catalog `apply` is available for an end-to-end publication.
-- Do not use any `delete-*`, `restore-*`, `cancel-job`, or force overwrite operation unless separately requested.
-- Do not build images, deploy Kubernetes, or change service configuration in this workflow.
-- Do not claim a successful publication from `--dry-run` output.
-- Do not claim Chronon compile/backfill from a successful first-phase Operator build.
-- Do not use `build-dataset` to serve a single realtime inference cutoff.