@zhuoyuezs/ml-platform 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DEVELOPMENT.md +189 -0
- package/README.md +103 -0
- package/checksums.json +110 -0
- package/package.json +29 -0
- package/release-policy.json +31 -0
- package/release.json +42 -0
- package/runtime/business-client/README.md +14 -0
- package/runtime/business-client/package-lock.json +19 -0
- package/runtime/business-client/package.json +21 -0
- package/runtime/business-client/src/catalog.js +184 -0
- package/runtime/business-client/src/cli.js +225 -0
- package/runtime/business-client/src/config.js +52 -0
- package/runtime/business-client/src/http.js +137 -0
- package/scripts/lib.js +819 -0
- package/scripts/main.js +92 -0
- package/skills/feature-management/SKILL.md +265 -0
- package/skills/feature-management/agents/openai.yaml +4 -0
- package/skills/feature-management/assets/catalog-template/catalog.json +23 -0
- package/skills/feature-management/assets/catalog-template/datasets/example_temperature_training.v1.json +24 -0
- package/skills/feature-management/assets/catalog-template/feature_sets/example_temperature_core.v1.json +13 -0
- package/skills/feature-management/assets/catalog-template/features/example_temperature_mean_5m.v1.json +21 -0
- package/skills/feature-management/assets/catalog-template/operator_package/pyproject.toml +12 -0
- package/skills/feature-management/assets/catalog-template/operator_package/src/business_feature_operator_template/__init__.py +39 -0
- package/skills/feature-management/assets/catalog-template/operator_package/tests/test_operator.py +56 -0
- package/skills/feature-management/assets/catalog-template/operators/example_temperature_features.v1.json +43 -0
- package/skills/feature-management/assets/catalog-template/parameters/example_temperature.v1.json +57 -0
- package/skills/feature-management/references/commands.md +244 -0
- package/skills/feature-management/references/contracts.md +682 -0
- package/skills/feature-management/references/operator-authoring.md +167 -0
package/scripts/main.js
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
"use strict";
|
|
4
|
+
|
|
5
|
+
const path = require("path");
|
|
6
|
+
const { spawnSync } = require("child_process");
|
|
7
|
+
const {
|
|
8
|
+
doctor,
|
|
9
|
+
install,
|
|
10
|
+
parseOptions,
|
|
11
|
+
status,
|
|
12
|
+
} = require("./lib");
|
|
13
|
+
|
|
14
|
+
const MANAGEMENT_COMMANDS = new Set(["install", "upgrade", "status", "doctor"]);
|
|
15
|
+
const BUSINESS_ENTRY = path.resolve(__dirname, "..", "runtime", "business-client", "src", "cli.js");
|
|
16
|
+
|
|
17
|
+
function usage() {
|
|
18
|
+
return `用法:
|
|
19
|
+
ml-platform install [--agent codex|pi|custom] [--scope user|project] [--skills-dir PATH] [--project-dir PATH] [--state-dir PATH] [--bin-dir PATH] [--upgrade] [--allow-downgrade]
|
|
20
|
+
ml-platform upgrade [install options]
|
|
21
|
+
ml-platform status [--agent codex|pi|custom] [--scope user|project] [--skills-dir PATH] [--project-dir PATH] [--state-dir PATH] [--bin-dir PATH]
|
|
22
|
+
ml-platform doctor [status options] [--api-url URL]
|
|
23
|
+
ml-platform <business-command> [args]
|
|
24
|
+
|
|
25
|
+
安装管理命令:
|
|
26
|
+
install, upgrade, status, doctor
|
|
27
|
+
|
|
28
|
+
其余已注册业务命令由同一份 JavaScript Business CLI 执行。
|
|
29
|
+
`;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function printJson(payload) {
|
|
33
|
+
process.stdout.write(`${JSON.stringify(payload, null, 2)}\n`);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function loadBusinessCommands() {
|
|
37
|
+
try {
|
|
38
|
+
const { BUSINESS_COMMANDS } = require(BUSINESS_ENTRY);
|
|
39
|
+
if (!(BUSINESS_COMMANDS instanceof Set)) throw new Error("BUSINESS_COMMANDS export is missing");
|
|
40
|
+
return BUSINESS_COMMANDS;
|
|
41
|
+
} catch (error) {
|
|
42
|
+
throw new Error(`Business CLI 不完整: ${error.message}`);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function dispatchBusiness(argv) {
|
|
47
|
+
const commands = loadBusinessCommands();
|
|
48
|
+
if (!argv.some((arg) => commands.has(arg))) {
|
|
49
|
+
throw new Error(`未知命令: ${argv.find((arg) => !arg.startsWith("-")) || argv[0]}`);
|
|
50
|
+
}
|
|
51
|
+
const result = spawnSync(process.execPath, [BUSINESS_ENTRY, ...argv], {
|
|
52
|
+
stdio: "inherit",
|
|
53
|
+
env: process.env,
|
|
54
|
+
});
|
|
55
|
+
if (result.error) throw result.error;
|
|
56
|
+
return result.status === null ? 1 : result.status;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function main(argv) {
|
|
60
|
+
const [command, ...rest] = argv;
|
|
61
|
+
if (!command || command === "help" || command === "--help" || command === "-h") {
|
|
62
|
+
process.stdout.write(usage());
|
|
63
|
+
return 0;
|
|
64
|
+
}
|
|
65
|
+
if (!MANAGEMENT_COMMANDS.has(command)) return dispatchBusiness(argv);
|
|
66
|
+
|
|
67
|
+
const options = parseOptions(rest);
|
|
68
|
+
if (command === "install" || command === "upgrade") {
|
|
69
|
+
if (command === "upgrade") options.upgrade = true;
|
|
70
|
+
printJson(install(options));
|
|
71
|
+
return 0;
|
|
72
|
+
}
|
|
73
|
+
if (command === "status") {
|
|
74
|
+
const result = status(options);
|
|
75
|
+
printJson(result);
|
|
76
|
+
return result.ok ? 0 : 1;
|
|
77
|
+
}
|
|
78
|
+
const result = doctor(options);
|
|
79
|
+
printJson(result);
|
|
80
|
+
return result.ok ? 0 : 1;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
if (require.main === module) {
|
|
84
|
+
try {
|
|
85
|
+
process.exitCode = main(process.argv.slice(2));
|
|
86
|
+
} catch (error) {
|
|
87
|
+
process.stderr.write(`${JSON.stringify({ ok: false, error: error.message }, null, 2)}\n`);
|
|
88
|
+
process.exitCode = 2;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
module.exports = { BUSINESS_ENTRY, MANAGEMENT_COMMANDS, dispatchBusiness, main, usage };
|
|
@@ -0,0 +1,265 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: feature-management
|
|
3
|
+
description: Manage projects, versioned business feature assets, dataset rowsets and endpoint eligibility, and causal realtime feature reads through the deployed ML Platform API using the ml-platform CLI installed with the same release. Use when a business user asks an agent to manage an empty Project, define or change Parameter source contracts, author/package/register an Operator wheel, define single-column Features, order a FeatureSet, create a DatasetManifest or catalog, configure training/validation/test rowsets or abnormal-data endpoint exclusion, configure realtime freshness policy, run catalog dry-run/apply, build a dataset, fetch one inference row, or inspect an artifact without downloading the platform source repository. Do not use for model training, versioned Registry asset deletion, image builds, or Kubernetes deployment.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Feature Management
|
|
7
|
+
|
|
8
|
+
Build the public asset chain without bypassing its versioned contracts:
|
|
9
|
+
|
|
10
|
+
```text
|
|
11
|
+
Parameter -----> Feature
|
|
12
|
+
Operator ------> Feature -> FeatureSet -> DatasetManifest -> DatasetArtifact
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
Treat `Recipe` and public `Feature.compute` as removed. Treat `computation_hash` as internal execution metadata, never as a business-managed asset.
|
|
16
|
+
|
|
17
|
+
## Load The Right Context
|
|
18
|
+
|
|
19
|
+
1. Resolve the Skill root as the directory containing this `SKILL.md`.
|
|
20
|
+
2. Read [references/contracts.md](references/contracts.md) before creating or changing JSON assets.
|
|
21
|
+
3. Read [references/operator-authoring.md](references/operator-authoring.md) whenever creating or changing Operator code or a wheel.
|
|
22
|
+
4. Read [references/commands.md](references/commands.md) before discovering assets, validating, publishing, building, fetching realtime data, or downloading.
|
|
23
|
+
5. Use [assets/catalog-template](assets/catalog-template) as a copyable starting point for a new end-to-end catalog. Rename every `example_*` identifier and update every referenced path before validation.
|
|
24
|
+
|
|
25
|
+
## Initialize The Client
|
|
26
|
+
|
|
27
|
+
Use the `ml-platform` executable on `PATH` for every platform command. The formal distribution contract requires the npm release to install the CLI and this Skill together; that packaging stage is not implemented yet. The Skill does not contain a second CLI runtime, launcher, or installer. During repository development, provide the current JavaScript CLI through an isolated test `PATH`.
|
|
28
|
+
|
|
29
|
+
Before any platform operation, verify the executable and its version:
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
command -v ml-platform
|
|
33
|
+
ml-platform version
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
If the executable is missing or cannot start, stop and report that the ML Platform release installation is incomplete. Do not download a client, search for a repository checkout, or fall back to another executable.
|
|
37
|
+
|
|
38
|
+
Before any server-profile discovery, dry-run, publication, build, or artifact
|
|
39
|
+
operation, configure the deployed API URL exactly as follows:
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
ml-platform configure --api-url http://10.36.9.212:30620
|
|
43
|
+
ml-platform --profile server health
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Use `http://10.36.9.212:30620` when operating against the current deployed
|
|
47
|
+
environment. Placeholder host and port values elsewhere in the references are
|
|
48
|
+
generic templates, not this environment's configured target. The API is
|
|
49
|
+
unauthenticated, so access it only from the approved internal network.
|
|
50
|
+
|
|
51
|
+
Install and upgrade the CLI and Skill only through the explicit commands provided by `@zhuoyuezs/ml-platform`. Restart the Agent session after a successful Skill upgrade so the definition is reloaded. Never copy files over an installed Skill directory manually.
|
|
52
|
+
|
|
53
|
+
## Establish Intent
|
|
54
|
+
|
|
55
|
+
Collect only missing information. Do not invent a source table, source field, formula, time boundary, unit, valid range, missing-data rule, or owner.
|
|
56
|
+
|
|
57
|
+
Confirm:
|
|
58
|
+
|
|
59
|
+
- the target project (namespace) for the assets; use the built-in `default` only when the user has no dedicated project. On creation the project comes from the optional `project` field in each asset spec (defaults to `default`); reads and filters use the `--project` command option;
|
|
60
|
+
- business meaning, stable asset names, owner, and intended consumers;
|
|
61
|
+
- Parameter source adapter and credential-free source mapping;
|
|
62
|
+
- feature formula, exact Parameter dependencies, windows, inclusion rules, rounding, null behavior, and output dtype;
|
|
63
|
+
- whether an approved Operator already implements the formula;
|
|
64
|
+
- FeatureSet column order;
|
|
65
|
+
- dataset mode, read policy, time range, grid, prediction horizon/cutoff contract, and explicit Parameter outputs;
|
|
66
|
+
- named training/validation/test rowsets, abnormal windows, endpoint-policy scope,
|
|
67
|
+
and the model context lookback used to decide endpoint eligibility;
|
|
68
|
+
- realtime freshness and missing-input policy when serving inference data;
|
|
69
|
+
- target profile/API and whether the user authorizes publish and/or build actions.
|
|
70
|
+
|
|
71
|
+
Separate preparation from mutation. Creating files and running local tests or `apply --dry-run` does not authorize publishing a wheel, changing the Registry, or submitting a build Job.
|
|
72
|
+
|
|
73
|
+
## Discover Existing Assets
|
|
74
|
+
|
|
75
|
+
Check the selected profile and list existing Parameter, Operator, Feature, FeatureSet, and Dataset versions before choosing names. Scope discovery to the target project with `--project`, because the true registry key is `project/name:version` and the same `name:version` may exist independently under another project. Registry list commands are paginated; search by stable identifier or follow every page until `offset + len(items) >= total`. Reuse an exact immutable version only when its full content matches. Never reference an asset in another project; cross-project references are rejected.
|
|
76
|
+
|
|
77
|
+
Create a new version when source semantics, formula code, config meaning, inputs, output dtype, time behavior, quality rules, or column order change. Never overwrite an immutable version or use suffixes such as `new`, `final`, or `test2`.
|
|
78
|
+
|
|
79
|
+
## Upgrade Dependency Chains
|
|
80
|
+
|
|
81
|
+
Treat a version change as a release of its affected reverse-dependency closure, not as an isolated asset bump.
|
|
82
|
+
|
|
83
|
+
1. Record the intended old-to-new version mapping and the semantic reason for every changed asset.
|
|
84
|
+
2. Follow exact references from the changed asset through every affected downstream consumer. For a Parameter change, publish a new Operator version when its input schema names that Parameter version, then new versions of every affected Feature, FeatureSet, and DatasetManifest. For an Operator change, start with its affected Features.
|
|
85
|
+
3. Republish only the affected closure, but explicitly list and justify every downstream asset intentionally retained on an older dependency.
|
|
86
|
+
4. Keep every reference exact. Never rewrite an existing immutable version, infer `latest`, or assume that increasing a Feature or FeatureSet version also upgrades its dependencies.
|
|
87
|
+
5. Treat `apply --dry-run` as an existence and structural-compatibility check. It does not prove that the declared versions are the intended business release.
|
|
88
|
+
|
|
89
|
+
Use this dependency order for an affected chain:
|
|
90
|
+
|
|
91
|
+
```text
|
|
92
|
+
Parameter -> Operator -> Feature -> FeatureSet -> DatasetManifest
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
## Build The Catalog
|
|
96
|
+
|
|
97
|
+
Keep one business domain in one user-owned catalog directory:
|
|
98
|
+
|
|
99
|
+
```text
|
|
100
|
+
<catalog>/
|
|
101
|
+
catalog.json
|
|
102
|
+
parameters/
|
|
103
|
+
operators/
|
|
104
|
+
operator_package/
|
|
105
|
+
features/
|
|
106
|
+
feature_sets/
|
|
107
|
+
datasets/
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
Keep platform demos and V95 built-ins unchanged. Do not put unrelated business assets under `src/data_platform_demo/builtin_catalogs/v95`.
|
|
111
|
+
|
|
112
|
+
### Define Parameters
|
|
113
|
+
|
|
114
|
+
Create one stable, versioned source contract for each independently readable value. Keep credentials out of JSON. Use only supported direct adapters and explicit source identity. Put source data validity in `quality_rules`; put dataset-specific transformations in preprocess Operators.
|
|
115
|
+
|
|
116
|
+
Use Parameter `rounding` when every consumer must receive the same fixed-point value. The platform applies `mode` (`half_up` or `half_even`) and `decimals` after source normalization and before quality checks, source caching, replay, and Operator execution. Prefer an unrounded source expression, do not repeat the same rounding in an Operator, and publish a new Parameter plus its affected reverse-dependency closure when the rule changes.
|
|
117
|
+
|
|
118
|
+
Do not represent a rolling mean, lag, ratio, trend, or model input as a Parameter. Those are Features.
|
|
119
|
+
|
|
120
|
+
### Define Operators
|
|
121
|
+
|
|
122
|
+
First reuse a registered Operator if its version, input schema, output schema, formula semantics, and runtime contract match exactly.
|
|
123
|
+
|
|
124
|
+
When a new Operator is required:
|
|
125
|
+
|
|
126
|
+
1. Write a standalone pure-Python package with a `module:function` entrypoint.
|
|
127
|
+
2. Make the Feature entrypoint accept one context and return a pandas DataFrame.
|
|
128
|
+
3. Consume only declared `context.inputs` and `context.metric_frames`.
|
|
129
|
+
4. Return `event_time` exactly equal to `context.target_times` plus the requested physical output columns.
|
|
130
|
+
5. Implement explicit causal boundaries; never use future rows accidentally.
|
|
131
|
+
6. Keep execution deterministic, network-free, credential-free, and bounded by `timeout_seconds`.
|
|
132
|
+
7. Add formula, boundary, missing-value, requested-output, and ordering tests.
|
|
133
|
+
8. Build a `py3-none-any` wheel and reference it from `catalog.json`.
|
|
134
|
+
9. Leave `code_hash`, `package_uri`, and `code_artifact` null in the draft spec; catalog publication fills them from the uploaded wheel.
|
|
135
|
+
|
|
136
|
+
Use a meaningful immutable `function_hash`. Change the Operator version whenever executable behavior changes.
|
|
137
|
+
|
|
138
|
+
### Define Features
|
|
139
|
+
|
|
140
|
+
Create one JSON file per single output column. Bind exactly one Operator version and physical `output_column`. List only the Parameter versions actually required by that column's formula.
|
|
141
|
+
|
|
142
|
+
Features may share the same Operator and config. The planner will merge their exact inputs and call the Operator once per computation group. Do not duplicate every Operator input into every Feature merely to make schemas look uniform.
|
|
143
|
+
|
|
144
|
+
### Define The FeatureSet
|
|
145
|
+
|
|
146
|
+
Reference immutable Feature versions in the exact consumer column order. Keep the list nonempty and duplicate-free. Do not copy formulas, inputs, or Operator configuration into the FeatureSet.
|
|
147
|
+
|
|
148
|
+
### Define The DatasetManifest
|
|
149
|
+
|
|
150
|
+
Reference exactly one FeatureSet. Use `parameters` only for Parameter columns that must also appear explicitly in `parameter_dataset.parquet`; Feature dependencies are resolved automatically.
|
|
151
|
+
|
|
152
|
+
Declare one dataset-wide `prediction` contract when Features use a forecast cutoff. Operators consume `context.cutoff_times`; do not copy the same prediction horizon into every Feature config. Check the selected Operator's `input_schema.prediction` limits and satisfy an explicit-horizon requirement before publishing or building.
|
|
153
|
+
|
|
154
|
+
Use `snapshot` for reproducible training data, `as_of` for historical visibility replay, and `latest` for current inference-style reads. Use a fresh `dataset_version` when validating a new release or intentionally requesting a new immutable dataset contract.
|
|
155
|
+
|
|
156
|
+
Treat row construction and endpoint eligibility as separate contracts. Use
|
|
157
|
+
`rowset_splits` to label candidate target instants such as `training`,
|
|
158
|
+
`validation`, and `test`. When abnormal observations should make only selected
|
|
159
|
+
endpoints unusable, keep `abnormal_windows.policy=mark_only` and declare
|
|
160
|
+
`endpoint_policy`; do not delete canonical Feature rows. Declare
|
|
161
|
+
`endpoint_policy.context_lookback` from the model sequence contract, independently
|
|
162
|
+
of Operator source-history requirements. Read [references/contracts.md](references/contracts.md)
|
|
163
|
+
for the cutoff formula, half-open overlap boundaries, scope behavior, and
|
|
164
|
+
`rowset_membership.parquet` schema.
|
|
165
|
+
|
|
166
|
+
For realtime inference, declare `realtime_fetch` only when its effective policy
|
|
167
|
+
must be part of the contract; leaving it unset preserves existing manifest
|
|
168
|
+
hashes. Keep `allow_missing=false` unless the user explicitly accepts missing
|
|
169
|
+
required inputs. Read [references/contracts.md](references/contracts.md) for the
|
|
170
|
+
freshness, tail-edge, gap-fill, and rowset contracts.
|
|
171
|
+
|
|
172
|
+
## Validate Before Publishing
|
|
173
|
+
|
|
174
|
+
Run, in order:
|
|
175
|
+
|
|
176
|
+
1. Operator unit tests.
|
|
177
|
+
2. Wheel build and wheel filename verification.
|
|
178
|
+
3. Catalog path, schema, dependency, immutability, and package validation with `apply --dry-run` against the same target profile intended for publication.
|
|
179
|
+
4. Compare the catalog's exact dependency closure with the intended old-to-new version mapping. Reject any affected downstream reference that still points to an old version unless its retention is explicit and justified.
|
|
180
|
+
5. A human-readable summary of planned new, unchanged, retained, and conflicting assets.
|
|
181
|
+
|
|
182
|
+
Stop on any error. Do not weaken schema validation, fabricate a missing dependency, change an existing version in place, or switch profiles to make validation pass.
|
|
183
|
+
|
|
184
|
+
## Publish And Build
|
|
185
|
+
|
|
186
|
+
Publish only after the user explicitly approves Registry and wheel changes. Use catalog `apply` so publication follows Parameter -> Operator -> Feature -> FeatureSet -> Dataset order.
|
|
187
|
+
|
|
188
|
+
After publication, resolve every new DatasetManifest and compare its exact Parameter, Operator, Feature, and FeatureSet versions with the pre-publication mapping. Stop if an affected old key or any unexpected version remains. Resolve the manifest before building. Submit a build only when requested. For server builds, wait for the terminal Job state and download the exact artifact by `dataset_id + manifest_hash`.
|
|
189
|
+
|
|
190
|
+
Do not delete versioned Registry assets, cancel Jobs, rebuild images, modify Kubernetes, or change service configuration as part of this workflow unless the user separately and explicitly requests that action. An empty Project may be soft-deleted only on an explicit request; rely on the server to reject deletion when resources still exist.
|
|
191
|
+
|
|
192
|
+
## Fetch Realtime Inference Data
|
|
193
|
+
|
|
194
|
+
Use `fetch-inference-data` only when the user requests one causal-cutoff read.
|
|
195
|
+
Confirm the cutoff and prediction horizon. The command must call
|
|
196
|
+
`POST /inference-data/fetch`; it must not submit a batch build, write parquet, or
|
|
197
|
+
publish a DatasetArtifact.
|
|
198
|
+
|
|
199
|
+
Verify that the response reports the requested cutoff and derived target time,
|
|
200
|
+
`uses_post_cutoff_data=false`, `contract.manifest_hash`, a replayable `as_of`
|
|
201
|
+
manifest, and per-Parameter freshness evidence. Report tolerated missing or
|
|
202
|
+
gap-filled inputs as degraded freshness, not as a normal read.
|
|
203
|
+
|
|
204
|
+
## Verify The Artifact
|
|
205
|
+
|
|
206
|
+
Require all of the following before reporting success:
|
|
207
|
+
|
|
208
|
+
- Job status is `succeeded`;
|
|
209
|
+
- `validation.json` has `ok: true` and no errors;
|
|
210
|
+
- `validation.feature_nulls` is read, not skipped: `ok: true` can coexist with
|
|
211
|
+
null model inputs reported as warnings. Report `total_null_cells`, and for each
|
|
212
|
+
affected Feature its `null_count`, event-time range, and upstream Parameter
|
|
213
|
+
keys. Set `output.strict_feature_nulls` when any null should fail the build
|
|
214
|
+
instead;
|
|
215
|
+
- row count is nonzero and matches the requested half-open time grid when no
|
|
216
|
+
documented row filtering applies. When `abnormal_windows` uses a drop policy,
|
|
217
|
+
reconcile the lower count against `abnormal_windows.rows_before`,
|
|
218
|
+
`rows_after`, and `dropped_rows` in the resolved manifest. With `mark_only`,
|
|
219
|
+
require `dropped_rows=0` and preserve the full candidate rowset;
|
|
220
|
+
- when `endpoint_policy` is set, require `abnormal_windows.policy=mark_only`,
|
|
221
|
+
preserve every candidate row in `feature_dataset.parquet` and
|
|
222
|
+
`parameter_dataset.parquet`, and require `rowset_membership.parquet`;
|
|
223
|
+
- verify membership counts globally and by rowset, inspect every ineligible
|
|
224
|
+
row's `reason_codes`, and confirm that the configured compatibility column in
|
|
225
|
+
both canonical datasets equals `policy_applied AND eligible`; it is false for
|
|
226
|
+
pass-through rowsets outside the policy scope;
|
|
227
|
+
- for a partitioned build, verify that merged membership, endpoint-policy
|
|
228
|
+
quality statistics, lineage, and `policy_hash` match a single build of the
|
|
229
|
+
same manifest;
|
|
230
|
+
- `feature_dataset.parquet` starts with `event_time`, optional `furnace_id`, then FeatureSet columns in exact order;
|
|
231
|
+
- resolved lineage records every Parameter version, Feature version, Operator version, code hash, and computation hash, exactly matches the approved dependency-version mapping, and contains no stale affected key;
|
|
232
|
+
- artifact lineage contains the same approved dependency closure as the resolved manifest;
|
|
233
|
+
- execution metadata shows the expected number of computation groups and requested physical columns;
|
|
234
|
+
- when `source_read` is set, `latency_stats.fetch_timing_summary` shows the chunk
|
|
235
|
+
windows tiling the fetch range, and any retries are accounted for;
|
|
236
|
+
- missing values and warnings are reported, not silently repaired.
|
|
237
|
+
|
|
238
|
+
The manifest hash returned by `resolve-manifest` is the artifact key: it stays
|
|
239
|
+
the same after the build, so it is usable for cache reuse and polling before
|
|
240
|
+
submission. Distinguish the two lookup failures rather than treating both as "not
|
|
241
|
+
ready" — `422` with `invalid_artifact_key` means the key can never name an
|
|
242
|
+
artifact, while `404` with `artifact_not_found` or `artifact_not_built` means
|
|
243
|
+
nothing is built under a valid key, and lists any in-flight Jobs.
|
|
244
|
+
|
|
245
|
+
State the current runtime truth: first-phase public builds execute registered Python Operators through `FeatureOperatorRunner`. Do not claim that Chronon compile/backfill or Chronon-native feature computation occurred unless the artifact contains and passes those explicit execution records.
|
|
246
|
+
|
|
247
|
+
## Report The Outcome
|
|
248
|
+
|
|
249
|
+
Return a concise business-facing summary containing:
|
|
250
|
+
|
|
251
|
+
```text
|
|
252
|
+
catalog and version
|
|
253
|
+
new / unchanged assets by type
|
|
254
|
+
planned versus resolved dependency versions and any explicitly retained old versions
|
|
255
|
+
Operator package filename and SHA-256
|
|
256
|
+
FeatureSet and ordered feature count
|
|
257
|
+
dataset id, version, manifest hash, and Job id
|
|
258
|
+
row and column counts
|
|
259
|
+
rowset candidate / policy-applied / eligible / ineligible counts when endpoint_policy is declared
|
|
260
|
+
validation status, warnings, and missing-data summary
|
|
261
|
+
artifact location
|
|
262
|
+
actual compute backend
|
|
263
|
+
```
|
|
264
|
+
|
|
265
|
+
If the work stops before publication, distinguish generated files, locally validated files, dry-run validation, and remotely published assets.
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schema_version": "ml_data_platform.catalog/v1",
|
|
3
|
+
"name": "example_business_feature_catalog",
|
|
4
|
+
"version": "v1",
|
|
5
|
+
"parameters": [
|
|
6
|
+
"parameters/example_temperature.v1.json"
|
|
7
|
+
],
|
|
8
|
+
"operators": [
|
|
9
|
+
{
|
|
10
|
+
"spec": "operators/example_temperature_features.v1.json",
|
|
11
|
+
"package": "operator_package/dist/business_feature_operator_template-1.0.0-py3-none-any.whl"
|
|
12
|
+
}
|
|
13
|
+
],
|
|
14
|
+
"features": [
|
|
15
|
+
"features/example_temperature_mean_5m.v1.json"
|
|
16
|
+
],
|
|
17
|
+
"feature_sets": [
|
|
18
|
+
"feature_sets/example_temperature_core.v1.json"
|
|
19
|
+
],
|
|
20
|
+
"datasets": [
|
|
21
|
+
"datasets/example_temperature_training.v1.json"
|
|
22
|
+
]
|
|
23
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schema_version": "ml_data_platform.dataset_manifest/v1",
|
|
3
|
+
"dataset_id": "example_temperature_training",
|
|
4
|
+
"dataset_version": "v1",
|
|
5
|
+
"mode": "training",
|
|
6
|
+
"read_policy": "snapshot",
|
|
7
|
+
"time_range": {
|
|
8
|
+
"start": "2026-07-01T00:00:00+08:00",
|
|
9
|
+
"end": "2026-07-02T00:00:00+08:00",
|
|
10
|
+
"grid": "10min"
|
|
11
|
+
},
|
|
12
|
+
"parameters": [],
|
|
13
|
+
"feature_set": {
|
|
14
|
+
"name": "example_temperature_core",
|
|
15
|
+
"version": "v1"
|
|
16
|
+
},
|
|
17
|
+
"preprocess": [],
|
|
18
|
+
"output": {
|
|
19
|
+
"format": "parquet",
|
|
20
|
+
"include_stats": true,
|
|
21
|
+
"include_lineage": true,
|
|
22
|
+
"engine": "chronon"
|
|
23
|
+
}
|
|
24
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schema_version": "ml_data_platform.feature_set/v1",
|
|
3
|
+
"name": "example_temperature_core",
|
|
4
|
+
"version": "v1",
|
|
5
|
+
"features": [
|
|
6
|
+
{
|
|
7
|
+
"name": "example_temperature_mean_5m",
|
|
8
|
+
"version": "v1"
|
|
9
|
+
}
|
|
10
|
+
],
|
|
11
|
+
"owner": "replace_with_business_owner",
|
|
12
|
+
"description": "Ordered example temperature features."
|
|
13
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schema_version": "ml_data_platform.feature/v1",
|
|
3
|
+
"name": "example_temperature_mean_5m",
|
|
4
|
+
"version": "v1",
|
|
5
|
+
"inputs": [
|
|
6
|
+
{
|
|
7
|
+
"parameter": "example_temperature",
|
|
8
|
+
"version": "v1"
|
|
9
|
+
}
|
|
10
|
+
],
|
|
11
|
+
"operator": "example_temperature_features",
|
|
12
|
+
"operator_version": "v1",
|
|
13
|
+
"config": {
|
|
14
|
+
"window": "5min"
|
|
15
|
+
},
|
|
16
|
+
"output_column": "example_temperature_mean_5m",
|
|
17
|
+
"output_dtype": "float64",
|
|
18
|
+
"offline_online_supported": false,
|
|
19
|
+
"owner": "replace_with_business_owner",
|
|
20
|
+
"description": "Mean example temperature in the causal interval (target-5min, target]."
|
|
21
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "business-feature-operator-template"
|
|
7
|
+
version = "1.0.0"
|
|
8
|
+
requires-python = ">=3.10"
|
|
9
|
+
dependencies = ["pandas>=2.3.3"]
|
|
10
|
+
|
|
11
|
+
[tool.hatch.build.targets.wheel]
|
|
12
|
+
packages = ["src/business_feature_operator_template"]
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""Example business Feature Operator. Replace names and formulas before use."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
import pandas as pd
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
SUPPORTED_OUTPUTS = {"example_temperature_mean_5m"}
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def compute_features(context: Any) -> pd.DataFrame:
|
|
14
|
+
requested = list(dict.fromkeys(context.requested_output_columns))
|
|
15
|
+
unknown = sorted(set(requested) - SUPPORTED_OUTPUTS)
|
|
16
|
+
if unknown:
|
|
17
|
+
raise ValueError(f"unsupported output columns: {unknown}")
|
|
18
|
+
|
|
19
|
+
input_keys = {item.parameter: item.key for item in context.inputs}
|
|
20
|
+
parameter_key = input_keys.get("example_temperature")
|
|
21
|
+
if parameter_key is None:
|
|
22
|
+
raise ValueError("example_temperature input is required")
|
|
23
|
+
|
|
24
|
+
frame = context.metric_frames[parameter_key]
|
|
25
|
+
values = pd.to_numeric(
|
|
26
|
+
frame.set_index("timestamp")["value"],
|
|
27
|
+
errors="coerce",
|
|
28
|
+
).sort_index()
|
|
29
|
+
if values.index.has_duplicates:
|
|
30
|
+
values = values.groupby(level=0).last()
|
|
31
|
+
|
|
32
|
+
window = pd.Timedelta(str(context.config["window"]))
|
|
33
|
+
output: dict[str, object] = {"event_time": context.target_times}
|
|
34
|
+
if "example_temperature_mean_5m" in requested:
|
|
35
|
+
output["example_temperature_mean_5m"] = [
|
|
36
|
+
values.loc[(values.index > target - window) & (values.index <= target)].mean()
|
|
37
|
+
for target in context.target_times
|
|
38
|
+
]
|
|
39
|
+
return pd.DataFrame(output)
|
package/skills/feature-management/assets/catalog-template/operator_package/tests/test_operator.py
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import unittest
|
|
4
|
+
from types import SimpleNamespace
|
|
5
|
+
|
|
6
|
+
import pandas as pd
|
|
7
|
+
|
|
8
|
+
from business_feature_operator_template import compute_features
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class ComputeFeaturesTest(unittest.TestCase):
|
|
12
|
+
def context(self, requested: list[str] | None = None) -> SimpleNamespace:
|
|
13
|
+
source = pd.DataFrame(
|
|
14
|
+
{
|
|
15
|
+
"timestamp": pd.DatetimeIndex(
|
|
16
|
+
[
|
|
17
|
+
"2026-07-01 09:54:00",
|
|
18
|
+
"2026-07-01 09:56:00",
|
|
19
|
+
"2026-07-01 10:00:00",
|
|
20
|
+
"2026-07-01 10:01:00",
|
|
21
|
+
]
|
|
22
|
+
),
|
|
23
|
+
"value": [10.0, 20.0, 30.0, 40.0],
|
|
24
|
+
}
|
|
25
|
+
)
|
|
26
|
+
return SimpleNamespace(
|
|
27
|
+
requested_output_columns=requested or ["example_temperature_mean_5m"],
|
|
28
|
+
inputs=[
|
|
29
|
+
SimpleNamespace(
|
|
30
|
+
parameter="example_temperature",
|
|
31
|
+
key="example_temperature:v1",
|
|
32
|
+
)
|
|
33
|
+
],
|
|
34
|
+
metric_frames={"example_temperature:v1": source},
|
|
35
|
+
target_times=pd.DatetimeIndex(
|
|
36
|
+
["2026-07-01 10:00:00", "2026-07-01 10:02:00"]
|
|
37
|
+
),
|
|
38
|
+
config={"window": "5min"},
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
def test_uses_open_left_and_closed_right_window(self) -> None:
|
|
42
|
+
result = compute_features(self.context())
|
|
43
|
+
|
|
44
|
+
self.assertEqual(
|
|
45
|
+
list(result.columns),
|
|
46
|
+
["event_time", "example_temperature_mean_5m"],
|
|
47
|
+
)
|
|
48
|
+
self.assertEqual(result["example_temperature_mean_5m"].tolist(), [25.0, 35.0])
|
|
49
|
+
|
|
50
|
+
def test_rejects_unknown_requested_output(self) -> None:
|
|
51
|
+
with self.assertRaisesRegex(ValueError, "unsupported output columns"):
|
|
52
|
+
compute_features(self.context(["unknown_feature"]))
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
if __name__ == "__main__":
|
|
56
|
+
unittest.main()
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schema_version": "ml_data_platform.operator/v1",
|
|
3
|
+
"name": "example_temperature_features",
|
|
4
|
+
"version": "v1",
|
|
5
|
+
"type": "feature",
|
|
6
|
+
"function_hash": "example_temperature_features.formulas.v1",
|
|
7
|
+
"entrypoint": "business_feature_operator_template:compute_features",
|
|
8
|
+
"code_hash": null,
|
|
9
|
+
"package_uri": null,
|
|
10
|
+
"code_artifact": null,
|
|
11
|
+
"input_schema": {
|
|
12
|
+
"parameters": [
|
|
13
|
+
"example_temperature:v1"
|
|
14
|
+
]
|
|
15
|
+
},
|
|
16
|
+
"output_schema": {
|
|
17
|
+
"columns": [
|
|
18
|
+
"event_time",
|
|
19
|
+
"example_temperature_mean_5m"
|
|
20
|
+
]
|
|
21
|
+
},
|
|
22
|
+
"config_schema": {
|
|
23
|
+
"properties": {
|
|
24
|
+
"window": {
|
|
25
|
+
"type": "string"
|
|
26
|
+
}
|
|
27
|
+
},
|
|
28
|
+
"required": [
|
|
29
|
+
"window"
|
|
30
|
+
]
|
|
31
|
+
},
|
|
32
|
+
"runtime": {
|
|
33
|
+
"engine": "python_entrypoint",
|
|
34
|
+
"network": "none"
|
|
35
|
+
},
|
|
36
|
+
"resources": {
|
|
37
|
+
"timeout_seconds": 60
|
|
38
|
+
},
|
|
39
|
+
"deterministic": true,
|
|
40
|
+
"supports_batch": true,
|
|
41
|
+
"supports_online": false,
|
|
42
|
+
"owner": "replace_with_business_owner"
|
|
43
|
+
}
|
package/skills/feature-management/assets/catalog-template/parameters/example_temperature.v1.json
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schema_version": "ml_data_platform.parameter/v1",
|
|
3
|
+
"name": "example_temperature",
|
|
4
|
+
"display_name": "Example temperature",
|
|
5
|
+
"version": "v1",
|
|
6
|
+
"data_type": "time_series",
|
|
7
|
+
"unit": "degC",
|
|
8
|
+
"expected_frequency": "1min",
|
|
9
|
+
"source": {
|
|
10
|
+
"adapter": "postgresql_direct",
|
|
11
|
+
"mode": "direct_column",
|
|
12
|
+
"schema": "process_data",
|
|
13
|
+
"table": "sensor_readings",
|
|
14
|
+
"time_column": "event_time",
|
|
15
|
+
"value_column": "temperature",
|
|
16
|
+
"metric_name": "example_temperature",
|
|
17
|
+
"unit_column": "unit",
|
|
18
|
+
"filters": {
|
|
19
|
+
"furnace_id": "BF12"
|
|
20
|
+
}
|
|
21
|
+
},
|
|
22
|
+
"time_semantics": {
|
|
23
|
+
"event_time_field": "event_time",
|
|
24
|
+
"ingested_at_field": "ingested_at",
|
|
25
|
+
"timezone": "Asia/Shanghai",
|
|
26
|
+
"availability": {
|
|
27
|
+
"strategy": "source_field",
|
|
28
|
+
"field": "ingested_at",
|
|
29
|
+
"accuracy": "exact"
|
|
30
|
+
}
|
|
31
|
+
},
|
|
32
|
+
"availability_sla": {
|
|
33
|
+
"max_delay": "PT10M"
|
|
34
|
+
},
|
|
35
|
+
"value_field": "value",
|
|
36
|
+
"quality_rules": {
|
|
37
|
+
"valid_range": [0.0, 2000.0],
|
|
38
|
+
"allow_missing": true,
|
|
39
|
+
"rules": [
|
|
40
|
+
{
|
|
41
|
+
"id": "finite_source_values",
|
|
42
|
+
"stage": "normalized_source",
|
|
43
|
+
"check": {
|
|
44
|
+
"type": "finite"
|
|
45
|
+
},
|
|
46
|
+
"acceptance": {
|
|
47
|
+
"max_violation_rate": 0.0,
|
|
48
|
+
"min_evaluated_rows": 1,
|
|
49
|
+
"min_comparable_rate": 0.0
|
|
50
|
+
},
|
|
51
|
+
"enforcement": "fail",
|
|
52
|
+
"description": "Reject non-finite source values before preprocessing."
|
|
53
|
+
}
|
|
54
|
+
]
|
|
55
|
+
},
|
|
56
|
+
"owner": "replace_with_business_owner"
|
|
57
|
+
}
|