@zhuoyuezs/ml-platform 0.1.8 → 0.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -3
- package/package.json +1 -1
- package/checksums.json +0 -155
- package/release.json +0 -52
- package/runtime/business-client/README.md +0 -78
- package/runtime/business-client/package-lock.json +0 -19
- package/runtime/business-client/package.json +0 -23
- package/runtime/business-client/src/catalog.js +0 -206
- package/runtime/business-client/src/cli.js +0 -533
- package/runtime/business-client/src/config.js +0 -56
- package/runtime/business-client/src/http.js +0 -254
- package/skills/feature-management/SKILL.md +0 -479
- package/skills/feature-management/agents/openai.yaml +0 -4
- package/skills/feature-management/assets/catalog-template/catalog.json +0 -23
- package/skills/feature-management/assets/catalog-template/datasets/example_temperature_training.v1.json +0 -40
- package/skills/feature-management/assets/catalog-template/feature_sets/example_temperature_core.v1.json +0 -14
- package/skills/feature-management/assets/catalog-template/features/example_temperature_mean_5m.v1.json +0 -28
- package/skills/feature-management/assets/catalog-template/operator_package/pyproject.toml +0 -12
- package/skills/feature-management/assets/catalog-template/operator_package/src/business_feature_operator_template/__init__.py +0 -39
- package/skills/feature-management/assets/catalog-template/operator_package/tests/test_operator.py +0 -83
- package/skills/feature-management/assets/catalog-template/operators/example_temperature_features.v1.json +0 -58
- package/skills/feature-management/assets/catalog-template/parameters/example_temperature.v1.json +0 -58
- package/skills/feature-management/references/commands.md +0 -358
- package/skills/feature-management/references/contracts.md +0 -719
- package/skills/feature-management/references/operator-authoring.md +0 -175
- package/skills/feature-management/references/platform-capability-guide.md +0 -75
- package/skills/feature-management/references/supervised-datasets.md +0 -101
- package/skills/model-lifecycle-management/SKILL.md +0 -38
- package/skills/model-lifecycle-management/agents/openai.yaml +0 -4
- package/skills/model-lifecycle-management/references/discovery.md +0 -89
- package/skills/model-lifecycle-management/references/evaluation.md +0 -172
- package/skills/model-lifecycle-management/references/packaging.md +0 -51
- package/skills/model-lifecycle-management/references/training-contracts.md +0 -139
- package/skills/model-lifecycle-management/references/training.md +0 -81
|
@@ -1,479 +0,0 @@
|
|
|
1
|
-
---
|
|
2
|
-
name: feature-management
|
|
3
|
-
description: Manage projects, versioned business feature assets and their lifecycle, dataset rowsets and endpoint eligibility, and causal realtime row or sequence-context reads through the deployed ML Platform API using the ml-platform CLI installed with the same release. Use when a business user asks an agent to manage an empty Project, define or change Parameter source contracts, author/package/register an Operator wheel, define single-column Features, order a FeatureSet, create a DatasetManifest or catalog, inspect references, soft-delete or restore a Parameter/Feature/FeatureSet version, configure training/validation/test rowsets or abnormal-data endpoint exclusion, configure realtime freshness policy, run catalog dry-run/apply, build a dataset, fetch inference data, or inspect an artifact without downloading the platform source repository. Do not use for model training, Operator/Dataset deletion, image builds, or Kubernetes deployment.
|
|
4
|
-
---
|
|
5
|
-
|
|
6
|
-
# Feature Management
|
|
7
|
-
|
|
8
|
-
Build the public asset chain without bypassing its versioned contracts:
|
|
9
|
-
|
|
10
|
-
```text
|
|
11
|
-
Parameter -----> Feature
|
|
12
|
-
Operator ------> Feature -> FeatureSet -> DatasetManifest -> DatasetArtifact
|
|
13
|
-
```
|
|
14
|
-
|
|
15
|
-
Treat `Recipe` and public `Feature.compute` as removed. Treat `computation_hash` as internal execution metadata, never as a business-managed asset.
|
|
16
|
-
|
|
17
|
-
## Platform Role And Evidence Boundaries
|
|
18
|
-
|
|
19
|
-
The ML Platform is a deterministic data-contract and data-delivery platform. It
|
|
20
|
-
registers versioned source Parameters, runs approved deterministic Operators,
|
|
21
|
-
orders single-column Features into FeatureSets, resolves DatasetManifests, and
|
|
22
|
-
materializes batch DatasetArtifacts or serves one causal realtime inference row.
|
|
23
|
-
It records versions, lineage, quality, missingness, freshness, and execution
|
|
24
|
-
metadata so an algorithm project can consume data without owning source routing
|
|
25
|
-
or cache details.
|
|
26
|
-
|
|
27
|
-
This Skill does not train, evaluate, or serve models; own complete business feature
|
|
28
|
-
engineering; provision arbitrary source tables; or administer Kubernetes and
|
|
29
|
-
platform services. Training, evaluation and model packaging are handled by the
|
|
30
|
-
separate `model-lifecycle-management` Skill; this scope limit is not a claim that
|
|
31
|
-
the platform lacks those capabilities. Read
|
|
32
|
-
[references/platform-capability-guide.md](references/platform-capability-guide.md)
|
|
33
|
-
when the user needs the detailed platform boundary or evidence-level explanation.
|
|
34
|
-
|
|
35
|
-
Keep these evidence rules active in every workflow:
|
|
36
|
-
|
|
37
|
-
- Platform defaults and capabilities are not business evidence.
|
|
38
|
-
- Dry-run, publish, resolve, build, metadata inspection, downloaded-file
|
|
39
|
-
inspection, and realtime fetch are separate evidence checkpoints; success at
|
|
40
|
-
one checkpoint does not imply success at the next.
|
|
41
|
-
- `validation.ok=true` does not by itself prove source semantics, causal formula
|
|
42
|
-
correctness, Parquet contents, or numeric parity.
|
|
43
|
-
- Dry-run, publish, and resolve may not detect a missing source relation. Do not
|
|
44
|
-
report source-backed success until build/fetch returns the required evidence,
|
|
45
|
-
and never rename a Parameter or remove fields merely to bypass a failed read.
|
|
46
|
-
|
|
47
|
-
## Load The Right Context
|
|
48
|
-
|
|
49
|
-
1. Resolve the Skill root as the directory containing this `SKILL.md`.
|
|
50
|
-
2. Read [references/contracts.md](references/contracts.md) before creating or changing JSON assets.
|
|
51
|
-
3. Read [references/operator-authoring.md](references/operator-authoring.md) whenever creating or changing Operator code or a wheel.
|
|
52
|
-
4. Read [references/commands.md](references/commands.md) immediately before a platform CLI action: discovery, validation, publication, build, realtime fetch, or artifact download. A local-only draft that is forbidden to call the CLI does not need this reference.
|
|
53
|
-
5. Use [assets/catalog-template](assets/catalog-template) as a copyable starting point for a new end-to-end catalog. Rename every `example_*` identifier and update every referenced path before validation.
|
|
54
|
-
6. Read [references/platform-capability-guide.md](references/platform-capability-guide.md) only when selecting an asset type or explaining a platform capability. Do not load it merely to write a confirmed local draft; it is never evidence for unresolved business semantics.
|
|
55
|
-
|
|
56
|
-
## Initialize The Client
|
|
57
|
-
|
|
58
|
-
Use the `ml-platform` executable on `PATH` for every platform command. The formal npm release installs
|
|
59
|
-
the CLI and this pure Skill together. The Skill does not contain a second CLI runtime, launcher, or
|
|
60
|
-
installer. During repository development, provide the current JavaScript CLI through an isolated test
|
|
61
|
-
`PATH`.
|
|
62
|
-
|
|
63
|
-
Before any platform operation, verify the executable and its version:
|
|
64
|
-
|
|
65
|
-
```bash
|
|
66
|
-
command -v ml-platform
|
|
67
|
-
ml-platform version
|
|
68
|
-
```
|
|
69
|
-
|
|
70
|
-
If the executable is missing or cannot start, stop and report that the ML Platform release installation is incomplete. Do not download a client, search for a repository checkout, or fall back to another executable.
|
|
71
|
-
|
|
72
|
-
Before any server-profile discovery, dry-run, publication, build, or artifact
|
|
73
|
-
operation, inspect the effective API configuration:
|
|
74
|
-
|
|
75
|
-
```bash
|
|
76
|
-
ml-platform show-config
|
|
77
|
-
ml-platform --profile server health
|
|
78
|
-
```
|
|
79
|
-
|
|
80
|
-
Use the global `--profile server` form shown above, before the business
|
|
81
|
-
command. Do not append `--profile` after a business command. Before discovery,
|
|
82
|
-
check the installed release with `ml-platform --help` and the relevant
|
|
83
|
-
subcommand's `--help`; CLI releases may differ in supported flags. In the
|
|
84
|
-
deployed 0.6.x CLI, registry `list-*` commands emit JSON by default and do not
|
|
85
|
-
accept a `--json` flag.
|
|
86
|
-
|
|
87
|
-
If the installed release rejects the global `--profile server` form shown above,
|
|
88
|
-
record the CLI syntax error and inspect `show-config`/subcommand help before
|
|
89
|
-
retrying the same read-only command without that flag when supported. Do not
|
|
90
|
-
switch clients or classify a syntax error as an API outage; keep each command
|
|
91
|
-
separate so CLI compatibility remains distinguishable from network health.
|
|
92
|
-
|
|
93
|
-
The effective API target is resolved in this order: explicit `--api-url`,
|
|
94
|
-
`ML_PLATFORM_API_URL`, then the saved value written by `ml-platform configure`.
|
|
95
|
-
If no target is configured, ask the user for an approved API URL before making
|
|
96
|
-
server requests. Do not hardcode an environment-specific address in a Skill,
|
|
97
|
-
catalog, or prompt. Keep database, MinIO, and source credentials out of commands;
|
|
98
|
-
use the server's managed Secrets.
|
|
99
|
-
|
|
100
|
-
Install and upgrade the CLI and Skill only through the explicit commands provided by `@zhuoyuezs/ml-platform`. Restart the Agent session after a successful Skill upgrade so the definition is reloaded. Never copy files over an installed Skill directory manually.
|
|
101
|
-
|
|
102
|
-
## Establish Intent
|
|
103
|
-
|
|
104
|
-
Collect only missing information. Do not invent a source table, source field, formula, time boundary, unit, valid range, missing-data rule, or owner.
|
|
105
|
-
|
|
106
|
-
Confirm:
|
|
107
|
-
|
|
108
|
-
- the target project (namespace) for the assets; use the built-in `default` only when the user has no dedicated project. After the target is confirmed, write its explicit `project` field in every Parameter, Operator, Feature, FeatureSet and DatasetManifest draft; never rely on the schema default. Reads and filters use the `--project` command option;
|
|
109
|
-
- business meaning, stable asset names, owner, and intended consumers;
|
|
110
|
-
- Parameter source adapter and credential-free source mapping;
|
|
111
|
-
- feature formula, exact Parameter dependencies, windows, inclusion rules, rounding, null behavior, and output dtype;
|
|
112
|
-
- whether an approved Operator already implements the formula;
|
|
113
|
-
- FeatureSet column order;
|
|
114
|
-
- dataset mode, read policy, time range, grid, prediction horizon/cutoff contract, and explicit Parameter outputs;
|
|
115
|
-
- named training/validation/test rowsets, abnormal windows, endpoint-policy scope,
|
|
116
|
-
and the model context lookback used to decide endpoint eligibility;
|
|
117
|
-
- realtime freshness and missing-input policy when serving inference data;
|
|
118
|
-
- target profile/API and whether the user authorizes publish and/or build actions.
|
|
119
|
-
|
|
120
|
-
Separate preparation from mutation. Creating files and running local tests or `apply --dry-run` does not authorize publishing a wheel, changing the Registry, or submitting a build Job.
|
|
121
|
-
|
|
122
|
-
Soft-delete and restore are separate Registry mutations. Before soft-deleting a
|
|
123
|
-
Parameter, Feature, or FeatureSet version, inspect its lifecycle state and both
|
|
124
|
-
direct and transitive references, report the affected immutable consumers, and
|
|
125
|
-
obtain explicit authorization for the exact `project/name:version`. Supply an
|
|
126
|
-
actor, reason, and stable idempotency key. Soft deletion preserves historical
|
|
127
|
-
references and hashes but hides the version from normal discovery and prevents
|
|
128
|
-
new consumers from binding it. Restore only after checking that all of the
|
|
129
|
-
resource's own dependencies are active. Never describe these operations as
|
|
130
|
-
physical deletion.
|
|
131
|
-
|
|
132
|
-
A label such as "10-minute mean" confirms neither its input Parameter nor a
|
|
133
|
-
complete formula. Before drafting an Operator or Feature, ask for the exact
|
|
134
|
-
versioned input Parameter(s), or for an approved source contract from which
|
|
135
|
-
they will be created. Do not infer an input from the Feature name or a similar
|
|
136
|
-
existing asset.
|
|
137
|
-
|
|
138
|
-
### Semantic Asset Review Gate
|
|
139
|
-
|
|
140
|
-
Treat resource correctness as a separate gate from workflow correctness. Before
|
|
141
|
-
creating Catalog JSON, write a local semantic asset review with one row per
|
|
142
|
-
Parameter, Operator, Feature and Dataset field:
|
|
143
|
-
|
|
144
|
-
```text
|
|
145
|
-
asset_key | business_meaning | source/evidence | confirmed_by | unresolved | proposed_value
|
|
146
|
-
```
|
|
147
|
-
|
|
148
|
-
The review must distinguish three sources: confirmed business facts, facts
|
|
149
|
-
observed from the selected Project Registry/CLI, and agent proposals. A Skill
|
|
150
|
-
reference, schema default, similarly named old asset, or platform capability is
|
|
151
|
-
not business evidence. Do not convert a proposal into `proposed_value` without
|
|
152
|
-
explicit user confirmation or an exact match in the selected Project.
|
|
153
|
-
|
|
154
|
-
Stop with `need_business_confirmation` when any required source mapping, field,
|
|
155
|
-
unit, formula, time conversion, missing/quality policy, FeatureSet order,
|
|
156
|
-
dataset read policy, rowset/endpoint policy, owner, or approval boundary is
|
|
157
|
-
unresolved. Do not create placeholder Catalog files to make the workflow look
|
|
158
|
-
complete.
|
|
159
|
-
|
|
160
|
-
For a requested Feature, `exact Parameter dependencies` is an independent
|
|
161
|
-
required contract item. If the user names a transform or window but not its
|
|
162
|
-
input Parameter(s), include `input_parameter` in the unresolved questions and
|
|
163
|
-
stop. Do this before treating an Operator formula as complete: a time window,
|
|
164
|
-
cutoff rule, or Feature name never identifies the input.
|
|
165
|
-
|
|
166
|
-
After confirmation, show the proposed dependency closure and obtain approval of
|
|
167
|
-
the semantic asset plan before generating Catalog files. A successful local
|
|
168
|
-
schema check or remote dry-run is not semantic approval.
|
|
169
|
-
|
|
170
|
-
Before emitting or writing a draft, mechanically check that every asset carries
|
|
171
|
-
the confirmed `project` field, that every Feature input carries that same
|
|
172
|
-
`project`, and that every Operator `input_schema.parameters` reference is
|
|
173
|
-
qualified as `project/parameter:version`. The schema's `default` values are not
|
|
174
|
-
valid substitutes for an explicit non-default target. When a confirmed Feature
|
|
175
|
-
config defines window closure, cutoff inclusion, duplicate handling, or empty
|
|
176
|
-
window behavior, copy all of those fields into the Feature config rather than
|
|
177
|
-
reducing it to the window string.
|
|
178
|
-
|
|
179
|
-
## Discover Existing Assets
|
|
180
|
-
|
|
181
|
-
Check the selected profile and list existing Parameter, Operator, Feature, FeatureSet, and Dataset versions before choosing names. Scope discovery to the target project with `--project`, because the true registry key is `project/name:version` and the same `name:version` may exist independently under another project. Run each `list-*` command as a separate command so one failed check cannot hide the status of the others. Do not add `--json` unless the installed subcommand help explicitly advertises it; supported releases emit machine-readable JSON by default. Registry list commands are paginated; search by stable identifier or follow every page until `offset + len(items) >= total`. Reuse an exact immutable version only when its full content matches. Never reference an asset in another project; cross-project references are rejected.
|
|
182
|
-
|
|
183
|
-
CLI readiness is a hard gate for every Registry conclusion. If `command -v`,
|
|
184
|
-
`version`, `show-config`, or server `health` fails, returns a nonzero status, or
|
|
185
|
-
does not identify the intended target, stop with `platform_not_ready`. Do not
|
|
186
|
-
list assets, select reuse versus authoring, dry-run, publish, build, fetch, or
|
|
187
|
-
report a Registry fact from prompt text, a remembered response, or a similar
|
|
188
|
-
local asset. Report the failed command and its configured target; resume only
|
|
189
|
-
after the user/environment restores the same target.
|
|
190
|
-
|
|
191
|
-
Create a new version when source semantics, formula code, config meaning, inputs, output dtype, time behavior, quality rules, or column order change. Never overwrite an immutable version or use suffixes such as `new`, `final`, or `test2`.
|
|
192
|
-
|
|
193
|
-
### Operate An Existing Registered Dataset
|
|
194
|
-
|
|
195
|
-
For diagnosis, resolve, build, Job lookup, and artifact lookup of an already
|
|
196
|
-
registered DatasetManifest, use the API through `ml-platform`. Do not search the
|
|
197
|
-
platform source checkout or a local Catalog directory for a matching JSON file.
|
|
198
|
-
|
|
199
|
-
1. Run `show-config` and `health` to record the effective API source.
|
|
200
|
-
2. Run `list-projects`, then `list-datasets --project <project>` to discover
|
|
201
|
-
candidates from the Registry.
|
|
202
|
-
3. Ask the user to confirm the exact `project`, `dataset_id`, and
|
|
203
|
-
`dataset_version` when more than one valid candidate exists. Never infer a
|
|
204
|
-
target from a similar local filename.
|
|
205
|
-
4. Run `get-dataset <dataset_id> <dataset_version> --project <project>` and
|
|
206
|
-
inspect the returned registered contract.
|
|
207
|
-
5. Run `resolve-dataset` for a read-only dependency and manifest-hash check.
|
|
208
|
-
6. Submit exactly one `build-registered-dataset ... --wait` only after the user
|
|
209
|
-
authorizes that exact registered version.
|
|
210
|
-
7. Use the returned Job ID with `get-job`, then use the returned manifest hash
|
|
211
|
-
with `get-dataset-artifact` after success.
|
|
212
|
-
|
|
213
|
-
Local `resolve-manifest` and `build-dataset <file>` remain authoring tools for a
|
|
214
|
-
new or changed Catalog contract before it is registered. They are not required
|
|
215
|
-
for operating an existing Registry version.
|
|
216
|
-
|
|
217
|
-
## Upgrade Dependency Chains
|
|
218
|
-
|
|
219
|
-
Treat a version change as a release of its affected reverse-dependency closure, not as an isolated asset bump.
|
|
220
|
-
|
|
221
|
-
1. Record the intended old-to-new version mapping and the semantic reason for every changed asset.
|
|
222
|
-
2. Follow exact references from the changed asset through every affected downstream consumer. For a Parameter change, publish a new Operator version when its input schema names that Parameter version, then new versions of every affected Feature, FeatureSet, and DatasetManifest. For an Operator change, start with its affected Features.
|
|
223
|
-
3. Republish only the affected closure, but explicitly list and justify every downstream asset intentionally retained on an older dependency.
|
|
224
|
-
4. Keep every reference exact. Never rewrite an existing immutable version, infer `latest`, or assume that increasing a Feature or FeatureSet version also upgrades its dependencies.
|
|
225
|
-
5. Treat `apply --dry-run` as an existence and structural-compatibility check. It does not prove that the declared versions are the intended business release.
|
|
226
|
-
|
|
227
|
-
Use this dependency order for an affected chain:
|
|
228
|
-
|
|
229
|
-
```text
|
|
230
|
-
Parameter -> Operator -> Feature -> FeatureSet -> DatasetManifest
|
|
231
|
-
```
|
|
232
|
-
|
|
233
|
-
## Build The Catalog
|
|
234
|
-
|
|
235
|
-
Keep one business domain in one user-owned catalog directory:
|
|
236
|
-
|
|
237
|
-
```text
|
|
238
|
-
<catalog>/
|
|
239
|
-
catalog.json
|
|
240
|
-
parameters/
|
|
241
|
-
operators/
|
|
242
|
-
operator_package/
|
|
243
|
-
features/
|
|
244
|
-
feature_sets/
|
|
245
|
-
datasets/
|
|
246
|
-
```
|
|
247
|
-
|
|
248
|
-
Keep platform demos and V95 built-ins unchanged. Do not put unrelated business assets under `src/data_platform_demo/builtin_catalogs/v95`.
|
|
249
|
-
|
|
250
|
-
### Define Parameters
|
|
251
|
-
|
|
252
|
-
Use the platform-capability guide's asset table before choosing the resource
|
|
253
|
-
type. For each Parameter record the business signal, exact source evidence,
|
|
254
|
-
unit, time/availability semantics and missing policy in the semantic review.
|
|
255
|
-
Never infer a source table, column, measurement, tag, unit or timezone from a
|
|
256
|
-
Parameter name.
|
|
257
|
-
|
|
258
|
-
Create one stable, versioned source contract for each independently readable value. Keep credentials out of JSON. Use only supported direct adapters and explicit source identity. Put source data validity in `quality_rules`; put dataset-specific transformations in preprocess Operators.
|
|
259
|
-
|
|
260
|
-
When the approved source contract supplies SQL, copy its query text and named
|
|
261
|
-
parameters exactly into `source.sql`/`source.params`. Do not retype, beautify,
|
|
262
|
-
rename an identifier, change a join key, or "simplify" an approved query while
|
|
263
|
-
drafting. SQL whitespace may be formatted only when a reviewed formatter proves
|
|
264
|
-
the token stream is unchanged; every source identifier and `<= %(end)s` boundary
|
|
265
|
-
remains part of the Parameter contract.
|
|
266
|
-
|
|
267
|
-
For SQL Parameters, the runtime contract is a single statement: use the exact
|
|
268
|
-
`source.sql` field and a mapping under `source.params` for every named
|
|
269
|
-
placeholder other than the reserved `start` and `end` window parameters. Do not
|
|
270
|
-
use `source.parameters` (it is not a supported alias and may be silently
|
|
271
|
-
dropped by older CLI/server releases). Do not leave a trailing semicolon; the
|
|
272
|
-
deployed Worker rejects SQL containing multiple statements. Verify the resolved
|
|
273
|
-
Parameter still contains the named params before authorizing a build.
|
|
274
|
-
|
|
275
|
-
Use Parameter `rounding` when every consumer must receive the same fixed-point value. The platform applies `mode` (`half_up` or `half_even`) and `decimals` after source normalization and before quality checks, source caching, replay, and Operator execution. Prefer an unrounded source expression, do not repeat the same rounding in an Operator, and publish a new Parameter plus its affected reverse-dependency closure when the rule changes.
|
|
276
|
-
|
|
277
|
-
Do not represent a rolling mean, lag, ratio, trend, or model input as a Parameter. Those are Features.
|
|
278
|
-
|
|
279
|
-
### Define Operators
|
|
280
|
-
|
|
281
|
-
First reuse a registered Operator if its version, input schema, output schema, formula semantics, and runtime contract match exactly.
|
|
282
|
-
|
|
283
|
-
When a new Operator is required:
|
|
284
|
-
|
|
285
|
-
1. Write a standalone pure-Python package with a `module:function` entrypoint.
|
|
286
|
-
2. Make the Feature entrypoint accept one context and return a pandas DataFrame.
|
|
287
|
-
3. Consume only declared `context.inputs` and `context.metric_frames`.
|
|
288
|
-
4. Return `event_time` exactly equal to `context.target_times` plus the requested physical output columns.
|
|
289
|
-
5. Implement explicit causal boundaries; never use future rows accidentally.
|
|
290
|
-
6. Keep execution deterministic, network-free, credential-free, and bounded by `timeout_seconds`.
|
|
291
|
-
7. Add formula, boundary, missing-value, requested-output, and ordering tests.
|
|
292
|
-
8. Build a `py3-none-any` wheel and reference it from `catalog.json`.
|
|
293
|
-
9. Leave `code_hash`, `package_uri`, and `code_artifact` null in the draft spec; catalog publication fills them from the uploaded wheel.
|
|
294
|
-
|
|
295
|
-
Use a meaningful immutable `function_hash`. Change the Operator version whenever executable behavior changes.
|
|
296
|
-
|
|
297
|
-
### Define Features
|
|
298
|
-
|
|
299
|
-
Create one JSON file per single output column. Set the confirmed `project` on the
|
|
300
|
-
Feature and on every nested Feature input; a FeatureInput without `project`
|
|
301
|
-
silently targets `default`. Bind exactly one Operator version and physical
|
|
302
|
-
`output_column`. List only the Parameter versions actually required by that
|
|
303
|
-
column's formula.
|
|
304
|
-
|
|
305
|
-
Features may share the same Operator and config. The planner will merge their exact inputs and call the Operator once per computation group. Do not duplicate every Operator input into every Feature merely to make schemas look uniform.
|
|
306
|
-
|
|
307
|
-
### Define The FeatureSet
|
|
308
|
-
|
|
309
|
-
Reference immutable Feature versions in the exact consumer column order. Keep the list nonempty and duplicate-free. Do not copy formulas, inputs, or Operator configuration into the FeatureSet.
|
|
310
|
-
|
|
311
|
-
### Define The DatasetManifest
|
|
312
|
-
|
|
313
|
-
For supervised datasets, read
|
|
314
|
-
[references/supervised-datasets.md](references/supervised-datasets.md) before
|
|
315
|
-
publishing the first version. Settle labels, clocks, split bindings and the final
|
|
316
|
-
DataSchema before expensive training.
|
|
317
|
-
|
|
318
|
-
Do not use DatasetManifest as a catch-all for unresolved semantics. Confirm the
|
|
319
|
-
dataset purpose and consumer, then separately confirm the target grid, horizon,
|
|
320
|
-
read policy, explicit Parameter outputs, rowset or rowset splits, abnormal windows and
|
|
321
|
-
endpoint eligibility. Keep the semantic asset review beside the draft Catalog
|
|
322
|
-
so a reviewer can compare business decisions with generated fields.
|
|
323
|
-
|
|
324
|
-
Reference exactly one FeatureSet. Use `parameters` only for Parameter columns that must also appear explicitly in `parameter_dataset.parquet`; Feature dependencies are resolved automatically.
|
|
325
|
-
|
|
326
|
-
Map the current Manifest schema exactly rather than carrying descriptive fields
|
|
327
|
-
from Parameter, Feature, or FeatureSet into it. A target Parameter reference is
|
|
328
|
-
**nested**; never flatten its `version` or `project` onto `target`:
|
|
329
|
-
|
|
330
|
-
```json
|
|
331
|
-
"target": {
|
|
332
|
-
"parameter": {"parameter": "hot_metal_si", "version": "v1", "project": "<project>"},
|
|
333
|
-
"si_time_source": "dispatch_time",
|
|
334
|
-
"offset_minutes": 20,
|
|
335
|
-
"interpolation": {"method": "linear"}
|
|
336
|
-
}
|
|
337
|
-
```
|
|
338
|
-
|
|
339
|
-
`target.interpolation` is `{"method": "linear"}`, `{"method": "none"}` or
|
|
340
|
-
`{"method": "forward_fill"}` -- neither it nor `target.parameter` accepts a
|
|
341
|
-
bare string. The current DatasetManifest has no top-level `owner`,
|
|
342
|
-
`description`, `si_null_policy`, or `preserve_all_candidate_rows`. Map a
|
|
343
|
-
business statement only to an existing schema field (for example `mark_only`,
|
|
344
|
-
`endpoint_policy`, a Feature config, or a Parameter missing policy); otherwise
|
|
345
|
-
keep it in the semantic review and ask for a platform/schema change. Validate
|
|
346
|
-
the generated object against the current schema before running `apply --dry-run`.
|
|
347
|
-
|
|
348
|
-
When a confirmed missing-data policy applies to a Feature-only Parameter, put it
|
|
349
|
-
in `parameter_missing_policies`, using `parameter`, `version`, `project`, and
|
|
350
|
-
`policy`. Do not copy the `ParameterRequest` names `name` and `missing_policy`
|
|
351
|
-
into that list. Omit the section when no such policy is confirmed.
|
|
352
|
-
|
|
353
|
-
Declare one dataset-wide `prediction` contract when Features use a forecast cutoff. Operators consume `context.cutoff_times`; do not copy the same prediction horizon into every Feature config. Check the selected Operator's `input_schema.prediction` limits and satisfy an explicit-horizon requirement before publishing or building.
|
|
354
|
-
|
|
355
|
-
Use `snapshot` for reproducible training data, `as_of` for historical visibility replay, and `latest` for current inference-style reads. Use a fresh `dataset_version` when validating a new release or intentionally requesting a new immutable dataset contract.
|
|
356
|
-
|
|
357
|
-
Treat row construction and endpoint eligibility as separate contracts. Use
|
|
358
|
-
`rowset_splits` to label candidate target instants such as `training`,
|
|
359
|
-
`validation`, and `test`. When abnormal observations should make only selected
|
|
360
|
-
endpoints unusable, keep `abnormal_windows.policy=mark_only` and declare
|
|
361
|
-
`endpoint_policy`; do not delete canonical Feature rows. Declare
|
|
362
|
-
`endpoint_policy.context_lookback` from the model sequence contract, independently
|
|
363
|
-
of Operator source-history requirements. Read [references/contracts.md](references/contracts.md)
|
|
364
|
-
for the cutoff formula, half-open overlap boundaries, scope behavior, and
|
|
365
|
-
`rowset_membership.parquet` schema.
|
|
366
|
-
|
|
367
|
-
For realtime inference, declare `realtime_fetch` only when its effective policy
|
|
368
|
-
must be part of the contract; leaving it unset preserves existing manifest
|
|
369
|
-
hashes. Keep `allow_missing=false` unless the user explicitly accepts missing
|
|
370
|
-
required inputs. Read [references/contracts.md](references/contracts.md) for the
|
|
371
|
-
freshness, tail-edge, gap-fill, and rowset contracts.
|
|
372
|
-
|
|
373
|
-
## Validate Before Publishing
|
|
374
|
-
|
|
375
|
-
Run, in order:
|
|
376
|
-
|
|
377
|
-
1. Semantic asset review: no required unresolved field and every proposed value has a source or user confirmation.
|
|
378
|
-
2. Operator unit tests. Before building, turn every applicable item in
|
|
379
|
-
`operator-authoring.md` into a separately named test; do not let one broad
|
|
380
|
-
happy-path assertion substitute for cutoff-before/at/after, duplicate and
|
|
381
|
-
missing input, empty history, requested-output, event-time order, or
|
|
382
|
-
dtype/rounding coverage. The template is only a starting point.
|
|
383
|
-
3. Wheel build and wheel filename verification.
|
|
384
|
-
4. Catalog path, schema, dependency, immutability, and package validation with `apply --dry-run` against the same target profile intended for publication.
|
|
385
|
-
5. Compare the catalog's exact dependency closure with the intended old-to-new version mapping. Reject any affected downstream reference that still points to an old version unless its retention is explicit and justified.
|
|
386
|
-
6. A human-readable summary of planned new, unchanged, retained, and conflicting assets.
|
|
387
|
-
|
|
388
|
-
Stop on any error. Do not weaken schema validation, fabricate a missing dependency, change an existing version in place, or switch profiles to make validation pass.
|
|
389
|
-
|
|
390
|
-
## Publish And Build
|
|
391
|
-
|
|
392
|
-
Publish only after the user explicitly approves Registry and wheel changes. Use catalog `apply` so publication follows Parameter -> Operator -> Feature -> FeatureSet -> Dataset order.
|
|
393
|
-
|
|
394
|
-
After publication, resolve every new DatasetManifest and compare its exact Parameter, Operator, Feature, and FeatureSet versions with the pre-publication mapping. Stop if an affected old key or any unexpected version remains. Resolve the manifest before building. Submit a build only when requested. For the current partitioned materialization runtime, use `--max-parallelism 1` unless a separately verified platform capability says otherwise; do not retry or resubmit a failed Job until its error and already-completed partitions are understood. For server builds, wait for the terminal Job state and download the exact artifact by `dataset_id + manifest_hash`.
|
|
395
|
-
|
|
396
|
-
Do not delete versioned Registry assets, cancel Jobs, rebuild images, modify Kubernetes, or change service configuration as part of this workflow unless the user separately and explicitly requests that action. An empty Project may be soft-deleted only on an explicit request; rely on the server to reject deletion when resources still exist.
|
|
397
|
-
|
|
398
|
-
## Fetch Realtime Inference Data
|
|
399
|
-
|
|
400
|
-
Use `fetch-inference-data` only when the user requests one causal-cutoff read.
|
|
401
|
-
Confirm the cutoff and prediction horizon. The command must call
|
|
402
|
-
`POST /inference-data/fetch`; it must not submit a batch build, write parquet, or
|
|
403
|
-
publish a DatasetArtifact.
|
|
404
|
-
|
|
405
|
-
Verify that the response reports the requested cutoff and derived target time,
|
|
406
|
-
`uses_post_cutoff_data=false`, `contract.manifest_hash`, a replayable `as_of`
|
|
407
|
-
manifest, and per-Parameter freshness evidence. Report tolerated missing or
|
|
408
|
-
gap-filled inputs as degraded freshness, not as a normal read.
|
|
409
|
-
|
|
410
|
-
## Verify The Artifact
|
|
411
|
-
|
|
412
|
-
Treat Registry metadata inspection and downloaded-file inspection as separate
|
|
413
|
-
checkpoints. A successful `get-dataset-artifact` can establish the immutable
|
|
414
|
-
identity, object inventory, row count, validation summary, and lineage metadata;
|
|
415
|
-
it cannot establish Parquet schema, column order, or file-content hashes. If a
|
|
416
|
-
download times out, report whether no response/progress was observed or whether
|
|
417
|
-
some body bytes were received, preserve the metadata-only evidence, and leave
|
|
418
|
-
downloaded-file validation explicitly incomplete.
|
|
419
|
-
|
|
420
|
-
Require all of the following before reporting success:
|
|
421
|
-
|
|
422
|
-
- Job status is `succeeded`;
|
|
423
|
-
- `validation.json` has `ok: true` and no errors;
|
|
424
|
-
- `validation.feature_nulls` is read, not skipped: `ok: true` can coexist with
|
|
425
|
-
null model inputs reported as warnings. Report `total_null_cells`, and for each
|
|
426
|
-
affected Feature its `null_count`, event-time range, and upstream Parameter
|
|
427
|
-
keys. Set `output.strict_feature_nulls` when any null should fail the build
|
|
428
|
-
instead;
|
|
429
|
-
- row count is nonzero and matches the requested half-open time grid when no
|
|
430
|
-
documented row filtering applies. When `abnormal_windows` uses a drop policy,
|
|
431
|
-
reconcile the lower count against `abnormal_windows.rows_before`,
|
|
432
|
-
`rows_after`, and `dropped_rows` in the resolved manifest. With `mark_only`,
|
|
433
|
-
require `dropped_rows=0` and preserve the full candidate rowset;
|
|
434
|
-
- when `endpoint_policy` is set, require `abnormal_windows.policy=mark_only`,
|
|
435
|
-
preserve every candidate row in `feature_dataset.parquet` and
|
|
436
|
-
`parameter_dataset.parquet`, and require `rowset_membership.parquet`;
|
|
437
|
-
- verify membership counts globally and by rowset, inspect every ineligible
|
|
438
|
-
row's `reason_codes`, and confirm that the configured compatibility column in
|
|
439
|
-
both canonical datasets equals `policy_applied AND eligible`; it is false for
|
|
440
|
-
pass-through rowsets outside the policy scope;
|
|
441
|
-
- for a partitioned build, verify that merged membership, endpoint-policy
|
|
442
|
-
quality statistics, lineage, and `policy_hash` match a single build of the
|
|
443
|
-
same manifest;
|
|
444
|
-
- `feature_dataset.parquet` starts with `event_time`, optional `furnace_id`, then FeatureSet columns in exact order;
|
|
445
|
-
- resolved lineage records every Parameter version, Feature version, Operator version, code hash, and computation hash, exactly matches the approved dependency-version mapping, and contains no stale affected key;
|
|
446
|
-
- artifact lineage contains the same approved dependency closure as the resolved manifest;
|
|
447
|
-
- execution metadata shows the expected number of computation groups and requested physical columns;
|
|
448
|
-
- when `source_read` is set, `latency_stats.fetch_timing_summary` shows the chunk
|
|
449
|
-
windows tiling the fetch range, and any retries are accounted for;
|
|
450
|
-
- missing values and warnings are reported, not silently repaired.
|
|
451
|
-
|
|
452
|
-
The manifest hash returned by `resolve-manifest` or `resolve-dataset` is the artifact key: it stays
|
|
453
|
-
the same after the build, so it is usable for cache reuse and polling before
|
|
454
|
-
submission. Distinguish the two lookup failures rather than treating both as "not
|
|
455
|
-
ready" — `422` with `invalid_artifact_key` means the key can never name an
|
|
456
|
-
artifact, while `404` with `artifact_not_found` or `artifact_not_built` means
|
|
457
|
-
nothing is built under a valid key, and lists any in-flight Jobs.
|
|
458
|
-
|
|
459
|
-
State the current runtime truth: first-phase public builds execute registered Python Operators through `FeatureOperatorRunner`. Do not claim that Chronon compile/backfill or Chronon-native feature computation occurred unless the artifact contains and passes those explicit execution records.
|
|
460
|
-
|
|
461
|
-
## Report The Outcome
|
|
462
|
-
|
|
463
|
-
Return a concise business-facing summary containing:
|
|
464
|
-
|
|
465
|
-
```text
|
|
466
|
-
catalog and version
|
|
467
|
-
new / unchanged assets by type
|
|
468
|
-
planned versus resolved dependency versions and any explicitly retained old versions
|
|
469
|
-
Operator package filename and SHA-256
|
|
470
|
-
FeatureSet and ordered feature count
|
|
471
|
-
dataset id, version, manifest hash, and Job id
|
|
472
|
-
row and column counts
|
|
473
|
-
rowset candidate / policy-applied / eligible / ineligible counts when endpoint_policy is declared
|
|
474
|
-
validation status, warnings, and missing-data summary
|
|
475
|
-
artifact location
|
|
476
|
-
actual compute backend
|
|
477
|
-
```
|
|
478
|
-
|
|
479
|
-
If the work stops before publication, distinguish generated files, locally validated files, dry-run validation, and remotely published assets.
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"schema_version": "ml_data_platform.catalog/v1",
|
|
3
|
-
"name": "example_business_feature_catalog",
|
|
4
|
-
"version": "v1",
|
|
5
|
-
"parameters": [
|
|
6
|
-
"parameters/example_temperature.v1.json"
|
|
7
|
-
],
|
|
8
|
-
"operators": [
|
|
9
|
-
{
|
|
10
|
-
"spec": "operators/example_temperature_features.v1.json",
|
|
11
|
-
"package": "operator_package/dist/business_feature_operator_template-1.0.0-py3-none-any.whl"
|
|
12
|
-
}
|
|
13
|
-
],
|
|
14
|
-
"features": [
|
|
15
|
-
"features/example_temperature_mean_5m.v1.json"
|
|
16
|
-
],
|
|
17
|
-
"feature_sets": [
|
|
18
|
-
"feature_sets/example_temperature_core.v1.json"
|
|
19
|
-
],
|
|
20
|
-
"datasets": [
|
|
21
|
-
"datasets/example_temperature_training.v1.json"
|
|
22
|
-
]
|
|
23
|
-
}
|
|
@@ -1,40 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"schema_version": "ml_data_platform.dataset_manifest/v1",
|
|
3
|
-
"project": "replace_with_project",
|
|
4
|
-
"dataset_id": "example_temperature_training",
|
|
5
|
-
"dataset_version": "v1",
|
|
6
|
-
"mode": "training",
|
|
7
|
-
"read_policy": "snapshot",
|
|
8
|
-
"time_range": {
|
|
9
|
-
"start": "2026-07-01T00:00:00+08:00",
|
|
10
|
-
"end": "2026-07-02T00:00:00+08:00",
|
|
11
|
-
"grid": "10min"
|
|
12
|
-
},
|
|
13
|
-
"prediction": {
|
|
14
|
-
"horizon": "0min"
|
|
15
|
-
},
|
|
16
|
-
"rowset": {
|
|
17
|
-
"strategy": "fixed_grid",
|
|
18
|
-
"grid": "10min"
|
|
19
|
-
},
|
|
20
|
-
"parameters": [],
|
|
21
|
-
"parameter_missing_policies": [
|
|
22
|
-
{
|
|
23
|
-
"parameter": "example_temperature",
|
|
24
|
-
"version": "v1",
|
|
25
|
-
"project": "replace_with_project",
|
|
26
|
-
"policy": "report_only"
|
|
27
|
-
}
|
|
28
|
-
],
|
|
29
|
-
"feature_set": {
|
|
30
|
-
"name": "example_temperature_core",
|
|
31
|
-
"version": "v1"
|
|
32
|
-
},
|
|
33
|
-
"preprocess": [],
|
|
34
|
-
"output": {
|
|
35
|
-
"format": "parquet",
|
|
36
|
-
"include_stats": true,
|
|
37
|
-
"include_lineage": true,
|
|
38
|
-
"engine": "chronon"
|
|
39
|
-
}
|
|
40
|
-
}
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"schema_version": "ml_data_platform.feature_set/v1",
|
|
3
|
-
"project": "replace_with_project",
|
|
4
|
-
"name": "example_temperature_core",
|
|
5
|
-
"version": "v1",
|
|
6
|
-
"features": [
|
|
7
|
-
{
|
|
8
|
-
"name": "example_temperature_mean_5m",
|
|
9
|
-
"version": "v1"
|
|
10
|
-
}
|
|
11
|
-
],
|
|
12
|
-
"owner": "replace_with_business_owner",
|
|
13
|
-
"description": "Ordered example temperature features."
|
|
14
|
-
}
|
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"schema_version": "ml_data_platform.feature/v1",
|
|
3
|
-
"project": "replace_with_project",
|
|
4
|
-
"name": "example_temperature_mean_5m",
|
|
5
|
-
"version": "v1",
|
|
6
|
-
"inputs": [
|
|
7
|
-
{
|
|
8
|
-
"parameter": "example_temperature",
|
|
9
|
-
"version": "v1",
|
|
10
|
-
"project": "replace_with_project"
|
|
11
|
-
}
|
|
12
|
-
],
|
|
13
|
-
"operator": "example_temperature_features",
|
|
14
|
-
"operator_version": "v1",
|
|
15
|
-
"config": {
|
|
16
|
-
"window": "5min",
|
|
17
|
-
"window_closed": "right",
|
|
18
|
-
"cutoff_included": true,
|
|
19
|
-
"post_cutoff_allowed": false,
|
|
20
|
-
"duplicate_event_time": "last",
|
|
21
|
-
"empty_window": "null"
|
|
22
|
-
},
|
|
23
|
-
"output_column": "example_temperature_mean_5m",
|
|
24
|
-
"output_dtype": "float64",
|
|
25
|
-
"offline_online_supported": false,
|
|
26
|
-
"owner": "replace_with_business_owner",
|
|
27
|
-
"description": "Mean example temperature in the causal interval (target-5min, target]."
|
|
28
|
-
}
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
[build-system]
|
|
2
|
-
requires = ["hatchling"]
|
|
3
|
-
build-backend = "hatchling.build"
|
|
4
|
-
|
|
5
|
-
[project]
|
|
6
|
-
name = "business-feature-operator-template"
|
|
7
|
-
version = "1.0.0"
|
|
8
|
-
requires-python = ">=3.10"
|
|
9
|
-
dependencies = ["pandas>=2.3.3"]
|
|
10
|
-
|
|
11
|
-
[tool.hatch.build.targets.wheel]
|
|
12
|
-
packages = ["src/business_feature_operator_template"]
|
|
@@ -1,39 +0,0 @@
|
|
|
1
|
-
"""Example business Feature Operator. Replace names and formulas before use."""
|
|
2
|
-
|
|
3
|
-
from __future__ import annotations
|
|
4
|
-
|
|
5
|
-
from typing import Any
|
|
6
|
-
|
|
7
|
-
import pandas as pd
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
SUPPORTED_OUTPUTS = {"example_temperature_mean_5m"}
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
def compute_features(context: Any) -> pd.DataFrame:
|
|
14
|
-
requested = list(dict.fromkeys(context.requested_output_columns))
|
|
15
|
-
unknown = sorted(set(requested) - SUPPORTED_OUTPUTS)
|
|
16
|
-
if unknown:
|
|
17
|
-
raise ValueError(f"unsupported output columns: {unknown}")
|
|
18
|
-
|
|
19
|
-
input_keys = {item.parameter: item.key for item in context.inputs}
|
|
20
|
-
parameter_key = input_keys.get("example_temperature")
|
|
21
|
-
if parameter_key is None:
|
|
22
|
-
raise ValueError("example_temperature input is required")
|
|
23
|
-
|
|
24
|
-
frame = context.metric_frames[parameter_key]
|
|
25
|
-
values = pd.to_numeric(
|
|
26
|
-
frame.set_index("timestamp")["value"],
|
|
27
|
-
errors="coerce",
|
|
28
|
-
).sort_index()
|
|
29
|
-
if values.index.has_duplicates:
|
|
30
|
-
values = values.groupby(level=0).last()
|
|
31
|
-
|
|
32
|
-
window = pd.Timedelta(str(context.config["window"]))
|
|
33
|
-
output: dict[str, object] = {"event_time": context.target_times}
|
|
34
|
-
if "example_temperature_mean_5m" in requested:
|
|
35
|
-
output["example_temperature_mean_5m"] = [
|
|
36
|
-
values.loc[(values.index > target - window) & (values.index <= target)].mean()
|
|
37
|
-
for target in context.target_times
|
|
38
|
-
]
|
|
39
|
-
return pd.DataFrame(output)
|