@zhuoyuezs/ml-platform 0.2.7 → 0.2.9

package/runtime/business-client/src/http.js

@@ -1,5 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+const crypto = require("crypto");
 const fs = require("fs");
 const os = require("os");
 const path = require("path");
@@ -16,14 +17,120 @@ function apiErrorDetail(result, fallback) {
         return result.detail;
     return JSON.stringify(result, null, 2);
 }
+// Render the body of a failed binary download with the same rules as JSON calls,
+// so structured FastAPI `detail` objects never collapse to "[object Object]".
+async function responseErrorDetail(response, signal) {
+    let text = "";
+    try {
+        text = String(await (signal ? abortable(response.text(), signal) : response.text()));
+    }
+    catch (_) { /* fall back to status text */ }
+    let parsed;
+    try {
+        parsed = JSON.parse(text);
+    }
+    catch (_) {
+        parsed = undefined;
+    }
+    return apiErrorDetail(parsed, text.trim() || response.statusText || "request failed");
+}
+function safeRelativeArtifactPath(relative) {
+    const normalized = String(relative).replace(/\\/g, "/");
+    const parts = normalized.split("/");
+    if (!normalized || normalized.startsWith("/") || parts.some((part) => part === "" || part === "." || part === "..")) {
+        throw new Error(`artifact path escapes output directory: ${JSON.stringify(String(relative))}`);
+    }
+    return { normalized, parts };
+}
+// Connection-failure policy. A failure is retried only when it happened before
+// any response headers arrived AND either
+//   - the method is idempotent (GET/HEAD): transient connection errors and the
+//     per-attempt timeout are retried with bounded backoff; or
+//   - any method: the TCP connection was never established (connection refused
+//     or connect timeout), which proves no request bytes reached the server.
+// Every other failure of a non-idempotent request (for example a reset after
+// the request was written) is reported once, because the server may already
+// have applied it.
+const IDEMPOTENT_METHODS = new Set(["GET", "HEAD"]);
+const UNSENT_CONNECT_CODES = new Set(["ECONNREFUSED", "UND_ERR_CONNECT_TIMEOUT"]);
+const TRANSIENT_CONNECTION_CODES = new Set([
+    ...UNSENT_CONNECT_CODES, "UND_ERR_SOCKET", "UND_ERR_CLOSED", "ECONNRESET", "EPIPE", "ETIMEDOUT", "ECONNABORTED",
+    "EHOSTUNREACH", "ENETUNREACH", "ENETDOWN", "EAI_AGAIN",
+]);
+const DEFAULT_RETRY_DELAYS_MS = [500, 1000, 2000];
+// Error codes along the `cause` chain (undici wraps socket errors as
+// `TypeError: fetch failed` with the real error in `cause`; happy-eyeballs
+// connects may report an AggregateError with per-address errors).
+function errorCodes(error, seen = new Set()) {
+    if (!error || typeof error !== "object" || seen.has(error))
+        return [];
+    seen.add(error);
+    const own = typeof error.code === "string" ? [error.code] : [];
+    const nested = Array.isArray(error.errors) ? error.errors.flatMap((item) => errorCodes(item, seen)) : [];
+    return [...own, ...nested, ...errorCodes(error.cause, seen)];
+}
+function describeError(error) {
+    if (!(error instanceof Error))
+        return String(error);
+    const details = [];
+    const seen = new Set([error]);
+    let cause = error.cause;
+    while (cause && typeof cause === "object" && !seen.has(cause)) {
+        seen.add(cause);
+        const message = cause.message || (Array.isArray(cause.errors) ? cause.errors.map((item) => item?.message).filter(Boolean).join("; ") : "");
+        details.push([cause.code, message].filter(Boolean).join(": "));
+        cause = cause.cause;
+    }
+    const suffix = details.filter(Boolean);
+    return suffix.length ? `${error.message} (${suffix.join(" <- ")})` : error.message;
+}
+function retryableFailure(method, error, timedOut) {
+    if (IDEMPOTENT_METHODS.has(method))
+        return timedOut || errorCodes(error).some((code) => TRANSIENT_CONNECTION_CODES.has(code));
+    if (timedOut)
+        return false;
+    const codes = errorCodes(error);
+    return codes.length > 0 && codes.every((code) => UNSENT_CONNECT_CODES.has(code));
+}
 class PlatformApiClient {
-    constructor(baseUrl, timeoutSeconds = 30) {
+    constructor(baseUrl, timeoutSeconds = 30, { retryDelaysMs = DEFAULT_RETRY_DELAYS_MS } = {}) {
         this.baseUrl = String(baseUrl).trim().replace(/\/+$/, "");
         if (!/^https?:\/\//.test(this.baseUrl))
             throw new Error("server profile api_url must start with http:// or https://");
         if (!(timeoutSeconds > 0))
             throw new Error("request timeout must be positive");
         this.timeoutMs = timeoutSeconds * 1000;
+        this.retryDelaysMs = [...retryDelaysMs];
+    }
+    // Send one request and return once response headers arrive. The caller owns
+    // the returned per-attempt timeout and must call `done()` after reading the
+    // body. Each request asks for `Connection: close`, so no pooled keep-alive
+    // socket is ever reused after the server (uvicorn closes idle sockets after
+    // 5 s) or a VPN/NAT hop has dropped it.
+    async open(method, url, { headers = {}, body } = {}) {
+        for (let attempt = 0;; attempt += 1) {
+            const controller = new AbortController();
+            let timedOut = false;
+            const timer = setTimeout(() => { timedOut = true; controller.abort(); }, this.timeoutMs);
+            try {
+                const response = await fetch(url, { method, headers: { ...headers, Connection: "close" }, body, signal: controller.signal });
+                return { response, signal: controller.signal, done: () => clearTimeout(timer) };
+            }
+            catch (error) {
+                clearTimeout(timer);
+                const detail = timedOut ? `request timed out after ${this.timeoutMs / 1000}s (${describeError(error)})` : describeError(error);
+                const delay = this.retryDelaysMs[attempt];
+                if (delay !== undefined && retryableFailure(method, error, timedOut)) {
+                    process.stderr.write(`platform API ${method} ${url}: ${detail}; retrying in ${delay}ms (attempt ${attempt + 2}/${this.retryDelaysMs.length + 1})\n`);
+                    await new Promise((resolve) => setTimeout(resolve, delay));
+                    continue;
+                }
+                const attempts = attempt ? ` after ${attempt + 1} attempts` : "";
+                const ambiguous = !IDEMPOTENT_METHODS.has(method) && !retryableFailure(method, error, timedOut)
+                    ? "; not retried because the server may have received it; inspect platform state before resubmitting" : "";
+                throw new Error(`platform API request failed: ${method} ${url}: ${detail}${attempts}${ambiguous}`);
+            }
+        }
     }
     url(endpoint, params = {}) {
         const url = new URL(endpoint.replace(/^\//, ""), `${this.baseUrl}/`);
@@ -34,8 +141,6 @@ class PlatformApiClient {
         return url;
     }
     async request(method, endpoint, { params, payload, body, contentType, requestHeaders } = {}) {
-        const controller = new AbortController();
-        const timer = setTimeout(() => controller.abort(), this.timeoutMs);
         const headers = { Accept: "application/json" };
         Object.assign(headers, requestHeaders || {});
         let requestBody = body;
@@ -45,17 +150,17 @@ class PlatformApiClient {
         }
         else if (contentType)
             headers["Content-Type"] = contentType;
-        let response;
+        const { response, done } = await this.open(method, this.url(endpoint, params), { headers, body: requestBody });
+        let text;
         try {
-            response = await fetch(this.url(endpoint, params), { method, headers, body: requestBody, signal: controller.signal });
+            text = await response.text();
         }
         catch (error) {
-            throw new Error(`platform API request failed: ${method} ${this.url(endpoint, params)}: ${error.message}`);
+            throw new Error(`platform API request failed: ${method} ${this.url(endpoint, params)}: reading response: ${describeError(error)}`);
         }
         finally {
-            clearTimeout(timer);
+            done();
         }
-        const text = await response.text();
         let result;
         try {
             result = JSON.parse(text);
@@ -115,20 +220,15 @@ class PlatformApiClient {
             fs.rmSync(target, { recursive: true, force: true });
         }
         fs.mkdirSync(target, { recursive: true });
-        const controller = new AbortController();
-        const timer = setTimeout(() => controller.abort(), this.timeoutMs);
         const temporaryDir = fs.mkdtempSync(path.join(os.tmpdir(), "data-platform-artifact-download-"));
         const temporary = path.join(temporaryDir, "artifact.zip");
-        let response;
+        let done = () => { };
         try {
-            response = await fetch(this.url(`/dataset-artifacts/${encodeURIComponent(project)}/${encodeURIComponent(datasetId)}/${encodeURIComponent(manifestHash)}/download`), { headers: { Accept: "application/zip" }, signal: controller.signal });
+            const opened = await this.open("GET", this.url(`/dataset-artifacts/${encodeURIComponent(project)}/${encodeURIComponent(datasetId)}/${encodeURIComponent(manifestHash)}/download`), { headers: { Accept: "application/zip" } });
+            const { response, signal } = opened;
+            done = opened.done;
             if (!response.ok) {
-                let detail = response.statusText;
-                try {
-                    detail = JSON.parse(await abortable(response.text(), controller.signal))?.detail ?? detail;
-                }
-                catch (_) { /* use status text */ }
-                throw new Error(`artifact download failed: HTTP ${response.status}: ${detail}`);
+                throw new Error(`artifact download failed: HTTP ${response.status}: ${await responseErrorDetail(response, signal)}`);
             }
             const total = contentLength(response);
             let lastReported = -1;
@@ -139,7 +239,7 @@ class PlatformApiClient {
                 }
             };
             report(0);
-            await writeResponseBody(response, temporary, controller.signal, report);
+            await writeResponseBody(response, temporary, signal, report);
             const files = await extractArchive(temporary, target);
             return { dataset_id: datasetId, manifest_hash: manifestHash, output_dir: target, file_count: files.length, files };
         }
@@ -147,10 +247,10 @@ class PlatformApiClient {
             fs.rmSync(target, { recursive: true, force: true });
             if (error instanceof Error && error.message.startsWith("artifact download failed:"))
                 throw error;
-            throw new Error(`artifact download failed: ${error.message}`);
+            throw new Error(`artifact download failed: ${describeError(error)}`);
         }
         finally {
-            clearTimeout(timer);
+            done();
             fs.rmSync(temporaryDir, { recursive: true, force: true });
         }
     }
@@ -183,22 +283,15 @@ class PlatformApiClient {
                 }
                 fs.mkdirSync(path.dirname(destination), { recursive: true });
                 const endpoint = `/dataset-artifacts/${encodeURIComponent(project)}/${encodeURIComponent(datasetId)}/${encodeURIComponent(manifestHash)}/files/${parts.map(encodeURIComponent).join("/")}`;
-                const controller = new AbortController();
-                const timer = setTimeout(() => controller.abort(), this.timeoutMs);
+                const { response, signal, done } = await this.open("GET", this.url(endpoint), { headers: { Accept: "application/octet-stream" } });
                 try {
-                    const response = await fetch(this.url(endpoint), { headers: { Accept: "application/octet-stream" }, signal: controller.signal });
                     if (!response.ok) {
-                        let detail = response.statusText;
-                        try {
-                            detail = JSON.parse(await response.text())?.detail ?? detail;
-                        }
-                        catch (_) { /* status text */ }
-                        throw new Error(`artifact file download failed: HTTP ${response.status}: ${detail}`);
+                        throw new Error(`artifact file download failed: HTTP ${response.status}: ${await responseErrorDetail(response, signal)}`);
                     }
                     fs.writeFileSync(destination, Buffer.from(await response.arrayBuffer()), { flag: "wx" });
                 }
                 finally {
-                    clearTimeout(timer);
+                    done();
                 }
             }
         }
@@ -208,6 +301,71 @@ class PlatformApiClient {
         }
         return { project, dataset_id: datasetId, manifest_hash: manifestHash, output_dir: targetRoot, file_count: files.length, files: files.sort() };
     }
+    // Download indexed ModelArtifact/EvaluationResult files one by one and verify
+    // each against the sha256 recorded in the registry index before keeping it.
+    async downloadLifecycleArtifactFiles(collection, resourceId, index, relativePaths, outputDir, { force = false, project = undefined } = {}) {
+        if (!["model-artifacts", "evaluation-results"].includes(collection))
+            throw new Error(`unsupported artifact collection: ${collection}`);
+        if (!index || typeof index !== "object" || Array.isArray(index))
+            throw new Error(`${collection} record has no file index`);
+        const available = Object.keys(index).sort();
+        const requested = relativePaths && relativePaths.length ? [...new Set(relativePaths.map((value) => String(value)))] : available;
+        if (!requested.length)
+            throw new Error(`${collection} record indexes no files: ${resourceId}`);
+        const plan = requested.map((relative) => safeRelativeArtifactPath(relative));
+        const missing = plan.filter((item) => !Object.prototype.hasOwnProperty.call(index, item.normalized)).map((item) => item.normalized);
+        if (missing.length)
+            throw new Error(`file is not indexed by ${collection}/${resourceId}: ${missing.join(", ")}; available: ${available.join(", ")}`);
+        const targetRoot = path.resolve(expandHome(outputDir));
+        if (fs.existsSync(targetRoot) && fs.readdirSync(targetRoot).length) {
+            if (!force)
+                throw new Error(`download output directory is not empty: ${targetRoot}`);
+            fs.rmSync(targetRoot, { recursive: true, force: true });
+        }
+        fs.mkdirSync(targetRoot, { recursive: true });
+        const downloaded = [];
+        try {
+            for (const { normalized, parts } of plan) {
+                const destination = path.resolve(targetRoot, ...parts);
+                if (!destination.startsWith(`${targetRoot}${path.sep}`))
+                    throw new Error(`artifact path escapes output directory: ${JSON.stringify(normalized)}`);
+                fs.mkdirSync(path.dirname(destination), { recursive: true });
+                const endpoint = `/${collection}/${encodeURIComponent(resourceId)}/files/${parts.map(encodeURIComponent).join("/")}`;
+                const partial = `${destination}.partial`;
+                let done = () => { };
+                try {
+                    const opened = await this.open("GET", this.url(endpoint, { project }), { headers: { Accept: "application/octet-stream" } });
+                    const { response, signal } = opened;
+                    done = opened.done;
+                    if (!response.ok)
+                        throw new Error(`artifact file download failed: ${normalized}: HTTP ${response.status}: ${await responseErrorDetail(response, signal)}`);
+                    const hash = crypto.createHash("sha256");
+                    let size = 0;
+                    await writeResponseBody(response, partial, signal, () => { }, (chunk) => { hash.update(chunk); size += chunk.length; });
+                    const actual = `sha256:${hash.digest("hex")}`;
+                    const expected = String(index[normalized]);
+                    const served = response.headers?.get?.("x-artifact-sha256");
+                    if (actual !== expected || (served && served !== expected)) {
+                        throw new Error(`artifact file checksum mismatch: ${normalized}: registry ${expected}, downloaded ${actual}${served && served !== expected ? `, server header ${served}` : ""}`);
+                    }
+                    fs.renameSync(partial, destination);
+                    downloaded.push({ name: normalized, sha256: actual, size_bytes: size });
+                }
+                finally {
+                    done();
+                    fs.rmSync(partial, { force: true });
+                }
+            }
+        }
+        catch (error) {
+            fs.rmSync(targetRoot, { recursive: true, force: true });
+            if (error instanceof Error && /^artifact (file|path)/.test(error.message))
+                throw error;
+            throw new Error(`artifact file download failed: ${error instanceof Error ? error.message : String(error)}`);
+        }
+        downloaded.sort((left, right) => left.name.localeCompare(right.name));
+        return { output_dir: targetRoot, file_count: downloaded.length, verified: true, files: downloaded };
+    }
 }
 function contentLength(response) {
     const raw = response.headers?.get?.("content-length") ?? response.headers?.["content-length"];
@@ -228,13 +386,15 @@ function abortable(value, signal) {
         Promise.resolve(value).then((result) => { cleanup(); resolve(result); }, (error) => { cleanup(); reject(error); });
     });
 }
-async function writeResponseBody(response, outputPath, signal, onProgress) {
+async function writeResponseBody(response, outputPath, signal, onProgress, onChunk = undefined) {
     const handle = fs.openSync(outputPath, "w");
     let received = 0;
     try {
         const write = (value) => {
             const chunk = Buffer.from(value);
             fs.writeSync(handle, chunk);
+            if (onChunk)
+                onChunk(chunk);
             received += chunk.length;
             onProgress(received);
         };

package/skills/feature-management/SKILL.md

@@ -82,8 +82,9 @@ Use the global `--profile server` form shown above, before the business
 command. Do not append `--profile` after a business command. Before discovery,
 check the installed release with `ml-platform --help` and the relevant
 subcommand's `--help`; CLI releases may differ in supported flags. In the
-deployed 0.6.x CLI, registry `list-*` commands emit JSON by default and do not
-accept a `--json` flag.
+current CLI release line (`@zhuoyuezs/ml-platform` 0.8.x; confirm with
+`ml-platform version`), registry `list-*` commands emit JSON by default and
+reject an unknown `--json` flag.
 
 If the installed release rejects the global `--profile server` form shown above,
 record the CLI syntax error and inspect `show-config`/subcommand help before
@@ -116,7 +117,10 @@ Confirm:
 - dataset mode, read policy, time range, grid, prediction horizon/cutoff contract, and explicit Parameter outputs;
 - named training/validation/test rowsets, abnormal windows, endpoint-policy scope,
   and the model context lookback used to decide endpoint eligibility;
-- realtime freshness and missing-input policy when serving inference data;
+- realtime freshness and missing-input policy when serving inference data,
+  chosen per model and per Parameter from the data owner's answers on
+  timestamp meaning, arrival schedule and ingestion-time columns (see the
+  realtime policy questions in [references/contracts.md](references/contracts.md));
 - target profile/API and whether the user authorizes publish and/or build actions.
 
 Separate preparation from mutation. Creating files and running local tests or `apply --dry-run` does not authorize publishing a wheel, changing the Registry, or submitting a build Job.
@@ -380,9 +384,15 @@ for the cutoff formula, half-open overlap boundaries, scope behavior, and
 
 For realtime inference, declare `realtime_fetch` only when its effective policy
 must be part of the contract; leaving it unset preserves existing manifest
-hashes. Keep `allow_missing=false` unless the user explicitly accepts missing
-required inputs. Read [references/contracts.md](references/contracts.md) for the
-freshness, tail-edge, gap-fill, and rowset contracts.
+hashes. The environment defaults are strict and suit short-horizon
+forecasting; for a new model whose Features are served online, choose the
+policy deliberately and declare every field it relies on. Keep
+`allow_missing=false` unless the user explicitly accepts missing required
+inputs. Read [references/contracts.md](references/contracts.md) for the
+freshness, tail-edge, gap-fill, and rowset contracts and the policy method.
+An Operator served online, especially one that aggregates a whole period,
+must follow the Online Period Aggregates method in
+[references/operator-authoring.md](references/operator-authoring.md).
 
 ## Validate Before Publishing
 
@@ -405,7 +415,7 @@ Stop on any error. Do not weaken schema validation, fabricate a missing dependen
 
 Publish only after the user explicitly approves Registry and wheel changes. Use catalog `apply` so publication follows Parameter -> Operator -> Feature -> FeatureSet -> Dataset order.
 
-After publication, resolve every new DatasetManifest and compare its exact Parameter, Operator, Feature, and FeatureSet versions with the pre-publication mapping. Stop if an affected old key or any unexpected version remains. Resolve the manifest before building. Submit a build only when requested. Before choosing partition controls, use the build command's read-only `--plan` capability described in [commands.md](references/commands.md), inspect its estimates and warnings, and prefer platform automatic sizing over a copied daily example. Explicit overrides need source/resource evidence; preserve the submitted Job's concrete plan when resuming. For the current partitioned materialization runtime, use `--max-parallelism 1` unless a separately verified platform capability says otherwise; do not retry or resubmit a failed Job until its error and already-completed partitions are understood. For server builds, wait for the terminal Job state and download the exact artifact by `dataset_id + manifest_hash`.
+After publication, resolve every new DatasetManifest and compare its exact Parameter, Operator, Feature, and FeatureSet versions with the pre-publication mapping. Stop if an affected old key or any unexpected version remains. Resolve the manifest before building. Submit a build only when requested. Before choosing partition controls, use the build command's read-only `--plan` capability described in [commands.md](references/commands.md), inspect its estimates and warnings, and prefer platform automatic sizing over a copied daily example. Explicit overrides need source/resource evidence; preserve the submitted Job's concrete plan when resuming. For the current partitioned materialization runtime, use `--max-parallelism 1` unless a separately verified platform capability says otherwise. If any selected Operator needs the complete requested window (for example full-window interpolation or smoothing), build with `--partition-duration none --max-parallelism 1` and confirm an unpartitioned `--plan` first, because the CLI default is `auto`; do not retry or resubmit a failed Job until its error and already-completed partitions are understood. For server builds, wait for the terminal Job state and download the exact artifact by `dataset_id + manifest_hash`.
 
 Do not delete versioned Registry assets, cancel Jobs, rebuild images, modify Kubernetes, or change service configuration as part of this workflow unless the user separately and explicitly requests that action. An empty Project may be soft-deleted only on an explicit request; rely on the server to reject deletion when resources still exist.
 

package/skills/feature-management/references/commands.md

@@ -51,8 +51,9 @@ ml-platform --profile server list-datasets -q <stable-dataset-id>
 ```
 
 Run each command separately. Check `ml-platform --help` and the specific
-subcommand help before adding flags. In the deployed 0.6.x CLI, `list-*`
-commands print JSON by default and do not accept a `--json` flag.
+subcommand help before adding flags. In the current CLI release line
+(`@zhuoyuezs/ml-platform` 0.8.x; confirm with `ml-platform version`), `list-*`
+commands print JSON by default and reject an unknown `--json` flag.
 If a deployed release rejects the global `--profile server` prefix, treat that
 as CLI compatibility drift: record the syntax error, inspect `show-config` and
 the subcommand help, then retry the same read-only command without the rejected
@@ -107,8 +108,8 @@ empty project. Project commands are server-only and do not modify local storage.
 - `apply --project <project>` scopes the comparison snapshot the catalog is
   validated against (omit to compare against `default`);
 - `list-* --project <project>` filters discovery to that project;
-- `get-dataset-artifact` / `download-dataset-artifact` take the project as their
-  first positional argument (defaults to `default`).
+- `get-dataset-artifact` / `download-dataset-artifact` take `--project <project>`
+  (defaults to `default`).
 
 Before choosing names, discover within the target project: identical names in a
 different project are unrelated and must not be reused across projects to mean the
@@ -247,6 +248,25 @@ sizing is not a resource guarantee. Final merge memory is not bounded by a
 partition's estimated rows. A long prediction horizon is not itself a reason to
 use daily partitions: inspect the reported source read amplification.
 
+Partitioned execution gives each partition only its declared
+`history_requirements` lookback and label overlap. An Operator whose result for
+one row depends on the complete requested window (for example full-window
+interpolation, smoothing, normalization or ranking) is not partition-invariant,
+and no Operator field tells the planner so. Build such a manifest with
+`--partition-duration none --max-parallelism 1`, because both build commands
+default to `auto`. First run the same command with `--plan` and confirm
+`execution_spec.partition_duration` is `null`, `estimated_partition_count` is
+`1`, and the reasons report unpartitioned execution; then submit with the same
+flags. Prefer reformulating the Operator as a causal, bounded-lookback
+calculation when the business rule allows it.
+
+```bash
+ml-platform --profile server \
+  build-registered-dataset <dataset_id> <dataset_version> \
+  --project <project> \
+  --partition-duration none --max-parallelism 1 --plan
+```
+
 `--partition-duration` is an execution control and does not change the
 DatasetManifest hash. The current implementation is sequential, so keep
 `--max-parallelism 1`. Completed partitions are checkpointed and resumed by
@@ -255,8 +275,12 @@ resolved partition duration. Changing it requires a new execution, rather than
 reusing old checkpoint boundaries. Changing partition size does not authorize
 cancelling or resubmitting an already-running Job.
 
-Record the returned `job_id`, `dataset_id`, and resolved `manifest_hash`; for a registered Dataset
-also retain `registered_manifest_hash` for FeatureRetrievalSpec binding. If a separate wait is needed:
+Record the returned `job_id`, `dataset_id`, and the artifact manifest hash; for a registered Dataset
+also retain `registered_manifest_hash` for FeatureRetrievalSpec binding. A succeeded
+`--wait` or `wait-job` result carries `artifact.manifest_hash`, the DatasetArtifact key used by
+`get-dataset-artifact`, `download-dataset-artifact` and training/evaluation dataset bindings. The
+Job's top-level `manifest_hash` identifies the submitted manifest; it is not an artifact key and
+returns `artifact_not_found` there. If a separate wait is needed:
 
 ```bash
 ml-platform --profile server wait-job <job_id>
@@ -294,6 +318,23 @@ write parquet, or publish a DatasetArtifact. Treat HTTP `503` as retryable sourc
 freshness/missing-data failure and HTTP `400` as a contract error that requires a
 manifest or request change.
 
+Before choosing `realtime_fetch` lag budgets or a schedule time, observe how far
+each Parameter's source has arrived:
+
+```bash
+ml-platform --profile server get-parameter-freshness <name> <version> \
+  --project <project> [--lookback 7d] [--end 2026-10-28T00:30:00+08:00] [--online]
+```
+
+It reads `[end - lookback, end]` (end inclusive, `lookback` at most `31d`) through
+the Parameter's source adapter without cache, build, or artifact, and returns
+`latest_event_time`, `lag_hours` (the value a realtime read compares with its lag
+budget at that cutoff), `latest_value_event_time`, `missing_trailing_points`,
+`available_at_watermark`/`ingested_at_watermark` when the Parameter declares
+availability, and `source_field_checks`. `--online` reads `online_source` when
+declared. Run it at the intended schedule time on several days; one reading does
+not establish a publication pattern.
+
 For a deployment-bound row or full sequence context, prepare one JSON object
 containing `feature_retrieval`, `cutoff_time`, and `entity_keys`, then run:
 
@@ -359,7 +400,9 @@ ml-platform --profile server \
 ```
 
 `--project` defaults to `default`; pass the owning project when the dataset lives
-elsewhere, or the lookup returns `artifact_not_found`.
+elsewhere, or the lookup returns `artifact_not_found`. `<manifest_hash>` is the
+artifact manifest hash (`list-dataset-artifacts`, or `artifact.manifest_hash`
+from a succeeded build wait), not the registered manifest hash.
 
 The download command writes progress to stderr after response headers arrive.
 Interpret a timeout with no progress line as a request/header or first-byte

package/skills/feature-management/references/contracts.md

@@ -139,6 +139,17 @@ supported field.
 当前严格 `as_of` 只接受 `exact` 或有合同支撑的 `upper_bound`；修改这些语义需要发布
 新的 Parameter 业务版本。
 
+`time_semantics.ingested_at_field` 和 `event_time_field` 只是标签：读取时不会查询它们，
+除非 `availability.field` 指向同一列。没有 `availability` 时，读取既没有
+`ingested_at`/`available_at` 证据，严格可见性读取（`as_of` 回放、
+`availability_boundary=cutoff_time`）也会拒绝该 Parameter。只把源表真实存在的入库列写进
+`availability`；源表没有入库时间时，保持 `availability` 为空或使用经数据 owner 确认的
+`fixed_delay`，不要声明不存在的列。发布新 Parameter 时，若 `source.connection_id` 是平台
+托管连接，平台会检查实际查询的列（`time_column`、`value_column`、metric/unit 列、
+`filters` 键、`availability.field`、`availability.revision_id_field`）：缺列直接拒绝
+（`source_fields_missing`），不存在的 `ingested_at_field`/`event_time_field` 只作为警告
+返回在 `source_field_checks` 中。已发布版本可用 `get-parameter-freshness` 查看同一检查。
+
 未来预报、排程、日历等 known-future 数据允许 `available_at < event_time`。这不是时钟错误：
 `event_time` 是未来目标时刻，`available_at` 是该修订何时已可读取。模型
 `feature_lookup` 会读取 cutoff 后的未来事件窗口，但仍严格过滤
@@ -380,8 +391,14 @@ Current schema rules:
 - Use `parameters` only to request explicit Parameter output columns.
 - Declare one dataset-wide `prediction` contract for forecast Features. `horizon`
   accepts non-negative whole-minute durations such as `0min`, `10min`, or `2h`.
-  The platform always derives `cutoff_times = target_times - horizon` and records
-  that derivation in resolved lineage; it is not a user-configurable policy.
+  For Operator inputs, each materialized row's `event_time` is its causal
+  feature cutoff (`context.cutoff_times` equals `context.target_times`), and the
+  horizon sets `label_time = event_time + horizon` without moving those reads.
+  `endpoint_policy` eligibility is separate and uses the `cutoff = t - h` rule
+  documented in its section. The platform records this in resolved
+  lineage (`prediction_time_semantics: event_time_is_causal_cutoff`,
+  `label_time_derivation: prediction_time_plus_horizon`); it is not a
+  user-configurable policy.
 
 ### Optional sections
 
@@ -442,11 +459,14 @@ attempts, rows, and elapsed time land in
 ```json
 "realtime_fetch": {
   "max_workers": 4,
+  "deadline_seconds": 30.0,
+  "chunk_days": null,
   "min_lookback": "1h",
   "validate_freshness": true,
   "max_source_lag_hours": 3.0,
   "max_source_lag_hours_by_parameter": {},
   "allow_missing": false,
+  "allow_missing_by_parameter": {},
   "allow_tail_edge_missing": true,
   "max_tail_edge_missing": 1,
   "tail_edge_missing_max_age_minutes": 65.0,
@@ -467,11 +487,109 @@ offline `source` instead.
 
 Keep the default strict: `allow_missing=false`. Tail-edge and bounded internal
 gap-fill tolerance may return a row, but the response must mark
-`freshness.degraded=true` and include validation evidence. A freshness or
+`freshness.degraded=true` and include validation evidence.
+
+Per-Parameter keys in `max_source_lag_hours_by_parameter` and
+`allow_missing_by_parameter` are full Parameter keys (`project/name:version`);
+other spellings are ignored with a validation warning. `allow_missing_by_parameter`
+overrides `allow_missing` for one Parameter in either direction and is omitted
+from the manifest while empty, so adding the field does not change existing
+hashes. A tolerated Parameter still marks the read degraded, and a deployment
+with `freshness_policy=strict` rejects degraded reads.
+
+The tail-edge and internal-gap-fill allowances are absolute lag ceilings measured
+from the cutoff, not additions to the lag budget: a source past its budget is
+tolerated when its lag is at most `max(tail_edge_missing_max_age_minutes,
+max_tail_edge_missing * expected_frequency)` or at most
+`internal_gap_fill_max_age_hours`. With the defaults (5h gap-fill ceiling) a
+Parameter budget below 5h therefore degrades rather than fails; set
+`allow_internal_gap_fill=false` (per manifest) when a budget must be hard.
+Neither allowance changes values: realtime rows use the same alignment and
+Operator code as a batch build, so these settings cannot create
+training/serving skew. Choose budgets from observed arrival with
+`get-parameter-freshness`. A freshness or
 missing-data condition that may recover returns HTTP `503`; an invalid contract
 returns HTTP `400`. Declaring `realtime_fetch` changes the manifest hash; leaving
 it unset preserves older contract hashes and uses environment fallbacks.
 
+`deadline_seconds` bounds the whole read. `chunk_days` pages long source reads
+and falls back to `source_read.chunk_days` when unset.
+
+#### Choosing a realtime fetch policy
+
+The values above are also the environment defaults (`DATA_PLATFORM_REALTIME_*`).
+They are strict and suit short-horizon forecasting on minute-level data. They
+are a process fallback, not a decision for a particular model. Choose the
+policy for each model from its data and its use, and per Parameter where
+sources differ.
+
+How the checks work:
+
+- For each Parameter, the platform takes the newest source row at or before
+  the cutoff (the watermark) and computes `lag = cutoff - watermark`.
+- The read is normal when the lag is within `max_source_lag_hours`, or within
+  the Parameter's entry in `max_source_lag_hours_by_parameter` (keyed
+  `project/name:version`).
+- A larger lag is still accepted, marked `freshness.degraded=true`, when
+  `allow_tail_edge_missing` is on and the lag is within the larger of
+  `tail_edge_missing_max_age_minutes` and `max_tail_edge_missing` times the
+  Parameter's `expected_frequency`, or when `allow_internal_gap_fill` is on and
+  the lag is within `internal_gap_fill_max_age_hours`. With the defaults, a
+  source up to 5 hours behind is accepted as degraded. Set
+  `allow_internal_gap_fill: false` when the lag budget must be a hard limit.
+- A Parameter with no rows in its window fails unless `allow_missing` is true.
+  `allow_missing` applies to every Parameter and also accepts any stale
+  source; there is no per-Parameter switch. Do not turn it on to tolerate one
+  optional input.
+- None of these settings changes values. Holes inside the lookback reach the
+  Operator unchanged, so the Operator's frozen missing-data rule
+  ([operator-authoring.md](operator-authoring.md), Online Period Aggregates)
+  is what keeps batch builds and realtime reads equal.
+
+Decide with the user:
+
+- Whether a late or missing source should fail visibly (HTTP `503`, rerun after
+  the data recovers) or return a degraded row. A long aggregate (for example a
+  daily mean) may barely change when one point is late, but a period that is
+  missing its last hours because a source stalled usually should fail.
+- When the read runs relative to each source's arrival. The lag budget must
+  cover the latest normal arrival: schedule the read after a batch-loaded
+  source is complete, or give that Parameter a larger budget.
+- Whether the read finishes within `deadline_seconds`. A long lookback over
+  several Parameters may need more than 30 seconds; measure it with
+  `fetch-inference-data` at a historical cutoff before choosing.
+
+Record the decision in the inference DatasetManifest. Declare every field the
+model relies on, even when it equals the current default: resolution is per
+field, and an omitted field takes the process environment value, which can
+differ between deployments. Note the reason for each value in the catalog
+review. Changing the policy needs a new inference manifest and package
+version, but not retraining. In each response, `policy.resolved_from` shows
+whether a field came from `manifest` or `environment`; check that nothing you
+meant to declare came from `environment`.
+
+The `fetch-inference-data` options `--max-source-lag-hours`,
+`--no-validate-freshness` and `--allow-missing` replace the manifest's whole
+`realtime_fetch` section for that request; fields you do not pass fall back to
+the environment, not to the manifest. Use them for diagnosis only.
+
+Ask the data owner, and do not guess:
+
+1. What each source timestamp means: interval start, interval end, or the
+   measurement instant, and in which timezone. This decides period boundaries
+   and the expected lag (an hourly record stamped at the hour start has a
+   watermark one hour before a midnight cutoff even when the day is complete).
+2. How data arrives: streaming or batch loads, when the previous period is
+   complete on a normal day, and how late it can be.
+3. Whether late corrections or backfills are written after first arrival;
+   these make a rerun at the same cutoff return a different row.
+4. Whether the table has a trustworthy ingestion-time column. Confirm the
+   column exists (for example with the `data-source-exploration` Skill) before
+   declaring `time_semantics.ingested_at_field` or an availability strategy
+   that reads it; never copy that field from a template.
+5. Which inputs may legitimately be absent (for example one faulty sensor) and
+   which must never be (for example a state indicator used by a business rule).
+
 `abnormal_windows` excludes target windows whose data must not be trusted. The
 policy and the lookback apply to the whole window set, not to individual
 windows: a dataset is either a filtered rowset or a marked full rowset.
@@ -494,9 +612,9 @@ windows: a dataset is either a filtered rowset or a marked full rowset.
 | `mark_only` | Drop nothing; record what is affected in lineage and validation. |
 
 How far past a window to drop is derived from the widest resolved Operator history
-requirement. Cutoff-relative history contributes `prediction.horizon + lookback`;
-target-time-relative history contributes its declared lookback directly. The
-maximum is used for source fetch and abnormal-window propagation. Do not restate
+requirement: the larger of the cutoff-anchored and target-time-anchored
+lookbacks (the horizon is not added, because the Operator cutoff is the row's
+`event_time`). The maximum is used for source fetch and abnormal-window propagation. Do not restate
 that number unless you want it asserted: an optional `input_lookback_hours` is
 checked against the derived value and a mismatch is rejected. Note that a window
 near the end of the target range can have its lookback clipped by the range end,