modelrig 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +94 -0
- package/README.md +41 -11
- package/data/PROVENANCE.md +19 -0
- package/data/pricing-manual.json +22 -0
- package/data/pricing-map.json +46362 -0
- package/data/pricing-overlay.json +419 -0
- package/dist/adapters/anthropic.d.ts +50 -6
- package/dist/adapters/anthropic.d.ts.map +1 -1
- package/dist/adapters/anthropic.js +123 -29
- package/dist/adapters/build-adapters.d.ts +11 -0
- package/dist/adapters/build-adapters.d.ts.map +1 -0
- package/dist/adapters/build-adapters.js +45 -0
- package/dist/adapters/deepseek.d.ts +6 -0
- package/dist/adapters/deepseek.d.ts.map +1 -1
- package/dist/adapters/deepseek.js +54 -1
- package/dist/adapters/gemini.d.ts.map +1 -1
- package/dist/adapters/gemini.js +46 -6
- package/dist/adapters/grok.d.ts +9 -0
- package/dist/adapters/grok.d.ts.map +1 -1
- package/dist/adapters/grok.js +67 -3
- package/dist/adapters/openai-compat.d.ts +6 -0
- package/dist/adapters/openai-compat.d.ts.map +1 -1
- package/dist/adapters/openai-compat.js +58 -1
- package/dist/adapters/openai-stream.d.ts +58 -0
- package/dist/adapters/openai-stream.d.ts.map +1 -0
- package/dist/adapters/openai-stream.js +120 -0
- package/dist/adapters/openai.d.ts +6 -0
- package/dist/adapters/openai.d.ts.map +1 -1
- package/dist/adapters/openai.js +60 -1
- package/dist/adapters/sampling.d.ts +28 -0
- package/dist/adapters/sampling.d.ts.map +1 -0
- package/dist/adapters/sampling.js +34 -0
- package/dist/adapters/tool-mapping.d.ts +92 -0
- package/dist/adapters/tool-mapping.d.ts.map +1 -0
- package/dist/adapters/tool-mapping.js +143 -0
- package/dist/artifacts/artifact.d.ts +137 -0
- package/dist/artifacts/artifact.d.ts.map +1 -0
- package/dist/artifacts/artifact.js +377 -0
- package/dist/artifacts/content-uploader.d.ts +87 -0
- package/dist/artifacts/content-uploader.d.ts.map +1 -0
- package/dist/artifacts/content-uploader.js +124 -0
- package/dist/artifacts/index.d.ts +63 -0
- package/dist/artifacts/index.d.ts.map +1 -0
- package/dist/artifacts/index.js +80 -0
- package/dist/artifacts/run-context.d.ts +129 -0
- package/dist/artifacts/run-context.d.ts.map +1 -0
- package/dist/artifacts/run-context.js +208 -0
- package/dist/artifacts/scrub.d.ts +60 -0
- package/dist/artifacts/scrub.d.ts.map +1 -0
- package/dist/artifacts/scrub.js +139 -0
- package/dist/artifacts/step.d.ts +26 -0
- package/dist/artifacts/step.d.ts.map +1 -0
- package/dist/artifacts/step.js +37 -0
- package/dist/artifacts/taint.d.ts +63 -0
- package/dist/artifacts/taint.d.ts.map +1 -0
- package/dist/artifacts/taint.js +76 -0
- package/dist/bakeoff/cache-sim.d.ts +99 -0
- package/dist/bakeoff/cache-sim.d.ts.map +1 -0
- package/dist/bakeoff/cache-sim.js +126 -0
- package/dist/bakeoff/eval-cases.d.ts +96 -0
- package/dist/bakeoff/eval-cases.d.ts.map +1 -0
- package/dist/bakeoff/eval-cases.js +105 -0
- package/dist/bakeoff/report.d.ts +41 -0
- package/dist/bakeoff/report.d.ts.map +1 -0
- package/dist/bakeoff/report.js +142 -0
- package/dist/bakeoff/samples.d.ts +15 -0
- package/dist/bakeoff/samples.d.ts.map +1 -1
- package/dist/bakeoff/samples.js +20 -0
- package/dist/cli.d.ts +1 -3
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +134 -116
- package/dist/config.d.ts +100 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +104 -0
- package/dist/deprecations.d.ts +32 -0
- package/dist/deprecations.d.ts.map +1 -0
- package/dist/deprecations.js +59 -0
- package/dist/gateway/http-protocol.d.ts +175 -0
- package/dist/gateway/http-protocol.d.ts.map +1 -0
- package/dist/gateway/http-protocol.js +105 -0
- package/dist/index.d.ts +105 -18
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +266 -138
- package/dist/init/finder.d.ts +15 -0
- package/dist/init/finder.d.ts.map +1 -1
- package/dist/init/finder.js +46 -0
- package/dist/init/init-cli.d.ts +10 -0
- package/dist/init/init-cli.d.ts.map +1 -0
- package/dist/init/init-cli.js +86 -0
- package/dist/init/observe-cli.d.ts +19 -0
- package/dist/init/observe-cli.d.ts.map +1 -0
- package/dist/init/observe-cli.js +73 -0
- package/dist/init/observe.d.ts +81 -0
- package/dist/init/observe.d.ts.map +1 -0
- package/dist/init/observe.js +245 -0
- package/dist/registry/build.d.ts +94 -1
- package/dist/registry/build.d.ts.map +1 -1
- package/dist/registry/build.js +125 -4
- package/dist/registry/cache-policy.d.ts +94 -0
- package/dist/registry/cache-policy.d.ts.map +1 -0
- package/dist/registry/cache-policy.js +238 -0
- package/dist/registry/call-notes.d.ts.map +1 -1
- package/dist/registry/call-notes.js +40 -2
- package/dist/registry/effective-rate.d.ts +151 -0
- package/dist/registry/effective-rate.d.ts.map +1 -0
- package/dist/registry/effective-rate.js +156 -0
- package/dist/registry/facts.d.ts.map +1 -1
- package/dist/registry/facts.js +16 -0
- package/dist/registry/parity.d.ts +26 -0
- package/dist/registry/parity.d.ts.map +1 -1
- package/dist/registry/parity.js +43 -2
- package/dist/registry/pricing-shape.d.ts +113 -0
- package/dist/registry/pricing-shape.d.ts.map +1 -0
- package/dist/registry/pricing-shape.js +123 -0
- package/dist/registry/pricing.d.ts +25 -1
- package/dist/registry/pricing.d.ts.map +1 -1
- package/dist/registry/pricing.js +64 -6
- package/dist/rigindex/bootstrap.d.ts +36 -0
- package/dist/rigindex/bootstrap.d.ts.map +1 -0
- package/dist/rigindex/bootstrap.js +97 -0
- package/dist/rigindex/fit.d.ts +24 -0
- package/dist/rigindex/fit.d.ts.map +1 -0
- package/dist/rigindex/fit.js +76 -0
- package/dist/rigindex/gates.d.ts +52 -0
- package/dist/rigindex/gates.d.ts.map +1 -0
- package/dist/rigindex/gates.js +109 -0
- package/dist/rigindex/graph.d.ts +31 -0
- package/dist/rigindex/graph.d.ts.map +1 -0
- package/dist/rigindex/graph.js +120 -0
- package/dist/rigindex/index.d.ts +23 -0
- package/dist/rigindex/index.d.ts.map +1 -0
- package/dist/rigindex/index.js +112 -0
- package/dist/rigindex/types.d.ts +76 -0
- package/dist/rigindex/types.d.ts.map +1 -0
- package/dist/rigindex/types.js +8 -0
- package/dist/routes/cache-lint.d.ts +41 -0
- package/dist/routes/cache-lint.d.ts.map +1 -0
- package/dist/routes/cache-lint.js +81 -0
- package/dist/routes/loader.d.ts +37 -0
- package/dist/routes/loader.d.ts.map +1 -1
- package/dist/routes/loader.js +138 -0
- package/dist/routes/resolved-bundle.d.ts +79 -0
- package/dist/routes/resolved-bundle.d.ts.map +1 -0
- package/dist/routes/resolved-bundle.js +148 -0
- package/dist/routes/sampling-config.d.ts +32 -0
- package/dist/routes/sampling-config.d.ts.map +1 -0
- package/dist/routes/sampling-config.js +91 -0
- package/dist/run/ajv.d.ts +22 -0
- package/dist/run/ajv.d.ts.map +1 -0
- package/dist/run/ajv.js +32 -0
- package/dist/run/cache-handle.d.ts +58 -0
- package/dist/run/cache-handle.d.ts.map +1 -0
- package/dist/run/cache-handle.js +54 -0
- package/dist/run/cache-stamp.d.ts +65 -0
- package/dist/run/cache-stamp.d.ts.map +1 -0
- package/dist/run/cache-stamp.js +0 -0
- package/dist/run/capture-write.d.ts +33 -0
- package/dist/run/capture-write.d.ts.map +1 -0
- package/dist/run/capture-write.js +55 -0
- package/dist/run/cost.d.ts.map +1 -1
- package/dist/run/cost.js +1 -1
- package/dist/run/dispatch.d.ts +99 -0
- package/dist/run/dispatch.d.ts.map +1 -0
- package/dist/run/dispatch.js +191 -0
- package/dist/run/raw.d.ts +88 -0
- package/dist/run/raw.d.ts.map +1 -0
- package/dist/run/raw.js +286 -0
- package/dist/run/record-attempt.d.ts +40 -0
- package/dist/run/record-attempt.d.ts.map +1 -0
- package/dist/run/record-attempt.js +99 -0
- package/dist/run/resolve.d.ts +75 -2
- package/dist/run/resolve.d.ts.map +1 -1
- package/dist/run/resolve.js +126 -4
- package/dist/run/run.d.ts +14 -0
- package/dist/run/run.d.ts.map +1 -1
- package/dist/run/run.js +0 -0
- package/dist/run/stream.d.ts +49 -0
- package/dist/run/stream.d.ts.map +1 -0
- package/dist/run/stream.js +87 -0
- package/dist/run/validate.d.ts.map +1 -1
- package/dist/run/validate.js +6 -7
- package/dist/savings/cache-ledger.d.ts +108 -0
- package/dist/savings/cache-ledger.d.ts.map +1 -0
- package/dist/savings/cache-ledger.js +174 -0
- package/dist/savings/cache-reconcile.d.ts +64 -0
- package/dist/savings/cache-reconcile.d.ts.map +1 -0
- package/dist/savings/cache-reconcile.js +59 -0
- package/dist/savings/cache-stats.d.ts +63 -0
- package/dist/savings/cache-stats.d.ts.map +1 -0
- package/dist/savings/cache-stats.js +108 -0
- package/dist/savings/emit.d.ts +13 -0
- package/dist/savings/emit.d.ts.map +1 -1
- package/dist/savings/emit.js +25 -0
- package/dist/savings/swap-execute.d.ts +14 -0
- package/dist/savings/swap-execute.d.ts.map +1 -1
- package/dist/savings/swap-execute.js +44 -1
- package/dist/savings/swaps-client.d.ts +14 -0
- package/dist/savings/swaps-client.d.ts.map +1 -1
- package/dist/savings/swaps-client.js +16 -2
- package/dist/settings/api-fetcher.d.ts +39 -0
- package/dist/settings/api-fetcher.d.ts.map +1 -0
- package/dist/settings/api-fetcher.js +92 -0
- package/dist/settings/index.d.ts +10 -0
- package/dist/settings/index.d.ts.map +1 -0
- package/dist/settings/index.js +15 -0
- package/dist/settings/task-settings.d.ts +91 -0
- package/dist/settings/task-settings.d.ts.map +1 -0
- package/dist/settings/task-settings.js +116 -0
- package/dist/status.d.ts +77 -0
- package/dist/status.d.ts.map +1 -0
- package/dist/status.js +113 -0
- package/dist/telemetry/artifact-store.d.ts +48 -0
- package/dist/telemetry/artifact-store.d.ts.map +1 -0
- package/dist/telemetry/artifact-store.js +189 -0
- package/dist/telemetry/cache-miss-streak.d.ts +79 -0
- package/dist/telemetry/cache-miss-streak.d.ts.map +1 -0
- package/dist/telemetry/cache-miss-streak.js +99 -0
- package/dist/telemetry/capture.d.ts +3 -0
- package/dist/telemetry/capture.d.ts.map +1 -1
- package/dist/telemetry/capture.js +15 -8
- package/dist/telemetry/exporter-setup.d.ts +28 -0
- package/dist/telemetry/exporter-setup.d.ts.map +1 -0
- package/dist/telemetry/exporter-setup.js +34 -0
- package/dist/telemetry/exporter.d.ts +75 -0
- package/dist/telemetry/exporter.d.ts.map +1 -1
- package/dist/telemetry/exporter.js +324 -47
- package/dist/telemetry/health.d.ts +76 -0
- package/dist/telemetry/health.d.ts.map +1 -0
- package/dist/telemetry/health.js +92 -0
- package/dist/telemetry/ingest-protocol.d.ts +152 -3
- package/dist/telemetry/ingest-protocol.d.ts.map +1 -1
- package/dist/telemetry/ingest-protocol.js +204 -4
- package/dist/telemetry/load-error-report.d.ts +24 -0
- package/dist/telemetry/load-error-report.d.ts.map +1 -0
- package/dist/telemetry/load-error-report.js +50 -0
- package/dist/telemetry/otel-map.d.ts +121 -0
- package/dist/telemetry/otel-map.d.ts.map +1 -0
- package/dist/telemetry/otel-map.js +224 -0
- package/dist/telemetry/otel-normalize.d.ts +104 -0
- package/dist/telemetry/otel-normalize.d.ts.map +1 -0
- package/dist/telemetry/otel-normalize.js +530 -0
- package/dist/telemetry/otlp-emitter.d.ts +52 -0
- package/dist/telemetry/otlp-emitter.d.ts.map +1 -0
- package/dist/telemetry/otlp-emitter.js +144 -0
- package/dist/telemetry/otlp-proto.d.ts +29 -0
- package/dist/telemetry/otlp-proto.d.ts.map +1 -0
- package/dist/telemetry/otlp-proto.js +266 -0
- package/dist/telemetry/sinks.d.ts +37 -1
- package/dist/telemetry/sinks.d.ts.map +1 -1
- package/dist/telemetry/sinks.js +121 -30
- package/dist/telemetry/store.d.ts +28 -1
- package/dist/telemetry/store.d.ts.map +1 -1
- package/dist/telemetry/store.js +218 -3
- package/dist/tenancy/api-key-format.d.ts +15 -5
- package/dist/tenancy/api-key-format.d.ts.map +1 -1
- package/dist/tenancy/api-key-format.js +24 -5
- package/dist/types/config-teaching.d.ts +62 -0
- package/dist/types/config-teaching.d.ts.map +1 -0
- package/dist/types/config-teaching.js +183 -0
- package/dist/types/provider.d.ts +122 -1
- package/dist/types/provider.d.ts.map +1 -1
- package/dist/types/provider.js +21 -1
- package/dist/types/route.d.ts +79 -1
- package/dist/types/route.d.ts.map +1 -1
- package/dist/types/telemetry.d.ts +117 -0
- package/dist/types/telemetry.d.ts.map +1 -1
- package/dist/validate-cli.d.ts +12 -0
- package/dist/validate-cli.d.ts.map +1 -0
- package/dist/validate-cli.js +57 -0
- package/dist/verify/capture.d.ts +66 -0
- package/dist/verify/capture.d.ts.map +1 -0
- package/dist/verify/capture.js +180 -0
- package/dist/verify/cli-run.d.ts +23 -0
- package/dist/verify/cli-run.d.ts.map +1 -0
- package/dist/verify/cli-run.js +57 -0
- package/dist/verify/diff.d.ts +48 -0
- package/dist/verify/diff.d.ts.map +1 -0
- package/dist/verify/diff.js +157 -0
- package/dist/verify/index.d.ts +17 -0
- package/dist/verify/index.d.ts.map +1 -0
- package/dist/verify/index.js +24 -0
- package/dist/verify/report.d.ts +14 -0
- package/dist/verify/report.d.ts.map +1 -0
- package/dist/verify/report.js +88 -0
- package/dist/verify/types.d.ts +101 -0
- package/dist/verify/types.d.ts.map +1 -0
- package/dist/verify/types.js +17 -0
- package/dist/verify/verify.d.ts +29 -0
- package/dist/verify/verify.d.ts.map +1 -0
- package/dist/verify/verify.js +148 -0
- package/dist/watch/evaluate.d.ts +162 -3
- package/dist/watch/evaluate.d.ts.map +1 -1
- package/dist/watch/evaluate.js +61 -0
- package/dist/watch/monitor.d.ts +116 -0
- package/dist/watch/monitor.d.ts.map +1 -0
- package/dist/watch/monitor.js +199 -0
- package/dist/watch/proposals.d.ts +20 -4
- package/dist/watch/proposals.d.ts.map +1 -1
- package/dist/watch/proposals.js +31 -3
- package/dist/watch/run.d.ts +33 -0
- package/dist/watch/run.d.ts.map +1 -1
- package/dist/watch/run.js +39 -0
- package/package.json +7 -1
- package/registry/LICENSE +20 -0
- package/registry/leaderboard.json +3639 -0
- package/registry/registry.json +7263 -0
- package/dist/telemetry/supabase-exporter.d.ts +0 -64
- package/dist/telemetry/supabase-exporter.d.ts.map +0 -1
- package/dist/telemetry/supabase-exporter.js +0 -211
|
@@ -0,0 +1,377 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Artifact namespace — Expansion E2 chunk B (spec §2.2, §9).
|
|
4
|
+
*
|
|
5
|
+
* `save/load/link/evaluate` over a durable work product's METADATA. The
|
|
6
|
+
* serialized value is hashed (sha256, node:crypto — no new dep) and the bytes
|
|
7
|
+
* are DISCARDED; nothing but the metadata row is persisted (the 00030
|
|
8
|
+
* counter-annotation). Lineage v0: artifacts saved within a run get
|
|
9
|
+
* `derived_from` edges to artifacts `load`ed since the last save (coverage
|
|
10
|
+
* measured, never promised). Instrumentation never throws into customer code.
|
|
11
|
+
*
|
|
12
|
+
* ── zeroRetention (ratified invariant, named test) ────────────────────────
|
|
13
|
+
* A run with `zeroRetention: true` emits NO artifact rows of any kind: `save`
|
|
14
|
+
* refuses fail-closed at THIS gate (before any row is built) with a counted,
|
|
15
|
+
* logged skip. Structural, not policy-checked at the server.
|
|
16
|
+
*/
|
|
17
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
18
|
+
exports.createArtifactApi = createArtifactApi;
|
|
19
|
+
const node_crypto_1 = require("node:crypto");
|
|
20
|
+
const fingerprint_1 = require("../registry/fingerprint");
|
|
21
|
+
const run_context_1 = require("./run-context");
|
|
22
|
+
const scrub_1 = require("./scrub");
|
|
23
|
+
/** Map an artifact `type` to a content-type for the object store. */
|
|
24
|
+
function contentTypeFor(type) {
|
|
25
|
+
const t = type.toLowerCase();
|
|
26
|
+
if (t === "json")
|
|
27
|
+
return "application/json";
|
|
28
|
+
if (t === "text" || t === "txt" || t === "md" || t === "markdown")
|
|
29
|
+
return "text/plain; charset=utf-8";
|
|
30
|
+
if (t === "csv")
|
|
31
|
+
return "text/csv";
|
|
32
|
+
if (t === "html")
|
|
33
|
+
return "text/html; charset=utf-8";
|
|
34
|
+
return "application/octet-stream";
|
|
35
|
+
}
|
|
36
|
+
function idOf(ref) {
|
|
37
|
+
return typeof ref === "string" ? ref : ref.id;
|
|
38
|
+
}
|
|
39
|
+
function warn(scope, err) {
|
|
40
|
+
console.warn(`[modelrig artifacts] ${scope} buffer write failed (dropped, run unaffected): ` +
|
|
41
|
+
`${err instanceof Error ? err.message : String(err)}`);
|
|
42
|
+
}
|
|
43
|
+
/** A disabled/inert namespace — every entry is a no-op (spec §1.4). */
|
|
44
|
+
function inertApi() {
|
|
45
|
+
return {
|
|
46
|
+
save: () => null,
|
|
47
|
+
load: () => { },
|
|
48
|
+
link: () => { },
|
|
49
|
+
evaluate: () => { },
|
|
50
|
+
promoteToEval: () => null,
|
|
51
|
+
batch: (fn) => fn(),
|
|
52
|
+
flushContent: () => Promise.resolve(),
|
|
53
|
+
retryPendingContent: () => Promise.resolve(),
|
|
54
|
+
coverage: () => ({
|
|
55
|
+
saves: 0,
|
|
56
|
+
inferredEdges: 0,
|
|
57
|
+
skippedZeroRetention: 0,
|
|
58
|
+
contentUploaded: 0,
|
|
59
|
+
contentDegraded: 0,
|
|
60
|
+
contentBlocked: 0,
|
|
61
|
+
contentPending: 0,
|
|
62
|
+
scrubFindings: 0,
|
|
63
|
+
}),
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
/** Bounded post-flush retries for a `pending` (404-until-durable) content upload
|
|
67
|
+
* before it becomes a counted skip (E4 chunk 0, flag 2). Two retries with the
|
|
68
|
+
* exporter interval between them is a multi-flush window for the metadata row to
|
|
69
|
+
* land — long enough for a real export lag, short enough to bound the wait. */
|
|
70
|
+
const CONTENT_RETRY_BUDGET = 2;
|
|
71
|
+
function createArtifactApi(deps) {
|
|
72
|
+
if (!deps.enabled)
|
|
73
|
+
return inertApi();
|
|
74
|
+
const { store, content } = deps;
|
|
75
|
+
let saves = 0;
|
|
76
|
+
let inferredEdges = 0;
|
|
77
|
+
let skippedZeroRetention = 0;
|
|
78
|
+
let contentUploaded = 0;
|
|
79
|
+
let contentDegraded = 0;
|
|
80
|
+
let contentBlocked = 0;
|
|
81
|
+
let scrubFindings = 0;
|
|
82
|
+
/** In-flight uploads; flushContent awaits them (graceful shutdown + tests). */
|
|
83
|
+
const pendingUploads = new Set();
|
|
84
|
+
/** Uploads that came back `pending` (grant 404 — metadata not yet durable),
|
|
85
|
+
* awaiting the exporter's post-flush retry hook. `retriesLeft` decrements once
|
|
86
|
+
* per hook call; at 0 the item becomes a counted skip (E4 chunk 0, flag 2). */
|
|
87
|
+
const pendingContent = [];
|
|
88
|
+
/** artifact id → the task it was saved with THIS session (E4 chunk A). Lets
|
|
89
|
+
* `promoteToEval(ref)` resolve the case's task without the caller re-supplying
|
|
90
|
+
* it. A promote of an artifact from a prior session must pass `opts.task`. */
|
|
91
|
+
const savedTasks = new Map();
|
|
92
|
+
/** Fire one content upload off the hot path. NEVER rejects into caller code —
|
|
93
|
+
* every outcome updates a counter (or queues a `pending` retry) and resolves. */
|
|
94
|
+
function fireUpload(input, upload) {
|
|
95
|
+
const p = upload(input)
|
|
96
|
+
.then((outcome) => {
|
|
97
|
+
if (outcome.kind === "uploaded")
|
|
98
|
+
contentUploaded += 1;
|
|
99
|
+
// A 404-until-durable race: queue for the post-flush retry rather than
|
|
100
|
+
// counting a degrade — the row simply has not been exported yet.
|
|
101
|
+
else if (outcome.kind === "pending") {
|
|
102
|
+
pendingContent.push({ input, retriesLeft: CONTENT_RETRY_BUDGET });
|
|
103
|
+
}
|
|
104
|
+
else
|
|
105
|
+
contentDegraded += 1; // "degraded" | "blocked"
|
|
106
|
+
})
|
|
107
|
+
.catch(() => {
|
|
108
|
+
contentDegraded += 1;
|
|
109
|
+
})
|
|
110
|
+
.finally(() => {
|
|
111
|
+
pendingUploads.delete(p);
|
|
112
|
+
});
|
|
113
|
+
pendingUploads.add(p);
|
|
114
|
+
}
|
|
115
|
+
/** Re-attempt every queued `pending` upload exactly once (the exporter's
|
|
116
|
+
* post-flush hook). Items that stay pending are re-queued with a decremented
|
|
117
|
+
* budget; at 0 they become a counted skip. Off/empty ⇒ a resolved no-op. */
|
|
118
|
+
async function retryPendingContent() {
|
|
119
|
+
if (content === undefined || pendingContent.length === 0)
|
|
120
|
+
return;
|
|
121
|
+
// Snapshot then clear: an item that stays pending is re-pushed with a lower
|
|
122
|
+
// budget, so THIS pass is exactly one retry per currently-queued item.
|
|
123
|
+
const batch = pendingContent.splice(0, pendingContent.length);
|
|
124
|
+
for (const item of batch) {
|
|
125
|
+
let outcome;
|
|
126
|
+
try {
|
|
127
|
+
outcome = await content.upload(item.input);
|
|
128
|
+
}
|
|
129
|
+
catch {
|
|
130
|
+
contentDegraded += 1; // an unexpected throw is a counted skip, never rethrown
|
|
131
|
+
continue;
|
|
132
|
+
}
|
|
133
|
+
if (outcome.kind === "uploaded") {
|
|
134
|
+
contentUploaded += 1;
|
|
135
|
+
}
|
|
136
|
+
else if (outcome.kind === "pending") {
|
|
137
|
+
const retriesLeft = item.retriesLeft - 1;
|
|
138
|
+
if (retriesLeft > 0) {
|
|
139
|
+
pendingContent.push({ input: item.input, retriesLeft });
|
|
140
|
+
}
|
|
141
|
+
else {
|
|
142
|
+
contentDegraded += 1; // exhausted the bounded retries → metadata-only
|
|
143
|
+
console.warn(`[modelrig artifacts] content for artifact ${item.input.artifactId} still not durable ` +
|
|
144
|
+
`after ${CONTENT_RETRY_BUDGET} post-flush retries — metadata-only (counted skip).`);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
else {
|
|
148
|
+
contentDegraded += 1; // degraded | blocked on retry → counted skip
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
return {
|
|
153
|
+
save(value, opts) {
|
|
154
|
+
const ctx = (0, run_context_1.getActiveRunContext)();
|
|
155
|
+
// zeroRetention: refuse fail-closed BEFORE hashing or building any row.
|
|
156
|
+
if (ctx !== null && ctx.zeroRetention) {
|
|
157
|
+
skippedZeroRetention += 1;
|
|
158
|
+
console.warn(`[modelrig artifacts] zeroRetention run — artifact "${opts.name}" NOT recorded ` +
|
|
159
|
+
"(fail-closed at the SDK gate; no metadata, no hash persisted).");
|
|
160
|
+
return null;
|
|
161
|
+
}
|
|
162
|
+
// Hash the serialization, then DISCARD the bytes — metadata + hash only.
|
|
163
|
+
const serialized = typeof value === "string" ? value : JSON.stringify(value ?? null);
|
|
164
|
+
const bytes = Buffer.byteLength(serialized, "utf8");
|
|
165
|
+
const contentHash = `sha256:${(0, node_crypto_1.createHash)("sha256").update(serialized).digest("hex")}`;
|
|
166
|
+
const schema = opts.schema ?? null;
|
|
167
|
+
const id = (0, node_crypto_1.randomUUID)();
|
|
168
|
+
const version = opts.supersedes !== undefined ? opts.supersedes.version + 1 : 1;
|
|
169
|
+
// Custody applies only when a rig content policy exists AND the caller left
|
|
170
|
+
// the bytes to us (a caller-supplied contentRef = their own store; managed
|
|
171
|
+
// custody must not double-home it).
|
|
172
|
+
const custodyApplies = content !== undefined && opts.contentRef === undefined;
|
|
173
|
+
// Scrub is computed HERE (before the row is written) so its counts-only
|
|
174
|
+
// summary can ride the immutable artifact row (00039). Only when custody
|
|
175
|
+
// applies; null otherwise (custody off ⇒ nothing was scanned).
|
|
176
|
+
const scrubbed = custodyApplies && content !== undefined ? (0, scrub_1.scrubContent)(serialized, content.scrub) : null;
|
|
177
|
+
const scrubSummary = scrubbed !== null && content !== undefined ? (0, scrub_1.summarizeScrub)(content.scrub, scrubbed) : null;
|
|
178
|
+
if (scrubbed !== null)
|
|
179
|
+
scrubFindings += scrubbed.findings.reduce((n, f) => n + f.count, 0);
|
|
180
|
+
try {
|
|
181
|
+
store.recordArtifact({
|
|
182
|
+
id,
|
|
183
|
+
runId: ctx?.runId ?? null,
|
|
184
|
+
// E3: step attribution is now an explicit caller-supplied key (see
|
|
185
|
+
// ArtifactSaveOptions.stepKey). Absent ⇒ null, exactly as before.
|
|
186
|
+
stepKey: opts.stepKey ?? null,
|
|
187
|
+
attemptNo: opts.attemptNo ?? null,
|
|
188
|
+
task: opts.task ?? null,
|
|
189
|
+
name: opts.name,
|
|
190
|
+
type: opts.type,
|
|
191
|
+
version,
|
|
192
|
+
supersedesArtifactId: opts.supersedes?.id ?? null,
|
|
193
|
+
declaredSchema: schema,
|
|
194
|
+
fingerprintCell: (0, fingerprint_1.fingerprintSchema)(schema)?.cellKey ?? null,
|
|
195
|
+
contentRef: opts.contentRef ?? null,
|
|
196
|
+
contentHash,
|
|
197
|
+
contentBytes: bytes,
|
|
198
|
+
dataClassification: opts.classification ?? null,
|
|
199
|
+
expiresAt: opts.expiresAt ?? null,
|
|
200
|
+
// Per-row scrub findings ride the original (scanned) row (00039).
|
|
201
|
+
scrub: scrubSummary,
|
|
202
|
+
});
|
|
203
|
+
saves += 1;
|
|
204
|
+
if (opts.task !== undefined)
|
|
205
|
+
savedTasks.set(id, opts.task); // for promoteToEval (chunk A)
|
|
206
|
+
// Lineage v0: derived_from edges to everything loaded since the last save.
|
|
207
|
+
if (ctx !== null) {
|
|
208
|
+
for (const loadedId of ctx.takeLoadedIds()) {
|
|
209
|
+
if (loadedId === id)
|
|
210
|
+
continue; // edges_no_self
|
|
211
|
+
store.recordArtifactEdge({ fromArtifactId: id, toArtifactId: loadedId, kind: "derived_from" });
|
|
212
|
+
inferredEdges += 1;
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
catch (err) {
|
|
217
|
+
warn("save", err);
|
|
218
|
+
return null;
|
|
219
|
+
}
|
|
220
|
+
// ── Content custody (E2-C) ──────────────────────────────────────────
|
|
221
|
+
// Off by default: only when a rig has configured content custody. The
|
|
222
|
+
// metadata row above is written regardless — content is layered on, never
|
|
223
|
+
// a precondition. zeroRetention already returned above, so we never reach
|
|
224
|
+
// here for a zeroRetention run (structural, not re-checked).
|
|
225
|
+
if (content !== undefined && scrubbed !== null) {
|
|
226
|
+
if (scrubbed.blocked) {
|
|
227
|
+
// `block` mode found PII/PHI → content upload refused (counted). The
|
|
228
|
+
// artifact keeps its metadata + hash; it simply stores no content.
|
|
229
|
+
contentBlocked += 1;
|
|
230
|
+
console.warn(`[modelrig artifacts] scrub blocked content for "${opts.name}" ` +
|
|
231
|
+
`(${scrubbed.findings.length} finding type(s)); metadata + hash kept, no content stored.`);
|
|
232
|
+
}
|
|
233
|
+
else if (content.scrub === "redact" && scrubbed.findings.length > 0 && scrubbed.text !== null) {
|
|
234
|
+
// Redaction is a SUPERSEDING version (immutability): the original row
|
|
235
|
+
// keeps its hash and stores NO content; a new version records the
|
|
236
|
+
// masked content's hash and uploads the masked bytes.
|
|
237
|
+
const maskedSerialized = scrubbed.text;
|
|
238
|
+
const maskedBytes = Buffer.byteLength(maskedSerialized, "utf8");
|
|
239
|
+
const maskedHash = `sha256:${(0, node_crypto_1.createHash)("sha256").update(maskedSerialized).digest("hex")}`;
|
|
240
|
+
const redactedId = (0, node_crypto_1.randomUUID)();
|
|
241
|
+
const redactedVersion = version + 1;
|
|
242
|
+
try {
|
|
243
|
+
store.recordArtifact({
|
|
244
|
+
id: redactedId,
|
|
245
|
+
runId: ctx?.runId ?? null,
|
|
246
|
+
stepKey: opts.stepKey ?? null,
|
|
247
|
+
attemptNo: opts.attemptNo ?? null,
|
|
248
|
+
task: opts.task ?? null,
|
|
249
|
+
name: opts.name,
|
|
250
|
+
type: opts.type,
|
|
251
|
+
version: redactedVersion,
|
|
252
|
+
supersedesArtifactId: id, // supersedes the original (ruling c column)
|
|
253
|
+
declaredSchema: schema,
|
|
254
|
+
fingerprintCell: (0, fingerprint_1.fingerprintSchema)(schema)?.cellKey ?? null,
|
|
255
|
+
contentRef: null,
|
|
256
|
+
contentHash: maskedHash,
|
|
257
|
+
contentBytes: maskedBytes,
|
|
258
|
+
dataClassification: opts.classification ?? null,
|
|
259
|
+
expiresAt: opts.expiresAt ?? null,
|
|
260
|
+
// Findings ride the original scanned row; the masked version stores
|
|
261
|
+
// no PII, so its own scrub is null (avoid double-counting).
|
|
262
|
+
scrub: null,
|
|
263
|
+
});
|
|
264
|
+
saves += 1;
|
|
265
|
+
fireUpload({
|
|
266
|
+
artifactId: redactedId,
|
|
267
|
+
contentType: contentTypeFor(opts.type),
|
|
268
|
+
body: Buffer.from(maskedSerialized, "utf8"),
|
|
269
|
+
contentHash: maskedHash,
|
|
270
|
+
}, content.upload);
|
|
271
|
+
// The live version is the redacted one — hand its handle back.
|
|
272
|
+
return { id: redactedId, name: opts.name, type: opts.type, version: redactedVersion };
|
|
273
|
+
}
|
|
274
|
+
catch (err) {
|
|
275
|
+
warn("save (redacted version)", err);
|
|
276
|
+
// Fall through to returning the original handle (metadata-only).
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
else {
|
|
280
|
+
// off / detect / redact-with-no-findings: upload the bytes verbatim.
|
|
281
|
+
fireUpload({
|
|
282
|
+
artifactId: id,
|
|
283
|
+
contentType: contentTypeFor(opts.type),
|
|
284
|
+
body: Buffer.from(scrubbed.text ?? serialized, "utf8"),
|
|
285
|
+
contentHash,
|
|
286
|
+
}, content.upload);
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
return { id, name: opts.name, type: opts.type, version };
|
|
290
|
+
},
|
|
291
|
+
load(ref) {
|
|
292
|
+
(0, run_context_1.getActiveRunContext)()?.noteLoad(idOf(ref));
|
|
293
|
+
},
|
|
294
|
+
link(from, to, kind) {
|
|
295
|
+
const fromId = idOf(from);
|
|
296
|
+
const toId = idOf(to);
|
|
297
|
+
if (fromId === toId)
|
|
298
|
+
return; // edges_no_self
|
|
299
|
+
try {
|
|
300
|
+
store.recordArtifactEdge({ fromArtifactId: fromId, toArtifactId: toId, kind });
|
|
301
|
+
}
|
|
302
|
+
catch (err) {
|
|
303
|
+
warn("link", err);
|
|
304
|
+
}
|
|
305
|
+
},
|
|
306
|
+
evaluate(ref, opts) {
|
|
307
|
+
try {
|
|
308
|
+
store.recordArtifactEvaluation({
|
|
309
|
+
id: (0, node_crypto_1.randomUUID)(),
|
|
310
|
+
artifactId: idOf(ref),
|
|
311
|
+
runId: null,
|
|
312
|
+
evaluator: opts.evaluator,
|
|
313
|
+
evaluatorKind: opts.kind,
|
|
314
|
+
score: opts.score ?? null,
|
|
315
|
+
passed: opts.passed ?? null,
|
|
316
|
+
label: opts.label ?? null,
|
|
317
|
+
explanation: opts.explanation ?? null,
|
|
318
|
+
createdBy: null,
|
|
319
|
+
});
|
|
320
|
+
}
|
|
321
|
+
catch (err) {
|
|
322
|
+
warn("evaluate", err);
|
|
323
|
+
}
|
|
324
|
+
},
|
|
325
|
+
promoteToEval(ref, opts) {
|
|
326
|
+
const sourceArtifactId = idOf(ref);
|
|
327
|
+
// Resolve the task: explicit wins, else the task this artifact was saved
|
|
328
|
+
// with this session. Unresolvable ⇒ a warned no-op (eval_cases needs task).
|
|
329
|
+
const task = opts?.task ?? savedTasks.get(sourceArtifactId);
|
|
330
|
+
if (task === undefined) {
|
|
331
|
+
console.warn(`[modelrig artifacts] promoteToEval("${sourceArtifactId}") skipped: no task known for this ` +
|
|
332
|
+
"artifact (it was not saved this session) — pass { task } to promote it.");
|
|
333
|
+
return null;
|
|
334
|
+
}
|
|
335
|
+
const correctedArtifactId = opts?.corrected !== undefined ? idOf(opts.corrected) : null;
|
|
336
|
+
const origin = opts?.origin ?? (correctedArtifactId !== null ? "human-corrected" : "promoted-failure");
|
|
337
|
+
const id = (0, node_crypto_1.randomUUID)();
|
|
338
|
+
try {
|
|
339
|
+
store.recordEvalCase({
|
|
340
|
+
id,
|
|
341
|
+
task,
|
|
342
|
+
sourceArtifactId,
|
|
343
|
+
correctedArtifactId,
|
|
344
|
+
origin,
|
|
345
|
+
note: opts?.note ?? null,
|
|
346
|
+
createdBy: null,
|
|
347
|
+
});
|
|
348
|
+
}
|
|
349
|
+
catch (err) {
|
|
350
|
+
warn("promoteToEval", err);
|
|
351
|
+
return null;
|
|
352
|
+
}
|
|
353
|
+
return id;
|
|
354
|
+
},
|
|
355
|
+
batch(fn) {
|
|
356
|
+
return store.batch(fn);
|
|
357
|
+
},
|
|
358
|
+
async flushContent() {
|
|
359
|
+
// Settle every in-flight upload. Individual failures already resolve (they
|
|
360
|
+
// update contentDegraded), so allSettled never throws here.
|
|
361
|
+
await Promise.allSettled([...pendingUploads]);
|
|
362
|
+
},
|
|
363
|
+
retryPendingContent,
|
|
364
|
+
coverage() {
|
|
365
|
+
return {
|
|
366
|
+
saves,
|
|
367
|
+
inferredEdges,
|
|
368
|
+
skippedZeroRetention,
|
|
369
|
+
contentUploaded,
|
|
370
|
+
contentDegraded,
|
|
371
|
+
contentBlocked,
|
|
372
|
+
contentPending: pendingContent.length,
|
|
373
|
+
scrubFindings,
|
|
374
|
+
};
|
|
375
|
+
},
|
|
376
|
+
};
|
|
377
|
+
}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Managed content upload path — E2-C chunk A (spec §2.4).
|
|
3
|
+
*
|
|
4
|
+
* When a rig has opted a route into content custody, `save` records the metadata
|
|
5
|
+
* row (as always) and THEN hands the serialized bytes here. This module talks to
|
|
6
|
+
* modelrig-server's grant endpoint, receives a short-lived presigned PUT, and
|
|
7
|
+
* uploads the bytes straight to the object store — the bytes NEVER touch our
|
|
8
|
+
* control-plane database, only the customer-designated store.
|
|
9
|
+
*
|
|
10
|
+
* ── THE POISON LESSON, APPLIED ────────────────────────────────────────────
|
|
11
|
+
* A deterministic refusal (403 posture-off, 409 already has content, 413 too
|
|
12
|
+
* big, 422 malformed) is NOT retried — it will fail identically forever, and
|
|
13
|
+
* retrying a poison request is how a fail-closed refusal turns into a hot loop.
|
|
14
|
+
* Every refusal DEGRADES to metadata-only: the artifact still exists (its row
|
|
15
|
+
* was written before we were called), it simply has no stored content.
|
|
16
|
+
* Degradation is counted and warned, never thrown — this runs off the
|
|
17
|
+
* customer's hot path and must never surface as a failed run.
|
|
18
|
+
*
|
|
19
|
+
* ── THE 404-UNTIL-DURABLE RACE (E4 chunk 0, flag 2) ───────────────────────
|
|
20
|
+
* A 404 is the ONE 4xx that is NOT deterministic. The grant endpoint verifies
|
|
21
|
+
* the artifact metadata row exists (and belongs to the org — T11) before
|
|
22
|
+
* issuing a URL, but that row reaches the control plane on the exporter's NEXT
|
|
23
|
+
* flush — so a content upload fired during `save` races ahead of its own
|
|
24
|
+
* metadata and 404s until the row lands. TRANSIENT by nature: it clears once
|
|
25
|
+
* the metadata is durable. So a 404 returns a distinct `pending` outcome that
|
|
26
|
+
* the exporter's post-flush hook re-attempts (bounded: 2 retries, then a
|
|
27
|
+
* counted skip). A genuinely foreign/missing artifact 404s the same way and
|
|
28
|
+
* simply exhausts those retries into the metadata-only end state — delayed, but
|
|
29
|
+
* never a hot loop.
|
|
30
|
+
*
|
|
31
|
+
* ── C2 ────────────────────────────────────────────────────────────────────
|
|
32
|
+
* The api key rides in an Authorization header, never a URL. The presigned URL
|
|
33
|
+
* and the content hash are NEVER logged. Warnings name the artifact + a status,
|
|
34
|
+
* nothing else.
|
|
35
|
+
*/
|
|
36
|
+
export type ContentUploadOutcome =
|
|
37
|
+
/** Bytes are in the store; `contentRef` is the server's canonical pointer. */
|
|
38
|
+
{
|
|
39
|
+
readonly kind: "uploaded";
|
|
40
|
+
readonly contentRef: string;
|
|
41
|
+
}
|
|
42
|
+
/** Metadata-only fallback (deterministic refusal or transient failure). The
|
|
43
|
+
* `status` is the grant HTTP status when there was one, else null. */
|
|
44
|
+
| {
|
|
45
|
+
readonly kind: "degraded";
|
|
46
|
+
readonly reason: string;
|
|
47
|
+
readonly status: number | null;
|
|
48
|
+
}
|
|
49
|
+
/** The artifact metadata row is not yet durable in the control plane — a 404
|
|
50
|
+
* from the grant endpoint (which verifies the row before issuing a URL, T11).
|
|
51
|
+
* TRANSIENT: it clears once the exporter flushes the metadata. NOT counted as
|
|
52
|
+
* a degrade; the caller queues it for the post-flush hook to re-attempt
|
|
53
|
+
* (bounded 2 retries, then a counted skip). E4 chunk 0, flag 2. */
|
|
54
|
+
| {
|
|
55
|
+
readonly kind: "pending";
|
|
56
|
+
readonly reason: string;
|
|
57
|
+
readonly status: 404;
|
|
58
|
+
}
|
|
59
|
+
/** The scrub layer BLOCKED upload (chunk B). Distinct from degraded: this is a
|
|
60
|
+
* policy refusal, not a failure. */
|
|
61
|
+
| {
|
|
62
|
+
readonly kind: "blocked";
|
|
63
|
+
readonly reason: string;
|
|
64
|
+
};
|
|
65
|
+
export interface ContentUploadConfig {
|
|
66
|
+
/** Full grant endpoint, e.g. https://server/v1/artifacts/uploads. */
|
|
67
|
+
readonly uploadsUrl: string;
|
|
68
|
+
/** `rig_sk_` key carrying the `artifact-write` scope. Header only (C2). */
|
|
69
|
+
readonly apiKey: string;
|
|
70
|
+
/** Client-side ceiling — a short-circuit before the round trip; the server
|
|
71
|
+
* enforces the authoritative one. */
|
|
72
|
+
readonly maxBytes: number;
|
|
73
|
+
}
|
|
74
|
+
export interface ContentUploadInput {
|
|
75
|
+
readonly artifactId: string;
|
|
76
|
+
readonly contentType: string;
|
|
77
|
+
/** Serialized bytes to upload — already scrubbed by the caller (chunk B). */
|
|
78
|
+
readonly body: Buffer;
|
|
79
|
+
/** `sha256:<hex>` of `body`. */
|
|
80
|
+
readonly contentHash: string;
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Upload one artifact's content. NEVER throws — every failure resolves to a
|
|
84
|
+
* `degraded` outcome the caller records as a metadata-only skip.
|
|
85
|
+
*/
|
|
86
|
+
export declare function uploadArtifactContent(input: ContentUploadInput, config: ContentUploadConfig, fetchFn?: typeof fetch): Promise<ContentUploadOutcome>;
|
|
87
|
+
//# sourceMappingURL=content-uploader.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"content-uploader.d.ts","sourceRoot":"","sources":["../../src/artifacts/content-uploader.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAEH,MAAM,MAAM,oBAAoB;AAC9B,8EAA8E;AAC5E;IAAE,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;CAAE;AAC5D;sEACsE;GACpE;IAAE,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE;AACxF;;;;mEAImE;GACjE;IAAE,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,GAAG,CAAA;CAAE;AAC7E;oCACoC;GAClC;IAAE,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAE1D,MAAM,WAAW,mBAAmB;IAClC,qEAAqE;IACrE,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,2EAA2E;IAC3E,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB;yCACqC;IACrC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,6EAA6E;IAC7E,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,gCAAgC;IAChC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAeD;;;GAGG;AACH,wBAAsB,qBAAqB,CACzC,KAAK,EAAE,kBAAkB,EACzB,MAAM,EAAE,mBAAmB,EAC3B,OAAO,GAAE,OAAO,KAAa,GAC5B,OAAO,CAAC,oBAAoB,CAAC,CAoE/B"}
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Managed content upload path — E2-C chunk A (spec §2.4).
|
|
4
|
+
*
|
|
5
|
+
* When a rig has opted a route into content custody, `save` records the metadata
|
|
6
|
+
* row (as always) and THEN hands the serialized bytes here. This module talks to
|
|
7
|
+
* modelrig-server's grant endpoint, receives a short-lived presigned PUT, and
|
|
8
|
+
* uploads the bytes straight to the object store — the bytes NEVER touch our
|
|
9
|
+
* control-plane database, only the customer-designated store.
|
|
10
|
+
*
|
|
11
|
+
* ── THE POISON LESSON, APPLIED ────────────────────────────────────────────
|
|
12
|
+
* A deterministic refusal (403 posture-off, 409 already has content, 413 too
|
|
13
|
+
* big, 422 malformed) is NOT retried — it will fail identically forever, and
|
|
14
|
+
* retrying a poison request is how a fail-closed refusal turns into a hot loop.
|
|
15
|
+
* Every refusal DEGRADES to metadata-only: the artifact still exists (its row
|
|
16
|
+
* was written before we were called), it simply has no stored content.
|
|
17
|
+
* Degradation is counted and warned, never thrown — this runs off the
|
|
18
|
+
* customer's hot path and must never surface as a failed run.
|
|
19
|
+
*
|
|
20
|
+
* ── THE 404-UNTIL-DURABLE RACE (E4 chunk 0, flag 2) ───────────────────────
|
|
21
|
+
* A 404 is the ONE 4xx that is NOT deterministic. The grant endpoint verifies
|
|
22
|
+
* the artifact metadata row exists (and belongs to the org — T11) before
|
|
23
|
+
* issuing a URL, but that row reaches the control plane on the exporter's NEXT
|
|
24
|
+
* flush — so a content upload fired during `save` races ahead of its own
|
|
25
|
+
* metadata and 404s until the row lands. TRANSIENT by nature: it clears once
|
|
26
|
+
* the metadata is durable. So a 404 returns a distinct `pending` outcome that
|
|
27
|
+
* the exporter's post-flush hook re-attempts (bounded: 2 retries, then a
|
|
28
|
+
* counted skip). A genuinely foreign/missing artifact 404s the same way and
|
|
29
|
+
* simply exhausts those retries into the metadata-only end state — delayed, but
|
|
30
|
+
* never a hot loop.
|
|
31
|
+
*
|
|
32
|
+
* ── C2 ────────────────────────────────────────────────────────────────────
|
|
33
|
+
* The api key rides in an Authorization header, never a URL. The presigned URL
|
|
34
|
+
* and the content hash are NEVER logged. Warnings name the artifact + a status,
|
|
35
|
+
* nothing else.
|
|
36
|
+
*/
|
|
37
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
38
|
+
exports.uploadArtifactContent = uploadArtifactContent;
|
|
39
|
+
/** A 4xx is a deterministic refusal — the request will fail identically forever,
|
|
40
|
+
* so it must never be retried (the poison lesson) — with ONE exception: 404,
|
|
41
|
+
* the metadata-not-yet-durable race, handled separately as `pending`. */
|
|
42
|
+
function isDeterministicRefusal(status) {
|
|
43
|
+
return status >= 400 && status < 500;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Upload one artifact's content. NEVER throws — every failure resolves to a
|
|
47
|
+
* `degraded` outcome the caller records as a metadata-only skip.
|
|
48
|
+
*/
|
|
49
|
+
async function uploadArtifactContent(input, config, fetchFn = fetch) {
|
|
50
|
+
// Client-side ceiling short-circuit: the server would 413 this; skip the trip.
|
|
51
|
+
if (input.body.byteLength > config.maxBytes) {
|
|
52
|
+
return {
|
|
53
|
+
kind: "degraded",
|
|
54
|
+
reason: `content is ${input.body.byteLength} bytes, over the ${config.maxBytes} ceiling`,
|
|
55
|
+
status: 413,
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
// 1) Request the grant. A 4xx is deterministic → degrade, no retry.
|
|
59
|
+
let grantRes;
|
|
60
|
+
try {
|
|
61
|
+
grantRes = await fetchFn(config.uploadsUrl, {
|
|
62
|
+
method: "POST",
|
|
63
|
+
headers: {
|
|
64
|
+
"content-type": "application/json",
|
|
65
|
+
authorization: `Bearer ${config.apiKey}`,
|
|
66
|
+
},
|
|
67
|
+
body: JSON.stringify({
|
|
68
|
+
artifact_id: input.artifactId,
|
|
69
|
+
content_type: input.contentType,
|
|
70
|
+
bytes: input.body.byteLength,
|
|
71
|
+
content_hash: input.contentHash,
|
|
72
|
+
}),
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
catch (err) {
|
|
76
|
+
// Transient (network) — degrade without retry this pass; the next save's
|
|
77
|
+
// upload is an independent attempt. We do NOT loop here.
|
|
78
|
+
return { kind: "degraded", reason: `grant request failed: ${errMsg(err)}`, status: null };
|
|
79
|
+
}
|
|
80
|
+
if (!grantRes.ok) {
|
|
81
|
+
// 404 is the metadata-not-yet-durable race (T11), the ONE transient 4xx: the
|
|
82
|
+
// row reaches the control plane on the exporter's next flush, so we hand back
|
|
83
|
+
// `pending` for the post-flush hook to re-attempt (bounded). Every other 4xx
|
|
84
|
+
// is a deterministic refusal and 5xx a server error — both degrade here, no
|
|
85
|
+
// retry this pass. The distinction is only in the reason we log.
|
|
86
|
+
if (grantRes.status === 404) {
|
|
87
|
+
return { kind: "pending", reason: "artifact metadata not yet durable (grant 404)", status: 404 };
|
|
88
|
+
}
|
|
89
|
+
const kind = isDeterministicRefusal(grantRes.status) ? "refused" : "unavailable";
|
|
90
|
+
return { kind: "degraded", reason: `grant ${kind} (status ${grantRes.status})`, status: grantRes.status };
|
|
91
|
+
}
|
|
92
|
+
let grant;
|
|
93
|
+
try {
|
|
94
|
+
grant = (await grantRes.json());
|
|
95
|
+
}
|
|
96
|
+
catch {
|
|
97
|
+
return { kind: "degraded", reason: "grant response was not JSON", status: grantRes.status };
|
|
98
|
+
}
|
|
99
|
+
if (typeof grant.upload_url !== "string" || typeof grant.content_ref !== "string") {
|
|
100
|
+
return { kind: "degraded", reason: "grant response missing upload_url/content_ref", status: grantRes.status };
|
|
101
|
+
}
|
|
102
|
+
const putHeaders = isStringRecord(grant.headers) ? grant.headers : {};
|
|
103
|
+
// 2) PUT the bytes straight to the store. Non-2xx or network → degrade.
|
|
104
|
+
let putRes;
|
|
105
|
+
try {
|
|
106
|
+
putRes = await fetchFn(grant.upload_url, { method: "PUT", headers: putHeaders, body: input.body });
|
|
107
|
+
}
|
|
108
|
+
catch (err) {
|
|
109
|
+
return { kind: "degraded", reason: `content PUT failed: ${errMsg(err)}`, status: null };
|
|
110
|
+
}
|
|
111
|
+
if (!putRes.ok) {
|
|
112
|
+
return { kind: "degraded", reason: `content PUT rejected (status ${putRes.status})`, status: putRes.status };
|
|
113
|
+
}
|
|
114
|
+
return { kind: "uploaded", contentRef: grant.content_ref };
|
|
115
|
+
}
|
|
116
|
+
function errMsg(err) {
|
|
117
|
+
return err instanceof Error ? err.message : String(err);
|
|
118
|
+
}
|
|
119
|
+
function isStringRecord(v) {
|
|
120
|
+
return (typeof v === "object" &&
|
|
121
|
+
v !== null &&
|
|
122
|
+
!Array.isArray(v) &&
|
|
123
|
+
Object.values(v).every((x) => typeof x === "string"));
|
|
124
|
+
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Artifact / execution-graph namespace (Expansion E2) — the public barrel.
|
|
3
|
+
*
|
|
4
|
+
* Lives here (not spread through `index.ts`) so the SDK entrypoint stays under
|
|
5
|
+
* the LOC cap and the run/artifact surface has one home. Chunk A ships the run
|
|
6
|
+
* lifecycle + `@step`; chunk B adds the artifact primitives (save/load/evaluate).
|
|
7
|
+
*/
|
|
8
|
+
import type { ArtifactBuffer, RunNamespace } from "./run-context";
|
|
9
|
+
import type { ArtifactApi, ArtifactContentDeps } from "./artifact";
|
|
10
|
+
import type { ArtifactContentConfig, ArtifactsConfig } from "../config";
|
|
11
|
+
import type { ArtifactStore } from "../telemetry/artifact-store";
|
|
12
|
+
export { getActiveRunContext, createRunNamespace } from "./run-context";
|
|
13
|
+
export type { RunNamespace, RunHandle, RunStartOptions, RunEvaluateOptions, RunStatus, ActiveRunContext, ArtifactBuffer, GroundTruthStepInput, } from "./run-context";
|
|
14
|
+
export { step } from "./step";
|
|
15
|
+
export { createArtifactApi } from "./artifact";
|
|
16
|
+
export { openArtifactStore } from "../telemetry/artifact-store";
|
|
17
|
+
export type { ArtifactStore } from "../telemetry/artifact-store";
|
|
18
|
+
export type { ArtifactApi, ArtifactHandle, ArtifactSaveOptions, ArtifactEvaluateOptions, ArtifactPromoteOptions, ArtifactContentDeps, ArtifactContentUploader, } from "./artifact";
|
|
19
|
+
export { uploadArtifactContent } from "./content-uploader";
|
|
20
|
+
export type { ContentUploadConfig, ContentUploadInput, ContentUploadOutcome, } from "./content-uploader";
|
|
21
|
+
export { scrubContent, luhnValid } from "./scrub";
|
|
22
|
+
export type { ScrubMode, ScrubFinding, ScrubResult, ScrubDetectorType } from "./scrub";
|
|
23
|
+
export { computeDownstreamClosure, TAINT_DEFAULT_MAX_DEPTH, TAINT_DEFAULT_CAP, } from "./taint";
|
|
24
|
+
export type { TaintEdge, Supersession, TaintNode, TaintClosure, TaintOptions, } from "./taint";
|
|
25
|
+
export type { RunRow, StepRow, ArtifactRow, ArtifactEdgeRow, ArtifactEvaluationRow, EvalCaseRow, EvalCaseOrigin, } from "../types/telemetry";
|
|
26
|
+
/** The rig-bound artifact surface exposed as `rig.artifacts`: the run lifecycle
|
|
27
|
+
* (chunk A) + the artifact primitives (chunk B). */
|
|
28
|
+
export interface ArtifactNamespace {
|
|
29
|
+
readonly run: RunNamespace;
|
|
30
|
+
readonly artifact: ArtifactApi;
|
|
31
|
+
}
|
|
32
|
+
/** Build the rig-bound artifact namespace AND the exporter's post-flush retry
|
|
33
|
+
* hook from resolved config (E4 chunk 0, flag 2). Lives here so the SDK
|
|
34
|
+
* entrypoint stays under the LOC cap. The hook is present only when content
|
|
35
|
+
* custody is configured; otherwise it would be a no-op. */
|
|
36
|
+
export declare function wireRigArtifacts(deps: {
|
|
37
|
+
readonly buffer: ArtifactBuffer;
|
|
38
|
+
readonly enabled: boolean;
|
|
39
|
+
readonly artifactStore: ArtifactStore | null;
|
|
40
|
+
readonly artifacts?: ArtifactsConfig;
|
|
41
|
+
}): {
|
|
42
|
+
readonly namespace: ArtifactNamespace;
|
|
43
|
+
readonly onArtifactsFlushed?: () => Promise<void>;
|
|
44
|
+
};
|
|
45
|
+
/**
|
|
46
|
+
* Build the content-custody deps (E2-C) from resolved config, or undefined when
|
|
47
|
+
* custody is off. Kept here (not in the SDK entrypoint) so index.ts stays under
|
|
48
|
+
* the LOC cap (P1) and the content wiring lives beside the namespace it feeds.
|
|
49
|
+
*/
|
|
50
|
+
export declare function buildArtifactContentDeps(content: ArtifactContentConfig | undefined): ArtifactContentDeps | undefined;
|
|
51
|
+
/** Build the rig-bound artifact namespace. Disabled ⇒ every entry is inert
|
|
52
|
+
* (run.start is a no-op, save returns null, no rows written) — spec §1.4. The
|
|
53
|
+
* artifact store is present only when enabled (index.ts opens it then). */
|
|
54
|
+
export declare function createArtifactNamespace(deps: {
|
|
55
|
+
readonly buffer: ArtifactBuffer;
|
|
56
|
+
readonly artifactStore?: ArtifactStore;
|
|
57
|
+
readonly enabled: boolean;
|
|
58
|
+
/** Content-custody config (E2-C). Absent = metadata + hash only; the upload
|
|
59
|
+
* deps are built here (buildArtifactContentDeps) so the SDK entrypoint stays
|
|
60
|
+
* under the LOC cap. */
|
|
61
|
+
readonly content?: ArtifactContentConfig;
|
|
62
|
+
}): ArtifactNamespace;
|
|
63
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/artifacts/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAElE,OAAO,KAAK,EAAE,WAAW,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAGnE,OAAO,KAAK,EAAE,qBAAqB,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACxE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAEjE,OAAO,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACxE,YAAY,EACV,YAAY,EACZ,SAAS,EACT,eAAe,EACf,kBAAkB,EAClB,SAAS,EACT,gBAAgB,EAChB,cAAc,EACd,oBAAoB,GACrB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAC;AAC9B,OAAO,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AAChE,YAAY,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACjE,YAAY,EACV,WAAW,EACX,cAAc,EACd,mBAAmB,EACnB,uBAAuB,EACvB,sBAAsB,EACtB,mBAAmB,EACnB,uBAAuB,GACxB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,YAAY,EACV,mBAAmB,EACnB,kBAAkB,EAClB,oBAAoB,GACrB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAClD,YAAY,EAAE,SAAS,EAAE,YAAY,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AACvF,OAAO,EACL,wBAAwB,EACxB,uBAAuB,EACvB,iBAAiB,GAClB,MAAM,SAAS,CAAC;AACjB,YAAY,EACV,SAAS,EACT,YAAY,EACZ,SAAS,EACT,YAAY,EACZ,YAAY,GACb,MAAM,SAAS,CAAC;AACjB,YAAY,EACV,MAAM,EACN,OAAO,EACP,WAAW,EACX,eAAe,EACf,qBAAqB,EACrB,WAAW,EACX,cAAc,GACf,MAAM,oBAAoB,CAAC;AAE5B;oDACoD;AACpD,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,GAAG,EAAE,YAAY,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC;CAChC;AAED;;;2DAG2D;AAC3D,wBAAgB,gBAAgB,CAAC,IAAI,EAAE;IACrC,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,aAAa,EAAE,aAAa,GAAG,IAAI,CAAC;IAC7C,QAAQ,CAAC,SAAS,CAAC,EAAE,eAAe,CAAC;CACtC,GAAG;IAAE,QAAQ,CAAC,SAAS,EAAE,iBAAiB,CAAC;IAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;CAAE,CAY/F;AAED;;;;GAIG;AACH,wBAAgB,wBAAwB,CACtC,OAAO,EAAE,qBAAqB,GAAG,SAAS,GACzC,mBAAmB,GAAG,SAAS,CAWjC;AAED;;2EAE2E;AAC3E,wBAAgB,uBAAuB,CAAC,IAAI,EAAE;IAC5C,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,QAAQ,CAAC,aAAa,CAAC,EAAE,aAAa,CAAC;IACvC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B;;4BAEwB;IACxB,QAAQ,CAAC,OAAO,CAAC,EAAE,qBAAqB,CAAC;CAC1C,GAAG,iBAAiB,CAYpB"}
|