modelrig 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +94 -0
- package/README.md +41 -11
- package/data/PROVENANCE.md +19 -0
- package/data/pricing-manual.json +22 -0
- package/data/pricing-map.json +46362 -0
- package/data/pricing-overlay.json +419 -0
- package/dist/adapters/anthropic.d.ts +50 -6
- package/dist/adapters/anthropic.d.ts.map +1 -1
- package/dist/adapters/anthropic.js +123 -29
- package/dist/adapters/build-adapters.d.ts +11 -0
- package/dist/adapters/build-adapters.d.ts.map +1 -0
- package/dist/adapters/build-adapters.js +45 -0
- package/dist/adapters/deepseek.d.ts +6 -0
- package/dist/adapters/deepseek.d.ts.map +1 -1
- package/dist/adapters/deepseek.js +54 -1
- package/dist/adapters/gemini.d.ts.map +1 -1
- package/dist/adapters/gemini.js +46 -6
- package/dist/adapters/grok.d.ts +9 -0
- package/dist/adapters/grok.d.ts.map +1 -1
- package/dist/adapters/grok.js +67 -3
- package/dist/adapters/openai-compat.d.ts +6 -0
- package/dist/adapters/openai-compat.d.ts.map +1 -1
- package/dist/adapters/openai-compat.js +58 -1
- package/dist/adapters/openai-stream.d.ts +58 -0
- package/dist/adapters/openai-stream.d.ts.map +1 -0
- package/dist/adapters/openai-stream.js +120 -0
- package/dist/adapters/openai.d.ts +6 -0
- package/dist/adapters/openai.d.ts.map +1 -1
- package/dist/adapters/openai.js +60 -1
- package/dist/adapters/sampling.d.ts +28 -0
- package/dist/adapters/sampling.d.ts.map +1 -0
- package/dist/adapters/sampling.js +34 -0
- package/dist/adapters/tool-mapping.d.ts +92 -0
- package/dist/adapters/tool-mapping.d.ts.map +1 -0
- package/dist/adapters/tool-mapping.js +143 -0
- package/dist/artifacts/artifact.d.ts +137 -0
- package/dist/artifacts/artifact.d.ts.map +1 -0
- package/dist/artifacts/artifact.js +377 -0
- package/dist/artifacts/content-uploader.d.ts +87 -0
- package/dist/artifacts/content-uploader.d.ts.map +1 -0
- package/dist/artifacts/content-uploader.js +124 -0
- package/dist/artifacts/index.d.ts +63 -0
- package/dist/artifacts/index.d.ts.map +1 -0
- package/dist/artifacts/index.js +80 -0
- package/dist/artifacts/run-context.d.ts +129 -0
- package/dist/artifacts/run-context.d.ts.map +1 -0
- package/dist/artifacts/run-context.js +208 -0
- package/dist/artifacts/scrub.d.ts +60 -0
- package/dist/artifacts/scrub.d.ts.map +1 -0
- package/dist/artifacts/scrub.js +139 -0
- package/dist/artifacts/step.d.ts +26 -0
- package/dist/artifacts/step.d.ts.map +1 -0
- package/dist/artifacts/step.js +37 -0
- package/dist/artifacts/taint.d.ts +63 -0
- package/dist/artifacts/taint.d.ts.map +1 -0
- package/dist/artifacts/taint.js +76 -0
- package/dist/bakeoff/cache-sim.d.ts +99 -0
- package/dist/bakeoff/cache-sim.d.ts.map +1 -0
- package/dist/bakeoff/cache-sim.js +126 -0
- package/dist/bakeoff/eval-cases.d.ts +96 -0
- package/dist/bakeoff/eval-cases.d.ts.map +1 -0
- package/dist/bakeoff/eval-cases.js +105 -0
- package/dist/bakeoff/report.d.ts +41 -0
- package/dist/bakeoff/report.d.ts.map +1 -0
- package/dist/bakeoff/report.js +142 -0
- package/dist/bakeoff/samples.d.ts +15 -0
- package/dist/bakeoff/samples.d.ts.map +1 -1
- package/dist/bakeoff/samples.js +20 -0
- package/dist/cli.d.ts +1 -3
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +134 -116
- package/dist/config.d.ts +100 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +104 -0
- package/dist/deprecations.d.ts +32 -0
- package/dist/deprecations.d.ts.map +1 -0
- package/dist/deprecations.js +59 -0
- package/dist/gateway/http-protocol.d.ts +175 -0
- package/dist/gateway/http-protocol.d.ts.map +1 -0
- package/dist/gateway/http-protocol.js +105 -0
- package/dist/index.d.ts +105 -18
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +266 -138
- package/dist/init/finder.d.ts +15 -0
- package/dist/init/finder.d.ts.map +1 -1
- package/dist/init/finder.js +46 -0
- package/dist/init/init-cli.d.ts +10 -0
- package/dist/init/init-cli.d.ts.map +1 -0
- package/dist/init/init-cli.js +86 -0
- package/dist/init/observe-cli.d.ts +19 -0
- package/dist/init/observe-cli.d.ts.map +1 -0
- package/dist/init/observe-cli.js +73 -0
- package/dist/init/observe.d.ts +81 -0
- package/dist/init/observe.d.ts.map +1 -0
- package/dist/init/observe.js +245 -0
- package/dist/registry/build.d.ts +94 -1
- package/dist/registry/build.d.ts.map +1 -1
- package/dist/registry/build.js +125 -4
- package/dist/registry/cache-policy.d.ts +94 -0
- package/dist/registry/cache-policy.d.ts.map +1 -0
- package/dist/registry/cache-policy.js +238 -0
- package/dist/registry/call-notes.d.ts.map +1 -1
- package/dist/registry/call-notes.js +40 -2
- package/dist/registry/effective-rate.d.ts +151 -0
- package/dist/registry/effective-rate.d.ts.map +1 -0
- package/dist/registry/effective-rate.js +156 -0
- package/dist/registry/facts.d.ts.map +1 -1
- package/dist/registry/facts.js +16 -0
- package/dist/registry/parity.d.ts +26 -0
- package/dist/registry/parity.d.ts.map +1 -1
- package/dist/registry/parity.js +43 -2
- package/dist/registry/pricing-shape.d.ts +113 -0
- package/dist/registry/pricing-shape.d.ts.map +1 -0
- package/dist/registry/pricing-shape.js +123 -0
- package/dist/registry/pricing.d.ts +25 -1
- package/dist/registry/pricing.d.ts.map +1 -1
- package/dist/registry/pricing.js +64 -6
- package/dist/rigindex/bootstrap.d.ts +36 -0
- package/dist/rigindex/bootstrap.d.ts.map +1 -0
- package/dist/rigindex/bootstrap.js +97 -0
- package/dist/rigindex/fit.d.ts +24 -0
- package/dist/rigindex/fit.d.ts.map +1 -0
- package/dist/rigindex/fit.js +76 -0
- package/dist/rigindex/gates.d.ts +52 -0
- package/dist/rigindex/gates.d.ts.map +1 -0
- package/dist/rigindex/gates.js +109 -0
- package/dist/rigindex/graph.d.ts +31 -0
- package/dist/rigindex/graph.d.ts.map +1 -0
- package/dist/rigindex/graph.js +120 -0
- package/dist/rigindex/index.d.ts +23 -0
- package/dist/rigindex/index.d.ts.map +1 -0
- package/dist/rigindex/index.js +112 -0
- package/dist/rigindex/types.d.ts +76 -0
- package/dist/rigindex/types.d.ts.map +1 -0
- package/dist/rigindex/types.js +8 -0
- package/dist/routes/cache-lint.d.ts +41 -0
- package/dist/routes/cache-lint.d.ts.map +1 -0
- package/dist/routes/cache-lint.js +81 -0
- package/dist/routes/loader.d.ts +37 -0
- package/dist/routes/loader.d.ts.map +1 -1
- package/dist/routes/loader.js +138 -0
- package/dist/routes/resolved-bundle.d.ts +79 -0
- package/dist/routes/resolved-bundle.d.ts.map +1 -0
- package/dist/routes/resolved-bundle.js +148 -0
- package/dist/routes/sampling-config.d.ts +32 -0
- package/dist/routes/sampling-config.d.ts.map +1 -0
- package/dist/routes/sampling-config.js +91 -0
- package/dist/run/ajv.d.ts +22 -0
- package/dist/run/ajv.d.ts.map +1 -0
- package/dist/run/ajv.js +32 -0
- package/dist/run/cache-handle.d.ts +58 -0
- package/dist/run/cache-handle.d.ts.map +1 -0
- package/dist/run/cache-handle.js +54 -0
- package/dist/run/cache-stamp.d.ts +65 -0
- package/dist/run/cache-stamp.d.ts.map +1 -0
- package/dist/run/cache-stamp.js +0 -0
- package/dist/run/capture-write.d.ts +33 -0
- package/dist/run/capture-write.d.ts.map +1 -0
- package/dist/run/capture-write.js +55 -0
- package/dist/run/cost.d.ts.map +1 -1
- package/dist/run/cost.js +1 -1
- package/dist/run/dispatch.d.ts +99 -0
- package/dist/run/dispatch.d.ts.map +1 -0
- package/dist/run/dispatch.js +191 -0
- package/dist/run/raw.d.ts +88 -0
- package/dist/run/raw.d.ts.map +1 -0
- package/dist/run/raw.js +286 -0
- package/dist/run/record-attempt.d.ts +40 -0
- package/dist/run/record-attempt.d.ts.map +1 -0
- package/dist/run/record-attempt.js +99 -0
- package/dist/run/resolve.d.ts +75 -2
- package/dist/run/resolve.d.ts.map +1 -1
- package/dist/run/resolve.js +126 -4
- package/dist/run/run.d.ts +14 -0
- package/dist/run/run.d.ts.map +1 -1
- package/dist/run/run.js +0 -0
- package/dist/run/stream.d.ts +49 -0
- package/dist/run/stream.d.ts.map +1 -0
- package/dist/run/stream.js +87 -0
- package/dist/run/validate.d.ts.map +1 -1
- package/dist/run/validate.js +6 -7
- package/dist/savings/cache-ledger.d.ts +108 -0
- package/dist/savings/cache-ledger.d.ts.map +1 -0
- package/dist/savings/cache-ledger.js +174 -0
- package/dist/savings/cache-reconcile.d.ts +64 -0
- package/dist/savings/cache-reconcile.d.ts.map +1 -0
- package/dist/savings/cache-reconcile.js +59 -0
- package/dist/savings/cache-stats.d.ts +63 -0
- package/dist/savings/cache-stats.d.ts.map +1 -0
- package/dist/savings/cache-stats.js +108 -0
- package/dist/savings/emit.d.ts +13 -0
- package/dist/savings/emit.d.ts.map +1 -1
- package/dist/savings/emit.js +25 -0
- package/dist/savings/swap-execute.d.ts +14 -0
- package/dist/savings/swap-execute.d.ts.map +1 -1
- package/dist/savings/swap-execute.js +44 -1
- package/dist/savings/swaps-client.d.ts +14 -0
- package/dist/savings/swaps-client.d.ts.map +1 -1
- package/dist/savings/swaps-client.js +16 -2
- package/dist/settings/api-fetcher.d.ts +39 -0
- package/dist/settings/api-fetcher.d.ts.map +1 -0
- package/dist/settings/api-fetcher.js +92 -0
- package/dist/settings/index.d.ts +10 -0
- package/dist/settings/index.d.ts.map +1 -0
- package/dist/settings/index.js +15 -0
- package/dist/settings/task-settings.d.ts +91 -0
- package/dist/settings/task-settings.d.ts.map +1 -0
- package/dist/settings/task-settings.js +116 -0
- package/dist/status.d.ts +77 -0
- package/dist/status.d.ts.map +1 -0
- package/dist/status.js +113 -0
- package/dist/telemetry/artifact-store.d.ts +48 -0
- package/dist/telemetry/artifact-store.d.ts.map +1 -0
- package/dist/telemetry/artifact-store.js +189 -0
- package/dist/telemetry/cache-miss-streak.d.ts +79 -0
- package/dist/telemetry/cache-miss-streak.d.ts.map +1 -0
- package/dist/telemetry/cache-miss-streak.js +99 -0
- package/dist/telemetry/capture.d.ts +3 -0
- package/dist/telemetry/capture.d.ts.map +1 -1
- package/dist/telemetry/capture.js +15 -8
- package/dist/telemetry/exporter-setup.d.ts +28 -0
- package/dist/telemetry/exporter-setup.d.ts.map +1 -0
- package/dist/telemetry/exporter-setup.js +34 -0
- package/dist/telemetry/exporter.d.ts +75 -0
- package/dist/telemetry/exporter.d.ts.map +1 -1
- package/dist/telemetry/exporter.js +324 -47
- package/dist/telemetry/health.d.ts +76 -0
- package/dist/telemetry/health.d.ts.map +1 -0
- package/dist/telemetry/health.js +92 -0
- package/dist/telemetry/ingest-protocol.d.ts +152 -3
- package/dist/telemetry/ingest-protocol.d.ts.map +1 -1
- package/dist/telemetry/ingest-protocol.js +204 -4
- package/dist/telemetry/load-error-report.d.ts +24 -0
- package/dist/telemetry/load-error-report.d.ts.map +1 -0
- package/dist/telemetry/load-error-report.js +50 -0
- package/dist/telemetry/otel-map.d.ts +121 -0
- package/dist/telemetry/otel-map.d.ts.map +1 -0
- package/dist/telemetry/otel-map.js +224 -0
- package/dist/telemetry/otel-normalize.d.ts +104 -0
- package/dist/telemetry/otel-normalize.d.ts.map +1 -0
- package/dist/telemetry/otel-normalize.js +530 -0
- package/dist/telemetry/otlp-emitter.d.ts +52 -0
- package/dist/telemetry/otlp-emitter.d.ts.map +1 -0
- package/dist/telemetry/otlp-emitter.js +144 -0
- package/dist/telemetry/otlp-proto.d.ts +29 -0
- package/dist/telemetry/otlp-proto.d.ts.map +1 -0
- package/dist/telemetry/otlp-proto.js +266 -0
- package/dist/telemetry/sinks.d.ts +37 -1
- package/dist/telemetry/sinks.d.ts.map +1 -1
- package/dist/telemetry/sinks.js +121 -30
- package/dist/telemetry/store.d.ts +28 -1
- package/dist/telemetry/store.d.ts.map +1 -1
- package/dist/telemetry/store.js +218 -3
- package/dist/tenancy/api-key-format.d.ts +15 -5
- package/dist/tenancy/api-key-format.d.ts.map +1 -1
- package/dist/tenancy/api-key-format.js +24 -5
- package/dist/types/config-teaching.d.ts +62 -0
- package/dist/types/config-teaching.d.ts.map +1 -0
- package/dist/types/config-teaching.js +183 -0
- package/dist/types/provider.d.ts +122 -1
- package/dist/types/provider.d.ts.map +1 -1
- package/dist/types/provider.js +21 -1
- package/dist/types/route.d.ts +79 -1
- package/dist/types/route.d.ts.map +1 -1
- package/dist/types/telemetry.d.ts +117 -0
- package/dist/types/telemetry.d.ts.map +1 -1
- package/dist/validate-cli.d.ts +12 -0
- package/dist/validate-cli.d.ts.map +1 -0
- package/dist/validate-cli.js +57 -0
- package/dist/verify/capture.d.ts +66 -0
- package/dist/verify/capture.d.ts.map +1 -0
- package/dist/verify/capture.js +180 -0
- package/dist/verify/cli-run.d.ts +23 -0
- package/dist/verify/cli-run.d.ts.map +1 -0
- package/dist/verify/cli-run.js +57 -0
- package/dist/verify/diff.d.ts +48 -0
- package/dist/verify/diff.d.ts.map +1 -0
- package/dist/verify/diff.js +157 -0
- package/dist/verify/index.d.ts +17 -0
- package/dist/verify/index.d.ts.map +1 -0
- package/dist/verify/index.js +24 -0
- package/dist/verify/report.d.ts +14 -0
- package/dist/verify/report.d.ts.map +1 -0
- package/dist/verify/report.js +88 -0
- package/dist/verify/types.d.ts +101 -0
- package/dist/verify/types.d.ts.map +1 -0
- package/dist/verify/types.js +17 -0
- package/dist/verify/verify.d.ts +29 -0
- package/dist/verify/verify.d.ts.map +1 -0
- package/dist/verify/verify.js +148 -0
- package/dist/watch/evaluate.d.ts +162 -3
- package/dist/watch/evaluate.d.ts.map +1 -1
- package/dist/watch/evaluate.js +61 -0
- package/dist/watch/monitor.d.ts +116 -0
- package/dist/watch/monitor.d.ts.map +1 -0
- package/dist/watch/monitor.js +199 -0
- package/dist/watch/proposals.d.ts +20 -4
- package/dist/watch/proposals.d.ts.map +1 -1
- package/dist/watch/proposals.js +31 -3
- package/dist/watch/run.d.ts +33 -0
- package/dist/watch/run.d.ts.map +1 -1
- package/dist/watch/run.js +39 -0
- package/package.json +7 -1
- package/registry/LICENSE +20 -0
- package/registry/leaderboard.json +3639 -0
- package/registry/registry.json +7263 -0
- package/dist/telemetry/supabase-exporter.d.ts +0 -64
- package/dist/telemetry/supabase-exporter.d.ts.map +0 -1
- package/dist/telemetry/supabase-exporter.js +0 -211
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Artifact / execution-graph namespace (Expansion E2) — the public barrel.
|
|
4
|
+
*
|
|
5
|
+
* Lives here (not spread through `index.ts`) so the SDK entrypoint stays under
|
|
6
|
+
* the LOC cap and the run/artifact surface has one home. Chunk A ships the run
|
|
7
|
+
* lifecycle + `@step`; chunk B adds the artifact primitives (save/load/evaluate).
|
|
8
|
+
*/
|
|
9
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
|
+
exports.TAINT_DEFAULT_CAP = exports.TAINT_DEFAULT_MAX_DEPTH = exports.computeDownstreamClosure = exports.luhnValid = exports.scrubContent = exports.uploadArtifactContent = exports.openArtifactStore = exports.createArtifactApi = exports.step = exports.createRunNamespace = exports.getActiveRunContext = void 0;
|
|
11
|
+
exports.wireRigArtifacts = wireRigArtifacts;
|
|
12
|
+
exports.buildArtifactContentDeps = buildArtifactContentDeps;
|
|
13
|
+
exports.createArtifactNamespace = createArtifactNamespace;
|
|
14
|
+
const run_context_1 = require("./run-context");
|
|
15
|
+
const artifact_1 = require("./artifact");
|
|
16
|
+
const content_uploader_1 = require("./content-uploader");
|
|
17
|
+
var run_context_2 = require("./run-context");
|
|
18
|
+
Object.defineProperty(exports, "getActiveRunContext", { enumerable: true, get: function () { return run_context_2.getActiveRunContext; } });
|
|
19
|
+
Object.defineProperty(exports, "createRunNamespace", { enumerable: true, get: function () { return run_context_2.createRunNamespace; } });
|
|
20
|
+
var step_1 = require("./step");
|
|
21
|
+
Object.defineProperty(exports, "step", { enumerable: true, get: function () { return step_1.step; } });
|
|
22
|
+
var artifact_2 = require("./artifact");
|
|
23
|
+
Object.defineProperty(exports, "createArtifactApi", { enumerable: true, get: function () { return artifact_2.createArtifactApi; } });
|
|
24
|
+
var artifact_store_1 = require("../telemetry/artifact-store");
|
|
25
|
+
Object.defineProperty(exports, "openArtifactStore", { enumerable: true, get: function () { return artifact_store_1.openArtifactStore; } });
|
|
26
|
+
var content_uploader_2 = require("./content-uploader");
|
|
27
|
+
Object.defineProperty(exports, "uploadArtifactContent", { enumerable: true, get: function () { return content_uploader_2.uploadArtifactContent; } });
|
|
28
|
+
var scrub_1 = require("./scrub");
|
|
29
|
+
Object.defineProperty(exports, "scrubContent", { enumerable: true, get: function () { return scrub_1.scrubContent; } });
|
|
30
|
+
Object.defineProperty(exports, "luhnValid", { enumerable: true, get: function () { return scrub_1.luhnValid; } });
|
|
31
|
+
var taint_1 = require("./taint");
|
|
32
|
+
Object.defineProperty(exports, "computeDownstreamClosure", { enumerable: true, get: function () { return taint_1.computeDownstreamClosure; } });
|
|
33
|
+
Object.defineProperty(exports, "TAINT_DEFAULT_MAX_DEPTH", { enumerable: true, get: function () { return taint_1.TAINT_DEFAULT_MAX_DEPTH; } });
|
|
34
|
+
Object.defineProperty(exports, "TAINT_DEFAULT_CAP", { enumerable: true, get: function () { return taint_1.TAINT_DEFAULT_CAP; } });
|
|
35
|
+
/** Build the rig-bound artifact namespace AND the exporter's post-flush retry
|
|
36
|
+
* hook from resolved config (E4 chunk 0, flag 2). Lives here so the SDK
|
|
37
|
+
* entrypoint stays under the LOC cap. The hook is present only when content
|
|
38
|
+
* custody is configured; otherwise it would be a no-op. */
|
|
39
|
+
function wireRigArtifacts(deps) {
|
|
40
|
+
const namespace = createArtifactNamespace({
|
|
41
|
+
buffer: deps.buffer,
|
|
42
|
+
enabled: deps.enabled,
|
|
43
|
+
...(deps.artifactStore !== null ? { artifactStore: deps.artifactStore } : {}),
|
|
44
|
+
...(deps.artifacts?.content !== undefined ? { content: deps.artifacts.content } : {}),
|
|
45
|
+
});
|
|
46
|
+
const hookOn = deps.enabled && deps.artifacts?.content !== undefined;
|
|
47
|
+
return {
|
|
48
|
+
namespace,
|
|
49
|
+
...(hookOn ? { onArtifactsFlushed: () => namespace.artifact.retryPendingContent() } : {}),
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Build the content-custody deps (E2-C) from resolved config, or undefined when
|
|
54
|
+
* custody is off. Kept here (not in the SDK entrypoint) so index.ts stays under
|
|
55
|
+
* the LOC cap (P1) and the content wiring lives beside the namespace it feeds.
|
|
56
|
+
*/
|
|
57
|
+
function buildArtifactContentDeps(content) {
|
|
58
|
+
if (content === undefined)
|
|
59
|
+
return undefined;
|
|
60
|
+
return {
|
|
61
|
+
scrub: content.scrub,
|
|
62
|
+
upload: (input) => (0, content_uploader_1.uploadArtifactContent)(input, { uploadsUrl: content.uploadsUrl, apiKey: content.apiKey, maxBytes: content.maxBytes }, fetch),
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
/** Build the rig-bound artifact namespace. Disabled ⇒ every entry is inert
|
|
66
|
+
* (run.start is a no-op, save returns null, no rows written) — spec §1.4. The
|
|
67
|
+
* artifact store is present only when enabled (index.ts opens it then). */
|
|
68
|
+
function createArtifactNamespace(deps) {
|
|
69
|
+
const contentDeps = deps.enabled ? buildArtifactContentDeps(deps.content) : undefined;
|
|
70
|
+
return {
|
|
71
|
+
run: (0, run_context_1.createRunNamespace)(deps),
|
|
72
|
+
artifact: (0, artifact_1.createArtifactApi)({
|
|
73
|
+
// A disabled namespace never opens a store; createArtifactApi(enabled:false)
|
|
74
|
+
// ignores the store and returns an inert api, so the cast is never reached.
|
|
75
|
+
store: deps.artifactStore,
|
|
76
|
+
enabled: deps.enabled && deps.artifactStore !== undefined,
|
|
77
|
+
...(contentDeps !== undefined ? { content: contentDeps } : {}),
|
|
78
|
+
}),
|
|
79
|
+
};
|
|
80
|
+
}
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Run context — Expansion E2 chunk A (spec §1.4, §9).
|
|
3
|
+
*
|
|
4
|
+
* A RUN is one execution of a customer pipeline; instrumentation hangs off an
|
|
5
|
+
* ambient run context held in an `AsyncLocalStorage` (a Node builtin — NO new
|
|
6
|
+
* npm dependency). `run.start()` mints the context and `run.end()` closes it;
|
|
7
|
+
* everything that happens in between — `rig.run()` auto-steps, `@step` scopes,
|
|
8
|
+
* and (chunk B) artifact save/load lineage — reads the SAME ambient context.
|
|
9
|
+
*
|
|
10
|
+
* ── FAIL-OPEN, ALWAYS ─────────────────────────────────────────────────────
|
|
11
|
+
* Instrumentation NEVER changes return values and NEVER throws into customer
|
|
12
|
+
* code (the exporter's swallow-and-log doctrine, extended). A telemetry-buffer
|
|
13
|
+
* write that fails is logged and dropped; the customer's pipeline is untouched.
|
|
14
|
+
*
|
|
15
|
+
* ── GATING ────────────────────────────────────────────────────────────────
|
|
16
|
+
* The namespace is inert unless the rig enabled it (`MODELRIG_ARTIFACTS=1`).
|
|
17
|
+
* A disabled namespace's `run.start()` sets NO context, so `getActiveRunContext`
|
|
18
|
+
* returns null everywhere and the run loop's auto-step is never reached —
|
|
19
|
+
* v1 wire, zero new rows, zero behaviour change (spec §1.4).
|
|
20
|
+
*/
|
|
21
|
+
import type { ArtifactEvaluationRow, RunRow, StepRow } from "../types/telemetry";
|
|
22
|
+
/** The narrow buffer the run/artifact namespace writes through — the local
|
|
23
|
+
* telemetry store satisfies it. Kept narrow so this module depends on the row
|
|
24
|
+
* shapes, not the whole store surface (and cannot reach captures). */
|
|
25
|
+
export interface ArtifactBuffer {
|
|
26
|
+
recordRun(row: RunRow): void;
|
|
27
|
+
recordStep(row: StepRow): void;
|
|
28
|
+
}
|
|
29
|
+
export type RunStatus = RunRow["status"];
|
|
30
|
+
export interface RunStartOptions {
|
|
31
|
+
/** Customer's pipeline name — the run's semantic identity. */
|
|
32
|
+
readonly pipeline: string;
|
|
33
|
+
readonly pipelineVersion?: string;
|
|
34
|
+
readonly environment?: string;
|
|
35
|
+
/** Bridges feedback.episode_key (run-level verdicts). Optional. */
|
|
36
|
+
readonly episodeKey?: string;
|
|
37
|
+
/** W3C traceparent for cross-service correlation (spec §1.4). When present the
|
|
38
|
+
* run adopts its trace id, so artifacts crossing a service boundary (chunk B)
|
|
39
|
+
* can be tied back to this run. Never changes the (uuid) run id. */
|
|
40
|
+
readonly traceparent?: string;
|
|
41
|
+
/** Zero-retention run (chunk B, ratified invariant): NO artifact rows of any
|
|
42
|
+
* kind are emitted — artifact.save refuses fail-closed at the SDK gate with a
|
|
43
|
+
* counted, logged skip. Set by the caller or derived from a route's
|
|
44
|
+
* `require: zero_retention`. */
|
|
45
|
+
readonly zeroRetention?: boolean;
|
|
46
|
+
}
|
|
47
|
+
/** One ground-truth step attempt the run loop asks the context to record. The
|
|
48
|
+
* context assigns `attempt_no` per step key so the caller need not track it. */
|
|
49
|
+
export interface GroundTruthStepInput {
|
|
50
|
+
readonly stepKey: string;
|
|
51
|
+
readonly task: string | null;
|
|
52
|
+
readonly inferenceId: string;
|
|
53
|
+
readonly startedAt: string;
|
|
54
|
+
readonly endedAt: string;
|
|
55
|
+
readonly failureClass: string | null;
|
|
56
|
+
}
|
|
57
|
+
export interface ActiveRunContext {
|
|
58
|
+
readonly runId: string;
|
|
59
|
+
/** Trace id adopted from a traceparent at run.start, if any (chunk B use). */
|
|
60
|
+
readonly traceId: string | null;
|
|
61
|
+
/** Zero-retention gate (chunk B): true ⇒ artifact.save refuses fail-closed. */
|
|
62
|
+
readonly zeroRetention: boolean;
|
|
63
|
+
/** Record that an artifact was loaded in the current run (chunk B lineage). */
|
|
64
|
+
noteLoad(artifactId: string): void;
|
|
65
|
+
/** Return AND clear the loaded-artifact ids — the set a subsequent save
|
|
66
|
+
* derives `derived_from` edges to (loads "consumed" by that save). v0
|
|
67
|
+
* lineage: run-scoped, cleared on each save; coverage measured, never promised. */
|
|
68
|
+
takeLoadedIds(): string[];
|
|
69
|
+
/** Mint a unique step key for ONE rig.run()/@step invocation. Retries of that
|
|
70
|
+
* invocation reuse the returned key (attempt_no distinguishes them); a later
|
|
71
|
+
* invocation gets a fresh key, so two calls never collapse into one step. */
|
|
72
|
+
nextStepKey(label: string): string;
|
|
73
|
+
/** Record one ground-truth step attempt (the rig.run auto-step). attempt_no is
|
|
74
|
+
* assigned here, per step key. Fail-open — a buffer error is logged, dropped,
|
|
75
|
+
* and never thrown into the run loop. */
|
|
76
|
+
emitGroundTruthStep(input: GroundTruthStepInput): void;
|
|
77
|
+
}
|
|
78
|
+
/** The active run context, or null when none is set (namespace disabled, or no
|
|
79
|
+
* run.start on this async path). Read by the run loop's auto-step seam. */
|
|
80
|
+
export declare function getActiveRunContext(): ActiveRunContext | null;
|
|
81
|
+
/** A run-level evaluation the caller attaches (E5 chunk V.2) — the run-subject
|
|
82
|
+
* mirror of ArtifactEvaluateOptions. The subject is the run, so there is no ref. */
|
|
83
|
+
export interface RunEvaluateOptions {
|
|
84
|
+
readonly evaluator: string;
|
|
85
|
+
readonly kind: "deterministic" | "model-judge" | "human";
|
|
86
|
+
readonly score?: number;
|
|
87
|
+
readonly passed?: boolean;
|
|
88
|
+
readonly label?: string;
|
|
89
|
+
readonly explanation?: string;
|
|
90
|
+
}
|
|
91
|
+
/** A handle a caller holds to end a run it started. */
|
|
92
|
+
export interface RunHandle {
|
|
93
|
+
readonly runId: string;
|
|
94
|
+
end(status?: RunStatus): void;
|
|
95
|
+
/** Attach a run-level evaluation to THIS run (E5 V.2). Works after end() —
|
|
96
|
+
* verdicts are often post-hoc. Fail-open + inert when the namespace is off. */
|
|
97
|
+
evaluate(opts: RunEvaluateOptions): void;
|
|
98
|
+
}
|
|
99
|
+
/** The `run` namespace, bound to one rig's buffer + gate. Disabled ⇒ inert. */
|
|
100
|
+
export interface RunNamespace {
|
|
101
|
+
/** Begin a run via the imperative handle API. Uses `AsyncLocalStorage.enterWith`,
|
|
102
|
+
* which has NO lexical exit — safe for one run at a time on a request/async
|
|
103
|
+
* path, but CONCURRENT overlapping `start()` calls (e.g. `Promise.all` over
|
|
104
|
+
* items) clobber each other's ambient context and cross-attribute steps.
|
|
105
|
+
* For concurrent runs use `scope()`. Disabled ⇒ no-op inert handle. */
|
|
106
|
+
start(opts: RunStartOptions): RunHandle;
|
|
107
|
+
/** End the active run with a terminal status (default "succeeded"). */
|
|
108
|
+
end(status?: RunStatus): void;
|
|
109
|
+
/** Attach a run-level evaluation to the ACTIVE run (E5 V.2) — the run-subject
|
|
110
|
+
* mirror of `artifact.evaluate`. No active run ⇒ a warned no-op (fail-open). */
|
|
111
|
+
evaluate(opts: RunEvaluateOptions): void;
|
|
112
|
+
/** Concurrency-safe run scope. Runs `fn` inside `AsyncLocalStorage.run`, so
|
|
113
|
+
* overlapping `scope()` calls each keep their OWN ambient run context — the
|
|
114
|
+
* correct primitive for `Promise.all` over items. Ends the run "succeeded" on
|
|
115
|
+
* return, "failed" if `fn` throws (then rethrows). Disabled ⇒ runs `fn` with
|
|
116
|
+
* no context. */
|
|
117
|
+
scope<T>(opts: RunStartOptions, fn: () => Promise<T>): Promise<T>;
|
|
118
|
+
}
|
|
119
|
+
export declare function createRunNamespace(deps: {
|
|
120
|
+
readonly buffer: ArtifactBuffer;
|
|
121
|
+
readonly enabled: boolean;
|
|
122
|
+
/** The evaluation writer (E5 V.2) — the artifact store's recordArtifactEvaluation.
|
|
123
|
+
* Absent (disabled / no store) ⇒ run.evaluate is inert. Kept OFF the narrow
|
|
124
|
+
* ArtifactBuffer so that contract stays "cannot reach anything but run/step". */
|
|
125
|
+
readonly artifactStore?: {
|
|
126
|
+
recordArtifactEvaluation(row: ArtifactEvaluationRow): void;
|
|
127
|
+
};
|
|
128
|
+
}): RunNamespace;
|
|
129
|
+
//# sourceMappingURL=run-context.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run-context.d.ts","sourceRoot":"","sources":["../../src/artifacts/run-context.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAIH,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAGjF;;sEAEsE;AACtE,MAAM,WAAW,cAAc;IAC7B,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,UAAU,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI,CAAC;CAChC;AAED,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;AAEzC,MAAM,WAAW,eAAe;IAC9B,8DAA8D;IAC9D,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,mEAAmE;IACnE,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B;;wEAEoE;IACpE,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B;;;oCAGgC;IAChC,QAAQ,CAAC,aAAa,CAAC,EAAE,OAAO,CAAC;CAClC;AAED;gFACgF;AAChF,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CACtC;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,8EAA8E;IAC9E,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,+EAA+E;IAC/E,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;IAChC,+EAA+E;IAC/E,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC;;uFAEmF;IACnF,aAAa,IAAI,MAAM,EAAE,CAAC;IAC1B;;iFAE6E;IAC7E,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC;IACnC;;6CAEyC;IACzC,mBAAmB,CAAC,KAAK,EAAE,oBAAoB,GAAG,IAAI,CAAC;CACxD;AAaD;2EAC2E;AAC3E,wBAAgB,mBAAmB,IAAI,gBAAgB,GAAG,IAAI,CAG7D;AAED;oFACoF;AACpF,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,IAAI,EAAE,eAAe,GAAG,aAAa,GAAG,OAAO,CAAC;IACzD,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CAC/B;AAED,uDAAuD;AACvD,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,GAAG,CAAC,MAAM,CAAC,EAAE,SAAS,GAAG,IAAI,CAAC;IAC9B;mFAC+E;IAC/E,QAAQ,CAAC,IAAI,EAAE,kBAAkB,GAAG,IAAI,CAAC;CAC1C;AAED,+EAA+E;AAC/E,MAAM,WAAW,YAAY;IAC3B;;;;2EAIuE;IACvE,KAAK,CAAC,IAAI,EAAE,eAAe,GAAG,SAAS,CAAC;IACxC,uEAAuE;IACvE,GAAG,CAAC,MAAM,CAAC,EAAE,SAAS,GAAG,IAAI,CAAC;IAC9B;oFACgF;IAChF,QAAQ,CAAC,IAAI,EAAE,kBAAkB,GAAG,IAAI,CAAC;IACzC;;;;qBAIiB;IACjB,KAAK,CAAC,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;CACnE;AASD,wBAAgB,kBAAkB,CAAC,IAAI,EAAE;IACvC,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B;;qFAEiF;IACjF,QAAQ,CAAC,aAAa,CAAC,EAAE;QAAE,wBAAwB,CAAC,GAAG,EAAE,qBAAqB,GAAG,IAAI,CAAA;KAAE,CAAC;CACzF,GAAG,YAAY,CAyKf"}
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Run context — Expansion E2 chunk A (spec §1.4, §9).
|
|
4
|
+
*
|
|
5
|
+
* A RUN is one execution of a customer pipeline; instrumentation hangs off an
|
|
6
|
+
* ambient run context held in an `AsyncLocalStorage` (a Node builtin — NO new
|
|
7
|
+
* npm dependency). `run.start()` mints the context and `run.end()` closes it;
|
|
8
|
+
* everything that happens in between — `rig.run()` auto-steps, `@step` scopes,
|
|
9
|
+
* and (chunk B) artifact save/load lineage — reads the SAME ambient context.
|
|
10
|
+
*
|
|
11
|
+
* ── FAIL-OPEN, ALWAYS ─────────────────────────────────────────────────────
|
|
12
|
+
* Instrumentation NEVER changes return values and NEVER throws into customer
|
|
13
|
+
* code (the exporter's swallow-and-log doctrine, extended). A telemetry-buffer
|
|
14
|
+
* write that fails is logged and dropped; the customer's pipeline is untouched.
|
|
15
|
+
*
|
|
16
|
+
* ── GATING ────────────────────────────────────────────────────────────────
|
|
17
|
+
* The namespace is inert unless the rig enabled it (`MODELRIG_ARTIFACTS=1`).
|
|
18
|
+
* A disabled namespace's `run.start()` sets NO context, so `getActiveRunContext`
|
|
19
|
+
* returns null everywhere and the run loop's auto-step is never reached —
|
|
20
|
+
* v1 wire, zero new rows, zero behaviour change (spec §1.4).
|
|
21
|
+
*/
|
|
22
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
23
|
+
exports.getActiveRunContext = getActiveRunContext;
|
|
24
|
+
exports.createRunNamespace = createRunNamespace;
|
|
25
|
+
const node_async_hooks_1 = require("node:async_hooks");
|
|
26
|
+
const node_crypto_1 = require("node:crypto");
|
|
27
|
+
const otel_map_1 = require("../telemetry/otel-map");
|
|
28
|
+
const storage = new node_async_hooks_1.AsyncLocalStorage();
|
|
29
|
+
/** The active run context, or null when none is set (namespace disabled, or no
|
|
30
|
+
* run.start on this async path). Read by the run loop's auto-step seam. */
|
|
31
|
+
function getActiveRunContext() {
|
|
32
|
+
const ctx = storage.getStore();
|
|
33
|
+
return ctx !== undefined && !ctx.ended ? ctx : null;
|
|
34
|
+
}
|
|
35
|
+
function warnOnce(scope, err) {
|
|
36
|
+
console.warn(`[modelrig artifacts] ${scope} buffer write failed (dropped, run unaffected): ` +
|
|
37
|
+
`${err instanceof Error ? err.message : String(err)}`);
|
|
38
|
+
}
|
|
39
|
+
function createRunNamespace(deps) {
|
|
40
|
+
const { buffer, enabled } = deps;
|
|
41
|
+
const evalStore = deps.artifactStore;
|
|
42
|
+
/** Write ONE run-subject evaluation (artifactId null, runId set). Fail-open. */
|
|
43
|
+
function writeRunEvaluation(runId, opts) {
|
|
44
|
+
if (!enabled || evalStore === undefined || runId === "")
|
|
45
|
+
return;
|
|
46
|
+
try {
|
|
47
|
+
evalStore.recordArtifactEvaluation({
|
|
48
|
+
id: (0, node_crypto_1.randomUUID)(),
|
|
49
|
+
artifactId: null,
|
|
50
|
+
runId,
|
|
51
|
+
evaluator: opts.evaluator,
|
|
52
|
+
evaluatorKind: opts.kind,
|
|
53
|
+
score: opts.score ?? null,
|
|
54
|
+
passed: opts.passed ?? null,
|
|
55
|
+
label: opts.label ?? null,
|
|
56
|
+
explanation: opts.explanation ?? null,
|
|
57
|
+
createdBy: null,
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
catch (err) {
|
|
61
|
+
warnOnce("run.evaluate", err);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
const inertHandle = { runId: "", end() { }, evaluate() { } };
|
|
65
|
+
function buildContext(startRow, traceId, zeroRetention) {
|
|
66
|
+
const runId = startRow.id;
|
|
67
|
+
const attemptByStep = new Map();
|
|
68
|
+
const loaded = new Set();
|
|
69
|
+
let stepSeq = 0;
|
|
70
|
+
return {
|
|
71
|
+
runId,
|
|
72
|
+
traceId,
|
|
73
|
+
zeroRetention,
|
|
74
|
+
ended: false,
|
|
75
|
+
buffer,
|
|
76
|
+
startRow,
|
|
77
|
+
noteLoad(artifactId) {
|
|
78
|
+
loaded.add(artifactId);
|
|
79
|
+
},
|
|
80
|
+
takeLoadedIds() {
|
|
81
|
+
const ids = [...loaded];
|
|
82
|
+
loaded.clear();
|
|
83
|
+
return ids;
|
|
84
|
+
},
|
|
85
|
+
nextStepKey(label) {
|
|
86
|
+
stepSeq += 1;
|
|
87
|
+
return `${label}#${stepSeq}`;
|
|
88
|
+
},
|
|
89
|
+
emitGroundTruthStep(input) {
|
|
90
|
+
const attemptNo = (attemptByStep.get(input.stepKey) ?? 0) + 1;
|
|
91
|
+
attemptByStep.set(input.stepKey, attemptNo);
|
|
92
|
+
try {
|
|
93
|
+
buffer.recordStep({
|
|
94
|
+
runId,
|
|
95
|
+
stepKey: input.stepKey,
|
|
96
|
+
attemptNo,
|
|
97
|
+
task: input.task,
|
|
98
|
+
provenance: "ground-truth",
|
|
99
|
+
inferenceId: input.inferenceId,
|
|
100
|
+
spanTraceId: null,
|
|
101
|
+
spanId: null,
|
|
102
|
+
startedAt: input.startedAt,
|
|
103
|
+
endedAt: input.endedAt,
|
|
104
|
+
failureClass: input.failureClass,
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
catch (err) {
|
|
108
|
+
warnOnce("step", err);
|
|
109
|
+
}
|
|
110
|
+
},
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
/** Build the context + record the `running` row. Shared by start() and
|
|
114
|
+
* scope() so the two entry points agree on identity + buffering. */
|
|
115
|
+
function newRun(opts) {
|
|
116
|
+
const startRow = {
|
|
117
|
+
id: (0, node_crypto_1.randomUUID)(),
|
|
118
|
+
pipeline: opts.pipeline,
|
|
119
|
+
pipelineVersion: opts.pipelineVersion ?? null,
|
|
120
|
+
environment: opts.environment ?? null,
|
|
121
|
+
episodeKey: opts.episodeKey ?? null,
|
|
122
|
+
startedAt: new Date().toISOString(),
|
|
123
|
+
endedAt: null,
|
|
124
|
+
status: "running",
|
|
125
|
+
};
|
|
126
|
+
const traceId = opts.traceparent !== undefined
|
|
127
|
+
? ((0, otel_map_1.parseTraceparent)(opts.traceparent)?.traceId ?? null)
|
|
128
|
+
: null;
|
|
129
|
+
const ctx = buildContext(startRow, traceId, opts.zeroRetention === true);
|
|
130
|
+
try {
|
|
131
|
+
buffer.recordRun(startRow);
|
|
132
|
+
}
|
|
133
|
+
catch (err) {
|
|
134
|
+
warnOnce("run", err);
|
|
135
|
+
}
|
|
136
|
+
return ctx;
|
|
137
|
+
}
|
|
138
|
+
return {
|
|
139
|
+
start(opts) {
|
|
140
|
+
if (!enabled)
|
|
141
|
+
return inertHandle;
|
|
142
|
+
const ctx = newRun(opts);
|
|
143
|
+
storage.enterWith(ctx);
|
|
144
|
+
return {
|
|
145
|
+
runId: ctx.runId,
|
|
146
|
+
end: (status) => endContext(ctx, status),
|
|
147
|
+
evaluate: (opts2) => writeRunEvaluation(ctx.runId, opts2),
|
|
148
|
+
};
|
|
149
|
+
},
|
|
150
|
+
end(status) {
|
|
151
|
+
const ctx = storage.getStore();
|
|
152
|
+
if (ctx !== undefined)
|
|
153
|
+
endContext(ctx, status);
|
|
154
|
+
},
|
|
155
|
+
evaluate(opts) {
|
|
156
|
+
const ctx = storage.getStore();
|
|
157
|
+
// `ended` counts as gone — same rule as getActiveRunContext. `enterWith`
|
|
158
|
+
// has no lexical exit, so an ended run's context stays ambient on this
|
|
159
|
+
// async path; without this check ns.evaluate re-attributes verdicts to
|
|
160
|
+
// the ended run (post-end verdicts belong to the HANDLE, which closes
|
|
161
|
+
// over its runId — that path is unaffected).
|
|
162
|
+
if (ctx === undefined || ctx.ended) {
|
|
163
|
+
console.warn("[modelrig artifacts] run.evaluate() with no active run — dropped");
|
|
164
|
+
return;
|
|
165
|
+
}
|
|
166
|
+
writeRunEvaluation(ctx.runId, opts);
|
|
167
|
+
},
|
|
168
|
+
async scope(opts, fn) {
|
|
169
|
+
if (!enabled)
|
|
170
|
+
return fn();
|
|
171
|
+
const ctx = newRun(opts);
|
|
172
|
+
// storage.run gives THIS run its own ambient context for the whole of fn,
|
|
173
|
+
// isolated from any sibling scope() running concurrently — no enterWith
|
|
174
|
+
// cross-attribution. End on the way out, success or failure.
|
|
175
|
+
return storage.run(ctx, async () => {
|
|
176
|
+
try {
|
|
177
|
+
const result = await fn();
|
|
178
|
+
endContext(ctx, "succeeded");
|
|
179
|
+
return result;
|
|
180
|
+
}
|
|
181
|
+
catch (err) {
|
|
182
|
+
endContext(ctx, "failed");
|
|
183
|
+
throw err;
|
|
184
|
+
}
|
|
185
|
+
});
|
|
186
|
+
},
|
|
187
|
+
};
|
|
188
|
+
function endContext(ctx, status) {
|
|
189
|
+
if (ctx.ended)
|
|
190
|
+
return;
|
|
191
|
+
ctx.ended = true;
|
|
192
|
+
// Update the projection in place: same identity as the start row, now with
|
|
193
|
+
// an ended_at and a terminal status. recordRun bumps export_seq so this
|
|
194
|
+
// terminal state re-exports past the `:runs` watermark. Emitting the full
|
|
195
|
+
// row (not a sparse one) is required — the server upsert overwrites every
|
|
196
|
+
// column it carries, so a blank pipeline here would clobber the real one.
|
|
197
|
+
try {
|
|
198
|
+
buffer.recordRun({
|
|
199
|
+
...ctx.startRow,
|
|
200
|
+
endedAt: new Date().toISOString(),
|
|
201
|
+
status: status ?? "succeeded",
|
|
202
|
+
});
|
|
203
|
+
}
|
|
204
|
+
catch (err) {
|
|
205
|
+
warnOnce("run", err);
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Scrub v1 — deterministic PII/PHI detection over artifact content BEFORE upload
|
|
3
|
+
* (E2-C chunk B, spec §3.1 / Expansion spec §7.4).
|
|
4
|
+
*
|
|
5
|
+
* Four modes, per route (default `detect` when content custody is on):
|
|
6
|
+
* off — no scan; upload verbatim.
|
|
7
|
+
* detect — scan; record FINDINGS as counts-by-type; upload verbatim. The
|
|
8
|
+
* findings are metadata (how many SSNs, how many emails), NEVER the
|
|
9
|
+
* matched values — recording the value would move the PII into the
|
|
10
|
+
* control-plane DB, the exact thing custody is meant to contain.
|
|
11
|
+
* redact — scan; upload a MASKED copy; the finding counts still travel.
|
|
12
|
+
* block — scan; if anything is found, REFUSE the upload (counted); the
|
|
13
|
+
* artifact keeps its metadata + hash, it simply stores no content.
|
|
14
|
+
*
|
|
15
|
+
* ── DETERMINISTIC ≠ DLP (honest limits) ───────────────────────────────────
|
|
16
|
+
* These are regex/checksum detectors, not a model. They catch well-formed SSNs,
|
|
17
|
+
* Luhn-valid PANs, emails, E.164/US phones, and labelled MRNs. They WILL miss
|
|
18
|
+
* obfuscated or free-form PII (that is scrub v2, model-assisted, out of scope).
|
|
19
|
+
* The custody docs page states this plainly — a caller must not read a clean
|
|
20
|
+
* `detect` result as a guarantee of no PII.
|
|
21
|
+
*
|
|
22
|
+
* NEVER logs a matched value. Findings are `{type, count}` only.
|
|
23
|
+
*/
|
|
24
|
+
export type ScrubMode = "off" | "detect" | "redact" | "block";
|
|
25
|
+
/** A finding is a COUNT by type — never a value (the load-bearing invariant). */
|
|
26
|
+
export interface ScrubFinding {
|
|
27
|
+
readonly type: ScrubDetectorType;
|
|
28
|
+
readonly count: number;
|
|
29
|
+
}
|
|
30
|
+
export type ScrubDetectorType = "ssn" | "pan" | "email" | "phone" | "mrn";
|
|
31
|
+
export interface ScrubResult {
|
|
32
|
+
/** Findings as counts-by-type, sorted by type. Empty when nothing matched. */
|
|
33
|
+
readonly findings: readonly ScrubFinding[];
|
|
34
|
+
/** The text to upload: identical to input for off/detect, masked for redact,
|
|
35
|
+
* and null for block-with-findings (nothing is uploaded). */
|
|
36
|
+
readonly text: string | null;
|
|
37
|
+
/** True when `block` mode refused because findings were present. */
|
|
38
|
+
readonly blocked: boolean;
|
|
39
|
+
}
|
|
40
|
+
/** The per-artifact scrub record persisted on `artifacts.scrub` (migration
|
|
41
|
+
* 00039). COUNTS ONLY — the `findings` are `{type,count}`, never a matched
|
|
42
|
+
* value (the load-bearing invariant, same as the SDK counters). `mode` records
|
|
43
|
+
* WHICH policy ran; `blocked` whether `block` mode refused the upload. */
|
|
44
|
+
export interface ScrubSummary {
|
|
45
|
+
readonly mode: ScrubMode;
|
|
46
|
+
readonly blocked: boolean;
|
|
47
|
+
readonly findings: readonly ScrubFinding[];
|
|
48
|
+
}
|
|
49
|
+
/** Build the persisted summary from a scrub result. The only producer of the
|
|
50
|
+
* `artifacts.scrub` value, so the counts-not-values invariant lives in one
|
|
51
|
+
* place — it simply forwards `ScrubResult.findings`, which are already counts. */
|
|
52
|
+
export declare function summarizeScrub(mode: ScrubMode, result: ScrubResult): ScrubSummary;
|
|
53
|
+
/** Luhn checksum — the difference between "16 digits" and "a card number". */
|
|
54
|
+
export declare function luhnValid(digits: string): boolean;
|
|
55
|
+
/**
|
|
56
|
+
* Scan `text` and produce a scrub result for `mode`. Pure and deterministic.
|
|
57
|
+
* Never returns or logs a matched value.
|
|
58
|
+
*/
|
|
59
|
+
export declare function scrubContent(text: string, mode: ScrubMode): ScrubResult;
|
|
60
|
+
//# sourceMappingURL=scrub.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scrub.d.ts","sourceRoot":"","sources":["../../src/artifacts/scrub.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,MAAM,MAAM,SAAS,GAAG,KAAK,GAAG,QAAQ,GAAG,QAAQ,GAAG,OAAO,CAAC;AAE9D,iFAAiF;AACjF,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,iBAAiB,CAAC;IACjC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,MAAM,iBAAiB,GAAG,KAAK,GAAG,KAAK,GAAG,OAAO,GAAG,OAAO,GAAG,KAAK,CAAC;AAE1E,MAAM,WAAW,WAAW;IAC1B,8EAA8E;IAC9E,QAAQ,CAAC,QAAQ,EAAE,SAAS,YAAY,EAAE,CAAC;IAC3C;iEAC6D;IAC7D,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,oEAAoE;IACpE,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;CAC3B;AAED;;;0EAG0E;AAC1E,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,SAAS,YAAY,EAAE,CAAC;CAC5C;AAED;;kFAEkF;AAClF,wBAAgB,cAAc,CAAC,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,WAAW,GAAG,YAAY,CAEjF;AAWD,8EAA8E;AAC9E,wBAAgB,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAejD;AA0CD;;;GAGG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,GAAG,WAAW,CA6CvE"}
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Scrub v1 — deterministic PII/PHI detection over artifact content BEFORE upload
|
|
4
|
+
* (E2-C chunk B, spec §3.1 / Expansion spec §7.4).
|
|
5
|
+
*
|
|
6
|
+
* Four modes, per route (default `detect` when content custody is on):
|
|
7
|
+
* off — no scan; upload verbatim.
|
|
8
|
+
* detect — scan; record FINDINGS as counts-by-type; upload verbatim. The
|
|
9
|
+
* findings are metadata (how many SSNs, how many emails), NEVER the
|
|
10
|
+
* matched values — recording the value would move the PII into the
|
|
11
|
+
* control-plane DB, the exact thing custody is meant to contain.
|
|
12
|
+
* redact — scan; upload a MASKED copy; the finding counts still travel.
|
|
13
|
+
* block — scan; if anything is found, REFUSE the upload (counted); the
|
|
14
|
+
* artifact keeps its metadata + hash, it simply stores no content.
|
|
15
|
+
*
|
|
16
|
+
* ── DETERMINISTIC ≠ DLP (honest limits) ───────────────────────────────────
|
|
17
|
+
* These are regex/checksum detectors, not a model. They catch well-formed SSNs,
|
|
18
|
+
* Luhn-valid PANs, emails, E.164/US phones, and labelled MRNs. They WILL miss
|
|
19
|
+
* obfuscated or free-form PII (that is scrub v2, model-assisted, out of scope).
|
|
20
|
+
* The custody docs page states this plainly — a caller must not read a clean
|
|
21
|
+
* `detect` result as a guarantee of no PII.
|
|
22
|
+
*
|
|
23
|
+
* NEVER logs a matched value. Findings are `{type, count}` only.
|
|
24
|
+
*/
|
|
25
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
+
exports.summarizeScrub = summarizeScrub;
|
|
27
|
+
exports.luhnValid = luhnValid;
|
|
28
|
+
exports.scrubContent = scrubContent;
|
|
29
|
+
/** Build the persisted summary from a scrub result. The only producer of the
|
|
30
|
+
* `artifacts.scrub` value, so the counts-not-values invariant lives in one
|
|
31
|
+
* place — it simply forwards `ScrubResult.findings`, which are already counts. */
|
|
32
|
+
function summarizeScrub(mode, result) {
|
|
33
|
+
return { mode, blocked: result.blocked, findings: result.findings };
|
|
34
|
+
}
|
|
35
|
+
/** Luhn checksum — the difference between "16 digits" and "a card number". */
|
|
36
|
+
function luhnValid(digits) {
|
|
37
|
+
const only = digits.replace(/\D/g, "");
|
|
38
|
+
if (only.length < 13 || only.length > 19)
|
|
39
|
+
return false;
|
|
40
|
+
let sum = 0;
|
|
41
|
+
let dbl = false;
|
|
42
|
+
for (let i = only.length - 1; i >= 0; i--) {
|
|
43
|
+
let d = only.charCodeAt(i) - 48;
|
|
44
|
+
if (dbl) {
|
|
45
|
+
d *= 2;
|
|
46
|
+
if (d > 9)
|
|
47
|
+
d -= 9;
|
|
48
|
+
}
|
|
49
|
+
sum += d;
|
|
50
|
+
dbl = !dbl;
|
|
51
|
+
}
|
|
52
|
+
return sum % 10 === 0;
|
|
53
|
+
}
|
|
54
|
+
// Order matters: PAN before phone/SSN so a 16-digit run is claimed as a card,
|
|
55
|
+
// not split. Each pattern is intentionally conservative — a false positive that
|
|
56
|
+
// masks a non-secret is cheaper than a false negative, but a pattern loose
|
|
57
|
+
// enough to match ordinary prose would make `block` unusable.
|
|
58
|
+
const DETECTORS = [
|
|
59
|
+
{
|
|
60
|
+
// PAN: 13–16 digits, optionally split by spaces/dashes in 4s, Luhn-valid.
|
|
61
|
+
type: "pan",
|
|
62
|
+
re: /\b(?:\d[ -]?){12,18}\d\b/g,
|
|
63
|
+
accept: luhnValid,
|
|
64
|
+
mask: "[REDACTED-PAN]",
|
|
65
|
+
},
|
|
66
|
+
{
|
|
67
|
+
// US SSN: AAA-GG-SSSS with dashes (the delimited form; bare 9-digit runs are
|
|
68
|
+
// too ambiguous to claim without a false-positive flood).
|
|
69
|
+
type: "ssn",
|
|
70
|
+
re: /\b\d{3}-\d{2}-\d{4}\b/g,
|
|
71
|
+
mask: "[REDACTED-SSN]",
|
|
72
|
+
},
|
|
73
|
+
{
|
|
74
|
+
type: "email",
|
|
75
|
+
re: /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/g,
|
|
76
|
+
mask: "[REDACTED-EMAIL]",
|
|
77
|
+
},
|
|
78
|
+
{
|
|
79
|
+
// Phone: E.164 (+1…) or US formats (parens/dashes/dots/spaces). Requires a
|
|
80
|
+
// grouping delimiter or leading + so bare 10-digit ids don't all match.
|
|
81
|
+
type: "phone",
|
|
82
|
+
re: /(?:\+\d{1,3}[ -]?)?(?:\(\d{3}\)|\d{3})[ .-]\d{3}[ .-]\d{4}\b/g,
|
|
83
|
+
mask: "[REDACTED-PHONE]",
|
|
84
|
+
},
|
|
85
|
+
{
|
|
86
|
+
// MRN-ish: an explicit label followed by an id. A heuristic by nature — the
|
|
87
|
+
// label is what makes it claimable without a false-positive flood.
|
|
88
|
+
type: "mrn",
|
|
89
|
+
re: /\bMRN\b\s*[:#]?\s*[A-Za-z0-9-]{4,}/gi,
|
|
90
|
+
mask: "[REDACTED-MRN]",
|
|
91
|
+
},
|
|
92
|
+
];
|
|
93
|
+
/**
|
|
94
|
+
* Scan `text` and produce a scrub result for `mode`. Pure and deterministic.
|
|
95
|
+
* Never returns or logs a matched value.
|
|
96
|
+
*/
|
|
97
|
+
function scrubContent(text, mode) {
|
|
98
|
+
if (mode === "off") {
|
|
99
|
+
return { findings: [], text, blocked: false };
|
|
100
|
+
}
|
|
101
|
+
const counts = new Map();
|
|
102
|
+
let masked = text;
|
|
103
|
+
for (const det of DETECTORS) {
|
|
104
|
+
// Fresh regex state per pass (global flag carries lastIndex otherwise).
|
|
105
|
+
const re = new RegExp(det.re.source, det.re.flags);
|
|
106
|
+
let count = 0;
|
|
107
|
+
const matched = [];
|
|
108
|
+
for (const m of text.matchAll(re)) {
|
|
109
|
+
const value = m[0];
|
|
110
|
+
if (det.accept && !det.accept(value))
|
|
111
|
+
continue;
|
|
112
|
+
count += 1;
|
|
113
|
+
if (mode === "redact")
|
|
114
|
+
matched.push(value);
|
|
115
|
+
}
|
|
116
|
+
if (count > 0) {
|
|
117
|
+
counts.set(det.type, count);
|
|
118
|
+
if (mode === "redact") {
|
|
119
|
+
for (const value of matched) {
|
|
120
|
+
// Replace literal occurrences with the mask. split/join avoids
|
|
121
|
+
// regex-escaping the value AND avoids putting it in any log.
|
|
122
|
+
masked = masked.split(value).join(det.mask);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
const findings = [...counts.entries()]
|
|
128
|
+
.map(([type, count]) => ({ type, count }))
|
|
129
|
+
.sort((a, b) => (a.type < b.type ? -1 : a.type > b.type ? 1 : 0));
|
|
130
|
+
const hasFindings = findings.length > 0;
|
|
131
|
+
if (mode === "block" && hasFindings) {
|
|
132
|
+
return { findings, text: null, blocked: true };
|
|
133
|
+
}
|
|
134
|
+
if (mode === "redact") {
|
|
135
|
+
return { findings, text: masked, blocked: false };
|
|
136
|
+
}
|
|
137
|
+
// detect (or block with no findings): upload verbatim, report counts.
|
|
138
|
+
return { findings, text, blocked: false };
|
|
139
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@step` / `step()` — execution scope for NON-LLM stages (spec §1.4, §9).
|
|
3
|
+
*
|
|
4
|
+
* A step is an execution operation ("what happened"). For LLM calls `rig.run()`
|
|
5
|
+
* auto-creates a ground-truth step (its inference row already exists); this
|
|
6
|
+
* wrapper is for the OTHER stages — retrieval, transforms, tool calls, external
|
|
7
|
+
* APIs — which have no inference row.
|
|
8
|
+
*
|
|
9
|
+
* ── WHY THIS WRITES NO STEP ROW THIS MISSION ──────────────────────────────
|
|
10
|
+
* The `steps` CHECK requires EXACTLY ONE reference: ground-truth (an inference)
|
|
11
|
+
* XOR attested (a foreign OTLP span). A pure non-LLM stage has neither — its
|
|
12
|
+
* span reference arrives with OTLP INGEST (E3), and nothing writes `attested`
|
|
13
|
+
* rows this mission. So `step()` establishes a NAMED execution scope inside the
|
|
14
|
+
* active run context (chunk B attributes artifact save/load lineage to it) and
|
|
15
|
+
* runs the body; it does not persist a row until it has a valid reference.
|
|
16
|
+
*
|
|
17
|
+
* Instrumentation never changes the wrapped function's return value, and never
|
|
18
|
+
* throws into it — a disabled or absent run context just runs the body.
|
|
19
|
+
*/
|
|
20
|
+
/**
|
|
21
|
+
* Run `fn` as a named execution step within the active run context. Returns the
|
|
22
|
+
* body's value unchanged (a promise if the body is async). Outside a run context
|
|
23
|
+
* (namespace disabled, or no run.start on this path) it simply runs the body.
|
|
24
|
+
*/
|
|
25
|
+
export declare function step<T>(key: string, fn: () => T): T;
|
|
26
|
+
//# sourceMappingURL=step.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"step.d.ts","sourceRoot":"","sources":["../../src/artifacts/step.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAIH;;;;GAIG;AACH,wBAAgB,IAAI,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,CAOnD"}
|