modelrig 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +123 -0
- package/README.md +65 -1
- package/dist/artifacts/run-context.d.ts +10 -0
- package/dist/artifacts/run-context.d.ts.map +1 -1
- package/dist/artifacts/run-context.js +12 -0
- package/dist/bakeoff/disagreement.d.ts +93 -0
- package/dist/bakeoff/disagreement.d.ts.map +1 -0
- package/dist/bakeoff/disagreement.js +140 -0
- package/dist/bakeoff/engine.d.ts.map +1 -1
- package/dist/bakeoff/engine.js +58 -2
- package/dist/bakeoff/pairwise.d.ts +102 -0
- package/dist/bakeoff/pairwise.d.ts.map +1 -0
- package/dist/bakeoff/pairwise.js +214 -0
- package/dist/bakeoff/replay.d.ts +6 -0
- package/dist/bakeoff/replay.d.ts.map +1 -1
- package/dist/bakeoff/replay.js +6 -2
- package/dist/bakeoff/report.d.ts +7 -0
- package/dist/bakeoff/report.d.ts.map +1 -1
- package/dist/bakeoff/report.js +46 -0
- package/dist/bakeoff/types.d.ts +24 -0
- package/dist/bakeoff/types.d.ts.map +1 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +9 -9
- package/dist/mask/apply.d.ts +80 -0
- package/dist/mask/apply.d.ts.map +1 -0
- package/dist/mask/apply.js +228 -0
- package/dist/mask/detect.d.ts +38 -0
- package/dist/mask/detect.d.ts.map +1 -0
- package/dist/mask/detect.js +167 -0
- package/dist/mask/index.d.ts +21 -0
- package/dist/mask/index.d.ts.map +1 -0
- package/dist/mask/index.js +54 -0
- package/dist/mask/mask-policy-cli.d.ts +11 -0
- package/dist/mask/mask-policy-cli.d.ts.map +1 -0
- package/dist/mask/mask-policy-cli.js +33 -0
- package/dist/mask/policy.d.ts +38 -0
- package/dist/mask/policy.d.ts.map +1 -0
- package/dist/mask/policy.js +187 -0
- package/dist/mask/proof.d.ts +54 -0
- package/dist/mask/proof.d.ts.map +1 -0
- package/dist/mask/proof.js +120 -0
- package/dist/mask/receipt.d.ts +30 -0
- package/dist/mask/receipt.d.ts.map +1 -0
- package/dist/mask/receipt.js +49 -0
- package/dist/mask/seam.d.ts +53 -0
- package/dist/mask/seam.d.ts.map +1 -0
- package/dist/mask/seam.js +120 -0
- package/dist/mask/types.d.ts +81 -0
- package/dist/mask/types.d.ts.map +1 -0
- package/dist/mask/types.js +10 -0
- package/dist/mask/vault.d.ts +50 -0
- package/dist/mask/vault.d.ts.map +1 -0
- package/dist/mask/vault.js +81 -0
- package/dist/receipts/types.d.ts +65 -0
- package/dist/receipts/types.d.ts.map +1 -0
- package/dist/receipts/types.js +134 -0
- package/dist/routes/loader-warnings.d.ts +8 -0
- package/dist/routes/loader-warnings.d.ts.map +1 -1
- package/dist/routes/loader-warnings.js +23 -0
- package/dist/routes/loader.d.ts +1 -1
- package/dist/routes/loader.d.ts.map +1 -1
- package/dist/routes/loader.js +24 -25
- package/dist/routes/mask-loader.d.ts +18 -0
- package/dist/routes/mask-loader.d.ts.map +1 -0
- package/dist/routes/mask-loader.js +39 -0
- package/dist/routes/objective-loader.d.ts.map +1 -1
- package/dist/routes/objective-loader.js +51 -17
- package/dist/routes/screen-loader.d.ts +23 -0
- package/dist/routes/screen-loader.d.ts.map +1 -0
- package/dist/routes/screen-loader.js +79 -0
- package/dist/routes/template.d.ts +10 -0
- package/dist/routes/template.d.ts.map +1 -1
- package/dist/routes/template.js +10 -3
- package/dist/run/accept.d.ts +4 -0
- package/dist/run/accept.d.ts.map +1 -1
- package/dist/run/accept.js +2 -0
- package/dist/run/dispatch.d.ts +35 -0
- package/dist/run/dispatch.d.ts.map +1 -1
- package/dist/run/dispatch.js +33 -0
- package/dist/run/grounding.d.ts +47 -2
- package/dist/run/grounding.d.ts.map +1 -1
- package/dist/run/grounding.js +113 -6
- package/dist/run/raw.d.ts.map +1 -1
- package/dist/run/raw.js +10 -3
- package/dist/run/record-attempt.d.ts.map +1 -1
- package/dist/run/record-attempt.js +12 -1
- package/dist/run/run.d.ts.map +1 -1
- package/dist/run/run.js +25 -19
- package/dist/screen/provenance.d.ts +59 -0
- package/dist/screen/provenance.d.ts.map +1 -0
- package/dist/screen/provenance.js +104 -0
- package/dist/screen/types.d.ts +71 -0
- package/dist/screen/types.d.ts.map +1 -0
- package/dist/screen/types.js +9 -0
- package/dist/status-cli.d.ts +11 -0
- package/dist/status-cli.d.ts.map +1 -0
- package/dist/status-cli.js +21 -0
- package/dist/status-runs.d.ts +38 -0
- package/dist/status-runs.d.ts.map +1 -0
- package/dist/status-runs.js +80 -0
- package/dist/status.d.ts +7 -2
- package/dist/status.d.ts.map +1 -1
- package/dist/status.js +11 -2
- package/dist/telemetry/ingest-protocol.d.ts +8 -0
- package/dist/telemetry/ingest-protocol.d.ts.map +1 -1
- package/dist/telemetry/store-migrations.d.ts.map +1 -1
- package/dist/telemetry/store-migrations.js +11 -0
- package/dist/telemetry/store.d.ts.map +1 -1
- package/dist/telemetry/store.js +12 -3
- package/dist/telemetry/supabase-exporter.d.ts +64 -0
- package/dist/telemetry/supabase-exporter.d.ts.map +1 -0
- package/dist/telemetry/supabase-exporter.js +211 -0
- package/dist/types/provider.d.ts +11 -0
- package/dist/types/provider.d.ts.map +1 -1
- package/dist/types/provider.js +9 -0
- package/dist/types/route.d.ts +55 -0
- package/dist/types/route.d.ts.map +1 -1
- package/dist/types/telemetry.d.ts +7 -0
- package/dist/types/telemetry.d.ts.map +1 -1
- package/dist/validate-cli.d.ts.map +1 -1
- package/dist/validate-cli.js +13 -0
- package/dist/verify/types.d.ts +6 -0
- package/dist/verify/types.d.ts.map +1 -1
- package/docs/agent-skill/SKILL.md +131 -0
- package/docs/how-it-fits.md +34 -0
- package/docs/instrumentation.md +221 -0
- package/docs/llms.txt +276 -0
- package/docs/migration-playbook.md +512 -0
- package/docs/quickstart.md +333 -0
- package/package.json +18 -2
- package/receipts.d.ts +12 -0
package/dist/run/dispatch.js
CHANGED
|
@@ -16,11 +16,13 @@ exports.jsonModeCoaching = jsonModeCoaching;
|
|
|
16
16
|
exports.buildAdapterRequest = buildAdapterRequest;
|
|
17
17
|
exports.buildRepairAdapterRequest = buildRepairAdapterRequest;
|
|
18
18
|
exports.dispatchCandidate = dispatchCandidate;
|
|
19
|
+
exports.buildDispatchMaybeMasked = buildDispatchMaybeMasked;
|
|
19
20
|
exports.emitToolCallResult = emitToolCallResult;
|
|
20
21
|
const node_crypto_1 = require("node:crypto");
|
|
21
22
|
const attempt_meta_1 = require("../telemetry/attempt-meta");
|
|
22
23
|
const cache_stamp_1 = require("./cache-stamp");
|
|
23
24
|
const cache_handle_1 = require("./cache-handle");
|
|
25
|
+
const seam_1 = require("../mask/seam");
|
|
24
26
|
/** Assemble the recorder's `metaInputs` from a completed dispatch (WI-1) — the
|
|
25
27
|
* adapter's native meta plus the rendered request (system+user+schema, hashed)
|
|
26
28
|
* and the output text (hashed). One helper so the routed lane's several record
|
|
@@ -179,6 +181,37 @@ async function dispatchCandidate(adapter, req, onDelta) {
|
|
|
179
181
|
},
|
|
180
182
|
});
|
|
181
183
|
}
|
|
184
|
+
/**
|
|
185
|
+
* Build the primary AdapterRequest, mask its outgoing user slot when the route
|
|
186
|
+
* has an active policy (spec §1.2 — "rewrites the AdapterRequest before dispatch
|
|
187
|
+
* and rehydrates the AdapterResult after"), dispatch, and time it. With no mask
|
|
188
|
+
* context this is byte-identical to `buildAdapterRequest` + `dispatchCandidate`
|
|
189
|
+
* (the golden test). The trusted system slot is never masked (B3 keeps untrusted
|
|
190
|
+
* content in the user path).
|
|
191
|
+
*/
|
|
192
|
+
async function buildDispatchMaybeMasked(adapter, parts, onDelta, mask) {
|
|
193
|
+
const built = buildAdapterRequest(parts);
|
|
194
|
+
if (mask === null) {
|
|
195
|
+
const started = Date.now();
|
|
196
|
+
const result = await dispatchCandidate(adapter, built, onDelta);
|
|
197
|
+
return { request: built, result, latencyMs: Date.now() - started, seam: null };
|
|
198
|
+
}
|
|
199
|
+
const seam = (0, seam_1.createMaskSeam)(mask.policy, mask.vault);
|
|
200
|
+
const outcome = seam.maskRequest(built);
|
|
201
|
+
if (outcome.kind !== "ok") {
|
|
202
|
+
// Fail closed: no request is sent. The run loop refuses pre-dispatch.
|
|
203
|
+
return {
|
|
204
|
+
request: built,
|
|
205
|
+
result: { ok: false, failure: { class: "invariant_violation", message: `masking failed closed (${outcome.kind})`, provider: parts.candidate.provider, model: parts.candidate.model } },
|
|
206
|
+
latencyMs: 0,
|
|
207
|
+
seam,
|
|
208
|
+
maskRefusal: outcome.kind,
|
|
209
|
+
};
|
|
210
|
+
}
|
|
211
|
+
const started = Date.now();
|
|
212
|
+
const result = await dispatchCandidate(adapter, outcome.request, onDelta);
|
|
213
|
+
return { request: outcome.request, result, latencyMs: Date.now() - started, seam };
|
|
214
|
+
}
|
|
182
215
|
/**
|
|
183
216
|
* Record a tool-call turn and build its RunResult (spec §4.4). A tool call is
|
|
184
217
|
* NOT a schema-validated JSON turn: it records a normal (non-failure) telemetry
|
package/dist/run/grounding.d.ts
CHANGED
|
@@ -8,7 +8,14 @@
|
|
|
8
8
|
* The Tavily-style provider ports the PATTERN from InferWealth's production
|
|
9
9
|
* websearch provider (query-length cap, basic/advanced depth, result shape)
|
|
10
10
|
* — logic only, no cross-package imports; key arrives via config.
|
|
11
|
+
*
|
|
12
|
+
* Screening S1/S5 (SC-0 B3): the `grounding.assembly: spotlight` knob routes
|
|
13
|
+
* search results through the provenance assembler as an untrusted user-slot
|
|
14
|
+
* block instead of concatenating them into the system prompt (INJ-1), and
|
|
15
|
+
* `extractCitations` allowlists model-output URLs to the injected sources (INJ-6).
|
|
11
16
|
*/
|
|
17
|
+
import type { RouteBundle } from "../types/route";
|
|
18
|
+
import { type ProviderId } from "../types/provider";
|
|
12
19
|
export interface SearchResultItem {
|
|
13
20
|
readonly title: string;
|
|
14
21
|
readonly url: string;
|
|
@@ -50,7 +57,45 @@ export interface UrlCitation {
|
|
|
50
57
|
readonly title?: string;
|
|
51
58
|
}
|
|
52
59
|
/** Extract URLs cited in the model's output and normalize to url_citation
|
|
53
|
-
* annotations, enriching with titles from the injected sources.
|
|
54
|
-
*
|
|
60
|
+
* annotations, enriching with titles from the injected sources.
|
|
61
|
+
*
|
|
62
|
+
* Screening S5 (SC-0 B3, INJ-6): ALLOWLISTED — only URLs that appear in the
|
|
63
|
+
* injected `sources` survive; a URL the model emitted that was not among the
|
|
64
|
+
* grounded sources is dropped (counted by `countDroppedCitations`). A model can
|
|
65
|
+
* no longer smuggle an attacker-controlled link into the caller's citation set. */
|
|
55
66
|
export declare function extractCitations(outputText: string, sources: readonly SearchResultItem[]): UrlCitation[];
|
|
67
|
+
/** Count of model-output URLs dropped by the S5 allowlist (not in the injected
|
|
68
|
+
* sources) — rides the attempt envelope as `meta.grounding.citations_dropped`. */
|
|
69
|
+
export declare function countDroppedCitations(outputText: string, sources: readonly SearchResultItem[]): number;
|
|
70
|
+
/** The grounding slice of the attempt meta: allowlisted citations plus the
|
|
71
|
+
* dropped-count (S5). Built once per successful grounded attempt so `run.ts`
|
|
72
|
+
* stays at its LOC ceiling (spec §10(i) — the seam lives here, not in run.ts). */
|
|
73
|
+
export declare function groundingMeta(outputText: string, sources: readonly SearchResultItem[]): {
|
|
74
|
+
citations: UrlCitation[];
|
|
75
|
+
grounding: {
|
|
76
|
+
citations_dropped: number;
|
|
77
|
+
};
|
|
78
|
+
};
|
|
79
|
+
/** Per-result snippet cap for the spotlighted search block (S1) — bounds the
|
|
80
|
+
* untrusted payload a single result can inject. */
|
|
81
|
+
export declare const SEARCH_SNIPPET_MAX_CHARS = 500;
|
|
82
|
+
/** Screening S1 (SC-0 B3): the SYSTEM-slot append for grounding.
|
|
83
|
+
* `legacy` (default) = the byte-identical pre-B3 injection block; `spotlight` =
|
|
84
|
+
* nothing (the results move to the user slot via `untrustedUserBlock`, and the
|
|
85
|
+
* untrusted-input policy sentence is owned by the provenance assembler and added
|
|
86
|
+
* to the system slot via `untrustedSystemAppend` — screening F2, single owner). */
|
|
87
|
+
export declare function groundingSystemAppend(assembly: RouteBundle["groundingAssembly"], results: readonly SearchResultItem[], retrievedAt: string): string;
|
|
88
|
+
/** Screening S1 + S2 (SC-0 B3): the USER-slot untrusted block appended after the
|
|
89
|
+
* user directive. Assembles the untrusted segments into a single spotlighted,
|
|
90
|
+
* JSON-encoded, delimiter-wrapped block via the provenance assembler. Returns ""
|
|
91
|
+
* when there is nothing untrusted (byte-identical default). */
|
|
92
|
+
export declare function untrustedUserBlock(route: Pick<RouteBundle, "groundingAssembly" | "variableTrust">, input: Record<string, unknown>, injectedSources: readonly SearchResultItem[] | null, provider: ProviderId): string;
|
|
93
|
+
/** Screening F2 (SC-0 B3): the SYSTEM-slot untrusted-input policy sentence. The
|
|
94
|
+
* provenance assembler is the SINGLE owner of this sentence (spec §2.2); this
|
|
95
|
+
* surfaces its system-slot output whenever ANY untrusted segment is delivered —
|
|
96
|
+
* so an `x-trust: untrusted` variable that is NOT referenced in the system
|
|
97
|
+
* template still gets its "this is data" framing. Returns "" when there is
|
|
98
|
+
* nothing untrusted (byte-identical default). The assembler's system slot does
|
|
99
|
+
* not depend on the nonce or the block shape, so fixed placeholders suffice. */
|
|
100
|
+
export declare function untrustedSystemAppend(route: Pick<RouteBundle, "groundingAssembly" | "variableTrust">, input: Record<string, unknown>, injectedSources: readonly SearchResultItem[] | null): string;
|
|
56
101
|
//# sourceMappingURL=grounding.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"grounding.d.ts","sourceRoot":"","sources":["../../src/run/grounding.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"grounding.d.ts","sourceRoot":"","sources":["../../src/run/grounding.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAGH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAqB,KAAK,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAGvE,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;CACjC;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,OAAO,EAAE,SAAS,gBAAgB,EAAE,CAAC;IAC9C,mEAAmE;IACnE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;CACzF;AAED,sEAAsE;AACtE,eAAO,MAAM,sBAAsB,MAAM,CAAC;AAE1C,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEpD;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB;uDACmD;IACnD,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,GAAG,UAAU,CAAC;IAC5C,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,sDAAsD;IACtD,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,KAAK,CAAC;CACjC;AAaD,wBAAgB,0BAA0B,CAAC,MAAM,EAAE,kBAAkB,GAAG,cAAc,CAoCrF;AAED;gEACgE;AAChE,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,SAAS,gBAAgB,EAAE,EAAE,WAAW,EAAE,MAAM,GAAG,MAAM,CAYtG;AAED,yEAAyE;AACzE,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC;IAC9B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;CACzB;AAWD;;;;;;mFAMmF;AACnF,wBAAgB,gBAAgB,CAC9B,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,SAAS,gBAAgB,EAAE,GACnC,WAAW,EAAE,CAYf;AAED;kFACkF;AAClF,wBAAgB,qBAAqB,CACnC,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,SAAS,gBAAgB,EAAE,GACnC,MAAM,CAGR;AAED;;kFAEkF;AAClF,wBAAgB,aAAa,CAC3B,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,SAAS,gBAAgB,EAAE,GACnC;IAAE,SAAS,EAAE,WAAW,EAAE,CAAC;IAAC,SAAS,EAAE;QAAE,iBAAiB,EAAE,MAAM,CAAA;KAAE,CAAA;CAAE,CAKxE;AAED;mDACmD;AACnD,eAAO,MAAM,wBAAwB,MAAM,CAAC;AAM5C;;;;mFAImF;AACnF,wBAAgB,qBAAqB,CACnC,QAAQ,EAAE,WAAW,CAAC,mBAAmB,CAAC,EAC1C,OAAO,EAAE,SAAS,gBAAgB,EAAE,EACpC,WAAW,EAAE,MAAM,GAClB,MAAM,CAGR;AAiCD;;;+DAG+D;AAC/D,wBAAgB,kBAAkB,CAChC,KAAK,EAAE,IAAI,CAAC,WAAW,EAAE,mBAAmB,GAAG,eAAe,CAAC,EAC/D,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC9B,eAAe,EAAE,SAAS,gBAAgB,EAAE,GAAG,IAAI,EACnD,QAAQ,EAAE,UAAU,GACnB,MAAM,CAKR;AAED;;;;;;gFAMgF;AAChF,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,IAAI,CAAC,WAAW,EAAE,mBAAmB,GAAG,eAAe,CAAC,EAC/D,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC9B,eAAe,EAAE,SAAS,gBAAgB,EAAE,GAAG,IAAI,GAClD,MAAM,CAKR"}
|
package/dist/run/grounding.js
CHANGED
|
@@ -9,13 +9,26 @@
|
|
|
9
9
|
* The Tavily-style provider ports the PATTERN from InferWealth's production
|
|
10
10
|
* websearch provider (query-length cap, basic/advanced depth, result shape)
|
|
11
11
|
* — logic only, no cross-package imports; key arrives via config.
|
|
12
|
+
*
|
|
13
|
+
* Screening S1/S5 (SC-0 B3): the `grounding.assembly: spotlight` knob routes
|
|
14
|
+
* search results through the provenance assembler as an untrusted user-slot
|
|
15
|
+
* block instead of concatenating them into the system prompt (INJ-1), and
|
|
16
|
+
* `extractCitations` allowlists model-output URLs to the injected sources (INJ-6).
|
|
12
17
|
*/
|
|
13
18
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
14
|
-
exports.SEARCH_QUERY_MAX_CHARS = void 0;
|
|
19
|
+
exports.SEARCH_SNIPPET_MAX_CHARS = exports.SEARCH_QUERY_MAX_CHARS = void 0;
|
|
15
20
|
exports.capSearchQuery = capSearchQuery;
|
|
16
21
|
exports.createTavilySearchProvider = createTavilySearchProvider;
|
|
17
22
|
exports.searchInjectionBlock = searchInjectionBlock;
|
|
18
23
|
exports.extractCitations = extractCitations;
|
|
24
|
+
exports.countDroppedCitations = countDroppedCitations;
|
|
25
|
+
exports.groundingMeta = groundingMeta;
|
|
26
|
+
exports.groundingSystemAppend = groundingSystemAppend;
|
|
27
|
+
exports.untrustedUserBlock = untrustedUserBlock;
|
|
28
|
+
exports.untrustedSystemAppend = untrustedSystemAppend;
|
|
29
|
+
const node_crypto_1 = require("node:crypto");
|
|
30
|
+
const provider_1 = require("../types/provider");
|
|
31
|
+
const provenance_1 = require("../screen/provenance");
|
|
19
32
|
/** Ported cap: the Tavily API hard-rejects queries over 400 chars. */
|
|
20
33
|
exports.SEARCH_QUERY_MAX_CHARS = 400;
|
|
21
34
|
function capSearchQuery(query) {
|
|
@@ -71,14 +84,24 @@ function searchInjectionBlock(results, retrievedAt) {
|
|
|
71
84
|
lines.join("\n"));
|
|
72
85
|
}
|
|
73
86
|
const URL_PATTERN = /https?:\/\/[^\s"'<>)\]\\]+/g;
|
|
87
|
+
/** URLs referenced in a text, deduped and with trailing punctuation stripped
|
|
88
|
+
* (same normalization as the probe suite). */
|
|
89
|
+
function citationUrls(outputText) {
|
|
90
|
+
const raw = outputText.match(URL_PATTERN) ?? [];
|
|
91
|
+
return [...new Set(raw.map((u) => u.replace(/[.,;:]+$/, "")))];
|
|
92
|
+
}
|
|
74
93
|
/** Extract URLs cited in the model's output and normalize to url_citation
|
|
75
|
-
* annotations, enriching with titles from the injected sources.
|
|
76
|
-
*
|
|
94
|
+
* annotations, enriching with titles from the injected sources.
|
|
95
|
+
*
|
|
96
|
+
* Screening S5 (SC-0 B3, INJ-6): ALLOWLISTED — only URLs that appear in the
|
|
97
|
+
* injected `sources` survive; a URL the model emitted that was not among the
|
|
98
|
+
* grounded sources is dropped (counted by `countDroppedCitations`). A model can
|
|
99
|
+
* no longer smuggle an attacker-controlled link into the caller's citation set. */
|
|
77
100
|
function extractCitations(outputText, sources) {
|
|
78
|
-
const raw = outputText.match(URL_PATTERN) ?? [];
|
|
79
|
-
const urls = [...new Set(raw.map((u) => u.replace(/[.,;:]+$/, "")))];
|
|
80
101
|
const titleByUrl = new Map(sources.map((s) => [s.url, s.title]));
|
|
81
|
-
return
|
|
102
|
+
return citationUrls(outputText)
|
|
103
|
+
.filter((url) => titleByUrl.has(url))
|
|
104
|
+
.map((url) => {
|
|
82
105
|
const title = titleByUrl.get(url);
|
|
83
106
|
return {
|
|
84
107
|
type: "url_citation",
|
|
@@ -87,3 +110,87 @@ function extractCitations(outputText, sources) {
|
|
|
87
110
|
};
|
|
88
111
|
});
|
|
89
112
|
}
|
|
113
|
+
/** Count of model-output URLs dropped by the S5 allowlist (not in the injected
|
|
114
|
+
* sources) — rides the attempt envelope as `meta.grounding.citations_dropped`. */
|
|
115
|
+
function countDroppedCitations(outputText, sources) {
|
|
116
|
+
const allowed = new Set(sources.map((s) => s.url));
|
|
117
|
+
return citationUrls(outputText).filter((url) => !allowed.has(url)).length;
|
|
118
|
+
}
|
|
119
|
+
/** The grounding slice of the attempt meta: allowlisted citations plus the
|
|
120
|
+
* dropped-count (S5). Built once per successful grounded attempt so `run.ts`
|
|
121
|
+
* stays at its LOC ceiling (spec §10(i) — the seam lives here, not in run.ts). */
|
|
122
|
+
function groundingMeta(outputText, sources) {
|
|
123
|
+
return {
|
|
124
|
+
citations: extractCitations(outputText, sources),
|
|
125
|
+
grounding: { citations_dropped: countDroppedCitations(outputText, sources) },
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
/** Per-result snippet cap for the spotlighted search block (S1) — bounds the
|
|
129
|
+
* untrusted payload a single result can inject. */
|
|
130
|
+
exports.SEARCH_SNIPPET_MAX_CHARS = 500;
|
|
131
|
+
function capSnippet(snippet) {
|
|
132
|
+
return snippet.length <= exports.SEARCH_SNIPPET_MAX_CHARS ? snippet : snippet.slice(0, exports.SEARCH_SNIPPET_MAX_CHARS);
|
|
133
|
+
}
|
|
134
|
+
/** Screening S1 (SC-0 B3): the SYSTEM-slot append for grounding.
|
|
135
|
+
* `legacy` (default) = the byte-identical pre-B3 injection block; `spotlight` =
|
|
136
|
+
* nothing (the results move to the user slot via `untrustedUserBlock`, and the
|
|
137
|
+
* untrusted-input policy sentence is owned by the provenance assembler and added
|
|
138
|
+
* to the system slot via `untrustedSystemAppend` — screening F2, single owner). */
|
|
139
|
+
function groundingSystemAppend(assembly, results, retrievedAt) {
|
|
140
|
+
if (assembly === "spotlight")
|
|
141
|
+
return "";
|
|
142
|
+
return searchInjectionBlock(results, retrievedAt); // legacy default — byte-identical
|
|
143
|
+
}
|
|
144
|
+
/** Screening S1 + S2 (SC-0 B3): the untrusted segments a route+input contribute
|
|
145
|
+
* — (a) each `x-trust: untrusted` variable (S2) and (b), when
|
|
146
|
+
* `grounding.assembly: spotlight`, the search results (S1). Shared by the system-
|
|
147
|
+
* and user-slot builders so the two slots agree on "is anything untrusted
|
|
148
|
+
* present" (screening F2). Empty ⇒ nothing untrusted (byte-identical default). */
|
|
149
|
+
function collectUntrustedSegments(route, input, injectedSources) {
|
|
150
|
+
const segments = [];
|
|
151
|
+
// S2: each variable annotated `x-trust: untrusted` moves out of the system slot.
|
|
152
|
+
for (const [name, trust] of Object.entries(route.variableTrust ?? {})) {
|
|
153
|
+
if (trust !== "untrusted")
|
|
154
|
+
continue;
|
|
155
|
+
const value = input[name];
|
|
156
|
+
if (value === undefined)
|
|
157
|
+
continue;
|
|
158
|
+
const text = typeof value === "string" ? value : JSON.stringify(value);
|
|
159
|
+
segments.push({ source: "variable", tier: "untrusted", text });
|
|
160
|
+
}
|
|
161
|
+
// S1: spotlight routes search results through the assembler as `search` data.
|
|
162
|
+
if (route.groundingAssembly === "spotlight" && injectedSources !== null && injectedSources.length > 0) {
|
|
163
|
+
const payload = {
|
|
164
|
+
source: "web_search",
|
|
165
|
+
retrieved_at: new Date().toISOString(),
|
|
166
|
+
results: injectedSources.map((r) => ({ title: r.title, url: r.url, snippet: capSnippet(r.snippet) })),
|
|
167
|
+
};
|
|
168
|
+
segments.push({ source: "search", tier: "untrusted", text: JSON.stringify(payload) });
|
|
169
|
+
}
|
|
170
|
+
return segments;
|
|
171
|
+
}
|
|
172
|
+
/** Screening S1 + S2 (SC-0 B3): the USER-slot untrusted block appended after the
|
|
173
|
+
* user directive. Assembles the untrusted segments into a single spotlighted,
|
|
174
|
+
* JSON-encoded, delimiter-wrapped block via the provenance assembler. Returns ""
|
|
175
|
+
* when there is nothing untrusted (byte-identical default). */
|
|
176
|
+
function untrustedUserBlock(route, input, injectedSources, provider) {
|
|
177
|
+
const segments = collectUntrustedSegments(route, input, injectedSources);
|
|
178
|
+
if (segments.length === 0)
|
|
179
|
+
return "";
|
|
180
|
+
const assembled = (0, provenance_1.assemblePrompt)(segments, { nonce: (0, node_crypto_1.randomUUID)(), untrustedBlock: (0, provider_1.untrustedBlockFor)(provider) });
|
|
181
|
+
return `\n\n${assembled.user}`;
|
|
182
|
+
}
|
|
183
|
+
/** Screening F2 (SC-0 B3): the SYSTEM-slot untrusted-input policy sentence. The
|
|
184
|
+
* provenance assembler is the SINGLE owner of this sentence (spec §2.2); this
|
|
185
|
+
* surfaces its system-slot output whenever ANY untrusted segment is delivered —
|
|
186
|
+
* so an `x-trust: untrusted` variable that is NOT referenced in the system
|
|
187
|
+
* template still gets its "this is data" framing. Returns "" when there is
|
|
188
|
+
* nothing untrusted (byte-identical default). The assembler's system slot does
|
|
189
|
+
* not depend on the nonce or the block shape, so fixed placeholders suffice. */
|
|
190
|
+
function untrustedSystemAppend(route, input, injectedSources) {
|
|
191
|
+
const segments = collectUntrustedSegments(route, input, injectedSources);
|
|
192
|
+
if (segments.length === 0)
|
|
193
|
+
return "";
|
|
194
|
+
const { system } = (0, provenance_1.assemblePrompt)(segments, { nonce: "", untrustedBlock: "user_json" });
|
|
195
|
+
return system === "" ? "" : `\n\n${system}`;
|
|
196
|
+
}
|
package/dist/run/raw.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"raw.d.ts","sourceRoot":"","sources":["../../src/run/raw.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;
|
|
1
|
+
{"version":3,"file":"raw.d.ts","sourceRoot":"","sources":["../../src/run/raw.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AAwBH,OAAO,EAAiC,KAAK,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACrF,OAAO,KAAK,EACV,cAAc,EAGd,YAAY,EAGZ,UAAU,EACV,eAAe,EACf,WAAW,EACX,UAAU,EACV,cAAc,EACf,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAW,SAAS,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAGlE,OAAO,KAAK,EAAkB,eAAe,EAAE,MAAM,UAAU,CAAC;AAChE,OAAO,KAAK,EAA+B,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAEtF;;;sCAGsC;AACtC,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;iFAC6E;IAC7E,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC;IACxB;;;;;iFAK6E;IAC7E,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;IAClC;;;6EAGyE;IACzE,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IAC3C,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;IACjC;;;;;;;0BAOsB;IACtB,QAAQ,CAAC,KAAK,CAAC,EAAE;QAAE,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACxE;;;;;;;iDAO6C;IAC7C,QAAQ,CAAC,SAAS,CAAC,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAA;KAAE,CAAC;IACjD;;;;mFAI+E;IAC/E,QAAQ,CAAC,WAAW,CAAC,EAAE,WAAW,CAAC;IACnC;;;;qFAIiF;IACjF,QAAQ,CAAC,SAAS,CAAC,EAAE,eAAe,CAAC;IACrC;;;;;;;;sEAQkE;IAClE,QAAQ,CAAC,cAAc,CAAC,EAAE,aAAa,CAAC;CACzC;AAED,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,SAAS,EAAE,cAAc,CAAC;IACnC;;+EAE2E;IAC3E,QAAQ,CAAC,sBAAsB,CAAC,EAAE,MAAM,CAAC;IACzC;;sEAEkE;IAClE,QAAQ,CAAC,QAAQ,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC;CACxC;AAiED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,WAAW,GAAG,IAAI,CAiDhE;AAED,4EAA4E;AAC5E,wBAAgB,eAAe,CAAC,SAAS,EAAE,YAAY,EAAE,KAAK,EAAE,WAAW,GAAG,cAAc,CAsC3F;AAgQD;wDACwD;AACxD,wBAAsB,MAAM,CAAC,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,SAAS,CAAC,CAYrF;AAcD;;;;;;;GAOG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,WAAW,GAAG,eAAe,CAiDlF"}
|
package/dist/run/raw.js
CHANGED
|
@@ -281,8 +281,9 @@ function recordRawRow(telemetry, p) {
|
|
|
281
281
|
* step is ever recorded; the dispatch is byte-identical to pre-G1.
|
|
282
282
|
*/
|
|
283
283
|
function prepareRawStep(deps, input) {
|
|
284
|
-
(0, manifest_1.warnMissingRequiredDimensions)(deps.manifest ?? null, input.tags ?? {});
|
|
285
284
|
const runCtx = (0, run_context_1.getActiveRunContext)();
|
|
285
|
+
// Warn on the EFFECTIVE tags: an auto-stamped run_id (RC-1) is not missing.
|
|
286
|
+
(0, manifest_1.warnMissingRequiredDimensions)(deps.manifest ?? null, (0, run_context_1.withRunIdTag)(input.tags ?? {}, runCtx));
|
|
286
287
|
if (runCtx === null)
|
|
287
288
|
return null;
|
|
288
289
|
const rawModelId = `${input.provider}/${input.model}`;
|
|
@@ -308,8 +309,14 @@ function finalizeRaw(deps, input, result, latencyMs, step) {
|
|
|
308
309
|
const callId = (0, node_crypto_1.randomUUID)();
|
|
309
310
|
// CE-2 F2: the declared tier is reported truthfully on the row AND the meta.
|
|
310
311
|
const requestedTier = input.serviceTier ?? "standard";
|
|
311
|
-
//
|
|
312
|
-
|
|
312
|
+
// RC-1: `run_id` from the ambient run (caller's value wins); lane is SDK-owned
|
|
313
|
+
// and cannot be overridden by a caller-supplied tag.
|
|
314
|
+
// GS-1: `tags.step` from the ambient step key (step.stepKey = caller's stepKey
|
|
315
|
+
// ?? tags.step ?? the auto provider/model#n key) when a run is active and the
|
|
316
|
+
// caller set none — spread FIRST so a caller value wins, and omitted entirely
|
|
317
|
+
// outside a run context so the row stays byte-identical (parity with RC-1).
|
|
318
|
+
const stepTag = step !== null ? { step: step.stepKey } : {};
|
|
319
|
+
const tags = { ...stepTag, ...(0, run_context_1.withRunIdTag)(input.tags ?? {}, step?.runCtx ?? null), lane: "raw" };
|
|
313
320
|
// WI-1 (spec §2, §6.3 lane parity): the attempt envelope, built by the SAME
|
|
314
321
|
// producer + rendered-request canonicalization the routed lane uses. The raw
|
|
315
322
|
// lane never sends an output schema (unvalidated by design), so schema = null.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"record-attempt.d.ts","sourceRoot":"","sources":["../../src/run/record-attempt.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AACnD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AACjE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAE3C,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACvE,OAAO,KAAK,EAAgB,aAAa,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAGtF,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAGpD,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,SAAS,EAAE,cAAc,CAAC;IACnC,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B;;;gDAG4C;IAC5C,QAAQ,CAAC,MAAM,EAAE,eAAe,CAAC;IACjC,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;IAChC,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,kFAAkF;IAClF,QAAQ,CAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACzC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC;;qFAEiF;IACjF,QAAQ,CAAC,KAAK,CAAC,EAAE,UAAU,CAAC,OAAO,CAAC,CAAC;IACrC;;;wEAGoE;IACpE,QAAQ,CAAC,kBAAkB,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;CAClD;AAED;;;;;GAKG;AACH,wBAAgB,MAAM,CACpB,OAAO,EAAE,gBAAgB,EACzB,WAAW,EAAE,OAAO,EACpB,cAAc,EAAE,MAAM,GAAG,IAAI,EAC7B,iBAAiB,EAAE,GAAG,CAAC,MAAM,CAAC,GAC7B,aAAa,CAUf;AAED;;yCAEyC;AACzC,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,iBAAiB,GAAG,CAAC,OAAO,EAAE,gBAAgB,KAAK,MAAM,
|
|
1
|
+
{"version":3,"file":"record-attempt.d.ts","sourceRoot":"","sources":["../../src/run/record-attempt.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AACnD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AACjE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAE3C,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACvE,OAAO,KAAK,EAAgB,aAAa,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAGtF,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAGpD,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,SAAS,EAAE,cAAc,CAAC;IACnC,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B;;;gDAG4C;IAC5C,QAAQ,CAAC,MAAM,EAAE,eAAe,CAAC;IACjC,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;IAChC,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,kFAAkF;IAClF,QAAQ,CAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACzC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC;;qFAEiF;IACjF,QAAQ,CAAC,KAAK,CAAC,EAAE,UAAU,CAAC,OAAO,CAAC,CAAC;IACrC;;;wEAGoE;IACpE,QAAQ,CAAC,kBAAkB,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;CAClD;AAED;;;;;GAKG;AACH,wBAAgB,MAAM,CACpB,OAAO,EAAE,gBAAgB,EACzB,WAAW,EAAE,OAAO,EACpB,cAAc,EAAE,MAAM,GAAG,IAAI,EAC7B,iBAAiB,EAAE,GAAG,CAAC,MAAM,CAAC,GAC7B,aAAa,CAUf;AAED;;yCAEyC;AACzC,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,iBAAiB,GAAG,CAAC,OAAO,EAAE,gBAAgB,KAAK,MAAM,CA8HlG"}
|
|
@@ -109,6 +109,14 @@ function createRecordAttempt(deps) {
|
|
|
109
109
|
...(attempt.tokensCacheWrite1h !== undefined ? { tokensCacheWrite1h: attempt.tokensCacheWrite1h } : {}),
|
|
110
110
|
},
|
|
111
111
|
});
|
|
112
|
+
// GS-1: auto-stamp `tags.step` from the ambient step key — the run graph's
|
|
113
|
+
// step, made a groupable tag — when a run is active and the caller set none.
|
|
114
|
+
// Mirrors withRunIdTag exactly: a caller value wins (effectiveTags carries it,
|
|
115
|
+
// and it is spread after this base), and OUTSIDE a run context nothing is added
|
|
116
|
+
// so the row stays byte-identical. `autoStepKey` already resolves the caller's
|
|
117
|
+
// stepKey ?? tags.step ?? the auto `route#n` key, so a caller's own step lands
|
|
118
|
+
// unchanged and the tag agrees with the ground-truth step row.
|
|
119
|
+
const stepTag = runCtx !== null && autoStepKey !== null ? { step: autoStepKey } : {};
|
|
112
120
|
const row = {
|
|
113
121
|
id: (0, node_crypto_1.randomUUID)(),
|
|
114
122
|
ts: new Date().toISOString(),
|
|
@@ -133,7 +141,7 @@ function createRecordAttempt(deps) {
|
|
|
133
141
|
attemptsJson: JSON.stringify(ledger.snapshot()),
|
|
134
142
|
// `rung` is stamped LAST so a caller's extraTags (Q1 quality tags) can
|
|
135
143
|
// never override the recorder-owned rung.
|
|
136
|
-
tagsJson: JSON.stringify({ ...effectiveTags, ...attempt.extraTags, rung }),
|
|
144
|
+
tagsJson: JSON.stringify({ ...stepTag, ...effectiveTags, ...attempt.extraTags, rung }),
|
|
137
145
|
fingerprintCell: route.fingerprintCell,
|
|
138
146
|
servedVariant,
|
|
139
147
|
repairedBy: attempt.repairedBy ?? null,
|
|
@@ -142,6 +150,9 @@ function createRecordAttempt(deps) {
|
|
|
142
150
|
attemptSeq: ladderKey.attemptSeq,
|
|
143
151
|
candidateIndex: ladderKey.candidateIndex,
|
|
144
152
|
meta,
|
|
153
|
+
// CS-1 (spec §6): the counts-only mask receipt for this attempt, or absent
|
|
154
|
+
// when the route does not mask. NEVER a matched value.
|
|
155
|
+
...(attempt.mask !== undefined ? { mask: attempt.mask } : {}),
|
|
145
156
|
};
|
|
146
157
|
telemetry.record(row);
|
|
147
158
|
// Mirror this inference into a ground-truth step attempt (fail-open — the
|
package/dist/run/run.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../src/run/run.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAMrD,OAAO,KAAK,EAAE,eAAe,EAAgB,MAAM,mBAAmB,CAAC;AAGvE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEtD,OAAO,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC5D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,KAAK,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAS3D,OAAO,KAAK,EAAE,cAAc,EAAoC,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../src/run/run.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAMrD,OAAO,KAAK,EAAE,eAAe,EAAgB,MAAM,mBAAmB,CAAC;AAGvE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEtD,OAAO,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC5D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,KAAK,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAS3D,OAAO,KAAK,EAAE,cAAc,EAAoC,MAAM,aAAa,CAAC;AAUpF,OAAO,KAAK,EAAe,eAAe,EAAE,MAAM,YAAY,CAAC;AAQ/D,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AAEtE,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IACxD,QAAQ,CAAC,SAAS,EAAE,cAAc,CAAC;IACnC,QAAQ,CAAC,YAAY,EAAE,gBAAgB,CAAC;IACxC,+EAA+E;IAC/E,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAC;IACpC;;gDAE4C;IAC5C,QAAQ,CAAC,sBAAsB,CAAC,EAAE,MAAM,CAAC;IACzC,0EAA0E;IAC1E,QAAQ,CAAC,QAAQ,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC;IACvC;2EACuE;IACvE,QAAQ,CAAC,QAAQ,CAAC,EAAE,YAAY,GAAG,IAAI,CAAC;IACxC;;0BAEsB;IACtB,QAAQ,CAAC,MAAM,CAAC,EAAE,cAAc,GAAG,IAAI,CAAC;IACxC,qDAAqD;IACrD,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/C;;;mFAG+E;IAC/E,QAAQ,CAAC,aAAa,CAAC,EAAE,eAAe,CAAC;IACzC;;;;;uDAKmD;IACnD,QAAQ,CAAC,QAAQ,CAAC,EAAE,oBAAoB,GAAG,IAAI,CAAC;IAChD;;;;2EAIuE;IACvE,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC;CACzB;AAsBD,wBAAsB,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,CAwkBhG"}
|
package/dist/run/run.js
CHANGED
|
@@ -24,6 +24,7 @@ const repair_1 = require("./repair");
|
|
|
24
24
|
const grounding_1 = require("./grounding");
|
|
25
25
|
const validate_1 = require("./validate");
|
|
26
26
|
const dispatch_1 = require("./dispatch");
|
|
27
|
+
const vault_1 = require("../mask/vault");
|
|
27
28
|
const record_attempt_1 = require("./record-attempt");
|
|
28
29
|
const ladder_key_1 = require("./ladder-key");
|
|
29
30
|
const quality_gate_1 = require("./quality-gate");
|
|
@@ -52,9 +53,13 @@ async function run(deps, routeName, opts) {
|
|
|
52
53
|
const variant = (0, variant_1.resolveVariant)(baseRoute, opts.variant, mode);
|
|
53
54
|
const servedVariant = variant?.name ?? null;
|
|
54
55
|
const route = (0, resolve_1.applyRunRequirements)((0, variant_1.effectiveRoute)(baseRoute, variant), opts.zeroRetention === true);
|
|
55
|
-
|
|
56
|
+
// RC-1: inside a run context the SDK stamps `run_id` (episodeKey, else run id)
|
|
57
|
+
// when the caller passed none — the bridge between tags and the run graph.
|
|
58
|
+
const runCtx = (0, run_context_1.getActiveRunContext)();
|
|
59
|
+
const runTags = (0, run_context_1.withRunIdTag)(opts.tags, runCtx);
|
|
60
|
+
const effectiveTags = route.require.includes("zero_retention") ? { ...runTags, zdr_enforced: "true" } : runTags;
|
|
56
61
|
// Data-hygiene warning (spec WS5): once per process per dimension.
|
|
57
|
-
(0, manifest_1.warnMissingRequiredDimensions)(deps.manifest ?? null,
|
|
62
|
+
(0, manifest_1.warnMissingRequiredDimensions)(deps.manifest ?? null, effectiveTags);
|
|
58
63
|
const ledger = (0, retry_1.createRetryLedger)(route.policy);
|
|
59
64
|
const sleep = deps.sleep ?? DEFAULT_SLEEP;
|
|
60
65
|
const guard = opts.budget
|
|
@@ -70,7 +75,6 @@ async function run(deps, routeName, opts) {
|
|
|
70
75
|
// Auto-step (E2 chunk A, spec §1.4): with an active run context, each attempt
|
|
71
76
|
// exit that records an inference also records a ground-truth step referencing
|
|
72
77
|
// it (attempt-per-row, ruling c). Inert without a run.start on this path.
|
|
73
|
-
const runCtx = (0, run_context_1.getActiveRunContext)();
|
|
74
78
|
const autoStepKey = runCtx !== null ? (0, run_context_1.resolveStepKey)(opts.stepKey, opts.tags.step, () => runCtx.nextStepKey(route.route)) : null;
|
|
75
79
|
// Per-call ladder key (spec §2.1 R0): one call_id, a monotonic attempt_seq per
|
|
76
80
|
// row, and the current candidate_index (advanced by nextCandidate below).
|
|
@@ -233,7 +237,7 @@ async function run(deps, routeName, opts) {
|
|
|
233
237
|
repairedBy,
|
|
234
238
|
callId: ladder.callId,
|
|
235
239
|
...(injectedSources !== null
|
|
236
|
-
?
|
|
240
|
+
? (0, grounding_1.groundingMeta)(result.text, injectedSources)
|
|
237
241
|
: {}),
|
|
238
242
|
},
|
|
239
243
|
},
|
|
@@ -258,6 +262,7 @@ async function run(deps, routeName, opts) {
|
|
|
258
262
|
// retries; cost accrues to the envelope and its own telemetry row.
|
|
259
263
|
let searchResponse = null;
|
|
260
264
|
let injectedSources = null;
|
|
265
|
+
const maskCtx = route.mask !== undefined ? { policy: route.mask, vault: (0, vault_1.createRunVault)() } : null; // CS-1: run-scoped vault; null ⇒ byte-identical
|
|
261
266
|
while (candidate !== null) {
|
|
262
267
|
// Runtime layer of the candidate-set invariant: even a compromised
|
|
263
268
|
// iterator cannot reach an adapter with a pair outside the declared list.
|
|
@@ -286,7 +291,7 @@ async function run(deps, routeName, opts) {
|
|
|
286
291
|
const flags = deps.capabilities(candidate.provider, candidate.model);
|
|
287
292
|
let systemPrompt = (0, template_1.renderTemplate)(route.systemTemplate, {
|
|
288
293
|
variables: opts.input,
|
|
289
|
-
capabilities: flags,
|
|
294
|
+
capabilities: flags, variableTrust: route.variableTrust,
|
|
290
295
|
});
|
|
291
296
|
// Variant augmentation (WS1): appended after capability-conditionals,
|
|
292
297
|
// before json coaching — promptAppend first, then the reasoning scaffold.
|
|
@@ -337,7 +342,7 @@ async function run(deps, routeName, opts) {
|
|
|
337
342
|
continue;
|
|
338
343
|
}
|
|
339
344
|
}
|
|
340
|
-
systemPrompt += (0, grounding_1.
|
|
345
|
+
systemPrompt += (0, grounding_1.groundingSystemAppend)(route.groundingAssembly, searchResponse.results, new Date().toISOString());
|
|
341
346
|
injectedSources = searchResponse.results;
|
|
342
347
|
}
|
|
343
348
|
// json_mode rung (route-level) or a candidate without native strict
|
|
@@ -359,21 +364,19 @@ async function run(deps, routeName, opts) {
|
|
|
359
364
|
// CE-1 (spec §2.2, run/cache-handle.ts): `suppressed` drops the customer handle after a cache_invalid.
|
|
360
365
|
const candidateKey = `${candidate.provider} ${candidate.model}`;
|
|
361
366
|
const suppressed = droppedCacheHandle.has(candidateKey);
|
|
362
|
-
//
|
|
363
|
-
|
|
364
|
-
// buffers via call() — the returned result is identical either way.
|
|
365
|
-
const request = (0, dispatch_1.buildAdapterRequest)({
|
|
367
|
+
// Build + mask (spec §1.2 — user slot only) + dispatch as one seam; null mask ⇒ byte-identical.
|
|
368
|
+
const dispatched = await (0, dispatch_1.buildDispatchMaybeMasked)(adapter, {
|
|
366
369
|
route,
|
|
367
370
|
candidate,
|
|
368
|
-
systemPrompt,
|
|
369
|
-
userDirective: USER_DIRECTIVE,
|
|
371
|
+
systemPrompt: systemPrompt + (0, grounding_1.untrustedSystemAppend)(route, opts.input, injectedSources),
|
|
372
|
+
userDirective: USER_DIRECTIVE + (0, grounding_1.untrustedUserBlock)(route, opts.input, injectedSources, candidate.provider),
|
|
370
373
|
nativeGrounded,
|
|
371
374
|
opts,
|
|
372
375
|
suppressCustomerCache: suppressed,
|
|
373
|
-
});
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
const latencyMs
|
|
376
|
+
}, deps.onStreamDelta, maskCtx);
|
|
377
|
+
if (!dispatched.result.ok && dispatched.maskRefusal !== undefined)
|
|
378
|
+
failRun(candidate, dispatched.result.failure, 0);
|
|
379
|
+
const { result, latencyMs, seam: maskSeam } = dispatched;
|
|
377
380
|
if (result.ok) {
|
|
378
381
|
const { costUsd, envelopeChargeUsd, pricingMissing } = (0, cost_1.attemptCost)(candidate.provider, candidate.model, result.usage, unpricedRatePerTok);
|
|
379
382
|
guard?.settle(envelopeChargeUsd);
|
|
@@ -421,15 +424,18 @@ async function run(deps, routeName, opts) {
|
|
|
421
424
|
continue;
|
|
422
425
|
}
|
|
423
426
|
const acceptTags = gate !== null ? (0, quality_gate_1.qualityTags)(gate) : undefined;
|
|
424
|
-
const
|
|
427
|
+
const output = maskSeam !== null ? maskSeam.rehydrateOutput(outcome.parsed) : outcome.parsed; // rehydrate post-validate (spec §5)
|
|
428
|
+
const maskReceipt = maskSeam?.receipt(); // read AFTER rehydrate so counts are final
|
|
429
|
+
const inferenceId = recordAttempt({ ...(0, accept_1.successAttemptRow)(candidate, result.usage, costUsd, latencyMs, pricingMissing, null, metaInputs, acceptTags), ...(maskReceipt !== undefined ? { mask: maskReceipt } : {}) });
|
|
425
430
|
captureAttempt(inferenceId, result.text);
|
|
426
431
|
const meta = (0, accept_1.buildSuccessMeta)({
|
|
427
432
|
inferenceId, route, candidate, usage: result.usage, costUsd, latencyMs,
|
|
428
433
|
attemptsByClass: ledger.snapshot(), servedVariant, callId: ladder.callId,
|
|
429
|
-
...(injectedSources !== null ?
|
|
434
|
+
...(injectedSources !== null ? (0, grounding_1.groundingMeta)(result.text, injectedSources) : {}),
|
|
430
435
|
...(gate !== null ? { quality: { score: gate.score, verdict: "pass", judgedBy: null } } : {}),
|
|
436
|
+
...(maskReceipt !== undefined ? { mask: maskReceipt } : {}),
|
|
431
437
|
});
|
|
432
|
-
return { output
|
|
438
|
+
return { output, meta };
|
|
433
439
|
}
|
|
434
440
|
// Schema-invalid output — a real (billed) attempt that failed.
|
|
435
441
|
const failure = {
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Screening — content provenance / assembly (WIRED 2026-08-23, screening B3).
|
|
3
|
+
* Spec §2. The load-bearing DETERMINISTIC control: every segment the SDK
|
|
4
|
+
* assembles carries {source, tier}, and the ASSEMBLER — not a detector —
|
|
5
|
+
* decides where each tier may appear. The structural invariant it enforces is
|
|
6
|
+
* "untrusted content never occupies the system slot"; trusted (`template`) and
|
|
7
|
+
* semi (`variable` by default) render into the system slot as today, so a route
|
|
8
|
+
* with no `x-trust` annotation and `grounding.assembly: legacy` is byte-identical.
|
|
9
|
+
*
|
|
10
|
+
* This is the SC-0 seam that must land before CS-1 Chunk A (masking wraps the
|
|
11
|
+
* same assembled request). It replaces, behind the `grounding.assembly:
|
|
12
|
+
* spotlight` knob and the `x-trust: untrusted` opt-in, the ad-hoc
|
|
13
|
+
* `systemPrompt += …` concatenations audited as INJ-1 (src/run/run.ts:475),
|
|
14
|
+
* INJ-3 (src/routes/template.ts:92-103 → src/run/run.ts:420-431) and INJ-2
|
|
15
|
+
* (packages/modelrig-server raw-completions.ts:60-76).
|
|
16
|
+
*/
|
|
17
|
+
import type { ContentSource, TrustTier } from "../detect/detector";
|
|
18
|
+
/** One assembled segment with its provenance. The assembler groups by tier so a
|
|
19
|
+
* receipt can attribute a finding to a source without holding the text
|
|
20
|
+
* downstream (the ledger here is ephemeral, used only at assembly time). */
|
|
21
|
+
export interface ContentSegment {
|
|
22
|
+
readonly source: ContentSource;
|
|
23
|
+
readonly tier: TrustTier;
|
|
24
|
+
readonly text: string;
|
|
25
|
+
}
|
|
26
|
+
/** The two provider slots after assembly. Untrusted segments are spotlighted
|
|
27
|
+
* into the user slot (JSON-encoded block, randomized delimiter) or, for an
|
|
28
|
+
* adapter that reports `untrusted_block: tool_result`, framed as a synthetic
|
|
29
|
+
* tool_result envelope (Anthropic guidance, research §2.3). */
|
|
30
|
+
export interface AssembledPrompt {
|
|
31
|
+
readonly system: string;
|
|
32
|
+
readonly user: string;
|
|
33
|
+
readonly segments: readonly ContentSegment[];
|
|
34
|
+
}
|
|
35
|
+
/** The policy sentence placed in the SYSTEM slot whenever an untrusted block is
|
|
36
|
+
* spotlighted into the user slot (spec §2.2). Verbatim — the docs page and the
|
|
37
|
+
* assembler share this constant. Not outward marketing copy (claim-lint §7 does
|
|
38
|
+
* not apply); it is prompt scaffolding delivered to the model. */
|
|
39
|
+
export declare const UNTRUSTED_POLICY_SENTENCE: string;
|
|
40
|
+
/**
|
|
41
|
+
* Default trust tier for a source (spec §2.1). A route's `screen.sources` /
|
|
42
|
+
* `x-trust` may TIGHTEN (never loosen) these.
|
|
43
|
+
*/
|
|
44
|
+
export declare function defaultTier(source: ContentSource): TrustTier;
|
|
45
|
+
/** Strip Unicode tag characters (U+E0000–U+E007F) — the invisible plane abused
|
|
46
|
+
* to smuggle instructions past a human reviewer — from untrusted text before it
|
|
47
|
+
* is spotlighted (spec §2.2). */
|
|
48
|
+
export declare function stripUnicodeTags(text: string): string;
|
|
49
|
+
/**
|
|
50
|
+
* Assemble segments into the two slots, enforcing "untrusted never in system"
|
|
51
|
+
* and spotlighting untrusted content per the adapter's `untrusted_block`
|
|
52
|
+
* capability. Randomized delimiter per run; JSON-encode untrusted blocks; strip
|
|
53
|
+
* Unicode tag chars. Pure over the segment list + a per-run nonce.
|
|
54
|
+
*/
|
|
55
|
+
export declare function assemblePrompt(segments: readonly ContentSegment[], opts: {
|
|
56
|
+
readonly nonce: string;
|
|
57
|
+
readonly untrustedBlock: "tool_result" | "user_json";
|
|
58
|
+
}): AssembledPrompt;
|
|
59
|
+
//# sourceMappingURL=provenance.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"provenance.d.ts","sourceRoot":"","sources":["../../src/screen/provenance.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAEnE;;4EAE4E;AAC5E,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC;IAC/B,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAED;;;+DAG+D;AAC/D,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,SAAS,cAAc,EAAE,CAAC;CAC9C;AAED;;;kEAGkE;AAClE,eAAO,MAAM,yBAAyB,QAEkC,CAAC;AAEzE;;;GAGG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,aAAa,GAAG,SAAS,CAmB5D;AAED;;iCAEiC;AACjC,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAErD;AAiBD;;;;;GAKG;AACH,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,SAAS,cAAc,EAAE,EACnC,IAAI,EAAE;IAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,cAAc,EAAE,aAAa,GAAG,WAAW,CAAA;CAAE,GACrF,eAAe,CA8BjB"}
|