@responsibleai/agent-hooks 0.1.0-beta.1 → 0.1.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +67 -2
- package/binding.d.ts +25 -0
- package/binding.js +58 -54
- package/dist/ctk/index.d.ts +44 -5
- package/dist/ctk/index.js +5 -1
- package/dist/ctk/reference.d.ts +15 -1
- package/dist/ctk/reference.declaration.json +33 -0
- package/dist/ctk/reference.js +84 -31
- package/dist/ctk/runner.d.ts +34 -4
- package/dist/ctk/runner.js +564 -36
- package/dist/declaration.d.ts +393 -0
- package/dist/declaration.js +752 -0
- package/dist/emitter.d.ts +68 -14
- package/dist/emitter.js +279 -42
- package/dist/index.d.ts +14 -3
- package/dist/index.js +26 -1
- package/dist/native.d.ts +8 -1
- package/dist/native.js +9 -1
- package/package.json +6 -6
package/dist/ctk/reference.js
CHANGED
|
@@ -7,57 +7,110 @@
|
|
|
7
7
|
* The simplest possible conformant agent loop; exists so the
|
|
8
8
|
* CTK can self-test without a real framework. Port of
|
|
9
9
|
* `sdk/python/python/agent_hooks/ctk/reference.py`.
|
|
10
|
+
*
|
|
11
|
+
* Every emitter it builds goes through the host declaration loader
|
|
12
|
+
* (§7.7): for a field-based vector it writes the vector's mode,
|
|
13
|
+
* composition and provider into a copy of its own document
|
|
14
|
+
* (`reference.declaration.json`) and binds the scripted interceptors
|
|
15
|
+
* through a `ctk.instance` kind, so the field-based vectors exercise
|
|
16
|
+
* the loader too.
|
|
10
17
|
*/
|
|
18
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
19
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
20
|
+
};
|
|
11
21
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
-
exports.ReferenceHarness = void 0;
|
|
22
|
+
exports.ReferenceHarness = exports.REFERENCE_DECLARATION = void 0;
|
|
13
23
|
const node_crypto_1 = require("node:crypto");
|
|
14
24
|
const index_1 = require("../index");
|
|
15
25
|
const builder_1 = require("../builder");
|
|
16
26
|
const emitter_1 = require("../emitter");
|
|
17
|
-
const
|
|
27
|
+
const runner_1 = require("./runner");
|
|
28
|
+
const reference_declaration_json_1 = __importDefault(require("./reference.declaration.json"));
|
|
29
|
+
/** The reference harness's own declaration (§7.7.9), with an explicit
|
|
30
|
+
* surface a claim can cite. */
|
|
31
|
+
exports.REFERENCE_DECLARATION = Object.freeze(reference_declaration_json_1.default);
|
|
18
32
|
class ReferenceHarness {
|
|
19
33
|
name = "reference-agent";
|
|
20
|
-
|
|
34
|
+
// host_declaration: every emitter is built through the loader.
|
|
35
|
+
// JSON.parse rounds integers beyond 2^53 before any guard can run,
|
|
36
|
+
// so neither int64_json nor bigint_json is declared (§4.4).
|
|
37
|
+
capabilities = new Set([
|
|
38
|
+
"model_calls",
|
|
39
|
+
"tool_calls",
|
|
40
|
+
"host_declaration",
|
|
41
|
+
]);
|
|
21
42
|
scenario;
|
|
22
43
|
emitter;
|
|
23
44
|
builder;
|
|
24
45
|
toolLog = [];
|
|
25
|
-
|
|
46
|
+
hostSurface() {
|
|
47
|
+
return index_1.HostSurface.fromCapabilities(this.capabilities, "continue");
|
|
48
|
+
}
|
|
49
|
+
declaration() {
|
|
50
|
+
return JSON.parse(JSON.stringify(exports.REFERENCE_DECLARATION));
|
|
51
|
+
}
|
|
52
|
+
startSession(scenario, emitter) {
|
|
26
53
|
this.scenario = scenario;
|
|
27
54
|
this.toolLog = [];
|
|
28
|
-
|
|
29
|
-
em.setComposition(composition);
|
|
30
|
-
if (redactForApproval.length > 0) {
|
|
31
|
-
// §9 redaction seam, CTK convention: each listed path is replaced
|
|
32
|
-
// with "[redacted]" via the §5.2/§4.3 transform machinery;
|
|
33
|
-
// unresolvable paths are left untouched.
|
|
34
|
-
em.setApprovalRedactor((ctx) => {
|
|
35
|
-
let out = JSON.stringify(ctx);
|
|
36
|
-
for (const path of redactForApproval) {
|
|
37
|
-
try {
|
|
38
|
-
out = native_1.native.applyTransformCtx(out, path, '"[redacted]"');
|
|
39
|
-
}
|
|
40
|
-
catch {
|
|
41
|
-
/* unresolvable at this point — skip */
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
return JSON.parse(out);
|
|
45
|
-
});
|
|
46
|
-
}
|
|
47
|
-
// §13.2: "ctk-fault" is a custom provider that throws, pinning the
|
|
48
|
-
// §10.1 provider-failure rule (deny context_invalid pre-dispatch).
|
|
49
|
-
em.setIdentityProvider(identityProvider === 'ctk-fault'
|
|
50
|
-
? { name: 'ctk-fault', fn: () => { throw new Error('ctk scripted provider fault'); } }
|
|
51
|
-
: identityProvider);
|
|
52
|
-
for (const i of interceptors)
|
|
53
|
-
em.register(i);
|
|
54
|
-
this.emitter = em;
|
|
55
|
+
this.emitter = emitter;
|
|
55
56
|
this.builder = new builder_1.AgentContextBuilder({
|
|
56
57
|
agentId: "ref-agent",
|
|
57
58
|
framework: "reference-agent",
|
|
58
59
|
sessionId: (0, node_crypto_1.randomUUID)(),
|
|
59
60
|
});
|
|
60
61
|
}
|
|
62
|
+
setup(scenario, interceptors, resolver, mode, composition, identityProvider, redactForApproval = []) {
|
|
63
|
+
// Field-based vector: write the vector's configuration into a copy
|
|
64
|
+
// of the reference document and bind the interceptors by index
|
|
65
|
+
// through the `ctk.instance` kind.
|
|
66
|
+
const doc = this.declaration();
|
|
67
|
+
const cfg = doc["configuration"];
|
|
68
|
+
cfg["mode"] = mode;
|
|
69
|
+
cfg["composition"] = composition;
|
|
70
|
+
const registry = index_1.HostRegistry.forConformance(this.hostSurface());
|
|
71
|
+
if (identityProvider === "ctk-fault") {
|
|
72
|
+
// §13.2: "ctk-fault" is a custom provider that throws, pinning the
|
|
73
|
+
// §10.1 provider-failure rule (deny context_invalid pre-dispatch).
|
|
74
|
+
registry.identityProvider("ctk-fault", () => {
|
|
75
|
+
throw new Error("ctk scripted provider fault");
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
cfg["identity_provider"] = identityProvider;
|
|
79
|
+
const approval = { resolver: null, redactor: null };
|
|
80
|
+
if (resolver !== null) {
|
|
81
|
+
registry.approvalResolver("ctk-scripted", resolver);
|
|
82
|
+
approval["resolver"] = "ctk-scripted";
|
|
83
|
+
}
|
|
84
|
+
if (redactForApproval.length > 0) {
|
|
85
|
+
const paths = [...redactForApproval];
|
|
86
|
+
registry.approvalRedactor("ctk-redact", (ctx) => (0, runner_1.redactPaths)(ctx, paths));
|
|
87
|
+
approval["redactor"] = "ctk-redact";
|
|
88
|
+
}
|
|
89
|
+
cfg["approval"] = approval;
|
|
90
|
+
const slots = [...interceptors];
|
|
91
|
+
registry.kind("ctk.instance", (config, ctx) => {
|
|
92
|
+
const index = config?.["index"];
|
|
93
|
+
if (typeof index !== "number" || !Number.isInteger(index) || index < 0) {
|
|
94
|
+
throw new Error("config.index must be an unsigned integer");
|
|
95
|
+
}
|
|
96
|
+
const instance = slots[index];
|
|
97
|
+
if (instance === undefined) {
|
|
98
|
+
throw new Error(`no interceptor instance ${index} for binding ${ctx.id}`);
|
|
99
|
+
}
|
|
100
|
+
slots[index] = undefined;
|
|
101
|
+
return instance;
|
|
102
|
+
});
|
|
103
|
+
doc["bindings"] = interceptors.map((_, i) => ({
|
|
104
|
+
id: `interceptor-${i}`,
|
|
105
|
+
kind: "ctk.instance",
|
|
106
|
+
config: { index: i },
|
|
107
|
+
}));
|
|
108
|
+
this.startSession(scenario, emitter_1.InterceptionEmitter.fromDeclarationValue(doc, registry));
|
|
109
|
+
}
|
|
110
|
+
setupDeclared(scenario, document, registry) {
|
|
111
|
+
// A refusal (DeclarationError) propagates to the runner (§7.7.9).
|
|
112
|
+
this.startSession(scenario, emitter_1.InterceptionEmitter.fromDeclarationValue(document, registry));
|
|
113
|
+
}
|
|
61
114
|
async run() {
|
|
62
115
|
const s = this.scenario;
|
|
63
116
|
const em = this.emitter;
|
package/dist/ctk/runner.d.ts
CHANGED
|
@@ -3,18 +3,48 @@
|
|
|
3
3
|
*
|
|
4
4
|
* The assertion engine, capability skip check, and scripted
|
|
5
5
|
* interceptor/resolver evaluation live in the Rust core (native.ctk*).
|
|
6
|
-
* This module keeps only vector globbing, the recording wrapper,
|
|
7
|
-
*
|
|
6
|
+
* This module keeps only vector globbing, the recording wrapper, the
|
|
7
|
+
* host declaration seam (§7.7.9: the CTK registry, the construction
|
|
8
|
+
* path proof and the `load` outcome) and the orchestration loop that
|
|
9
|
+
* calls the native `Harness`.
|
|
8
10
|
*/
|
|
9
|
-
import { JsonValue } from "../index";
|
|
11
|
+
import { AgentContext, DeclarationBuilder, HostRegistry, JsonValue } from "../index";
|
|
10
12
|
import type { Harness } from "./index";
|
|
11
13
|
export interface VectorResult {
|
|
12
14
|
id: string;
|
|
13
15
|
title: string;
|
|
16
|
+
part?: string;
|
|
14
17
|
status: "pass" | "fail" | "skip";
|
|
15
18
|
detail: string;
|
|
16
19
|
failures: string[];
|
|
17
20
|
}
|
|
18
21
|
export declare function loadVectors(dir: string): JsonValue[];
|
|
19
|
-
|
|
22
|
+
/** §9 redaction seam, CTK convention: each listed path is replaced
|
|
23
|
+
* with "[redacted]" via the §5.2/§4.3 transform machinery; a path that
|
|
24
|
+
* does not resolve at the escalating point is left untouched. */
|
|
25
|
+
export declare function redactPaths(ctx: AgentContext, paths: readonly string[]): AgentContext;
|
|
26
|
+
/** Rebuild a document through {@link DeclarationBuilder}, member by
|
|
27
|
+
* member (§7.7.7, the code path). Members the typed setters cannot
|
|
28
|
+
* express exactly (an unknown member, a wrong type) go through
|
|
29
|
+
* {@link DeclarationBuilder.raw}, so the result is validated like the
|
|
30
|
+
* file it came from. Mirrors the Rust runner's `builder_from_value`. */
|
|
31
|
+
export declare function builderFromValue(doc: JsonValue): DeclarationBuilder;
|
|
32
|
+
/** Resolve `doc` through the four construction paths (§7.7.7) and
|
|
33
|
+
* compare: value, JSON text, a temporary file and the builder. Equal
|
|
34
|
+
* canonical forms, or equal refusal classes, prove the paths
|
|
35
|
+
* equivalent. Returns `[equivalent, detail]`. */
|
|
36
|
+
export declare function provePaths(doc: JsonValue, registry: HostRegistry, tag: string): Promise<[boolean, string]>;
|
|
37
|
+
/** What a run assesses the harness against: its capabilities and
|
|
38
|
+
* posture, read from its resolved declaration when it ships one
|
|
39
|
+
* (§7.7.9) and from the Harness fields otherwise. */
|
|
40
|
+
export interface AssessedSurface {
|
|
41
|
+
capabilities: string[];
|
|
42
|
+
posture: string;
|
|
43
|
+
}
|
|
44
|
+
/** Resolve the harness's own declaration against its code surface.
|
|
45
|
+
* Throws {@link DeclarationError} on refusal. {@link runVectors} calls
|
|
46
|
+
* this once before the first vector, as §7.7.9 asks; {@link runVector}
|
|
47
|
+
* calls it when no resolved surface is handed in. */
|
|
48
|
+
export declare function assessHarness(harness: Harness): AssessedSurface;
|
|
49
|
+
export declare function runVector(harness: Harness, vector: JsonValue, assessed?: AssessedSurface): Promise<VectorResult>;
|
|
20
50
|
export declare function runVectors(harnessFactory: () => Harness, vectors: JsonValue[]): Promise<VectorResult[]>;
|