@attest-ai/core 0.4.2 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/index.cjs +2595 -0
- package/dist/cjs/index.cjs.map +1 -0
- package/dist/cjs/index.d.cts +1111 -0
- package/dist/cjs/proto/index.cjs +208 -0
- package/dist/cjs/proto/index.cjs.map +1 -0
- package/dist/cjs/proto/index.d.cts +269 -0
- package/dist/esm/cli.d.ts +9 -0
- package/dist/esm/cli.js +154 -0
- package/dist/esm/cli.js.map +1 -0
- package/dist/esm/index.d.ts +1111 -0
- package/dist/esm/index.js +2490 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/proto/index.d.ts +269 -0
- package/dist/esm/proto/index.js +152 -0
- package/dist/esm/proto/index.js.map +1 -0
- package/package.json +37 -11
- package/dist/adapters/anthropic.d.ts +0 -31
- package/dist/adapters/anthropic.d.ts.map +0 -1
- package/dist/adapters/anthropic.js +0 -42
- package/dist/adapters/anthropic.js.map +0 -1
- package/dist/adapters/gemini.d.ts +0 -30
- package/dist/adapters/gemini.d.ts.map +0 -1
- package/dist/adapters/gemini.js +0 -41
- package/dist/adapters/gemini.js.map +0 -1
- package/dist/adapters/index.d.ts +0 -11
- package/dist/adapters/index.d.ts.map +0 -1
- package/dist/adapters/index.js +0 -7
- package/dist/adapters/index.js.map +0 -1
- package/dist/adapters/manual.d.ts +0 -9
- package/dist/adapters/manual.d.ts.map +0 -1
- package/dist/adapters/manual.js +0 -16
- package/dist/adapters/manual.js.map +0 -1
- package/dist/adapters/ollama.d.ts +0 -19
- package/dist/adapters/ollama.d.ts.map +0 -1
- package/dist/adapters/ollama.js +0 -30
- package/dist/adapters/ollama.js.map +0 -1
- package/dist/adapters/openai.d.ts +0 -34
- package/dist/adapters/openai.d.ts.map +0 -1
- package/dist/adapters/openai.js +0 -42
- package/dist/adapters/openai.js.map +0 -1
- package/dist/adapters/otel.d.ts +0 -25
- package/dist/adapters/otel.d.ts.map +0 -1
- package/dist/adapters/otel.js +0 -133
- package/dist/adapters/otel.js.map +0 -1
- package/dist/agent.d.ts +0 -14
- package/dist/agent.d.ts.map +0 -1
- package/dist/agent.js +0 -54
- package/dist/agent.js.map +0 -1
- package/dist/client.d.ts +0 -19
- package/dist/client.d.ts.map +0 -1
- package/dist/client.js +0 -132
- package/dist/client.js.map +0 -1
- package/dist/context.d.ts +0 -4
- package/dist/context.d.ts.map +0 -1
- package/dist/context.js +0 -3
- package/dist/context.js.map +0 -1
- package/dist/delegate.d.ts +0 -3
- package/dist/delegate.d.ts.map +0 -1
- package/dist/delegate.js +0 -30
- package/dist/delegate.js.map +0 -1
- package/dist/engine-downloader.d.ts +0 -7
- package/dist/engine-downloader.d.ts.map +0 -1
- package/dist/engine-downloader.js +0 -136
- package/dist/engine-downloader.js.map +0 -1
- package/dist/engine-manager.d.ts +0 -26
- package/dist/engine-manager.d.ts.map +0 -1
- package/dist/engine-manager.js +0 -172
- package/dist/engine-manager.js.map +0 -1
- package/dist/expect.d.ts +0 -133
- package/dist/expect.d.ts.map +0 -1
- package/dist/expect.js +0 -330
- package/dist/expect.js.map +0 -1
- package/dist/index.d.ts +0 -15
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js +0 -14
- package/dist/index.js.map +0 -1
- package/dist/proto/codec.d.ts +0 -20
- package/dist/proto/codec.d.ts.map +0 -1
- package/dist/proto/codec.js +0 -71
- package/dist/proto/codec.js.map +0 -1
- package/dist/proto/constants.d.ts +0 -21
- package/dist/proto/constants.d.ts.map +0 -1
- package/dist/proto/constants.js +0 -25
- package/dist/proto/constants.js.map +0 -1
- package/dist/proto/errors.d.ts +0 -8
- package/dist/proto/errors.d.ts.map +0 -1
- package/dist/proto/errors.js +0 -13
- package/dist/proto/errors.js.map +0 -1
- package/dist/proto/index.d.ts +0 -5
- package/dist/proto/index.d.ts.map +0 -1
- package/dist/proto/index.js +0 -5
- package/dist/proto/index.js.map +0 -1
- package/dist/proto/types.d.ts +0 -104
- package/dist/proto/types.d.ts.map +0 -1
- package/dist/proto/types.js +0 -2
- package/dist/proto/types.js.map +0 -1
- package/dist/result.d.ts +0 -15
- package/dist/result.d.ts.map +0 -1
- package/dist/result.js +0 -32
- package/dist/result.js.map +0 -1
- package/dist/tier.d.ts +0 -5
- package/dist/tier.d.ts.map +0 -1
- package/dist/tier.js +0 -10
- package/dist/tier.js.map +0 -1
- package/dist/trace-tree.d.ts +0 -20
- package/dist/trace-tree.d.ts.map +0 -1
- package/dist/trace-tree.js +0 -92
- package/dist/trace-tree.js.map +0 -1
- package/dist/trace.d.ts +0 -40
- package/dist/trace.d.ts.map +0 -1
- package/dist/trace.js +0 -85
- package/dist/trace.js.map +0 -1
- package/dist/version.d.ts +0 -3
- package/dist/version.d.ts.map +0 -1
- package/dist/version.js +0 -3
- package/dist/version.js.map +0 -1
package/dist/engine-manager.d.ts
DELETED
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
import { type ChildProcess } from "node:child_process";
|
|
2
|
-
import { type Interface as ReadlineInterface } from "node:readline";
|
|
3
|
-
import type { InitializeResult } from "./proto/types.js";
|
|
4
|
-
export declare class EngineManager {
|
|
5
|
-
private enginePath;
|
|
6
|
-
private readonly logLevel;
|
|
7
|
-
private process;
|
|
8
|
-
private reader;
|
|
9
|
-
private initialized;
|
|
10
|
-
private requestId;
|
|
11
|
-
private initResult;
|
|
12
|
-
constructor(options?: {
|
|
13
|
-
enginePath?: string;
|
|
14
|
-
logLevel?: string;
|
|
15
|
-
});
|
|
16
|
-
start(): Promise<InitializeResult>;
|
|
17
|
-
stop(): Promise<void>;
|
|
18
|
-
sendRequest(method: string, params: Record<string, unknown>): Promise<unknown>;
|
|
19
|
-
private sendRequestInternal;
|
|
20
|
-
get isRunning(): boolean;
|
|
21
|
-
/** Expose process for AttestClient reader loop access */
|
|
22
|
-
get childProcess(): ChildProcess | null;
|
|
23
|
-
/** Expose readline interface for AttestClient reader loop */
|
|
24
|
-
get readlineInterface(): ReadlineInterface | null;
|
|
25
|
-
}
|
|
26
|
-
//# sourceMappingURL=engine-manager.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"engine-manager.d.ts","sourceRoot":"","sources":["../src/engine-manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAmB,KAAK,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACxE,OAAO,EAAmB,KAAK,SAAS,IAAI,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAIrF,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AA+DzD,qBAAa,aAAa;IACxB,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,OAAO,CAA6B;IAC5C,OAAO,CAAC,MAAM,CAAkC;IAChD,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,UAAU,CAAiC;gBAEvC,OAAO,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE;IAK1D,KAAK,IAAI,OAAO,CAAC,gBAAgB,CAAC;IA6BlC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAiCrB,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC;YAOtE,mBAAmB;IAmCjC,IAAI,SAAS,IAAI,OAAO,CAEvB;IAED,yDAAyD;IACzD,IAAI,YAAY,IAAI,YAAY,GAAG,IAAI,CAEtC;IAED,6DAA6D;IAC7D,IAAI,iBAAiB,IAAI,iBAAiB,GAAG,IAAI,CAEhD;CACF"}
|
package/dist/engine-manager.js
DELETED
|
@@ -1,172 +0,0 @@
|
|
|
1
|
-
import { spawn, execSync } from "node:child_process";
|
|
2
|
-
import { createInterface } from "node:readline";
|
|
3
|
-
import { existsSync, accessSync, constants as fsConstants } from "node:fs";
|
|
4
|
-
import { resolve, join, dirname } from "node:path";
|
|
5
|
-
import { fileURLToPath } from "node:url";
|
|
6
|
-
import { decodeResponse, encodeRequest, extractResult } from "./proto/codec.js";
|
|
7
|
-
import { VERSION } from "./version.js";
|
|
8
|
-
import { cachedEnginePath, downloadEngine } from "./engine-downloader.js";
|
|
9
|
-
const ENGINE_BINARY_NAME = "attest-engine";
|
|
10
|
-
async function findEngineBinary() {
|
|
11
|
-
// 1. Explicit env var override
|
|
12
|
-
const envPath = process.env["ATTEST_ENGINE_PATH"];
|
|
13
|
-
if (envPath) {
|
|
14
|
-
if (!existsSync(envPath)) {
|
|
15
|
-
throw new Error(`ATTEST_ENGINE_PATH is set to '${envPath}' but the file does not exist.`);
|
|
16
|
-
}
|
|
17
|
-
return envPath;
|
|
18
|
-
}
|
|
19
|
-
// 2. Check PATH via which/where
|
|
20
|
-
try {
|
|
21
|
-
const cmd = process.platform === "win32" ? "where" : "which";
|
|
22
|
-
const found = execSync(`${cmd} ${ENGINE_BINARY_NAME}`, { encoding: "utf-8" }).trim();
|
|
23
|
-
if (found)
|
|
24
|
-
return found;
|
|
25
|
-
}
|
|
26
|
-
catch {
|
|
27
|
-
// not on PATH
|
|
28
|
-
}
|
|
29
|
-
// 3. Cached download in ~/.attest/bin/
|
|
30
|
-
const cached = cachedEnginePath();
|
|
31
|
-
if (cached)
|
|
32
|
-
return cached;
|
|
33
|
-
// 4. Monorepo dev layout (../../bin/ relative to this file)
|
|
34
|
-
// 5. CWD ./bin/
|
|
35
|
-
const candidates = [
|
|
36
|
-
join(dirname(fileURLToPath(import.meta.url)), "..", "..", "..", "..", "..", "bin", ENGINE_BINARY_NAME),
|
|
37
|
-
join(process.cwd(), "bin", ENGINE_BINARY_NAME)
|
|
38
|
-
];
|
|
39
|
-
for (const candidate of candidates) {
|
|
40
|
-
const resolved = resolve(candidate);
|
|
41
|
-
try {
|
|
42
|
-
accessSync(resolved, fsConstants.X_OK);
|
|
43
|
-
if (existsSync(resolved))
|
|
44
|
-
return resolved;
|
|
45
|
-
}
|
|
46
|
-
catch {
|
|
47
|
-
// not accessible
|
|
48
|
-
}
|
|
49
|
-
}
|
|
50
|
-
// 6. Auto-download
|
|
51
|
-
const noDownload = process.env["ATTEST_ENGINE_NO_DOWNLOAD"];
|
|
52
|
-
if (noDownload === "1" || noDownload === "true" || noDownload === "yes") {
|
|
53
|
-
throw new Error(`Cannot find '${ENGINE_BINARY_NAME}' binary and auto-download is disabled ` +
|
|
54
|
-
"(ATTEST_ENGINE_NO_DOWNLOAD is set). Install the engine manually:\n" +
|
|
55
|
-
" 1. Download from https://github.com/attest-framework/attest/releases\n" +
|
|
56
|
-
" 2. Place the binary on your PATH or set ATTEST_ENGINE_PATH");
|
|
57
|
-
}
|
|
58
|
-
return downloadEngine();
|
|
59
|
-
}
|
|
60
|
-
export class EngineManager {
|
|
61
|
-
enginePath;
|
|
62
|
-
logLevel;
|
|
63
|
-
process = null;
|
|
64
|
-
reader = null;
|
|
65
|
-
initialized = false;
|
|
66
|
-
requestId = 0;
|
|
67
|
-
initResult = null;
|
|
68
|
-
constructor(options) {
|
|
69
|
-
this.enginePath = options?.enginePath ?? "";
|
|
70
|
-
this.logLevel = options?.logLevel ?? "warn";
|
|
71
|
-
}
|
|
72
|
-
async start() {
|
|
73
|
-
if (!this.enginePath) {
|
|
74
|
-
this.enginePath = await findEngineBinary();
|
|
75
|
-
}
|
|
76
|
-
this.process = spawn(this.enginePath, [`--log-level=${this.logLevel}`], {
|
|
77
|
-
stdio: ["pipe", "pipe", "pipe"],
|
|
78
|
-
});
|
|
79
|
-
this.reader = createInterface({ input: this.process.stdout });
|
|
80
|
-
const result = await this.sendRequestInternal("initialize", {
|
|
81
|
-
sdk_name: "attest-typescript",
|
|
82
|
-
sdk_version: VERSION,
|
|
83
|
-
protocol_version: 1,
|
|
84
|
-
required_capabilities: ["layers_1_4"],
|
|
85
|
-
preferred_encoding: "json",
|
|
86
|
-
});
|
|
87
|
-
this.initResult = result;
|
|
88
|
-
if (!this.initResult.compatible) {
|
|
89
|
-
throw new Error(`Engine incompatible. Missing capabilities: ${JSON.stringify(this.initResult.missing)}`);
|
|
90
|
-
}
|
|
91
|
-
this.initialized = true;
|
|
92
|
-
return this.initResult;
|
|
93
|
-
}
|
|
94
|
-
async stop() {
|
|
95
|
-
if (this.process === null)
|
|
96
|
-
return;
|
|
97
|
-
if (this.initialized) {
|
|
98
|
-
try {
|
|
99
|
-
await this.sendRequestInternal("shutdown", {});
|
|
100
|
-
}
|
|
101
|
-
catch {
|
|
102
|
-
// shutdown failed, will kill
|
|
103
|
-
}
|
|
104
|
-
}
|
|
105
|
-
if (this.reader) {
|
|
106
|
-
this.reader.close();
|
|
107
|
-
this.reader = null;
|
|
108
|
-
}
|
|
109
|
-
if (this.process.exitCode === null) {
|
|
110
|
-
this.process.kill("SIGTERM");
|
|
111
|
-
await new Promise((res) => {
|
|
112
|
-
const timer = setTimeout(() => {
|
|
113
|
-
this.process?.kill("SIGKILL");
|
|
114
|
-
}, 5000);
|
|
115
|
-
this.process.on("exit", () => {
|
|
116
|
-
clearTimeout(timer);
|
|
117
|
-
res();
|
|
118
|
-
});
|
|
119
|
-
});
|
|
120
|
-
}
|
|
121
|
-
this.initialized = false;
|
|
122
|
-
this.process = null;
|
|
123
|
-
}
|
|
124
|
-
async sendRequest(method, params) {
|
|
125
|
-
if (!this.initialized && method !== "initialize") {
|
|
126
|
-
throw new Error("Engine not initialized. Call start() first.");
|
|
127
|
-
}
|
|
128
|
-
return this.sendRequestInternal(method, params);
|
|
129
|
-
}
|
|
130
|
-
async sendRequestInternal(method, params) {
|
|
131
|
-
if (this.process === null || this.process.stdin === null || this.reader === null) {
|
|
132
|
-
throw new Error("Engine process not started.");
|
|
133
|
-
}
|
|
134
|
-
this.requestId += 1;
|
|
135
|
-
const requestStr = encodeRequest(this.requestId, method, params);
|
|
136
|
-
return new Promise((resolve, reject) => {
|
|
137
|
-
const onLine = (line) => {
|
|
138
|
-
try {
|
|
139
|
-
const response = decodeResponse(line);
|
|
140
|
-
resolve(extractResult(response));
|
|
141
|
-
}
|
|
142
|
-
catch (err) {
|
|
143
|
-
reject(err);
|
|
144
|
-
}
|
|
145
|
-
};
|
|
146
|
-
this.reader.once("line", onLine);
|
|
147
|
-
const ok = this.process.stdin.write(requestStr, (err) => {
|
|
148
|
-
if (err) {
|
|
149
|
-
this.reader.removeListener("line", onLine);
|
|
150
|
-
reject(err);
|
|
151
|
-
}
|
|
152
|
-
});
|
|
153
|
-
if (!ok) {
|
|
154
|
-
this.process.stdin.once("drain", () => {
|
|
155
|
-
// write will proceed after drain
|
|
156
|
-
});
|
|
157
|
-
}
|
|
158
|
-
});
|
|
159
|
-
}
|
|
160
|
-
get isRunning() {
|
|
161
|
-
return this.process !== null && this.process.exitCode === null;
|
|
162
|
-
}
|
|
163
|
-
/** Expose process for AttestClient reader loop access */
|
|
164
|
-
get childProcess() {
|
|
165
|
-
return this.process;
|
|
166
|
-
}
|
|
167
|
-
/** Expose readline interface for AttestClient reader loop */
|
|
168
|
-
get readlineInterface() {
|
|
169
|
-
return this.reader;
|
|
170
|
-
}
|
|
171
|
-
}
|
|
172
|
-
//# sourceMappingURL=engine-manager.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"engine-manager.js","sourceRoot":"","sources":["../src/engine-manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAqB,MAAM,oBAAoB,CAAC;AACxE,OAAO,EAAE,eAAe,EAAuC,MAAM,eAAe,CAAC;AACrF,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,SAAS,IAAI,WAAW,EAAE,MAAM,SAAS,CAAC;AAC3E,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACnD,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAChF,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAE1E,MAAM,kBAAkB,GAAG,eAAe,CAAC;AAE3C,KAAK,UAAU,gBAAgB;IAC7B,+BAA+B;IAC/B,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;IAClD,IAAI,OAAO,EAAE,CAAC;QACZ,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CACb,iCAAiC,OAAO,gCAAgC,CACzE,CAAC;QACJ,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,gCAAgC;IAChC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC;QAC7D,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,GAAG,IAAI,kBAAkB,EAAE,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACrF,IAAI,KAAK;YAAE,OAAO,KAAK,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACP,cAAc;IAChB,CAAC;IAED,uCAAuC;IACvC,MAAM,MAAM,GAAG,gBAAgB,EAAE,CAAC;IAClC,IAAI,MAAM;QAAE,OAAO,MAAM,CAAC;IAE1B,4DAA4D;IAC5D,gBAAgB;IAChB,MAAM,UAAU,GAAG;QACjB,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,kBAAkB,CAAC;QACtG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,kBAAkB,CAAC;KAC/C,CAAC;IAEF,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;QACpC,IAAI,CAAC;YACH,UAAU,CAAC,QAAQ,EAAE,WAAW,CAAC,IAAI,CAAC,CAAC;YACvC,IAAI,UAAU,CAAC,QAAQ,CAAC;gBAAE,OAAO,QAAQ,CAAC;QAC5C,CAAC;QAAC,MAAM,CAAC;YACP,iBAAiB;QACnB,CAAC;IACH,CAAC;IAED,mBAAmB;IACnB,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;IAC5D,IAAI,UAAU,KAAK,GAAG,IAAI,UAAU,KAAK,MAAM,IAAI,UAAU,KAAK,KAAK,EAAE,CAAC;QACxE,MAAM,IAAI,KAAK,CACb,gBAAgB,kBAAkB,yCAAyC;YAC3E,oEAAoE;YACpE,0EAA0E;YAC1E,8DAA8D,CAC/D,CAAC;IACJ,CAAC;IAED,OAAO,cAAc,EAAE,CAAC;AAC1B,CAAC;AAED,MAAM,OAAO,aAAa;IAChB,UAAU,CAAS;IACV,QAAQ,CAAS;IAC1B,OAAO,GAAwB,IAAI,CAAC;IACpC,MAAM,GAA6B,IAAI,CAAC;IACxC,WAAW,GAAG,KAAK,CAAC;IACpB,SAAS,GAAG,CAAC,CAAC;IACd,UAAU,GAA4B,IAAI,CAAC;IAEnD,YAAY,OAAoD;QAC9D,IAAI,CAAC,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,EAAE,CAAC;QAC5C,IAAI,CAAC,QAAQ,GAAG,OAAO,EAAE,QAAQ,IAAI,MAAM,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YACrB,IAAI,CAAC,UAAU,GAAG,MAAM,gBAAgB,EAAE,CAAC;QAC7C,CAAC;QAED,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,eAAe,IAAI,CAAC,QAAQ,EAAE,CAAC,EAAE;YACtE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;SAChC,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,MAAO,EAAE,CAAC,CAAC;QAE/D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,YAAY,EAAE;YAC1D,QAAQ,EAAE,mBAAmB;YAC7B,WAAW,EAAE,OAAO;YACpB,gBAAgB,EAAE,CAAC;YACnB,qBAAqB,EAAE,CAAC,YAAY,CAAC;YACrC,kBAAkB,EAAE,MAAM;SAC3B,CAAC,CAAC;QAEH,IAAI,CAAC,UAAU,GAAG,MAA0B,CAAC;QAC7C,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,UAAU,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CACb,8CAA8C,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CACxF,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QACxB,OAAO,IAAI,CAAC,UAAU,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI;YAAE,OAAO;QAElC,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,mBAAmB,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;YACjD,CAAC;YAAC,MAAM,CAAC;gBACP,6BAA6B;YAC/B,CAAC;QACH,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YACpB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACrB,CAAC;QAED,IAAI,IAAI,CAAC,OAAO,CAAC,QAAQ,KAAK,IAAI,EAAE,CAAC;YACnC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC7B,MAAM,IAAI,OAAO,CAAO,CAAC,GAAG,EAAE,EAAE;gBAC9B,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;oBAC5B,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;gBAChC,CAAC,EAAE,IAAI,CAAC,CAAC;gBACT,IAAI,CAAC,OAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;oBAC5B,YAAY,CAAC,KAAK,CAAC,CAAC;oBACpB,GAAG,EAAE,CAAC;gBACR,CAAC,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;QACL,CAAC;QAED,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;QACzB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,MAAc,EAAE,MAA+B;QAC/D,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,MAAM,KAAK,YAAY,EAAE,CAAC;YACjD,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;QACjE,CAAC;QACD,OAAO,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAClD,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,MAAc,EAAE,MAA+B;QAC/E,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,KAAK,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;YACjF,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QACjD,CAAC;QAED,IAAI,CAAC,SAAS,IAAI,CAAC,CAAC;QACpB,MAAM,UAAU,GAAG,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;QAEjE,OAAO,IAAI,OAAO,CAAU,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC9C,MAAM,MAAM,GAAG,CAAC,IAAY,EAAQ,EAAE;gBACpC,IAAI,CAAC;oBACH,MAAM,QAAQ,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;oBACtC,OAAO,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAC;gBACnC,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,MAAM,CAAC,GAAG,CAAC,CAAC;gBACd,CAAC;YACH,CAAC,CAAC;YAEF,IAAI,CAAC,MAAO,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;YAElC,MAAM,EAAE,GAAG,IAAI,CAAC,OAAQ,CAAC,KAAM,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC,GAA6B,EAAE,EAAE;gBAClF,IAAI,GAAG,EAAE,CAAC;oBACR,IAAI,CAAC,MAAO,CAAC,cAAc,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;oBAC5C,MAAM,CAAC,GAAG,CAAC,CAAC;gBACd,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,EAAE,EAAE,CAAC;gBACR,IAAI,CAAC,OAAQ,CAAC,KAAM,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE;oBACtC,iCAAiC;gBACnC,CAAC,CAAC,CAAC;YACL,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,OAAO,KAAK,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,QAAQ,KAAK,IAAI,CAAC;IACjE,CAAC;IAED,yDAAyD;IACzD,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,6DAA6D;IAC7D,IAAI,iBAAiB;QACnB,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;CACF"}
|
package/dist/expect.d.ts
DELETED
|
@@ -1,133 +0,0 @@
|
|
|
1
|
-
import type { Assertion, Trace } from "./proto/types.js";
|
|
2
|
-
import { AgentResult } from "./result.js";
|
|
3
|
-
export declare class ExpectChain {
|
|
4
|
-
private readonly _result;
|
|
5
|
-
private readonly _assertions;
|
|
6
|
-
constructor(result: AgentResult);
|
|
7
|
-
get assertions(): Assertion[];
|
|
8
|
-
get trace(): Trace;
|
|
9
|
-
private add;
|
|
10
|
-
outputMatchesSchema(schema: Record<string, unknown>): this;
|
|
11
|
-
outputFieldMatchesSchema(field: string, schema: Record<string, unknown>): this;
|
|
12
|
-
toolArgsMatchSchema(toolName: string, schema: Record<string, unknown>): this;
|
|
13
|
-
toolResultMatchesSchema(toolName: string, schema: Record<string, unknown>): this;
|
|
14
|
-
costUnder(maxCost: number, opts?: {
|
|
15
|
-
soft?: boolean;
|
|
16
|
-
}): this;
|
|
17
|
-
latencyUnder(maxMs: number, opts?: {
|
|
18
|
-
soft?: boolean;
|
|
19
|
-
}): this;
|
|
20
|
-
tokensUnder(maxTokens: number, opts?: {
|
|
21
|
-
soft?: boolean;
|
|
22
|
-
}): this;
|
|
23
|
-
tokensBetween(minTokens: number, maxTokens: number, opts?: {
|
|
24
|
-
soft?: boolean;
|
|
25
|
-
}): this;
|
|
26
|
-
stepCount(operator: string, value: number, opts?: {
|
|
27
|
-
soft?: boolean;
|
|
28
|
-
}): this;
|
|
29
|
-
toolCallCount(operator: string, value: number, opts?: {
|
|
30
|
-
soft?: boolean;
|
|
31
|
-
}): this;
|
|
32
|
-
constraint(field: string, operator: string, opts?: {
|
|
33
|
-
value?: number;
|
|
34
|
-
min?: number;
|
|
35
|
-
max?: number;
|
|
36
|
-
soft?: boolean;
|
|
37
|
-
}): this;
|
|
38
|
-
toolsCalledInOrder(tools: string[], opts?: {
|
|
39
|
-
soft?: boolean;
|
|
40
|
-
}): this;
|
|
41
|
-
toolsCalledExactly(tools: string[], opts?: {
|
|
42
|
-
soft?: boolean;
|
|
43
|
-
}): this;
|
|
44
|
-
noToolLoops(tool: string, maxRepetitions?: number, opts?: {
|
|
45
|
-
soft?: boolean;
|
|
46
|
-
}): this;
|
|
47
|
-
noDuplicateTools(opts?: {
|
|
48
|
-
soft?: boolean;
|
|
49
|
-
}): this;
|
|
50
|
-
requiredTools(tools: string[], opts?: {
|
|
51
|
-
soft?: boolean;
|
|
52
|
-
}): this;
|
|
53
|
-
forbiddenTools(tools: string[], opts?: {
|
|
54
|
-
soft?: boolean;
|
|
55
|
-
}): this;
|
|
56
|
-
outputContains(value: string, opts?: {
|
|
57
|
-
caseSensitive?: boolean;
|
|
58
|
-
soft?: boolean;
|
|
59
|
-
}): this;
|
|
60
|
-
outputNotContains(value: string, opts?: {
|
|
61
|
-
caseSensitive?: boolean;
|
|
62
|
-
soft?: boolean;
|
|
63
|
-
}): this;
|
|
64
|
-
outputMatchesRegex(pattern: string, opts?: {
|
|
65
|
-
soft?: boolean;
|
|
66
|
-
}): this;
|
|
67
|
-
outputHasAllKeywords(keywords: string[], opts?: {
|
|
68
|
-
caseSensitive?: boolean;
|
|
69
|
-
soft?: boolean;
|
|
70
|
-
}): this;
|
|
71
|
-
outputHasAnyKeyword(keywords: string[], opts?: {
|
|
72
|
-
caseSensitive?: boolean;
|
|
73
|
-
soft?: boolean;
|
|
74
|
-
}): this;
|
|
75
|
-
outputForbids(terms: string[]): this;
|
|
76
|
-
contentContains(target: string, value: string, opts?: {
|
|
77
|
-
caseSensitive?: boolean;
|
|
78
|
-
soft?: boolean;
|
|
79
|
-
}): this;
|
|
80
|
-
outputSimilarTo(reference: string, opts?: {
|
|
81
|
-
threshold?: number;
|
|
82
|
-
model?: string;
|
|
83
|
-
soft?: boolean;
|
|
84
|
-
}): this;
|
|
85
|
-
passesJudge(criteria: string, opts?: {
|
|
86
|
-
rubric?: string;
|
|
87
|
-
threshold?: number;
|
|
88
|
-
model?: string;
|
|
89
|
-
soft?: boolean;
|
|
90
|
-
}): this;
|
|
91
|
-
agentCalled(agentId: string, opts?: {
|
|
92
|
-
soft?: boolean;
|
|
93
|
-
}): this;
|
|
94
|
-
delegationDepth(maxDepth: number, opts?: {
|
|
95
|
-
soft?: boolean;
|
|
96
|
-
}): this;
|
|
97
|
-
agentOutputContains(agentId: string, value: string, opts?: {
|
|
98
|
-
caseSensitive?: boolean;
|
|
99
|
-
soft?: boolean;
|
|
100
|
-
}): this;
|
|
101
|
-
crossAgentDataFlow(fromAgent: string, toAgent: string, field: string, opts?: {
|
|
102
|
-
soft?: boolean;
|
|
103
|
-
}): this;
|
|
104
|
-
followsTransitions(transitions: [string, string][], opts?: {
|
|
105
|
-
soft?: boolean;
|
|
106
|
-
}): this;
|
|
107
|
-
aggregateCostUnder(maxCost: number, opts?: {
|
|
108
|
-
soft?: boolean;
|
|
109
|
-
}): this;
|
|
110
|
-
aggregateTokensUnder(maxTokens: number, opts?: {
|
|
111
|
-
soft?: boolean;
|
|
112
|
-
}): this;
|
|
113
|
-
agentOrderedBefore(agentA: string, agentB: string, opts?: {
|
|
114
|
-
soft?: boolean;
|
|
115
|
-
}): this;
|
|
116
|
-
agentsOverlap(agentA: string, agentB: string, opts?: {
|
|
117
|
-
soft?: boolean;
|
|
118
|
-
}): this;
|
|
119
|
-
agentWallTimeUnder(agentId: string, maxMs: number, opts?: {
|
|
120
|
-
soft?: boolean;
|
|
121
|
-
}): this;
|
|
122
|
-
orderedAgents(groups: string[][], opts?: {
|
|
123
|
-
soft?: boolean;
|
|
124
|
-
}): this;
|
|
125
|
-
}
|
|
126
|
-
/**
|
|
127
|
-
* Create an assertion chain for an agent result.
|
|
128
|
-
*
|
|
129
|
-
* Accepts either an {@link AgentResult} or a raw {@link Trace}. When a
|
|
130
|
-
* `Trace` is passed it is automatically wrapped in an `AgentResult`.
|
|
131
|
-
*/
|
|
132
|
-
export declare function attestExpect(result: AgentResult | Trace): ExpectChain;
|
|
133
|
-
//# sourceMappingURL=expect.d.ts.map
|
package/dist/expect.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"expect.d.ts","sourceRoot":"","sources":["../src/expect.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAUzD,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE1C,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAc;IACtC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAmB;gBAEnC,MAAM,EAAE,WAAW;IAI/B,IAAI,UAAU,IAAI,SAAS,EAAE,CAE5B;IAED,IAAI,KAAK,IAAI,KAAK,CAEjB;IAED,OAAO,CAAC,GAAG;IAWX,mBAAmB,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAI1D,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAI9E,mBAAmB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAO5E,uBAAuB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAShF,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI;IAS3D,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI;IAS5D,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI;IAS/D,aAAa,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI;IAUpF,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI;IAS3E,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI;IAS/E,UAAU,CACR,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,GACpE,IAAI;IAcP,kBAAkB,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI;IAQpE,kBAAkB,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI;IAQpE,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,cAAc,SAAI,EAAE,IAAI,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI;IAS9E,gBAAgB,CAAC,IAAI,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI;IAIjD,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI;IAQ/D,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI;IAUhE,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,aAAa,CAAC,EAAE,OAAO,CAAC;QAAC,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI;IAUvF,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,aAAa,CAAC,EAAE,OAAO,CAAC;QAAC,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI;IAU1F,kBAAkB,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI;IASpE,oBAAoB,CAClB,QAAQ,EAAE,MAAM,EAAE,EAClB,IAAI,CAAC,EAAE;QAAE,aAAa,CAAC,EAAE,OAAO,CAAC;QAAC,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,GACjD,IAAI;IAUP,mBAAmB,CACjB,QAAQ,EAAE,MAAM,EAAE,EAClB,IAAI,CAAC,EAAE;QAAE,aAAa,CAAC,EAAE,OAAO,CAAC;QAAC,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,GACjD,IAAI;IAUP,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI;IAQpC,eAAe,CACb,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EACb,IAAI,CAAC,EAAE;QAAE,aAAa,CAAC,EAAE,OAAO,CAAC;QAAC,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,GACjD,IAAI;IAYP,eAAe,CACb,SAAS,EAAE,MAAM,EACjB,IAAI,CAAC,EAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,GAC5D,IAAI;IAYP,WAAW,CACT,QAAQ,EAAE,MAAM,EAChB,IAAI,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,GAC7E,IAAI;IAaP,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI;IAQ7D,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI;IAQlE,mBAAmB,CACjB,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,MAAM,EACb,IAAI,CAAC,EAAE;QAAE,aAAa,CAAC,EAAE,OAAO,CAAC;QAAC,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,GACjD,IAAI;IAUP,kBAAkB,CAChB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,MAAM,EACb,IAAI,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,GACxB,IAAI;IAUP,kBAAkB,CAAC,WAAW,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI;IAQpF,kBAAkB,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI;IASpE,oBAAoB,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI;IASxE,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI;IASnF,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI;IAS9E,kBAAkB,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI;IASnF,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI;CAOnE;AAED;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,WAAW,GAAG,KAAK,GAAG,WAAW,CAMrE"}
|
package/dist/expect.js
DELETED
|
@@ -1,330 +0,0 @@
|
|
|
1
|
-
import { randomUUID } from "node:crypto";
|
|
2
|
-
import { TYPE_CONSTRAINT, TYPE_CONTENT, TYPE_EMBEDDING, TYPE_LLM_JUDGE, TYPE_SCHEMA, TYPE_TRACE, TYPE_TRACE_TREE, } from "./proto/constants.js";
|
|
3
|
-
import { AgentResult } from "./result.js";
|
|
4
|
-
export class ExpectChain {
|
|
5
|
-
_result;
|
|
6
|
-
_assertions = [];
|
|
7
|
-
constructor(result) {
|
|
8
|
-
this._result = result;
|
|
9
|
-
}
|
|
10
|
-
get assertions() {
|
|
11
|
-
return [...this._assertions];
|
|
12
|
-
}
|
|
13
|
-
get trace() {
|
|
14
|
-
return this._result.trace;
|
|
15
|
-
}
|
|
16
|
-
add(assertionType, spec) {
|
|
17
|
-
this._assertions.push({
|
|
18
|
-
assertion_id: `assert_${randomUUID().replace(/-/g, "").slice(0, 8)}`,
|
|
19
|
-
type: assertionType,
|
|
20
|
-
spec,
|
|
21
|
-
});
|
|
22
|
-
return this;
|
|
23
|
-
}
|
|
24
|
-
// -- Layer 1: Schema --
|
|
25
|
-
outputMatchesSchema(schema) {
|
|
26
|
-
return this.add(TYPE_SCHEMA, { target: "output.structured", schema });
|
|
27
|
-
}
|
|
28
|
-
outputFieldMatchesSchema(field, schema) {
|
|
29
|
-
return this.add(TYPE_SCHEMA, { target: `output.${field}`, schema });
|
|
30
|
-
}
|
|
31
|
-
toolArgsMatchSchema(toolName, schema) {
|
|
32
|
-
return this.add(TYPE_SCHEMA, {
|
|
33
|
-
target: `steps[?name=='${toolName}'].args`,
|
|
34
|
-
schema,
|
|
35
|
-
});
|
|
36
|
-
}
|
|
37
|
-
toolResultMatchesSchema(toolName, schema) {
|
|
38
|
-
return this.add(TYPE_SCHEMA, {
|
|
39
|
-
target: `steps[?name=='${toolName}'].result`,
|
|
40
|
-
schema,
|
|
41
|
-
});
|
|
42
|
-
}
|
|
43
|
-
// -- Layer 2: Constraints --
|
|
44
|
-
costUnder(maxCost, opts) {
|
|
45
|
-
return this.add(TYPE_CONSTRAINT, {
|
|
46
|
-
field: "metadata.cost_usd",
|
|
47
|
-
operator: "lte",
|
|
48
|
-
value: maxCost,
|
|
49
|
-
soft: opts?.soft ?? false,
|
|
50
|
-
});
|
|
51
|
-
}
|
|
52
|
-
latencyUnder(maxMs, opts) {
|
|
53
|
-
return this.add(TYPE_CONSTRAINT, {
|
|
54
|
-
field: "metadata.latency_ms",
|
|
55
|
-
operator: "lte",
|
|
56
|
-
value: maxMs,
|
|
57
|
-
soft: opts?.soft ?? false,
|
|
58
|
-
});
|
|
59
|
-
}
|
|
60
|
-
tokensUnder(maxTokens, opts) {
|
|
61
|
-
return this.add(TYPE_CONSTRAINT, {
|
|
62
|
-
field: "metadata.total_tokens",
|
|
63
|
-
operator: "lte",
|
|
64
|
-
value: maxTokens,
|
|
65
|
-
soft: opts?.soft ?? false,
|
|
66
|
-
});
|
|
67
|
-
}
|
|
68
|
-
tokensBetween(minTokens, maxTokens, opts) {
|
|
69
|
-
return this.add(TYPE_CONSTRAINT, {
|
|
70
|
-
field: "metadata.total_tokens",
|
|
71
|
-
operator: "between",
|
|
72
|
-
min: minTokens,
|
|
73
|
-
max: maxTokens,
|
|
74
|
-
soft: opts?.soft ?? false,
|
|
75
|
-
});
|
|
76
|
-
}
|
|
77
|
-
stepCount(operator, value, opts) {
|
|
78
|
-
return this.add(TYPE_CONSTRAINT, {
|
|
79
|
-
field: "steps.length",
|
|
80
|
-
operator,
|
|
81
|
-
value,
|
|
82
|
-
soft: opts?.soft ?? false,
|
|
83
|
-
});
|
|
84
|
-
}
|
|
85
|
-
toolCallCount(operator, value, opts) {
|
|
86
|
-
return this.add(TYPE_CONSTRAINT, {
|
|
87
|
-
field: "steps[?type=='tool_call'].length",
|
|
88
|
-
operator,
|
|
89
|
-
value,
|
|
90
|
-
soft: opts?.soft ?? false,
|
|
91
|
-
});
|
|
92
|
-
}
|
|
93
|
-
constraint(field, operator, opts) {
|
|
94
|
-
const spec = {
|
|
95
|
-
field,
|
|
96
|
-
operator,
|
|
97
|
-
soft: opts?.soft ?? false,
|
|
98
|
-
};
|
|
99
|
-
if (opts?.value !== undefined)
|
|
100
|
-
spec.value = opts.value;
|
|
101
|
-
if (opts?.min !== undefined)
|
|
102
|
-
spec.min = opts.min;
|
|
103
|
-
if (opts?.max !== undefined)
|
|
104
|
-
spec.max = opts.max;
|
|
105
|
-
return this.add(TYPE_CONSTRAINT, spec);
|
|
106
|
-
}
|
|
107
|
-
// -- Layer 3: Trace --
|
|
108
|
-
toolsCalledInOrder(tools, opts) {
|
|
109
|
-
return this.add(TYPE_TRACE, {
|
|
110
|
-
check: "contains_in_order",
|
|
111
|
-
tools,
|
|
112
|
-
soft: opts?.soft ?? false,
|
|
113
|
-
});
|
|
114
|
-
}
|
|
115
|
-
toolsCalledExactly(tools, opts) {
|
|
116
|
-
return this.add(TYPE_TRACE, {
|
|
117
|
-
check: "exact_order",
|
|
118
|
-
tools,
|
|
119
|
-
soft: opts?.soft ?? false,
|
|
120
|
-
});
|
|
121
|
-
}
|
|
122
|
-
noToolLoops(tool, maxRepetitions = 1, opts) {
|
|
123
|
-
return this.add(TYPE_TRACE, {
|
|
124
|
-
check: "loop_detection",
|
|
125
|
-
tool,
|
|
126
|
-
max_repetitions: maxRepetitions,
|
|
127
|
-
soft: opts?.soft ?? false,
|
|
128
|
-
});
|
|
129
|
-
}
|
|
130
|
-
noDuplicateTools(opts) {
|
|
131
|
-
return this.add(TYPE_TRACE, { check: "no_duplicates", soft: opts?.soft ?? false });
|
|
132
|
-
}
|
|
133
|
-
requiredTools(tools, opts) {
|
|
134
|
-
return this.add(TYPE_TRACE, {
|
|
135
|
-
check: "required_tools",
|
|
136
|
-
tools,
|
|
137
|
-
soft: opts?.soft ?? false,
|
|
138
|
-
});
|
|
139
|
-
}
|
|
140
|
-
forbiddenTools(tools, opts) {
|
|
141
|
-
return this.add(TYPE_TRACE, {
|
|
142
|
-
check: "forbidden_tools",
|
|
143
|
-
tools,
|
|
144
|
-
soft: opts?.soft ?? false,
|
|
145
|
-
});
|
|
146
|
-
}
|
|
147
|
-
// -- Layer 4: Content --
|
|
148
|
-
outputContains(value, opts) {
|
|
149
|
-
return this.add(TYPE_CONTENT, {
|
|
150
|
-
target: "output.message",
|
|
151
|
-
check: "contains",
|
|
152
|
-
value,
|
|
153
|
-
case_sensitive: opts?.caseSensitive ?? false,
|
|
154
|
-
soft: opts?.soft ?? false,
|
|
155
|
-
});
|
|
156
|
-
}
|
|
157
|
-
outputNotContains(value, opts) {
|
|
158
|
-
return this.add(TYPE_CONTENT, {
|
|
159
|
-
target: "output.message",
|
|
160
|
-
check: "not_contains",
|
|
161
|
-
value,
|
|
162
|
-
case_sensitive: opts?.caseSensitive ?? false,
|
|
163
|
-
soft: opts?.soft ?? false,
|
|
164
|
-
});
|
|
165
|
-
}
|
|
166
|
-
outputMatchesRegex(pattern, opts) {
|
|
167
|
-
return this.add(TYPE_CONTENT, {
|
|
168
|
-
target: "output.message",
|
|
169
|
-
check: "regex_match",
|
|
170
|
-
value: pattern,
|
|
171
|
-
soft: opts?.soft ?? false,
|
|
172
|
-
});
|
|
173
|
-
}
|
|
174
|
-
outputHasAllKeywords(keywords, opts) {
|
|
175
|
-
return this.add(TYPE_CONTENT, {
|
|
176
|
-
target: "output.message",
|
|
177
|
-
check: "keyword_all",
|
|
178
|
-
values: keywords,
|
|
179
|
-
case_sensitive: opts?.caseSensitive ?? false,
|
|
180
|
-
soft: opts?.soft ?? false,
|
|
181
|
-
});
|
|
182
|
-
}
|
|
183
|
-
outputHasAnyKeyword(keywords, opts) {
|
|
184
|
-
return this.add(TYPE_CONTENT, {
|
|
185
|
-
target: "output.message",
|
|
186
|
-
check: "keyword_any",
|
|
187
|
-
values: keywords,
|
|
188
|
-
case_sensitive: opts?.caseSensitive ?? false,
|
|
189
|
-
soft: opts?.soft ?? false,
|
|
190
|
-
});
|
|
191
|
-
}
|
|
192
|
-
outputForbids(terms) {
|
|
193
|
-
return this.add(TYPE_CONTENT, {
|
|
194
|
-
target: "output.message",
|
|
195
|
-
check: "forbidden",
|
|
196
|
-
values: terms,
|
|
197
|
-
});
|
|
198
|
-
}
|
|
199
|
-
contentContains(target, value, opts) {
|
|
200
|
-
return this.add(TYPE_CONTENT, {
|
|
201
|
-
target,
|
|
202
|
-
check: "contains",
|
|
203
|
-
value,
|
|
204
|
-
case_sensitive: opts?.caseSensitive ?? false,
|
|
205
|
-
soft: opts?.soft ?? false,
|
|
206
|
-
});
|
|
207
|
-
}
|
|
208
|
-
// -- Layer 5: Embedding Similarity --
|
|
209
|
-
outputSimilarTo(reference, opts) {
|
|
210
|
-
return this.add(TYPE_EMBEDDING, {
|
|
211
|
-
target: "output.message",
|
|
212
|
-
reference,
|
|
213
|
-
threshold: opts?.threshold ?? 0.8,
|
|
214
|
-
model: opts?.model,
|
|
215
|
-
soft: opts?.soft ?? false,
|
|
216
|
-
});
|
|
217
|
-
}
|
|
218
|
-
// -- Layer 6: LLM Judge --
|
|
219
|
-
passesJudge(criteria, opts) {
|
|
220
|
-
return this.add(TYPE_LLM_JUDGE, {
|
|
221
|
-
target: "output.message",
|
|
222
|
-
criteria,
|
|
223
|
-
rubric: opts?.rubric ?? "default",
|
|
224
|
-
threshold: opts?.threshold ?? 0.8,
|
|
225
|
-
model: opts?.model,
|
|
226
|
-
soft: opts?.soft ?? false,
|
|
227
|
-
});
|
|
228
|
-
}
|
|
229
|
-
// -- Layer 7: Trace Tree (Multi-Agent) --
|
|
230
|
-
agentCalled(agentId, opts) {
|
|
231
|
-
return this.add(TYPE_TRACE_TREE, {
|
|
232
|
-
check: "agent_called",
|
|
233
|
-
agent_id: agentId,
|
|
234
|
-
soft: opts?.soft ?? false,
|
|
235
|
-
});
|
|
236
|
-
}
|
|
237
|
-
delegationDepth(maxDepth, opts) {
|
|
238
|
-
return this.add(TYPE_TRACE_TREE, {
|
|
239
|
-
check: "delegation_depth",
|
|
240
|
-
max_depth: maxDepth,
|
|
241
|
-
soft: opts?.soft ?? false,
|
|
242
|
-
});
|
|
243
|
-
}
|
|
244
|
-
agentOutputContains(agentId, value, opts) {
|
|
245
|
-
return this.add(TYPE_TRACE_TREE, {
|
|
246
|
-
check: "agent_output_contains",
|
|
247
|
-
agent_id: agentId,
|
|
248
|
-
value,
|
|
249
|
-
case_sensitive: opts?.caseSensitive ?? false,
|
|
250
|
-
soft: opts?.soft ?? false,
|
|
251
|
-
});
|
|
252
|
-
}
|
|
253
|
-
crossAgentDataFlow(fromAgent, toAgent, field, opts) {
|
|
254
|
-
return this.add(TYPE_TRACE_TREE, {
|
|
255
|
-
check: "cross_agent_data_flow",
|
|
256
|
-
from_agent: fromAgent,
|
|
257
|
-
to_agent: toAgent,
|
|
258
|
-
field,
|
|
259
|
-
soft: opts?.soft ?? false,
|
|
260
|
-
});
|
|
261
|
-
}
|
|
262
|
-
followsTransitions(transitions, opts) {
|
|
263
|
-
return this.add(TYPE_TRACE_TREE, {
|
|
264
|
-
check: "follows_transitions",
|
|
265
|
-
transitions,
|
|
266
|
-
soft: opts?.soft ?? false,
|
|
267
|
-
});
|
|
268
|
-
}
|
|
269
|
-
aggregateCostUnder(maxCost, opts) {
|
|
270
|
-
return this.add(TYPE_TRACE_TREE, {
|
|
271
|
-
check: "aggregate_cost",
|
|
272
|
-
operator: "lte",
|
|
273
|
-
value: maxCost,
|
|
274
|
-
soft: opts?.soft ?? false,
|
|
275
|
-
});
|
|
276
|
-
}
|
|
277
|
-
aggregateTokensUnder(maxTokens, opts) {
|
|
278
|
-
return this.add(TYPE_TRACE_TREE, {
|
|
279
|
-
check: "aggregate_tokens",
|
|
280
|
-
operator: "lte",
|
|
281
|
-
value: maxTokens,
|
|
282
|
-
soft: opts?.soft ?? false,
|
|
283
|
-
});
|
|
284
|
-
}
|
|
285
|
-
agentOrderedBefore(agentA, agentB, opts) {
|
|
286
|
-
return this.add(TYPE_TRACE_TREE, {
|
|
287
|
-
check: "agent_ordered_before",
|
|
288
|
-
agent_a: agentA,
|
|
289
|
-
agent_b: agentB,
|
|
290
|
-
soft: opts?.soft ?? false,
|
|
291
|
-
});
|
|
292
|
-
}
|
|
293
|
-
agentsOverlap(agentA, agentB, opts) {
|
|
294
|
-
return this.add(TYPE_TRACE_TREE, {
|
|
295
|
-
check: "agents_overlap",
|
|
296
|
-
agent_a: agentA,
|
|
297
|
-
agent_b: agentB,
|
|
298
|
-
soft: opts?.soft ?? false,
|
|
299
|
-
});
|
|
300
|
-
}
|
|
301
|
-
agentWallTimeUnder(agentId, maxMs, opts) {
|
|
302
|
-
return this.add(TYPE_TRACE_TREE, {
|
|
303
|
-
check: "agent_wall_time_under",
|
|
304
|
-
agent_id: agentId,
|
|
305
|
-
max_ms: maxMs,
|
|
306
|
-
soft: opts?.soft ?? false,
|
|
307
|
-
});
|
|
308
|
-
}
|
|
309
|
-
orderedAgents(groups, opts) {
|
|
310
|
-
return this.add(TYPE_TRACE_TREE, {
|
|
311
|
-
check: "ordered_agents",
|
|
312
|
-
groups,
|
|
313
|
-
soft: opts?.soft ?? false,
|
|
314
|
-
});
|
|
315
|
-
}
|
|
316
|
-
}
|
|
317
|
-
/**
|
|
318
|
-
* Create an assertion chain for an agent result.
|
|
319
|
-
*
|
|
320
|
-
* Accepts either an {@link AgentResult} or a raw {@link Trace}. When a
|
|
321
|
-
* `Trace` is passed it is automatically wrapped in an `AgentResult`.
|
|
322
|
-
*/
|
|
323
|
-
export function attestExpect(result) {
|
|
324
|
-
if (result instanceof AgentResult) {
|
|
325
|
-
return new ExpectChain(result);
|
|
326
|
-
}
|
|
327
|
-
// Treat as Trace — wrap in AgentResult
|
|
328
|
-
return new ExpectChain(new AgentResult(result));
|
|
329
|
-
}
|
|
330
|
-
//# sourceMappingURL=expect.js.map
|