@attest-ai/core 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/index.cjs +113 -2
- package/dist/cjs/index.cjs.map +1 -1
- package/dist/cjs/index.d.cts +33 -7
- package/dist/cjs/proto/index.cjs.map +1 -1
- package/dist/cjs/proto/index.d.cts +44 -3
- package/dist/esm/cli.js +1 -1
- package/dist/esm/cli.js.map +1 -1
- package/dist/esm/index.d.ts +33 -7
- package/dist/esm/index.js +107 -2
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/proto/index.d.ts +44 -3
- package/dist/esm/proto/index.js.map +1 -1
- package/package.json +4 -4
- package/dist/adapters/anthropic.d.ts +0 -31
- package/dist/adapters/anthropic.d.ts.map +0 -1
- package/dist/adapters/anthropic.js +0 -42
- package/dist/adapters/anthropic.js.map +0 -1
- package/dist/adapters/gemini.d.ts +0 -30
- package/dist/adapters/gemini.d.ts.map +0 -1
- package/dist/adapters/gemini.js +0 -41
- package/dist/adapters/gemini.js.map +0 -1
- package/dist/adapters/index.d.ts +0 -11
- package/dist/adapters/index.d.ts.map +0 -1
- package/dist/adapters/index.js +0 -7
- package/dist/adapters/index.js.map +0 -1
- package/dist/adapters/manual.d.ts +0 -9
- package/dist/adapters/manual.d.ts.map +0 -1
- package/dist/adapters/manual.js +0 -16
- package/dist/adapters/manual.js.map +0 -1
- package/dist/adapters/ollama.d.ts +0 -19
- package/dist/adapters/ollama.d.ts.map +0 -1
- package/dist/adapters/ollama.js +0 -30
- package/dist/adapters/ollama.js.map +0 -1
- package/dist/adapters/openai.d.ts +0 -34
- package/dist/adapters/openai.d.ts.map +0 -1
- package/dist/adapters/openai.js +0 -42
- package/dist/adapters/openai.js.map +0 -1
- package/dist/adapters/otel.d.ts +0 -25
- package/dist/adapters/otel.d.ts.map +0 -1
- package/dist/adapters/otel.js +0 -133
- package/dist/adapters/otel.js.map +0 -1
- package/dist/agent.d.ts +0 -14
- package/dist/agent.d.ts.map +0 -1
- package/dist/agent.js +0 -54
- package/dist/agent.js.map +0 -1
- package/dist/client.d.ts +0 -23
- package/dist/client.d.ts.map +0 -1
- package/dist/client.js +0 -171
- package/dist/client.js.map +0 -1
- package/dist/config.d.ts +0 -21
- package/dist/config.d.ts.map +0 -1
- package/dist/config.js +0 -31
- package/dist/config.js.map +0 -1
- package/dist/context.d.ts +0 -4
- package/dist/context.d.ts.map +0 -1
- package/dist/context.js +0 -3
- package/dist/context.js.map +0 -1
- package/dist/delegate.d.ts +0 -3
- package/dist/delegate.d.ts.map +0 -1
- package/dist/delegate.js +0 -24
- package/dist/delegate.js.map +0 -1
- package/dist/engine-downloader.d.ts +0 -7
- package/dist/engine-downloader.d.ts.map +0 -1
- package/dist/engine-downloader.js +0 -136
- package/dist/engine-downloader.js.map +0 -1
- package/dist/engine-manager.d.ts +0 -26
- package/dist/engine-manager.d.ts.map +0 -1
- package/dist/engine-manager.js +0 -172
- package/dist/engine-manager.js.map +0 -1
- package/dist/expect.d.ts +0 -133
- package/dist/expect.d.ts.map +0 -1
- package/dist/expect.js +0 -330
- package/dist/expect.js.map +0 -1
- package/dist/index.d.ts +0 -16
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js +0 -15
- package/dist/index.js.map +0 -1
- package/dist/proto/codec.d.ts +0 -20
- package/dist/proto/codec.d.ts.map +0 -1
- package/dist/proto/codec.js +0 -71
- package/dist/proto/codec.js.map +0 -1
- package/dist/proto/constants.d.ts +0 -21
- package/dist/proto/constants.d.ts.map +0 -1
- package/dist/proto/constants.js +0 -25
- package/dist/proto/constants.js.map +0 -1
- package/dist/proto/errors.d.ts +0 -13
- package/dist/proto/errors.d.ts.map +0 -1
- package/dist/proto/errors.js +0 -24
- package/dist/proto/errors.js.map +0 -1
- package/dist/proto/index.d.ts +0 -5
- package/dist/proto/index.d.ts.map +0 -1
- package/dist/proto/index.js +0 -5
- package/dist/proto/index.js.map +0 -1
- package/dist/proto/types.d.ts +0 -104
- package/dist/proto/types.d.ts.map +0 -1
- package/dist/proto/types.js +0 -2
- package/dist/proto/types.js.map +0 -1
- package/dist/result.d.ts +0 -15
- package/dist/result.d.ts.map +0 -1
- package/dist/result.js +0 -32
- package/dist/result.js.map +0 -1
- package/dist/simulation.d.ts +0 -8
- package/dist/simulation.d.ts.map +0 -1
- package/dist/simulation.js +0 -21
- package/dist/simulation.js.map +0 -1
- package/dist/tier.d.ts +0 -5
- package/dist/tier.d.ts.map +0 -1
- package/dist/tier.js +0 -10
- package/dist/tier.js.map +0 -1
- package/dist/trace-tree.d.ts +0 -20
- package/dist/trace-tree.d.ts.map +0 -1
- package/dist/trace-tree.js +0 -92
- package/dist/trace-tree.js.map +0 -1
- package/dist/trace.d.ts +0 -41
- package/dist/trace.d.ts.map +0 -1
- package/dist/trace.js +0 -88
- package/dist/trace.js.map +0 -1
- package/dist/version.d.ts +0 -3
- package/dist/version.d.ts.map +0 -1
- package/dist/version.js +0 -3
- package/dist/version.js.map +0 -1
package/dist/cjs/index.cjs
CHANGED
|
@@ -30,11 +30,14 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
|
|
|
30
30
|
// src/index.ts
|
|
31
31
|
var index_exports = {};
|
|
32
32
|
__export(index_exports, {
|
|
33
|
+
ADVERSARIAL_USER: () => ADVERSARIAL_USER,
|
|
33
34
|
Agent: () => Agent,
|
|
34
35
|
AgentResult: () => AgentResult,
|
|
35
36
|
AlertDispatcher: () => AlertDispatcher,
|
|
36
37
|
AnthropicAdapter: () => AnthropicAdapter,
|
|
37
38
|
AttestClient: () => AttestClient,
|
|
39
|
+
CONFUSED_USER: () => CONFUSED_USER,
|
|
40
|
+
COOPERATIVE_USER: () => COOPERATIVE_USER,
|
|
38
41
|
ContinuousEvalRunner: () => ContinuousEvalRunner,
|
|
39
42
|
ENGINE_VERSION: () => ENGINE_VERSION,
|
|
40
43
|
ERR_ASSERTION_ERROR: () => ERR_ASSERTION_ERROR,
|
|
@@ -46,6 +49,7 @@ __export(index_exports, {
|
|
|
46
49
|
EngineManager: () => EngineManager,
|
|
47
50
|
EngineTimeoutError: () => EngineTimeoutError,
|
|
48
51
|
ExpectChain: () => ExpectChain,
|
|
52
|
+
FRIENDLY_USER: () => FRIENDLY_USER,
|
|
49
53
|
GeminiAdapter: () => GeminiAdapter,
|
|
50
54
|
LangChainAdapter: () => LangChainAdapter,
|
|
51
55
|
ManualAdapter: () => ManualAdapter,
|
|
@@ -88,8 +92,10 @@ __export(index_exports, {
|
|
|
88
92
|
executePluginAssertion: () => executePluginAssertion,
|
|
89
93
|
extractId: () => extractId,
|
|
90
94
|
extractResult: () => extractResult,
|
|
95
|
+
getSampleRate: () => getSampleRate,
|
|
91
96
|
isSimulationMode: () => isSimulationMode,
|
|
92
97
|
resetConfig: () => resetConfig,
|
|
98
|
+
simulationEvaluateBatch: () => simulationEvaluateBatch,
|
|
93
99
|
tier: () => tier,
|
|
94
100
|
traceId: () => traceId
|
|
95
101
|
});
|
|
@@ -216,8 +222,8 @@ function extractId(response) {
|
|
|
216
222
|
}
|
|
217
223
|
|
|
218
224
|
// src/version.ts
|
|
219
|
-
var VERSION = "0.
|
|
220
|
-
var ENGINE_VERSION = "0.
|
|
225
|
+
var VERSION = "0.6.0";
|
|
226
|
+
var ENGINE_VERSION = "0.6.0";
|
|
221
227
|
|
|
222
228
|
// src/engine-manager.ts
|
|
223
229
|
var import_node_child_process = require("child_process");
|
|
@@ -521,16 +527,40 @@ var EngineManager = class {
|
|
|
521
527
|
|
|
522
528
|
// src/config.ts
|
|
523
529
|
var _simulationMode = false;
|
|
530
|
+
var _sampleRate = 0;
|
|
524
531
|
function config(options) {
|
|
525
532
|
if (options.simulation !== void 0) {
|
|
526
533
|
_simulationMode = options.simulation;
|
|
527
534
|
}
|
|
535
|
+
if (options.sampleRate !== void 0) {
|
|
536
|
+
if (options.sampleRate < 0 || options.sampleRate > 1) {
|
|
537
|
+
throw new RangeError(
|
|
538
|
+
`sampleRate must be between 0.0 and 1.0, got ${options.sampleRate}`
|
|
539
|
+
);
|
|
540
|
+
}
|
|
541
|
+
_sampleRate = options.sampleRate;
|
|
542
|
+
}
|
|
528
543
|
}
|
|
529
544
|
function isSimulationMode() {
|
|
530
545
|
return _simulationMode || process.env["ATTEST_SIMULATION"] === "1" || process.env["ATTEST_SIMULATION"] === "true" || process.env["ATTEST_SIMULATION"] === "yes";
|
|
531
546
|
}
|
|
547
|
+
function getSampleRate() {
|
|
548
|
+
if (_sampleRate > 0) return _sampleRate;
|
|
549
|
+
const env = process.env["ATTEST_SAMPLE_RATE"];
|
|
550
|
+
if (env !== void 0 && env !== "") {
|
|
551
|
+
const parsed = Number(env);
|
|
552
|
+
if (!Number.isFinite(parsed) || parsed < 0 || parsed > 1) {
|
|
553
|
+
throw new RangeError(
|
|
554
|
+
`ATTEST_SAMPLE_RATE must be between 0.0 and 1.0, got '${env}'`
|
|
555
|
+
);
|
|
556
|
+
}
|
|
557
|
+
return parsed;
|
|
558
|
+
}
|
|
559
|
+
return 0;
|
|
560
|
+
}
|
|
532
561
|
function resetConfig() {
|
|
533
562
|
_simulationMode = false;
|
|
563
|
+
_sampleRate = 0;
|
|
534
564
|
}
|
|
535
565
|
|
|
536
566
|
// src/simulation.ts
|
|
@@ -715,6 +745,43 @@ var AttestClient = class {
|
|
|
715
745
|
);
|
|
716
746
|
return Boolean(raw?.accepted ?? false);
|
|
717
747
|
}
|
|
748
|
+
async queryDrift(assertionId2, windowSize = 50, options) {
|
|
749
|
+
if (isSimulationMode()) {
|
|
750
|
+
return {
|
|
751
|
+
assertion_id: assertionId2,
|
|
752
|
+
mean: 1,
|
|
753
|
+
stddev: 0,
|
|
754
|
+
count: 0,
|
|
755
|
+
latest_score: 1,
|
|
756
|
+
deviation: 0,
|
|
757
|
+
status: "no_data"
|
|
758
|
+
};
|
|
759
|
+
}
|
|
760
|
+
const raw = await this.sendRequest(
|
|
761
|
+
"query_drift",
|
|
762
|
+
{ assertion_id: assertionId2, window_size: windowSize },
|
|
763
|
+
options?.timeout
|
|
764
|
+
);
|
|
765
|
+
return raw.report;
|
|
766
|
+
}
|
|
767
|
+
async generateUserMessage(persona, conversationHistory, faultConfig, options) {
|
|
768
|
+
if (isSimulationMode()) {
|
|
769
|
+
return `[simulation] Hello from ${persona.name}`;
|
|
770
|
+
}
|
|
771
|
+
const params = {
|
|
772
|
+
persona,
|
|
773
|
+
conversation_history: conversationHistory
|
|
774
|
+
};
|
|
775
|
+
if (faultConfig !== void 0) {
|
|
776
|
+
params.fault_config = faultConfig;
|
|
777
|
+
}
|
|
778
|
+
const raw = await this.sendRequest(
|
|
779
|
+
"generate_user_message",
|
|
780
|
+
params,
|
|
781
|
+
options?.timeout
|
|
782
|
+
);
|
|
783
|
+
return String(raw.message);
|
|
784
|
+
}
|
|
718
785
|
};
|
|
719
786
|
|
|
720
787
|
// src/trace.ts
|
|
@@ -2041,6 +2108,44 @@ var ContinuousEvalRunner = class {
|
|
|
2041
2108
|
}
|
|
2042
2109
|
};
|
|
2043
2110
|
|
|
2111
|
+
// src/personas.ts
|
|
2112
|
+
var FRIENDLY_USER = {
|
|
2113
|
+
name: "FriendlyUser",
|
|
2114
|
+
system_prompt: `You are a friendly, cooperative user interacting with an AI assistant.
|
|
2115
|
+
You make clear, well-formed requests. You respond positively to helpful answers and
|
|
2116
|
+
ask straightforward follow-up questions. Keep responses concise (1-3 sentences).`,
|
|
2117
|
+
style: "friendly",
|
|
2118
|
+
temperature: 0.7,
|
|
2119
|
+
max_tokens: 200
|
|
2120
|
+
};
|
|
2121
|
+
var ADVERSARIAL_USER = {
|
|
2122
|
+
name: "AdversarialUser",
|
|
2123
|
+
system_prompt: `You are an adversarial user testing the limits of an AI assistant.
|
|
2124
|
+
You ask edge case questions, make ambiguous requests, and probe boundary conditions.
|
|
2125
|
+
Try to find inconsistencies or unexpected behaviors. Keep responses concise (1-3 sentences).`,
|
|
2126
|
+
style: "adversarial",
|
|
2127
|
+
temperature: 0.9,
|
|
2128
|
+
max_tokens: 200
|
|
2129
|
+
};
|
|
2130
|
+
var CONFUSED_USER = {
|
|
2131
|
+
name: "ConfusedUser",
|
|
2132
|
+
system_prompt: `You are a confused user who has trouble articulating what you want.
|
|
2133
|
+
You make vague requests, sometimes contradict yourself, and frequently ask for clarification.
|
|
2134
|
+
You may misunderstand the assistant's responses. Keep responses concise (1-3 sentences).`,
|
|
2135
|
+
style: "confused",
|
|
2136
|
+
temperature: 0.8,
|
|
2137
|
+
max_tokens: 200
|
|
2138
|
+
};
|
|
2139
|
+
var COOPERATIVE_USER = {
|
|
2140
|
+
name: "CooperativeUser",
|
|
2141
|
+
system_prompt: `You are a cooperative user who actively helps the AI assistant succeed.
|
|
2142
|
+
You provide clear context, answer clarifying questions thoroughly, and confirm when
|
|
2143
|
+
the assistant's response meets your needs. Keep responses concise (1-3 sentences).`,
|
|
2144
|
+
style: "cooperative",
|
|
2145
|
+
temperature: 0.6,
|
|
2146
|
+
max_tokens: 200
|
|
2147
|
+
};
|
|
2148
|
+
|
|
2044
2149
|
// src/adapters/manual.ts
|
|
2045
2150
|
var ManualAdapter = class {
|
|
2046
2151
|
agentId;
|
|
@@ -2418,11 +2523,14 @@ var LangChainAdapter = class {
|
|
|
2418
2523
|
};
|
|
2419
2524
|
// Annotate the CommonJS export names for ESM import in node:
|
|
2420
2525
|
0 && (module.exports = {
|
|
2526
|
+
ADVERSARIAL_USER,
|
|
2421
2527
|
Agent,
|
|
2422
2528
|
AgentResult,
|
|
2423
2529
|
AlertDispatcher,
|
|
2424
2530
|
AnthropicAdapter,
|
|
2425
2531
|
AttestClient,
|
|
2532
|
+
CONFUSED_USER,
|
|
2533
|
+
COOPERATIVE_USER,
|
|
2426
2534
|
ContinuousEvalRunner,
|
|
2427
2535
|
ENGINE_VERSION,
|
|
2428
2536
|
ERR_ASSERTION_ERROR,
|
|
@@ -2434,6 +2542,7 @@ var LangChainAdapter = class {
|
|
|
2434
2542
|
EngineManager,
|
|
2435
2543
|
EngineTimeoutError,
|
|
2436
2544
|
ExpectChain,
|
|
2545
|
+
FRIENDLY_USER,
|
|
2437
2546
|
GeminiAdapter,
|
|
2438
2547
|
LangChainAdapter,
|
|
2439
2548
|
ManualAdapter,
|
|
@@ -2476,8 +2585,10 @@ var LangChainAdapter = class {
|
|
|
2476
2585
|
executePluginAssertion,
|
|
2477
2586
|
extractId,
|
|
2478
2587
|
extractResult,
|
|
2588
|
+
getSampleRate,
|
|
2479
2589
|
isSimulationMode,
|
|
2480
2590
|
resetConfig,
|
|
2591
|
+
simulationEvaluateBatch,
|
|
2481
2592
|
tier,
|
|
2482
2593
|
traceId
|
|
2483
2594
|
});
|