@fevex/core 0.1.0-alpha.0 → 0.1.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +349 -49
- package/dist/agents/index.d.mts +1 -22
- package/dist/evals/index.d.mts +145 -0
- package/dist/evals/index.mjs +419 -0
- package/dist/fevex-Dz6GBz9o.d.mts +32 -0
- package/dist/index-BzxHfyYV.d.mts +23 -0
- package/dist/index-CZGyhNIQ.d.mts +47 -0
- package/dist/index-C_Zpr13f.d.mts +124 -0
- package/dist/index-DGWOGXzc.d.mts +181 -0
- package/dist/index-QTkd6a7C.d.mts +20 -0
- package/dist/index-ojCY3lwm.d.mts +79 -0
- package/dist/index.d.mts +27 -15
- package/dist/index.mjs +1444 -197
- package/dist/model-stream-u2xMsEaf.mjs +251 -0
- package/dist/models/index.d.mts +24 -5
- package/dist/observability/index.d.mts +2 -0
- package/dist/observability/index.mjs +1 -0
- package/dist/policies/index.d.mts +2 -0
- package/dist/policies/index.mjs +6 -0
- package/dist/runtime/index.d.mts +2 -17
- package/dist/runtime/index.mjs +92 -1
- package/dist/testing/index.d.mts +7 -5
- package/dist/testing/index.mjs +172 -11
- package/dist/tools/index.d.mts +3 -31
- package/package.json +15 -3
- package/dist/index-D_RBJmVt.d.mts +0 -44
package/dist/testing/index.mjs
CHANGED
|
@@ -1,44 +1,64 @@
|
|
|
1
|
+
import { t as readModelStream } from "../model-stream-u2xMsEaf.mjs";
|
|
1
2
|
//#region src/testing/index.ts
|
|
2
3
|
function fakeModel(...responses) {
|
|
3
4
|
const calls = [];
|
|
4
5
|
return {
|
|
5
6
|
calls,
|
|
6
|
-
async
|
|
7
|
+
async *stream(input) {
|
|
7
8
|
input.signal?.throwIfAborted();
|
|
8
9
|
const response = responses[calls.length];
|
|
9
10
|
calls.push(input);
|
|
10
11
|
if (!response) throw new Error(`fakeModel has no response for call ${calls.length}`);
|
|
11
|
-
|
|
12
|
+
if (response.output !== void 0) {
|
|
13
|
+
const delta = typeof response.output === "string" ? response.output : JSON.stringify(response.output);
|
|
14
|
+
if (delta) yield {
|
|
15
|
+
type: "output.delta",
|
|
16
|
+
delta
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
yield {
|
|
20
|
+
type: "completed",
|
|
21
|
+
result: response
|
|
22
|
+
};
|
|
12
23
|
}
|
|
13
24
|
};
|
|
14
25
|
}
|
|
15
|
-
const contractOutputSchema = {
|
|
26
|
+
const contractOutputSchema = {
|
|
27
|
+
type: "object",
|
|
28
|
+
properties: { answer: { type: "string" } },
|
|
29
|
+
required: ["answer"],
|
|
30
|
+
additionalProperties: false
|
|
31
|
+
};
|
|
16
32
|
const contractToolInputSchema = {
|
|
17
33
|
type: "object",
|
|
18
34
|
properties: { query: { type: "string" } },
|
|
19
|
-
required: ["query"]
|
|
35
|
+
required: ["query"],
|
|
36
|
+
additionalProperties: false
|
|
20
37
|
};
|
|
21
38
|
function assert(condition, message) {
|
|
22
39
|
if (!condition) throw new TypeError(message);
|
|
23
40
|
}
|
|
24
41
|
async function testModelGateway(model, contract = {}) {
|
|
25
|
-
const output = contract.output ?? "ok";
|
|
42
|
+
const output = contract.output ?? { answer: "ok" };
|
|
26
43
|
const toolCall = contract.toolCall ?? {
|
|
27
44
|
id: "call-1",
|
|
28
45
|
name: "lookup",
|
|
29
46
|
input: { query: "value" }
|
|
30
47
|
};
|
|
31
|
-
const
|
|
48
|
+
const outputStream = readModelStream(model, {
|
|
32
49
|
messages: [{
|
|
33
50
|
role: "user",
|
|
34
51
|
content: "Return a final answer."
|
|
35
52
|
}],
|
|
36
53
|
outputSchema: contractOutputSchema
|
|
37
54
|
});
|
|
55
|
+
const firstOutput = await outputStream.next();
|
|
56
|
+
assert(!firstOutput.done && firstOutput.value.length > 0, "ModelGateway must stream an output delta before completed");
|
|
57
|
+
const outputResult = await collectModelResult(outputStream);
|
|
38
58
|
assert(outputResult.output !== void 0, "ModelGateway must return output for a final answer");
|
|
39
|
-
assert(outputResult.output === output, "ModelGateway returned an unexpected output");
|
|
59
|
+
assert(JSON.stringify(outputResult.output) === JSON.stringify(output), "ModelGateway returned an unexpected output");
|
|
40
60
|
if (contract.usage) assert(outputResult.usage !== void 0, "ModelGateway must return usage when the contract requires it");
|
|
41
|
-
const toolResult = await model
|
|
61
|
+
const toolResult = await collectModelResult(model, {
|
|
42
62
|
messages: [{
|
|
43
63
|
role: "user",
|
|
44
64
|
content: "Call lookup."
|
|
@@ -52,9 +72,14 @@ async function testModelGateway(model, contract = {}) {
|
|
|
52
72
|
assert(toolResult.toolCalls?.length === 1, "ModelGateway must return one tool call");
|
|
53
73
|
assert(toolResult.toolCalls[0]?.id === toolCall.id, "ModelGateway returned an unexpected tool call id");
|
|
54
74
|
assert(toolResult.toolCalls[0]?.name === toolCall.name, "ModelGateway returned an unexpected tool name");
|
|
75
|
+
if (model.stateCodec && toolResult.providerState !== void 0) {
|
|
76
|
+
const serialized = model.stateCodec.serialize(toolResult.providerState);
|
|
77
|
+
const restored = model.stateCodec.restore(structuredClone(serialized));
|
|
78
|
+
assert(JSON.stringify(model.stateCodec.serialize(restored)) === JSON.stringify(serialized), "ModelGateway stateCodec must preserve provider state across a JSON round-trip");
|
|
79
|
+
}
|
|
55
80
|
const controller = new AbortController();
|
|
56
81
|
controller.abort();
|
|
57
|
-
await model
|
|
82
|
+
await collectModelResult(model, {
|
|
58
83
|
messages: [{
|
|
59
84
|
role: "user",
|
|
60
85
|
content: "This call is aborted."
|
|
@@ -63,7 +88,7 @@ async function testModelGateway(model, contract = {}) {
|
|
|
63
88
|
}).then(() => {
|
|
64
89
|
throw new TypeError("ModelGateway must reject aborted calls");
|
|
65
90
|
}, () => {});
|
|
66
|
-
if (contract.error) await model
|
|
91
|
+
if (contract.error) await collectModelResult(model, { messages: [{
|
|
67
92
|
role: "user",
|
|
68
93
|
content: "Propagate an error."
|
|
69
94
|
}] }).then(() => {
|
|
@@ -72,5 +97,141 @@ async function testModelGateway(model, contract = {}) {
|
|
|
72
97
|
assert(error === contract.error, "ModelGateway must preserve provider error identity");
|
|
73
98
|
});
|
|
74
99
|
}
|
|
100
|
+
async function collectModelResult(modelOrStream, input) {
|
|
101
|
+
const stream = "stream" in modelOrStream ? readModelStream(modelOrStream, input) : modelOrStream;
|
|
102
|
+
while (true) {
|
|
103
|
+
const next = await stream.next();
|
|
104
|
+
if (next.done) return next.value;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
async function testRunStore(store) {
|
|
108
|
+
const suffix = crypto.randomUUID();
|
|
109
|
+
const runId = `contract-run-${suffix}`;
|
|
110
|
+
const sessionId = `contract-session-${suffix}`;
|
|
111
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
112
|
+
const session = {
|
|
113
|
+
id: sessionId,
|
|
114
|
+
history: [],
|
|
115
|
+
createdAt: now,
|
|
116
|
+
updatedAt: now
|
|
117
|
+
};
|
|
118
|
+
const run = {
|
|
119
|
+
id: runId,
|
|
120
|
+
sessionId,
|
|
121
|
+
agentName: "contract-agent",
|
|
122
|
+
status: "running",
|
|
123
|
+
revision: 0,
|
|
124
|
+
createdAt: now,
|
|
125
|
+
updatedAt: now
|
|
126
|
+
};
|
|
127
|
+
await store.saveSession(session);
|
|
128
|
+
await store.saveRun(run);
|
|
129
|
+
const snapshot = await store.getRun(runId);
|
|
130
|
+
assert(snapshot?.status === "running", "RunStore must return saved runs");
|
|
131
|
+
snapshot.status = "failed";
|
|
132
|
+
assert((await store.getRun(runId))?.status === "running", "RunStore must not expose mutable run references");
|
|
133
|
+
const firstEvent = {
|
|
134
|
+
id: `event-1-${suffix}`,
|
|
135
|
+
sequence: 1,
|
|
136
|
+
type: "run.started",
|
|
137
|
+
runId,
|
|
138
|
+
timestamp: now
|
|
139
|
+
};
|
|
140
|
+
await store.appendEvent(firstEvent);
|
|
141
|
+
const checkpoint = {
|
|
142
|
+
runId,
|
|
143
|
+
definitionHash: "definition",
|
|
144
|
+
messages: [{
|
|
145
|
+
role: "user",
|
|
146
|
+
content: "input"
|
|
147
|
+
}],
|
|
148
|
+
inputContent: "input",
|
|
149
|
+
step: 1,
|
|
150
|
+
toolCallCount: 0,
|
|
151
|
+
seenToolCallIds: ["tool-call"],
|
|
152
|
+
pendingTools: [{
|
|
153
|
+
call: {
|
|
154
|
+
id: "tool-call",
|
|
155
|
+
name: "lookup",
|
|
156
|
+
input: { query: "value" }
|
|
157
|
+
},
|
|
158
|
+
input: { query: "value" },
|
|
159
|
+
idempotencyKey: `key-${suffix}`,
|
|
160
|
+
attempt: 1
|
|
161
|
+
}],
|
|
162
|
+
pendingIndex: 0
|
|
163
|
+
};
|
|
164
|
+
const toolExecution = {
|
|
165
|
+
runId,
|
|
166
|
+
toolCallId: "tool-call",
|
|
167
|
+
toolName: "lookup",
|
|
168
|
+
input: { query: "value" },
|
|
169
|
+
status: "completed",
|
|
170
|
+
attempt: 1,
|
|
171
|
+
idempotencyKey: `key-${suffix}`,
|
|
172
|
+
output: { answer: "ok" },
|
|
173
|
+
updatedAt: now
|
|
174
|
+
};
|
|
175
|
+
session.history.push({
|
|
176
|
+
role: "system",
|
|
177
|
+
content: "committed"
|
|
178
|
+
});
|
|
179
|
+
const secondEvent = {
|
|
180
|
+
id: `event-2-${suffix}`,
|
|
181
|
+
sequence: 2,
|
|
182
|
+
type: "tool.completed",
|
|
183
|
+
runId,
|
|
184
|
+
timestamp: now,
|
|
185
|
+
payload: {
|
|
186
|
+
step: 1,
|
|
187
|
+
toolCallId: "tool-call",
|
|
188
|
+
toolName: "lookup"
|
|
189
|
+
}
|
|
190
|
+
};
|
|
191
|
+
assert(await store.commitExecution({
|
|
192
|
+
expectedRevision: 0,
|
|
193
|
+
run,
|
|
194
|
+
session,
|
|
195
|
+
checkpoint,
|
|
196
|
+
toolExecution,
|
|
197
|
+
events: [secondEvent]
|
|
198
|
+
}), "RunStore must commit a matching revision");
|
|
199
|
+
assert(run.revision === 1, "RunStore must advance the committed run revision");
|
|
200
|
+
assert((await store.getSession(sessionId))?.history.length === 1 && (await store.getCheckpoint(runId))?.definitionHash === "definition" && (await store.getToolExecution(runId, "tool-call"))?.status === "completed", "RunStore atomic commits must include session, checkpoint, and tool ledger");
|
|
201
|
+
assert(!await store.commitExecution({
|
|
202
|
+
expectedRevision: 0,
|
|
203
|
+
run: {
|
|
204
|
+
...run,
|
|
205
|
+
status: "failed"
|
|
206
|
+
},
|
|
207
|
+
events: [{
|
|
208
|
+
id: `stale-event-${suffix}`,
|
|
209
|
+
sequence: 3,
|
|
210
|
+
type: "run.failed",
|
|
211
|
+
runId,
|
|
212
|
+
timestamp: now,
|
|
213
|
+
payload: { error: "stale" }
|
|
214
|
+
}]
|
|
215
|
+
}), "RunStore must reject stale revisions");
|
|
216
|
+
const events = await store.listEvents(runId, { after: firstEvent.id });
|
|
217
|
+
assert(events.length === 1 && events[0]?.id === secondEvent.id, "RunStore cursors must return stable ordered suffixes");
|
|
218
|
+
const lease1 = {
|
|
219
|
+
runId,
|
|
220
|
+
ownerId: `owner-1-${suffix}`,
|
|
221
|
+
expiresAt: new Date(Date.now() + 3e4).toISOString()
|
|
222
|
+
};
|
|
223
|
+
const lease2 = {
|
|
224
|
+
runId,
|
|
225
|
+
ownerId: `owner-2-${suffix}`,
|
|
226
|
+
expiresAt: new Date(Date.now() + 3e4).toISOString()
|
|
227
|
+
};
|
|
228
|
+
assert(await store.acquireLease(lease1), "RunStore must acquire a free lease");
|
|
229
|
+
assert(!await store.acquireLease(lease2), "RunStore must reject a competing lease");
|
|
230
|
+
assert(!await store.renewLease(lease2), "RunStore must reject renewal by another owner");
|
|
231
|
+
assert(await store.renewLease(lease1), "RunStore must renew a matching lease");
|
|
232
|
+
await store.releaseLease(runId, lease1.ownerId);
|
|
233
|
+
assert(await store.acquireLease(lease2), "RunStore must release a lease for another owner");
|
|
234
|
+
await store.releaseLease(runId, lease2.ownerId);
|
|
235
|
+
}
|
|
75
236
|
//#endregion
|
|
76
|
-
export { fakeModel, testModelGateway };
|
|
237
|
+
export { fakeModel, testModelGateway, testRunStore };
|
package/dist/tools/index.d.mts
CHANGED
|
@@ -1,31 +1,3 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { a as
|
|
3
|
-
|
|
4
|
-
//#region src/tools/index.d.ts
|
|
5
|
-
interface ToolExecutionContext {
|
|
6
|
-
runId: RunId;
|
|
7
|
-
toolCallId: string;
|
|
8
|
-
context?: ExecutionContext;
|
|
9
|
-
signal?: AbortSignal;
|
|
10
|
-
}
|
|
11
|
-
interface ToolDefinition<TInput = unknown, TOutput = unknown> {
|
|
12
|
-
name: string;
|
|
13
|
-
description?: string;
|
|
14
|
-
inputSchema?: StandardSchemaV1<unknown, TInput>;
|
|
15
|
-
outputSchema?: StandardSchemaV1<unknown, TOutput>;
|
|
16
|
-
execute(input: TInput, context: ToolExecutionContext): TOutput | Promise<TOutput>;
|
|
17
|
-
}
|
|
18
|
-
type ToolWithSchemas<TInputSchema extends StandardSchemaV1, TOutputSchema extends StandardSchemaV1> = Omit<ToolDefinition<StandardSchemaV1.InferOutput<TInputSchema>, StandardSchemaV1.InferOutput<TOutputSchema>>, 'inputSchema' | 'outputSchema'> & {
|
|
19
|
-
inputSchema: TInputSchema;
|
|
20
|
-
outputSchema: TOutputSchema;
|
|
21
|
-
};
|
|
22
|
-
type ToolWithInputSchema<TInputSchema extends StandardSchemaV1> = Omit<ToolDefinition<StandardSchemaV1.InferOutput<TInputSchema>, unknown>, 'inputSchema'> & {
|
|
23
|
-
inputSchema: TInputSchema;
|
|
24
|
-
};
|
|
25
|
-
declare function defineTool<TInputSchema extends StandardSchemaV1, TOutputSchema extends StandardSchemaV1, TDefinition extends ToolWithSchemas<TInputSchema, TOutputSchema>>(tool: ToolWithSchemas<TInputSchema, TOutputSchema> & ExactDefinition<TDefinition, ToolWithSchemas<TInputSchema, TOutputSchema>>): TDefinition;
|
|
26
|
-
declare function defineTool<TInputSchema extends StandardSchemaV1, TDefinition extends ToolWithInputSchema<TInputSchema>>(tool: ToolWithInputSchema<TInputSchema> & ExactDefinition<TDefinition, ToolWithInputSchema<TInputSchema>>): TDefinition;
|
|
27
|
-
declare function defineTool<TDefinition extends ToolDefinition>(tool: ToolDefinition & ExactDefinition<TDefinition, ToolDefinition>): TDefinition;
|
|
28
|
-
declare function toToolSpec(tool: ToolDefinition, inputSchema?: JsonObject): ToolSpec;
|
|
29
|
-
type ToolResult = JsonValue;
|
|
30
|
-
//#endregion
|
|
31
|
-
export { type ToolCall, ToolDefinition, ToolExecutionContext, ToolResult, type ToolSpec, defineTool, toToolSpec };
|
|
1
|
+
import { h as ToolSpec, m as ToolCall } from "../index-C_Zpr13f.mjs";
|
|
2
|
+
import { a as ToolResult, c as defineTool, i as ToolIdempotency, l as toToolSpec, n as ToolDefinition, o as ToolRetryPolicy, r as ToolExecutionContext, s as ToolRisk, t as ToolApproval } from "../index-CZGyhNIQ.mjs";
|
|
3
|
+
export { ToolApproval, type ToolCall, ToolDefinition, ToolExecutionContext, ToolIdempotency, ToolResult, ToolRetryPolicy, ToolRisk, type ToolSpec, defineTool, toToolSpec };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@fevex/core",
|
|
3
|
-
"version": "0.1.0-alpha.
|
|
3
|
+
"version": "0.1.0-alpha.1",
|
|
4
4
|
"description": "Agent Engineering Framework for code-first, provider-neutral AI agents.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -43,15 +43,27 @@
|
|
|
43
43
|
"types": "./dist/runtime/index.d.mts",
|
|
44
44
|
"import": "./dist/runtime/index.mjs"
|
|
45
45
|
},
|
|
46
|
+
"./policies": {
|
|
47
|
+
"types": "./dist/policies/index.d.mts",
|
|
48
|
+
"import": "./dist/policies/index.mjs"
|
|
49
|
+
},
|
|
50
|
+
"./observability": {
|
|
51
|
+
"types": "./dist/observability/index.d.mts",
|
|
52
|
+
"import": "./dist/observability/index.mjs"
|
|
53
|
+
},
|
|
54
|
+
"./evals": {
|
|
55
|
+
"types": "./dist/evals/index.d.mts",
|
|
56
|
+
"import": "./dist/evals/index.mjs"
|
|
57
|
+
},
|
|
46
58
|
"./testing": {
|
|
47
59
|
"types": "./dist/testing/index.d.mts",
|
|
48
60
|
"import": "./dist/testing/index.mjs"
|
|
49
61
|
}
|
|
50
62
|
},
|
|
51
63
|
"scripts": {
|
|
52
|
-
"build": "tsdown src/index.ts src/agents/index.ts src/models/index.ts src/tools/index.ts src/runtime/index.ts src/testing/index.ts --format esm --dts",
|
|
64
|
+
"build": "tsdown src/index.ts src/agents/index.ts src/models/index.ts src/tools/index.ts src/runtime/index.ts src/policies/index.ts src/observability/index.ts src/evals/index.ts src/testing/index.ts --format esm --dts --clean",
|
|
53
65
|
"prepack": "bun run build",
|
|
54
|
-
"test": "bun test src/index.test.ts src/testing/index.test.ts",
|
|
66
|
+
"test": "bun test src/index.test.ts src/internal/json.test.ts src/internal/model-stream.test.ts src/observability/index.test.ts src/evals/index.test.ts src/testing/index.test.ts",
|
|
55
67
|
"typecheck": "tsc -p tsconfig.json --noEmit"
|
|
56
68
|
},
|
|
57
69
|
"publishConfig": {
|
|
@@ -1,44 +0,0 @@
|
|
|
1
|
-
//#region src/core/index.d.ts
|
|
2
|
-
type JsonPrimitive = string | number | boolean | null;
|
|
3
|
-
type JsonObject = {
|
|
4
|
-
[key: string]: JsonValue;
|
|
5
|
-
};
|
|
6
|
-
type JsonValue = JsonPrimitive | JsonValue[] | JsonObject;
|
|
7
|
-
type RunId = string;
|
|
8
|
-
declare const SCHEMA_NOT_TRANSPORTABLE = "SCHEMA_NOT_TRANSPORTABLE";
|
|
9
|
-
interface ExecutionContext {
|
|
10
|
-
namespace?: string;
|
|
11
|
-
actor?: {
|
|
12
|
-
id: string;
|
|
13
|
-
type?: string;
|
|
14
|
-
};
|
|
15
|
-
attributes?: JsonObject;
|
|
16
|
-
prompt?: JsonObject;
|
|
17
|
-
}
|
|
18
|
-
type AgentMessageRole = 'system' | 'user' | 'assistant' | 'tool';
|
|
19
|
-
interface AgentMessage {
|
|
20
|
-
role: AgentMessageRole;
|
|
21
|
-
content: string;
|
|
22
|
-
name?: string;
|
|
23
|
-
toolCallId?: string;
|
|
24
|
-
toolCalls?: ToolCall[];
|
|
25
|
-
}
|
|
26
|
-
interface ToolSpec {
|
|
27
|
-
name: string;
|
|
28
|
-
description?: string;
|
|
29
|
-
inputSchema?: JsonObject;
|
|
30
|
-
}
|
|
31
|
-
interface ToolCall {
|
|
32
|
-
id: string;
|
|
33
|
-
name: string;
|
|
34
|
-
input: JsonValue;
|
|
35
|
-
}
|
|
36
|
-
type AgentEventType = 'run.started' | 'model.completed' | 'tool.completed' | 'tool.failed' | 'run.completed' | 'run.failed';
|
|
37
|
-
interface AgentEvent<TPayload extends JsonValue = JsonObject> {
|
|
38
|
-
type: AgentEventType;
|
|
39
|
-
runId: RunId;
|
|
40
|
-
timestamp: string;
|
|
41
|
-
payload?: TPayload;
|
|
42
|
-
}
|
|
43
|
-
//#endregion
|
|
44
|
-
export { ExecutionContext as a, JsonValue as c, ToolCall as d, ToolSpec as f, AgentMessageRole as i, RunId as l, AgentEventType as n, JsonObject as o, AgentMessage as r, JsonPrimitive as s, AgentEvent as t, SCHEMA_NOT_TRANSPORTABLE as u };
|