@spine-event-engine/testing 2.0.0-snapshot.22 → 2.0.0-snapshot.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +64 -15
- package/REFERENCE.md +47 -5
- package/dist/ai-test/ai-test-backend.d.ts +167 -0
- package/dist/ai-test/ai-test-backend.d.ts.map +1 -0
- package/dist/ai-test/ai-test-backend.js +134 -0
- package/dist/ai-test/ai-test-backend.js.map +1 -0
- package/dist/ai-test/scripted-attempts.d.ts +261 -0
- package/dist/ai-test/scripted-attempts.d.ts.map +1 -0
- package/dist/ai-test/scripted-attempts.js +607 -0
- package/dist/ai-test/scripted-attempts.js.map +1 -0
- package/dist/black-box/black-box.d.ts +22 -2
- package/dist/black-box/black-box.d.ts.map +1 -1
- package/dist/black-box/black-box.js +103 -11
- package/dist/black-box/black-box.js.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +7 -5
package/README.md
CHANGED
|
@@ -1,26 +1,75 @@
|
|
|
1
1
|
# Black-box testing for Spine applications
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
Node client boundary used by an application. `BlackBox` is the end-user test
|
|
5
|
-
API; applications do not need internal test utilities.
|
|
3
|
+
`BlackBox` tests a Bounded Context through the same local server and Node client path that an application uses. Post a Command, then observe its Event or query result. This matters for asynchronous handlers: accepting a Command does not mean its later work has finished.
|
|
6
4
|
|
|
7
|
-
|
|
8
|
-
[REFERENCE.md documentation for agents](REFERENCE.md).
|
|
9
|
-
|
|
10
|
-
This is an experimental snapshot package. Use Node 24 or newer and an
|
|
11
|
-
application Bounded Context before writing BlackBox tests.
|
|
5
|
+
This experimental snapshot package needs Node 24 or newer and a built application Bounded Context.
|
|
12
6
|
|
|
13
7
|
```sh
|
|
14
8
|
pnpm add -D @spine-event-engine/testing@snapshot
|
|
15
9
|
```
|
|
16
10
|
|
|
17
|
-
|
|
11
|
+
For exact methods and failure behavior, see the [coding-agent reference](REFERENCE.md).
|
|
12
|
+
|
|
13
|
+
## Test a model-assisted outcome
|
|
14
|
+
|
|
15
|
+
The [warehouse support example](https://github.com/SpineEventEngine/spine-ts/blob/master/examples/support/README.md) asks an Agent to draft a reply. A test can supply a controlled model response with `AiTestBackend`, post `DraftSupportReply`, and wait for `SupportReplySuggested`. This exercises the real Agent, Event delivery, and output validation without a provider account.
|
|
16
|
+
|
|
17
|
+
<!-- prettier-ignore-start -->
|
|
18
|
+
|
|
19
|
+
<!-- docs-snippet-path: examples/support/test/support-blackbox.test.ts -->
|
|
20
|
+
|
|
21
|
+
```ts
|
|
22
|
+
import { create, equals } from "@bufbuild/protobuf";
|
|
23
|
+
import { AnyMessages } from "@spine-event-engine/core";
|
|
24
|
+
import { AiTestBackend, BlackBox } from "@spine-event-engine/testing";
|
|
25
|
+
import {
|
|
26
|
+
DraftSupportReplySchema,
|
|
27
|
+
type DraftSupportReply,
|
|
28
|
+
} from "../generated/spine/examples/support/commands_pb.js";
|
|
29
|
+
import { SupportReplySuggestedSchema } from "../generated/spine/examples/support/events_pb.js";
|
|
30
|
+
import {
|
|
31
|
+
SupportReplySchema,
|
|
32
|
+
SupportTicketIdSchema,
|
|
33
|
+
} from "../generated/spine/examples/support/types_pb.js";
|
|
34
|
+
import { draftSupportReply } from "../dist/src/index.js";
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Scripts a proposal and waits for that exact domain outcome.
|
|
38
|
+
*
|
|
39
|
+
* @param box Running application test boundary.
|
|
40
|
+
* @param backend Controlled model deployment.
|
|
41
|
+
* @param command Draft request with ticket facts and conversation.
|
|
42
|
+
* @returns Matching proposal Events after asynchronous delivery.
|
|
43
|
+
*/
|
|
44
|
+
async function submitDraft(box: BlackBox, backend: AiTestBackend, command: DraftSupportReply) {
|
|
45
|
+
const reply = create(SupportReplySchema, {
|
|
46
|
+
subject: "Shipping label printing is blocked",
|
|
47
|
+
body: "Both stations still fail after restarting. What error appears when printing?",
|
|
48
|
+
});
|
|
49
|
+
backend.forModel(draftSupportReply).respondWith(reply);
|
|
50
|
+
const acknowledgement = await box.asGuest().post(DraftSupportReplySchema, command);
|
|
51
|
+
if (acknowledgement.kind !== "ok") throw new Error("Draft request was not accepted.");
|
|
52
|
+
const suggested = await box.eventually(
|
|
53
|
+
() => box.assertEvents()
|
|
54
|
+
.flatMap((event) => event.message === undefined
|
|
55
|
+
? []
|
|
56
|
+
: [AnyMessages.unpack(event.message, SupportReplySuggestedSchema)])
|
|
57
|
+
.filter((event) => event !== undefined),
|
|
58
|
+
(events) => events.some((event) =>
|
|
59
|
+
event.reply !== undefined &&
|
|
60
|
+
event.id !== undefined && command.id !== undefined &&
|
|
61
|
+
event.conversation?.value === command.conversation?.value &&
|
|
62
|
+
equals(SupportReplySchema, event.reply, reply) &&
|
|
63
|
+
equals(SupportTicketIdSchema, event.id, command.id)),
|
|
64
|
+
);
|
|
65
|
+
backend.assertSatisfied();
|
|
66
|
+
return suggested;
|
|
67
|
+
}
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
<!-- prettier-ignore-end -->
|
|
18
71
|
|
|
19
|
-
-
|
|
20
|
-
- ✅ Posts commands, reads query-side views, and observes subscriptions as a
|
|
21
|
-
user.
|
|
22
|
-
- ✅ Runs on an ephemeral local server with predictable cleanup.
|
|
23
|
-
- ✅ Waits for genuinely asynchronous results with bounded polling.
|
|
72
|
+
The Command acknowledgement only says the request was accepted. The test waits for the proposed-reply Event and checks its complete reply, ticket ID, and conversation; an earlier ticket-opened Event cannot satisfy that condition. The complete test also queries the review Projection and reads the Agent's recorded interaction history. Script malformed text or a blank reply to check the failure and correction paths.
|
|
24
73
|
|
|
25
74
|
## ✅ Run one command through a BlackBox
|
|
26
75
|
|
|
@@ -101,7 +150,7 @@ async function inspectPartnerEvent<Schema extends GenMessage<Message>>(
|
|
|
101
150
|
|
|
102
151
|
Snapshots are taken at the call time. Use `eventually()` when detached handling
|
|
103
152
|
may produce output later. Input Commands, setup Events, received external
|
|
104
|
-
Events, and
|
|
153
|
+
Events, and rejection Events from unsuccessful handlers are excluded; only admitted produced
|
|
105
154
|
Commands and committed produced Events appear in their production order.
|
|
106
155
|
|
|
107
156
|
## ⚠️ Test boundary
|
package/REFERENCE.md
CHANGED
|
@@ -5,10 +5,38 @@ This reference describes the public test API for coding agents.
|
|
|
5
5
|
## Entry point
|
|
6
6
|
|
|
7
7
|
Import `BlackBox`, `BlackBoxClosedError`, `BlackBoxTimeoutError`, and their
|
|
8
|
-
option/scope types
|
|
8
|
+
option/scope types, plus `AiTestBackend` when scripting a model deployment, from
|
|
9
|
+
`@spine-event-engine/testing`. Do not use the package's
|
|
9
10
|
internal test access or the server's `BoundedContextFixture` as application
|
|
10
11
|
test APIs.
|
|
11
12
|
|
|
13
|
+
## Scripted AI backend
|
|
14
|
+
|
|
15
|
+
Create `AiTestBackend` with a credential-free `ModelRef` and either the
|
|
16
|
+
`generation` or `decision` kind, then register its `registration` in the
|
|
17
|
+
application `AiRegistry`. `forModel(model)` requires an `AiModel.define()` result
|
|
18
|
+
of the same kind. For generation, queue `respondWith(value)` or
|
|
19
|
+
`respondWithText(text)`; for decisions, queue `respondWithDecision(result)`.
|
|
20
|
+
Wrong-kind calls fail when scripting. `failWith(failure)` and `refuse()` queue
|
|
21
|
+
safe failures; the runtime supplies the diagnostic ID even when the supplied
|
|
22
|
+
failure carries one. `withUsage(usage)` defensively copies known token counts
|
|
23
|
+
onto the preceding response. `delay()` queues a pause and returns a gate whose
|
|
24
|
+
`release()` resumes it; runtime cancellation also ends the wait.
|
|
25
|
+
|
|
26
|
+
Each admitted physical request crosses `beginAttempt`, `reserveTransport`, and
|
|
27
|
+
`finishAttempt` or the runtime's failure path. Typed generation output still
|
|
28
|
+
passes through the runtime response parser, Proto constraints, and application
|
|
29
|
+
validation; decision answers pass through the runtime decision admission.
|
|
30
|
+
`requests()` gives immutable per-request snapshots with capability name, Agent
|
|
31
|
+
call name, one-based attempt, prepared input ProtoJSON, correction target and
|
|
32
|
+
issues, local validation issues, and advertised tool names. Recovery reuse does
|
|
33
|
+
not produce a new backend request. `assertSatisfied()` fails for any queued
|
|
34
|
+
response left unused or any admitted request without a matching script. This
|
|
35
|
+
includes a request waiting at a gate. If cancellation interrupts a reserved
|
|
36
|
+
response, the scripted observation remains pending and satisfaction does not
|
|
37
|
+
pass. This dependency does not replace BlackBox, establish delivery or state, or provide an
|
|
38
|
+
MCP tool transport.
|
|
39
|
+
|
|
12
40
|
`BlackBox.from(contextOrBuilder, options?)` accepts a built `BoundedContext` or
|
|
13
41
|
`BoundedContextBuilder`, builds a builder asynchronously, starts a local
|
|
14
42
|
`Server`, and connects a Node client. The returned box is ready to use. Options
|
|
@@ -29,9 +57,19 @@ timestamp, and external marker, reaches only external handlers, and is not
|
|
|
29
57
|
captured as produced output. `assertCommands()` and `assertEvents()` return
|
|
30
58
|
independently cloned snapshots of admitted produced signals in admission order.
|
|
31
59
|
They exclude test inputs, external inputs, system events, stored-event replay,
|
|
32
|
-
and
|
|
60
|
+
and output from unsuccessful handlers. Use `eventually()` when detached handling has not yet
|
|
33
61
|
admitted a produced signal.
|
|
34
62
|
|
|
63
|
+
`readAgentHistory(repositoryOrAgentClass, entityId, request)` reads the registered Agent's
|
|
64
|
+
full retained history newest first. It uses the same `HistoryRead` page size and
|
|
65
|
+
opaque cursor as protected Agent history; a cursor from another Agent or view
|
|
66
|
+
is rejected. `readSystemEvents(ids)` reads exact persisted System Event IDs from
|
|
67
|
+
the paired System Context, in requested order, and returns independent envelope copies;
|
|
68
|
+
missing IDs are omitted. Both methods use the BlackBox's fixed tenant and fail after close. History
|
|
69
|
+
reads also reject a repository from another Bounded Context. System reads fail when System Event
|
|
70
|
+
recording is unavailable. These audit reads are separate from `assertEvents()`,
|
|
71
|
+
which observes produced domain output.
|
|
72
|
+
|
|
35
73
|
## Waiting and lifecycle
|
|
36
74
|
|
|
37
75
|
`eventually(read, accept, options?)` retries `read` until `accept` returns true
|
|
@@ -43,9 +81,13 @@ stops waiting when close begins.
|
|
|
43
81
|
operations, cancels tracked subscriptions, closes the client, then closes the
|
|
44
82
|
local server. Operations begun after close starts throw `BlackBoxClosedError`.
|
|
45
83
|
|
|
46
|
-
Subscriptions
|
|
47
|
-
ended with `cancel()`. The
|
|
48
|
-
|
|
84
|
+
Subscriptions returned by `createSubscription` are inactive until `activate()`
|
|
85
|
+
and must be ended with `cancel()`. The same method accepts several Event schemas,
|
|
86
|
+
and activation accepts the optional `activationTimeoutMs`.
|
|
87
|
+
`postAndSubscribe` activates its selected subscriptions before posting, so an
|
|
88
|
+
accepted result contains an already active handle. It uses the same optional
|
|
89
|
+
first-update notification and continuing updates as the Node client. The box
|
|
90
|
+
tracks these handles for cleanup if a test fails before canceling them.
|
|
49
91
|
|
|
50
92
|
## Limits
|
|
51
93
|
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
import type { MessageShape } from "@bufbuild/protobuf";
|
|
2
|
+
import type { MessageSchema } from "@spine-event-engine/core";
|
|
3
|
+
import type { AiDecisionResult, AiFailure, AiModel as TypedModel, ModelRef } from "@spine-event-engine/ai";
|
|
4
|
+
import { type AiBackendRegistration } from "@spine-event-engine/ai/spi/adapter";
|
|
5
|
+
import type { AiUsage } from "@spine-event-engine/proto/agent";
|
|
6
|
+
/**
|
|
7
|
+
* One actual physical request recorded at the scripted backend boundary.
|
|
8
|
+
*/
|
|
9
|
+
export interface AiTestRequest {
|
|
10
|
+
/**
|
|
11
|
+
* Application capability name.
|
|
12
|
+
*/
|
|
13
|
+
readonly modelName: string;
|
|
14
|
+
/**
|
|
15
|
+
* Named Agent call supplied by the runtime.
|
|
16
|
+
*/
|
|
17
|
+
readonly call: string;
|
|
18
|
+
/**
|
|
19
|
+
* One-based physical request number for the logical operation.
|
|
20
|
+
*/
|
|
21
|
+
readonly attempt: number;
|
|
22
|
+
/**
|
|
23
|
+
* Bounded ProtoJSON of the application facts prepared for the backend.
|
|
24
|
+
*/
|
|
25
|
+
readonly inputJson: string;
|
|
26
|
+
/**
|
|
27
|
+
* Correction target when this physical request corrects a prior attempt.
|
|
28
|
+
*/
|
|
29
|
+
readonly corrects?: string;
|
|
30
|
+
/**
|
|
31
|
+
* Actual local validation issues observed for this response.
|
|
32
|
+
*/
|
|
33
|
+
readonly validationIssues: readonly {
|
|
34
|
+
readonly code: string;
|
|
35
|
+
readonly path: string;
|
|
36
|
+
readonly message: string;
|
|
37
|
+
}[];
|
|
38
|
+
/**
|
|
39
|
+
* Issues supplied to this corrective attempt from its preceding response.
|
|
40
|
+
*/
|
|
41
|
+
readonly correctionIssues: readonly {
|
|
42
|
+
readonly code: string;
|
|
43
|
+
readonly path: string;
|
|
44
|
+
readonly message: string;
|
|
45
|
+
}[];
|
|
46
|
+
/**
|
|
47
|
+
* Permitted tool names advertised by this generation capability.
|
|
48
|
+
*/
|
|
49
|
+
readonly toolNames: readonly string[];
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Externally released pause on one scripted physical request.
|
|
53
|
+
*/
|
|
54
|
+
export interface AiTestGate {
|
|
55
|
+
/**
|
|
56
|
+
* Completes the paused response once.
|
|
57
|
+
*/
|
|
58
|
+
release(): void;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Response queue for one factory-created application capability.
|
|
62
|
+
*
|
|
63
|
+
* @typeParam O Generated output descriptor.
|
|
64
|
+
*/
|
|
65
|
+
export interface AiTestResponses<O extends MessageSchema> {
|
|
66
|
+
/**
|
|
67
|
+
* Queues typed output, serialized before the runtime parses and validates it.
|
|
68
|
+
*
|
|
69
|
+
* @param value Typed generation output.
|
|
70
|
+
* @returns This response queue.
|
|
71
|
+
*/
|
|
72
|
+
respondWith(value: MessageShape<O>): this;
|
|
73
|
+
/**
|
|
74
|
+
* Queues exact raw text for output and correction testing.
|
|
75
|
+
*
|
|
76
|
+
* @param text Exact provider candidate.
|
|
77
|
+
* @returns This response queue.
|
|
78
|
+
*/
|
|
79
|
+
respondWithText(text: string): this;
|
|
80
|
+
/**
|
|
81
|
+
* Queues provider decision answers for normal decision admission.
|
|
82
|
+
*
|
|
83
|
+
* @param result Provider answers keyed by question ID.
|
|
84
|
+
* @returns This response queue.
|
|
85
|
+
*/
|
|
86
|
+
respondWithDecision(result: AiDecisionResult): this;
|
|
87
|
+
/**
|
|
88
|
+
* Queues a safe failure category; runtime allocates the diagnostic ID.
|
|
89
|
+
*
|
|
90
|
+
* @param failure Safe category and retry policy; its diagnostic ID is ignored.
|
|
91
|
+
* @returns This response queue.
|
|
92
|
+
*/
|
|
93
|
+
failWith(failure: AiFailure): this;
|
|
94
|
+
/**
|
|
95
|
+
* Queues a provider refusal; runtime allocates its diagnostic ID.
|
|
96
|
+
*
|
|
97
|
+
* @returns This response queue.
|
|
98
|
+
*/
|
|
99
|
+
refuse(): this;
|
|
100
|
+
/**
|
|
101
|
+
* Applies known provider usage to the most recently queued response.
|
|
102
|
+
*
|
|
103
|
+
* @param usage Present token counts, defensively copied.
|
|
104
|
+
* @returns This response queue.
|
|
105
|
+
*/
|
|
106
|
+
withUsage(usage: AiUsage): this;
|
|
107
|
+
/**
|
|
108
|
+
* Queues a pause before the next response until release or cancellation.
|
|
109
|
+
*
|
|
110
|
+
* @returns The gate controlling this pause.
|
|
111
|
+
*/
|
|
112
|
+
delay(): AiTestGate;
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Scripted external model dependency registered by the normal AI registry.
|
|
116
|
+
*/
|
|
117
|
+
export declare class AiTestBackend {
|
|
118
|
+
#private;
|
|
119
|
+
/**
|
|
120
|
+
* Normal factory-proven deployment registration.
|
|
121
|
+
*/
|
|
122
|
+
readonly registration: AiBackendRegistration;
|
|
123
|
+
/**
|
|
124
|
+
* Creates the factory-proven registration for this scripted deployment.
|
|
125
|
+
*
|
|
126
|
+
* @param ref Credential-free deployment reference.
|
|
127
|
+
* @param kind Generation or decision backend kind.
|
|
128
|
+
*/
|
|
129
|
+
private constructor();
|
|
130
|
+
/**
|
|
131
|
+
* Creates a network-free deployment selected by normal registry rules.
|
|
132
|
+
*
|
|
133
|
+
* @param options Deployment reference and API kind.
|
|
134
|
+
* @returns Factory-proven scripted backend.
|
|
135
|
+
*/
|
|
136
|
+
static create(options: {
|
|
137
|
+
readonly ref: ModelRef;
|
|
138
|
+
readonly kind: "generation" | "decision";
|
|
139
|
+
}): AiTestBackend;
|
|
140
|
+
/**
|
|
141
|
+
* Queues responses for a factory-created capability of this deployment kind.
|
|
142
|
+
*
|
|
143
|
+
* @typeParam I Generated input descriptor.
|
|
144
|
+
* @typeParam O Generated output descriptor.
|
|
145
|
+
* @param model Factory-created capability.
|
|
146
|
+
* @returns Queue writer for this capability.
|
|
147
|
+
*/
|
|
148
|
+
forModel<I extends MessageSchema, O extends MessageSchema>(model: TypedModel<I, O>): AiTestResponses<O>;
|
|
149
|
+
/**
|
|
150
|
+
* Returns immutable snapshots of actual backend dispatches.
|
|
151
|
+
*
|
|
152
|
+
* @returns Physical request observations.
|
|
153
|
+
*/
|
|
154
|
+
requests(): readonly AiTestRequest[];
|
|
155
|
+
/**
|
|
156
|
+
* Checks for unused scripts, unexpected requests, and pending dispatches.
|
|
157
|
+
*/
|
|
158
|
+
assertSatisfied(): void;
|
|
159
|
+
/**
|
|
160
|
+
* Executes bounded generation corrections through runtime journal and validation barriers.
|
|
161
|
+
*
|
|
162
|
+
* @param request Selected authenticated backend invocation.
|
|
163
|
+
* @returns Admitted output or runtime-recorded failure.
|
|
164
|
+
*/
|
|
165
|
+
private execute;
|
|
166
|
+
}
|
|
167
|
+
//# sourceMappingURL=ai-test-backend.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ai-test-backend.d.ts","sourceRoot":"","sources":["../../src/ai-test/ai-test-backend.ts"],"names":[],"mappings":"AAcA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,KAAK,EACV,gBAAgB,EAChB,SAAS,EACT,OAAO,IAAI,UAAU,EAErB,QAAQ,EACT,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAIL,KAAK,qBAAqB,EAC3B,MAAM,oCAAoC,CAAC;AAE5C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iCAAiC,CAAC;AAG/D;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B;;OAEG;IACH,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAE3B;;OAEG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAEtB;;OAEG;IACH,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IAEzB;;OAEG;IACH,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAE3B;;OAEG;IACH,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAE3B;;OAEG;IACH,QAAQ,CAAC,gBAAgB,EAAE,SAAS;QAClC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;KAC1B,EAAE,CAAC;IAEJ;;OAEG;IACH,QAAQ,CAAC,gBAAgB,EAAE,SAAS;QAClC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;KAC1B,EAAE,CAAC;IAEJ;;OAEG;IACH,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;CACvC;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB;;OAEG;IACH,OAAO,IAAI,IAAI,CAAC;CACjB;AAED;;;;GAIG;AACH,MAAM,WAAW,eAAe,CAAC,CAAC,SAAS,aAAa;IACtD;;;;;OAKG;IACH,WAAW,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAE1C;;;;;OAKG;IACH,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAEpC;;;;;OAKG;IACH,mBAAmB,CAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI,CAAC;IAEpD;;;;;OAKG;IACH,QAAQ,CAAC,OAAO,EAAE,SAAS,GAAG,IAAI,CAAC;IAEnC;;;;OAIG;IACH,MAAM,IAAI,IAAI,CAAC;IAEf;;;;;OAKG;IACH,SAAS,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAAC;IAEhC;;;;OAIG;IACH,KAAK,IAAI,UAAU,CAAC;CACrB;AAED;;GAEG;AACH,qBAAa,aAAa;;IACxB;;OAEG;IACH,QAAQ,CAAC,YAAY,EAAE,qBAAqB,CAAC;IAM7C;;;;;OAKG;IACH,OAAO;IAiBP;;;;;OAKG;IACH,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE;QACrB,QAAQ,CAAC,GAAG,EAAE,QAAQ,CAAC;QACvB,QAAQ,CAAC,IAAI,EAAE,YAAY,GAAG,UAAU,CAAC;KAC1C,GAAG,aAAa;IAIjB;;;;;;;OAOG;IACH,QAAQ,CAAC,CAAC,SAAS,aAAa,EAAE,CAAC,SAAS,aAAa,EACvD,KAAK,EAAE,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC,GACtB,eAAe,CAAC,CAAC,CAAC;IAoBrB;;;;OAIG;IACH,QAAQ,IAAI,SAAS,aAAa,EAAE;IAQpC;;OAEG;IACH,eAAe,IAAI,IAAI;IAcvB;;;;;OAKG;YACW,OAAO;CAwBtB"}
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright 2026, CodeMatters. All rights reserved.
|
|
3
|
+
*
|
|
4
|
+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
|
|
5
|
+
* in compliance with the License. You may obtain a copy of the License at
|
|
6
|
+
*
|
|
7
|
+
* https://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
*
|
|
9
|
+
* Unless required by applicable law or agreed to in writing, software distributed under the License
|
|
10
|
+
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
|
11
|
+
* or implied. See the License for the specific language governing permissions and limitations under
|
|
12
|
+
* the License.
|
|
13
|
+
*/
|
|
14
|
+
import { createBackendRegistration, } from "@spine-event-engine/ai/spi/adapter";
|
|
15
|
+
import { isAiModel } from "@spine-event-engine/ai/spi/runtime";
|
|
16
|
+
import { ScriptedAttempts } from "./scripted-attempts.js";
|
|
17
|
+
/**
|
|
18
|
+
* Scripted external model dependency registered by the normal AI registry.
|
|
19
|
+
*/
|
|
20
|
+
export class AiTestBackend {
|
|
21
|
+
/**
|
|
22
|
+
* Normal factory-proven deployment registration.
|
|
23
|
+
*/
|
|
24
|
+
registration;
|
|
25
|
+
#queues = new Map();
|
|
26
|
+
#records = [];
|
|
27
|
+
/**
|
|
28
|
+
* Creates the factory-proven registration for this scripted deployment.
|
|
29
|
+
*
|
|
30
|
+
* @param ref Credential-free deployment reference.
|
|
31
|
+
* @param kind Generation or decision backend kind.
|
|
32
|
+
*/
|
|
33
|
+
constructor(ref, kind) {
|
|
34
|
+
this.registration = createBackendRegistration({
|
|
35
|
+
ref,
|
|
36
|
+
kind,
|
|
37
|
+
supports: (definition) => definition.kind === kind,
|
|
38
|
+
resolveIdentity: () => ({
|
|
39
|
+
provider: "scripted",
|
|
40
|
+
account: "test",
|
|
41
|
+
endpoint: "local",
|
|
42
|
+
model: this.registration.ref.name?.value ?? "",
|
|
43
|
+
}),
|
|
44
|
+
authorizeUse: () => true,
|
|
45
|
+
connect: (_scope, identity) => ({ model: this, identity }),
|
|
46
|
+
execute: (request) => this.execute(request),
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Creates a network-free deployment selected by normal registry rules.
|
|
51
|
+
*
|
|
52
|
+
* @param options Deployment reference and API kind.
|
|
53
|
+
* @returns Factory-proven scripted backend.
|
|
54
|
+
*/
|
|
55
|
+
static create(options) {
|
|
56
|
+
return new AiTestBackend(options.ref, options.kind);
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Queues responses for a factory-created capability of this deployment kind.
|
|
60
|
+
*
|
|
61
|
+
* @typeParam I Generated input descriptor.
|
|
62
|
+
* @typeParam O Generated output descriptor.
|
|
63
|
+
* @param model Factory-created capability.
|
|
64
|
+
* @returns Queue writer for this capability.
|
|
65
|
+
*/
|
|
66
|
+
forModel(model) {
|
|
67
|
+
if (!isAiModel(model))
|
|
68
|
+
throw new TypeError("Scripted capability must come from AiModel.define().");
|
|
69
|
+
const candidate = model.definition;
|
|
70
|
+
if (candidate.kind !== this.registration.kind)
|
|
71
|
+
throw new TypeError("Scripted response kind does not match backend kind.");
|
|
72
|
+
const key = ScriptedAttempts.key(candidate.name, candidate.version);
|
|
73
|
+
let queue = this.#queues.get(key);
|
|
74
|
+
if (queue === undefined) {
|
|
75
|
+
queue = {
|
|
76
|
+
definition: candidate,
|
|
77
|
+
scripts: [],
|
|
78
|
+
};
|
|
79
|
+
this.#queues.set(key, queue);
|
|
80
|
+
}
|
|
81
|
+
else if (queue.definition !== candidate) {
|
|
82
|
+
throw new TypeError("Scripted capability revision already has a different definition.");
|
|
83
|
+
}
|
|
84
|
+
return ScriptedAttempts.responses(queue, candidate.output);
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Returns immutable snapshots of actual backend dispatches.
|
|
88
|
+
*
|
|
89
|
+
* @returns Physical request observations.
|
|
90
|
+
*/
|
|
91
|
+
requests() {
|
|
92
|
+
return Object.freeze(this.#records
|
|
93
|
+
.filter((record) => record.dispatched)
|
|
94
|
+
.map((record) => ScriptedAttempts.snapshot(record)));
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* Checks for unused scripts, unexpected requests, and pending dispatches.
|
|
98
|
+
*/
|
|
99
|
+
assertSatisfied() {
|
|
100
|
+
const unused = [...this.#queues.values()].reduce((count, queue) => count + queue.scripts.length, 0);
|
|
101
|
+
const unexpected = this.#records.filter((record) => record.unexpected).length;
|
|
102
|
+
const pending = this.#records.filter((record) => record.dispatched && !record.settled).length;
|
|
103
|
+
if (unused || unexpected || pending)
|
|
104
|
+
throw new Error(`Scripted AI backend: ${String(unused)} unused response(s), ` +
|
|
105
|
+
`${String(unexpected)} unexpected request(s), ${String(pending)} pending request(s).`);
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Executes bounded generation corrections through runtime journal and validation barriers.
|
|
109
|
+
*
|
|
110
|
+
* @param request Selected authenticated backend invocation.
|
|
111
|
+
* @returns Admitted output or runtime-recorded failure.
|
|
112
|
+
*/
|
|
113
|
+
async execute(request) {
|
|
114
|
+
const key = ScriptedAttempts.key(request.definition.name, request.definition.version);
|
|
115
|
+
const candidate = this.#queues.get(key);
|
|
116
|
+
const queue = candidate?.definition === request.definition ? candidate : undefined;
|
|
117
|
+
const operation = request.operationId.value;
|
|
118
|
+
let prior = this.#records.findLast((candidate) => candidate.operation === operation && candidate.dispatched);
|
|
119
|
+
for (;;) {
|
|
120
|
+
const record = ScriptedAttempts.record(request, (prior?.attempt ?? 0) + 1, prior);
|
|
121
|
+
this.#records.push(record);
|
|
122
|
+
const outcome = await ScriptedAttempts.execute(request, record, queue);
|
|
123
|
+
record.settled = true;
|
|
124
|
+
if (outcome.ok ||
|
|
125
|
+
request.definition.kind !== "generation" ||
|
|
126
|
+
outcome.failure.code !== "INVALID_OUTPUT" ||
|
|
127
|
+
(request.definition.limits.modelRequests !== undefined &&
|
|
128
|
+
record.attempt >= request.definition.limits.modelRequests))
|
|
129
|
+
return outcome;
|
|
130
|
+
prior = record;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
//# sourceMappingURL=ai-test-backend.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ai-test-backend.js","sourceRoot":"","sources":["../../src/ai-test/ai-test-backend.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAWH,OAAO,EACL,yBAAyB,GAI1B,MAAM,oCAAoC,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,oCAAoC,CAAC;AAE/D,OAAO,EAAE,gBAAgB,EAAwC,MAAM,wBAAwB,CAAC;AA8HhG;;GAEG;AACH,MAAM,OAAO,aAAa;IACxB;;OAEG;IACM,YAAY,CAAwB;IAEpC,OAAO,GAAG,IAAI,GAAG,EAAuB,CAAC;IAEzC,QAAQ,GAAoB,EAAE,CAAC;IAExC;;;;;OAKG;IACH,YAAoB,GAAa,EAAE,IAA+B;QAChE,IAAI,CAAC,YAAY,GAAG,yBAAyB,CAAC;YAC5C,GAAG;YACH,IAAI;YACJ,QAAQ,EAAE,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,KAAK,IAAI;YAClD,eAAe,EAAE,GAAG,EAAE,CAAC,CAAC;gBACtB,QAAQ,EAAE,UAAU;gBACpB,OAAO,EAAE,MAAM;gBACf,QAAQ,EAAE,OAAO;gBACjB,KAAK,EAAE,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,IAAI,EAAE;aAC/C,CAAC;YACF,YAAY,EAAE,GAAG,EAAE,CAAC,IAAI;YACxB,OAAO,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;YAC1D,OAAO,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC;SAC5C,CAAC,CAAC;IACL,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,MAAM,CAAC,OAGb;QACC,OAAO,IAAI,aAAa,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IACtD,CAAC;IAED;;;;;;;OAOG;IACH,QAAQ,CACN,KAAuB;QAEvB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC;YACnB,MAAM,IAAI,SAAS,CAAC,sDAAsD,CAAC,CAAC;QAC9E,MAAM,SAAS,GAAG,KAAK,CAAC,UAAU,CAAC;QACnC,IAAI,SAAS,CAAC,IAAI,KAAK,IAAI,CAAC,YAAY,CAAC,IAAI;YAC3C,MAAM,IAAI,SAAS,CAAC,qDAAqD,CAAC,CAAC;QAC7E,MAAM,GAAG,GAAG,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC;QACpE,IAAI,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAClC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,KAAK,GAAG;gBACN,UAAU,EAAE,SAAuE;gBACnF,OAAO,EAAE,EAAE;aACZ,CAAC;YACF,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC/B,CAAC;aAAM,IAAI,KAAK,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YAC1C,MAAM,IAAI,SAAS,CAAC,kEAAkE,CAAC,CAAC;QAC1F,CAAC;QACD,OAAO,gBAAgB,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;IAC7D,CAAC;IAED;;;;OAIG;IACH,QAAQ;QACN,OAAO,MAAM,CAAC,MAAM,CAClB,IAAI,CAAC,QAAQ;aACV,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;aACrC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,gBAAgB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CACtD,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,eAAe;QACb,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAC9C,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,EAC9C,CAAC,CACF,CAAC;QACF,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC;QAC9E,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;QAC9F,IAAI,MAAM,IAAI,UAAU,IAAI,OAAO;YACjC,MAAM,IAAI,KAAK,CACb,wBAAwB,MAAM,CAAC,MAAM,CAAC,uBAAuB;gBAC3D,GAAG,MAAM,CAAC,UAAU,CAAC,2BAA2B,MAAM,CAAC,OAAO,CAAC,sBAAsB,CACxF,CAAC;IACN,CAAC;IAED;;;;;OAKG;IACK,KAAK,CAAC,OAAO,CAAC,OAA2B;QAC/C,MAAM,GAAG,GAAG,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QACtF,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,KAAK,GAAG,SAAS,EAAE,UAAU,KAAK,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;QACnF,MAAM,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC;QAC5C,IAAI,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAChC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,SAAS,KAAK,SAAS,IAAI,SAAS,CAAC,UAAU,CACzE,CAAC;QACF,SAAS,CAAC;YACR,MAAM,MAAM,GAAG,gBAAgB,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,OAAO,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC;YAClF,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC3B,MAAM,OAAO,GAAG,MAAM,gBAAgB,CAAC,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;YACvE,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;YACtB,IACE,OAAO,CAAC,EAAE;gBACV,OAAO,CAAC,UAAU,CAAC,IAAI,KAAK,YAAY;gBACxC,OAAO,CAAC,OAAO,CAAC,IAAI,KAAK,gBAAgB;gBACzC,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,aAAa,KAAK,SAAS;oBACpD,MAAM,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC;gBAE5D,OAAO,OAAO,CAAC;YACjB,KAAK,GAAG,MAAM,CAAC;QACjB,CAAC;IACH,CAAC;CACF"}
|