@artemiskit/sdk 0.6.0 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/README.md +14 -3
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +36 -9
- package/dist/types-only.d.ts +1 -1
- package/dist/types-only.d.ts.map +1 -1
- package/dist/workflow.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/__tests__/workflow-recovery.test.ts +116 -0
- package/src/__tests__/workflow.test.ts +149 -2
- package/src/index.ts +32 -0
- package/src/types-only.ts +24 -0
- package/src/workflow.ts +28 -10
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,23 @@
|
|
|
1
1
|
# @artemiskit/sdk
|
|
2
2
|
|
|
3
|
+
## 0.6.2
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- Add declared workflow faults and bounded retries, private local checkpoints with explicit pause/resume, and version 3 recovery evidence. Preserve workflow identity, tool authority, original deadlines and cumulative budgets across restart; refuse ambiguous pending operations. Expose the same recovery controls in CLI and SDK while retaining version 1/2 record compatibility.
|
|
8
|
+
- Updated dependencies
|
|
9
|
+
- @artemiskit/core@0.6.3
|
|
10
|
+
- @artemiskit/redteam@0.3.11
|
|
11
|
+
|
|
12
|
+
## 0.6.1
|
|
13
|
+
|
|
14
|
+
### Patch Changes
|
|
15
|
+
|
|
16
|
+
- Complete the 0.6.2 independent workflow outcome milestone: verify final artifacts, bounded JSON schemas, state, tool traces and policy before optional explicitly configured semantic judging. Keep target and judge identity, usage and limits separate; expose task validity and coverage, strict V1/V2 saved-record reading, SDK composition and CLI outcome exits without changing historical prompt-response reports.
|
|
17
|
+
- Updated dependencies
|
|
18
|
+
- @artemiskit/core@0.6.2
|
|
19
|
+
- @artemiskit/redteam@0.3.10
|
|
20
|
+
|
|
3
21
|
## 0.6.0
|
|
4
22
|
|
|
5
23
|
### Minor Changes
|
package/README.md
CHANGED
|
@@ -57,16 +57,27 @@ const session = await kit.createWorkflowSession({
|
|
|
57
57
|
preflight: true,
|
|
58
58
|
});
|
|
59
59
|
const result = await session.run();
|
|
60
|
-
console.log(result.record); // bounded metadata
|
|
60
|
+
console.log(result.record); // bounded metadata with separate execution and task outcomes
|
|
61
61
|
// session.cancel() stops new work and initiates bounded cleanup.
|
|
62
62
|
// result.state and result.transcript contain sensitive working data.
|
|
63
63
|
```
|
|
64
64
|
|
|
65
|
-
`kit.runWorkflow(options)` creates and runs a
|
|
65
|
+
`kit.runWorkflow(options)` creates and runs a session in one call; fresh execution is the default. Supply a `client` or
|
|
66
66
|
`target` to use an existing trusted adapter or target. The same native controls apply through the
|
|
67
67
|
CLI and SDK: declared tools and permissions, cumulative budgets, explicit preflight, cancellation,
|
|
68
68
|
and cleanup evidence. Workflows use fresh simulated state or a disposable Docker filesystem.
|
|
69
|
-
|
|
69
|
+
The 0.6.2 milestone independently verifies declared artifacts, state, schemas, traces and policy.
|
|
70
|
+
Use explicit `semanticJudge` configuration for declared qualitative criteria. Target and judge usage
|
|
71
|
+
remain separate; `readWorkflowRecord` validates V1/V2/V3 saved metadata without making model calls.
|
|
72
|
+
See [workflow outcomes](https://github.com/code-sensei/artemiskit/blob/main/docs/workflow-outcomes.md)
|
|
73
|
+
for usage, denominators, compatibility and publication status.
|
|
74
|
+
|
|
75
|
+
For explicit private recovery, pass `checkpoint: { directory, mode: 'create', configurationId }`
|
|
76
|
+
and optionally `pauseAfterActions`. A later process uses the same authorized configuration and
|
|
77
|
+
`mode: 'resume'`. `session.pause()` requests a safe boundary; `session.cancel()` still cancels.
|
|
78
|
+
Checkpoints retain sensitive conversation and state with cumulative budgets and the original
|
|
79
|
+
deadline. A paused result is not task success. See the [0.6.3 recovery guide](https://github.com/code-sensei/artemiskit/blob/main/docs/workflow-recovery.md)
|
|
80
|
+
for complete examples, fault schedules, refusal conditions and qualification status.
|
|
70
81
|
|
|
71
82
|
The root and `/types` entry points expose workflow result/session/event types. Low-level
|
|
72
83
|
`createAgentWorkflowSession` and `runAgentWorkflow` are available for explicit target composition.
|
package/dist/index.d.ts
CHANGED
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
* Programmatic SDK for ArtemisKit LLM evaluation toolkit
|
|
4
4
|
*/
|
|
5
5
|
export { ArtemisKit } from './artemiskit.js';
|
|
6
|
-
export { AgentWorkflowSchema, WorkflowPolicySchema, WORKFLOW_TOOL_IDS, validateAgentWorkflow, parseAgentWorkflow, loadAgentWorkflow, listWorkflowTools, getWorkflowTool, executeSimulatedTool, createModelClientTarget, createAgentWorkflowSession, runAgentWorkflow, createSimulatedWorkflowEnvironment, createDockerWorkflowEnvironment, createDockerWorkflowEnvironmentFactory, WORKFLOW_SANDBOX_IMAGE, WorkflowEnvironmentInitializationError, } from '@artemiskit/core';
|
|
7
|
-
export type { AgentWorkflow, WorkflowPolicy, WorkflowJson, WorkflowToolId, WorkflowResource, WorkflowToolDescriptor, SimulatedToolEvidence, SimulatedToolResult, AgentTarget, AgentTurnRequest, AgentTurnResult, AgentTargetCapabilities, AgentTargetFailure, AgentWorkflowExecution, AgentWorkflowReason, AgentWorkflowEvent, AgentWorkflowRecord, AgentWorkflowResult, AgentWorkflowSessionOptions, AgentWorkflowSession, WorkflowState, WorkflowEnvironment, WorkflowEnvironmentFactory, WorkflowEnvironmentCleanup, DockerWorkflowEnvironmentOptions, } from '@artemiskit/core';
|
|
6
|
+
export { AgentWorkflowSchema, WorkflowPolicySchema, WorkflowFaultsSchema, WorkflowRetrySchema, WORKFLOW_TOOL_IDS, validateAgentWorkflow, parseAgentWorkflow, loadAgentWorkflow, listWorkflowTools, getWorkflowTool, executeSimulatedTool, createModelClientTarget, createAgentWorkflowSession, runAgentWorkflow, evaluateWorkflowDeterministicOutcomes, evaluateWorkflowSemantics, isValidWorkflowJudgeOptions, decideWorkflowOutcome, validWorkflowAssertionCounts, readWorkflowRecord, createSimulatedWorkflowEnvironment, createDockerWorkflowEnvironment, createDockerWorkflowEnvironmentFactory, WORKFLOW_SANDBOX_IMAGE, WorkflowEnvironmentInitializationError, } from '@artemiskit/core';
|
|
7
|
+
export type { AgentWorkflow, WorkflowPolicy, WorkflowJson, WorkflowToolId, WorkflowResource, WorkflowToolDescriptor, SimulatedToolEvidence, SimulatedToolResult, AgentTarget, AgentTurnRequest, AgentTurnResult, AgentTargetCapabilities, AgentTargetFailure, AgentWorkflowExecution, AgentWorkflowReason, AgentWorkflowEvent, AgentWorkflowRecord, AgentWorkflowExecutionRecord, AgentWorkflowEvidence, WorkflowDeterministicStatus, WorkflowDeterministicReason, WorkflowDeterministicAssertion, WorkflowDeterministicSummary, WorkflowJudgeOptions, WorkflowSemanticOptions, WorkflowSemanticReason, WorkflowSemanticAssertion, WorkflowSemanticIdentity, WorkflowSemanticEvaluation, WorkflowOutcomeStatus, WorkflowAssertionCounts, WorkflowOutcomeDecision, WorkflowOutcomeAssessment, SavedWorkflowRecord, HistoricalWorkflowRecord, AgentWorkflowResult, AgentWorkflowSessionOptions, WorkflowCheckpointOptions, WorkflowRecoveryEvidence, WorkflowFault, WorkflowFaults, WorkflowFaultKind, WorkflowRetry, AgentWorkflowSession, WorkflowState, WorkflowEnvironment, WorkflowEnvironmentFactory, WorkflowEnvironmentCleanup, DockerWorkflowEnvironmentOptions, } from '@artemiskit/core';
|
|
8
8
|
export type { ArtemisKitConfig, WorkflowRunOptions, RunOptions, RedTeamOptions, StressOptions, RunResult, RedTeamResult, StressResult, CaseStartEvent, CaseCompleteEvent, ProgressEvent, RedTeamMutationStartEvent, RedTeamMutationCompleteEvent, StressRequestCompleteEvent, CaseStartHandler, CaseCompleteHandler, ProgressHandler, RedTeamMutationStartHandler, RedTeamMutationCompleteHandler, StressRequestCompleteHandler, ArtemisKitEvents, ArtemisKitEventName, ValidateOptions, ValidationResult, ValidationError, ValidationWarning, ScenarioValidation, CompareOptions, CompareResult, RunSummary, ComparisonDetails, } from './types.js';
|
|
9
9
|
export type { Scenario, TestCase, Expected, Provider, ProviderConfig, ChatMessageType, Variables, } from '@artemiskit/core';
|
|
10
10
|
export type { CaseResult, CaseRedactionInfo, RunManifest, RunConfig, RunMetrics, RedTeamManifest, RedTeamConfig, RedTeamMetrics, RedTeamCaseResult, RedTeamStatus, RedTeamSeverity, StressManifest, StressConfig, StressMetrics, StressRequestResult, AnyManifest, ResolvedConfig, ConfigSource, ManifestRedactionInfo, CostEstimateInfo, GitInfo, ProvenanceInfo, } from '@artemiskit/core';
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAG1C,OAAO,EACL,mBAAmB,EACnB,oBAAoB,EACpB,iBAAiB,EACjB,qBAAqB,EACrB,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,eAAe,EACf,oBAAoB,EACpB,uBAAuB,EACvB,0BAA0B,EAC1B,gBAAgB,EAChB,kCAAkC,EAClC,+BAA+B,EAC/B,sCAAsC,EACtC,sBAAsB,EACtB,sCAAsC,GACvC,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,aAAa,EACb,cAAc,EACd,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,sBAAsB,EACtB,qBAAqB,EACrB,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,eAAe,EACf,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,2BAA2B,EAC3B,oBAAoB,EACpB,aAAa,EACb,mBAAmB,EACnB,0BAA0B,EAC1B,0BAA0B,EAC1B,gCAAgC,GACjC,MAAM,kBAAkB,CAAC;AAG1B,YAAY,EAEV,gBAAgB,EAChB,kBAAkB,EAClB,UAAU,EACV,cAAc,EACd,aAAa,EAEb,SAAS,EACT,aAAa,EACb,YAAY,EAEZ,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,yBAAyB,EACzB,4BAA4B,EAC5B,0BAA0B,EAE1B,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAE5B,gBAAgB,EAChB,mBAAmB,EAEnB,eAAe,EACf,gBAAgB,EAChB,eAAe,EACf,iBAAiB,EACjB,kBAAkB,EAElB,cAAc,EACd,aAAa,EACb,UAAU,EACV,iBAAiB,GAClB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,cAAc,EACd,eAAe,EACf,SAAS,GACV,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,UAAU,EACV,iBAAiB,EAEjB,WAAW,EACX,SAAS,EACT,UAAU,EAEV,eAAe,EACf,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,eAAe,EAEf,cAAc,EACd,YAAY,EACZ,aAAa,EACb,mBAAmB,EAEnB,WAAW,EACX,cAAc,EACd,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,OAAO,EACP,cAAc,GACf,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,WAAW,EACX,iBAAiB,EAEjB,eAAe,EACf,cAAc,EACd,UAAU,EAEV,WAAW,EAEX,kBAAkB,EAClB,cAAc,EACd,QAAQ,EAER,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,YAAY,GACb,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMrF,YAAY,EACV,cAAc,EACd,aAAa,EACb,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EAEtB,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EACtB,eAAe,EACf,mBAAmB,EACnB,cAAc,EACd,uBAAuB,GACxB,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMxD,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAMtF,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAM7E,OAAO,EAAE,kBAAkB,EAAE,KAAK,kBAAkB,EAAE,KAAK,aAAa,EAAE,MAAM,YAAY,CAAC;AAM7F,OAAO,EAEL,WAAW,EACX,eAAe,EACf,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,oBAAoB,EAEpB,aAAa,EACb,MAAM,EAEN,cAAc,EACd,cAAc,EACd,aAAa,EACb,oBAAoB,GACrB,MAAM,SAAS,CAAC;AAEjB,YAAY,EAEV,YAAY,EACZ,eAAe,EAEf,SAAS,EACT,eAAe,EACf,mBAAmB,EACnB,oBAAoB,EACpB,eAAe,EAEf,WAAW,EACX,eAAe,EACf,aAAa,EACb,cAAc,EACd,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,OAAO,EAEL,aAAa,EAEb,eAAe,EAEf,aAAa,EAEb,YAAY,GACb,MAAM,aAAa,CAAC;AAErB,YAAY,EACV,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,aAAa,CAAC;AAMrB,OAAO,EAEL,eAAe,EACf,eAAe,EAEf,QAAQ,EACR,QAAQ,EAER,YAAY,EACZ,SAAS,EACT,SAAS,EACT,QAAQ,EACR,UAAU,EAEV,KAAK,EACL,QAAQ,EACR,WAAW,EACX,KAAK,EACL,KAAK,EACL,UAAU,EACV,QAAQ,EACR,UAAU,EACV,MAAM,EACN,KAAK,EACL,KAAK,GACN,MAAM,YAAY,CAAC;AAMpB,OAAO,EAEL,QAAQ,EACR,cAAc,EAEd,qBAAqB,EAErB,iBAAiB,EACjB,uBAAuB,EAEvB,mBAAmB,EACnB,oBAAoB,EACpB,iBAAiB,EAEjB,eAAe,EACf,4BAA4B,EAE5B,gBAAgB,EAChB,sBAAsB,EAEtB,eAAe,EACf,wBAAwB,EACxB,SAAS,EACT,kBAAkB,EAClB,aAAa,EACb,4BAA4B,EAC5B,gBAAgB,EAEhB,YAAY,EACZ,4BAA4B,EAE5B,UAAU,EACV,WAAW,EACX,cAAc,EACd,mBAAmB,EACnB,aAAa,EACb,sBAAsB,EACtB,eAAe,EACf,qBAAqB,EAErB,cAAc,EACd,gBAAgB,EAChB,WAAW,GACZ,MAAM,YAAY,CAAC;AAGpB,YAAY,EAEV,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,sBAAsB,EACtB,gBAAgB,EAChB,iBAAiB,EACjB,oBAAoB,EAEpB,YAAY,EACZ,iBAAiB,EACjB,eAAe,EACf,aAAa,EACb,SAAS,EACT,eAAe,EAEf,UAAU,EACV,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,eAAe,EACf,eAAe,EAEf,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EAEnB,oBAAoB,EACpB,cAAc,EAEd,YAAY,EACZ,OAAO,EACP,WAAW,EACX,kBAAkB,EAClB,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,eAAe,EAEf,eAAe,EACf,mBAAmB,EACnB,YAAY,EAEZ,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EAEpB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EAEpB,aAAa,EACb,0BAA0B,EAE1B,qBAAqB,EACrB,kBAAkB,EAClB,eAAe,EAEf,uBAAuB,EACvB,aAAa,EACb,kBAAkB,EAClB,eAAe,EACf,wBAAwB,EAExB,kBAAkB,EAClB,oBAAoB,EACpB,cAAc,EACd,cAAc,EACd,mBAAmB,EACnB,4BAA4B,EAC5B,yBAAyB,EACzB,kCAAkC,EAClC,2BAA2B,EAC3B,mBAAmB,EACnB,uBAAuB,EACvB,eAAe,EACf,eAAe,EACf,gBAAgB,EAChB,yBAAyB,EACzB,sBAAsB,GACvB,MAAM,YAAY,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAG1C,OAAO,EACL,mBAAmB,EACnB,oBAAoB,EACpB,oBAAoB,EACpB,mBAAmB,EACnB,iBAAiB,EACjB,qBAAqB,EACrB,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,eAAe,EACf,oBAAoB,EACpB,uBAAuB,EACvB,0BAA0B,EAC1B,gBAAgB,EAChB,qCAAqC,EACrC,yBAAyB,EACzB,2BAA2B,EAC3B,qBAAqB,EACrB,4BAA4B,EAC5B,kBAAkB,EAClB,kCAAkC,EAClC,+BAA+B,EAC/B,sCAAsC,EACtC,sBAAsB,EACtB,sCAAsC,GACvC,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,aAAa,EACb,cAAc,EACd,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,sBAAsB,EACtB,qBAAqB,EACrB,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,eAAe,EACf,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,kBAAkB,EAClB,mBAAmB,EACnB,4BAA4B,EAC5B,qBAAqB,EACrB,2BAA2B,EAC3B,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAC5B,oBAAoB,EACpB,uBAAuB,EACvB,sBAAsB,EACtB,yBAAyB,EACzB,wBAAwB,EACxB,0BAA0B,EAC1B,qBAAqB,EACrB,uBAAuB,EACvB,uBAAuB,EACvB,yBAAyB,EACzB,mBAAmB,EACnB,wBAAwB,EACxB,mBAAmB,EACnB,2BAA2B,EAC3B,yBAAyB,EACzB,wBAAwB,EACxB,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EACpB,aAAa,EACb,mBAAmB,EACnB,0BAA0B,EAC1B,0BAA0B,EAC1B,gCAAgC,GACjC,MAAM,kBAAkB,CAAC;AAG1B,YAAY,EAEV,gBAAgB,EAChB,kBAAkB,EAClB,UAAU,EACV,cAAc,EACd,aAAa,EAEb,SAAS,EACT,aAAa,EACb,YAAY,EAEZ,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,yBAAyB,EACzB,4BAA4B,EAC5B,0BAA0B,EAE1B,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAE5B,gBAAgB,EAChB,mBAAmB,EAEnB,eAAe,EACf,gBAAgB,EAChB,eAAe,EACf,iBAAiB,EACjB,kBAAkB,EAElB,cAAc,EACd,aAAa,EACb,UAAU,EACV,iBAAiB,GAClB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,cAAc,EACd,eAAe,EACf,SAAS,GACV,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,UAAU,EACV,iBAAiB,EAEjB,WAAW,EACX,SAAS,EACT,UAAU,EAEV,eAAe,EACf,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,eAAe,EAEf,cAAc,EACd,YAAY,EACZ,aAAa,EACb,mBAAmB,EAEnB,WAAW,EACX,cAAc,EACd,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,OAAO,EACP,cAAc,GACf,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,WAAW,EACX,iBAAiB,EAEjB,eAAe,EACf,cAAc,EACd,UAAU,EAEV,WAAW,EAEX,kBAAkB,EAClB,cAAc,EACd,QAAQ,EAER,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,YAAY,GACb,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMrF,YAAY,EACV,cAAc,EACd,aAAa,EACb,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EAEtB,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EACtB,eAAe,EACf,mBAAmB,EACnB,cAAc,EACd,uBAAuB,GACxB,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMxD,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAMtF,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAM7E,OAAO,EAAE,kBAAkB,EAAE,KAAK,kBAAkB,EAAE,KAAK,aAAa,EAAE,MAAM,YAAY,CAAC;AAM7F,OAAO,EAEL,WAAW,EACX,eAAe,EACf,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,oBAAoB,EAEpB,aAAa,EACb,MAAM,EAEN,cAAc,EACd,cAAc,EACd,aAAa,EACb,oBAAoB,GACrB,MAAM,SAAS,CAAC;AAEjB,YAAY,EAEV,YAAY,EACZ,eAAe,EAEf,SAAS,EACT,eAAe,EACf,mBAAmB,EACnB,oBAAoB,EACpB,eAAe,EAEf,WAAW,EACX,eAAe,EACf,aAAa,EACb,cAAc,EACd,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,OAAO,EAEL,aAAa,EAEb,eAAe,EAEf,aAAa,EAEb,YAAY,GACb,MAAM,aAAa,CAAC;AAErB,YAAY,EACV,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,aAAa,CAAC;AAMrB,OAAO,EAEL,eAAe,EACf,eAAe,EAEf,QAAQ,EACR,QAAQ,EAER,YAAY,EACZ,SAAS,EACT,SAAS,EACT,QAAQ,EACR,UAAU,EAEV,KAAK,EACL,QAAQ,EACR,WAAW,EACX,KAAK,EACL,KAAK,EACL,UAAU,EACV,QAAQ,EACR,UAAU,EACV,MAAM,EACN,KAAK,EACL,KAAK,GACN,MAAM,YAAY,CAAC;AAMpB,OAAO,EAEL,QAAQ,EACR,cAAc,EAEd,qBAAqB,EAErB,iBAAiB,EACjB,uBAAuB,EAEvB,mBAAmB,EACnB,oBAAoB,EACpB,iBAAiB,EAEjB,eAAe,EACf,4BAA4B,EAE5B,gBAAgB,EAChB,sBAAsB,EAEtB,eAAe,EACf,wBAAwB,EACxB,SAAS,EACT,kBAAkB,EAClB,aAAa,EACb,4BAA4B,EAC5B,gBAAgB,EAEhB,YAAY,EACZ,4BAA4B,EAE5B,UAAU,EACV,WAAW,EACX,cAAc,EACd,mBAAmB,EACnB,aAAa,EACb,sBAAsB,EACtB,eAAe,EACf,qBAAqB,EAErB,cAAc,EACd,gBAAgB,EAChB,WAAW,GACZ,MAAM,YAAY,CAAC;AAGpB,YAAY,EAEV,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,sBAAsB,EACtB,gBAAgB,EAChB,iBAAiB,EACjB,oBAAoB,EAEpB,YAAY,EACZ,iBAAiB,EACjB,eAAe,EACf,aAAa,EACb,SAAS,EACT,eAAe,EAEf,UAAU,EACV,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,eAAe,EACf,eAAe,EAEf,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EAEnB,oBAAoB,EACpB,cAAc,EAEd,YAAY,EACZ,OAAO,EACP,WAAW,EACX,kBAAkB,EAClB,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,eAAe,EAEf,eAAe,EACf,mBAAmB,EACnB,YAAY,EAEZ,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EAEpB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EAEpB,aAAa,EACb,0BAA0B,EAE1B,qBAAqB,EACrB,kBAAkB,EAClB,eAAe,EAEf,uBAAuB,EACvB,aAAa,EACb,kBAAkB,EAClB,eAAe,EACf,wBAAwB,EAExB,kBAAkB,EAClB,oBAAoB,EACpB,cAAc,EACd,cAAc,EACd,mBAAmB,EACnB,4BAA4B,EAC5B,yBAAyB,EACzB,kCAAkC,EAClC,2BAA2B,EAC3B,mBAAmB,EACnB,uBAAuB,EACvB,eAAe,EACf,eAAe,EACf,gBAAgB,EAChB,yBAAyB,EACzB,sBAAsB,GACvB,MAAM,YAAY,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -7653,21 +7653,29 @@ async function prepareWorkflowSession(options, config) {
|
|
|
7653
7653
|
const workflow = file ? await loadAgentWorkflow(file) : validateAgentWorkflow(options.workflow);
|
|
7654
7654
|
let target = options.target;
|
|
7655
7655
|
if (!target) {
|
|
7656
|
-
|
|
7657
|
-
|
|
7658
|
-
|
|
7659
|
-
|
|
7656
|
+
if (options.client)
|
|
7657
|
+
target = createModelClientTarget(options.client);
|
|
7658
|
+
else {
|
|
7659
|
+
let initialized;
|
|
7660
|
+
const getTarget = () => {
|
|
7661
|
+
initialized ??= createAdapter({
|
|
7660
7662
|
...config.providerConfig,
|
|
7661
7663
|
...options.providerConfig,
|
|
7662
7664
|
provider: workflow.target.provider,
|
|
7663
7665
|
defaultModel: workflow.target.model,
|
|
7664
7666
|
maxRetries: 0
|
|
7667
|
+
}).then(createModelClientTarget).catch(() => {
|
|
7668
|
+
throw new ArtemisError("Unable to initialize workflow target", "PROVIDER_UNAVAILABLE");
|
|
7665
7669
|
});
|
|
7666
|
-
|
|
7667
|
-
|
|
7668
|
-
|
|
7670
|
+
return initialized;
|
|
7671
|
+
};
|
|
7672
|
+
target = options.checkpoint ? {
|
|
7673
|
+
provider: workflow.target.provider,
|
|
7674
|
+
capabilities: async (limits, signal) => (await getTarget()).capabilities(limits, signal),
|
|
7675
|
+
turn: async (request, signal) => (await getTarget()).turn(request, signal),
|
|
7676
|
+
drain: async (limits) => initialized ? (await initialized).drain?.(limits) ?? { pendingOperations: 0 } : { pendingOperations: 0 }
|
|
7677
|
+
} : await getTarget();
|
|
7669
7678
|
}
|
|
7670
|
-
target = createModelClientTarget(client);
|
|
7671
7679
|
}
|
|
7672
7680
|
return createAgentWorkflowSession({
|
|
7673
7681
|
workflow,
|
|
@@ -7678,7 +7686,10 @@ async function prepareWorkflowSession(options, config) {
|
|
|
7678
7686
|
preflightOnly: options.preflightOnly,
|
|
7679
7687
|
signal: options.signal,
|
|
7680
7688
|
cleanupTimeoutMs: options.cleanupTimeoutMs,
|
|
7681
|
-
|
|
7689
|
+
semanticJudge: options.semanticJudge,
|
|
7690
|
+
onEvent: options.onEvent,
|
|
7691
|
+
checkpoint: options.checkpoint,
|
|
7692
|
+
pauseAfterActions: options.pauseAfterActions
|
|
7682
7693
|
});
|
|
7683
7694
|
}
|
|
7684
7695
|
|
|
@@ -8506,6 +8517,8 @@ function percentile(sortedArray, p) {
|
|
|
8506
8517
|
import {
|
|
8507
8518
|
AgentWorkflowSchema,
|
|
8508
8519
|
WorkflowPolicySchema,
|
|
8520
|
+
WorkflowFaultsSchema,
|
|
8521
|
+
WorkflowRetrySchema,
|
|
8509
8522
|
WORKFLOW_TOOL_IDS,
|
|
8510
8523
|
validateAgentWorkflow as validateAgentWorkflow2,
|
|
8511
8524
|
parseAgentWorkflow,
|
|
@@ -8516,6 +8529,12 @@ import {
|
|
|
8516
8529
|
createModelClientTarget as createModelClientTarget2,
|
|
8517
8530
|
createAgentWorkflowSession as createAgentWorkflowSession2,
|
|
8518
8531
|
runAgentWorkflow,
|
|
8532
|
+
evaluateWorkflowDeterministicOutcomes,
|
|
8533
|
+
evaluateWorkflowSemantics,
|
|
8534
|
+
isValidWorkflowJudgeOptions,
|
|
8535
|
+
decideWorkflowOutcome,
|
|
8536
|
+
validWorkflowAssertionCounts,
|
|
8537
|
+
readWorkflowRecord,
|
|
8519
8538
|
createSimulatedWorkflowEnvironment,
|
|
8520
8539
|
createDockerWorkflowEnvironment,
|
|
8521
8540
|
createDockerWorkflowEnvironmentFactory,
|
|
@@ -15257,12 +15276,14 @@ function createGuardian(config = {}) {
|
|
|
15257
15276
|
export {
|
|
15258
15277
|
validatePolicy,
|
|
15259
15278
|
validateAgentWorkflow2 as validateAgentWorkflow,
|
|
15279
|
+
validWorkflowAssertionCounts,
|
|
15260
15280
|
testCase,
|
|
15261
15281
|
similarity,
|
|
15262
15282
|
scenario,
|
|
15263
15283
|
runAgentWorkflow,
|
|
15264
15284
|
regexCase,
|
|
15265
15285
|
regex,
|
|
15286
|
+
readWorkflowRecord,
|
|
15266
15287
|
parsePolicy,
|
|
15267
15288
|
parseAgentWorkflow,
|
|
15268
15289
|
notContains,
|
|
@@ -15275,6 +15296,7 @@ export {
|
|
|
15275
15296
|
listWorkflowTools,
|
|
15276
15297
|
jsonSchema,
|
|
15277
15298
|
jsonCase,
|
|
15299
|
+
isValidWorkflowJudgeOptions,
|
|
15278
15300
|
isStressResult,
|
|
15279
15301
|
isStressManifestType,
|
|
15280
15302
|
isStressManifest,
|
|
@@ -15296,12 +15318,15 @@ export {
|
|
|
15296
15318
|
executeSimulatedTool,
|
|
15297
15319
|
exactCase,
|
|
15298
15320
|
exact,
|
|
15321
|
+
evaluateWorkflowSemantics,
|
|
15322
|
+
evaluateWorkflowDeterministicOutcomes,
|
|
15299
15323
|
detectPII,
|
|
15300
15324
|
detectInjection,
|
|
15301
15325
|
defineStorage,
|
|
15302
15326
|
definePlugin,
|
|
15303
15327
|
defineEvaluator,
|
|
15304
15328
|
defineAdapter,
|
|
15329
|
+
decideWorkflowOutcome,
|
|
15305
15330
|
createSimulatedWorkflowEnvironment,
|
|
15306
15331
|
createSemanticValidator,
|
|
15307
15332
|
createPIIGuardrail,
|
|
@@ -15326,7 +15351,9 @@ export {
|
|
|
15326
15351
|
artemiskitMatchers,
|
|
15327
15352
|
anyOf,
|
|
15328
15353
|
allOf,
|
|
15354
|
+
WorkflowRetrySchema,
|
|
15329
15355
|
WorkflowPolicySchema,
|
|
15356
|
+
WorkflowFaultsSchema,
|
|
15330
15357
|
WorkflowEnvironmentInitializationError,
|
|
15331
15358
|
WORKFLOW_TOOL_IDS,
|
|
15332
15359
|
WORKFLOW_SANDBOX_IMAGE,
|
package/dist/types-only.d.ts
CHANGED
|
@@ -28,5 +28,5 @@ export type { ArtemisKitMatchers, MatcherResult } from './matchers/index.js';
|
|
|
28
28
|
export type { ProviderName, ExpectationType, AnyResult, ExtractRunCases, ExtractRedTeamCases, ExtractStressResults, ExtractManifest, DeepPartial, PartialScenario, RequireFields, OptionalFields, StrictAdapterConfig, } from './utils/index.js';
|
|
29
29
|
export type { AdapterContract, EvaluatorContract, StorageContract, AdapterFactory, EvaluatorFactory, StorageFactory, ArtemisKitPlugin, } from './contracts/index.js';
|
|
30
30
|
export type { ScenarioBuilder, TestCaseBuilder } from './builders/index.js';
|
|
31
|
-
export type { AgentWorkflow, WorkflowPolicy, WorkflowJson, WorkflowToolId, WorkflowResource, WorkflowToolDescriptor, SimulatedToolEvidence, SimulatedToolResult, AgentTarget, AgentTurnRequest, AgentTurnResult, AgentTargetCapabilities, AgentTargetFailure, AgentWorkflowExecution, AgentWorkflowReason, AgentWorkflowEvent, AgentWorkflowRecord, AgentWorkflowResult, AgentWorkflowSessionOptions, AgentWorkflowSession, WorkflowState, WorkflowEnvironment, WorkflowEnvironmentFactory, WorkflowEnvironmentCleanup, DockerWorkflowEnvironmentOptions, WorkflowEnvironmentInitializationError, } from '@artemiskit/core';
|
|
31
|
+
export type { AgentWorkflow, WorkflowPolicy, WorkflowJson, WorkflowToolId, WorkflowResource, WorkflowToolDescriptor, SimulatedToolEvidence, SimulatedToolResult, AgentTarget, AgentTurnRequest, AgentTurnResult, AgentTargetCapabilities, AgentTargetFailure, AgentWorkflowExecution, AgentWorkflowReason, AgentWorkflowEvent, AgentWorkflowRecord, AgentWorkflowExecutionRecord, AgentWorkflowEvidence, WorkflowDeterministicStatus, WorkflowDeterministicReason, WorkflowDeterministicAssertion, WorkflowDeterministicSummary, WorkflowJudgeOptions, WorkflowSemanticOptions, WorkflowSemanticReason, WorkflowSemanticAssertion, WorkflowSemanticIdentity, WorkflowSemanticEvaluation, WorkflowOutcomeStatus, WorkflowAssertionCounts, WorkflowOutcomeDecision, WorkflowOutcomeAssessment, SavedWorkflowRecord, HistoricalWorkflowRecord, AgentWorkflowResult, AgentWorkflowSessionOptions, WorkflowCheckpointOptions, WorkflowRecoveryEvidence, WorkflowFault, WorkflowFaults, WorkflowFaultKind, WorkflowRetry, AgentWorkflowSession, WorkflowState, WorkflowEnvironment, WorkflowEnvironmentFactory, WorkflowEnvironmentCleanup, DockerWorkflowEnvironmentOptions, WorkflowEnvironmentInitializationError, } from '@artemiskit/core';
|
|
32
32
|
//# sourceMappingURL=types-only.d.ts.map
|
package/dist/types-only.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types-only.d.ts","sourceRoot":"","sources":["../src/types-only.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAMH,YAAY,EAEV,gBAAgB,EAChB,kBAAkB,EAClB,UAAU,EACV,cAAc,EACd,aAAa,EAEb,SAAS,EACT,aAAa,EACb,YAAY,EAEZ,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,yBAAyB,EACzB,4BAA4B,EAC5B,0BAA0B,EAE1B,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAE5B,gBAAgB,EAChB,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,cAAc,EACd,eAAe,EACf,SAAS,GACV,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,UAAU,EACV,iBAAiB,EAEjB,WAAW,EACX,SAAS,EACT,UAAU,EAEV,eAAe,EACf,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,eAAe,EAEf,cAAc,EACd,YAAY,EACZ,aAAa,EACb,mBAAmB,EAEnB,WAAW,EACX,cAAc,EACd,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,OAAO,EACP,cAAc,GACf,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,WAAW,EACX,iBAAiB,EAEjB,eAAe,EACf,cAAc,EACd,UAAU,EAEV,WAAW,EAEX,kBAAkB,EAClB,cAAc,EACd,QAAQ,EAER,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,YAAY,GACb,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMrF,YAAY,EACV,cAAc,EACd,aAAa,EACb,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EAEtB,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EACtB,eAAe,EACf,mBAAmB,EACnB,cAAc,EACd,uBAAuB,GACxB,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMxD,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAM7E,YAAY,EAEV,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,sBAAsB,EACtB,gBAAgB,EAChB,iBAAiB,EAEjB,YAAY,EACZ,iBAAiB,EACjB,eAAe,EACf,aAAa,EACb,SAAS,EACT,eAAe,EAEf,UAAU,EACV,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,eAAe,EACf,eAAe,EAEf,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EAEnB,oBAAoB,EACpB,cAAc,EAEd,YAAY,EACZ,OAAO,EACP,WAAW,EACX,kBAAkB,EAClB,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,eAAe,EAEf,eAAe,EACf,mBAAmB,EACnB,YAAY,EAEZ,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EAEpB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EAEpB,aAAa,EACb,0BAA0B,GAC3B,MAAM,YAAY,CAAC;AAMpB,YAAY,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAMpE,YAAY,EAEV,YAAY,EACZ,eAAe,EAEf,SAAS,EACT,eAAe,EACf,mBAAmB,EACnB,oBAAoB,EACpB,eAAe,EAEf,WAAW,EACX,eAAe,EACf,aAAa,EACb,cAAc,EACd,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,aAAa,CAAC;AAMrB,YAAY,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAEnE,YAAY,EACV,aAAa,EACb,cAAc,EACd,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,sBAAsB,EACtB,qBAAqB,EACrB,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,eAAe,EACf,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,2BAA2B,EAC3B,oBAAoB,EACpB,aAAa,EACb,mBAAmB,EACnB,0BAA0B,EAC1B,0BAA0B,EAC1B,gCAAgC,EAChC,sCAAsC,GACvC,MAAM,kBAAkB,CAAC"}
|
|
1
|
+
{"version":3,"file":"types-only.d.ts","sourceRoot":"","sources":["../src/types-only.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAMH,YAAY,EAEV,gBAAgB,EAChB,kBAAkB,EAClB,UAAU,EACV,cAAc,EACd,aAAa,EAEb,SAAS,EACT,aAAa,EACb,YAAY,EAEZ,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,yBAAyB,EACzB,4BAA4B,EAC5B,0BAA0B,EAE1B,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAE5B,gBAAgB,EAChB,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,cAAc,EACd,eAAe,EACf,SAAS,GACV,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,UAAU,EACV,iBAAiB,EAEjB,WAAW,EACX,SAAS,EACT,UAAU,EAEV,eAAe,EACf,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,eAAe,EAEf,cAAc,EACd,YAAY,EACZ,aAAa,EACb,mBAAmB,EAEnB,WAAW,EACX,cAAc,EACd,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,OAAO,EACP,cAAc,GACf,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,WAAW,EACX,iBAAiB,EAEjB,eAAe,EACf,cAAc,EACd,UAAU,EAEV,WAAW,EAEX,kBAAkB,EAClB,cAAc,EACd,QAAQ,EAER,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,YAAY,GACb,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMrF,YAAY,EACV,cAAc,EACd,aAAa,EACb,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EAEtB,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EACtB,eAAe,EACf,mBAAmB,EACnB,cAAc,EACd,uBAAuB,GACxB,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMxD,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAM7E,YAAY,EAEV,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,sBAAsB,EACtB,gBAAgB,EAChB,iBAAiB,EAEjB,YAAY,EACZ,iBAAiB,EACjB,eAAe,EACf,aAAa,EACb,SAAS,EACT,eAAe,EAEf,UAAU,EACV,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,eAAe,EACf,eAAe,EAEf,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EAEnB,oBAAoB,EACpB,cAAc,EAEd,YAAY,EACZ,OAAO,EACP,WAAW,EACX,kBAAkB,EAClB,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,eAAe,EAEf,eAAe,EACf,mBAAmB,EACnB,YAAY,EAEZ,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EAEpB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EAEpB,aAAa,EACb,0BAA0B,GAC3B,MAAM,YAAY,CAAC;AAMpB,YAAY,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAMpE,YAAY,EAEV,YAAY,EACZ,eAAe,EAEf,SAAS,EACT,eAAe,EACf,mBAAmB,EACnB,oBAAoB,EACpB,eAAe,EAEf,WAAW,EACX,eAAe,EACf,aAAa,EACb,cAAc,EACd,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,aAAa,CAAC;AAMrB,YAAY,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAEnE,YAAY,EACV,aAAa,EACb,cAAc,EACd,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,sBAAsB,EACtB,qBAAqB,EACrB,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,eAAe,EACf,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,kBAAkB,EAClB,mBAAmB,EACnB,4BAA4B,EAC5B,qBAAqB,EACrB,2BAA2B,EAC3B,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAC5B,oBAAoB,EACpB,uBAAuB,EACvB,sBAAsB,EACtB,yBAAyB,EACzB,wBAAwB,EACxB,0BAA0B,EAC1B,qBAAqB,EACrB,uBAAuB,EACvB,uBAAuB,EACvB,yBAAyB,EACzB,mBAAmB,EACnB,wBAAwB,EACxB,mBAAmB,EACnB,2BAA2B,EAC3B,yBAAyB,EACzB,wBAAwB,EACxB,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EACpB,aAAa,EACb,mBAAmB,EACnB,0BAA0B,EAC1B,0BAA0B,EAC1B,gCAAgC,EAChC,sCAAsC,GACvC,MAAM,kBAAkB,CAAC"}
|
package/dist/workflow.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"workflow.d.ts","sourceRoot":"","sources":["../src/workflow.ts"],"names":[],"mappings":"AACA,OAAO,
|
|
1
|
+
{"version":3,"file":"workflow.d.ts","sourceRoot":"","sources":["../src/workflow.ts"],"names":[],"mappings":"AACA,OAAO,EAGL,KAAK,oBAAoB,EAO1B,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAEpE,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,kBAAkB,EAC3B,MAAM,EAAE,gBAAgB,GACvB,OAAO,CAAC,oBAAoB,CAAC,CAmD/B"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@artemiskit/sdk",
|
|
3
|
-
"version": "0.6.
|
|
3
|
+
"version": "0.6.2",
|
|
4
4
|
"description": "Programmatic SDK for ArtemisKit LLM evaluation toolkit - integrate into your Node.js apps",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -71,8 +71,8 @@
|
|
|
71
71
|
"test:coverage": "bun test --coverage"
|
|
72
72
|
},
|
|
73
73
|
"dependencies": {
|
|
74
|
-
"@artemiskit/core": "0.6.
|
|
75
|
-
"@artemiskit/redteam": "0.3.
|
|
74
|
+
"@artemiskit/core": "0.6.3",
|
|
75
|
+
"@artemiskit/redteam": "0.3.11"
|
|
76
76
|
},
|
|
77
77
|
"devDependencies": {
|
|
78
78
|
"@types/bun": "^1.1.0",
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import { describe, expect, test } from 'bun:test';
|
|
2
|
+
import { mkdtemp } from 'node:fs/promises';
|
|
3
|
+
import { tmpdir } from 'node:os';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
import { type AgentTarget, validateAgentWorkflow } from '@artemiskit/core';
|
|
6
|
+
import { ArtemisKit } from '../artemiskit';
|
|
7
|
+
import { readWorkflowRecord } from '../index';
|
|
8
|
+
import type { WorkflowCheckpointOptions } from '../types-only';
|
|
9
|
+
|
|
10
|
+
describe('SDK workflow recovery options', () => {
|
|
11
|
+
test('forwards checkpoint and pause controls and preserves the same logical run', async () => {
|
|
12
|
+
const parent = await mkdtemp(join(tmpdir(), 'artemis-sdk-recovery-'));
|
|
13
|
+
const checkpoint: WorkflowCheckpointOptions = {
|
|
14
|
+
directory: join(parent, 'private'),
|
|
15
|
+
mode: 'create',
|
|
16
|
+
configurationId: 'sdk-fixture-v1',
|
|
17
|
+
};
|
|
18
|
+
const workflow = validateAgentWorkflow({
|
|
19
|
+
version: '1',
|
|
20
|
+
kind: 'agent_workflow',
|
|
21
|
+
name: 'sdk-recovery',
|
|
22
|
+
target: { provider: 'openai', model: 'fixture' },
|
|
23
|
+
environment: {
|
|
24
|
+
type: 'simulated',
|
|
25
|
+
policy: {
|
|
26
|
+
network: 'denied',
|
|
27
|
+
side_effects: 'denied',
|
|
28
|
+
permissions: { files: 'write' },
|
|
29
|
+
budgets: {
|
|
30
|
+
max_actions: 6,
|
|
31
|
+
max_model_requests: 2,
|
|
32
|
+
max_tool_calls: 2,
|
|
33
|
+
max_tokens: 6,
|
|
34
|
+
timeout_ms: 20000,
|
|
35
|
+
},
|
|
36
|
+
},
|
|
37
|
+
},
|
|
38
|
+
tools: ['write_file'],
|
|
39
|
+
workflow: {
|
|
40
|
+
system_instructions: 'Write two files.',
|
|
41
|
+
initial_state: { files: {} },
|
|
42
|
+
turns: [{ role: 'user', content: 'PRIVATE-WORKFLOW' }],
|
|
43
|
+
},
|
|
44
|
+
outcomes: {
|
|
45
|
+
deterministic: [{ type: 'file', path: 'two.txt', exists: true, equals: 'PRIVATE-FILE' }],
|
|
46
|
+
},
|
|
47
|
+
evidence: { trace: 'summary', artifacts: 'checksums', redact: true },
|
|
48
|
+
});
|
|
49
|
+
let calls = 0;
|
|
50
|
+
let capabilities = 0;
|
|
51
|
+
const target: AgentTarget = {
|
|
52
|
+
provider: 'openai',
|
|
53
|
+
capabilities: async () => {
|
|
54
|
+
capabilities++;
|
|
55
|
+
return { status: 'available', toolUse: true, transportCancellation: true };
|
|
56
|
+
},
|
|
57
|
+
turn: async (request) => {
|
|
58
|
+
calls++;
|
|
59
|
+
const continued = request.messages.some((message) => message.role === 'tool');
|
|
60
|
+
return {
|
|
61
|
+
status: 'completed',
|
|
62
|
+
id: `answer-${calls}`,
|
|
63
|
+
model: 'fixture',
|
|
64
|
+
latencyMs: 1,
|
|
65
|
+
tokens: { prompt: 2, completion: 1, total: 3 },
|
|
66
|
+
message: {
|
|
67
|
+
role: 'assistant',
|
|
68
|
+
content: continued ? 'Done.' : '',
|
|
69
|
+
...(continued
|
|
70
|
+
? {}
|
|
71
|
+
: {
|
|
72
|
+
tool_calls: ['one', 'two'].map((name) => ({
|
|
73
|
+
id: name,
|
|
74
|
+
type: 'function' as const,
|
|
75
|
+
function: {
|
|
76
|
+
name: 'write_file',
|
|
77
|
+
arguments: JSON.stringify({ path: `${name}.txt`, content: 'PRIVATE-FILE' }),
|
|
78
|
+
},
|
|
79
|
+
})),
|
|
80
|
+
}),
|
|
81
|
+
},
|
|
82
|
+
};
|
|
83
|
+
},
|
|
84
|
+
};
|
|
85
|
+
const kit = new ArtemisKit();
|
|
86
|
+
const paused = await kit.runWorkflow({ workflow, target, checkpoint, pauseAfterActions: 2 });
|
|
87
|
+
expect(paused.record.reason).toBe('checkpoint_paused');
|
|
88
|
+
expect(paused.record.budgets.actions).toBe(2);
|
|
89
|
+
expect(calls).toBe(1);
|
|
90
|
+
expect(readWorkflowRecord(paused.record)).toEqual(paused.record);
|
|
91
|
+
const bad = await kit.runWorkflow({
|
|
92
|
+
workflow,
|
|
93
|
+
target,
|
|
94
|
+
checkpoint: { ...checkpoint, mode: 'resume', configurationId: 'changed' },
|
|
95
|
+
});
|
|
96
|
+
expect(bad.record.execution).not.toBe('completed');
|
|
97
|
+
expect(capabilities).toBe(1);
|
|
98
|
+
expect(calls).toBe(1);
|
|
99
|
+
const session = await kit.createWorkflowSession({
|
|
100
|
+
workflow,
|
|
101
|
+
target,
|
|
102
|
+
checkpoint: { ...checkpoint, mode: 'resume' },
|
|
103
|
+
});
|
|
104
|
+
expect(typeof session.pause).toBe('function');
|
|
105
|
+
const resumed = await session.run();
|
|
106
|
+
expect(resumed.record.taskVerification).toBe('passed');
|
|
107
|
+
expect(resumed.record.budgets).toMatchObject({ actions: 4, toolCalls: 2, modelRequests: 2 });
|
|
108
|
+
expect(resumed.record.usage.reported.total).toBe(6);
|
|
109
|
+
expect(calls).toBe(2);
|
|
110
|
+
expect(resumed.record.recovery?.runId).toBe(paused.record.recovery?.runId);
|
|
111
|
+
expect(resumed.record.recovery?.attempts).toBe(2);
|
|
112
|
+
expect(JSON.stringify(resumed.record)).not.toContain('PRIVATE-');
|
|
113
|
+
expect(JSON.stringify(resumed.record)).not.toContain(checkpoint.directory);
|
|
114
|
+
expect(readWorkflowRecord(resumed.record)).toEqual(resumed.record);
|
|
115
|
+
});
|
|
116
|
+
});
|
|
@@ -12,7 +12,8 @@ import {
|
|
|
12
12
|
} from '@artemiskit/core';
|
|
13
13
|
import { OpenAIAdapter } from '../../../adapters/openai/src/client';
|
|
14
14
|
import { ArtemisKit } from '../artemiskit';
|
|
15
|
-
import
|
|
15
|
+
import { readWorkflowRecord } from '../index';
|
|
16
|
+
import type { WorkflowJudgeOptions, WorkflowRunOptions } from '../types-only';
|
|
16
17
|
|
|
17
18
|
const servers: ReturnType<typeof Bun.serve>[] = [];
|
|
18
19
|
afterEach(() => {
|
|
@@ -63,6 +64,152 @@ function target(): AgentTarget {
|
|
|
63
64
|
}
|
|
64
65
|
|
|
65
66
|
describe('SDK native workflow session', () => {
|
|
67
|
+
test('forwards explicit independent semantic judging with separate coverage, usage and record reading', async () => {
|
|
68
|
+
for (const verdict of ['pass', 'fail', 'malformed', 'unconfigured']) {
|
|
69
|
+
const scenario = workflow();
|
|
70
|
+
scenario.outcomes.deterministic = [{ type: 'file', path: 'result.txt', exists: false }];
|
|
71
|
+
scenario.outcomes.semantic = [
|
|
72
|
+
{ type: 'llm_judge', mode: 'strict_assurance', rubric: 'PRIVATE-RUBRIC' },
|
|
73
|
+
];
|
|
74
|
+
let calls = 0;
|
|
75
|
+
let closed = 0;
|
|
76
|
+
const semanticJudge: WorkflowJudgeOptions = {
|
|
77
|
+
provider: 'independent-judge',
|
|
78
|
+
model: 'judge-model',
|
|
79
|
+
limits: { maxRequests: 1, maxTokens: 100, maxOutputTokens: 10, timeoutMs: 1000 },
|
|
80
|
+
client: {
|
|
81
|
+
provider: 'independent-judge',
|
|
82
|
+
capabilities: async () => ({
|
|
83
|
+
streaming: false,
|
|
84
|
+
functionCalling: false,
|
|
85
|
+
toolUse: false,
|
|
86
|
+
maxContext: 10000,
|
|
87
|
+
}),
|
|
88
|
+
generate: async (request) => {
|
|
89
|
+
calls++;
|
|
90
|
+
expect(request.model).toBe('judge-model');
|
|
91
|
+
expect(request.maxRetries).toBe(0);
|
|
92
|
+
return {
|
|
93
|
+
id: 'judge',
|
|
94
|
+
model: 'observed-judge',
|
|
95
|
+
text: verdict === 'malformed' ? 'PRIVATE-BAD-RESPONSE' : JSON.stringify({ verdict }),
|
|
96
|
+
tokens: { prompt: 5, completion: 2, total: 7 },
|
|
97
|
+
latencyMs: 1,
|
|
98
|
+
};
|
|
99
|
+
},
|
|
100
|
+
close: async () => {
|
|
101
|
+
closed++;
|
|
102
|
+
},
|
|
103
|
+
},
|
|
104
|
+
};
|
|
105
|
+
const result = await new ArtemisKit().runWorkflow({
|
|
106
|
+
workflow: scenario,
|
|
107
|
+
target: target(),
|
|
108
|
+
...(verdict === 'unconfigured' ? {} : { semanticJudge }),
|
|
109
|
+
});
|
|
110
|
+
expect(result.record.execution).toBe('completed');
|
|
111
|
+
expect(result.record.taskVerification).toBe(
|
|
112
|
+
verdict === 'pass'
|
|
113
|
+
? 'passed'
|
|
114
|
+
: verdict === 'fail'
|
|
115
|
+
? 'failed'
|
|
116
|
+
: verdict === 'malformed'
|
|
117
|
+
? 'invalid'
|
|
118
|
+
: 'unavailable'
|
|
119
|
+
);
|
|
120
|
+
expect(result.record.usage.reported.total).toBe(3);
|
|
121
|
+
expect(result.record.outcomes.deterministic.counts.valid).toBe(1);
|
|
122
|
+
expect(result.record.outcomes.semantic.usage.reported.total).toBe(
|
|
123
|
+
verdict === 'unconfigured' ? 0 : 7
|
|
124
|
+
);
|
|
125
|
+
expect(calls).toBe(verdict === 'unconfigured' ? 0 : 1);
|
|
126
|
+
expect(closed).toBe(0);
|
|
127
|
+
expect(readWorkflowRecord(JSON.parse(JSON.stringify(result.record)))).toEqual(result.record);
|
|
128
|
+
expect(JSON.stringify(result.record)).not.toContain('PRIVATE-');
|
|
129
|
+
}
|
|
130
|
+
});
|
|
131
|
+
test('malformed explicit judge bounds stop before any target model call', async () => {
|
|
132
|
+
const scenario = workflow();
|
|
133
|
+
let turns = 0;
|
|
134
|
+
const custom = target();
|
|
135
|
+
custom.turn = async () => {
|
|
136
|
+
turns++;
|
|
137
|
+
throw new Error('must not call');
|
|
138
|
+
};
|
|
139
|
+
const result = await new ArtemisKit().runWorkflow({
|
|
140
|
+
workflow: scenario,
|
|
141
|
+
target: custom,
|
|
142
|
+
semanticJudge: {
|
|
143
|
+
provider: 'judge',
|
|
144
|
+
model: 'judge',
|
|
145
|
+
limits: { maxRequests: 0, maxTokens: 100, maxOutputTokens: 10, timeoutMs: 1000 },
|
|
146
|
+
client: {
|
|
147
|
+
provider: 'judge',
|
|
148
|
+
capabilities: async () => {
|
|
149
|
+
throw new Error('must not call');
|
|
150
|
+
},
|
|
151
|
+
generate: async () => {
|
|
152
|
+
throw new Error('must not call');
|
|
153
|
+
},
|
|
154
|
+
},
|
|
155
|
+
},
|
|
156
|
+
});
|
|
157
|
+
expect(turns).toBe(0);
|
|
158
|
+
expect(result.record.execution).toBe('invalid');
|
|
159
|
+
});
|
|
160
|
+
test('deterministic failure gates judge calls and session cancellation reaches the independent evaluator', async () => {
|
|
161
|
+
let started!: () => void;
|
|
162
|
+
const pending = new Promise<void>((resolve) => {
|
|
163
|
+
started = resolve;
|
|
164
|
+
});
|
|
165
|
+
let calls = 0;
|
|
166
|
+
const semanticJudge: WorkflowJudgeOptions = {
|
|
167
|
+
provider: 'judge',
|
|
168
|
+
model: 'judge',
|
|
169
|
+
limits: { maxRequests: 1, maxTokens: 100, maxOutputTokens: 10, timeoutMs: 1000 },
|
|
170
|
+
client: {
|
|
171
|
+
provider: 'judge',
|
|
172
|
+
capabilities: async () => ({
|
|
173
|
+
streaming: false,
|
|
174
|
+
functionCalling: false,
|
|
175
|
+
toolUse: false,
|
|
176
|
+
maxContext: 10000,
|
|
177
|
+
}),
|
|
178
|
+
generate: async (request) => {
|
|
179
|
+
calls++;
|
|
180
|
+
started();
|
|
181
|
+
await new Promise<void>((resolve) =>
|
|
182
|
+
request.signal?.addEventListener('abort', () => resolve(), { once: true })
|
|
183
|
+
);
|
|
184
|
+
throw new Error('PRIVATE-ABORT');
|
|
185
|
+
},
|
|
186
|
+
},
|
|
187
|
+
};
|
|
188
|
+
const scenario = workflow();
|
|
189
|
+
scenario.outcomes.semantic = [
|
|
190
|
+
{ type: 'llm_judge', mode: 'strict_assurance', rubric: 'PRIVATE-RUBRIC' },
|
|
191
|
+
];
|
|
192
|
+
const kit = new ArtemisKit();
|
|
193
|
+
const failed = await kit.runWorkflow({ workflow: scenario, target: target(), semanticJudge });
|
|
194
|
+
expect(failed.record.taskVerification).toBe('failed');
|
|
195
|
+
expect(calls).toBe(0);
|
|
196
|
+
scenario.outcomes.deterministic = [{ type: 'file', path: 'result.txt', exists: false }];
|
|
197
|
+
const session = await kit.createWorkflowSession({
|
|
198
|
+
workflow: scenario,
|
|
199
|
+
target: target(),
|
|
200
|
+
semanticJudge,
|
|
201
|
+
});
|
|
202
|
+
const finished = session.run();
|
|
203
|
+
await pending;
|
|
204
|
+
session.cancel();
|
|
205
|
+
const result = await finished;
|
|
206
|
+
expect(result.record.execution).toBe('completed');
|
|
207
|
+
expect(result.record.outcomes.cancelled).toBe(true);
|
|
208
|
+
expect(result.record.outcomes.reason).toBe('cancelled');
|
|
209
|
+
expect(result.record.outcomes.semantic.counts.valid).toBe(0);
|
|
210
|
+
expect(readWorkflowRecord(result.record)).toEqual(result.record);
|
|
211
|
+
});
|
|
212
|
+
|
|
66
213
|
test('configured adapter uses workflow identity and zero retries with real loopback multi-turn execution', async () => {
|
|
67
214
|
const bodies: Record<string, unknown>[] = [];
|
|
68
215
|
const server = Bun.serve({
|
|
@@ -154,7 +301,7 @@ describe('SDK native workflow session', () => {
|
|
|
154
301
|
const { result, events } = JSON.parse(stdout);
|
|
155
302
|
expect(bodies[0].model).toBe('workflow-authoritative');
|
|
156
303
|
expect(result.record.execution).toBe('completed');
|
|
157
|
-
expect(result.record.taskVerification).toBe('
|
|
304
|
+
expect(result.record.taskVerification).toBe('passed');
|
|
158
305
|
expect(result.record.budgets).toMatchObject({ modelRequests: 2, toolCalls: 1 });
|
|
159
306
|
expect(result.state?.files).toEqual({ 'result.txt': 'PRIVATE-STATE' });
|
|
160
307
|
expect(JSON.stringify(result.record)).not.toContain('PRIVATE-');
|
package/src/index.ts
CHANGED
|
@@ -10,6 +10,8 @@ export { ArtemisKit } from './artemiskit';
|
|
|
10
10
|
export {
|
|
11
11
|
AgentWorkflowSchema,
|
|
12
12
|
WorkflowPolicySchema,
|
|
13
|
+
WorkflowFaultsSchema,
|
|
14
|
+
WorkflowRetrySchema,
|
|
13
15
|
WORKFLOW_TOOL_IDS,
|
|
14
16
|
validateAgentWorkflow,
|
|
15
17
|
parseAgentWorkflow,
|
|
@@ -20,6 +22,12 @@ export {
|
|
|
20
22
|
createModelClientTarget,
|
|
21
23
|
createAgentWorkflowSession,
|
|
22
24
|
runAgentWorkflow,
|
|
25
|
+
evaluateWorkflowDeterministicOutcomes,
|
|
26
|
+
evaluateWorkflowSemantics,
|
|
27
|
+
isValidWorkflowJudgeOptions,
|
|
28
|
+
decideWorkflowOutcome,
|
|
29
|
+
validWorkflowAssertionCounts,
|
|
30
|
+
readWorkflowRecord,
|
|
23
31
|
createSimulatedWorkflowEnvironment,
|
|
24
32
|
createDockerWorkflowEnvironment,
|
|
25
33
|
createDockerWorkflowEnvironmentFactory,
|
|
@@ -44,8 +52,32 @@ export type {
|
|
|
44
52
|
AgentWorkflowReason,
|
|
45
53
|
AgentWorkflowEvent,
|
|
46
54
|
AgentWorkflowRecord,
|
|
55
|
+
AgentWorkflowExecutionRecord,
|
|
56
|
+
AgentWorkflowEvidence,
|
|
57
|
+
WorkflowDeterministicStatus,
|
|
58
|
+
WorkflowDeterministicReason,
|
|
59
|
+
WorkflowDeterministicAssertion,
|
|
60
|
+
WorkflowDeterministicSummary,
|
|
61
|
+
WorkflowJudgeOptions,
|
|
62
|
+
WorkflowSemanticOptions,
|
|
63
|
+
WorkflowSemanticReason,
|
|
64
|
+
WorkflowSemanticAssertion,
|
|
65
|
+
WorkflowSemanticIdentity,
|
|
66
|
+
WorkflowSemanticEvaluation,
|
|
67
|
+
WorkflowOutcomeStatus,
|
|
68
|
+
WorkflowAssertionCounts,
|
|
69
|
+
WorkflowOutcomeDecision,
|
|
70
|
+
WorkflowOutcomeAssessment,
|
|
71
|
+
SavedWorkflowRecord,
|
|
72
|
+
HistoricalWorkflowRecord,
|
|
47
73
|
AgentWorkflowResult,
|
|
48
74
|
AgentWorkflowSessionOptions,
|
|
75
|
+
WorkflowCheckpointOptions,
|
|
76
|
+
WorkflowRecoveryEvidence,
|
|
77
|
+
WorkflowFault,
|
|
78
|
+
WorkflowFaults,
|
|
79
|
+
WorkflowFaultKind,
|
|
80
|
+
WorkflowRetry,
|
|
49
81
|
AgentWorkflowSession,
|
|
50
82
|
WorkflowState,
|
|
51
83
|
WorkflowEnvironment,
|
package/src/types-only.ts
CHANGED
|
@@ -294,8 +294,32 @@ export type {
|
|
|
294
294
|
AgentWorkflowReason,
|
|
295
295
|
AgentWorkflowEvent,
|
|
296
296
|
AgentWorkflowRecord,
|
|
297
|
+
AgentWorkflowExecutionRecord,
|
|
298
|
+
AgentWorkflowEvidence,
|
|
299
|
+
WorkflowDeterministicStatus,
|
|
300
|
+
WorkflowDeterministicReason,
|
|
301
|
+
WorkflowDeterministicAssertion,
|
|
302
|
+
WorkflowDeterministicSummary,
|
|
303
|
+
WorkflowJudgeOptions,
|
|
304
|
+
WorkflowSemanticOptions,
|
|
305
|
+
WorkflowSemanticReason,
|
|
306
|
+
WorkflowSemanticAssertion,
|
|
307
|
+
WorkflowSemanticIdentity,
|
|
308
|
+
WorkflowSemanticEvaluation,
|
|
309
|
+
WorkflowOutcomeStatus,
|
|
310
|
+
WorkflowAssertionCounts,
|
|
311
|
+
WorkflowOutcomeDecision,
|
|
312
|
+
WorkflowOutcomeAssessment,
|
|
313
|
+
SavedWorkflowRecord,
|
|
314
|
+
HistoricalWorkflowRecord,
|
|
297
315
|
AgentWorkflowResult,
|
|
298
316
|
AgentWorkflowSessionOptions,
|
|
317
|
+
WorkflowCheckpointOptions,
|
|
318
|
+
WorkflowRecoveryEvidence,
|
|
319
|
+
WorkflowFault,
|
|
320
|
+
WorkflowFaults,
|
|
321
|
+
WorkflowFaultKind,
|
|
322
|
+
WorkflowRetry,
|
|
299
323
|
AgentWorkflowSession,
|
|
300
324
|
WorkflowState,
|
|
301
325
|
WorkflowEnvironment,
|
package/src/workflow.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { dirname, resolve } from 'node:path';
|
|
2
2
|
import {
|
|
3
3
|
type AdapterConfig,
|
|
4
|
+
type AgentTarget,
|
|
4
5
|
type AgentWorkflowSession,
|
|
5
6
|
ArtemisError,
|
|
6
7
|
createAdapter,
|
|
@@ -20,22 +21,36 @@ export async function prepareWorkflowSession(
|
|
|
20
21
|
const workflow = file ? await loadAgentWorkflow(file) : validateAgentWorkflow(options.workflow);
|
|
21
22
|
let target = options.target;
|
|
22
23
|
if (!target) {
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
24
|
+
if (options.client) target = createModelClientTarget(options.client);
|
|
25
|
+
else {
|
|
26
|
+
let initialized: Promise<AgentTarget> | undefined;
|
|
27
|
+
const getTarget = () => {
|
|
28
|
+
initialized ??= createAdapter({
|
|
27
29
|
...config.providerConfig,
|
|
28
30
|
...options.providerConfig,
|
|
29
31
|
provider: workflow.target.provider,
|
|
30
32
|
defaultModel: workflow.target.model,
|
|
31
33
|
maxRetries: 0,
|
|
32
|
-
} as AdapterConfig)
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
34
|
+
} as AdapterConfig)
|
|
35
|
+
.then(createModelClientTarget)
|
|
36
|
+
.catch(() => {
|
|
37
|
+
throw new ArtemisError('Unable to initialize workflow target', 'PROVIDER_UNAVAILABLE');
|
|
38
|
+
});
|
|
39
|
+
return initialized;
|
|
40
|
+
};
|
|
41
|
+
target = options.checkpoint
|
|
42
|
+
? {
|
|
43
|
+
provider: workflow.target.provider,
|
|
44
|
+
capabilities: async (limits, signal) =>
|
|
45
|
+
(await getTarget()).capabilities(limits, signal),
|
|
46
|
+
turn: async (request, signal) => (await getTarget()).turn(request, signal),
|
|
47
|
+
drain: async (limits) =>
|
|
48
|
+
initialized
|
|
49
|
+
? ((await initialized).drain?.(limits) ?? { pendingOperations: 0 })
|
|
50
|
+
: { pendingOperations: 0 },
|
|
51
|
+
}
|
|
52
|
+
: await getTarget();
|
|
37
53
|
}
|
|
38
|
-
target = createModelClientTarget(client);
|
|
39
54
|
}
|
|
40
55
|
return createAgentWorkflowSession({
|
|
41
56
|
workflow,
|
|
@@ -46,6 +61,9 @@ export async function prepareWorkflowSession(
|
|
|
46
61
|
preflightOnly: options.preflightOnly,
|
|
47
62
|
signal: options.signal,
|
|
48
63
|
cleanupTimeoutMs: options.cleanupTimeoutMs,
|
|
64
|
+
semanticJudge: options.semanticJudge,
|
|
49
65
|
onEvent: options.onEvent,
|
|
66
|
+
checkpoint: options.checkpoint,
|
|
67
|
+
pauseAfterActions: options.pauseAfterActions,
|
|
50
68
|
});
|
|
51
69
|
}
|