@artemiskit/sdk 0.6.0 → 0.6.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,23 @@
1
1
  # @artemiskit/sdk
2
2
 
3
+ ## 0.6.2
4
+
5
+ ### Patch Changes
6
+
7
+ - Add declared workflow faults and bounded retries, private local checkpoints with explicit pause/resume, and version 3 recovery evidence. Preserve workflow identity, tool authority, original deadlines and cumulative budgets across restart; refuse ambiguous pending operations. Expose the same recovery controls in CLI and SDK while retaining version 1/2 record compatibility.
8
+ - Updated dependencies
9
+ - @artemiskit/core@0.6.3
10
+ - @artemiskit/redteam@0.3.11
11
+
12
+ ## 0.6.1
13
+
14
+ ### Patch Changes
15
+
16
+ - Complete the 0.6.2 independent workflow outcome milestone: verify final artifacts, bounded JSON schemas, state, tool traces and policy before optional explicitly configured semantic judging. Keep target and judge identity, usage and limits separate; expose task validity and coverage, strict V1/V2 saved-record reading, SDK composition and CLI outcome exits without changing historical prompt-response reports.
17
+ - Updated dependencies
18
+ - @artemiskit/core@0.6.2
19
+ - @artemiskit/redteam@0.3.10
20
+
3
21
  ## 0.6.0
4
22
 
5
23
  ### Minor Changes
package/README.md CHANGED
@@ -57,16 +57,27 @@ const session = await kit.createWorkflowSession({
57
57
  preflight: true,
58
58
  });
59
59
  const result = await session.run();
60
- console.log(result.record); // bounded metadata; taskVerification is unavailable in 0.6.1
60
+ console.log(result.record); // bounded metadata with separate execution and task outcomes
61
61
  // session.cancel() stops new work and initiates bounded cleanup.
62
62
  // result.state and result.transcript contain sensitive working data.
63
63
  ```
64
64
 
65
- `kit.runWorkflow(options)` creates and runs a fresh session in one call. Supply a `client` or
65
+ `kit.runWorkflow(options)` creates and runs a session in one call; fresh execution is the default. Supply a `client` or
66
66
  `target` to use an existing trusted adapter or target. The same native controls apply through the
67
67
  CLI and SDK: declared tools and permissions, cumulative budgets, explicit preflight, cancellation,
68
68
  and cleanup evidence. Workflows use fresh simulated state or a disposable Docker filesystem.
69
- Independent task scoring follows in 0.6.2; normal execution does not establish a task pass.
69
+ The 0.6.2 milestone independently verifies declared artifacts, state, schemas, traces and policy.
70
+ Use explicit `semanticJudge` configuration for declared qualitative criteria. Target and judge usage
71
+ remain separate; `readWorkflowRecord` validates V1/V2/V3 saved metadata without making model calls.
72
+ See [workflow outcomes](https://github.com/code-sensei/artemiskit/blob/main/docs/workflow-outcomes.md)
73
+ for usage, denominators, compatibility and publication status.
74
+
75
+ For explicit private recovery, pass `checkpoint: { directory, mode: 'create', configurationId }`
76
+ and optionally `pauseAfterActions`. A later process uses the same authorized configuration and
77
+ `mode: 'resume'`. `session.pause()` requests a safe boundary; `session.cancel()` still cancels.
78
+ Checkpoints retain sensitive conversation and state with cumulative budgets and the original
79
+ deadline. A paused result is not task success. See the [0.6.3 recovery guide](https://github.com/code-sensei/artemiskit/blob/main/docs/workflow-recovery.md)
80
+ for complete examples, fault schedules, refusal conditions and qualification status.
70
81
 
71
82
  The root and `/types` entry points expose workflow result/session/event types. Low-level
72
83
  `createAgentWorkflowSession` and `runAgentWorkflow` are available for explicit target composition.
package/dist/index.d.ts CHANGED
@@ -3,8 +3,8 @@
3
3
  * Programmatic SDK for ArtemisKit LLM evaluation toolkit
4
4
  */
5
5
  export { ArtemisKit } from './artemiskit.js';
6
- export { AgentWorkflowSchema, WorkflowPolicySchema, WORKFLOW_TOOL_IDS, validateAgentWorkflow, parseAgentWorkflow, loadAgentWorkflow, listWorkflowTools, getWorkflowTool, executeSimulatedTool, createModelClientTarget, createAgentWorkflowSession, runAgentWorkflow, createSimulatedWorkflowEnvironment, createDockerWorkflowEnvironment, createDockerWorkflowEnvironmentFactory, WORKFLOW_SANDBOX_IMAGE, WorkflowEnvironmentInitializationError, } from '@artemiskit/core';
7
- export type { AgentWorkflow, WorkflowPolicy, WorkflowJson, WorkflowToolId, WorkflowResource, WorkflowToolDescriptor, SimulatedToolEvidence, SimulatedToolResult, AgentTarget, AgentTurnRequest, AgentTurnResult, AgentTargetCapabilities, AgentTargetFailure, AgentWorkflowExecution, AgentWorkflowReason, AgentWorkflowEvent, AgentWorkflowRecord, AgentWorkflowResult, AgentWorkflowSessionOptions, AgentWorkflowSession, WorkflowState, WorkflowEnvironment, WorkflowEnvironmentFactory, WorkflowEnvironmentCleanup, DockerWorkflowEnvironmentOptions, } from '@artemiskit/core';
6
+ export { AgentWorkflowSchema, WorkflowPolicySchema, WorkflowFaultsSchema, WorkflowRetrySchema, WORKFLOW_TOOL_IDS, validateAgentWorkflow, parseAgentWorkflow, loadAgentWorkflow, listWorkflowTools, getWorkflowTool, executeSimulatedTool, createModelClientTarget, createAgentWorkflowSession, runAgentWorkflow, evaluateWorkflowDeterministicOutcomes, evaluateWorkflowSemantics, isValidWorkflowJudgeOptions, decideWorkflowOutcome, validWorkflowAssertionCounts, readWorkflowRecord, createSimulatedWorkflowEnvironment, createDockerWorkflowEnvironment, createDockerWorkflowEnvironmentFactory, WORKFLOW_SANDBOX_IMAGE, WorkflowEnvironmentInitializationError, } from '@artemiskit/core';
7
+ export type { AgentWorkflow, WorkflowPolicy, WorkflowJson, WorkflowToolId, WorkflowResource, WorkflowToolDescriptor, SimulatedToolEvidence, SimulatedToolResult, AgentTarget, AgentTurnRequest, AgentTurnResult, AgentTargetCapabilities, AgentTargetFailure, AgentWorkflowExecution, AgentWorkflowReason, AgentWorkflowEvent, AgentWorkflowRecord, AgentWorkflowExecutionRecord, AgentWorkflowEvidence, WorkflowDeterministicStatus, WorkflowDeterministicReason, WorkflowDeterministicAssertion, WorkflowDeterministicSummary, WorkflowJudgeOptions, WorkflowSemanticOptions, WorkflowSemanticReason, WorkflowSemanticAssertion, WorkflowSemanticIdentity, WorkflowSemanticEvaluation, WorkflowOutcomeStatus, WorkflowAssertionCounts, WorkflowOutcomeDecision, WorkflowOutcomeAssessment, SavedWorkflowRecord, HistoricalWorkflowRecord, AgentWorkflowResult, AgentWorkflowSessionOptions, WorkflowCheckpointOptions, WorkflowRecoveryEvidence, WorkflowFault, WorkflowFaults, WorkflowFaultKind, WorkflowRetry, AgentWorkflowSession, WorkflowState, WorkflowEnvironment, WorkflowEnvironmentFactory, WorkflowEnvironmentCleanup, DockerWorkflowEnvironmentOptions, } from '@artemiskit/core';
8
8
  export type { ArtemisKitConfig, WorkflowRunOptions, RunOptions, RedTeamOptions, StressOptions, RunResult, RedTeamResult, StressResult, CaseStartEvent, CaseCompleteEvent, ProgressEvent, RedTeamMutationStartEvent, RedTeamMutationCompleteEvent, StressRequestCompleteEvent, CaseStartHandler, CaseCompleteHandler, ProgressHandler, RedTeamMutationStartHandler, RedTeamMutationCompleteHandler, StressRequestCompleteHandler, ArtemisKitEvents, ArtemisKitEventName, ValidateOptions, ValidationResult, ValidationError, ValidationWarning, ScenarioValidation, CompareOptions, CompareResult, RunSummary, ComparisonDetails, } from './types.js';
9
9
  export type { Scenario, TestCase, Expected, Provider, ProviderConfig, ChatMessageType, Variables, } from '@artemiskit/core';
10
10
  export type { CaseResult, CaseRedactionInfo, RunManifest, RunConfig, RunMetrics, RedTeamManifest, RedTeamConfig, RedTeamMetrics, RedTeamCaseResult, RedTeamStatus, RedTeamSeverity, StressManifest, StressConfig, StressMetrics, StressRequestResult, AnyManifest, ResolvedConfig, ConfigSource, ManifestRedactionInfo, CostEstimateInfo, GitInfo, ProvenanceInfo, } from '@artemiskit/core';
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAG1C,OAAO,EACL,mBAAmB,EACnB,oBAAoB,EACpB,iBAAiB,EACjB,qBAAqB,EACrB,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,eAAe,EACf,oBAAoB,EACpB,uBAAuB,EACvB,0BAA0B,EAC1B,gBAAgB,EAChB,kCAAkC,EAClC,+BAA+B,EAC/B,sCAAsC,EACtC,sBAAsB,EACtB,sCAAsC,GACvC,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,aAAa,EACb,cAAc,EACd,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,sBAAsB,EACtB,qBAAqB,EACrB,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,eAAe,EACf,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,2BAA2B,EAC3B,oBAAoB,EACpB,aAAa,EACb,mBAAmB,EACnB,0BAA0B,EAC1B,0BAA0B,EAC1B,gCAAgC,GACjC,MAAM,kBAAkB,CAAC;AAG1B,YAAY,EAEV,gBAAgB,EAChB,kBAAkB,EAClB,UAAU,EACV,cAAc,EACd,aAAa,EAEb,SAAS,EACT,aAAa,EACb,YAAY,EAEZ,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,yBAAyB,EACzB,4BAA4B,EAC5B,0BAA0B,EAE1B,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAE5B,gBAAgB,EAChB,mBAAmB,EAEnB,eAAe,EACf,gBAAgB,EAChB,eAAe,EACf,iBAAiB,EACjB,kBAAkB,EAElB,cAAc,EACd,aAAa,EACb,UAAU,EACV,iBAAiB,GAClB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,cAAc,EACd,eAAe,EACf,SAAS,GACV,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,UAAU,EACV,iBAAiB,EAEjB,WAAW,EACX,SAAS,EACT,UAAU,EAEV,eAAe,EACf,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,eAAe,EAEf,cAAc,EACd,YAAY,EACZ,aAAa,EACb,mBAAmB,EAEnB,WAAW,EACX,cAAc,EACd,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,OAAO,EACP,cAAc,GACf,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,WAAW,EACX,iBAAiB,EAEjB,eAAe,EACf,cAAc,EACd,UAAU,EAEV,WAAW,EAEX,kBAAkB,EAClB,cAAc,EACd,QAAQ,EAER,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,YAAY,GACb,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMrF,YAAY,EACV,cAAc,EACd,aAAa,EACb,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EAEtB,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EACtB,eAAe,EACf,mBAAmB,EACnB,cAAc,EACd,uBAAuB,GACxB,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMxD,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAMtF,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAM7E,OAAO,EAAE,kBAAkB,EAAE,KAAK,kBAAkB,EAAE,KAAK,aAAa,EAAE,MAAM,YAAY,CAAC;AAM7F,OAAO,EAEL,WAAW,EACX,eAAe,EACf,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,oBAAoB,EAEpB,aAAa,EACb,MAAM,EAEN,cAAc,EACd,cAAc,EACd,aAAa,EACb,oBAAoB,GACrB,MAAM,SAAS,CAAC;AAEjB,YAAY,EAEV,YAAY,EACZ,eAAe,EAEf,SAAS,EACT,eAAe,EACf,mBAAmB,EACnB,oBAAoB,EACpB,eAAe,EAEf,WAAW,EACX,eAAe,EACf,aAAa,EACb,cAAc,EACd,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,OAAO,EAEL,aAAa,EAEb,eAAe,EAEf,aAAa,EAEb,YAAY,GACb,MAAM,aAAa,CAAC;AAErB,YAAY,EACV,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,aAAa,CAAC;AAMrB,OAAO,EAEL,eAAe,EACf,eAAe,EAEf,QAAQ,EACR,QAAQ,EAER,YAAY,EACZ,SAAS,EACT,SAAS,EACT,QAAQ,EACR,UAAU,EAEV,KAAK,EACL,QAAQ,EACR,WAAW,EACX,KAAK,EACL,KAAK,EACL,UAAU,EACV,QAAQ,EACR,UAAU,EACV,MAAM,EACN,KAAK,EACL,KAAK,GACN,MAAM,YAAY,CAAC;AAMpB,OAAO,EAEL,QAAQ,EACR,cAAc,EAEd,qBAAqB,EAErB,iBAAiB,EACjB,uBAAuB,EAEvB,mBAAmB,EACnB,oBAAoB,EACpB,iBAAiB,EAEjB,eAAe,EACf,4BAA4B,EAE5B,gBAAgB,EAChB,sBAAsB,EAEtB,eAAe,EACf,wBAAwB,EACxB,SAAS,EACT,kBAAkB,EAClB,aAAa,EACb,4BAA4B,EAC5B,gBAAgB,EAEhB,YAAY,EACZ,4BAA4B,EAE5B,UAAU,EACV,WAAW,EACX,cAAc,EACd,mBAAmB,EACnB,aAAa,EACb,sBAAsB,EACtB,eAAe,EACf,qBAAqB,EAErB,cAAc,EACd,gBAAgB,EAChB,WAAW,GACZ,MAAM,YAAY,CAAC;AAGpB,YAAY,EAEV,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,sBAAsB,EACtB,gBAAgB,EAChB,iBAAiB,EACjB,oBAAoB,EAEpB,YAAY,EACZ,iBAAiB,EACjB,eAAe,EACf,aAAa,EACb,SAAS,EACT,eAAe,EAEf,UAAU,EACV,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,eAAe,EACf,eAAe,EAEf,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EAEnB,oBAAoB,EACpB,cAAc,EAEd,YAAY,EACZ,OAAO,EACP,WAAW,EACX,kBAAkB,EAClB,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,eAAe,EAEf,eAAe,EACf,mBAAmB,EACnB,YAAY,EAEZ,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EAEpB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EAEpB,aAAa,EACb,0BAA0B,EAE1B,qBAAqB,EACrB,kBAAkB,EAClB,eAAe,EAEf,uBAAuB,EACvB,aAAa,EACb,kBAAkB,EAClB,eAAe,EACf,wBAAwB,EAExB,kBAAkB,EAClB,oBAAoB,EACpB,cAAc,EACd,cAAc,EACd,mBAAmB,EACnB,4BAA4B,EAC5B,yBAAyB,EACzB,kCAAkC,EAClC,2BAA2B,EAC3B,mBAAmB,EACnB,uBAAuB,EACvB,eAAe,EACf,eAAe,EACf,gBAAgB,EAChB,yBAAyB,EACzB,sBAAsB,GACvB,MAAM,YAAY,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAG1C,OAAO,EACL,mBAAmB,EACnB,oBAAoB,EACpB,oBAAoB,EACpB,mBAAmB,EACnB,iBAAiB,EACjB,qBAAqB,EACrB,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,eAAe,EACf,oBAAoB,EACpB,uBAAuB,EACvB,0BAA0B,EAC1B,gBAAgB,EAChB,qCAAqC,EACrC,yBAAyB,EACzB,2BAA2B,EAC3B,qBAAqB,EACrB,4BAA4B,EAC5B,kBAAkB,EAClB,kCAAkC,EAClC,+BAA+B,EAC/B,sCAAsC,EACtC,sBAAsB,EACtB,sCAAsC,GACvC,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,aAAa,EACb,cAAc,EACd,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,sBAAsB,EACtB,qBAAqB,EACrB,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,eAAe,EACf,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,kBAAkB,EAClB,mBAAmB,EACnB,4BAA4B,EAC5B,qBAAqB,EACrB,2BAA2B,EAC3B,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAC5B,oBAAoB,EACpB,uBAAuB,EACvB,sBAAsB,EACtB,yBAAyB,EACzB,wBAAwB,EACxB,0BAA0B,EAC1B,qBAAqB,EACrB,uBAAuB,EACvB,uBAAuB,EACvB,yBAAyB,EACzB,mBAAmB,EACnB,wBAAwB,EACxB,mBAAmB,EACnB,2BAA2B,EAC3B,yBAAyB,EACzB,wBAAwB,EACxB,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EACpB,aAAa,EACb,mBAAmB,EACnB,0BAA0B,EAC1B,0BAA0B,EAC1B,gCAAgC,GACjC,MAAM,kBAAkB,CAAC;AAG1B,YAAY,EAEV,gBAAgB,EAChB,kBAAkB,EAClB,UAAU,EACV,cAAc,EACd,aAAa,EAEb,SAAS,EACT,aAAa,EACb,YAAY,EAEZ,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,yBAAyB,EACzB,4BAA4B,EAC5B,0BAA0B,EAE1B,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAE5B,gBAAgB,EAChB,mBAAmB,EAEnB,eAAe,EACf,gBAAgB,EAChB,eAAe,EACf,iBAAiB,EACjB,kBAAkB,EAElB,cAAc,EACd,aAAa,EACb,UAAU,EACV,iBAAiB,GAClB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,cAAc,EACd,eAAe,EACf,SAAS,GACV,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,UAAU,EACV,iBAAiB,EAEjB,WAAW,EACX,SAAS,EACT,UAAU,EAEV,eAAe,EACf,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,eAAe,EAEf,cAAc,EACd,YAAY,EACZ,aAAa,EACb,mBAAmB,EAEnB,WAAW,EACX,cAAc,EACd,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,OAAO,EACP,cAAc,GACf,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,WAAW,EACX,iBAAiB,EAEjB,eAAe,EACf,cAAc,EACd,UAAU,EAEV,WAAW,EAEX,kBAAkB,EAClB,cAAc,EACd,QAAQ,EAER,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,YAAY,GACb,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMrF,YAAY,EACV,cAAc,EACd,aAAa,EACb,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EAEtB,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EACtB,eAAe,EACf,mBAAmB,EACnB,cAAc,EACd,uBAAuB,GACxB,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMxD,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAMtF,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAM7E,OAAO,EAAE,kBAAkB,EAAE,KAAK,kBAAkB,EAAE,KAAK,aAAa,EAAE,MAAM,YAAY,CAAC;AAM7F,OAAO,EAEL,WAAW,EACX,eAAe,EACf,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,oBAAoB,EAEpB,aAAa,EACb,MAAM,EAEN,cAAc,EACd,cAAc,EACd,aAAa,EACb,oBAAoB,GACrB,MAAM,SAAS,CAAC;AAEjB,YAAY,EAEV,YAAY,EACZ,eAAe,EAEf,SAAS,EACT,eAAe,EACf,mBAAmB,EACnB,oBAAoB,EACpB,eAAe,EAEf,WAAW,EACX,eAAe,EACf,aAAa,EACb,cAAc,EACd,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,OAAO,EAEL,aAAa,EAEb,eAAe,EAEf,aAAa,EAEb,YAAY,GACb,MAAM,aAAa,CAAC;AAErB,YAAY,EACV,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,aAAa,CAAC;AAMrB,OAAO,EAEL,eAAe,EACf,eAAe,EAEf,QAAQ,EACR,QAAQ,EAER,YAAY,EACZ,SAAS,EACT,SAAS,EACT,QAAQ,EACR,UAAU,EAEV,KAAK,EACL,QAAQ,EACR,WAAW,EACX,KAAK,EACL,KAAK,EACL,UAAU,EACV,QAAQ,EACR,UAAU,EACV,MAAM,EACN,KAAK,EACL,KAAK,GACN,MAAM,YAAY,CAAC;AAMpB,OAAO,EAEL,QAAQ,EACR,cAAc,EAEd,qBAAqB,EAErB,iBAAiB,EACjB,uBAAuB,EAEvB,mBAAmB,EACnB,oBAAoB,EACpB,iBAAiB,EAEjB,eAAe,EACf,4BAA4B,EAE5B,gBAAgB,EAChB,sBAAsB,EAEtB,eAAe,EACf,wBAAwB,EACxB,SAAS,EACT,kBAAkB,EAClB,aAAa,EACb,4BAA4B,EAC5B,gBAAgB,EAEhB,YAAY,EACZ,4BAA4B,EAE5B,UAAU,EACV,WAAW,EACX,cAAc,EACd,mBAAmB,EACnB,aAAa,EACb,sBAAsB,EACtB,eAAe,EACf,qBAAqB,EAErB,cAAc,EACd,gBAAgB,EAChB,WAAW,GACZ,MAAM,YAAY,CAAC;AAGpB,YAAY,EAEV,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,sBAAsB,EACtB,gBAAgB,EAChB,iBAAiB,EACjB,oBAAoB,EAEpB,YAAY,EACZ,iBAAiB,EACjB,eAAe,EACf,aAAa,EACb,SAAS,EACT,eAAe,EAEf,UAAU,EACV,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,eAAe,EACf,eAAe,EAEf,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EAEnB,oBAAoB,EACpB,cAAc,EAEd,YAAY,EACZ,OAAO,EACP,WAAW,EACX,kBAAkB,EAClB,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,eAAe,EAEf,eAAe,EACf,mBAAmB,EACnB,YAAY,EAEZ,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EAEpB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EAEpB,aAAa,EACb,0BAA0B,EAE1B,qBAAqB,EACrB,kBAAkB,EAClB,eAAe,EAEf,uBAAuB,EACvB,aAAa,EACb,kBAAkB,EAClB,eAAe,EACf,wBAAwB,EAExB,kBAAkB,EAClB,oBAAoB,EACpB,cAAc,EACd,cAAc,EACd,mBAAmB,EACnB,4BAA4B,EAC5B,yBAAyB,EACzB,kCAAkC,EAClC,2BAA2B,EAC3B,mBAAmB,EACnB,uBAAuB,EACvB,eAAe,EACf,eAAe,EACf,gBAAgB,EAChB,yBAAyB,EACzB,sBAAsB,GACvB,MAAM,YAAY,CAAC"}
package/dist/index.js CHANGED
@@ -7653,21 +7653,29 @@ async function prepareWorkflowSession(options, config) {
7653
7653
  const workflow = file ? await loadAgentWorkflow(file) : validateAgentWorkflow(options.workflow);
7654
7654
  let target = options.target;
7655
7655
  if (!target) {
7656
- let client = options.client;
7657
- if (!client) {
7658
- try {
7659
- client = await createAdapter({
7656
+ if (options.client)
7657
+ target = createModelClientTarget(options.client);
7658
+ else {
7659
+ let initialized;
7660
+ const getTarget = () => {
7661
+ initialized ??= createAdapter({
7660
7662
  ...config.providerConfig,
7661
7663
  ...options.providerConfig,
7662
7664
  provider: workflow.target.provider,
7663
7665
  defaultModel: workflow.target.model,
7664
7666
  maxRetries: 0
7667
+ }).then(createModelClientTarget).catch(() => {
7668
+ throw new ArtemisError("Unable to initialize workflow target", "PROVIDER_UNAVAILABLE");
7665
7669
  });
7666
- } catch {
7667
- throw new ArtemisError("Unable to initialize workflow target", "PROVIDER_UNAVAILABLE");
7668
- }
7670
+ return initialized;
7671
+ };
7672
+ target = options.checkpoint ? {
7673
+ provider: workflow.target.provider,
7674
+ capabilities: async (limits, signal) => (await getTarget()).capabilities(limits, signal),
7675
+ turn: async (request, signal) => (await getTarget()).turn(request, signal),
7676
+ drain: async (limits) => initialized ? (await initialized).drain?.(limits) ?? { pendingOperations: 0 } : { pendingOperations: 0 }
7677
+ } : await getTarget();
7669
7678
  }
7670
- target = createModelClientTarget(client);
7671
7679
  }
7672
7680
  return createAgentWorkflowSession({
7673
7681
  workflow,
@@ -7678,7 +7686,10 @@ async function prepareWorkflowSession(options, config) {
7678
7686
  preflightOnly: options.preflightOnly,
7679
7687
  signal: options.signal,
7680
7688
  cleanupTimeoutMs: options.cleanupTimeoutMs,
7681
- onEvent: options.onEvent
7689
+ semanticJudge: options.semanticJudge,
7690
+ onEvent: options.onEvent,
7691
+ checkpoint: options.checkpoint,
7692
+ pauseAfterActions: options.pauseAfterActions
7682
7693
  });
7683
7694
  }
7684
7695
 
@@ -8506,6 +8517,8 @@ function percentile(sortedArray, p) {
8506
8517
  import {
8507
8518
  AgentWorkflowSchema,
8508
8519
  WorkflowPolicySchema,
8520
+ WorkflowFaultsSchema,
8521
+ WorkflowRetrySchema,
8509
8522
  WORKFLOW_TOOL_IDS,
8510
8523
  validateAgentWorkflow as validateAgentWorkflow2,
8511
8524
  parseAgentWorkflow,
@@ -8516,6 +8529,12 @@ import {
8516
8529
  createModelClientTarget as createModelClientTarget2,
8517
8530
  createAgentWorkflowSession as createAgentWorkflowSession2,
8518
8531
  runAgentWorkflow,
8532
+ evaluateWorkflowDeterministicOutcomes,
8533
+ evaluateWorkflowSemantics,
8534
+ isValidWorkflowJudgeOptions,
8535
+ decideWorkflowOutcome,
8536
+ validWorkflowAssertionCounts,
8537
+ readWorkflowRecord,
8519
8538
  createSimulatedWorkflowEnvironment,
8520
8539
  createDockerWorkflowEnvironment,
8521
8540
  createDockerWorkflowEnvironmentFactory,
@@ -15257,12 +15276,14 @@ function createGuardian(config = {}) {
15257
15276
  export {
15258
15277
  validatePolicy,
15259
15278
  validateAgentWorkflow2 as validateAgentWorkflow,
15279
+ validWorkflowAssertionCounts,
15260
15280
  testCase,
15261
15281
  similarity,
15262
15282
  scenario,
15263
15283
  runAgentWorkflow,
15264
15284
  regexCase,
15265
15285
  regex,
15286
+ readWorkflowRecord,
15266
15287
  parsePolicy,
15267
15288
  parseAgentWorkflow,
15268
15289
  notContains,
@@ -15275,6 +15296,7 @@ export {
15275
15296
  listWorkflowTools,
15276
15297
  jsonSchema,
15277
15298
  jsonCase,
15299
+ isValidWorkflowJudgeOptions,
15278
15300
  isStressResult,
15279
15301
  isStressManifestType,
15280
15302
  isStressManifest,
@@ -15296,12 +15318,15 @@ export {
15296
15318
  executeSimulatedTool,
15297
15319
  exactCase,
15298
15320
  exact,
15321
+ evaluateWorkflowSemantics,
15322
+ evaluateWorkflowDeterministicOutcomes,
15299
15323
  detectPII,
15300
15324
  detectInjection,
15301
15325
  defineStorage,
15302
15326
  definePlugin,
15303
15327
  defineEvaluator,
15304
15328
  defineAdapter,
15329
+ decideWorkflowOutcome,
15305
15330
  createSimulatedWorkflowEnvironment,
15306
15331
  createSemanticValidator,
15307
15332
  createPIIGuardrail,
@@ -15326,7 +15351,9 @@ export {
15326
15351
  artemiskitMatchers,
15327
15352
  anyOf,
15328
15353
  allOf,
15354
+ WorkflowRetrySchema,
15329
15355
  WorkflowPolicySchema,
15356
+ WorkflowFaultsSchema,
15330
15357
  WorkflowEnvironmentInitializationError,
15331
15358
  WORKFLOW_TOOL_IDS,
15332
15359
  WORKFLOW_SANDBOX_IMAGE,
@@ -28,5 +28,5 @@ export type { ArtemisKitMatchers, MatcherResult } from './matchers/index.js';
28
28
  export type { ProviderName, ExpectationType, AnyResult, ExtractRunCases, ExtractRedTeamCases, ExtractStressResults, ExtractManifest, DeepPartial, PartialScenario, RequireFields, OptionalFields, StrictAdapterConfig, } from './utils/index.js';
29
29
  export type { AdapterContract, EvaluatorContract, StorageContract, AdapterFactory, EvaluatorFactory, StorageFactory, ArtemisKitPlugin, } from './contracts/index.js';
30
30
  export type { ScenarioBuilder, TestCaseBuilder } from './builders/index.js';
31
- export type { AgentWorkflow, WorkflowPolicy, WorkflowJson, WorkflowToolId, WorkflowResource, WorkflowToolDescriptor, SimulatedToolEvidence, SimulatedToolResult, AgentTarget, AgentTurnRequest, AgentTurnResult, AgentTargetCapabilities, AgentTargetFailure, AgentWorkflowExecution, AgentWorkflowReason, AgentWorkflowEvent, AgentWorkflowRecord, AgentWorkflowResult, AgentWorkflowSessionOptions, AgentWorkflowSession, WorkflowState, WorkflowEnvironment, WorkflowEnvironmentFactory, WorkflowEnvironmentCleanup, DockerWorkflowEnvironmentOptions, WorkflowEnvironmentInitializationError, } from '@artemiskit/core';
31
+ export type { AgentWorkflow, WorkflowPolicy, WorkflowJson, WorkflowToolId, WorkflowResource, WorkflowToolDescriptor, SimulatedToolEvidence, SimulatedToolResult, AgentTarget, AgentTurnRequest, AgentTurnResult, AgentTargetCapabilities, AgentTargetFailure, AgentWorkflowExecution, AgentWorkflowReason, AgentWorkflowEvent, AgentWorkflowRecord, AgentWorkflowExecutionRecord, AgentWorkflowEvidence, WorkflowDeterministicStatus, WorkflowDeterministicReason, WorkflowDeterministicAssertion, WorkflowDeterministicSummary, WorkflowJudgeOptions, WorkflowSemanticOptions, WorkflowSemanticReason, WorkflowSemanticAssertion, WorkflowSemanticIdentity, WorkflowSemanticEvaluation, WorkflowOutcomeStatus, WorkflowAssertionCounts, WorkflowOutcomeDecision, WorkflowOutcomeAssessment, SavedWorkflowRecord, HistoricalWorkflowRecord, AgentWorkflowResult, AgentWorkflowSessionOptions, WorkflowCheckpointOptions, WorkflowRecoveryEvidence, WorkflowFault, WorkflowFaults, WorkflowFaultKind, WorkflowRetry, AgentWorkflowSession, WorkflowState, WorkflowEnvironment, WorkflowEnvironmentFactory, WorkflowEnvironmentCleanup, DockerWorkflowEnvironmentOptions, WorkflowEnvironmentInitializationError, } from '@artemiskit/core';
32
32
  //# sourceMappingURL=types-only.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"types-only.d.ts","sourceRoot":"","sources":["../src/types-only.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAMH,YAAY,EAEV,gBAAgB,EAChB,kBAAkB,EAClB,UAAU,EACV,cAAc,EACd,aAAa,EAEb,SAAS,EACT,aAAa,EACb,YAAY,EAEZ,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,yBAAyB,EACzB,4BAA4B,EAC5B,0BAA0B,EAE1B,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAE5B,gBAAgB,EAChB,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,cAAc,EACd,eAAe,EACf,SAAS,GACV,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,UAAU,EACV,iBAAiB,EAEjB,WAAW,EACX,SAAS,EACT,UAAU,EAEV,eAAe,EACf,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,eAAe,EAEf,cAAc,EACd,YAAY,EACZ,aAAa,EACb,mBAAmB,EAEnB,WAAW,EACX,cAAc,EACd,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,OAAO,EACP,cAAc,GACf,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,WAAW,EACX,iBAAiB,EAEjB,eAAe,EACf,cAAc,EACd,UAAU,EAEV,WAAW,EAEX,kBAAkB,EAClB,cAAc,EACd,QAAQ,EAER,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,YAAY,GACb,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMrF,YAAY,EACV,cAAc,EACd,aAAa,EACb,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EAEtB,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EACtB,eAAe,EACf,mBAAmB,EACnB,cAAc,EACd,uBAAuB,GACxB,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMxD,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAM7E,YAAY,EAEV,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,sBAAsB,EACtB,gBAAgB,EAChB,iBAAiB,EAEjB,YAAY,EACZ,iBAAiB,EACjB,eAAe,EACf,aAAa,EACb,SAAS,EACT,eAAe,EAEf,UAAU,EACV,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,eAAe,EACf,eAAe,EAEf,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EAEnB,oBAAoB,EACpB,cAAc,EAEd,YAAY,EACZ,OAAO,EACP,WAAW,EACX,kBAAkB,EAClB,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,eAAe,EAEf,eAAe,EACf,mBAAmB,EACnB,YAAY,EAEZ,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EAEpB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EAEpB,aAAa,EACb,0BAA0B,GAC3B,MAAM,YAAY,CAAC;AAMpB,YAAY,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAMpE,YAAY,EAEV,YAAY,EACZ,eAAe,EAEf,SAAS,EACT,eAAe,EACf,mBAAmB,EACnB,oBAAoB,EACpB,eAAe,EAEf,WAAW,EACX,eAAe,EACf,aAAa,EACb,cAAc,EACd,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,aAAa,CAAC;AAMrB,YAAY,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAEnE,YAAY,EACV,aAAa,EACb,cAAc,EACd,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,sBAAsB,EACtB,qBAAqB,EACrB,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,eAAe,EACf,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,2BAA2B,EAC3B,oBAAoB,EACpB,aAAa,EACb,mBAAmB,EACnB,0BAA0B,EAC1B,0BAA0B,EAC1B,gCAAgC,EAChC,sCAAsC,GACvC,MAAM,kBAAkB,CAAC"}
1
+ {"version":3,"file":"types-only.d.ts","sourceRoot":"","sources":["../src/types-only.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAMH,YAAY,EAEV,gBAAgB,EAChB,kBAAkB,EAClB,UAAU,EACV,cAAc,EACd,aAAa,EAEb,SAAS,EACT,aAAa,EACb,YAAY,EAEZ,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,yBAAyB,EACzB,4BAA4B,EAC5B,0BAA0B,EAE1B,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAE5B,gBAAgB,EAChB,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,cAAc,EACd,eAAe,EACf,SAAS,GACV,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,UAAU,EACV,iBAAiB,EAEjB,WAAW,EACX,SAAS,EACT,UAAU,EAEV,eAAe,EACf,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,eAAe,EAEf,cAAc,EACd,YAAY,EACZ,aAAa,EACb,mBAAmB,EAEnB,WAAW,EACX,cAAc,EACd,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,OAAO,EACP,cAAc,GACf,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,WAAW,EACX,iBAAiB,EAEjB,eAAe,EACf,cAAc,EACd,UAAU,EAEV,WAAW,EAEX,kBAAkB,EAClB,cAAc,EACd,QAAQ,EAER,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,YAAY,GACb,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMrF,YAAY,EACV,cAAc,EACd,aAAa,EACb,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EAEtB,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EACtB,eAAe,EACf,mBAAmB,EACnB,cAAc,EACd,uBAAuB,GACxB,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMxD,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAM7E,YAAY,EAEV,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,sBAAsB,EACtB,gBAAgB,EAChB,iBAAiB,EAEjB,YAAY,EACZ,iBAAiB,EACjB,eAAe,EACf,aAAa,EACb,SAAS,EACT,eAAe,EAEf,UAAU,EACV,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,eAAe,EACf,eAAe,EAEf,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EAEnB,oBAAoB,EACpB,cAAc,EAEd,YAAY,EACZ,OAAO,EACP,WAAW,EACX,kBAAkB,EAClB,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,eAAe,EAEf,eAAe,EACf,mBAAmB,EACnB,YAAY,EAEZ,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EAEpB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EAEpB,aAAa,EACb,0BAA0B,GAC3B,MAAM,YAAY,CAAC;AAMpB,YAAY,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAMpE,YAAY,EAEV,YAAY,EACZ,eAAe,EAEf,SAAS,EACT,eAAe,EACf,mBAAmB,EACnB,oBAAoB,EACpB,eAAe,EAEf,WAAW,EACX,eAAe,EACf,aAAa,EACb,cAAc,EACd,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,aAAa,CAAC;AAMrB,YAAY,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAEnE,YAAY,EACV,aAAa,EACb,cAAc,EACd,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,sBAAsB,EACtB,qBAAqB,EACrB,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,eAAe,EACf,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,kBAAkB,EAClB,mBAAmB,EACnB,4BAA4B,EAC5B,qBAAqB,EACrB,2BAA2B,EAC3B,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAC5B,oBAAoB,EACpB,uBAAuB,EACvB,sBAAsB,EACtB,yBAAyB,EACzB,wBAAwB,EACxB,0BAA0B,EAC1B,qBAAqB,EACrB,uBAAuB,EACvB,uBAAuB,EACvB,yBAAyB,EACzB,mBAAmB,EACnB,wBAAwB,EACxB,mBAAmB,EACnB,2BAA2B,EAC3B,yBAAyB,EACzB,wBAAwB,EACxB,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EACpB,aAAa,EACb,mBAAmB,EACnB,0BAA0B,EAC1B,0BAA0B,EAC1B,gCAAgC,EAChC,sCAAsC,GACvC,MAAM,kBAAkB,CAAC"}
@@ -1 +1 @@
1
- {"version":3,"file":"workflow.d.ts","sourceRoot":"","sources":["../src/workflow.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,KAAK,oBAAoB,EAO1B,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAEpE,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,kBAAkB,EAC3B,MAAM,EAAE,gBAAgB,GACvB,OAAO,CAAC,oBAAoB,CAAC,CAkC/B"}
1
+ {"version":3,"file":"workflow.d.ts","sourceRoot":"","sources":["../src/workflow.ts"],"names":[],"mappings":"AACA,OAAO,EAGL,KAAK,oBAAoB,EAO1B,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAEpE,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,kBAAkB,EAC3B,MAAM,EAAE,gBAAgB,GACvB,OAAO,CAAC,oBAAoB,CAAC,CAmD/B"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@artemiskit/sdk",
3
- "version": "0.6.0",
3
+ "version": "0.6.2",
4
4
  "description": "Programmatic SDK for ArtemisKit LLM evaluation toolkit - integrate into your Node.js apps",
5
5
  "type": "module",
6
6
  "license": "Apache-2.0",
@@ -71,8 +71,8 @@
71
71
  "test:coverage": "bun test --coverage"
72
72
  },
73
73
  "dependencies": {
74
- "@artemiskit/core": "0.6.1",
75
- "@artemiskit/redteam": "0.3.9"
74
+ "@artemiskit/core": "0.6.3",
75
+ "@artemiskit/redteam": "0.3.11"
76
76
  },
77
77
  "devDependencies": {
78
78
  "@types/bun": "^1.1.0",
@@ -0,0 +1,116 @@
1
+ import { describe, expect, test } from 'bun:test';
2
+ import { mkdtemp } from 'node:fs/promises';
3
+ import { tmpdir } from 'node:os';
4
+ import { join } from 'node:path';
5
+ import { type AgentTarget, validateAgentWorkflow } from '@artemiskit/core';
6
+ import { ArtemisKit } from '../artemiskit';
7
+ import { readWorkflowRecord } from '../index';
8
+ import type { WorkflowCheckpointOptions } from '../types-only';
9
+
10
+ describe('SDK workflow recovery options', () => {
11
+ test('forwards checkpoint and pause controls and preserves the same logical run', async () => {
12
+ const parent = await mkdtemp(join(tmpdir(), 'artemis-sdk-recovery-'));
13
+ const checkpoint: WorkflowCheckpointOptions = {
14
+ directory: join(parent, 'private'),
15
+ mode: 'create',
16
+ configurationId: 'sdk-fixture-v1',
17
+ };
18
+ const workflow = validateAgentWorkflow({
19
+ version: '1',
20
+ kind: 'agent_workflow',
21
+ name: 'sdk-recovery',
22
+ target: { provider: 'openai', model: 'fixture' },
23
+ environment: {
24
+ type: 'simulated',
25
+ policy: {
26
+ network: 'denied',
27
+ side_effects: 'denied',
28
+ permissions: { files: 'write' },
29
+ budgets: {
30
+ max_actions: 6,
31
+ max_model_requests: 2,
32
+ max_tool_calls: 2,
33
+ max_tokens: 6,
34
+ timeout_ms: 20000,
35
+ },
36
+ },
37
+ },
38
+ tools: ['write_file'],
39
+ workflow: {
40
+ system_instructions: 'Write two files.',
41
+ initial_state: { files: {} },
42
+ turns: [{ role: 'user', content: 'PRIVATE-WORKFLOW' }],
43
+ },
44
+ outcomes: {
45
+ deterministic: [{ type: 'file', path: 'two.txt', exists: true, equals: 'PRIVATE-FILE' }],
46
+ },
47
+ evidence: { trace: 'summary', artifacts: 'checksums', redact: true },
48
+ });
49
+ let calls = 0;
50
+ let capabilities = 0;
51
+ const target: AgentTarget = {
52
+ provider: 'openai',
53
+ capabilities: async () => {
54
+ capabilities++;
55
+ return { status: 'available', toolUse: true, transportCancellation: true };
56
+ },
57
+ turn: async (request) => {
58
+ calls++;
59
+ const continued = request.messages.some((message) => message.role === 'tool');
60
+ return {
61
+ status: 'completed',
62
+ id: `answer-${calls}`,
63
+ model: 'fixture',
64
+ latencyMs: 1,
65
+ tokens: { prompt: 2, completion: 1, total: 3 },
66
+ message: {
67
+ role: 'assistant',
68
+ content: continued ? 'Done.' : '',
69
+ ...(continued
70
+ ? {}
71
+ : {
72
+ tool_calls: ['one', 'two'].map((name) => ({
73
+ id: name,
74
+ type: 'function' as const,
75
+ function: {
76
+ name: 'write_file',
77
+ arguments: JSON.stringify({ path: `${name}.txt`, content: 'PRIVATE-FILE' }),
78
+ },
79
+ })),
80
+ }),
81
+ },
82
+ };
83
+ },
84
+ };
85
+ const kit = new ArtemisKit();
86
+ const paused = await kit.runWorkflow({ workflow, target, checkpoint, pauseAfterActions: 2 });
87
+ expect(paused.record.reason).toBe('checkpoint_paused');
88
+ expect(paused.record.budgets.actions).toBe(2);
89
+ expect(calls).toBe(1);
90
+ expect(readWorkflowRecord(paused.record)).toEqual(paused.record);
91
+ const bad = await kit.runWorkflow({
92
+ workflow,
93
+ target,
94
+ checkpoint: { ...checkpoint, mode: 'resume', configurationId: 'changed' },
95
+ });
96
+ expect(bad.record.execution).not.toBe('completed');
97
+ expect(capabilities).toBe(1);
98
+ expect(calls).toBe(1);
99
+ const session = await kit.createWorkflowSession({
100
+ workflow,
101
+ target,
102
+ checkpoint: { ...checkpoint, mode: 'resume' },
103
+ });
104
+ expect(typeof session.pause).toBe('function');
105
+ const resumed = await session.run();
106
+ expect(resumed.record.taskVerification).toBe('passed');
107
+ expect(resumed.record.budgets).toMatchObject({ actions: 4, toolCalls: 2, modelRequests: 2 });
108
+ expect(resumed.record.usage.reported.total).toBe(6);
109
+ expect(calls).toBe(2);
110
+ expect(resumed.record.recovery?.runId).toBe(paused.record.recovery?.runId);
111
+ expect(resumed.record.recovery?.attempts).toBe(2);
112
+ expect(JSON.stringify(resumed.record)).not.toContain('PRIVATE-');
113
+ expect(JSON.stringify(resumed.record)).not.toContain(checkpoint.directory);
114
+ expect(readWorkflowRecord(resumed.record)).toEqual(resumed.record);
115
+ });
116
+ });
@@ -12,7 +12,8 @@ import {
12
12
  } from '@artemiskit/core';
13
13
  import { OpenAIAdapter } from '../../../adapters/openai/src/client';
14
14
  import { ArtemisKit } from '../artemiskit';
15
- import type { WorkflowRunOptions } from '../types-only';
15
+ import { readWorkflowRecord } from '../index';
16
+ import type { WorkflowJudgeOptions, WorkflowRunOptions } from '../types-only';
16
17
 
17
18
  const servers: ReturnType<typeof Bun.serve>[] = [];
18
19
  afterEach(() => {
@@ -63,6 +64,152 @@ function target(): AgentTarget {
63
64
  }
64
65
 
65
66
  describe('SDK native workflow session', () => {
67
+ test('forwards explicit independent semantic judging with separate coverage, usage and record reading', async () => {
68
+ for (const verdict of ['pass', 'fail', 'malformed', 'unconfigured']) {
69
+ const scenario = workflow();
70
+ scenario.outcomes.deterministic = [{ type: 'file', path: 'result.txt', exists: false }];
71
+ scenario.outcomes.semantic = [
72
+ { type: 'llm_judge', mode: 'strict_assurance', rubric: 'PRIVATE-RUBRIC' },
73
+ ];
74
+ let calls = 0;
75
+ let closed = 0;
76
+ const semanticJudge: WorkflowJudgeOptions = {
77
+ provider: 'independent-judge',
78
+ model: 'judge-model',
79
+ limits: { maxRequests: 1, maxTokens: 100, maxOutputTokens: 10, timeoutMs: 1000 },
80
+ client: {
81
+ provider: 'independent-judge',
82
+ capabilities: async () => ({
83
+ streaming: false,
84
+ functionCalling: false,
85
+ toolUse: false,
86
+ maxContext: 10000,
87
+ }),
88
+ generate: async (request) => {
89
+ calls++;
90
+ expect(request.model).toBe('judge-model');
91
+ expect(request.maxRetries).toBe(0);
92
+ return {
93
+ id: 'judge',
94
+ model: 'observed-judge',
95
+ text: verdict === 'malformed' ? 'PRIVATE-BAD-RESPONSE' : JSON.stringify({ verdict }),
96
+ tokens: { prompt: 5, completion: 2, total: 7 },
97
+ latencyMs: 1,
98
+ };
99
+ },
100
+ close: async () => {
101
+ closed++;
102
+ },
103
+ },
104
+ };
105
+ const result = await new ArtemisKit().runWorkflow({
106
+ workflow: scenario,
107
+ target: target(),
108
+ ...(verdict === 'unconfigured' ? {} : { semanticJudge }),
109
+ });
110
+ expect(result.record.execution).toBe('completed');
111
+ expect(result.record.taskVerification).toBe(
112
+ verdict === 'pass'
113
+ ? 'passed'
114
+ : verdict === 'fail'
115
+ ? 'failed'
116
+ : verdict === 'malformed'
117
+ ? 'invalid'
118
+ : 'unavailable'
119
+ );
120
+ expect(result.record.usage.reported.total).toBe(3);
121
+ expect(result.record.outcomes.deterministic.counts.valid).toBe(1);
122
+ expect(result.record.outcomes.semantic.usage.reported.total).toBe(
123
+ verdict === 'unconfigured' ? 0 : 7
124
+ );
125
+ expect(calls).toBe(verdict === 'unconfigured' ? 0 : 1);
126
+ expect(closed).toBe(0);
127
+ expect(readWorkflowRecord(JSON.parse(JSON.stringify(result.record)))).toEqual(result.record);
128
+ expect(JSON.stringify(result.record)).not.toContain('PRIVATE-');
129
+ }
130
+ });
131
+ test('malformed explicit judge bounds stop before any target model call', async () => {
132
+ const scenario = workflow();
133
+ let turns = 0;
134
+ const custom = target();
135
+ custom.turn = async () => {
136
+ turns++;
137
+ throw new Error('must not call');
138
+ };
139
+ const result = await new ArtemisKit().runWorkflow({
140
+ workflow: scenario,
141
+ target: custom,
142
+ semanticJudge: {
143
+ provider: 'judge',
144
+ model: 'judge',
145
+ limits: { maxRequests: 0, maxTokens: 100, maxOutputTokens: 10, timeoutMs: 1000 },
146
+ client: {
147
+ provider: 'judge',
148
+ capabilities: async () => {
149
+ throw new Error('must not call');
150
+ },
151
+ generate: async () => {
152
+ throw new Error('must not call');
153
+ },
154
+ },
155
+ },
156
+ });
157
+ expect(turns).toBe(0);
158
+ expect(result.record.execution).toBe('invalid');
159
+ });
160
+ test('deterministic failure gates judge calls and session cancellation reaches the independent evaluator', async () => {
161
+ let started!: () => void;
162
+ const pending = new Promise<void>((resolve) => {
163
+ started = resolve;
164
+ });
165
+ let calls = 0;
166
+ const semanticJudge: WorkflowJudgeOptions = {
167
+ provider: 'judge',
168
+ model: 'judge',
169
+ limits: { maxRequests: 1, maxTokens: 100, maxOutputTokens: 10, timeoutMs: 1000 },
170
+ client: {
171
+ provider: 'judge',
172
+ capabilities: async () => ({
173
+ streaming: false,
174
+ functionCalling: false,
175
+ toolUse: false,
176
+ maxContext: 10000,
177
+ }),
178
+ generate: async (request) => {
179
+ calls++;
180
+ started();
181
+ await new Promise<void>((resolve) =>
182
+ request.signal?.addEventListener('abort', () => resolve(), { once: true })
183
+ );
184
+ throw new Error('PRIVATE-ABORT');
185
+ },
186
+ },
187
+ };
188
+ const scenario = workflow();
189
+ scenario.outcomes.semantic = [
190
+ { type: 'llm_judge', mode: 'strict_assurance', rubric: 'PRIVATE-RUBRIC' },
191
+ ];
192
+ const kit = new ArtemisKit();
193
+ const failed = await kit.runWorkflow({ workflow: scenario, target: target(), semanticJudge });
194
+ expect(failed.record.taskVerification).toBe('failed');
195
+ expect(calls).toBe(0);
196
+ scenario.outcomes.deterministic = [{ type: 'file', path: 'result.txt', exists: false }];
197
+ const session = await kit.createWorkflowSession({
198
+ workflow: scenario,
199
+ target: target(),
200
+ semanticJudge,
201
+ });
202
+ const finished = session.run();
203
+ await pending;
204
+ session.cancel();
205
+ const result = await finished;
206
+ expect(result.record.execution).toBe('completed');
207
+ expect(result.record.outcomes.cancelled).toBe(true);
208
+ expect(result.record.outcomes.reason).toBe('cancelled');
209
+ expect(result.record.outcomes.semantic.counts.valid).toBe(0);
210
+ expect(readWorkflowRecord(result.record)).toEqual(result.record);
211
+ });
212
+
66
213
  test('configured adapter uses workflow identity and zero retries with real loopback multi-turn execution', async () => {
67
214
  const bodies: Record<string, unknown>[] = [];
68
215
  const server = Bun.serve({
@@ -154,7 +301,7 @@ describe('SDK native workflow session', () => {
154
301
  const { result, events } = JSON.parse(stdout);
155
302
  expect(bodies[0].model).toBe('workflow-authoritative');
156
303
  expect(result.record.execution).toBe('completed');
157
- expect(result.record.taskVerification).toBe('unavailable');
304
+ expect(result.record.taskVerification).toBe('passed');
158
305
  expect(result.record.budgets).toMatchObject({ modelRequests: 2, toolCalls: 1 });
159
306
  expect(result.state?.files).toEqual({ 'result.txt': 'PRIVATE-STATE' });
160
307
  expect(JSON.stringify(result.record)).not.toContain('PRIVATE-');
package/src/index.ts CHANGED
@@ -10,6 +10,8 @@ export { ArtemisKit } from './artemiskit';
10
10
  export {
11
11
  AgentWorkflowSchema,
12
12
  WorkflowPolicySchema,
13
+ WorkflowFaultsSchema,
14
+ WorkflowRetrySchema,
13
15
  WORKFLOW_TOOL_IDS,
14
16
  validateAgentWorkflow,
15
17
  parseAgentWorkflow,
@@ -20,6 +22,12 @@ export {
20
22
  createModelClientTarget,
21
23
  createAgentWorkflowSession,
22
24
  runAgentWorkflow,
25
+ evaluateWorkflowDeterministicOutcomes,
26
+ evaluateWorkflowSemantics,
27
+ isValidWorkflowJudgeOptions,
28
+ decideWorkflowOutcome,
29
+ validWorkflowAssertionCounts,
30
+ readWorkflowRecord,
23
31
  createSimulatedWorkflowEnvironment,
24
32
  createDockerWorkflowEnvironment,
25
33
  createDockerWorkflowEnvironmentFactory,
@@ -44,8 +52,32 @@ export type {
44
52
  AgentWorkflowReason,
45
53
  AgentWorkflowEvent,
46
54
  AgentWorkflowRecord,
55
+ AgentWorkflowExecutionRecord,
56
+ AgentWorkflowEvidence,
57
+ WorkflowDeterministicStatus,
58
+ WorkflowDeterministicReason,
59
+ WorkflowDeterministicAssertion,
60
+ WorkflowDeterministicSummary,
61
+ WorkflowJudgeOptions,
62
+ WorkflowSemanticOptions,
63
+ WorkflowSemanticReason,
64
+ WorkflowSemanticAssertion,
65
+ WorkflowSemanticIdentity,
66
+ WorkflowSemanticEvaluation,
67
+ WorkflowOutcomeStatus,
68
+ WorkflowAssertionCounts,
69
+ WorkflowOutcomeDecision,
70
+ WorkflowOutcomeAssessment,
71
+ SavedWorkflowRecord,
72
+ HistoricalWorkflowRecord,
47
73
  AgentWorkflowResult,
48
74
  AgentWorkflowSessionOptions,
75
+ WorkflowCheckpointOptions,
76
+ WorkflowRecoveryEvidence,
77
+ WorkflowFault,
78
+ WorkflowFaults,
79
+ WorkflowFaultKind,
80
+ WorkflowRetry,
49
81
  AgentWorkflowSession,
50
82
  WorkflowState,
51
83
  WorkflowEnvironment,
package/src/types-only.ts CHANGED
@@ -294,8 +294,32 @@ export type {
294
294
  AgentWorkflowReason,
295
295
  AgentWorkflowEvent,
296
296
  AgentWorkflowRecord,
297
+ AgentWorkflowExecutionRecord,
298
+ AgentWorkflowEvidence,
299
+ WorkflowDeterministicStatus,
300
+ WorkflowDeterministicReason,
301
+ WorkflowDeterministicAssertion,
302
+ WorkflowDeterministicSummary,
303
+ WorkflowJudgeOptions,
304
+ WorkflowSemanticOptions,
305
+ WorkflowSemanticReason,
306
+ WorkflowSemanticAssertion,
307
+ WorkflowSemanticIdentity,
308
+ WorkflowSemanticEvaluation,
309
+ WorkflowOutcomeStatus,
310
+ WorkflowAssertionCounts,
311
+ WorkflowOutcomeDecision,
312
+ WorkflowOutcomeAssessment,
313
+ SavedWorkflowRecord,
314
+ HistoricalWorkflowRecord,
297
315
  AgentWorkflowResult,
298
316
  AgentWorkflowSessionOptions,
317
+ WorkflowCheckpointOptions,
318
+ WorkflowRecoveryEvidence,
319
+ WorkflowFault,
320
+ WorkflowFaults,
321
+ WorkflowFaultKind,
322
+ WorkflowRetry,
299
323
  AgentWorkflowSession,
300
324
  WorkflowState,
301
325
  WorkflowEnvironment,
package/src/workflow.ts CHANGED
@@ -1,6 +1,7 @@
1
1
  import { dirname, resolve } from 'node:path';
2
2
  import {
3
3
  type AdapterConfig,
4
+ type AgentTarget,
4
5
  type AgentWorkflowSession,
5
6
  ArtemisError,
6
7
  createAdapter,
@@ -20,22 +21,36 @@ export async function prepareWorkflowSession(
20
21
  const workflow = file ? await loadAgentWorkflow(file) : validateAgentWorkflow(options.workflow);
21
22
  let target = options.target;
22
23
  if (!target) {
23
- let client = options.client;
24
- if (!client) {
25
- try {
26
- client = await createAdapter({
24
+ if (options.client) target = createModelClientTarget(options.client);
25
+ else {
26
+ let initialized: Promise<AgentTarget> | undefined;
27
+ const getTarget = () => {
28
+ initialized ??= createAdapter({
27
29
  ...config.providerConfig,
28
30
  ...options.providerConfig,
29
31
  provider: workflow.target.provider,
30
32
  defaultModel: workflow.target.model,
31
33
  maxRetries: 0,
32
- } as AdapterConfig);
33
- } catch {
34
- // Provider factories may put credentials or configuration values in their diagnostics.
35
- throw new ArtemisError('Unable to initialize workflow target', 'PROVIDER_UNAVAILABLE');
36
- }
34
+ } as AdapterConfig)
35
+ .then(createModelClientTarget)
36
+ .catch(() => {
37
+ throw new ArtemisError('Unable to initialize workflow target', 'PROVIDER_UNAVAILABLE');
38
+ });
39
+ return initialized;
40
+ };
41
+ target = options.checkpoint
42
+ ? {
43
+ provider: workflow.target.provider,
44
+ capabilities: async (limits, signal) =>
45
+ (await getTarget()).capabilities(limits, signal),
46
+ turn: async (request, signal) => (await getTarget()).turn(request, signal),
47
+ drain: async (limits) =>
48
+ initialized
49
+ ? ((await initialized).drain?.(limits) ?? { pendingOperations: 0 })
50
+ : { pendingOperations: 0 },
51
+ }
52
+ : await getTarget();
37
53
  }
38
- target = createModelClientTarget(client);
39
54
  }
40
55
  return createAgentWorkflowSession({
41
56
  workflow,
@@ -46,6 +61,9 @@ export async function prepareWorkflowSession(
46
61
  preflightOnly: options.preflightOnly,
47
62
  signal: options.signal,
48
63
  cleanupTimeoutMs: options.cleanupTimeoutMs,
64
+ semanticJudge: options.semanticJudge,
49
65
  onEvent: options.onEvent,
66
+ checkpoint: options.checkpoint,
67
+ pauseAfterActions: options.pauseAfterActions,
50
68
  });
51
69
  }