@artemiskit/sdk 0.6.0 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,14 @@
1
1
  # @artemiskit/sdk
2
2
 
3
+ ## 0.6.1
4
+
5
+ ### Patch Changes
6
+
7
+ - Complete the 0.6.2 independent workflow outcome milestone: verify final artifacts, bounded JSON schemas, state, tool traces and policy before optional explicitly configured semantic judging. Keep target and judge identity, usage and limits separate; expose task validity and coverage, strict V1/V2 saved-record reading, SDK composition and CLI outcome exits without changing historical prompt-response reports.
8
+ - Updated dependencies
9
+ - @artemiskit/core@0.6.2
10
+ - @artemiskit/redteam@0.3.10
11
+
3
12
  ## 0.6.0
4
13
 
5
14
  ### Minor Changes
package/README.md CHANGED
@@ -57,7 +57,7 @@ const session = await kit.createWorkflowSession({
57
57
  preflight: true,
58
58
  });
59
59
  const result = await session.run();
60
- console.log(result.record); // bounded metadata; taskVerification is unavailable in 0.6.1
60
+ console.log(result.record); // bounded metadata with separate execution and task outcomes
61
61
  // session.cancel() stops new work and initiates bounded cleanup.
62
62
  // result.state and result.transcript contain sensitive working data.
63
63
  ```
@@ -66,7 +66,11 @@ console.log(result.record); // bounded metadata; taskVerification is unavailable
66
66
  `target` to use an existing trusted adapter or target. The same native controls apply through the
67
67
  CLI and SDK: declared tools and permissions, cumulative budgets, explicit preflight, cancellation,
68
68
  and cleanup evidence. Workflows use fresh simulated state or a disposable Docker filesystem.
69
- Independent task scoring follows in 0.6.2; normal execution does not establish a task pass.
69
+ The 0.6.2 milestone independently verifies declared artifacts, state, schemas, traces and policy.
70
+ Use explicit `semanticJudge` configuration for declared qualitative criteria. Target and judge usage
71
+ remain separate; `readWorkflowRecord` validates V1/V2 saved metadata without making model calls.
72
+ See [workflow outcomes](https://github.com/code-sensei/artemiskit/blob/main/docs/workflow-outcomes.md)
73
+ for usage, denominators, compatibility and publication status.
70
74
 
71
75
  The root and `/types` entry points expose workflow result/session/event types. Low-level
72
76
  `createAgentWorkflowSession` and `runAgentWorkflow` are available for explicit target composition.
package/dist/index.d.ts CHANGED
@@ -3,8 +3,8 @@
3
3
  * Programmatic SDK for ArtemisKit LLM evaluation toolkit
4
4
  */
5
5
  export { ArtemisKit } from './artemiskit.js';
6
- export { AgentWorkflowSchema, WorkflowPolicySchema, WORKFLOW_TOOL_IDS, validateAgentWorkflow, parseAgentWorkflow, loadAgentWorkflow, listWorkflowTools, getWorkflowTool, executeSimulatedTool, createModelClientTarget, createAgentWorkflowSession, runAgentWorkflow, createSimulatedWorkflowEnvironment, createDockerWorkflowEnvironment, createDockerWorkflowEnvironmentFactory, WORKFLOW_SANDBOX_IMAGE, WorkflowEnvironmentInitializationError, } from '@artemiskit/core';
7
- export type { AgentWorkflow, WorkflowPolicy, WorkflowJson, WorkflowToolId, WorkflowResource, WorkflowToolDescriptor, SimulatedToolEvidence, SimulatedToolResult, AgentTarget, AgentTurnRequest, AgentTurnResult, AgentTargetCapabilities, AgentTargetFailure, AgentWorkflowExecution, AgentWorkflowReason, AgentWorkflowEvent, AgentWorkflowRecord, AgentWorkflowResult, AgentWorkflowSessionOptions, AgentWorkflowSession, WorkflowState, WorkflowEnvironment, WorkflowEnvironmentFactory, WorkflowEnvironmentCleanup, DockerWorkflowEnvironmentOptions, } from '@artemiskit/core';
6
+ export { AgentWorkflowSchema, WorkflowPolicySchema, WORKFLOW_TOOL_IDS, validateAgentWorkflow, parseAgentWorkflow, loadAgentWorkflow, listWorkflowTools, getWorkflowTool, executeSimulatedTool, createModelClientTarget, createAgentWorkflowSession, runAgentWorkflow, evaluateWorkflowDeterministicOutcomes, evaluateWorkflowSemantics, isValidWorkflowJudgeOptions, decideWorkflowOutcome, validWorkflowAssertionCounts, readWorkflowRecord, createSimulatedWorkflowEnvironment, createDockerWorkflowEnvironment, createDockerWorkflowEnvironmentFactory, WORKFLOW_SANDBOX_IMAGE, WorkflowEnvironmentInitializationError, } from '@artemiskit/core';
7
+ export type { AgentWorkflow, WorkflowPolicy, WorkflowJson, WorkflowToolId, WorkflowResource, WorkflowToolDescriptor, SimulatedToolEvidence, SimulatedToolResult, AgentTarget, AgentTurnRequest, AgentTurnResult, AgentTargetCapabilities, AgentTargetFailure, AgentWorkflowExecution, AgentWorkflowReason, AgentWorkflowEvent, AgentWorkflowRecord, AgentWorkflowExecutionRecord, AgentWorkflowEvidence, WorkflowDeterministicStatus, WorkflowDeterministicReason, WorkflowDeterministicAssertion, WorkflowDeterministicSummary, WorkflowJudgeOptions, WorkflowSemanticOptions, WorkflowSemanticReason, WorkflowSemanticAssertion, WorkflowSemanticIdentity, WorkflowSemanticEvaluation, WorkflowOutcomeStatus, WorkflowAssertionCounts, WorkflowOutcomeDecision, WorkflowOutcomeAssessment, SavedWorkflowRecord, HistoricalWorkflowRecord, AgentWorkflowResult, AgentWorkflowSessionOptions, AgentWorkflowSession, WorkflowState, WorkflowEnvironment, WorkflowEnvironmentFactory, WorkflowEnvironmentCleanup, DockerWorkflowEnvironmentOptions, } from '@artemiskit/core';
8
8
  export type { ArtemisKitConfig, WorkflowRunOptions, RunOptions, RedTeamOptions, StressOptions, RunResult, RedTeamResult, StressResult, CaseStartEvent, CaseCompleteEvent, ProgressEvent, RedTeamMutationStartEvent, RedTeamMutationCompleteEvent, StressRequestCompleteEvent, CaseStartHandler, CaseCompleteHandler, ProgressHandler, RedTeamMutationStartHandler, RedTeamMutationCompleteHandler, StressRequestCompleteHandler, ArtemisKitEvents, ArtemisKitEventName, ValidateOptions, ValidationResult, ValidationError, ValidationWarning, ScenarioValidation, CompareOptions, CompareResult, RunSummary, ComparisonDetails, } from './types.js';
9
9
  export type { Scenario, TestCase, Expected, Provider, ProviderConfig, ChatMessageType, Variables, } from '@artemiskit/core';
10
10
  export type { CaseResult, CaseRedactionInfo, RunManifest, RunConfig, RunMetrics, RedTeamManifest, RedTeamConfig, RedTeamMetrics, RedTeamCaseResult, RedTeamStatus, RedTeamSeverity, StressManifest, StressConfig, StressMetrics, StressRequestResult, AnyManifest, ResolvedConfig, ConfigSource, ManifestRedactionInfo, CostEstimateInfo, GitInfo, ProvenanceInfo, } from '@artemiskit/core';
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAG1C,OAAO,EACL,mBAAmB,EACnB,oBAAoB,EACpB,iBAAiB,EACjB,qBAAqB,EACrB,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,eAAe,EACf,oBAAoB,EACpB,uBAAuB,EACvB,0BAA0B,EAC1B,gBAAgB,EAChB,kCAAkC,EAClC,+BAA+B,EAC/B,sCAAsC,EACtC,sBAAsB,EACtB,sCAAsC,GACvC,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,aAAa,EACb,cAAc,EACd,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,sBAAsB,EACtB,qBAAqB,EACrB,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,eAAe,EACf,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,2BAA2B,EAC3B,oBAAoB,EACpB,aAAa,EACb,mBAAmB,EACnB,0BAA0B,EAC1B,0BAA0B,EAC1B,gCAAgC,GACjC,MAAM,kBAAkB,CAAC;AAG1B,YAAY,EAEV,gBAAgB,EAChB,kBAAkB,EAClB,UAAU,EACV,cAAc,EACd,aAAa,EAEb,SAAS,EACT,aAAa,EACb,YAAY,EAEZ,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,yBAAyB,EACzB,4BAA4B,EAC5B,0BAA0B,EAE1B,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAE5B,gBAAgB,EAChB,mBAAmB,EAEnB,eAAe,EACf,gBAAgB,EAChB,eAAe,EACf,iBAAiB,EACjB,kBAAkB,EAElB,cAAc,EACd,aAAa,EACb,UAAU,EACV,iBAAiB,GAClB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,cAAc,EACd,eAAe,EACf,SAAS,GACV,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,UAAU,EACV,iBAAiB,EAEjB,WAAW,EACX,SAAS,EACT,UAAU,EAEV,eAAe,EACf,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,eAAe,EAEf,cAAc,EACd,YAAY,EACZ,aAAa,EACb,mBAAmB,EAEnB,WAAW,EACX,cAAc,EACd,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,OAAO,EACP,cAAc,GACf,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,WAAW,EACX,iBAAiB,EAEjB,eAAe,EACf,cAAc,EACd,UAAU,EAEV,WAAW,EAEX,kBAAkB,EAClB,cAAc,EACd,QAAQ,EAER,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,YAAY,GACb,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMrF,YAAY,EACV,cAAc,EACd,aAAa,EACb,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EAEtB,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EACtB,eAAe,EACf,mBAAmB,EACnB,cAAc,EACd,uBAAuB,GACxB,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMxD,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAMtF,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAM7E,OAAO,EAAE,kBAAkB,EAAE,KAAK,kBAAkB,EAAE,KAAK,aAAa,EAAE,MAAM,YAAY,CAAC;AAM7F,OAAO,EAEL,WAAW,EACX,eAAe,EACf,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,oBAAoB,EAEpB,aAAa,EACb,MAAM,EAEN,cAAc,EACd,cAAc,EACd,aAAa,EACb,oBAAoB,GACrB,MAAM,SAAS,CAAC;AAEjB,YAAY,EAEV,YAAY,EACZ,eAAe,EAEf,SAAS,EACT,eAAe,EACf,mBAAmB,EACnB,oBAAoB,EACpB,eAAe,EAEf,WAAW,EACX,eAAe,EACf,aAAa,EACb,cAAc,EACd,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,OAAO,EAEL,aAAa,EAEb,eAAe,EAEf,aAAa,EAEb,YAAY,GACb,MAAM,aAAa,CAAC;AAErB,YAAY,EACV,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,aAAa,CAAC;AAMrB,OAAO,EAEL,eAAe,EACf,eAAe,EAEf,QAAQ,EACR,QAAQ,EAER,YAAY,EACZ,SAAS,EACT,SAAS,EACT,QAAQ,EACR,UAAU,EAEV,KAAK,EACL,QAAQ,EACR,WAAW,EACX,KAAK,EACL,KAAK,EACL,UAAU,EACV,QAAQ,EACR,UAAU,EACV,MAAM,EACN,KAAK,EACL,KAAK,GACN,MAAM,YAAY,CAAC;AAMpB,OAAO,EAEL,QAAQ,EACR,cAAc,EAEd,qBAAqB,EAErB,iBAAiB,EACjB,uBAAuB,EAEvB,mBAAmB,EACnB,oBAAoB,EACpB,iBAAiB,EAEjB,eAAe,EACf,4BAA4B,EAE5B,gBAAgB,EAChB,sBAAsB,EAEtB,eAAe,EACf,wBAAwB,EACxB,SAAS,EACT,kBAAkB,EAClB,aAAa,EACb,4BAA4B,EAC5B,gBAAgB,EAEhB,YAAY,EACZ,4BAA4B,EAE5B,UAAU,EACV,WAAW,EACX,cAAc,EACd,mBAAmB,EACnB,aAAa,EACb,sBAAsB,EACtB,eAAe,EACf,qBAAqB,EAErB,cAAc,EACd,gBAAgB,EAChB,WAAW,GACZ,MAAM,YAAY,CAAC;AAGpB,YAAY,EAEV,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,sBAAsB,EACtB,gBAAgB,EAChB,iBAAiB,EACjB,oBAAoB,EAEpB,YAAY,EACZ,iBAAiB,EACjB,eAAe,EACf,aAAa,EACb,SAAS,EACT,eAAe,EAEf,UAAU,EACV,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,eAAe,EACf,eAAe,EAEf,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EAEnB,oBAAoB,EACpB,cAAc,EAEd,YAAY,EACZ,OAAO,EACP,WAAW,EACX,kBAAkB,EAClB,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,eAAe,EAEf,eAAe,EACf,mBAAmB,EACnB,YAAY,EAEZ,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EAEpB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EAEpB,aAAa,EACb,0BAA0B,EAE1B,qBAAqB,EACrB,kBAAkB,EAClB,eAAe,EAEf,uBAAuB,EACvB,aAAa,EACb,kBAAkB,EAClB,eAAe,EACf,wBAAwB,EAExB,kBAAkB,EAClB,oBAAoB,EACpB,cAAc,EACd,cAAc,EACd,mBAAmB,EACnB,4BAA4B,EAC5B,yBAAyB,EACzB,kCAAkC,EAClC,2BAA2B,EAC3B,mBAAmB,EACnB,uBAAuB,EACvB,eAAe,EACf,eAAe,EACf,gBAAgB,EAChB,yBAAyB,EACzB,sBAAsB,GACvB,MAAM,YAAY,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAG1C,OAAO,EACL,mBAAmB,EACnB,oBAAoB,EACpB,iBAAiB,EACjB,qBAAqB,EACrB,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,eAAe,EACf,oBAAoB,EACpB,uBAAuB,EACvB,0BAA0B,EAC1B,gBAAgB,EAChB,qCAAqC,EACrC,yBAAyB,EACzB,2BAA2B,EAC3B,qBAAqB,EACrB,4BAA4B,EAC5B,kBAAkB,EAClB,kCAAkC,EAClC,+BAA+B,EAC/B,sCAAsC,EACtC,sBAAsB,EACtB,sCAAsC,GACvC,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,aAAa,EACb,cAAc,EACd,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,sBAAsB,EACtB,qBAAqB,EACrB,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,eAAe,EACf,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,kBAAkB,EAClB,mBAAmB,EACnB,4BAA4B,EAC5B,qBAAqB,EACrB,2BAA2B,EAC3B,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAC5B,oBAAoB,EACpB,uBAAuB,EACvB,sBAAsB,EACtB,yBAAyB,EACzB,wBAAwB,EACxB,0BAA0B,EAC1B,qBAAqB,EACrB,uBAAuB,EACvB,uBAAuB,EACvB,yBAAyB,EACzB,mBAAmB,EACnB,wBAAwB,EACxB,mBAAmB,EACnB,2BAA2B,EAC3B,oBAAoB,EACpB,aAAa,EACb,mBAAmB,EACnB,0BAA0B,EAC1B,0BAA0B,EAC1B,gCAAgC,GACjC,MAAM,kBAAkB,CAAC;AAG1B,YAAY,EAEV,gBAAgB,EAChB,kBAAkB,EAClB,UAAU,EACV,cAAc,EACd,aAAa,EAEb,SAAS,EACT,aAAa,EACb,YAAY,EAEZ,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,yBAAyB,EACzB,4BAA4B,EAC5B,0BAA0B,EAE1B,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAE5B,gBAAgB,EAChB,mBAAmB,EAEnB,eAAe,EACf,gBAAgB,EAChB,eAAe,EACf,iBAAiB,EACjB,kBAAkB,EAElB,cAAc,EACd,aAAa,EACb,UAAU,EACV,iBAAiB,GAClB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,cAAc,EACd,eAAe,EACf,SAAS,GACV,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,UAAU,EACV,iBAAiB,EAEjB,WAAW,EACX,SAAS,EACT,UAAU,EAEV,eAAe,EACf,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,eAAe,EAEf,cAAc,EACd,YAAY,EACZ,aAAa,EACb,mBAAmB,EAEnB,WAAW,EACX,cAAc,EACd,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,OAAO,EACP,cAAc,GACf,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,WAAW,EACX,iBAAiB,EAEjB,eAAe,EACf,cAAc,EACd,UAAU,EAEV,WAAW,EAEX,kBAAkB,EAClB,cAAc,EACd,QAAQ,EAER,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,YAAY,GACb,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMrF,YAAY,EACV,cAAc,EACd,aAAa,EACb,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EAEtB,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EACtB,eAAe,EACf,mBAAmB,EACnB,cAAc,EACd,uBAAuB,GACxB,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMxD,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAMtF,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAM7E,OAAO,EAAE,kBAAkB,EAAE,KAAK,kBAAkB,EAAE,KAAK,aAAa,EAAE,MAAM,YAAY,CAAC;AAM7F,OAAO,EAEL,WAAW,EACX,eAAe,EACf,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,oBAAoB,EAEpB,aAAa,EACb,MAAM,EAEN,cAAc,EACd,cAAc,EACd,aAAa,EACb,oBAAoB,GACrB,MAAM,SAAS,CAAC;AAEjB,YAAY,EAEV,YAAY,EACZ,eAAe,EAEf,SAAS,EACT,eAAe,EACf,mBAAmB,EACnB,oBAAoB,EACpB,eAAe,EAEf,WAAW,EACX,eAAe,EACf,aAAa,EACb,cAAc,EACd,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,OAAO,EAEL,aAAa,EAEb,eAAe,EAEf,aAAa,EAEb,YAAY,GACb,MAAM,aAAa,CAAC;AAErB,YAAY,EACV,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,aAAa,CAAC;AAMrB,OAAO,EAEL,eAAe,EACf,eAAe,EAEf,QAAQ,EACR,QAAQ,EAER,YAAY,EACZ,SAAS,EACT,SAAS,EACT,QAAQ,EACR,UAAU,EAEV,KAAK,EACL,QAAQ,EACR,WAAW,EACX,KAAK,EACL,KAAK,EACL,UAAU,EACV,QAAQ,EACR,UAAU,EACV,MAAM,EACN,KAAK,EACL,KAAK,GACN,MAAM,YAAY,CAAC;AAMpB,OAAO,EAEL,QAAQ,EACR,cAAc,EAEd,qBAAqB,EAErB,iBAAiB,EACjB,uBAAuB,EAEvB,mBAAmB,EACnB,oBAAoB,EACpB,iBAAiB,EAEjB,eAAe,EACf,4BAA4B,EAE5B,gBAAgB,EAChB,sBAAsB,EAEtB,eAAe,EACf,wBAAwB,EACxB,SAAS,EACT,kBAAkB,EAClB,aAAa,EACb,4BAA4B,EAC5B,gBAAgB,EAEhB,YAAY,EACZ,4BAA4B,EAE5B,UAAU,EACV,WAAW,EACX,cAAc,EACd,mBAAmB,EACnB,aAAa,EACb,sBAAsB,EACtB,eAAe,EACf,qBAAqB,EAErB,cAAc,EACd,gBAAgB,EAChB,WAAW,GACZ,MAAM,YAAY,CAAC;AAGpB,YAAY,EAEV,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,sBAAsB,EACtB,gBAAgB,EAChB,iBAAiB,EACjB,oBAAoB,EAEpB,YAAY,EACZ,iBAAiB,EACjB,eAAe,EACf,aAAa,EACb,SAAS,EACT,eAAe,EAEf,UAAU,EACV,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,eAAe,EACf,eAAe,EAEf,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EAEnB,oBAAoB,EACpB,cAAc,EAEd,YAAY,EACZ,OAAO,EACP,WAAW,EACX,kBAAkB,EAClB,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,eAAe,EAEf,eAAe,EACf,mBAAmB,EACnB,YAAY,EAEZ,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EAEpB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EAEpB,aAAa,EACb,0BAA0B,EAE1B,qBAAqB,EACrB,kBAAkB,EAClB,eAAe,EAEf,uBAAuB,EACvB,aAAa,EACb,kBAAkB,EAClB,eAAe,EACf,wBAAwB,EAExB,kBAAkB,EAClB,oBAAoB,EACpB,cAAc,EACd,cAAc,EACd,mBAAmB,EACnB,4BAA4B,EAC5B,yBAAyB,EACzB,kCAAkC,EAClC,2BAA2B,EAC3B,mBAAmB,EACnB,uBAAuB,EACvB,eAAe,EACf,eAAe,EACf,gBAAgB,EAChB,yBAAyB,EACzB,sBAAsB,GACvB,MAAM,YAAY,CAAC"}
package/dist/index.js CHANGED
@@ -7678,6 +7678,7 @@ async function prepareWorkflowSession(options, config) {
7678
7678
  preflightOnly: options.preflightOnly,
7679
7679
  signal: options.signal,
7680
7680
  cleanupTimeoutMs: options.cleanupTimeoutMs,
7681
+ semanticJudge: options.semanticJudge,
7681
7682
  onEvent: options.onEvent
7682
7683
  });
7683
7684
  }
@@ -8516,6 +8517,12 @@ import {
8516
8517
  createModelClientTarget as createModelClientTarget2,
8517
8518
  createAgentWorkflowSession as createAgentWorkflowSession2,
8518
8519
  runAgentWorkflow,
8520
+ evaluateWorkflowDeterministicOutcomes,
8521
+ evaluateWorkflowSemantics,
8522
+ isValidWorkflowJudgeOptions,
8523
+ decideWorkflowOutcome,
8524
+ validWorkflowAssertionCounts,
8525
+ readWorkflowRecord,
8519
8526
  createSimulatedWorkflowEnvironment,
8520
8527
  createDockerWorkflowEnvironment,
8521
8528
  createDockerWorkflowEnvironmentFactory,
@@ -15257,12 +15264,14 @@ function createGuardian(config = {}) {
15257
15264
  export {
15258
15265
  validatePolicy,
15259
15266
  validateAgentWorkflow2 as validateAgentWorkflow,
15267
+ validWorkflowAssertionCounts,
15260
15268
  testCase,
15261
15269
  similarity,
15262
15270
  scenario,
15263
15271
  runAgentWorkflow,
15264
15272
  regexCase,
15265
15273
  regex,
15274
+ readWorkflowRecord,
15266
15275
  parsePolicy,
15267
15276
  parseAgentWorkflow,
15268
15277
  notContains,
@@ -15275,6 +15284,7 @@ export {
15275
15284
  listWorkflowTools,
15276
15285
  jsonSchema,
15277
15286
  jsonCase,
15287
+ isValidWorkflowJudgeOptions,
15278
15288
  isStressResult,
15279
15289
  isStressManifestType,
15280
15290
  isStressManifest,
@@ -15296,12 +15306,15 @@ export {
15296
15306
  executeSimulatedTool,
15297
15307
  exactCase,
15298
15308
  exact,
15309
+ evaluateWorkflowSemantics,
15310
+ evaluateWorkflowDeterministicOutcomes,
15299
15311
  detectPII,
15300
15312
  detectInjection,
15301
15313
  defineStorage,
15302
15314
  definePlugin,
15303
15315
  defineEvaluator,
15304
15316
  defineAdapter,
15317
+ decideWorkflowOutcome,
15305
15318
  createSimulatedWorkflowEnvironment,
15306
15319
  createSemanticValidator,
15307
15320
  createPIIGuardrail,
@@ -28,5 +28,5 @@ export type { ArtemisKitMatchers, MatcherResult } from './matchers/index.js';
28
28
  export type { ProviderName, ExpectationType, AnyResult, ExtractRunCases, ExtractRedTeamCases, ExtractStressResults, ExtractManifest, DeepPartial, PartialScenario, RequireFields, OptionalFields, StrictAdapterConfig, } from './utils/index.js';
29
29
  export type { AdapterContract, EvaluatorContract, StorageContract, AdapterFactory, EvaluatorFactory, StorageFactory, ArtemisKitPlugin, } from './contracts/index.js';
30
30
  export type { ScenarioBuilder, TestCaseBuilder } from './builders/index.js';
31
- export type { AgentWorkflow, WorkflowPolicy, WorkflowJson, WorkflowToolId, WorkflowResource, WorkflowToolDescriptor, SimulatedToolEvidence, SimulatedToolResult, AgentTarget, AgentTurnRequest, AgentTurnResult, AgentTargetCapabilities, AgentTargetFailure, AgentWorkflowExecution, AgentWorkflowReason, AgentWorkflowEvent, AgentWorkflowRecord, AgentWorkflowResult, AgentWorkflowSessionOptions, AgentWorkflowSession, WorkflowState, WorkflowEnvironment, WorkflowEnvironmentFactory, WorkflowEnvironmentCleanup, DockerWorkflowEnvironmentOptions, WorkflowEnvironmentInitializationError, } from '@artemiskit/core';
31
+ export type { AgentWorkflow, WorkflowPolicy, WorkflowJson, WorkflowToolId, WorkflowResource, WorkflowToolDescriptor, SimulatedToolEvidence, SimulatedToolResult, AgentTarget, AgentTurnRequest, AgentTurnResult, AgentTargetCapabilities, AgentTargetFailure, AgentWorkflowExecution, AgentWorkflowReason, AgentWorkflowEvent, AgentWorkflowRecord, AgentWorkflowExecutionRecord, AgentWorkflowEvidence, WorkflowDeterministicStatus, WorkflowDeterministicReason, WorkflowDeterministicAssertion, WorkflowDeterministicSummary, WorkflowJudgeOptions, WorkflowSemanticOptions, WorkflowSemanticReason, WorkflowSemanticAssertion, WorkflowSemanticIdentity, WorkflowSemanticEvaluation, WorkflowOutcomeStatus, WorkflowAssertionCounts, WorkflowOutcomeDecision, WorkflowOutcomeAssessment, SavedWorkflowRecord, HistoricalWorkflowRecord, AgentWorkflowResult, AgentWorkflowSessionOptions, AgentWorkflowSession, WorkflowState, WorkflowEnvironment, WorkflowEnvironmentFactory, WorkflowEnvironmentCleanup, DockerWorkflowEnvironmentOptions, WorkflowEnvironmentInitializationError, } from '@artemiskit/core';
32
32
  //# sourceMappingURL=types-only.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"types-only.d.ts","sourceRoot":"","sources":["../src/types-only.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAMH,YAAY,EAEV,gBAAgB,EAChB,kBAAkB,EAClB,UAAU,EACV,cAAc,EACd,aAAa,EAEb,SAAS,EACT,aAAa,EACb,YAAY,EAEZ,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,yBAAyB,EACzB,4BAA4B,EAC5B,0BAA0B,EAE1B,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAE5B,gBAAgB,EAChB,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,cAAc,EACd,eAAe,EACf,SAAS,GACV,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,UAAU,EACV,iBAAiB,EAEjB,WAAW,EACX,SAAS,EACT,UAAU,EAEV,eAAe,EACf,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,eAAe,EAEf,cAAc,EACd,YAAY,EACZ,aAAa,EACb,mBAAmB,EAEnB,WAAW,EACX,cAAc,EACd,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,OAAO,EACP,cAAc,GACf,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,WAAW,EACX,iBAAiB,EAEjB,eAAe,EACf,cAAc,EACd,UAAU,EAEV,WAAW,EAEX,kBAAkB,EAClB,cAAc,EACd,QAAQ,EAER,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,YAAY,GACb,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMrF,YAAY,EACV,cAAc,EACd,aAAa,EACb,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EAEtB,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EACtB,eAAe,EACf,mBAAmB,EACnB,cAAc,EACd,uBAAuB,GACxB,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMxD,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAM7E,YAAY,EAEV,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,sBAAsB,EACtB,gBAAgB,EAChB,iBAAiB,EAEjB,YAAY,EACZ,iBAAiB,EACjB,eAAe,EACf,aAAa,EACb,SAAS,EACT,eAAe,EAEf,UAAU,EACV,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,eAAe,EACf,eAAe,EAEf,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EAEnB,oBAAoB,EACpB,cAAc,EAEd,YAAY,EACZ,OAAO,EACP,WAAW,EACX,kBAAkB,EAClB,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,eAAe,EAEf,eAAe,EACf,mBAAmB,EACnB,YAAY,EAEZ,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EAEpB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EAEpB,aAAa,EACb,0BAA0B,GAC3B,MAAM,YAAY,CAAC;AAMpB,YAAY,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAMpE,YAAY,EAEV,YAAY,EACZ,eAAe,EAEf,SAAS,EACT,eAAe,EACf,mBAAmB,EACnB,oBAAoB,EACpB,eAAe,EAEf,WAAW,EACX,eAAe,EACf,aAAa,EACb,cAAc,EACd,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,aAAa,CAAC;AAMrB,YAAY,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAEnE,YAAY,EACV,aAAa,EACb,cAAc,EACd,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,sBAAsB,EACtB,qBAAqB,EACrB,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,eAAe,EACf,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,2BAA2B,EAC3B,oBAAoB,EACpB,aAAa,EACb,mBAAmB,EACnB,0BAA0B,EAC1B,0BAA0B,EAC1B,gCAAgC,EAChC,sCAAsC,GACvC,MAAM,kBAAkB,CAAC"}
1
+ {"version":3,"file":"types-only.d.ts","sourceRoot":"","sources":["../src/types-only.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAMH,YAAY,EAEV,gBAAgB,EAChB,kBAAkB,EAClB,UAAU,EACV,cAAc,EACd,aAAa,EAEb,SAAS,EACT,aAAa,EACb,YAAY,EAEZ,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,yBAAyB,EACzB,4BAA4B,EAC5B,0BAA0B,EAE1B,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAE5B,gBAAgB,EAChB,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,cAAc,EACd,eAAe,EACf,SAAS,GACV,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,UAAU,EACV,iBAAiB,EAEjB,WAAW,EACX,SAAS,EACT,UAAU,EAEV,eAAe,EACf,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,eAAe,EAEf,cAAc,EACd,YAAY,EACZ,aAAa,EACb,mBAAmB,EAEnB,WAAW,EACX,cAAc,EACd,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,OAAO,EACP,cAAc,GACf,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,WAAW,EACX,iBAAiB,EAEjB,eAAe,EACf,cAAc,EACd,UAAU,EAEV,WAAW,EAEX,kBAAkB,EAClB,cAAc,EACd,QAAQ,EAER,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,YAAY,GACb,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMrF,YAAY,EACV,cAAc,EACd,aAAa,EACb,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EAEtB,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EACtB,eAAe,EACf,mBAAmB,EACnB,cAAc,EACd,uBAAuB,GACxB,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMxD,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAM7E,YAAY,EAEV,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,sBAAsB,EACtB,gBAAgB,EAChB,iBAAiB,EAEjB,YAAY,EACZ,iBAAiB,EACjB,eAAe,EACf,aAAa,EACb,SAAS,EACT,eAAe,EAEf,UAAU,EACV,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,eAAe,EACf,eAAe,EAEf,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EAEnB,oBAAoB,EACpB,cAAc,EAEd,YAAY,EACZ,OAAO,EACP,WAAW,EACX,kBAAkB,EAClB,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,eAAe,EAEf,eAAe,EACf,mBAAmB,EACnB,YAAY,EAEZ,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EAEpB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EAEpB,aAAa,EACb,0BAA0B,GAC3B,MAAM,YAAY,CAAC;AAMpB,YAAY,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAMpE,YAAY,EAEV,YAAY,EACZ,eAAe,EAEf,SAAS,EACT,eAAe,EACf,mBAAmB,EACnB,oBAAoB,EACpB,eAAe,EAEf,WAAW,EACX,eAAe,EACf,aAAa,EACb,cAAc,EACd,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,aAAa,CAAC;AAMrB,YAAY,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAEnE,YAAY,EACV,aAAa,EACb,cAAc,EACd,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,sBAAsB,EACtB,qBAAqB,EACrB,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,eAAe,EACf,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,kBAAkB,EAClB,mBAAmB,EACnB,4BAA4B,EAC5B,qBAAqB,EACrB,2BAA2B,EAC3B,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAC5B,oBAAoB,EACpB,uBAAuB,EACvB,sBAAsB,EACtB,yBAAyB,EACzB,wBAAwB,EACxB,0BAA0B,EAC1B,qBAAqB,EACrB,uBAAuB,EACvB,uBAAuB,EACvB,yBAAyB,EACzB,mBAAmB,EACnB,wBAAwB,EACxB,mBAAmB,EACnB,2BAA2B,EAC3B,oBAAoB,EACpB,aAAa,EACb,mBAAmB,EACnB,0BAA0B,EAC1B,0BAA0B,EAC1B,gCAAgC,EAChC,sCAAsC,GACvC,MAAM,kBAAkB,CAAC"}
@@ -1 +1 @@
1
- {"version":3,"file":"workflow.d.ts","sourceRoot":"","sources":["../src/workflow.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,KAAK,oBAAoB,EAO1B,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAEpE,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,kBAAkB,EAC3B,MAAM,EAAE,gBAAgB,GACvB,OAAO,CAAC,oBAAoB,CAAC,CAkC/B"}
1
+ {"version":3,"file":"workflow.d.ts","sourceRoot":"","sources":["../src/workflow.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,KAAK,oBAAoB,EAO1B,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAEpE,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,kBAAkB,EAC3B,MAAM,EAAE,gBAAgB,GACvB,OAAO,CAAC,oBAAoB,CAAC,CAmC/B"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@artemiskit/sdk",
3
- "version": "0.6.0",
3
+ "version": "0.6.1",
4
4
  "description": "Programmatic SDK for ArtemisKit LLM evaluation toolkit - integrate into your Node.js apps",
5
5
  "type": "module",
6
6
  "license": "Apache-2.0",
@@ -71,8 +71,8 @@
71
71
  "test:coverage": "bun test --coverage"
72
72
  },
73
73
  "dependencies": {
74
- "@artemiskit/core": "0.6.1",
75
- "@artemiskit/redteam": "0.3.9"
74
+ "@artemiskit/core": "0.6.2",
75
+ "@artemiskit/redteam": "0.3.10"
76
76
  },
77
77
  "devDependencies": {
78
78
  "@types/bun": "^1.1.0",
@@ -12,7 +12,8 @@ import {
12
12
  } from '@artemiskit/core';
13
13
  import { OpenAIAdapter } from '../../../adapters/openai/src/client';
14
14
  import { ArtemisKit } from '../artemiskit';
15
- import type { WorkflowRunOptions } from '../types-only';
15
+ import { readWorkflowRecord } from '../index';
16
+ import type { WorkflowJudgeOptions, WorkflowRunOptions } from '../types-only';
16
17
 
17
18
  const servers: ReturnType<typeof Bun.serve>[] = [];
18
19
  afterEach(() => {
@@ -63,6 +64,152 @@ function target(): AgentTarget {
63
64
  }
64
65
 
65
66
  describe('SDK native workflow session', () => {
67
+ test('forwards explicit independent semantic judging with separate coverage, usage and record reading', async () => {
68
+ for (const verdict of ['pass', 'fail', 'malformed', 'unconfigured']) {
69
+ const scenario = workflow();
70
+ scenario.outcomes.deterministic = [{ type: 'file', path: 'result.txt', exists: false }];
71
+ scenario.outcomes.semantic = [
72
+ { type: 'llm_judge', mode: 'strict_assurance', rubric: 'PRIVATE-RUBRIC' },
73
+ ];
74
+ let calls = 0;
75
+ let closed = 0;
76
+ const semanticJudge: WorkflowJudgeOptions = {
77
+ provider: 'independent-judge',
78
+ model: 'judge-model',
79
+ limits: { maxRequests: 1, maxTokens: 100, maxOutputTokens: 10, timeoutMs: 1000 },
80
+ client: {
81
+ provider: 'independent-judge',
82
+ capabilities: async () => ({
83
+ streaming: false,
84
+ functionCalling: false,
85
+ toolUse: false,
86
+ maxContext: 10000,
87
+ }),
88
+ generate: async (request) => {
89
+ calls++;
90
+ expect(request.model).toBe('judge-model');
91
+ expect(request.maxRetries).toBe(0);
92
+ return {
93
+ id: 'judge',
94
+ model: 'observed-judge',
95
+ text: verdict === 'malformed' ? 'PRIVATE-BAD-RESPONSE' : JSON.stringify({ verdict }),
96
+ tokens: { prompt: 5, completion: 2, total: 7 },
97
+ latencyMs: 1,
98
+ };
99
+ },
100
+ close: async () => {
101
+ closed++;
102
+ },
103
+ },
104
+ };
105
+ const result = await new ArtemisKit().runWorkflow({
106
+ workflow: scenario,
107
+ target: target(),
108
+ ...(verdict === 'unconfigured' ? {} : { semanticJudge }),
109
+ });
110
+ expect(result.record.execution).toBe('completed');
111
+ expect(result.record.taskVerification).toBe(
112
+ verdict === 'pass'
113
+ ? 'passed'
114
+ : verdict === 'fail'
115
+ ? 'failed'
116
+ : verdict === 'malformed'
117
+ ? 'invalid'
118
+ : 'unavailable'
119
+ );
120
+ expect(result.record.usage.reported.total).toBe(3);
121
+ expect(result.record.outcomes.deterministic.counts.valid).toBe(1);
122
+ expect(result.record.outcomes.semantic.usage.reported.total).toBe(
123
+ verdict === 'unconfigured' ? 0 : 7
124
+ );
125
+ expect(calls).toBe(verdict === 'unconfigured' ? 0 : 1);
126
+ expect(closed).toBe(0);
127
+ expect(readWorkflowRecord(JSON.parse(JSON.stringify(result.record)))).toEqual(result.record);
128
+ expect(JSON.stringify(result.record)).not.toContain('PRIVATE-');
129
+ }
130
+ });
131
+ test('malformed explicit judge bounds stop before any target model call', async () => {
132
+ const scenario = workflow();
133
+ let turns = 0;
134
+ const custom = target();
135
+ custom.turn = async () => {
136
+ turns++;
137
+ throw new Error('must not call');
138
+ };
139
+ const result = await new ArtemisKit().runWorkflow({
140
+ workflow: scenario,
141
+ target: custom,
142
+ semanticJudge: {
143
+ provider: 'judge',
144
+ model: 'judge',
145
+ limits: { maxRequests: 0, maxTokens: 100, maxOutputTokens: 10, timeoutMs: 1000 },
146
+ client: {
147
+ provider: 'judge',
148
+ capabilities: async () => {
149
+ throw new Error('must not call');
150
+ },
151
+ generate: async () => {
152
+ throw new Error('must not call');
153
+ },
154
+ },
155
+ },
156
+ });
157
+ expect(turns).toBe(0);
158
+ expect(result.record.execution).toBe('invalid');
159
+ });
160
+ test('deterministic failure gates judge calls and session cancellation reaches the independent evaluator', async () => {
161
+ let started!: () => void;
162
+ const pending = new Promise<void>((resolve) => {
163
+ started = resolve;
164
+ });
165
+ let calls = 0;
166
+ const semanticJudge: WorkflowJudgeOptions = {
167
+ provider: 'judge',
168
+ model: 'judge',
169
+ limits: { maxRequests: 1, maxTokens: 100, maxOutputTokens: 10, timeoutMs: 1000 },
170
+ client: {
171
+ provider: 'judge',
172
+ capabilities: async () => ({
173
+ streaming: false,
174
+ functionCalling: false,
175
+ toolUse: false,
176
+ maxContext: 10000,
177
+ }),
178
+ generate: async (request) => {
179
+ calls++;
180
+ started();
181
+ await new Promise<void>((resolve) =>
182
+ request.signal?.addEventListener('abort', () => resolve(), { once: true })
183
+ );
184
+ throw new Error('PRIVATE-ABORT');
185
+ },
186
+ },
187
+ };
188
+ const scenario = workflow();
189
+ scenario.outcomes.semantic = [
190
+ { type: 'llm_judge', mode: 'strict_assurance', rubric: 'PRIVATE-RUBRIC' },
191
+ ];
192
+ const kit = new ArtemisKit();
193
+ const failed = await kit.runWorkflow({ workflow: scenario, target: target(), semanticJudge });
194
+ expect(failed.record.taskVerification).toBe('failed');
195
+ expect(calls).toBe(0);
196
+ scenario.outcomes.deterministic = [{ type: 'file', path: 'result.txt', exists: false }];
197
+ const session = await kit.createWorkflowSession({
198
+ workflow: scenario,
199
+ target: target(),
200
+ semanticJudge,
201
+ });
202
+ const finished = session.run();
203
+ await pending;
204
+ session.cancel();
205
+ const result = await finished;
206
+ expect(result.record.execution).toBe('completed');
207
+ expect(result.record.outcomes.cancelled).toBe(true);
208
+ expect(result.record.outcomes.reason).toBe('cancelled');
209
+ expect(result.record.outcomes.semantic.counts.valid).toBe(0);
210
+ expect(readWorkflowRecord(result.record)).toEqual(result.record);
211
+ });
212
+
66
213
  test('configured adapter uses workflow identity and zero retries with real loopback multi-turn execution', async () => {
67
214
  const bodies: Record<string, unknown>[] = [];
68
215
  const server = Bun.serve({
@@ -154,7 +301,7 @@ describe('SDK native workflow session', () => {
154
301
  const { result, events } = JSON.parse(stdout);
155
302
  expect(bodies[0].model).toBe('workflow-authoritative');
156
303
  expect(result.record.execution).toBe('completed');
157
- expect(result.record.taskVerification).toBe('unavailable');
304
+ expect(result.record.taskVerification).toBe('passed');
158
305
  expect(result.record.budgets).toMatchObject({ modelRequests: 2, toolCalls: 1 });
159
306
  expect(result.state?.files).toEqual({ 'result.txt': 'PRIVATE-STATE' });
160
307
  expect(JSON.stringify(result.record)).not.toContain('PRIVATE-');
package/src/index.ts CHANGED
@@ -20,6 +20,12 @@ export {
20
20
  createModelClientTarget,
21
21
  createAgentWorkflowSession,
22
22
  runAgentWorkflow,
23
+ evaluateWorkflowDeterministicOutcomes,
24
+ evaluateWorkflowSemantics,
25
+ isValidWorkflowJudgeOptions,
26
+ decideWorkflowOutcome,
27
+ validWorkflowAssertionCounts,
28
+ readWorkflowRecord,
23
29
  createSimulatedWorkflowEnvironment,
24
30
  createDockerWorkflowEnvironment,
25
31
  createDockerWorkflowEnvironmentFactory,
@@ -44,6 +50,24 @@ export type {
44
50
  AgentWorkflowReason,
45
51
  AgentWorkflowEvent,
46
52
  AgentWorkflowRecord,
53
+ AgentWorkflowExecutionRecord,
54
+ AgentWorkflowEvidence,
55
+ WorkflowDeterministicStatus,
56
+ WorkflowDeterministicReason,
57
+ WorkflowDeterministicAssertion,
58
+ WorkflowDeterministicSummary,
59
+ WorkflowJudgeOptions,
60
+ WorkflowSemanticOptions,
61
+ WorkflowSemanticReason,
62
+ WorkflowSemanticAssertion,
63
+ WorkflowSemanticIdentity,
64
+ WorkflowSemanticEvaluation,
65
+ WorkflowOutcomeStatus,
66
+ WorkflowAssertionCounts,
67
+ WorkflowOutcomeDecision,
68
+ WorkflowOutcomeAssessment,
69
+ SavedWorkflowRecord,
70
+ HistoricalWorkflowRecord,
47
71
  AgentWorkflowResult,
48
72
  AgentWorkflowSessionOptions,
49
73
  AgentWorkflowSession,
package/src/types-only.ts CHANGED
@@ -294,6 +294,24 @@ export type {
294
294
  AgentWorkflowReason,
295
295
  AgentWorkflowEvent,
296
296
  AgentWorkflowRecord,
297
+ AgentWorkflowExecutionRecord,
298
+ AgentWorkflowEvidence,
299
+ WorkflowDeterministicStatus,
300
+ WorkflowDeterministicReason,
301
+ WorkflowDeterministicAssertion,
302
+ WorkflowDeterministicSummary,
303
+ WorkflowJudgeOptions,
304
+ WorkflowSemanticOptions,
305
+ WorkflowSemanticReason,
306
+ WorkflowSemanticAssertion,
307
+ WorkflowSemanticIdentity,
308
+ WorkflowSemanticEvaluation,
309
+ WorkflowOutcomeStatus,
310
+ WorkflowAssertionCounts,
311
+ WorkflowOutcomeDecision,
312
+ WorkflowOutcomeAssessment,
313
+ SavedWorkflowRecord,
314
+ HistoricalWorkflowRecord,
297
315
  AgentWorkflowResult,
298
316
  AgentWorkflowSessionOptions,
299
317
  AgentWorkflowSession,
package/src/workflow.ts CHANGED
@@ -46,6 +46,7 @@ export async function prepareWorkflowSession(
46
46
  preflightOnly: options.preflightOnly,
47
47
  signal: options.signal,
48
48
  cleanupTimeoutMs: options.cleanupTimeoutMs,
49
+ semanticJudge: options.semanticJudge,
49
50
  onEvent: options.onEvent,
50
51
  });
51
52
  }