@artemiskit/sdk 0.6.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -0
- package/README.md +6 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +13 -0
- package/dist/types-only.d.ts +1 -1
- package/dist/types-only.d.ts.map +1 -1
- package/dist/workflow.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/__tests__/workflow.test.ts +149 -2
- package/src/index.ts +24 -0
- package/src/types-only.ts +18 -0
- package/src/workflow.ts +1 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,14 @@
|
|
|
1
1
|
# @artemiskit/sdk
|
|
2
2
|
|
|
3
|
+
## 0.6.1
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- Complete the 0.6.2 independent workflow outcome milestone: verify final artifacts, bounded JSON schemas, state, tool traces and policy before optional explicitly configured semantic judging. Keep target and judge identity, usage and limits separate; expose task validity and coverage, strict V1/V2 saved-record reading, SDK composition and CLI outcome exits without changing historical prompt-response reports.
|
|
8
|
+
- Updated dependencies
|
|
9
|
+
- @artemiskit/core@0.6.2
|
|
10
|
+
- @artemiskit/redteam@0.3.10
|
|
11
|
+
|
|
3
12
|
## 0.6.0
|
|
4
13
|
|
|
5
14
|
### Minor Changes
|
package/README.md
CHANGED
|
@@ -57,7 +57,7 @@ const session = await kit.createWorkflowSession({
|
|
|
57
57
|
preflight: true,
|
|
58
58
|
});
|
|
59
59
|
const result = await session.run();
|
|
60
|
-
console.log(result.record); // bounded metadata
|
|
60
|
+
console.log(result.record); // bounded metadata with separate execution and task outcomes
|
|
61
61
|
// session.cancel() stops new work and initiates bounded cleanup.
|
|
62
62
|
// result.state and result.transcript contain sensitive working data.
|
|
63
63
|
```
|
|
@@ -66,7 +66,11 @@ console.log(result.record); // bounded metadata; taskVerification is unavailable
|
|
|
66
66
|
`target` to use an existing trusted adapter or target. The same native controls apply through the
|
|
67
67
|
CLI and SDK: declared tools and permissions, cumulative budgets, explicit preflight, cancellation,
|
|
68
68
|
and cleanup evidence. Workflows use fresh simulated state or a disposable Docker filesystem.
|
|
69
|
-
|
|
69
|
+
The 0.6.2 milestone independently verifies declared artifacts, state, schemas, traces and policy.
|
|
70
|
+
Use explicit `semanticJudge` configuration for declared qualitative criteria. Target and judge usage
|
|
71
|
+
remain separate; `readWorkflowRecord` validates V1/V2 saved metadata without making model calls.
|
|
72
|
+
See [workflow outcomes](https://github.com/code-sensei/artemiskit/blob/main/docs/workflow-outcomes.md)
|
|
73
|
+
for usage, denominators, compatibility and publication status.
|
|
70
74
|
|
|
71
75
|
The root and `/types` entry points expose workflow result/session/event types. Low-level
|
|
72
76
|
`createAgentWorkflowSession` and `runAgentWorkflow` are available for explicit target composition.
|
package/dist/index.d.ts
CHANGED
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
* Programmatic SDK for ArtemisKit LLM evaluation toolkit
|
|
4
4
|
*/
|
|
5
5
|
export { ArtemisKit } from './artemiskit.js';
|
|
6
|
-
export { AgentWorkflowSchema, WorkflowPolicySchema, WORKFLOW_TOOL_IDS, validateAgentWorkflow, parseAgentWorkflow, loadAgentWorkflow, listWorkflowTools, getWorkflowTool, executeSimulatedTool, createModelClientTarget, createAgentWorkflowSession, runAgentWorkflow, createSimulatedWorkflowEnvironment, createDockerWorkflowEnvironment, createDockerWorkflowEnvironmentFactory, WORKFLOW_SANDBOX_IMAGE, WorkflowEnvironmentInitializationError, } from '@artemiskit/core';
|
|
7
|
-
export type { AgentWorkflow, WorkflowPolicy, WorkflowJson, WorkflowToolId, WorkflowResource, WorkflowToolDescriptor, SimulatedToolEvidence, SimulatedToolResult, AgentTarget, AgentTurnRequest, AgentTurnResult, AgentTargetCapabilities, AgentTargetFailure, AgentWorkflowExecution, AgentWorkflowReason, AgentWorkflowEvent, AgentWorkflowRecord, AgentWorkflowResult, AgentWorkflowSessionOptions, AgentWorkflowSession, WorkflowState, WorkflowEnvironment, WorkflowEnvironmentFactory, WorkflowEnvironmentCleanup, DockerWorkflowEnvironmentOptions, } from '@artemiskit/core';
|
|
6
|
+
export { AgentWorkflowSchema, WorkflowPolicySchema, WORKFLOW_TOOL_IDS, validateAgentWorkflow, parseAgentWorkflow, loadAgentWorkflow, listWorkflowTools, getWorkflowTool, executeSimulatedTool, createModelClientTarget, createAgentWorkflowSession, runAgentWorkflow, evaluateWorkflowDeterministicOutcomes, evaluateWorkflowSemantics, isValidWorkflowJudgeOptions, decideWorkflowOutcome, validWorkflowAssertionCounts, readWorkflowRecord, createSimulatedWorkflowEnvironment, createDockerWorkflowEnvironment, createDockerWorkflowEnvironmentFactory, WORKFLOW_SANDBOX_IMAGE, WorkflowEnvironmentInitializationError, } from '@artemiskit/core';
|
|
7
|
+
export type { AgentWorkflow, WorkflowPolicy, WorkflowJson, WorkflowToolId, WorkflowResource, WorkflowToolDescriptor, SimulatedToolEvidence, SimulatedToolResult, AgentTarget, AgentTurnRequest, AgentTurnResult, AgentTargetCapabilities, AgentTargetFailure, AgentWorkflowExecution, AgentWorkflowReason, AgentWorkflowEvent, AgentWorkflowRecord, AgentWorkflowExecutionRecord, AgentWorkflowEvidence, WorkflowDeterministicStatus, WorkflowDeterministicReason, WorkflowDeterministicAssertion, WorkflowDeterministicSummary, WorkflowJudgeOptions, WorkflowSemanticOptions, WorkflowSemanticReason, WorkflowSemanticAssertion, WorkflowSemanticIdentity, WorkflowSemanticEvaluation, WorkflowOutcomeStatus, WorkflowAssertionCounts, WorkflowOutcomeDecision, WorkflowOutcomeAssessment, SavedWorkflowRecord, HistoricalWorkflowRecord, AgentWorkflowResult, AgentWorkflowSessionOptions, AgentWorkflowSession, WorkflowState, WorkflowEnvironment, WorkflowEnvironmentFactory, WorkflowEnvironmentCleanup, DockerWorkflowEnvironmentOptions, } from '@artemiskit/core';
|
|
8
8
|
export type { ArtemisKitConfig, WorkflowRunOptions, RunOptions, RedTeamOptions, StressOptions, RunResult, RedTeamResult, StressResult, CaseStartEvent, CaseCompleteEvent, ProgressEvent, RedTeamMutationStartEvent, RedTeamMutationCompleteEvent, StressRequestCompleteEvent, CaseStartHandler, CaseCompleteHandler, ProgressHandler, RedTeamMutationStartHandler, RedTeamMutationCompleteHandler, StressRequestCompleteHandler, ArtemisKitEvents, ArtemisKitEventName, ValidateOptions, ValidationResult, ValidationError, ValidationWarning, ScenarioValidation, CompareOptions, CompareResult, RunSummary, ComparisonDetails, } from './types.js';
|
|
9
9
|
export type { Scenario, TestCase, Expected, Provider, ProviderConfig, ChatMessageType, Variables, } from '@artemiskit/core';
|
|
10
10
|
export type { CaseResult, CaseRedactionInfo, RunManifest, RunConfig, RunMetrics, RedTeamManifest, RedTeamConfig, RedTeamMetrics, RedTeamCaseResult, RedTeamStatus, RedTeamSeverity, StressManifest, StressConfig, StressMetrics, StressRequestResult, AnyManifest, ResolvedConfig, ConfigSource, ManifestRedactionInfo, CostEstimateInfo, GitInfo, ProvenanceInfo, } from '@artemiskit/core';
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAG1C,OAAO,EACL,mBAAmB,EACnB,oBAAoB,EACpB,iBAAiB,EACjB,qBAAqB,EACrB,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,eAAe,EACf,oBAAoB,EACpB,uBAAuB,EACvB,0BAA0B,EAC1B,gBAAgB,EAChB,kCAAkC,EAClC,+BAA+B,EAC/B,sCAAsC,EACtC,sBAAsB,EACtB,sCAAsC,GACvC,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,aAAa,EACb,cAAc,EACd,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,sBAAsB,EACtB,qBAAqB,EACrB,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,eAAe,EACf,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,2BAA2B,EAC3B,oBAAoB,EACpB,aAAa,EACb,mBAAmB,EACnB,0BAA0B,EAC1B,0BAA0B,EAC1B,gCAAgC,GACjC,MAAM,kBAAkB,CAAC;AAG1B,YAAY,EAEV,gBAAgB,EAChB,kBAAkB,EAClB,UAAU,EACV,cAAc,EACd,aAAa,EAEb,SAAS,EACT,aAAa,EACb,YAAY,EAEZ,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,yBAAyB,EACzB,4BAA4B,EAC5B,0BAA0B,EAE1B,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAE5B,gBAAgB,EAChB,mBAAmB,EAEnB,eAAe,EACf,gBAAgB,EAChB,eAAe,EACf,iBAAiB,EACjB,kBAAkB,EAElB,cAAc,EACd,aAAa,EACb,UAAU,EACV,iBAAiB,GAClB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,cAAc,EACd,eAAe,EACf,SAAS,GACV,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,UAAU,EACV,iBAAiB,EAEjB,WAAW,EACX,SAAS,EACT,UAAU,EAEV,eAAe,EACf,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,eAAe,EAEf,cAAc,EACd,YAAY,EACZ,aAAa,EACb,mBAAmB,EAEnB,WAAW,EACX,cAAc,EACd,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,OAAO,EACP,cAAc,GACf,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,WAAW,EACX,iBAAiB,EAEjB,eAAe,EACf,cAAc,EACd,UAAU,EAEV,WAAW,EAEX,kBAAkB,EAClB,cAAc,EACd,QAAQ,EAER,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,YAAY,GACb,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMrF,YAAY,EACV,cAAc,EACd,aAAa,EACb,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EAEtB,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EACtB,eAAe,EACf,mBAAmB,EACnB,cAAc,EACd,uBAAuB,GACxB,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMxD,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAMtF,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAM7E,OAAO,EAAE,kBAAkB,EAAE,KAAK,kBAAkB,EAAE,KAAK,aAAa,EAAE,MAAM,YAAY,CAAC;AAM7F,OAAO,EAEL,WAAW,EACX,eAAe,EACf,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,oBAAoB,EAEpB,aAAa,EACb,MAAM,EAEN,cAAc,EACd,cAAc,EACd,aAAa,EACb,oBAAoB,GACrB,MAAM,SAAS,CAAC;AAEjB,YAAY,EAEV,YAAY,EACZ,eAAe,EAEf,SAAS,EACT,eAAe,EACf,mBAAmB,EACnB,oBAAoB,EACpB,eAAe,EAEf,WAAW,EACX,eAAe,EACf,aAAa,EACb,cAAc,EACd,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,OAAO,EAEL,aAAa,EAEb,eAAe,EAEf,aAAa,EAEb,YAAY,GACb,MAAM,aAAa,CAAC;AAErB,YAAY,EACV,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,aAAa,CAAC;AAMrB,OAAO,EAEL,eAAe,EACf,eAAe,EAEf,QAAQ,EACR,QAAQ,EAER,YAAY,EACZ,SAAS,EACT,SAAS,EACT,QAAQ,EACR,UAAU,EAEV,KAAK,EACL,QAAQ,EACR,WAAW,EACX,KAAK,EACL,KAAK,EACL,UAAU,EACV,QAAQ,EACR,UAAU,EACV,MAAM,EACN,KAAK,EACL,KAAK,GACN,MAAM,YAAY,CAAC;AAMpB,OAAO,EAEL,QAAQ,EACR,cAAc,EAEd,qBAAqB,EAErB,iBAAiB,EACjB,uBAAuB,EAEvB,mBAAmB,EACnB,oBAAoB,EACpB,iBAAiB,EAEjB,eAAe,EACf,4BAA4B,EAE5B,gBAAgB,EAChB,sBAAsB,EAEtB,eAAe,EACf,wBAAwB,EACxB,SAAS,EACT,kBAAkB,EAClB,aAAa,EACb,4BAA4B,EAC5B,gBAAgB,EAEhB,YAAY,EACZ,4BAA4B,EAE5B,UAAU,EACV,WAAW,EACX,cAAc,EACd,mBAAmB,EACnB,aAAa,EACb,sBAAsB,EACtB,eAAe,EACf,qBAAqB,EAErB,cAAc,EACd,gBAAgB,EAChB,WAAW,GACZ,MAAM,YAAY,CAAC;AAGpB,YAAY,EAEV,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,sBAAsB,EACtB,gBAAgB,EAChB,iBAAiB,EACjB,oBAAoB,EAEpB,YAAY,EACZ,iBAAiB,EACjB,eAAe,EACf,aAAa,EACb,SAAS,EACT,eAAe,EAEf,UAAU,EACV,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,eAAe,EACf,eAAe,EAEf,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EAEnB,oBAAoB,EACpB,cAAc,EAEd,YAAY,EACZ,OAAO,EACP,WAAW,EACX,kBAAkB,EAClB,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,eAAe,EAEf,eAAe,EACf,mBAAmB,EACnB,YAAY,EAEZ,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EAEpB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EAEpB,aAAa,EACb,0BAA0B,EAE1B,qBAAqB,EACrB,kBAAkB,EAClB,eAAe,EAEf,uBAAuB,EACvB,aAAa,EACb,kBAAkB,EAClB,eAAe,EACf,wBAAwB,EAExB,kBAAkB,EAClB,oBAAoB,EACpB,cAAc,EACd,cAAc,EACd,mBAAmB,EACnB,4BAA4B,EAC5B,yBAAyB,EACzB,kCAAkC,EAClC,2BAA2B,EAC3B,mBAAmB,EACnB,uBAAuB,EACvB,eAAe,EACf,eAAe,EACf,gBAAgB,EAChB,yBAAyB,EACzB,sBAAsB,GACvB,MAAM,YAAY,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAG1C,OAAO,EACL,mBAAmB,EACnB,oBAAoB,EACpB,iBAAiB,EACjB,qBAAqB,EACrB,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,eAAe,EACf,oBAAoB,EACpB,uBAAuB,EACvB,0BAA0B,EAC1B,gBAAgB,EAChB,qCAAqC,EACrC,yBAAyB,EACzB,2BAA2B,EAC3B,qBAAqB,EACrB,4BAA4B,EAC5B,kBAAkB,EAClB,kCAAkC,EAClC,+BAA+B,EAC/B,sCAAsC,EACtC,sBAAsB,EACtB,sCAAsC,GACvC,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,aAAa,EACb,cAAc,EACd,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,sBAAsB,EACtB,qBAAqB,EACrB,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,eAAe,EACf,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,kBAAkB,EAClB,mBAAmB,EACnB,4BAA4B,EAC5B,qBAAqB,EACrB,2BAA2B,EAC3B,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAC5B,oBAAoB,EACpB,uBAAuB,EACvB,sBAAsB,EACtB,yBAAyB,EACzB,wBAAwB,EACxB,0BAA0B,EAC1B,qBAAqB,EACrB,uBAAuB,EACvB,uBAAuB,EACvB,yBAAyB,EACzB,mBAAmB,EACnB,wBAAwB,EACxB,mBAAmB,EACnB,2BAA2B,EAC3B,oBAAoB,EACpB,aAAa,EACb,mBAAmB,EACnB,0BAA0B,EAC1B,0BAA0B,EAC1B,gCAAgC,GACjC,MAAM,kBAAkB,CAAC;AAG1B,YAAY,EAEV,gBAAgB,EAChB,kBAAkB,EAClB,UAAU,EACV,cAAc,EACd,aAAa,EAEb,SAAS,EACT,aAAa,EACb,YAAY,EAEZ,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,yBAAyB,EACzB,4BAA4B,EAC5B,0BAA0B,EAE1B,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAE5B,gBAAgB,EAChB,mBAAmB,EAEnB,eAAe,EACf,gBAAgB,EAChB,eAAe,EACf,iBAAiB,EACjB,kBAAkB,EAElB,cAAc,EACd,aAAa,EACb,UAAU,EACV,iBAAiB,GAClB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,cAAc,EACd,eAAe,EACf,SAAS,GACV,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,UAAU,EACV,iBAAiB,EAEjB,WAAW,EACX,SAAS,EACT,UAAU,EAEV,eAAe,EACf,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,eAAe,EAEf,cAAc,EACd,YAAY,EACZ,aAAa,EACb,mBAAmB,EAEnB,WAAW,EACX,cAAc,EACd,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,OAAO,EACP,cAAc,GACf,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,WAAW,EACX,iBAAiB,EAEjB,eAAe,EACf,cAAc,EACd,UAAU,EAEV,WAAW,EAEX,kBAAkB,EAClB,cAAc,EACd,QAAQ,EAER,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,YAAY,GACb,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMrF,YAAY,EACV,cAAc,EACd,aAAa,EACb,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EAEtB,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EACtB,eAAe,EACf,mBAAmB,EACnB,cAAc,EACd,uBAAuB,GACxB,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMxD,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAMtF,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAM7E,OAAO,EAAE,kBAAkB,EAAE,KAAK,kBAAkB,EAAE,KAAK,aAAa,EAAE,MAAM,YAAY,CAAC;AAM7F,OAAO,EAEL,WAAW,EACX,eAAe,EACf,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,oBAAoB,EAEpB,aAAa,EACb,MAAM,EAEN,cAAc,EACd,cAAc,EACd,aAAa,EACb,oBAAoB,GACrB,MAAM,SAAS,CAAC;AAEjB,YAAY,EAEV,YAAY,EACZ,eAAe,EAEf,SAAS,EACT,eAAe,EACf,mBAAmB,EACnB,oBAAoB,EACpB,eAAe,EAEf,WAAW,EACX,eAAe,EACf,aAAa,EACb,cAAc,EACd,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,OAAO,EAEL,aAAa,EAEb,eAAe,EAEf,aAAa,EAEb,YAAY,GACb,MAAM,aAAa,CAAC;AAErB,YAAY,EACV,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,aAAa,CAAC;AAMrB,OAAO,EAEL,eAAe,EACf,eAAe,EAEf,QAAQ,EACR,QAAQ,EAER,YAAY,EACZ,SAAS,EACT,SAAS,EACT,QAAQ,EACR,UAAU,EAEV,KAAK,EACL,QAAQ,EACR,WAAW,EACX,KAAK,EACL,KAAK,EACL,UAAU,EACV,QAAQ,EACR,UAAU,EACV,MAAM,EACN,KAAK,EACL,KAAK,GACN,MAAM,YAAY,CAAC;AAMpB,OAAO,EAEL,QAAQ,EACR,cAAc,EAEd,qBAAqB,EAErB,iBAAiB,EACjB,uBAAuB,EAEvB,mBAAmB,EACnB,oBAAoB,EACpB,iBAAiB,EAEjB,eAAe,EACf,4BAA4B,EAE5B,gBAAgB,EAChB,sBAAsB,EAEtB,eAAe,EACf,wBAAwB,EACxB,SAAS,EACT,kBAAkB,EAClB,aAAa,EACb,4BAA4B,EAC5B,gBAAgB,EAEhB,YAAY,EACZ,4BAA4B,EAE5B,UAAU,EACV,WAAW,EACX,cAAc,EACd,mBAAmB,EACnB,aAAa,EACb,sBAAsB,EACtB,eAAe,EACf,qBAAqB,EAErB,cAAc,EACd,gBAAgB,EAChB,WAAW,GACZ,MAAM,YAAY,CAAC;AAGpB,YAAY,EAEV,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,sBAAsB,EACtB,gBAAgB,EAChB,iBAAiB,EACjB,oBAAoB,EAEpB,YAAY,EACZ,iBAAiB,EACjB,eAAe,EACf,aAAa,EACb,SAAS,EACT,eAAe,EAEf,UAAU,EACV,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,eAAe,EACf,eAAe,EAEf,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EAEnB,oBAAoB,EACpB,cAAc,EAEd,YAAY,EACZ,OAAO,EACP,WAAW,EACX,kBAAkB,EAClB,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,eAAe,EAEf,eAAe,EACf,mBAAmB,EACnB,YAAY,EAEZ,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EAEpB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EAEpB,aAAa,EACb,0BAA0B,EAE1B,qBAAqB,EACrB,kBAAkB,EAClB,eAAe,EAEf,uBAAuB,EACvB,aAAa,EACb,kBAAkB,EAClB,eAAe,EACf,wBAAwB,EAExB,kBAAkB,EAClB,oBAAoB,EACpB,cAAc,EACd,cAAc,EACd,mBAAmB,EACnB,4BAA4B,EAC5B,yBAAyB,EACzB,kCAAkC,EAClC,2BAA2B,EAC3B,mBAAmB,EACnB,uBAAuB,EACvB,eAAe,EACf,eAAe,EACf,gBAAgB,EAChB,yBAAyB,EACzB,sBAAsB,GACvB,MAAM,YAAY,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -7678,6 +7678,7 @@ async function prepareWorkflowSession(options, config) {
|
|
|
7678
7678
|
preflightOnly: options.preflightOnly,
|
|
7679
7679
|
signal: options.signal,
|
|
7680
7680
|
cleanupTimeoutMs: options.cleanupTimeoutMs,
|
|
7681
|
+
semanticJudge: options.semanticJudge,
|
|
7681
7682
|
onEvent: options.onEvent
|
|
7682
7683
|
});
|
|
7683
7684
|
}
|
|
@@ -8516,6 +8517,12 @@ import {
|
|
|
8516
8517
|
createModelClientTarget as createModelClientTarget2,
|
|
8517
8518
|
createAgentWorkflowSession as createAgentWorkflowSession2,
|
|
8518
8519
|
runAgentWorkflow,
|
|
8520
|
+
evaluateWorkflowDeterministicOutcomes,
|
|
8521
|
+
evaluateWorkflowSemantics,
|
|
8522
|
+
isValidWorkflowJudgeOptions,
|
|
8523
|
+
decideWorkflowOutcome,
|
|
8524
|
+
validWorkflowAssertionCounts,
|
|
8525
|
+
readWorkflowRecord,
|
|
8519
8526
|
createSimulatedWorkflowEnvironment,
|
|
8520
8527
|
createDockerWorkflowEnvironment,
|
|
8521
8528
|
createDockerWorkflowEnvironmentFactory,
|
|
@@ -15257,12 +15264,14 @@ function createGuardian(config = {}) {
|
|
|
15257
15264
|
export {
|
|
15258
15265
|
validatePolicy,
|
|
15259
15266
|
validateAgentWorkflow2 as validateAgentWorkflow,
|
|
15267
|
+
validWorkflowAssertionCounts,
|
|
15260
15268
|
testCase,
|
|
15261
15269
|
similarity,
|
|
15262
15270
|
scenario,
|
|
15263
15271
|
runAgentWorkflow,
|
|
15264
15272
|
regexCase,
|
|
15265
15273
|
regex,
|
|
15274
|
+
readWorkflowRecord,
|
|
15266
15275
|
parsePolicy,
|
|
15267
15276
|
parseAgentWorkflow,
|
|
15268
15277
|
notContains,
|
|
@@ -15275,6 +15284,7 @@ export {
|
|
|
15275
15284
|
listWorkflowTools,
|
|
15276
15285
|
jsonSchema,
|
|
15277
15286
|
jsonCase,
|
|
15287
|
+
isValidWorkflowJudgeOptions,
|
|
15278
15288
|
isStressResult,
|
|
15279
15289
|
isStressManifestType,
|
|
15280
15290
|
isStressManifest,
|
|
@@ -15296,12 +15306,15 @@ export {
|
|
|
15296
15306
|
executeSimulatedTool,
|
|
15297
15307
|
exactCase,
|
|
15298
15308
|
exact,
|
|
15309
|
+
evaluateWorkflowSemantics,
|
|
15310
|
+
evaluateWorkflowDeterministicOutcomes,
|
|
15299
15311
|
detectPII,
|
|
15300
15312
|
detectInjection,
|
|
15301
15313
|
defineStorage,
|
|
15302
15314
|
definePlugin,
|
|
15303
15315
|
defineEvaluator,
|
|
15304
15316
|
defineAdapter,
|
|
15317
|
+
decideWorkflowOutcome,
|
|
15305
15318
|
createSimulatedWorkflowEnvironment,
|
|
15306
15319
|
createSemanticValidator,
|
|
15307
15320
|
createPIIGuardrail,
|
package/dist/types-only.d.ts
CHANGED
|
@@ -28,5 +28,5 @@ export type { ArtemisKitMatchers, MatcherResult } from './matchers/index.js';
|
|
|
28
28
|
export type { ProviderName, ExpectationType, AnyResult, ExtractRunCases, ExtractRedTeamCases, ExtractStressResults, ExtractManifest, DeepPartial, PartialScenario, RequireFields, OptionalFields, StrictAdapterConfig, } from './utils/index.js';
|
|
29
29
|
export type { AdapterContract, EvaluatorContract, StorageContract, AdapterFactory, EvaluatorFactory, StorageFactory, ArtemisKitPlugin, } from './contracts/index.js';
|
|
30
30
|
export type { ScenarioBuilder, TestCaseBuilder } from './builders/index.js';
|
|
31
|
-
export type { AgentWorkflow, WorkflowPolicy, WorkflowJson, WorkflowToolId, WorkflowResource, WorkflowToolDescriptor, SimulatedToolEvidence, SimulatedToolResult, AgentTarget, AgentTurnRequest, AgentTurnResult, AgentTargetCapabilities, AgentTargetFailure, AgentWorkflowExecution, AgentWorkflowReason, AgentWorkflowEvent, AgentWorkflowRecord, AgentWorkflowResult, AgentWorkflowSessionOptions, AgentWorkflowSession, WorkflowState, WorkflowEnvironment, WorkflowEnvironmentFactory, WorkflowEnvironmentCleanup, DockerWorkflowEnvironmentOptions, WorkflowEnvironmentInitializationError, } from '@artemiskit/core';
|
|
31
|
+
export type { AgentWorkflow, WorkflowPolicy, WorkflowJson, WorkflowToolId, WorkflowResource, WorkflowToolDescriptor, SimulatedToolEvidence, SimulatedToolResult, AgentTarget, AgentTurnRequest, AgentTurnResult, AgentTargetCapabilities, AgentTargetFailure, AgentWorkflowExecution, AgentWorkflowReason, AgentWorkflowEvent, AgentWorkflowRecord, AgentWorkflowExecutionRecord, AgentWorkflowEvidence, WorkflowDeterministicStatus, WorkflowDeterministicReason, WorkflowDeterministicAssertion, WorkflowDeterministicSummary, WorkflowJudgeOptions, WorkflowSemanticOptions, WorkflowSemanticReason, WorkflowSemanticAssertion, WorkflowSemanticIdentity, WorkflowSemanticEvaluation, WorkflowOutcomeStatus, WorkflowAssertionCounts, WorkflowOutcomeDecision, WorkflowOutcomeAssessment, SavedWorkflowRecord, HistoricalWorkflowRecord, AgentWorkflowResult, AgentWorkflowSessionOptions, AgentWorkflowSession, WorkflowState, WorkflowEnvironment, WorkflowEnvironmentFactory, WorkflowEnvironmentCleanup, DockerWorkflowEnvironmentOptions, WorkflowEnvironmentInitializationError, } from '@artemiskit/core';
|
|
32
32
|
//# sourceMappingURL=types-only.d.ts.map
|
package/dist/types-only.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types-only.d.ts","sourceRoot":"","sources":["../src/types-only.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAMH,YAAY,EAEV,gBAAgB,EAChB,kBAAkB,EAClB,UAAU,EACV,cAAc,EACd,aAAa,EAEb,SAAS,EACT,aAAa,EACb,YAAY,EAEZ,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,yBAAyB,EACzB,4BAA4B,EAC5B,0BAA0B,EAE1B,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAE5B,gBAAgB,EAChB,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,cAAc,EACd,eAAe,EACf,SAAS,GACV,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,UAAU,EACV,iBAAiB,EAEjB,WAAW,EACX,SAAS,EACT,UAAU,EAEV,eAAe,EACf,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,eAAe,EAEf,cAAc,EACd,YAAY,EACZ,aAAa,EACb,mBAAmB,EAEnB,WAAW,EACX,cAAc,EACd,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,OAAO,EACP,cAAc,GACf,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,WAAW,EACX,iBAAiB,EAEjB,eAAe,EACf,cAAc,EACd,UAAU,EAEV,WAAW,EAEX,kBAAkB,EAClB,cAAc,EACd,QAAQ,EAER,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,YAAY,GACb,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMrF,YAAY,EACV,cAAc,EACd,aAAa,EACb,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EAEtB,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EACtB,eAAe,EACf,mBAAmB,EACnB,cAAc,EACd,uBAAuB,GACxB,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMxD,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAM7E,YAAY,EAEV,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,sBAAsB,EACtB,gBAAgB,EAChB,iBAAiB,EAEjB,YAAY,EACZ,iBAAiB,EACjB,eAAe,EACf,aAAa,EACb,SAAS,EACT,eAAe,EAEf,UAAU,EACV,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,eAAe,EACf,eAAe,EAEf,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EAEnB,oBAAoB,EACpB,cAAc,EAEd,YAAY,EACZ,OAAO,EACP,WAAW,EACX,kBAAkB,EAClB,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,eAAe,EAEf,eAAe,EACf,mBAAmB,EACnB,YAAY,EAEZ,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EAEpB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EAEpB,aAAa,EACb,0BAA0B,GAC3B,MAAM,YAAY,CAAC;AAMpB,YAAY,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAMpE,YAAY,EAEV,YAAY,EACZ,eAAe,EAEf,SAAS,EACT,eAAe,EACf,mBAAmB,EACnB,oBAAoB,EACpB,eAAe,EAEf,WAAW,EACX,eAAe,EACf,aAAa,EACb,cAAc,EACd,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,aAAa,CAAC;AAMrB,YAAY,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAEnE,YAAY,EACV,aAAa,EACb,cAAc,EACd,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,sBAAsB,EACtB,qBAAqB,EACrB,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,eAAe,EACf,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,2BAA2B,EAC3B,oBAAoB,EACpB,aAAa,EACb,mBAAmB,EACnB,0BAA0B,EAC1B,0BAA0B,EAC1B,gCAAgC,EAChC,sCAAsC,GACvC,MAAM,kBAAkB,CAAC"}
|
|
1
|
+
{"version":3,"file":"types-only.d.ts","sourceRoot":"","sources":["../src/types-only.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAMH,YAAY,EAEV,gBAAgB,EAChB,kBAAkB,EAClB,UAAU,EACV,cAAc,EACd,aAAa,EAEb,SAAS,EACT,aAAa,EACb,YAAY,EAEZ,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,yBAAyB,EACzB,4BAA4B,EAC5B,0BAA0B,EAE1B,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAE5B,gBAAgB,EAChB,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,cAAc,EACd,eAAe,EACf,SAAS,GACV,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,UAAU,EACV,iBAAiB,EAEjB,WAAW,EACX,SAAS,EACT,UAAU,EAEV,eAAe,EACf,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,eAAe,EAEf,cAAc,EACd,YAAY,EACZ,aAAa,EACb,mBAAmB,EAEnB,WAAW,EACX,cAAc,EACd,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,OAAO,EACP,cAAc,GACf,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,WAAW,EACX,iBAAiB,EAEjB,eAAe,EACf,cAAc,EACd,UAAU,EAEV,WAAW,EAEX,kBAAkB,EAClB,cAAc,EACd,QAAQ,EAER,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,YAAY,GACb,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMrF,YAAY,EACV,cAAc,EACd,aAAa,EACb,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EAEtB,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EACtB,eAAe,EACf,mBAAmB,EACnB,cAAc,EACd,uBAAuB,GACxB,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMxD,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAM7E,YAAY,EAEV,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,sBAAsB,EACtB,gBAAgB,EAChB,iBAAiB,EAEjB,YAAY,EACZ,iBAAiB,EACjB,eAAe,EACf,aAAa,EACb,SAAS,EACT,eAAe,EAEf,UAAU,EACV,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,eAAe,EACf,eAAe,EAEf,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EAEnB,oBAAoB,EACpB,cAAc,EAEd,YAAY,EACZ,OAAO,EACP,WAAW,EACX,kBAAkB,EAClB,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,eAAe,EAEf,eAAe,EACf,mBAAmB,EACnB,YAAY,EAEZ,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EAEpB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EAEpB,aAAa,EACb,0BAA0B,GAC3B,MAAM,YAAY,CAAC;AAMpB,YAAY,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAMpE,YAAY,EAEV,YAAY,EACZ,eAAe,EAEf,SAAS,EACT,eAAe,EACf,mBAAmB,EACnB,oBAAoB,EACpB,eAAe,EAEf,WAAW,EACX,eAAe,EACf,aAAa,EACb,cAAc,EACd,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,aAAa,CAAC;AAMrB,YAAY,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAEnE,YAAY,EACV,aAAa,EACb,cAAc,EACd,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,sBAAsB,EACtB,qBAAqB,EACrB,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,eAAe,EACf,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,kBAAkB,EAClB,mBAAmB,EACnB,4BAA4B,EAC5B,qBAAqB,EACrB,2BAA2B,EAC3B,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAC5B,oBAAoB,EACpB,uBAAuB,EACvB,sBAAsB,EACtB,yBAAyB,EACzB,wBAAwB,EACxB,0BAA0B,EAC1B,qBAAqB,EACrB,uBAAuB,EACvB,uBAAuB,EACvB,yBAAyB,EACzB,mBAAmB,EACnB,wBAAwB,EACxB,mBAAmB,EACnB,2BAA2B,EAC3B,oBAAoB,EACpB,aAAa,EACb,mBAAmB,EACnB,0BAA0B,EAC1B,0BAA0B,EAC1B,gCAAgC,EAChC,sCAAsC,GACvC,MAAM,kBAAkB,CAAC"}
|
package/dist/workflow.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"workflow.d.ts","sourceRoot":"","sources":["../src/workflow.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,KAAK,oBAAoB,EAO1B,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAEpE,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,kBAAkB,EAC3B,MAAM,EAAE,gBAAgB,GACvB,OAAO,CAAC,oBAAoB,CAAC,
|
|
1
|
+
{"version":3,"file":"workflow.d.ts","sourceRoot":"","sources":["../src/workflow.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,KAAK,oBAAoB,EAO1B,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAEpE,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,kBAAkB,EAC3B,MAAM,EAAE,gBAAgB,GACvB,OAAO,CAAC,oBAAoB,CAAC,CAmC/B"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@artemiskit/sdk",
|
|
3
|
-
"version": "0.6.
|
|
3
|
+
"version": "0.6.1",
|
|
4
4
|
"description": "Programmatic SDK for ArtemisKit LLM evaluation toolkit - integrate into your Node.js apps",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -71,8 +71,8 @@
|
|
|
71
71
|
"test:coverage": "bun test --coverage"
|
|
72
72
|
},
|
|
73
73
|
"dependencies": {
|
|
74
|
-
"@artemiskit/core": "0.6.
|
|
75
|
-
"@artemiskit/redteam": "0.3.
|
|
74
|
+
"@artemiskit/core": "0.6.2",
|
|
75
|
+
"@artemiskit/redteam": "0.3.10"
|
|
76
76
|
},
|
|
77
77
|
"devDependencies": {
|
|
78
78
|
"@types/bun": "^1.1.0",
|
|
@@ -12,7 +12,8 @@ import {
|
|
|
12
12
|
} from '@artemiskit/core';
|
|
13
13
|
import { OpenAIAdapter } from '../../../adapters/openai/src/client';
|
|
14
14
|
import { ArtemisKit } from '../artemiskit';
|
|
15
|
-
import
|
|
15
|
+
import { readWorkflowRecord } from '../index';
|
|
16
|
+
import type { WorkflowJudgeOptions, WorkflowRunOptions } from '../types-only';
|
|
16
17
|
|
|
17
18
|
const servers: ReturnType<typeof Bun.serve>[] = [];
|
|
18
19
|
afterEach(() => {
|
|
@@ -63,6 +64,152 @@ function target(): AgentTarget {
|
|
|
63
64
|
}
|
|
64
65
|
|
|
65
66
|
describe('SDK native workflow session', () => {
|
|
67
|
+
test('forwards explicit independent semantic judging with separate coverage, usage and record reading', async () => {
|
|
68
|
+
for (const verdict of ['pass', 'fail', 'malformed', 'unconfigured']) {
|
|
69
|
+
const scenario = workflow();
|
|
70
|
+
scenario.outcomes.deterministic = [{ type: 'file', path: 'result.txt', exists: false }];
|
|
71
|
+
scenario.outcomes.semantic = [
|
|
72
|
+
{ type: 'llm_judge', mode: 'strict_assurance', rubric: 'PRIVATE-RUBRIC' },
|
|
73
|
+
];
|
|
74
|
+
let calls = 0;
|
|
75
|
+
let closed = 0;
|
|
76
|
+
const semanticJudge: WorkflowJudgeOptions = {
|
|
77
|
+
provider: 'independent-judge',
|
|
78
|
+
model: 'judge-model',
|
|
79
|
+
limits: { maxRequests: 1, maxTokens: 100, maxOutputTokens: 10, timeoutMs: 1000 },
|
|
80
|
+
client: {
|
|
81
|
+
provider: 'independent-judge',
|
|
82
|
+
capabilities: async () => ({
|
|
83
|
+
streaming: false,
|
|
84
|
+
functionCalling: false,
|
|
85
|
+
toolUse: false,
|
|
86
|
+
maxContext: 10000,
|
|
87
|
+
}),
|
|
88
|
+
generate: async (request) => {
|
|
89
|
+
calls++;
|
|
90
|
+
expect(request.model).toBe('judge-model');
|
|
91
|
+
expect(request.maxRetries).toBe(0);
|
|
92
|
+
return {
|
|
93
|
+
id: 'judge',
|
|
94
|
+
model: 'observed-judge',
|
|
95
|
+
text: verdict === 'malformed' ? 'PRIVATE-BAD-RESPONSE' : JSON.stringify({ verdict }),
|
|
96
|
+
tokens: { prompt: 5, completion: 2, total: 7 },
|
|
97
|
+
latencyMs: 1,
|
|
98
|
+
};
|
|
99
|
+
},
|
|
100
|
+
close: async () => {
|
|
101
|
+
closed++;
|
|
102
|
+
},
|
|
103
|
+
},
|
|
104
|
+
};
|
|
105
|
+
const result = await new ArtemisKit().runWorkflow({
|
|
106
|
+
workflow: scenario,
|
|
107
|
+
target: target(),
|
|
108
|
+
...(verdict === 'unconfigured' ? {} : { semanticJudge }),
|
|
109
|
+
});
|
|
110
|
+
expect(result.record.execution).toBe('completed');
|
|
111
|
+
expect(result.record.taskVerification).toBe(
|
|
112
|
+
verdict === 'pass'
|
|
113
|
+
? 'passed'
|
|
114
|
+
: verdict === 'fail'
|
|
115
|
+
? 'failed'
|
|
116
|
+
: verdict === 'malformed'
|
|
117
|
+
? 'invalid'
|
|
118
|
+
: 'unavailable'
|
|
119
|
+
);
|
|
120
|
+
expect(result.record.usage.reported.total).toBe(3);
|
|
121
|
+
expect(result.record.outcomes.deterministic.counts.valid).toBe(1);
|
|
122
|
+
expect(result.record.outcomes.semantic.usage.reported.total).toBe(
|
|
123
|
+
verdict === 'unconfigured' ? 0 : 7
|
|
124
|
+
);
|
|
125
|
+
expect(calls).toBe(verdict === 'unconfigured' ? 0 : 1);
|
|
126
|
+
expect(closed).toBe(0);
|
|
127
|
+
expect(readWorkflowRecord(JSON.parse(JSON.stringify(result.record)))).toEqual(result.record);
|
|
128
|
+
expect(JSON.stringify(result.record)).not.toContain('PRIVATE-');
|
|
129
|
+
}
|
|
130
|
+
});
|
|
131
|
+
test('malformed explicit judge bounds stop before any target model call', async () => {
|
|
132
|
+
const scenario = workflow();
|
|
133
|
+
let turns = 0;
|
|
134
|
+
const custom = target();
|
|
135
|
+
custom.turn = async () => {
|
|
136
|
+
turns++;
|
|
137
|
+
throw new Error('must not call');
|
|
138
|
+
};
|
|
139
|
+
const result = await new ArtemisKit().runWorkflow({
|
|
140
|
+
workflow: scenario,
|
|
141
|
+
target: custom,
|
|
142
|
+
semanticJudge: {
|
|
143
|
+
provider: 'judge',
|
|
144
|
+
model: 'judge',
|
|
145
|
+
limits: { maxRequests: 0, maxTokens: 100, maxOutputTokens: 10, timeoutMs: 1000 },
|
|
146
|
+
client: {
|
|
147
|
+
provider: 'judge',
|
|
148
|
+
capabilities: async () => {
|
|
149
|
+
throw new Error('must not call');
|
|
150
|
+
},
|
|
151
|
+
generate: async () => {
|
|
152
|
+
throw new Error('must not call');
|
|
153
|
+
},
|
|
154
|
+
},
|
|
155
|
+
},
|
|
156
|
+
});
|
|
157
|
+
expect(turns).toBe(0);
|
|
158
|
+
expect(result.record.execution).toBe('invalid');
|
|
159
|
+
});
|
|
160
|
+
test('deterministic failure gates judge calls and session cancellation reaches the independent evaluator', async () => {
|
|
161
|
+
let started!: () => void;
|
|
162
|
+
const pending = new Promise<void>((resolve) => {
|
|
163
|
+
started = resolve;
|
|
164
|
+
});
|
|
165
|
+
let calls = 0;
|
|
166
|
+
const semanticJudge: WorkflowJudgeOptions = {
|
|
167
|
+
provider: 'judge',
|
|
168
|
+
model: 'judge',
|
|
169
|
+
limits: { maxRequests: 1, maxTokens: 100, maxOutputTokens: 10, timeoutMs: 1000 },
|
|
170
|
+
client: {
|
|
171
|
+
provider: 'judge',
|
|
172
|
+
capabilities: async () => ({
|
|
173
|
+
streaming: false,
|
|
174
|
+
functionCalling: false,
|
|
175
|
+
toolUse: false,
|
|
176
|
+
maxContext: 10000,
|
|
177
|
+
}),
|
|
178
|
+
generate: async (request) => {
|
|
179
|
+
calls++;
|
|
180
|
+
started();
|
|
181
|
+
await new Promise<void>((resolve) =>
|
|
182
|
+
request.signal?.addEventListener('abort', () => resolve(), { once: true })
|
|
183
|
+
);
|
|
184
|
+
throw new Error('PRIVATE-ABORT');
|
|
185
|
+
},
|
|
186
|
+
},
|
|
187
|
+
};
|
|
188
|
+
const scenario = workflow();
|
|
189
|
+
scenario.outcomes.semantic = [
|
|
190
|
+
{ type: 'llm_judge', mode: 'strict_assurance', rubric: 'PRIVATE-RUBRIC' },
|
|
191
|
+
];
|
|
192
|
+
const kit = new ArtemisKit();
|
|
193
|
+
const failed = await kit.runWorkflow({ workflow: scenario, target: target(), semanticJudge });
|
|
194
|
+
expect(failed.record.taskVerification).toBe('failed');
|
|
195
|
+
expect(calls).toBe(0);
|
|
196
|
+
scenario.outcomes.deterministic = [{ type: 'file', path: 'result.txt', exists: false }];
|
|
197
|
+
const session = await kit.createWorkflowSession({
|
|
198
|
+
workflow: scenario,
|
|
199
|
+
target: target(),
|
|
200
|
+
semanticJudge,
|
|
201
|
+
});
|
|
202
|
+
const finished = session.run();
|
|
203
|
+
await pending;
|
|
204
|
+
session.cancel();
|
|
205
|
+
const result = await finished;
|
|
206
|
+
expect(result.record.execution).toBe('completed');
|
|
207
|
+
expect(result.record.outcomes.cancelled).toBe(true);
|
|
208
|
+
expect(result.record.outcomes.reason).toBe('cancelled');
|
|
209
|
+
expect(result.record.outcomes.semantic.counts.valid).toBe(0);
|
|
210
|
+
expect(readWorkflowRecord(result.record)).toEqual(result.record);
|
|
211
|
+
});
|
|
212
|
+
|
|
66
213
|
test('configured adapter uses workflow identity and zero retries with real loopback multi-turn execution', async () => {
|
|
67
214
|
const bodies: Record<string, unknown>[] = [];
|
|
68
215
|
const server = Bun.serve({
|
|
@@ -154,7 +301,7 @@ describe('SDK native workflow session', () => {
|
|
|
154
301
|
const { result, events } = JSON.parse(stdout);
|
|
155
302
|
expect(bodies[0].model).toBe('workflow-authoritative');
|
|
156
303
|
expect(result.record.execution).toBe('completed');
|
|
157
|
-
expect(result.record.taskVerification).toBe('
|
|
304
|
+
expect(result.record.taskVerification).toBe('passed');
|
|
158
305
|
expect(result.record.budgets).toMatchObject({ modelRequests: 2, toolCalls: 1 });
|
|
159
306
|
expect(result.state?.files).toEqual({ 'result.txt': 'PRIVATE-STATE' });
|
|
160
307
|
expect(JSON.stringify(result.record)).not.toContain('PRIVATE-');
|
package/src/index.ts
CHANGED
|
@@ -20,6 +20,12 @@ export {
|
|
|
20
20
|
createModelClientTarget,
|
|
21
21
|
createAgentWorkflowSession,
|
|
22
22
|
runAgentWorkflow,
|
|
23
|
+
evaluateWorkflowDeterministicOutcomes,
|
|
24
|
+
evaluateWorkflowSemantics,
|
|
25
|
+
isValidWorkflowJudgeOptions,
|
|
26
|
+
decideWorkflowOutcome,
|
|
27
|
+
validWorkflowAssertionCounts,
|
|
28
|
+
readWorkflowRecord,
|
|
23
29
|
createSimulatedWorkflowEnvironment,
|
|
24
30
|
createDockerWorkflowEnvironment,
|
|
25
31
|
createDockerWorkflowEnvironmentFactory,
|
|
@@ -44,6 +50,24 @@ export type {
|
|
|
44
50
|
AgentWorkflowReason,
|
|
45
51
|
AgentWorkflowEvent,
|
|
46
52
|
AgentWorkflowRecord,
|
|
53
|
+
AgentWorkflowExecutionRecord,
|
|
54
|
+
AgentWorkflowEvidence,
|
|
55
|
+
WorkflowDeterministicStatus,
|
|
56
|
+
WorkflowDeterministicReason,
|
|
57
|
+
WorkflowDeterministicAssertion,
|
|
58
|
+
WorkflowDeterministicSummary,
|
|
59
|
+
WorkflowJudgeOptions,
|
|
60
|
+
WorkflowSemanticOptions,
|
|
61
|
+
WorkflowSemanticReason,
|
|
62
|
+
WorkflowSemanticAssertion,
|
|
63
|
+
WorkflowSemanticIdentity,
|
|
64
|
+
WorkflowSemanticEvaluation,
|
|
65
|
+
WorkflowOutcomeStatus,
|
|
66
|
+
WorkflowAssertionCounts,
|
|
67
|
+
WorkflowOutcomeDecision,
|
|
68
|
+
WorkflowOutcomeAssessment,
|
|
69
|
+
SavedWorkflowRecord,
|
|
70
|
+
HistoricalWorkflowRecord,
|
|
47
71
|
AgentWorkflowResult,
|
|
48
72
|
AgentWorkflowSessionOptions,
|
|
49
73
|
AgentWorkflowSession,
|
package/src/types-only.ts
CHANGED
|
@@ -294,6 +294,24 @@ export type {
|
|
|
294
294
|
AgentWorkflowReason,
|
|
295
295
|
AgentWorkflowEvent,
|
|
296
296
|
AgentWorkflowRecord,
|
|
297
|
+
AgentWorkflowExecutionRecord,
|
|
298
|
+
AgentWorkflowEvidence,
|
|
299
|
+
WorkflowDeterministicStatus,
|
|
300
|
+
WorkflowDeterministicReason,
|
|
301
|
+
WorkflowDeterministicAssertion,
|
|
302
|
+
WorkflowDeterministicSummary,
|
|
303
|
+
WorkflowJudgeOptions,
|
|
304
|
+
WorkflowSemanticOptions,
|
|
305
|
+
WorkflowSemanticReason,
|
|
306
|
+
WorkflowSemanticAssertion,
|
|
307
|
+
WorkflowSemanticIdentity,
|
|
308
|
+
WorkflowSemanticEvaluation,
|
|
309
|
+
WorkflowOutcomeStatus,
|
|
310
|
+
WorkflowAssertionCounts,
|
|
311
|
+
WorkflowOutcomeDecision,
|
|
312
|
+
WorkflowOutcomeAssessment,
|
|
313
|
+
SavedWorkflowRecord,
|
|
314
|
+
HistoricalWorkflowRecord,
|
|
297
315
|
AgentWorkflowResult,
|
|
298
316
|
AgentWorkflowSessionOptions,
|
|
299
317
|
AgentWorkflowSession,
|
package/src/workflow.ts
CHANGED