@artemiskit/sdk 0.5.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/README.md +29 -0
- package/dist/artemiskit.d.ts +5 -1
- package/dist/artemiskit.d.ts.map +1 -1
- package/dist/index.d.ts +3 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +90 -11
- package/dist/types-only.d.ts +2 -1
- package/dist/types-only.d.ts.map +1 -1
- package/dist/types.d.ts +12 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/workflow.d.ts +4 -0
- package/dist/workflow.d.ts.map +1 -0
- package/package.json +3 -3
- package/src/__tests__/workflow.test.ts +391 -0
- package/src/artemiskit.ts +12 -0
- package/src/index.ts +45 -1
- package/src/types-only.ts +48 -0
- package/src/types.ts +16 -0
- package/src/workflow.ts +52 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,35 @@
|
|
|
1
1
|
# @artemiskit/sdk
|
|
2
2
|
|
|
3
|
+
## 0.6.1
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- Complete the 0.6.2 independent workflow outcome milestone: verify final artifacts, bounded JSON schemas, state, tool traces and policy before optional explicitly configured semantic judging. Keep target and judge identity, usage and limits separate; expose task validity and coverage, strict V1/V2 saved-record reading, SDK composition and CLI outcome exits without changing historical prompt-response reports.
|
|
8
|
+
- Updated dependencies
|
|
9
|
+
- @artemiskit/core@0.6.2
|
|
10
|
+
- @artemiskit/redteam@0.3.10
|
|
11
|
+
|
|
12
|
+
## 0.6.0
|
|
13
|
+
|
|
14
|
+
### Minor Changes
|
|
15
|
+
|
|
16
|
+
- Add native controlled workflow execution for milestone 0.6.1. CLI workflow run/preflight and SDK
|
|
17
|
+
sessions share strict tool and permission enforcement, cumulative budgets, explicit tool-protocol
|
|
18
|
+
preflight, measured-usage states, bounded events, cancellation, and truthful cleanup evidence.
|
|
19
|
+
Run against detached simulated state or a disposable Docker filesystem with a fixed local image,
|
|
20
|
+
no host mounts or network, and no arbitrary command tool. Save bounded execution metadata separately
|
|
21
|
+
from explicitly requested sensitive state exports. Task verification remains unavailable until 0.6.2.
|
|
22
|
+
|
|
23
|
+
OpenAI and Ling workflow transports forward abort signals, disable automatic retries, and distinguish
|
|
24
|
+
missing or malformed token usage from explicitly measured zero. Preserve existing scenario APIs and
|
|
25
|
+
offline workflow validation. Package versions remain independent of the milestone label.
|
|
26
|
+
|
|
27
|
+
### Patch Changes
|
|
28
|
+
|
|
29
|
+
- Updated dependencies
|
|
30
|
+
- @artemiskit/core@0.6.1
|
|
31
|
+
- @artemiskit/redteam@0.3.9
|
|
32
|
+
|
|
3
33
|
## 0.5.0
|
|
4
34
|
|
|
5
35
|
### Minor Changes
|
package/README.md
CHANGED
|
@@ -48,6 +48,35 @@ console.log('All tests passed! ✅');
|
|
|
48
48
|
|
|
49
49
|
## API Reference
|
|
50
50
|
|
|
51
|
+
### Controlled agent workflows
|
|
52
|
+
|
|
53
|
+
```typescript
|
|
54
|
+
const session = await kit.createWorkflowSession({
|
|
55
|
+
workflow: './workflow.yaml',
|
|
56
|
+
providerConfig: { apiKey: process.env.OPENAI_API_KEY },
|
|
57
|
+
preflight: true,
|
|
58
|
+
});
|
|
59
|
+
const result = await session.run();
|
|
60
|
+
console.log(result.record); // bounded metadata with separate execution and task outcomes
|
|
61
|
+
// session.cancel() stops new work and initiates bounded cleanup.
|
|
62
|
+
// result.state and result.transcript contain sensitive working data.
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
`kit.runWorkflow(options)` creates and runs a fresh session in one call. Supply a `client` or
|
|
66
|
+
`target` to use an existing trusted adapter or target. The same native controls apply through the
|
|
67
|
+
CLI and SDK: declared tools and permissions, cumulative budgets, explicit preflight, cancellation,
|
|
68
|
+
and cleanup evidence. Workflows use fresh simulated state or a disposable Docker filesystem.
|
|
69
|
+
The 0.6.2 milestone independently verifies declared artifacts, state, schemas, traces and policy.
|
|
70
|
+
Use explicit `semanticJudge` configuration for declared qualitative criteria. Target and judge usage
|
|
71
|
+
remain separate; `readWorkflowRecord` validates V1/V2 saved metadata without making model calls.
|
|
72
|
+
See [workflow outcomes](https://github.com/code-sensei/artemiskit/blob/main/docs/workflow-outcomes.md)
|
|
73
|
+
for usage, denominators, compatibility and publication status.
|
|
74
|
+
|
|
75
|
+
The root and `/types` entry points expose workflow result/session/event types. Low-level
|
|
76
|
+
`createAgentWorkflowSession` and `runAgentWorkflow` are available for explicit target composition.
|
|
77
|
+
See the [execution guide](https://github.com/code-sensei/artemiskit/blob/main/docs/workflow-execution.md)
|
|
78
|
+
for configuration, Docker requirements, token-accounting limits, events, and safe persistence.
|
|
79
|
+
|
|
51
80
|
### ArtemisKit Class
|
|
52
81
|
|
|
53
82
|
#### Constructor
|
package/dist/artemiskit.d.ts
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* @artemiskit/sdk
|
|
3
3
|
* Main ArtemisKit class - programmatic API for LLM testing
|
|
4
4
|
*/
|
|
5
|
-
import type { ArtemisKitConfig, ArtemisKitEventName, ArtemisKitEvents, CaseCompleteHandler, CaseStartHandler, CompareOptions, CompareResult, ProgressHandler, RedTeamMutationCompleteHandler, RedTeamMutationStartHandler, RedTeamOptions, RedTeamResult, RunOptions, RunResult, StressOptions, StressRequestCompleteHandler, StressResult, ValidateOptions, ValidationResult } from './types.js';
|
|
5
|
+
import type { ArtemisKitConfig, ArtemisKitEventName, ArtemisKitEvents, CaseCompleteHandler, CaseStartHandler, CompareOptions, CompareResult, ProgressHandler, RedTeamMutationCompleteHandler, RedTeamMutationStartHandler, RedTeamOptions, RedTeamResult, RunOptions, RunResult, StressOptions, StressRequestCompleteHandler, StressResult, ValidateOptions, ValidationResult, WorkflowRunOptions } from './types.js';
|
|
6
6
|
/**
|
|
7
7
|
* ArtemisKit SDK - programmatic API for LLM evaluation testing
|
|
8
8
|
*
|
|
@@ -25,6 +25,10 @@ export declare class ArtemisKit {
|
|
|
25
25
|
private eventHandlers;
|
|
26
26
|
private storage;
|
|
27
27
|
constructor(config?: ArtemisKitConfig);
|
|
28
|
+
/** Create a cancellable native session without executing model turns or persisting results. */
|
|
29
|
+
createWorkflowSession(options: WorkflowRunOptions): Promise<import("@artemiskit/core").AgentWorkflowSession>;
|
|
30
|
+
/** Execute a workflow; task verification is explicitly unavailable until outcome scoring. */
|
|
31
|
+
runWorkflow(options: WorkflowRunOptions): Promise<import("@artemiskit/core").AgentWorkflowResult>;
|
|
28
32
|
/**
|
|
29
33
|
* Register an event handler
|
|
30
34
|
*/
|
package/dist/artemiskit.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"artemiskit.d.ts","sourceRoot":"","sources":["../src/artemiskit.ts"],"names":[],"mappings":"AAAA;;;GAGG;
|
|
1
|
+
{"version":3,"file":"artemiskit.d.ts","sourceRoot":"","sources":["../src/artemiskit.ts"],"names":[],"mappings":"AAAA;;;GAGG;AA0CH,OAAO,KAAK,EACV,gBAAgB,EAChB,mBAAmB,EACnB,gBAAgB,EAChB,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,aAAa,EACb,eAAe,EACf,8BAA8B,EAC9B,2BAA2B,EAC3B,cAAc,EACd,aAAa,EACb,UAAU,EACV,SAAS,EACT,aAAa,EACb,4BAA4B,EAC5B,YAAY,EACZ,eAAe,EACf,gBAAgB,EAChB,kBAAkB,EACnB,MAAM,SAAS,CAAC;AAsBjB;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAmB;IACjC,OAAO,CAAC,aAAa,CAA6D;IAClF,OAAO,CAAC,OAAO,CAA+B;gBAElC,MAAM,GAAE,gBAAqB;IAmBzC,+FAA+F;IACzF,qBAAqB,CAAC,OAAO,EAAE,kBAAkB;IAIvD,6FAA6F;IACvF,WAAW,CAAC,OAAO,EAAE,kBAAkB;IAQ7C;;OAEG;IACH,EAAE,CAAC,CAAC,SAAS,mBAAmB,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,IAAI;IAQhG;;OAEG;IACH,GAAG,CAAC,CAAC,SAAS,mBAAmB,EAC/B,KAAK,EAAE,CAAC,EACR,OAAO,EAAE,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC,KAAK,IAAI,GAC5C,IAAI;IAQP;;OAEG;IACH,IAAI,CAAC,CAAC,SAAS,mBAAmB,EAChC,KAAK,EAAE,CAAC,EACR,OAAO,EAAE,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC,KAAK,IAAI,GAC5C,IAAI;IAQP;;OAEG;IACH,OAAO,CAAC,IAAI;IAiBZ;;OAEG;IACH,WAAW,CAAC,OAAO,EAAE,gBAAgB,GAAG,IAAI;IAI5C;;OAEG;IACH,cAAc,CAAC,OAAO,EAAE,mBAAmB,GAAG,IAAI;IAIlD;;OAEG;IACH,UAAU,CAAC,OAAO,EAAE,eAAe,GAAG,IAAI;IAI1C;;OAEG;IACH,sBAAsB,CAAC,OAAO,EAAE,2BAA2B,GAAG,IAAI;IAIlE;;OAEG;IACH,yBAAyB,CAAC,OAAO,EAAE,8BAA8B,GAAG,IAAI;IAIxE;;OAEG;IACH,uBAAuB,CAAC,OAAO,EAAE,4BAA4B,GAAG,IAAI;IAQpE;;OAEG;IACG,GAAG,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC;IA+DlD;;OAEG;IACG,OAAO,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;IAgN9D;;OAEG;IACG,MAAM,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC;IAwO3D;;;;;;;;;;;;;OAaG;IACG,QAAQ,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA2LnE;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACG,OAAO,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;IA2H9D;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAmBzB;;OAEG;IACH,OAAO,CAAC,YAAY;IA4EpB;;OAEG;IACH,OAAO,CAAC,YAAY;IAepB;;OAEG;YACW,YAAY;IAO1B;;OAEG;YACW,YAAY;IAgB1B;;OAEG;IACH,OAAO,CAAC,cAAc;IA6BtB;;OAEG;IACH,qBAAqB,IAAI,MAAM,EAAE;CAGlC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -3,9 +3,9 @@
|
|
|
3
3
|
* Programmatic SDK for ArtemisKit LLM evaluation toolkit
|
|
4
4
|
*/
|
|
5
5
|
export { ArtemisKit } from './artemiskit.js';
|
|
6
|
-
export { AgentWorkflowSchema, WorkflowPolicySchema, WORKFLOW_TOOL_IDS, validateAgentWorkflow, parseAgentWorkflow, loadAgentWorkflow, listWorkflowTools, getWorkflowTool, executeSimulatedTool, createModelClientTarget, } from '@artemiskit/core';
|
|
7
|
-
export type { AgentWorkflow, WorkflowPolicy, WorkflowJson, WorkflowToolId, WorkflowResource, WorkflowToolDescriptor, SimulatedToolEvidence, SimulatedToolResult, AgentTarget, AgentTurnRequest, AgentTurnResult, AgentTargetCapabilities, AgentTargetFailure, } from '@artemiskit/core';
|
|
8
|
-
export type { ArtemisKitConfig, RunOptions, RedTeamOptions, StressOptions, RunResult, RedTeamResult, StressResult, CaseStartEvent, CaseCompleteEvent, ProgressEvent, RedTeamMutationStartEvent, RedTeamMutationCompleteEvent, StressRequestCompleteEvent, CaseStartHandler, CaseCompleteHandler, ProgressHandler, RedTeamMutationStartHandler, RedTeamMutationCompleteHandler, StressRequestCompleteHandler, ArtemisKitEvents, ArtemisKitEventName, ValidateOptions, ValidationResult, ValidationError, ValidationWarning, ScenarioValidation, CompareOptions, CompareResult, RunSummary, ComparisonDetails, } from './types.js';
|
|
6
|
+
export { AgentWorkflowSchema, WorkflowPolicySchema, WORKFLOW_TOOL_IDS, validateAgentWorkflow, parseAgentWorkflow, loadAgentWorkflow, listWorkflowTools, getWorkflowTool, executeSimulatedTool, createModelClientTarget, createAgentWorkflowSession, runAgentWorkflow, evaluateWorkflowDeterministicOutcomes, evaluateWorkflowSemantics, isValidWorkflowJudgeOptions, decideWorkflowOutcome, validWorkflowAssertionCounts, readWorkflowRecord, createSimulatedWorkflowEnvironment, createDockerWorkflowEnvironment, createDockerWorkflowEnvironmentFactory, WORKFLOW_SANDBOX_IMAGE, WorkflowEnvironmentInitializationError, } from '@artemiskit/core';
|
|
7
|
+
export type { AgentWorkflow, WorkflowPolicy, WorkflowJson, WorkflowToolId, WorkflowResource, WorkflowToolDescriptor, SimulatedToolEvidence, SimulatedToolResult, AgentTarget, AgentTurnRequest, AgentTurnResult, AgentTargetCapabilities, AgentTargetFailure, AgentWorkflowExecution, AgentWorkflowReason, AgentWorkflowEvent, AgentWorkflowRecord, AgentWorkflowExecutionRecord, AgentWorkflowEvidence, WorkflowDeterministicStatus, WorkflowDeterministicReason, WorkflowDeterministicAssertion, WorkflowDeterministicSummary, WorkflowJudgeOptions, WorkflowSemanticOptions, WorkflowSemanticReason, WorkflowSemanticAssertion, WorkflowSemanticIdentity, WorkflowSemanticEvaluation, WorkflowOutcomeStatus, WorkflowAssertionCounts, WorkflowOutcomeDecision, WorkflowOutcomeAssessment, SavedWorkflowRecord, HistoricalWorkflowRecord, AgentWorkflowResult, AgentWorkflowSessionOptions, AgentWorkflowSession, WorkflowState, WorkflowEnvironment, WorkflowEnvironmentFactory, WorkflowEnvironmentCleanup, DockerWorkflowEnvironmentOptions, } from '@artemiskit/core';
|
|
8
|
+
export type { ArtemisKitConfig, WorkflowRunOptions, RunOptions, RedTeamOptions, StressOptions, RunResult, RedTeamResult, StressResult, CaseStartEvent, CaseCompleteEvent, ProgressEvent, RedTeamMutationStartEvent, RedTeamMutationCompleteEvent, StressRequestCompleteEvent, CaseStartHandler, CaseCompleteHandler, ProgressHandler, RedTeamMutationStartHandler, RedTeamMutationCompleteHandler, StressRequestCompleteHandler, ArtemisKitEvents, ArtemisKitEventName, ValidateOptions, ValidationResult, ValidationError, ValidationWarning, ScenarioValidation, CompareOptions, CompareResult, RunSummary, ComparisonDetails, } from './types.js';
|
|
9
9
|
export type { Scenario, TestCase, Expected, Provider, ProviderConfig, ChatMessageType, Variables, } from '@artemiskit/core';
|
|
10
10
|
export type { CaseResult, CaseRedactionInfo, RunManifest, RunConfig, RunMetrics, RedTeamManifest, RedTeamConfig, RedTeamMetrics, RedTeamCaseResult, RedTeamStatus, RedTeamSeverity, StressManifest, StressConfig, StressMetrics, StressRequestResult, AnyManifest, ResolvedConfig, ConfigSource, ManifestRedactionInfo, CostEstimateInfo, GitInfo, ProvenanceInfo, } from '@artemiskit/core';
|
|
11
11
|
export type { ModelClient, ModelCapabilities, GenerateOptions, GenerateResult, TokenUsage, ChatMessage, FunctionDefinition, ToolDefinition, ToolCall, AdapterConfig, BaseAdapterConfig, OpenAIAdapterConfig, AzureOpenAIAdapterConfig, VercelAIAdapterConfig, AnthropicAdapterConfig, LangChainAdapterConfig, DeepAgentsAdapterConfig, ProviderType, } from '@artemiskit/core';
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAG1C,OAAO,EACL,mBAAmB,EACnB,oBAAoB,EACpB,iBAAiB,EACjB,qBAAqB,EACrB,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,eAAe,EACf,oBAAoB,EACpB,uBAAuB,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAG1C,OAAO,EACL,mBAAmB,EACnB,oBAAoB,EACpB,iBAAiB,EACjB,qBAAqB,EACrB,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,eAAe,EACf,oBAAoB,EACpB,uBAAuB,EACvB,0BAA0B,EAC1B,gBAAgB,EAChB,qCAAqC,EACrC,yBAAyB,EACzB,2BAA2B,EAC3B,qBAAqB,EACrB,4BAA4B,EAC5B,kBAAkB,EAClB,kCAAkC,EAClC,+BAA+B,EAC/B,sCAAsC,EACtC,sBAAsB,EACtB,sCAAsC,GACvC,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,aAAa,EACb,cAAc,EACd,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,sBAAsB,EACtB,qBAAqB,EACrB,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,eAAe,EACf,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,kBAAkB,EAClB,mBAAmB,EACnB,4BAA4B,EAC5B,qBAAqB,EACrB,2BAA2B,EAC3B,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAC5B,oBAAoB,EACpB,uBAAuB,EACvB,sBAAsB,EACtB,yBAAyB,EACzB,wBAAwB,EACxB,0BAA0B,EAC1B,qBAAqB,EACrB,uBAAuB,EACvB,uBAAuB,EACvB,yBAAyB,EACzB,mBAAmB,EACnB,wBAAwB,EACxB,mBAAmB,EACnB,2BAA2B,EAC3B,oBAAoB,EACpB,aAAa,EACb,mBAAmB,EACnB,0BAA0B,EAC1B,0BAA0B,EAC1B,gCAAgC,GACjC,MAAM,kBAAkB,CAAC;AAG1B,YAAY,EAEV,gBAAgB,EAChB,kBAAkB,EAClB,UAAU,EACV,cAAc,EACd,aAAa,EAEb,SAAS,EACT,aAAa,EACb,YAAY,EAEZ,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,yBAAyB,EACzB,4BAA4B,EAC5B,0BAA0B,EAE1B,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAE5B,gBAAgB,EAChB,mBAAmB,EAEnB,eAAe,EACf,gBAAgB,EAChB,eAAe,EACf,iBAAiB,EACjB,kBAAkB,EAElB,cAAc,EACd,aAAa,EACb,UAAU,EACV,iBAAiB,GAClB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,cAAc,EACd,eAAe,EACf,SAAS,GACV,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,UAAU,EACV,iBAAiB,EAEjB,WAAW,EACX,SAAS,EACT,UAAU,EAEV,eAAe,EACf,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,eAAe,EAEf,cAAc,EACd,YAAY,EACZ,aAAa,EACb,mBAAmB,EAEnB,WAAW,EACX,cAAc,EACd,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,OAAO,EACP,cAAc,GACf,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,WAAW,EACX,iBAAiB,EAEjB,eAAe,EACf,cAAc,EACd,UAAU,EAEV,WAAW,EAEX,kBAAkB,EAClB,cAAc,EACd,QAAQ,EAER,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,YAAY,GACb,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMrF,YAAY,EACV,cAAc,EACd,aAAa,EACb,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EAEtB,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EACtB,eAAe,EACf,mBAAmB,EACnB,cAAc,EACd,uBAAuB,GACxB,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMxD,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAMtF,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAM7E,OAAO,EAAE,kBAAkB,EAAE,KAAK,kBAAkB,EAAE,KAAK,aAAa,EAAE,MAAM,YAAY,CAAC;AAM7F,OAAO,EAEL,WAAW,EACX,eAAe,EACf,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,oBAAoB,EAEpB,aAAa,EACb,MAAM,EAEN,cAAc,EACd,cAAc,EACd,aAAa,EACb,oBAAoB,GACrB,MAAM,SAAS,CAAC;AAEjB,YAAY,EAEV,YAAY,EACZ,eAAe,EAEf,SAAS,EACT,eAAe,EACf,mBAAmB,EACnB,oBAAoB,EACpB,eAAe,EAEf,WAAW,EACX,eAAe,EACf,aAAa,EACb,cAAc,EACd,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,OAAO,EAEL,aAAa,EAEb,eAAe,EAEf,aAAa,EAEb,YAAY,GACb,MAAM,aAAa,CAAC;AAErB,YAAY,EACV,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,aAAa,CAAC;AAMrB,OAAO,EAEL,eAAe,EACf,eAAe,EAEf,QAAQ,EACR,QAAQ,EAER,YAAY,EACZ,SAAS,EACT,SAAS,EACT,QAAQ,EACR,UAAU,EAEV,KAAK,EACL,QAAQ,EACR,WAAW,EACX,KAAK,EACL,KAAK,EACL,UAAU,EACV,QAAQ,EACR,UAAU,EACV,MAAM,EACN,KAAK,EACL,KAAK,GACN,MAAM,YAAY,CAAC;AAMpB,OAAO,EAEL,QAAQ,EACR,cAAc,EAEd,qBAAqB,EAErB,iBAAiB,EACjB,uBAAuB,EAEvB,mBAAmB,EACnB,oBAAoB,EACpB,iBAAiB,EAEjB,eAAe,EACf,4BAA4B,EAE5B,gBAAgB,EAChB,sBAAsB,EAEtB,eAAe,EACf,wBAAwB,EACxB,SAAS,EACT,kBAAkB,EAClB,aAAa,EACb,4BAA4B,EAC5B,gBAAgB,EAEhB,YAAY,EACZ,4BAA4B,EAE5B,UAAU,EACV,WAAW,EACX,cAAc,EACd,mBAAmB,EACnB,aAAa,EACb,sBAAsB,EACtB,eAAe,EACf,qBAAqB,EAErB,cAAc,EACd,gBAAgB,EAChB,WAAW,GACZ,MAAM,YAAY,CAAC;AAGpB,YAAY,EAEV,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,sBAAsB,EACtB,gBAAgB,EAChB,iBAAiB,EACjB,oBAAoB,EAEpB,YAAY,EACZ,iBAAiB,EACjB,eAAe,EACf,aAAa,EACb,SAAS,EACT,eAAe,EAEf,UAAU,EACV,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,eAAe,EACf,eAAe,EAEf,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EAEnB,oBAAoB,EACpB,cAAc,EAEd,YAAY,EACZ,OAAO,EACP,WAAW,EACX,kBAAkB,EAClB,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,eAAe,EAEf,eAAe,EACf,mBAAmB,EACnB,YAAY,EAEZ,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EAEpB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EAEpB,aAAa,EACb,0BAA0B,EAE1B,qBAAqB,EACrB,kBAAkB,EAClB,eAAe,EAEf,uBAAuB,EACvB,aAAa,EACb,kBAAkB,EAClB,eAAe,EACf,wBAAwB,EAExB,kBAAkB,EAClB,oBAAoB,EACpB,cAAc,EACd,cAAc,EACd,mBAAmB,EACnB,4BAA4B,EAC5B,yBAAyB,EACzB,kCAAkC,EAClC,2BAA2B,EAC3B,mBAAmB,EACnB,uBAAuB,EACvB,eAAe,EACf,eAAe,EACf,gBAAgB,EAChB,yBAAyB,EACzB,sBAAsB,GACvB,MAAM,YAAY,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -7584,7 +7584,7 @@ function anyOf(...expectations) {
|
|
|
7584
7584
|
import {
|
|
7585
7585
|
assessComparisonEligibility,
|
|
7586
7586
|
runScenario as coreRunScenario,
|
|
7587
|
-
createAdapter,
|
|
7587
|
+
createAdapter as createAdapter2,
|
|
7588
7588
|
createStorageAdapter,
|
|
7589
7589
|
getGitInfo,
|
|
7590
7590
|
isComparisonAvailable,
|
|
@@ -7636,6 +7636,53 @@ function nanoid(size = 21) {
|
|
|
7636
7636
|
return id;
|
|
7637
7637
|
}
|
|
7638
7638
|
|
|
7639
|
+
// src/workflow.ts
|
|
7640
|
+
import { dirname, resolve } from "node:path";
|
|
7641
|
+
import {
|
|
7642
|
+
ArtemisError,
|
|
7643
|
+
createAdapter,
|
|
7644
|
+
createAgentWorkflowSession,
|
|
7645
|
+
createModelClientTarget,
|
|
7646
|
+
loadAgentWorkflow,
|
|
7647
|
+
validateAgentWorkflow
|
|
7648
|
+
} from "@artemiskit/core";
|
|
7649
|
+
async function prepareWorkflowSession(options, config) {
|
|
7650
|
+
if (options.client && options.target)
|
|
7651
|
+
throw new Error("Choose a workflow client or target");
|
|
7652
|
+
const file = typeof options.workflow === "string" ? resolve(options.workflow) : undefined;
|
|
7653
|
+
const workflow = file ? await loadAgentWorkflow(file) : validateAgentWorkflow(options.workflow);
|
|
7654
|
+
let target = options.target;
|
|
7655
|
+
if (!target) {
|
|
7656
|
+
let client = options.client;
|
|
7657
|
+
if (!client) {
|
|
7658
|
+
try {
|
|
7659
|
+
client = await createAdapter({
|
|
7660
|
+
...config.providerConfig,
|
|
7661
|
+
...options.providerConfig,
|
|
7662
|
+
provider: workflow.target.provider,
|
|
7663
|
+
defaultModel: workflow.target.model,
|
|
7664
|
+
maxRetries: 0
|
|
7665
|
+
});
|
|
7666
|
+
} catch {
|
|
7667
|
+
throw new ArtemisError("Unable to initialize workflow target", "PROVIDER_UNAVAILABLE");
|
|
7668
|
+
}
|
|
7669
|
+
}
|
|
7670
|
+
target = createModelClientTarget(client);
|
|
7671
|
+
}
|
|
7672
|
+
return createAgentWorkflowSession({
|
|
7673
|
+
workflow,
|
|
7674
|
+
target,
|
|
7675
|
+
fixtureRoot: options.fixtureRoot ?? (file ? dirname(file) : undefined),
|
|
7676
|
+
environmentFactory: options.environmentFactory,
|
|
7677
|
+
preflight: options.preflight,
|
|
7678
|
+
preflightOnly: options.preflightOnly,
|
|
7679
|
+
signal: options.signal,
|
|
7680
|
+
cleanupTimeoutMs: options.cleanupTimeoutMs,
|
|
7681
|
+
semanticJudge: options.semanticJudge,
|
|
7682
|
+
onEvent: options.onEvent
|
|
7683
|
+
});
|
|
7684
|
+
}
|
|
7685
|
+
|
|
7639
7686
|
// src/artemiskit.ts
|
|
7640
7687
|
var MUTATION_MAP = {
|
|
7641
7688
|
typo: TypoMutation,
|
|
@@ -7670,6 +7717,12 @@ class ArtemisKit {
|
|
|
7670
7717
|
this.storage = createStorageAdapter(config.storage);
|
|
7671
7718
|
}
|
|
7672
7719
|
}
|
|
7720
|
+
async createWorkflowSession(options) {
|
|
7721
|
+
return prepareWorkflowSession(options, this.config);
|
|
7722
|
+
}
|
|
7723
|
+
async runWorkflow(options) {
|
|
7724
|
+
return (await this.createWorkflowSession(options)).run();
|
|
7725
|
+
}
|
|
7673
7726
|
on(event, handler) {
|
|
7674
7727
|
if (!this.eventHandlers.has(event)) {
|
|
7675
7728
|
this.eventHandlers.set(event, new Set);
|
|
@@ -8414,7 +8467,7 @@ class ArtemisKit {
|
|
|
8414
8467
|
...this.config.providerConfig,
|
|
8415
8468
|
...options.providerConfig
|
|
8416
8469
|
};
|
|
8417
|
-
return
|
|
8470
|
+
return createAdapter2(adapterConfig);
|
|
8418
8471
|
}
|
|
8419
8472
|
buildMutations(mutationNames) {
|
|
8420
8473
|
if (!mutationNames || mutationNames.length === 0) {
|
|
@@ -8441,7 +8494,7 @@ class ArtemisKit {
|
|
|
8441
8494
|
}
|
|
8442
8495
|
}
|
|
8443
8496
|
function sleep(ms) {
|
|
8444
|
-
return new Promise((
|
|
8497
|
+
return new Promise((resolve2) => setTimeout(resolve2, ms));
|
|
8445
8498
|
}
|
|
8446
8499
|
function percentile(sortedArray, p) {
|
|
8447
8500
|
if (sortedArray.length === 0)
|
|
@@ -8455,13 +8508,26 @@ import {
|
|
|
8455
8508
|
AgentWorkflowSchema,
|
|
8456
8509
|
WorkflowPolicySchema,
|
|
8457
8510
|
WORKFLOW_TOOL_IDS,
|
|
8458
|
-
validateAgentWorkflow,
|
|
8511
|
+
validateAgentWorkflow as validateAgentWorkflow2,
|
|
8459
8512
|
parseAgentWorkflow,
|
|
8460
|
-
loadAgentWorkflow,
|
|
8513
|
+
loadAgentWorkflow as loadAgentWorkflow2,
|
|
8461
8514
|
listWorkflowTools,
|
|
8462
8515
|
getWorkflowTool,
|
|
8463
8516
|
executeSimulatedTool,
|
|
8464
|
-
createModelClientTarget
|
|
8517
|
+
createModelClientTarget as createModelClientTarget2,
|
|
8518
|
+
createAgentWorkflowSession as createAgentWorkflowSession2,
|
|
8519
|
+
runAgentWorkflow,
|
|
8520
|
+
evaluateWorkflowDeterministicOutcomes,
|
|
8521
|
+
evaluateWorkflowSemantics,
|
|
8522
|
+
isValidWorkflowJudgeOptions,
|
|
8523
|
+
decideWorkflowOutcome,
|
|
8524
|
+
validWorkflowAssertionCounts,
|
|
8525
|
+
readWorkflowRecord,
|
|
8526
|
+
createSimulatedWorkflowEnvironment,
|
|
8527
|
+
createDockerWorkflowEnvironment,
|
|
8528
|
+
createDockerWorkflowEnvironmentFactory,
|
|
8529
|
+
WORKFLOW_SANDBOX_IMAGE,
|
|
8530
|
+
WorkflowEnvironmentInitializationError
|
|
8465
8531
|
} from "@artemiskit/core";
|
|
8466
8532
|
import { isRunManifest, isRedTeamManifest, isStressManifest } from "@artemiskit/core";
|
|
8467
8533
|
|
|
@@ -10225,7 +10291,7 @@ function createInterceptor(client, config = {}) {
|
|
|
10225
10291
|
// src/guardian/policy.ts
|
|
10226
10292
|
var import_yaml = __toESM(require_dist(), 1);
|
|
10227
10293
|
import { existsSync, readFileSync } from "node:fs";
|
|
10228
|
-
import { resolve } from "node:path";
|
|
10294
|
+
import { resolve as resolve2 } from "node:path";
|
|
10229
10295
|
|
|
10230
10296
|
// ../../node_modules/zod/v3/external.js
|
|
10231
10297
|
var exports_external = {};
|
|
@@ -14277,7 +14343,7 @@ var GuardianPolicySchema = exports_external.object({
|
|
|
14277
14343
|
costLimits: CostLimitConfigSchema.optional()
|
|
14278
14344
|
});
|
|
14279
14345
|
function loadPolicy(filePath) {
|
|
14280
|
-
const absolutePath =
|
|
14346
|
+
const absolutePath = resolve2(filePath);
|
|
14281
14347
|
if (!existsSync(absolutePath)) {
|
|
14282
14348
|
throw new PolicyLoadError(`Policy file not found: ${absolutePath}`);
|
|
14283
14349
|
}
|
|
@@ -15197,12 +15263,15 @@ function createGuardian(config = {}) {
|
|
|
15197
15263
|
}
|
|
15198
15264
|
export {
|
|
15199
15265
|
validatePolicy,
|
|
15200
|
-
validateAgentWorkflow,
|
|
15266
|
+
validateAgentWorkflow2 as validateAgentWorkflow,
|
|
15267
|
+
validWorkflowAssertionCounts,
|
|
15201
15268
|
testCase,
|
|
15202
15269
|
similarity,
|
|
15203
15270
|
scenario,
|
|
15271
|
+
runAgentWorkflow,
|
|
15204
15272
|
regexCase,
|
|
15205
15273
|
regex,
|
|
15274
|
+
readWorkflowRecord,
|
|
15206
15275
|
parsePolicy,
|
|
15207
15276
|
parseAgentWorkflow,
|
|
15208
15277
|
notContains,
|
|
@@ -15210,11 +15279,12 @@ export {
|
|
|
15210
15279
|
mergePolicies,
|
|
15211
15280
|
matchPattern,
|
|
15212
15281
|
loadPolicy,
|
|
15213
|
-
loadAgentWorkflow,
|
|
15282
|
+
loadAgentWorkflow2 as loadAgentWorkflow,
|
|
15214
15283
|
llmGrade,
|
|
15215
15284
|
listWorkflowTools,
|
|
15216
15285
|
jsonSchema,
|
|
15217
15286
|
jsonCase,
|
|
15287
|
+
isValidWorkflowJudgeOptions,
|
|
15218
15288
|
isStressResult,
|
|
15219
15289
|
isStressManifestType,
|
|
15220
15290
|
isStressManifest,
|
|
@@ -15236,24 +15306,31 @@ export {
|
|
|
15236
15306
|
executeSimulatedTool,
|
|
15237
15307
|
exactCase,
|
|
15238
15308
|
exact,
|
|
15309
|
+
evaluateWorkflowSemantics,
|
|
15310
|
+
evaluateWorkflowDeterministicOutcomes,
|
|
15239
15311
|
detectPII,
|
|
15240
15312
|
detectInjection,
|
|
15241
15313
|
defineStorage,
|
|
15242
15314
|
definePlugin,
|
|
15243
15315
|
defineEvaluator,
|
|
15244
15316
|
defineAdapter,
|
|
15317
|
+
decideWorkflowOutcome,
|
|
15318
|
+
createSimulatedWorkflowEnvironment,
|
|
15245
15319
|
createSemanticValidator,
|
|
15246
15320
|
createPIIGuardrail,
|
|
15247
|
-
createModelClientTarget,
|
|
15321
|
+
createModelClientTarget2 as createModelClientTarget,
|
|
15248
15322
|
createInterceptor,
|
|
15249
15323
|
createIntentClassifier,
|
|
15250
15324
|
createInjectionGuardrail,
|
|
15251
15325
|
createGuardrails,
|
|
15252
15326
|
createGuardian,
|
|
15327
|
+
createDockerWorkflowEnvironmentFactory,
|
|
15328
|
+
createDockerWorkflowEnvironment,
|
|
15253
15329
|
createDefaultPolicy,
|
|
15254
15330
|
createDefaultActionValidator,
|
|
15255
15331
|
createCustomPatternGuardrail,
|
|
15256
15332
|
createContentFilterGuardrail,
|
|
15333
|
+
createAgentWorkflowSession2 as createAgentWorkflowSession,
|
|
15257
15334
|
containsCase,
|
|
15258
15335
|
contains,
|
|
15259
15336
|
calculateSuccessRate,
|
|
@@ -15263,7 +15340,9 @@ export {
|
|
|
15263
15340
|
anyOf,
|
|
15264
15341
|
allOf,
|
|
15265
15342
|
WorkflowPolicySchema,
|
|
15343
|
+
WorkflowEnvironmentInitializationError,
|
|
15266
15344
|
WORKFLOW_TOOL_IDS,
|
|
15345
|
+
WORKFLOW_SANDBOX_IMAGE,
|
|
15267
15346
|
TestCaseBuilder,
|
|
15268
15347
|
SemanticValidator,
|
|
15269
15348
|
ScenarioBuilder,
|
package/dist/types-only.d.ts
CHANGED
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
* } from '@artemiskit/sdk/types'
|
|
16
16
|
* ```
|
|
17
17
|
*/
|
|
18
|
-
export type { ArtemisKitConfig, RunOptions, RedTeamOptions, StressOptions, RunResult, RedTeamResult, StressResult, CaseStartEvent, CaseCompleteEvent, ProgressEvent, RedTeamMutationStartEvent, RedTeamMutationCompleteEvent, StressRequestCompleteEvent, CaseStartHandler, CaseCompleteHandler, ProgressHandler, RedTeamMutationStartHandler, RedTeamMutationCompleteHandler, StressRequestCompleteHandler, ArtemisKitEvents, ArtemisKitEventName, } from './types.js';
|
|
18
|
+
export type { ArtemisKitConfig, WorkflowRunOptions, RunOptions, RedTeamOptions, StressOptions, RunResult, RedTeamResult, StressResult, CaseStartEvent, CaseCompleteEvent, ProgressEvent, RedTeamMutationStartEvent, RedTeamMutationCompleteEvent, StressRequestCompleteEvent, CaseStartHandler, CaseCompleteHandler, ProgressHandler, RedTeamMutationStartHandler, RedTeamMutationCompleteHandler, StressRequestCompleteHandler, ArtemisKitEvents, ArtemisKitEventName, } from './types.js';
|
|
19
19
|
export type { Scenario, TestCase, Expected, Provider, ProviderConfig, ChatMessageType, Variables, } from '@artemiskit/core';
|
|
20
20
|
export type { CaseResult, CaseRedactionInfo, RunManifest, RunConfig, RunMetrics, RedTeamManifest, RedTeamConfig, RedTeamMetrics, RedTeamCaseResult, RedTeamStatus, RedTeamSeverity, StressManifest, StressConfig, StressMetrics, StressRequestResult, AnyManifest, ResolvedConfig, ConfigSource, ManifestRedactionInfo, CostEstimateInfo, GitInfo, ProvenanceInfo, } from '@artemiskit/core';
|
|
21
21
|
export type { ModelClient, ModelCapabilities, GenerateOptions, GenerateResult, TokenUsage, ChatMessage, FunctionDefinition, ToolDefinition, ToolCall, AdapterConfig, BaseAdapterConfig, OpenAIAdapterConfig, AzureOpenAIAdapterConfig, VercelAIAdapterConfig, AnthropicAdapterConfig, LangChainAdapterConfig, DeepAgentsAdapterConfig, ProviderType, } from '@artemiskit/core';
|
|
@@ -28,4 +28,5 @@ export type { ArtemisKitMatchers, MatcherResult } from './matchers/index.js';
|
|
|
28
28
|
export type { ProviderName, ExpectationType, AnyResult, ExtractRunCases, ExtractRedTeamCases, ExtractStressResults, ExtractManifest, DeepPartial, PartialScenario, RequireFields, OptionalFields, StrictAdapterConfig, } from './utils/index.js';
|
|
29
29
|
export type { AdapterContract, EvaluatorContract, StorageContract, AdapterFactory, EvaluatorFactory, StorageFactory, ArtemisKitPlugin, } from './contracts/index.js';
|
|
30
30
|
export type { ScenarioBuilder, TestCaseBuilder } from './builders/index.js';
|
|
31
|
+
export type { AgentWorkflow, WorkflowPolicy, WorkflowJson, WorkflowToolId, WorkflowResource, WorkflowToolDescriptor, SimulatedToolEvidence, SimulatedToolResult, AgentTarget, AgentTurnRequest, AgentTurnResult, AgentTargetCapabilities, AgentTargetFailure, AgentWorkflowExecution, AgentWorkflowReason, AgentWorkflowEvent, AgentWorkflowRecord, AgentWorkflowExecutionRecord, AgentWorkflowEvidence, WorkflowDeterministicStatus, WorkflowDeterministicReason, WorkflowDeterministicAssertion, WorkflowDeterministicSummary, WorkflowJudgeOptions, WorkflowSemanticOptions, WorkflowSemanticReason, WorkflowSemanticAssertion, WorkflowSemanticIdentity, WorkflowSemanticEvaluation, WorkflowOutcomeStatus, WorkflowAssertionCounts, WorkflowOutcomeDecision, WorkflowOutcomeAssessment, SavedWorkflowRecord, HistoricalWorkflowRecord, AgentWorkflowResult, AgentWorkflowSessionOptions, AgentWorkflowSession, WorkflowState, WorkflowEnvironment, WorkflowEnvironmentFactory, WorkflowEnvironmentCleanup, DockerWorkflowEnvironmentOptions, WorkflowEnvironmentInitializationError, } from '@artemiskit/core';
|
|
31
32
|
//# sourceMappingURL=types-only.d.ts.map
|
package/dist/types-only.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types-only.d.ts","sourceRoot":"","sources":["../src/types-only.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAMH,YAAY,EAEV,gBAAgB,EAChB,UAAU,EACV,cAAc,EACd,aAAa,EAEb,SAAS,EACT,aAAa,EACb,YAAY,EAEZ,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,yBAAyB,EACzB,4BAA4B,EAC5B,0BAA0B,EAE1B,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAE5B,gBAAgB,EAChB,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,cAAc,EACd,eAAe,EACf,SAAS,GACV,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,UAAU,EACV,iBAAiB,EAEjB,WAAW,EACX,SAAS,EACT,UAAU,EAEV,eAAe,EACf,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,eAAe,EAEf,cAAc,EACd,YAAY,EACZ,aAAa,EACb,mBAAmB,EAEnB,WAAW,EACX,cAAc,EACd,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,OAAO,EACP,cAAc,GACf,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,WAAW,EACX,iBAAiB,EAEjB,eAAe,EACf,cAAc,EACd,UAAU,EAEV,WAAW,EAEX,kBAAkB,EAClB,cAAc,EACd,QAAQ,EAER,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,YAAY,GACb,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMrF,YAAY,EACV,cAAc,EACd,aAAa,EACb,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EAEtB,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EACtB,eAAe,EACf,mBAAmB,EACnB,cAAc,EACd,uBAAuB,GACxB,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMxD,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAM7E,YAAY,EAEV,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,sBAAsB,EACtB,gBAAgB,EAChB,iBAAiB,EAEjB,YAAY,EACZ,iBAAiB,EACjB,eAAe,EACf,aAAa,EACb,SAAS,EACT,eAAe,EAEf,UAAU,EACV,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,eAAe,EACf,eAAe,EAEf,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EAEnB,oBAAoB,EACpB,cAAc,EAEd,YAAY,EACZ,OAAO,EACP,WAAW,EACX,kBAAkB,EAClB,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,eAAe,EAEf,eAAe,EACf,mBAAmB,EACnB,YAAY,EAEZ,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EAEpB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EAEpB,aAAa,EACb,0BAA0B,GAC3B,MAAM,YAAY,CAAC;AAMpB,YAAY,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAMpE,YAAY,EAEV,YAAY,EACZ,eAAe,EAEf,SAAS,EACT,eAAe,EACf,mBAAmB,EACnB,oBAAoB,EACpB,eAAe,EAEf,WAAW,EACX,eAAe,EACf,aAAa,EACb,cAAc,EACd,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,aAAa,CAAC;AAMrB,YAAY,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC"}
|
|
1
|
+
{"version":3,"file":"types-only.d.ts","sourceRoot":"","sources":["../src/types-only.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAMH,YAAY,EAEV,gBAAgB,EAChB,kBAAkB,EAClB,UAAU,EACV,cAAc,EACd,aAAa,EAEb,SAAS,EACT,aAAa,EACb,YAAY,EAEZ,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,yBAAyB,EACzB,4BAA4B,EAC5B,0BAA0B,EAE1B,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAE5B,gBAAgB,EAChB,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,cAAc,EACd,eAAe,EACf,SAAS,GACV,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,UAAU,EACV,iBAAiB,EAEjB,WAAW,EACX,SAAS,EACT,UAAU,EAEV,eAAe,EACf,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,eAAe,EAEf,cAAc,EACd,YAAY,EACZ,aAAa,EACb,mBAAmB,EAEnB,WAAW,EACX,cAAc,EACd,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,OAAO,EACP,cAAc,GACf,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,WAAW,EACX,iBAAiB,EAEjB,eAAe,EACf,cAAc,EACd,UAAU,EAEV,WAAW,EAEX,kBAAkB,EAClB,cAAc,EACd,QAAQ,EAER,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,YAAY,GACb,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMrF,YAAY,EACV,cAAc,EACd,aAAa,EACb,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EAEtB,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EACtB,eAAe,EACf,mBAAmB,EACnB,cAAc,EACd,uBAAuB,GACxB,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMxD,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAM7E,YAAY,EAEV,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,sBAAsB,EACtB,gBAAgB,EAChB,iBAAiB,EAEjB,YAAY,EACZ,iBAAiB,EACjB,eAAe,EACf,aAAa,EACb,SAAS,EACT,eAAe,EAEf,UAAU,EACV,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,eAAe,EACf,eAAe,EAEf,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EAEnB,oBAAoB,EACpB,cAAc,EAEd,YAAY,EACZ,OAAO,EACP,WAAW,EACX,kBAAkB,EAClB,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,eAAe,EAEf,eAAe,EACf,mBAAmB,EACnB,YAAY,EAEZ,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EAEpB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EAEpB,aAAa,EACb,0BAA0B,GAC3B,MAAM,YAAY,CAAC;AAMpB,YAAY,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAMpE,YAAY,EAEV,YAAY,EACZ,eAAe,EAEf,SAAS,EACT,eAAe,EACf,mBAAmB,EACnB,oBAAoB,EACpB,eAAe,EAEf,WAAW,EACX,eAAe,EACf,aAAa,EACb,cAAc,EACd,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,aAAa,CAAC;AAMrB,YAAY,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAEnE,YAAY,EACV,aAAa,EACb,cAAc,EACd,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,sBAAsB,EACtB,qBAAqB,EACrB,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,eAAe,EACf,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,kBAAkB,EAClB,mBAAmB,EACnB,4BAA4B,EAC5B,qBAAqB,EACrB,2BAA2B,EAC3B,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAC5B,oBAAoB,EACpB,uBAAuB,EACvB,sBAAsB,EACtB,yBAAyB,EACzB,wBAAwB,EACxB,0BAA0B,EAC1B,qBAAqB,EACrB,uBAAuB,EACvB,uBAAuB,EACvB,yBAAyB,EACzB,mBAAmB,EACnB,wBAAwB,EACxB,mBAAmB,EACnB,2BAA2B,EAC3B,oBAAoB,EACpB,aAAa,EACb,mBAAmB,EACnB,0BAA0B,EAC1B,0BAA0B,EAC1B,gCAAgC,EAChC,sCAAsC,GACvC,MAAM,kBAAkB,CAAC"}
|
package/dist/types.d.ts
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* @artemiskit/sdk
|
|
3
3
|
* Type definitions for the programmatic SDK
|
|
4
4
|
*/
|
|
5
|
-
import type { AdapterConfig, CaseResult, ComparisonEligibility, ModelClient, RedTeamManifest, RunManifest, StorageConfig, StressManifest, StressRequestResult, ToolExecutor } from '@artemiskit/core';
|
|
5
|
+
import type { AdapterConfig, AgentTarget, AgentWorkflow, AgentWorkflowSessionOptions, CaseResult, ComparisonEligibility, ModelClient, RedTeamManifest, RunManifest, StorageConfig, StressManifest, StressRequestResult, ToolExecutor } from '@artemiskit/core';
|
|
6
6
|
import type { RedactionConfig } from '@artemiskit/core';
|
|
7
7
|
import type { Scenario } from '@artemiskit/core';
|
|
8
8
|
/**
|
|
@@ -380,4 +380,15 @@ export interface CompareResult {
|
|
|
380
380
|
/** Threshold used for regression detection */
|
|
381
381
|
threshold: number;
|
|
382
382
|
}
|
|
383
|
+
/** Controlled workflow execution. Text/state remain in memory unless the caller explicitly saves them. */
|
|
384
|
+
export interface WorkflowRunOptions extends Omit<AgentWorkflowSessionOptions, 'workflow' | 'target'> {
|
|
385
|
+
/** Workflow object or YAML path. File fixtures default to the workflow file's directory. */
|
|
386
|
+
workflow: AgentWorkflow | string;
|
|
387
|
+
/** Explicit host-owned client. Its transport retry behavior remains the caller's responsibility. */
|
|
388
|
+
client?: ModelClient;
|
|
389
|
+
/** Explicit custom target, mutually exclusive with client. */
|
|
390
|
+
target?: AgentTarget;
|
|
391
|
+
/** Trusted transport configuration; workflow provider/model and zero retries remain authoritative. */
|
|
392
|
+
providerConfig?: Partial<AdapterConfig>;
|
|
393
|
+
}
|
|
383
394
|
//# sourceMappingURL=types.d.ts.map
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EACV,aAAa,EACb,UAAU,EACV,qBAAqB,EACrB,WAAW,EACX,eAAe,EACf,WAAW,EACX,aAAa,EACb,cAAc,EACd,mBAAmB,EACnB,YAAY,EACb,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAMjD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,UAAU,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,OAAO,GAAG,SAAS,GAAG,UAAU,CAAC;IACxC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC3C,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;IAChD,QAAQ,EAAE,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAC;IAC1D,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,0BAA0B;IACzC,MAAM,EAAE,mBAAmB,CAAC;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;CACpB;AAMD,MAAM,MAAM,gBAAgB,GAAG,CAAC,KAAK,EAAE,cAAc,KAAK,IAAI,CAAC;AAC/D,MAAM,MAAM,mBAAmB,GAAG,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,CAAC;AACrE,MAAM,MAAM,eAAe,GAAG,CAAC,KAAK,EAAE,aAAa,KAAK,IAAI,CAAC;AAC7D,MAAM,MAAM,2BAA2B,GAAG,CAAC,KAAK,EAAE,yBAAyB,KAAK,IAAI,CAAC;AACrF,MAAM,MAAM,8BAA8B,GAAG,CAAC,KAAK,EAAE,4BAA4B,KAAK,IAAI,CAAC;AAC3F,MAAM,MAAM,4BAA4B,GAAG,CAAC,KAAK,EAAE,0BAA0B,KAAK,IAAI,CAAC;AAMvF;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,yCAAyC;IACzC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,2BAA2B;IAC3B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,sCAAsC;IACtC,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,iCAAiC;IACjC,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,+CAA+C;IAC/C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,gCAAgC;IAChC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iCAAiC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,2EAA2E;IAC3E,YAAY,CAAC,EAAE,YAAY,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,wCAAwC;IACxC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,mCAAmC;IACnC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gCAAgC;IAChC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+BAA+B;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gCAAgC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,kDAAkD;IAClD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B;AAMD;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,6BAA6B;IAC7B,QAAQ,EAAE,WAAW,CAAC;IACtB,8BAA8B;IAC9B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,+BAA+B;IAC/B,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,6BAA6B;IAC7B,QAAQ,EAAE,eAAe,CAAC;IAC1B,4DAA4D;IAC5D,OAAO,EAAE,OAAO,CAAC;IACjB,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,gCAAgC;IAChC,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,6BAA6B;IAC7B,QAAQ,EAAE,cAAc,CAAC;IACzB,qDAAqD;IACrD,OAAO,EAAE,OAAO,CAAC;IACjB,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,mCAAmC;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,kCAAkC;IAClC,YAAY,EAAE,MAAM,CAAC;CACtB;AAMD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,cAAc,CAAC;IAC1B,YAAY,EAAE,iBAAiB,CAAC;IAChC,QAAQ,EAAE,aAAa,CAAC;IACxB,oBAAoB,EAAE,yBAAyB,CAAC;IAChD,uBAAuB,EAAE,4BAA4B,CAAC;IACtD,qBAAqB,EAAE,0BAA0B,CAAC;CACnD;AAED,MAAM,MAAM,mBAAmB,GAAG,MAAM,gBAAgB,CAAC;AAMzD;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B;;;OAGG;IACH,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAE5B;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,qCAAqC;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,oBAAoB;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,sBAAsB;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,gBAAgB;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,oBAAoB;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,2BAA2B;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,qCAAqC;IACrC,KAAK,EAAE,OAAO,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,gCAAgC;IAChC,KAAK,EAAE,OAAO,CAAC;IACf,6CAA6C;IAC7C,SAAS,EAAE,kBAAkB,EAAE,CAAC;IAChC,wBAAwB;IACxB,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,0CAA0C;IAC1C,QAAQ,EAAE,iBAAiB,EAAE,CAAC;CAC/B;AAMD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;;OAGG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,qBAAqB;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,6BAA6B;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,6BAA6B;IAC7B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,+DAA+D;IAC/D,mBAAmB,EAAE,OAAO,CAAC;IAC7B,iFAAiF;IACjF,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,0DAA0D;IAC1D,WAAW,EAAE,KAAK,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,cAAc,EAAE,MAAM,CAAC;QACvB,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC,CAAC;IACH,0DAA0D;IAC1D,SAAS,EAAE,KAAK,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,cAAc,EAAE,MAAM,CAAC;QACvB,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC,CAAC;IACH,kCAAkC;IAClC,SAAS,EAAE,KAAK,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;IACH,2EAA2E;IAC3E,UAAU,EAAE,KAAK,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;IACH,+EAA+E;IAC/E,YAAY,EAAE,KAAK,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,2BAA2B;IAC3B,QAAQ,EAAE,UAAU,CAAC;IACrB,0BAA0B;IAC1B,OAAO,EAAE,UAAU,CAAC;IACpB,0BAA0B;IAC1B,UAAU,EAAE,iBAAiB,CAAC;IAC9B,oEAAoE;IACpE,WAAW,EAAE,qBAAqB,CAAC;IACnC,6EAA6E;IAC7E,aAAa,EAAE,OAAO,CAAC;IACvB,8CAA8C;IAC9C,SAAS,EAAE,MAAM,CAAC;CACnB"}
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EACV,aAAa,EACb,WAAW,EACX,aAAa,EACb,2BAA2B,EAC3B,UAAU,EACV,qBAAqB,EACrB,WAAW,EACX,eAAe,EACf,WAAW,EACX,aAAa,EACb,cAAc,EACd,mBAAmB,EACnB,YAAY,EACb,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAMjD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,UAAU,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,OAAO,GAAG,SAAS,GAAG,UAAU,CAAC;IACxC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC3C,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;IAChD,QAAQ,EAAE,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAC;IAC1D,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,0BAA0B;IACzC,MAAM,EAAE,mBAAmB,CAAC;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;CACpB;AAMD,MAAM,MAAM,gBAAgB,GAAG,CAAC,KAAK,EAAE,cAAc,KAAK,IAAI,CAAC;AAC/D,MAAM,MAAM,mBAAmB,GAAG,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,CAAC;AACrE,MAAM,MAAM,eAAe,GAAG,CAAC,KAAK,EAAE,aAAa,KAAK,IAAI,CAAC;AAC7D,MAAM,MAAM,2BAA2B,GAAG,CAAC,KAAK,EAAE,yBAAyB,KAAK,IAAI,CAAC;AACrF,MAAM,MAAM,8BAA8B,GAAG,CAAC,KAAK,EAAE,4BAA4B,KAAK,IAAI,CAAC;AAC3F,MAAM,MAAM,4BAA4B,GAAG,CAAC,KAAK,EAAE,0BAA0B,KAAK,IAAI,CAAC;AAMvF;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,yCAAyC;IACzC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,2BAA2B;IAC3B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,sCAAsC;IACtC,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,iCAAiC;IACjC,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,+CAA+C;IAC/C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,gCAAgC;IAChC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iCAAiC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,2EAA2E;IAC3E,YAAY,CAAC,EAAE,YAAY,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,wCAAwC;IACxC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,mCAAmC;IACnC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gCAAgC;IAChC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+BAA+B;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gCAAgC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,kDAAkD;IAClD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B;AAMD;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,6BAA6B;IAC7B,QAAQ,EAAE,WAAW,CAAC;IACtB,8BAA8B;IAC9B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,+BAA+B;IAC/B,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,6BAA6B;IAC7B,QAAQ,EAAE,eAAe,CAAC;IAC1B,4DAA4D;IAC5D,OAAO,EAAE,OAAO,CAAC;IACjB,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,gCAAgC;IAChC,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,6BAA6B;IAC7B,QAAQ,EAAE,cAAc,CAAC;IACzB,qDAAqD;IACrD,OAAO,EAAE,OAAO,CAAC;IACjB,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,mCAAmC;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,kCAAkC;IAClC,YAAY,EAAE,MAAM,CAAC;CACtB;AAMD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,cAAc,CAAC;IAC1B,YAAY,EAAE,iBAAiB,CAAC;IAChC,QAAQ,EAAE,aAAa,CAAC;IACxB,oBAAoB,EAAE,yBAAyB,CAAC;IAChD,uBAAuB,EAAE,4BAA4B,CAAC;IACtD,qBAAqB,EAAE,0BAA0B,CAAC;CACnD;AAED,MAAM,MAAM,mBAAmB,GAAG,MAAM,gBAAgB,CAAC;AAMzD;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B;;;OAGG;IACH,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAE5B;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,qCAAqC;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,oBAAoB;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,sBAAsB;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,gBAAgB;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,oBAAoB;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,2BAA2B;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,qCAAqC;IACrC,KAAK,EAAE,OAAO,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,gCAAgC;IAChC,KAAK,EAAE,OAAO,CAAC;IACf,6CAA6C;IAC7C,SAAS,EAAE,kBAAkB,EAAE,CAAC;IAChC,wBAAwB;IACxB,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,0CAA0C;IAC1C,QAAQ,EAAE,iBAAiB,EAAE,CAAC;CAC/B;AAMD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;;OAGG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,qBAAqB;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,6BAA6B;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,6BAA6B;IAC7B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,+DAA+D;IAC/D,mBAAmB,EAAE,OAAO,CAAC;IAC7B,iFAAiF;IACjF,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,0DAA0D;IAC1D,WAAW,EAAE,KAAK,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,cAAc,EAAE,MAAM,CAAC;QACvB,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC,CAAC;IACH,0DAA0D;IAC1D,SAAS,EAAE,KAAK,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,cAAc,EAAE,MAAM,CAAC;QACvB,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC,CAAC;IACH,kCAAkC;IAClC,SAAS,EAAE,KAAK,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;IACH,2EAA2E;IAC3E,UAAU,EAAE,KAAK,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;IACH,+EAA+E;IAC/E,YAAY,EAAE,KAAK,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,2BAA2B;IAC3B,QAAQ,EAAE,UAAU,CAAC;IACrB,0BAA0B;IAC1B,OAAO,EAAE,UAAU,CAAC;IACpB,0BAA0B;IAC1B,UAAU,EAAE,iBAAiB,CAAC;IAC9B,oEAAoE;IACpE,WAAW,EAAE,qBAAqB,CAAC;IACnC,6EAA6E;IAC7E,aAAa,EAAE,OAAO,CAAC;IACvB,8CAA8C;IAC9C,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,0GAA0G;AAC1G,MAAM,WAAW,kBACf,SAAQ,IAAI,CAAC,2BAA2B,EAAE,UAAU,GAAG,QAAQ,CAAC;IAChE,4FAA4F;IAC5F,QAAQ,EAAE,aAAa,GAAG,MAAM,CAAC;IACjC,oGAAoG;IACpG,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,8DAA8D;IAC9D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,sGAAsG;IACtG,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;CACzC"}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import { type AgentWorkflowSession } from '@artemiskit/core';
|
|
2
|
+
import type { ArtemisKitConfig, WorkflowRunOptions } from './types.js';
|
|
3
|
+
export declare function prepareWorkflowSession(options: WorkflowRunOptions, config: ArtemisKitConfig): Promise<AgentWorkflowSession>;
|
|
4
|
+
//# sourceMappingURL=workflow.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"workflow.d.ts","sourceRoot":"","sources":["../src/workflow.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,KAAK,oBAAoB,EAO1B,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAEpE,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,kBAAkB,EAC3B,MAAM,EAAE,gBAAgB,GACvB,OAAO,CAAC,oBAAoB,CAAC,CAmC/B"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@artemiskit/sdk",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.1",
|
|
4
4
|
"description": "Programmatic SDK for ArtemisKit LLM evaluation toolkit - integrate into your Node.js apps",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -71,8 +71,8 @@
|
|
|
71
71
|
"test:coverage": "bun test --coverage"
|
|
72
72
|
},
|
|
73
73
|
"dependencies": {
|
|
74
|
-
"@artemiskit/core": "0.6.
|
|
75
|
-
"@artemiskit/redteam": "0.3.
|
|
74
|
+
"@artemiskit/core": "0.6.2",
|
|
75
|
+
"@artemiskit/redteam": "0.3.10"
|
|
76
76
|
},
|
|
77
77
|
"devDependencies": {
|
|
78
78
|
"@types/bun": "^1.1.0",
|
|
@@ -0,0 +1,391 @@
|
|
|
1
|
+
import { afterEach, describe, expect, test } from 'bun:test';
|
|
2
|
+
import { mkdtemp, readdir, writeFile } from 'node:fs/promises';
|
|
3
|
+
import { tmpdir } from 'node:os';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
import { fileURLToPath } from 'node:url';
|
|
6
|
+
import {
|
|
7
|
+
type AgentTarget,
|
|
8
|
+
type AgentWorkflow,
|
|
9
|
+
type AgentWorkflowEvent,
|
|
10
|
+
createSimulatedWorkflowEnvironment,
|
|
11
|
+
validateAgentWorkflow,
|
|
12
|
+
} from '@artemiskit/core';
|
|
13
|
+
import { OpenAIAdapter } from '../../../adapters/openai/src/client';
|
|
14
|
+
import { ArtemisKit } from '../artemiskit';
|
|
15
|
+
import { readWorkflowRecord } from '../index';
|
|
16
|
+
import type { WorkflowJudgeOptions, WorkflowRunOptions } from '../types-only';
|
|
17
|
+
|
|
18
|
+
const servers: ReturnType<typeof Bun.serve>[] = [];
|
|
19
|
+
afterEach(() => {
|
|
20
|
+
for (const server of servers.splice(0)) server.stop(true);
|
|
21
|
+
});
|
|
22
|
+
function workflow(): AgentWorkflow {
|
|
23
|
+
return validateAgentWorkflow({
|
|
24
|
+
version: '1',
|
|
25
|
+
kind: 'agent_workflow',
|
|
26
|
+
name: 'sdk-workflow',
|
|
27
|
+
target: {
|
|
28
|
+
provider: 'openai',
|
|
29
|
+
model: 'workflow-authoritative',
|
|
30
|
+
generation: { max_tokens: 256 },
|
|
31
|
+
},
|
|
32
|
+
environment: {
|
|
33
|
+
type: 'simulated',
|
|
34
|
+
policy: {
|
|
35
|
+
network: 'denied',
|
|
36
|
+
side_effects: 'denied',
|
|
37
|
+
permissions: { files: 'write' },
|
|
38
|
+
budgets: { max_actions: 10, max_tokens: 4096, timeout_ms: 1000 },
|
|
39
|
+
},
|
|
40
|
+
},
|
|
41
|
+
tools: ['write_file'],
|
|
42
|
+
workflow: {
|
|
43
|
+
system_instructions: 'PRIVATE-INSTRUCTIONS',
|
|
44
|
+
initial_state: { files: {} },
|
|
45
|
+
turns: [{ role: 'user', content: 'PRIVATE-PROMPT' }],
|
|
46
|
+
},
|
|
47
|
+
outcomes: { deterministic: [{ type: 'file', path: 'result.txt', exists: true }] },
|
|
48
|
+
evidence: { trace: 'summary', artifacts: 'checksums', redact: true },
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
function target(): AgentTarget {
|
|
52
|
+
return {
|
|
53
|
+
provider: 'custom',
|
|
54
|
+
capabilities: async () => ({ status: 'available', toolUse: true, transportCancellation: true }),
|
|
55
|
+
turn: async () => ({
|
|
56
|
+
status: 'completed',
|
|
57
|
+
id: 'fixture',
|
|
58
|
+
model: 'fixture',
|
|
59
|
+
message: { role: 'assistant', content: 'PRIVATE-ANSWER' },
|
|
60
|
+
tokens: { prompt: 2, completion: 1, total: 3 },
|
|
61
|
+
latencyMs: 1,
|
|
62
|
+
}),
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
describe('SDK native workflow session', () => {
|
|
67
|
+
test('forwards explicit independent semantic judging with separate coverage, usage and record reading', async () => {
|
|
68
|
+
for (const verdict of ['pass', 'fail', 'malformed', 'unconfigured']) {
|
|
69
|
+
const scenario = workflow();
|
|
70
|
+
scenario.outcomes.deterministic = [{ type: 'file', path: 'result.txt', exists: false }];
|
|
71
|
+
scenario.outcomes.semantic = [
|
|
72
|
+
{ type: 'llm_judge', mode: 'strict_assurance', rubric: 'PRIVATE-RUBRIC' },
|
|
73
|
+
];
|
|
74
|
+
let calls = 0;
|
|
75
|
+
let closed = 0;
|
|
76
|
+
const semanticJudge: WorkflowJudgeOptions = {
|
|
77
|
+
provider: 'independent-judge',
|
|
78
|
+
model: 'judge-model',
|
|
79
|
+
limits: { maxRequests: 1, maxTokens: 100, maxOutputTokens: 10, timeoutMs: 1000 },
|
|
80
|
+
client: {
|
|
81
|
+
provider: 'independent-judge',
|
|
82
|
+
capabilities: async () => ({
|
|
83
|
+
streaming: false,
|
|
84
|
+
functionCalling: false,
|
|
85
|
+
toolUse: false,
|
|
86
|
+
maxContext: 10000,
|
|
87
|
+
}),
|
|
88
|
+
generate: async (request) => {
|
|
89
|
+
calls++;
|
|
90
|
+
expect(request.model).toBe('judge-model');
|
|
91
|
+
expect(request.maxRetries).toBe(0);
|
|
92
|
+
return {
|
|
93
|
+
id: 'judge',
|
|
94
|
+
model: 'observed-judge',
|
|
95
|
+
text: verdict === 'malformed' ? 'PRIVATE-BAD-RESPONSE' : JSON.stringify({ verdict }),
|
|
96
|
+
tokens: { prompt: 5, completion: 2, total: 7 },
|
|
97
|
+
latencyMs: 1,
|
|
98
|
+
};
|
|
99
|
+
},
|
|
100
|
+
close: async () => {
|
|
101
|
+
closed++;
|
|
102
|
+
},
|
|
103
|
+
},
|
|
104
|
+
};
|
|
105
|
+
const result = await new ArtemisKit().runWorkflow({
|
|
106
|
+
workflow: scenario,
|
|
107
|
+
target: target(),
|
|
108
|
+
...(verdict === 'unconfigured' ? {} : { semanticJudge }),
|
|
109
|
+
});
|
|
110
|
+
expect(result.record.execution).toBe('completed');
|
|
111
|
+
expect(result.record.taskVerification).toBe(
|
|
112
|
+
verdict === 'pass'
|
|
113
|
+
? 'passed'
|
|
114
|
+
: verdict === 'fail'
|
|
115
|
+
? 'failed'
|
|
116
|
+
: verdict === 'malformed'
|
|
117
|
+
? 'invalid'
|
|
118
|
+
: 'unavailable'
|
|
119
|
+
);
|
|
120
|
+
expect(result.record.usage.reported.total).toBe(3);
|
|
121
|
+
expect(result.record.outcomes.deterministic.counts.valid).toBe(1);
|
|
122
|
+
expect(result.record.outcomes.semantic.usage.reported.total).toBe(
|
|
123
|
+
verdict === 'unconfigured' ? 0 : 7
|
|
124
|
+
);
|
|
125
|
+
expect(calls).toBe(verdict === 'unconfigured' ? 0 : 1);
|
|
126
|
+
expect(closed).toBe(0);
|
|
127
|
+
expect(readWorkflowRecord(JSON.parse(JSON.stringify(result.record)))).toEqual(result.record);
|
|
128
|
+
expect(JSON.stringify(result.record)).not.toContain('PRIVATE-');
|
|
129
|
+
}
|
|
130
|
+
});
|
|
131
|
+
test('malformed explicit judge bounds stop before any target model call', async () => {
|
|
132
|
+
const scenario = workflow();
|
|
133
|
+
let turns = 0;
|
|
134
|
+
const custom = target();
|
|
135
|
+
custom.turn = async () => {
|
|
136
|
+
turns++;
|
|
137
|
+
throw new Error('must not call');
|
|
138
|
+
};
|
|
139
|
+
const result = await new ArtemisKit().runWorkflow({
|
|
140
|
+
workflow: scenario,
|
|
141
|
+
target: custom,
|
|
142
|
+
semanticJudge: {
|
|
143
|
+
provider: 'judge',
|
|
144
|
+
model: 'judge',
|
|
145
|
+
limits: { maxRequests: 0, maxTokens: 100, maxOutputTokens: 10, timeoutMs: 1000 },
|
|
146
|
+
client: {
|
|
147
|
+
provider: 'judge',
|
|
148
|
+
capabilities: async () => {
|
|
149
|
+
throw new Error('must not call');
|
|
150
|
+
},
|
|
151
|
+
generate: async () => {
|
|
152
|
+
throw new Error('must not call');
|
|
153
|
+
},
|
|
154
|
+
},
|
|
155
|
+
},
|
|
156
|
+
});
|
|
157
|
+
expect(turns).toBe(0);
|
|
158
|
+
expect(result.record.execution).toBe('invalid');
|
|
159
|
+
});
|
|
160
|
+
test('deterministic failure gates judge calls and session cancellation reaches the independent evaluator', async () => {
|
|
161
|
+
let started!: () => void;
|
|
162
|
+
const pending = new Promise<void>((resolve) => {
|
|
163
|
+
started = resolve;
|
|
164
|
+
});
|
|
165
|
+
let calls = 0;
|
|
166
|
+
const semanticJudge: WorkflowJudgeOptions = {
|
|
167
|
+
provider: 'judge',
|
|
168
|
+
model: 'judge',
|
|
169
|
+
limits: { maxRequests: 1, maxTokens: 100, maxOutputTokens: 10, timeoutMs: 1000 },
|
|
170
|
+
client: {
|
|
171
|
+
provider: 'judge',
|
|
172
|
+
capabilities: async () => ({
|
|
173
|
+
streaming: false,
|
|
174
|
+
functionCalling: false,
|
|
175
|
+
toolUse: false,
|
|
176
|
+
maxContext: 10000,
|
|
177
|
+
}),
|
|
178
|
+
generate: async (request) => {
|
|
179
|
+
calls++;
|
|
180
|
+
started();
|
|
181
|
+
await new Promise<void>((resolve) =>
|
|
182
|
+
request.signal?.addEventListener('abort', () => resolve(), { once: true })
|
|
183
|
+
);
|
|
184
|
+
throw new Error('PRIVATE-ABORT');
|
|
185
|
+
},
|
|
186
|
+
},
|
|
187
|
+
};
|
|
188
|
+
const scenario = workflow();
|
|
189
|
+
scenario.outcomes.semantic = [
|
|
190
|
+
{ type: 'llm_judge', mode: 'strict_assurance', rubric: 'PRIVATE-RUBRIC' },
|
|
191
|
+
];
|
|
192
|
+
const kit = new ArtemisKit();
|
|
193
|
+
const failed = await kit.runWorkflow({ workflow: scenario, target: target(), semanticJudge });
|
|
194
|
+
expect(failed.record.taskVerification).toBe('failed');
|
|
195
|
+
expect(calls).toBe(0);
|
|
196
|
+
scenario.outcomes.deterministic = [{ type: 'file', path: 'result.txt', exists: false }];
|
|
197
|
+
const session = await kit.createWorkflowSession({
|
|
198
|
+
workflow: scenario,
|
|
199
|
+
target: target(),
|
|
200
|
+
semanticJudge,
|
|
201
|
+
});
|
|
202
|
+
const finished = session.run();
|
|
203
|
+
await pending;
|
|
204
|
+
session.cancel();
|
|
205
|
+
const result = await finished;
|
|
206
|
+
expect(result.record.execution).toBe('completed');
|
|
207
|
+
expect(result.record.outcomes.cancelled).toBe(true);
|
|
208
|
+
expect(result.record.outcomes.reason).toBe('cancelled');
|
|
209
|
+
expect(result.record.outcomes.semantic.counts.valid).toBe(0);
|
|
210
|
+
expect(readWorkflowRecord(result.record)).toEqual(result.record);
|
|
211
|
+
});
|
|
212
|
+
|
|
213
|
+
test('configured adapter uses workflow identity and zero retries with real loopback multi-turn execution', async () => {
|
|
214
|
+
const bodies: Record<string, unknown>[] = [];
|
|
215
|
+
const server = Bun.serve({
|
|
216
|
+
hostname: '127.0.0.1',
|
|
217
|
+
port: 0,
|
|
218
|
+
async fetch(request) {
|
|
219
|
+
bodies.push(await request.json());
|
|
220
|
+
const first = bodies.length === 1;
|
|
221
|
+
return Response.json({
|
|
222
|
+
id: 'fixture',
|
|
223
|
+
model: 'observed-model',
|
|
224
|
+
choices: [
|
|
225
|
+
{
|
|
226
|
+
index: 0,
|
|
227
|
+
message: {
|
|
228
|
+
role: 'assistant',
|
|
229
|
+
content: first ? null : 'PRIVATE-ANSWER',
|
|
230
|
+
...(first
|
|
231
|
+
? {
|
|
232
|
+
tool_calls: [
|
|
233
|
+
{
|
|
234
|
+
id: 'private-id',
|
|
235
|
+
type: 'function',
|
|
236
|
+
function: {
|
|
237
|
+
name: 'write_file',
|
|
238
|
+
arguments: '{"path":"result.txt","content":"PRIVATE-STATE"}',
|
|
239
|
+
},
|
|
240
|
+
},
|
|
241
|
+
],
|
|
242
|
+
}
|
|
243
|
+
: {}),
|
|
244
|
+
},
|
|
245
|
+
finish_reason: first ? 'tool_calls' : 'stop',
|
|
246
|
+
},
|
|
247
|
+
],
|
|
248
|
+
usage: { prompt_tokens: 2, completion_tokens: 1, total_tokens: 3 },
|
|
249
|
+
});
|
|
250
|
+
},
|
|
251
|
+
});
|
|
252
|
+
servers.push(server);
|
|
253
|
+
// Legacy SDK tests mock the core factory globally; qualify the real public boundary in a fresh process.
|
|
254
|
+
const directory = await mkdtemp(join(tmpdir(), 'artemis-sdk-client-'));
|
|
255
|
+
const workflowPath = join(directory, 'workflow.json');
|
|
256
|
+
await writeFile(workflowPath, JSON.stringify(workflow()));
|
|
257
|
+
const child = Bun.spawn(
|
|
258
|
+
[
|
|
259
|
+
process.execPath,
|
|
260
|
+
'-e',
|
|
261
|
+
`
|
|
262
|
+
import { ArtemisKit } from ${JSON.stringify(fileURLToPath(new URL('../artemiskit.ts', import.meta.url)))};
|
|
263
|
+
import { registerAdapter } from '@artemiskit/core';
|
|
264
|
+
import { OpenAIAdapter } from '@artemiskit/adapter-openai';
|
|
265
|
+
import assert from 'node:assert/strict';
|
|
266
|
+
registerAdapter('openai', async (config) => {
|
|
267
|
+
assert.equal(config.provider, 'openai');
|
|
268
|
+
assert.equal(config.defaultModel, 'workflow-authoritative');
|
|
269
|
+
assert.equal(config.maxRetries, 0);
|
|
270
|
+
return new OpenAIAdapter(config);
|
|
271
|
+
});
|
|
272
|
+
const kit = new ArtemisKit({ provider: 'anthropic', model: 'wrong', providerConfig: {
|
|
273
|
+
provider: 'openai', defaultModel: 'wrong-config', apiKey: 'dummy-only',
|
|
274
|
+
baseUrl: ${JSON.stringify(`http://127.0.0.1:${server.port}/v1`)}, maxRetries: 9,
|
|
275
|
+
}});
|
|
276
|
+
const events = [];
|
|
277
|
+
const result = await kit.runWorkflow({ workflow: ${JSON.stringify(workflowPath)}, onEvent: (event) => events.push(event) });
|
|
278
|
+
registerAdapter('openai', async () => { throw new Error('SYNTHETIC_PRIVATE_CONSTRUCTION_DETAIL'); });
|
|
279
|
+
await assert.rejects(kit.createWorkflowSession({ workflow: ${JSON.stringify(workflowPath)} }),
|
|
280
|
+
(error) => error.code === 'PROVIDER_UNAVAILABLE' &&
|
|
281
|
+
error.message === 'Unable to initialize workflow target' && error.cause === undefined);
|
|
282
|
+
console.log(JSON.stringify({ result, events }));
|
|
283
|
+
`,
|
|
284
|
+
],
|
|
285
|
+
{
|
|
286
|
+
cwd: fileURLToPath(new URL('../', import.meta.url)),
|
|
287
|
+
stdin: 'ignore',
|
|
288
|
+
stdout: 'pipe',
|
|
289
|
+
stderr: 'pipe',
|
|
290
|
+
env: { PATH: process.env.PATH },
|
|
291
|
+
}
|
|
292
|
+
);
|
|
293
|
+
const timer = setTimeout(() => child.kill(), 10000);
|
|
294
|
+
const [exit, stdout, stderr] = await Promise.all([
|
|
295
|
+
child.exited,
|
|
296
|
+
new Response(child.stdout).text(),
|
|
297
|
+
new Response(child.stderr).text(),
|
|
298
|
+
]);
|
|
299
|
+
clearTimeout(timer);
|
|
300
|
+
expect({ exit, stderr }).toEqual({ exit: 0, stderr: '' });
|
|
301
|
+
const { result, events } = JSON.parse(stdout);
|
|
302
|
+
expect(bodies[0].model).toBe('workflow-authoritative');
|
|
303
|
+
expect(result.record.execution).toBe('completed');
|
|
304
|
+
expect(result.record.taskVerification).toBe('passed');
|
|
305
|
+
expect(result.record.budgets).toMatchObject({ modelRequests: 2, toolCalls: 1 });
|
|
306
|
+
expect(result.state?.files).toEqual({ 'result.txt': 'PRIVATE-STATE' });
|
|
307
|
+
expect(JSON.stringify(result.record)).not.toContain('PRIVATE-');
|
|
308
|
+
expect(events.some((event) => event.type === 'tool_completed')).toBe(true);
|
|
309
|
+
});
|
|
310
|
+
test('file fixture roots, session events and environment overrides use the same engine without persistence', async () => {
|
|
311
|
+
const directory = await mkdtemp(join(tmpdir(), 'artemis-sdk-workflow-'));
|
|
312
|
+
const scenario = workflow();
|
|
313
|
+
scenario.workflow.initial_state = 'fixture.json';
|
|
314
|
+
await writeFile(join(directory, 'fixture.json'), '{"files":{"initial.txt":"PRIVATE"}}');
|
|
315
|
+
await writeFile(join(directory, 'workflow.json'), JSON.stringify(scenario));
|
|
316
|
+
let factories = 0;
|
|
317
|
+
const options: WorkflowRunOptions = {
|
|
318
|
+
workflow: join(directory, 'workflow.json'),
|
|
319
|
+
target: target(),
|
|
320
|
+
environmentFactory: async (args) => {
|
|
321
|
+
factories++;
|
|
322
|
+
return createSimulatedWorkflowEnvironment(args);
|
|
323
|
+
},
|
|
324
|
+
};
|
|
325
|
+
const session = await new ArtemisKit().createWorkflowSession(options);
|
|
326
|
+
expect(session.state).toBe('idle');
|
|
327
|
+
expect(factories).toBe(0);
|
|
328
|
+
const stream = (async () => {
|
|
329
|
+
const events: AgentWorkflowEvent[] = [];
|
|
330
|
+
for await (const event of session.events()) events.push(event);
|
|
331
|
+
return events;
|
|
332
|
+
})();
|
|
333
|
+
const result = await session.run();
|
|
334
|
+
expect(session.state).toBe('completed');
|
|
335
|
+
expect(result.state?.files).toEqual({ 'initial.txt': 'PRIVATE' });
|
|
336
|
+
expect(factories).toBe(1);
|
|
337
|
+
expect((await stream).at(-1)?.type).toBe('finished');
|
|
338
|
+
expect((await readdir(directory)).sort()).toEqual(['fixture.json', 'workflow.json']);
|
|
339
|
+
});
|
|
340
|
+
test('supports explicit client, rejects client+target and custom unsupported capabilities', async () => {
|
|
341
|
+
const unsupported = target();
|
|
342
|
+
unsupported.capabilities = async () => ({
|
|
343
|
+
status: 'available',
|
|
344
|
+
toolUse: false,
|
|
345
|
+
transportCancellation: false,
|
|
346
|
+
});
|
|
347
|
+
const kit = new ArtemisKit();
|
|
348
|
+
const client = new OpenAIAdapter({
|
|
349
|
+
provider: 'openai',
|
|
350
|
+
apiKey: 'dummy-only',
|
|
351
|
+
baseUrl: 'http://127.0.0.1:1',
|
|
352
|
+
maxRetries: 0,
|
|
353
|
+
});
|
|
354
|
+
await expect(
|
|
355
|
+
kit.createWorkflowSession({ workflow: workflow(), client, target: unsupported })
|
|
356
|
+
).rejects.toThrow('client or target');
|
|
357
|
+
expect(
|
|
358
|
+
(await kit.runWorkflow({ workflow: workflow(), target: unsupported })).record.execution
|
|
359
|
+
).toBe('unsupported');
|
|
360
|
+
const aborted = new AbortController();
|
|
361
|
+
aborted.abort();
|
|
362
|
+
expect(
|
|
363
|
+
(await kit.runWorkflow({ workflow: workflow(), client, signal: aborted.signal })).record
|
|
364
|
+
.execution
|
|
365
|
+
).toBe('cancelled');
|
|
366
|
+
});
|
|
367
|
+
test('cancellation and all SDK engine options stay host controlled', async () => {
|
|
368
|
+
let resolveStarted!: () => void;
|
|
369
|
+
const started = new Promise<void>((resolve) => {
|
|
370
|
+
resolveStarted = resolve;
|
|
371
|
+
});
|
|
372
|
+
const custom = target();
|
|
373
|
+
custom.turn = async (_request, signal) => {
|
|
374
|
+
resolveStarted();
|
|
375
|
+
await new Promise<void>((resolve) =>
|
|
376
|
+
signal?.addEventListener('abort', () => resolve(), { once: true })
|
|
377
|
+
);
|
|
378
|
+
return { status: 'error', code: 'aborted' };
|
|
379
|
+
};
|
|
380
|
+
const session = await new ArtemisKit().createWorkflowSession({
|
|
381
|
+
workflow: workflow(),
|
|
382
|
+
target: custom,
|
|
383
|
+
cleanupTimeoutMs: 20,
|
|
384
|
+
});
|
|
385
|
+
const result = session.run();
|
|
386
|
+
await started;
|
|
387
|
+
session.cancel();
|
|
388
|
+
expect((await result).record.execution).toBe('cancelled');
|
|
389
|
+
expect(session.state).toBe('completed');
|
|
390
|
+
});
|
|
391
|
+
});
|
package/src/artemiskit.ts
CHANGED
|
@@ -41,6 +41,7 @@ import {
|
|
|
41
41
|
UnsafeResponseDetector,
|
|
42
42
|
} from '@artemiskit/redteam';
|
|
43
43
|
import { nanoid } from 'nanoid';
|
|
44
|
+
import { prepareWorkflowSession } from './workflow';
|
|
44
45
|
|
|
45
46
|
import type {
|
|
46
47
|
ArtemisKitConfig,
|
|
@@ -62,6 +63,7 @@ import type {
|
|
|
62
63
|
StressResult,
|
|
63
64
|
ValidateOptions,
|
|
64
65
|
ValidationResult,
|
|
66
|
+
WorkflowRunOptions,
|
|
65
67
|
} from './types';
|
|
66
68
|
|
|
67
69
|
/**
|
|
@@ -125,6 +127,16 @@ export class ArtemisKit {
|
|
|
125
127
|
}
|
|
126
128
|
}
|
|
127
129
|
|
|
130
|
+
/** Create a cancellable native session without executing model turns or persisting results. */
|
|
131
|
+
async createWorkflowSession(options: WorkflowRunOptions) {
|
|
132
|
+
return prepareWorkflowSession(options, this.config);
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** Execute a workflow; task verification is explicitly unavailable until outcome scoring. */
|
|
136
|
+
async runWorkflow(options: WorkflowRunOptions) {
|
|
137
|
+
return (await this.createWorkflowSession(options)).run();
|
|
138
|
+
}
|
|
139
|
+
|
|
128
140
|
// ==========================================================================
|
|
129
141
|
// Event Emitter Methods
|
|
130
142
|
// ==========================================================================
|
package/src/index.ts
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
// Main class
|
|
7
7
|
export { ArtemisKit } from './artemiskit';
|
|
8
8
|
|
|
9
|
-
// Workflow authoring
|
|
9
|
+
// Workflow authoring, controlled native execution and tool primitives.
|
|
10
10
|
export {
|
|
11
11
|
AgentWorkflowSchema,
|
|
12
12
|
WorkflowPolicySchema,
|
|
@@ -18,6 +18,19 @@ export {
|
|
|
18
18
|
getWorkflowTool,
|
|
19
19
|
executeSimulatedTool,
|
|
20
20
|
createModelClientTarget,
|
|
21
|
+
createAgentWorkflowSession,
|
|
22
|
+
runAgentWorkflow,
|
|
23
|
+
evaluateWorkflowDeterministicOutcomes,
|
|
24
|
+
evaluateWorkflowSemantics,
|
|
25
|
+
isValidWorkflowJudgeOptions,
|
|
26
|
+
decideWorkflowOutcome,
|
|
27
|
+
validWorkflowAssertionCounts,
|
|
28
|
+
readWorkflowRecord,
|
|
29
|
+
createSimulatedWorkflowEnvironment,
|
|
30
|
+
createDockerWorkflowEnvironment,
|
|
31
|
+
createDockerWorkflowEnvironmentFactory,
|
|
32
|
+
WORKFLOW_SANDBOX_IMAGE,
|
|
33
|
+
WorkflowEnvironmentInitializationError,
|
|
21
34
|
} from '@artemiskit/core';
|
|
22
35
|
export type {
|
|
23
36
|
AgentWorkflow,
|
|
@@ -33,12 +46,43 @@ export type {
|
|
|
33
46
|
AgentTurnResult,
|
|
34
47
|
AgentTargetCapabilities,
|
|
35
48
|
AgentTargetFailure,
|
|
49
|
+
AgentWorkflowExecution,
|
|
50
|
+
AgentWorkflowReason,
|
|
51
|
+
AgentWorkflowEvent,
|
|
52
|
+
AgentWorkflowRecord,
|
|
53
|
+
AgentWorkflowExecutionRecord,
|
|
54
|
+
AgentWorkflowEvidence,
|
|
55
|
+
WorkflowDeterministicStatus,
|
|
56
|
+
WorkflowDeterministicReason,
|
|
57
|
+
WorkflowDeterministicAssertion,
|
|
58
|
+
WorkflowDeterministicSummary,
|
|
59
|
+
WorkflowJudgeOptions,
|
|
60
|
+
WorkflowSemanticOptions,
|
|
61
|
+
WorkflowSemanticReason,
|
|
62
|
+
WorkflowSemanticAssertion,
|
|
63
|
+
WorkflowSemanticIdentity,
|
|
64
|
+
WorkflowSemanticEvaluation,
|
|
65
|
+
WorkflowOutcomeStatus,
|
|
66
|
+
WorkflowAssertionCounts,
|
|
67
|
+
WorkflowOutcomeDecision,
|
|
68
|
+
WorkflowOutcomeAssessment,
|
|
69
|
+
SavedWorkflowRecord,
|
|
70
|
+
HistoricalWorkflowRecord,
|
|
71
|
+
AgentWorkflowResult,
|
|
72
|
+
AgentWorkflowSessionOptions,
|
|
73
|
+
AgentWorkflowSession,
|
|
74
|
+
WorkflowState,
|
|
75
|
+
WorkflowEnvironment,
|
|
76
|
+
WorkflowEnvironmentFactory,
|
|
77
|
+
WorkflowEnvironmentCleanup,
|
|
78
|
+
DockerWorkflowEnvironmentOptions,
|
|
36
79
|
} from '@artemiskit/core';
|
|
37
80
|
|
|
38
81
|
// Types
|
|
39
82
|
export type {
|
|
40
83
|
// Configuration
|
|
41
84
|
ArtemisKitConfig,
|
|
85
|
+
WorkflowRunOptions,
|
|
42
86
|
RunOptions,
|
|
43
87
|
RedTeamOptions,
|
|
44
88
|
StressOptions,
|
package/src/types-only.ts
CHANGED
|
@@ -23,6 +23,7 @@
|
|
|
23
23
|
export type {
|
|
24
24
|
// Configuration
|
|
25
25
|
ArtemisKitConfig,
|
|
26
|
+
WorkflowRunOptions,
|
|
26
27
|
RunOptions,
|
|
27
28
|
RedTeamOptions,
|
|
28
29
|
StressOptions,
|
|
@@ -274,3 +275,50 @@ export type {
|
|
|
274
275
|
// ============================================================================
|
|
275
276
|
|
|
276
277
|
export type { ScenarioBuilder, TestCaseBuilder } from './builders';
|
|
278
|
+
|
|
279
|
+
export type {
|
|
280
|
+
AgentWorkflow,
|
|
281
|
+
WorkflowPolicy,
|
|
282
|
+
WorkflowJson,
|
|
283
|
+
WorkflowToolId,
|
|
284
|
+
WorkflowResource,
|
|
285
|
+
WorkflowToolDescriptor,
|
|
286
|
+
SimulatedToolEvidence,
|
|
287
|
+
SimulatedToolResult,
|
|
288
|
+
AgentTarget,
|
|
289
|
+
AgentTurnRequest,
|
|
290
|
+
AgentTurnResult,
|
|
291
|
+
AgentTargetCapabilities,
|
|
292
|
+
AgentTargetFailure,
|
|
293
|
+
AgentWorkflowExecution,
|
|
294
|
+
AgentWorkflowReason,
|
|
295
|
+
AgentWorkflowEvent,
|
|
296
|
+
AgentWorkflowRecord,
|
|
297
|
+
AgentWorkflowExecutionRecord,
|
|
298
|
+
AgentWorkflowEvidence,
|
|
299
|
+
WorkflowDeterministicStatus,
|
|
300
|
+
WorkflowDeterministicReason,
|
|
301
|
+
WorkflowDeterministicAssertion,
|
|
302
|
+
WorkflowDeterministicSummary,
|
|
303
|
+
WorkflowJudgeOptions,
|
|
304
|
+
WorkflowSemanticOptions,
|
|
305
|
+
WorkflowSemanticReason,
|
|
306
|
+
WorkflowSemanticAssertion,
|
|
307
|
+
WorkflowSemanticIdentity,
|
|
308
|
+
WorkflowSemanticEvaluation,
|
|
309
|
+
WorkflowOutcomeStatus,
|
|
310
|
+
WorkflowAssertionCounts,
|
|
311
|
+
WorkflowOutcomeDecision,
|
|
312
|
+
WorkflowOutcomeAssessment,
|
|
313
|
+
SavedWorkflowRecord,
|
|
314
|
+
HistoricalWorkflowRecord,
|
|
315
|
+
AgentWorkflowResult,
|
|
316
|
+
AgentWorkflowSessionOptions,
|
|
317
|
+
AgentWorkflowSession,
|
|
318
|
+
WorkflowState,
|
|
319
|
+
WorkflowEnvironment,
|
|
320
|
+
WorkflowEnvironmentFactory,
|
|
321
|
+
WorkflowEnvironmentCleanup,
|
|
322
|
+
DockerWorkflowEnvironmentOptions,
|
|
323
|
+
WorkflowEnvironmentInitializationError,
|
|
324
|
+
} from '@artemiskit/core';
|
package/src/types.ts
CHANGED
|
@@ -5,6 +5,9 @@
|
|
|
5
5
|
|
|
6
6
|
import type {
|
|
7
7
|
AdapterConfig,
|
|
8
|
+
AgentTarget,
|
|
9
|
+
AgentWorkflow,
|
|
10
|
+
AgentWorkflowSessionOptions,
|
|
8
11
|
CaseResult,
|
|
9
12
|
ComparisonEligibility,
|
|
10
13
|
ModelClient,
|
|
@@ -448,3 +451,16 @@ export interface CompareResult {
|
|
|
448
451
|
/** Threshold used for regression detection */
|
|
449
452
|
threshold: number;
|
|
450
453
|
}
|
|
454
|
+
|
|
455
|
+
/** Controlled workflow execution. Text/state remain in memory unless the caller explicitly saves them. */
|
|
456
|
+
export interface WorkflowRunOptions
|
|
457
|
+
extends Omit<AgentWorkflowSessionOptions, 'workflow' | 'target'> {
|
|
458
|
+
/** Workflow object or YAML path. File fixtures default to the workflow file's directory. */
|
|
459
|
+
workflow: AgentWorkflow | string;
|
|
460
|
+
/** Explicit host-owned client. Its transport retry behavior remains the caller's responsibility. */
|
|
461
|
+
client?: ModelClient;
|
|
462
|
+
/** Explicit custom target, mutually exclusive with client. */
|
|
463
|
+
target?: AgentTarget;
|
|
464
|
+
/** Trusted transport configuration; workflow provider/model and zero retries remain authoritative. */
|
|
465
|
+
providerConfig?: Partial<AdapterConfig>;
|
|
466
|
+
}
|
package/src/workflow.ts
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { dirname, resolve } from 'node:path';
|
|
2
|
+
import {
|
|
3
|
+
type AdapterConfig,
|
|
4
|
+
type AgentWorkflowSession,
|
|
5
|
+
ArtemisError,
|
|
6
|
+
createAdapter,
|
|
7
|
+
createAgentWorkflowSession,
|
|
8
|
+
createModelClientTarget,
|
|
9
|
+
loadAgentWorkflow,
|
|
10
|
+
validateAgentWorkflow,
|
|
11
|
+
} from '@artemiskit/core';
|
|
12
|
+
import type { ArtemisKitConfig, WorkflowRunOptions } from './types';
|
|
13
|
+
|
|
14
|
+
export async function prepareWorkflowSession(
|
|
15
|
+
options: WorkflowRunOptions,
|
|
16
|
+
config: ArtemisKitConfig
|
|
17
|
+
): Promise<AgentWorkflowSession> {
|
|
18
|
+
if (options.client && options.target) throw new Error('Choose a workflow client or target');
|
|
19
|
+
const file = typeof options.workflow === 'string' ? resolve(options.workflow) : undefined;
|
|
20
|
+
const workflow = file ? await loadAgentWorkflow(file) : validateAgentWorkflow(options.workflow);
|
|
21
|
+
let target = options.target;
|
|
22
|
+
if (!target) {
|
|
23
|
+
let client = options.client;
|
|
24
|
+
if (!client) {
|
|
25
|
+
try {
|
|
26
|
+
client = await createAdapter({
|
|
27
|
+
...config.providerConfig,
|
|
28
|
+
...options.providerConfig,
|
|
29
|
+
provider: workflow.target.provider,
|
|
30
|
+
defaultModel: workflow.target.model,
|
|
31
|
+
maxRetries: 0,
|
|
32
|
+
} as AdapterConfig);
|
|
33
|
+
} catch {
|
|
34
|
+
// Provider factories may put credentials or configuration values in their diagnostics.
|
|
35
|
+
throw new ArtemisError('Unable to initialize workflow target', 'PROVIDER_UNAVAILABLE');
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
target = createModelClientTarget(client);
|
|
39
|
+
}
|
|
40
|
+
return createAgentWorkflowSession({
|
|
41
|
+
workflow,
|
|
42
|
+
target,
|
|
43
|
+
fixtureRoot: options.fixtureRoot ?? (file ? dirname(file) : undefined),
|
|
44
|
+
environmentFactory: options.environmentFactory,
|
|
45
|
+
preflight: options.preflight,
|
|
46
|
+
preflightOnly: options.preflightOnly,
|
|
47
|
+
signal: options.signal,
|
|
48
|
+
cleanupTimeoutMs: options.cleanupTimeoutMs,
|
|
49
|
+
semanticJudge: options.semanticJudge,
|
|
50
|
+
onEvent: options.onEvent,
|
|
51
|
+
});
|
|
52
|
+
}
|