@artemiskit/sdk 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/README.md +25 -0
- package/dist/artemiskit.d.ts +5 -1
- package/dist/artemiskit.d.ts.map +1 -1
- package/dist/index.d.ts +3 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +77 -11
- package/dist/types-only.d.ts +2 -1
- package/dist/types-only.d.ts.map +1 -1
- package/dist/types.d.ts +12 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/workflow.d.ts +4 -0
- package/dist/workflow.d.ts.map +1 -0
- package/package.json +3 -3
- package/src/__tests__/workflow.test.ts +244 -0
- package/src/artemiskit.ts +12 -0
- package/src/index.ts +21 -1
- package/src/types-only.ts +30 -0
- package/src/types.ts +16 -0
- package/src/workflow.ts +51 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,26 @@
|
|
|
1
1
|
# @artemiskit/sdk
|
|
2
2
|
|
|
3
|
+
## 0.6.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- Add native controlled workflow execution for milestone 0.6.1. CLI workflow run/preflight and SDK
|
|
8
|
+
sessions share strict tool and permission enforcement, cumulative budgets, explicit tool-protocol
|
|
9
|
+
preflight, measured-usage states, bounded events, cancellation, and truthful cleanup evidence.
|
|
10
|
+
Run against detached simulated state or a disposable Docker filesystem with a fixed local image,
|
|
11
|
+
no host mounts or network, and no arbitrary command tool. Save bounded execution metadata separately
|
|
12
|
+
from explicitly requested sensitive state exports. Task verification remains unavailable until 0.6.2.
|
|
13
|
+
|
|
14
|
+
OpenAI and Ling workflow transports forward abort signals, disable automatic retries, and distinguish
|
|
15
|
+
missing or malformed token usage from explicitly measured zero. Preserve existing scenario APIs and
|
|
16
|
+
offline workflow validation. Package versions remain independent of the milestone label.
|
|
17
|
+
|
|
18
|
+
### Patch Changes
|
|
19
|
+
|
|
20
|
+
- Updated dependencies
|
|
21
|
+
- @artemiskit/core@0.6.1
|
|
22
|
+
- @artemiskit/redteam@0.3.9
|
|
23
|
+
|
|
3
24
|
## 0.5.0
|
|
4
25
|
|
|
5
26
|
### Minor Changes
|
package/README.md
CHANGED
|
@@ -48,6 +48,31 @@ console.log('All tests passed! ✅');
|
|
|
48
48
|
|
|
49
49
|
## API Reference
|
|
50
50
|
|
|
51
|
+
### Controlled agent workflows
|
|
52
|
+
|
|
53
|
+
```typescript
|
|
54
|
+
const session = await kit.createWorkflowSession({
|
|
55
|
+
workflow: './workflow.yaml',
|
|
56
|
+
providerConfig: { apiKey: process.env.OPENAI_API_KEY },
|
|
57
|
+
preflight: true,
|
|
58
|
+
});
|
|
59
|
+
const result = await session.run();
|
|
60
|
+
console.log(result.record); // bounded metadata; taskVerification is unavailable in 0.6.1
|
|
61
|
+
// session.cancel() stops new work and initiates bounded cleanup.
|
|
62
|
+
// result.state and result.transcript contain sensitive working data.
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
`kit.runWorkflow(options)` creates and runs a fresh session in one call. Supply a `client` or
|
|
66
|
+
`target` to use an existing trusted adapter or target. The same native controls apply through the
|
|
67
|
+
CLI and SDK: declared tools and permissions, cumulative budgets, explicit preflight, cancellation,
|
|
68
|
+
and cleanup evidence. Workflows use fresh simulated state or a disposable Docker filesystem.
|
|
69
|
+
Independent task scoring follows in 0.6.2; normal execution does not establish a task pass.
|
|
70
|
+
|
|
71
|
+
The root and `/types` entry points expose workflow result/session/event types. Low-level
|
|
72
|
+
`createAgentWorkflowSession` and `runAgentWorkflow` are available for explicit target composition.
|
|
73
|
+
See the [execution guide](https://github.com/code-sensei/artemiskit/blob/main/docs/workflow-execution.md)
|
|
74
|
+
for configuration, Docker requirements, token-accounting limits, events, and safe persistence.
|
|
75
|
+
|
|
51
76
|
### ArtemisKit Class
|
|
52
77
|
|
|
53
78
|
#### Constructor
|
package/dist/artemiskit.d.ts
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* @artemiskit/sdk
|
|
3
3
|
* Main ArtemisKit class - programmatic API for LLM testing
|
|
4
4
|
*/
|
|
5
|
-
import type { ArtemisKitConfig, ArtemisKitEventName, ArtemisKitEvents, CaseCompleteHandler, CaseStartHandler, CompareOptions, CompareResult, ProgressHandler, RedTeamMutationCompleteHandler, RedTeamMutationStartHandler, RedTeamOptions, RedTeamResult, RunOptions, RunResult, StressOptions, StressRequestCompleteHandler, StressResult, ValidateOptions, ValidationResult } from './types.js';
|
|
5
|
+
import type { ArtemisKitConfig, ArtemisKitEventName, ArtemisKitEvents, CaseCompleteHandler, CaseStartHandler, CompareOptions, CompareResult, ProgressHandler, RedTeamMutationCompleteHandler, RedTeamMutationStartHandler, RedTeamOptions, RedTeamResult, RunOptions, RunResult, StressOptions, StressRequestCompleteHandler, StressResult, ValidateOptions, ValidationResult, WorkflowRunOptions } from './types.js';
|
|
6
6
|
/**
|
|
7
7
|
* ArtemisKit SDK - programmatic API for LLM evaluation testing
|
|
8
8
|
*
|
|
@@ -25,6 +25,10 @@ export declare class ArtemisKit {
|
|
|
25
25
|
private eventHandlers;
|
|
26
26
|
private storage;
|
|
27
27
|
constructor(config?: ArtemisKitConfig);
|
|
28
|
+
/** Create a cancellable native session without executing model turns or persisting results. */
|
|
29
|
+
createWorkflowSession(options: WorkflowRunOptions): Promise<import("@artemiskit/core").AgentWorkflowSession>;
|
|
30
|
+
/** Execute a workflow; task verification is explicitly unavailable until outcome scoring. */
|
|
31
|
+
runWorkflow(options: WorkflowRunOptions): Promise<import("@artemiskit/core").AgentWorkflowResult>;
|
|
28
32
|
/**
|
|
29
33
|
* Register an event handler
|
|
30
34
|
*/
|
package/dist/artemiskit.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"artemiskit.d.ts","sourceRoot":"","sources":["../src/artemiskit.ts"],"names":[],"mappings":"AAAA;;;GAGG;
|
|
1
|
+
{"version":3,"file":"artemiskit.d.ts","sourceRoot":"","sources":["../src/artemiskit.ts"],"names":[],"mappings":"AAAA;;;GAGG;AA0CH,OAAO,KAAK,EACV,gBAAgB,EAChB,mBAAmB,EACnB,gBAAgB,EAChB,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,aAAa,EACb,eAAe,EACf,8BAA8B,EAC9B,2BAA2B,EAC3B,cAAc,EACd,aAAa,EACb,UAAU,EACV,SAAS,EACT,aAAa,EACb,4BAA4B,EAC5B,YAAY,EACZ,eAAe,EACf,gBAAgB,EAChB,kBAAkB,EACnB,MAAM,SAAS,CAAC;AAsBjB;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAmB;IACjC,OAAO,CAAC,aAAa,CAA6D;IAClF,OAAO,CAAC,OAAO,CAA+B;gBAElC,MAAM,GAAE,gBAAqB;IAmBzC,+FAA+F;IACzF,qBAAqB,CAAC,OAAO,EAAE,kBAAkB;IAIvD,6FAA6F;IACvF,WAAW,CAAC,OAAO,EAAE,kBAAkB;IAQ7C;;OAEG;IACH,EAAE,CAAC,CAAC,SAAS,mBAAmB,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,IAAI;IAQhG;;OAEG;IACH,GAAG,CAAC,CAAC,SAAS,mBAAmB,EAC/B,KAAK,EAAE,CAAC,EACR,OAAO,EAAE,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC,KAAK,IAAI,GAC5C,IAAI;IAQP;;OAEG;IACH,IAAI,CAAC,CAAC,SAAS,mBAAmB,EAChC,KAAK,EAAE,CAAC,EACR,OAAO,EAAE,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC,KAAK,IAAI,GAC5C,IAAI;IAQP;;OAEG;IACH,OAAO,CAAC,IAAI;IAiBZ;;OAEG;IACH,WAAW,CAAC,OAAO,EAAE,gBAAgB,GAAG,IAAI;IAI5C;;OAEG;IACH,cAAc,CAAC,OAAO,EAAE,mBAAmB,GAAG,IAAI;IAIlD;;OAEG;IACH,UAAU,CAAC,OAAO,EAAE,eAAe,GAAG,IAAI;IAI1C;;OAEG;IACH,sBAAsB,CAAC,OAAO,EAAE,2BAA2B,GAAG,IAAI;IAIlE;;OAEG;IACH,yBAAyB,CAAC,OAAO,EAAE,8BAA8B,GAAG,IAAI;IAIxE;;OAEG;IACH,uBAAuB,CAAC,OAAO,EAAE,4BAA4B,GAAG,IAAI;IAQpE;;OAEG;IACG,GAAG,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC;IA+DlD;;OAEG;IACG,OAAO,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;IAgN9D;;OAEG;IACG,MAAM,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC;IAwO3D;;;;;;;;;;;;;OAaG;IACG,QAAQ,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA2LnE;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACG,OAAO,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;IA2H9D;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAmBzB;;OAEG;IACH,OAAO,CAAC,YAAY;IA4EpB;;OAEG;IACH,OAAO,CAAC,YAAY;IAepB;;OAEG;YACW,YAAY;IAO1B;;OAEG;YACW,YAAY;IAgB1B;;OAEG;IACH,OAAO,CAAC,cAAc;IA6BtB;;OAEG;IACH,qBAAqB,IAAI,MAAM,EAAE;CAGlC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -3,9 +3,9 @@
|
|
|
3
3
|
* Programmatic SDK for ArtemisKit LLM evaluation toolkit
|
|
4
4
|
*/
|
|
5
5
|
export { ArtemisKit } from './artemiskit.js';
|
|
6
|
-
export { AgentWorkflowSchema, WorkflowPolicySchema, WORKFLOW_TOOL_IDS, validateAgentWorkflow, parseAgentWorkflow, loadAgentWorkflow, listWorkflowTools, getWorkflowTool, executeSimulatedTool, createModelClientTarget, } from '@artemiskit/core';
|
|
7
|
-
export type { AgentWorkflow, WorkflowPolicy, WorkflowJson, WorkflowToolId, WorkflowResource, WorkflowToolDescriptor, SimulatedToolEvidence, SimulatedToolResult, AgentTarget, AgentTurnRequest, AgentTurnResult, AgentTargetCapabilities, AgentTargetFailure, } from '@artemiskit/core';
|
|
8
|
-
export type { ArtemisKitConfig, RunOptions, RedTeamOptions, StressOptions, RunResult, RedTeamResult, StressResult, CaseStartEvent, CaseCompleteEvent, ProgressEvent, RedTeamMutationStartEvent, RedTeamMutationCompleteEvent, StressRequestCompleteEvent, CaseStartHandler, CaseCompleteHandler, ProgressHandler, RedTeamMutationStartHandler, RedTeamMutationCompleteHandler, StressRequestCompleteHandler, ArtemisKitEvents, ArtemisKitEventName, ValidateOptions, ValidationResult, ValidationError, ValidationWarning, ScenarioValidation, CompareOptions, CompareResult, RunSummary, ComparisonDetails, } from './types.js';
|
|
6
|
+
export { AgentWorkflowSchema, WorkflowPolicySchema, WORKFLOW_TOOL_IDS, validateAgentWorkflow, parseAgentWorkflow, loadAgentWorkflow, listWorkflowTools, getWorkflowTool, executeSimulatedTool, createModelClientTarget, createAgentWorkflowSession, runAgentWorkflow, createSimulatedWorkflowEnvironment, createDockerWorkflowEnvironment, createDockerWorkflowEnvironmentFactory, WORKFLOW_SANDBOX_IMAGE, WorkflowEnvironmentInitializationError, } from '@artemiskit/core';
|
|
7
|
+
export type { AgentWorkflow, WorkflowPolicy, WorkflowJson, WorkflowToolId, WorkflowResource, WorkflowToolDescriptor, SimulatedToolEvidence, SimulatedToolResult, AgentTarget, AgentTurnRequest, AgentTurnResult, AgentTargetCapabilities, AgentTargetFailure, AgentWorkflowExecution, AgentWorkflowReason, AgentWorkflowEvent, AgentWorkflowRecord, AgentWorkflowResult, AgentWorkflowSessionOptions, AgentWorkflowSession, WorkflowState, WorkflowEnvironment, WorkflowEnvironmentFactory, WorkflowEnvironmentCleanup, DockerWorkflowEnvironmentOptions, } from '@artemiskit/core';
|
|
8
|
+
export type { ArtemisKitConfig, WorkflowRunOptions, RunOptions, RedTeamOptions, StressOptions, RunResult, RedTeamResult, StressResult, CaseStartEvent, CaseCompleteEvent, ProgressEvent, RedTeamMutationStartEvent, RedTeamMutationCompleteEvent, StressRequestCompleteEvent, CaseStartHandler, CaseCompleteHandler, ProgressHandler, RedTeamMutationStartHandler, RedTeamMutationCompleteHandler, StressRequestCompleteHandler, ArtemisKitEvents, ArtemisKitEventName, ValidateOptions, ValidationResult, ValidationError, ValidationWarning, ScenarioValidation, CompareOptions, CompareResult, RunSummary, ComparisonDetails, } from './types.js';
|
|
9
9
|
export type { Scenario, TestCase, Expected, Provider, ProviderConfig, ChatMessageType, Variables, } from '@artemiskit/core';
|
|
10
10
|
export type { CaseResult, CaseRedactionInfo, RunManifest, RunConfig, RunMetrics, RedTeamManifest, RedTeamConfig, RedTeamMetrics, RedTeamCaseResult, RedTeamStatus, RedTeamSeverity, StressManifest, StressConfig, StressMetrics, StressRequestResult, AnyManifest, ResolvedConfig, ConfigSource, ManifestRedactionInfo, CostEstimateInfo, GitInfo, ProvenanceInfo, } from '@artemiskit/core';
|
|
11
11
|
export type { ModelClient, ModelCapabilities, GenerateOptions, GenerateResult, TokenUsage, ChatMessage, FunctionDefinition, ToolDefinition, ToolCall, AdapterConfig, BaseAdapterConfig, OpenAIAdapterConfig, AzureOpenAIAdapterConfig, VercelAIAdapterConfig, AnthropicAdapterConfig, LangChainAdapterConfig, DeepAgentsAdapterConfig, ProviderType, } from '@artemiskit/core';
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAG1C,OAAO,EACL,mBAAmB,EACnB,oBAAoB,EACpB,iBAAiB,EACjB,qBAAqB,EACrB,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,eAAe,EACf,oBAAoB,EACpB,uBAAuB,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAG1C,OAAO,EACL,mBAAmB,EACnB,oBAAoB,EACpB,iBAAiB,EACjB,qBAAqB,EACrB,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,eAAe,EACf,oBAAoB,EACpB,uBAAuB,EACvB,0BAA0B,EAC1B,gBAAgB,EAChB,kCAAkC,EAClC,+BAA+B,EAC/B,sCAAsC,EACtC,sBAAsB,EACtB,sCAAsC,GACvC,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,aAAa,EACb,cAAc,EACd,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,sBAAsB,EACtB,qBAAqB,EACrB,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,eAAe,EACf,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,2BAA2B,EAC3B,oBAAoB,EACpB,aAAa,EACb,mBAAmB,EACnB,0BAA0B,EAC1B,0BAA0B,EAC1B,gCAAgC,GACjC,MAAM,kBAAkB,CAAC;AAG1B,YAAY,EAEV,gBAAgB,EAChB,kBAAkB,EAClB,UAAU,EACV,cAAc,EACd,aAAa,EAEb,SAAS,EACT,aAAa,EACb,YAAY,EAEZ,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,yBAAyB,EACzB,4BAA4B,EAC5B,0BAA0B,EAE1B,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAE5B,gBAAgB,EAChB,mBAAmB,EAEnB,eAAe,EACf,gBAAgB,EAChB,eAAe,EACf,iBAAiB,EACjB,kBAAkB,EAElB,cAAc,EACd,aAAa,EACb,UAAU,EACV,iBAAiB,GAClB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,cAAc,EACd,eAAe,EACf,SAAS,GACV,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,UAAU,EACV,iBAAiB,EAEjB,WAAW,EACX,SAAS,EACT,UAAU,EAEV,eAAe,EACf,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,eAAe,EAEf,cAAc,EACd,YAAY,EACZ,aAAa,EACb,mBAAmB,EAEnB,WAAW,EACX,cAAc,EACd,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,OAAO,EACP,cAAc,GACf,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,WAAW,EACX,iBAAiB,EAEjB,eAAe,EACf,cAAc,EACd,UAAU,EAEV,WAAW,EAEX,kBAAkB,EAClB,cAAc,EACd,QAAQ,EAER,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,YAAY,GACb,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMrF,YAAY,EACV,cAAc,EACd,aAAa,EACb,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EAEtB,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EACtB,eAAe,EACf,mBAAmB,EACnB,cAAc,EACd,uBAAuB,GACxB,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMxD,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAMtF,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAM7E,OAAO,EAAE,kBAAkB,EAAE,KAAK,kBAAkB,EAAE,KAAK,aAAa,EAAE,MAAM,YAAY,CAAC;AAM7F,OAAO,EAEL,WAAW,EACX,eAAe,EACf,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,oBAAoB,EAEpB,aAAa,EACb,MAAM,EAEN,cAAc,EACd,cAAc,EACd,aAAa,EACb,oBAAoB,GACrB,MAAM,SAAS,CAAC;AAEjB,YAAY,EAEV,YAAY,EACZ,eAAe,EAEf,SAAS,EACT,eAAe,EACf,mBAAmB,EACnB,oBAAoB,EACpB,eAAe,EAEf,WAAW,EACX,eAAe,EACf,aAAa,EACb,cAAc,EACd,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,OAAO,EAEL,aAAa,EAEb,eAAe,EAEf,aAAa,EAEb,YAAY,GACb,MAAM,aAAa,CAAC;AAErB,YAAY,EACV,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,aAAa,CAAC;AAMrB,OAAO,EAEL,eAAe,EACf,eAAe,EAEf,QAAQ,EACR,QAAQ,EAER,YAAY,EACZ,SAAS,EACT,SAAS,EACT,QAAQ,EACR,UAAU,EAEV,KAAK,EACL,QAAQ,EACR,WAAW,EACX,KAAK,EACL,KAAK,EACL,UAAU,EACV,QAAQ,EACR,UAAU,EACV,MAAM,EACN,KAAK,EACL,KAAK,GACN,MAAM,YAAY,CAAC;AAMpB,OAAO,EAEL,QAAQ,EACR,cAAc,EAEd,qBAAqB,EAErB,iBAAiB,EACjB,uBAAuB,EAEvB,mBAAmB,EACnB,oBAAoB,EACpB,iBAAiB,EAEjB,eAAe,EACf,4BAA4B,EAE5B,gBAAgB,EAChB,sBAAsB,EAEtB,eAAe,EACf,wBAAwB,EACxB,SAAS,EACT,kBAAkB,EAClB,aAAa,EACb,4BAA4B,EAC5B,gBAAgB,EAEhB,YAAY,EACZ,4BAA4B,EAE5B,UAAU,EACV,WAAW,EACX,cAAc,EACd,mBAAmB,EACnB,aAAa,EACb,sBAAsB,EACtB,eAAe,EACf,qBAAqB,EAErB,cAAc,EACd,gBAAgB,EAChB,WAAW,GACZ,MAAM,YAAY,CAAC;AAGpB,YAAY,EAEV,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,sBAAsB,EACtB,gBAAgB,EAChB,iBAAiB,EACjB,oBAAoB,EAEpB,YAAY,EACZ,iBAAiB,EACjB,eAAe,EACf,aAAa,EACb,SAAS,EACT,eAAe,EAEf,UAAU,EACV,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,eAAe,EACf,eAAe,EAEf,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EAEnB,oBAAoB,EACpB,cAAc,EAEd,YAAY,EACZ,OAAO,EACP,WAAW,EACX,kBAAkB,EAClB,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,eAAe,EAEf,eAAe,EACf,mBAAmB,EACnB,YAAY,EAEZ,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EAEpB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EAEpB,aAAa,EACb,0BAA0B,EAE1B,qBAAqB,EACrB,kBAAkB,EAClB,eAAe,EAEf,uBAAuB,EACvB,aAAa,EACb,kBAAkB,EAClB,eAAe,EACf,wBAAwB,EAExB,kBAAkB,EAClB,oBAAoB,EACpB,cAAc,EACd,cAAc,EACd,mBAAmB,EACnB,4BAA4B,EAC5B,yBAAyB,EACzB,kCAAkC,EAClC,2BAA2B,EAC3B,mBAAmB,EACnB,uBAAuB,EACvB,eAAe,EACf,eAAe,EACf,gBAAgB,EAChB,yBAAyB,EACzB,sBAAsB,GACvB,MAAM,YAAY,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -7584,7 +7584,7 @@ function anyOf(...expectations) {
|
|
|
7584
7584
|
import {
|
|
7585
7585
|
assessComparisonEligibility,
|
|
7586
7586
|
runScenario as coreRunScenario,
|
|
7587
|
-
createAdapter,
|
|
7587
|
+
createAdapter as createAdapter2,
|
|
7588
7588
|
createStorageAdapter,
|
|
7589
7589
|
getGitInfo,
|
|
7590
7590
|
isComparisonAvailable,
|
|
@@ -7636,6 +7636,52 @@ function nanoid(size = 21) {
|
|
|
7636
7636
|
return id;
|
|
7637
7637
|
}
|
|
7638
7638
|
|
|
7639
|
+
// src/workflow.ts
|
|
7640
|
+
import { dirname, resolve } from "node:path";
|
|
7641
|
+
import {
|
|
7642
|
+
ArtemisError,
|
|
7643
|
+
createAdapter,
|
|
7644
|
+
createAgentWorkflowSession,
|
|
7645
|
+
createModelClientTarget,
|
|
7646
|
+
loadAgentWorkflow,
|
|
7647
|
+
validateAgentWorkflow
|
|
7648
|
+
} from "@artemiskit/core";
|
|
7649
|
+
async function prepareWorkflowSession(options, config) {
|
|
7650
|
+
if (options.client && options.target)
|
|
7651
|
+
throw new Error("Choose a workflow client or target");
|
|
7652
|
+
const file = typeof options.workflow === "string" ? resolve(options.workflow) : undefined;
|
|
7653
|
+
const workflow = file ? await loadAgentWorkflow(file) : validateAgentWorkflow(options.workflow);
|
|
7654
|
+
let target = options.target;
|
|
7655
|
+
if (!target) {
|
|
7656
|
+
let client = options.client;
|
|
7657
|
+
if (!client) {
|
|
7658
|
+
try {
|
|
7659
|
+
client = await createAdapter({
|
|
7660
|
+
...config.providerConfig,
|
|
7661
|
+
...options.providerConfig,
|
|
7662
|
+
provider: workflow.target.provider,
|
|
7663
|
+
defaultModel: workflow.target.model,
|
|
7664
|
+
maxRetries: 0
|
|
7665
|
+
});
|
|
7666
|
+
} catch {
|
|
7667
|
+
throw new ArtemisError("Unable to initialize workflow target", "PROVIDER_UNAVAILABLE");
|
|
7668
|
+
}
|
|
7669
|
+
}
|
|
7670
|
+
target = createModelClientTarget(client);
|
|
7671
|
+
}
|
|
7672
|
+
return createAgentWorkflowSession({
|
|
7673
|
+
workflow,
|
|
7674
|
+
target,
|
|
7675
|
+
fixtureRoot: options.fixtureRoot ?? (file ? dirname(file) : undefined),
|
|
7676
|
+
environmentFactory: options.environmentFactory,
|
|
7677
|
+
preflight: options.preflight,
|
|
7678
|
+
preflightOnly: options.preflightOnly,
|
|
7679
|
+
signal: options.signal,
|
|
7680
|
+
cleanupTimeoutMs: options.cleanupTimeoutMs,
|
|
7681
|
+
onEvent: options.onEvent
|
|
7682
|
+
});
|
|
7683
|
+
}
|
|
7684
|
+
|
|
7639
7685
|
// src/artemiskit.ts
|
|
7640
7686
|
var MUTATION_MAP = {
|
|
7641
7687
|
typo: TypoMutation,
|
|
@@ -7670,6 +7716,12 @@ class ArtemisKit {
|
|
|
7670
7716
|
this.storage = createStorageAdapter(config.storage);
|
|
7671
7717
|
}
|
|
7672
7718
|
}
|
|
7719
|
+
async createWorkflowSession(options) {
|
|
7720
|
+
return prepareWorkflowSession(options, this.config);
|
|
7721
|
+
}
|
|
7722
|
+
async runWorkflow(options) {
|
|
7723
|
+
return (await this.createWorkflowSession(options)).run();
|
|
7724
|
+
}
|
|
7673
7725
|
on(event, handler) {
|
|
7674
7726
|
if (!this.eventHandlers.has(event)) {
|
|
7675
7727
|
this.eventHandlers.set(event, new Set);
|
|
@@ -8414,7 +8466,7 @@ class ArtemisKit {
|
|
|
8414
8466
|
...this.config.providerConfig,
|
|
8415
8467
|
...options.providerConfig
|
|
8416
8468
|
};
|
|
8417
|
-
return
|
|
8469
|
+
return createAdapter2(adapterConfig);
|
|
8418
8470
|
}
|
|
8419
8471
|
buildMutations(mutationNames) {
|
|
8420
8472
|
if (!mutationNames || mutationNames.length === 0) {
|
|
@@ -8441,7 +8493,7 @@ class ArtemisKit {
|
|
|
8441
8493
|
}
|
|
8442
8494
|
}
|
|
8443
8495
|
function sleep(ms) {
|
|
8444
|
-
return new Promise((
|
|
8496
|
+
return new Promise((resolve2) => setTimeout(resolve2, ms));
|
|
8445
8497
|
}
|
|
8446
8498
|
function percentile(sortedArray, p) {
|
|
8447
8499
|
if (sortedArray.length === 0)
|
|
@@ -8455,13 +8507,20 @@ import {
|
|
|
8455
8507
|
AgentWorkflowSchema,
|
|
8456
8508
|
WorkflowPolicySchema,
|
|
8457
8509
|
WORKFLOW_TOOL_IDS,
|
|
8458
|
-
validateAgentWorkflow,
|
|
8510
|
+
validateAgentWorkflow as validateAgentWorkflow2,
|
|
8459
8511
|
parseAgentWorkflow,
|
|
8460
|
-
loadAgentWorkflow,
|
|
8512
|
+
loadAgentWorkflow as loadAgentWorkflow2,
|
|
8461
8513
|
listWorkflowTools,
|
|
8462
8514
|
getWorkflowTool,
|
|
8463
8515
|
executeSimulatedTool,
|
|
8464
|
-
createModelClientTarget
|
|
8516
|
+
createModelClientTarget as createModelClientTarget2,
|
|
8517
|
+
createAgentWorkflowSession as createAgentWorkflowSession2,
|
|
8518
|
+
runAgentWorkflow,
|
|
8519
|
+
createSimulatedWorkflowEnvironment,
|
|
8520
|
+
createDockerWorkflowEnvironment,
|
|
8521
|
+
createDockerWorkflowEnvironmentFactory,
|
|
8522
|
+
WORKFLOW_SANDBOX_IMAGE,
|
|
8523
|
+
WorkflowEnvironmentInitializationError
|
|
8465
8524
|
} from "@artemiskit/core";
|
|
8466
8525
|
import { isRunManifest, isRedTeamManifest, isStressManifest } from "@artemiskit/core";
|
|
8467
8526
|
|
|
@@ -10225,7 +10284,7 @@ function createInterceptor(client, config = {}) {
|
|
|
10225
10284
|
// src/guardian/policy.ts
|
|
10226
10285
|
var import_yaml = __toESM(require_dist(), 1);
|
|
10227
10286
|
import { existsSync, readFileSync } from "node:fs";
|
|
10228
|
-
import { resolve } from "node:path";
|
|
10287
|
+
import { resolve as resolve2 } from "node:path";
|
|
10229
10288
|
|
|
10230
10289
|
// ../../node_modules/zod/v3/external.js
|
|
10231
10290
|
var exports_external = {};
|
|
@@ -14277,7 +14336,7 @@ var GuardianPolicySchema = exports_external.object({
|
|
|
14277
14336
|
costLimits: CostLimitConfigSchema.optional()
|
|
14278
14337
|
});
|
|
14279
14338
|
function loadPolicy(filePath) {
|
|
14280
|
-
const absolutePath =
|
|
14339
|
+
const absolutePath = resolve2(filePath);
|
|
14281
14340
|
if (!existsSync(absolutePath)) {
|
|
14282
14341
|
throw new PolicyLoadError(`Policy file not found: ${absolutePath}`);
|
|
14283
14342
|
}
|
|
@@ -15197,10 +15256,11 @@ function createGuardian(config = {}) {
|
|
|
15197
15256
|
}
|
|
15198
15257
|
export {
|
|
15199
15258
|
validatePolicy,
|
|
15200
|
-
validateAgentWorkflow,
|
|
15259
|
+
validateAgentWorkflow2 as validateAgentWorkflow,
|
|
15201
15260
|
testCase,
|
|
15202
15261
|
similarity,
|
|
15203
15262
|
scenario,
|
|
15263
|
+
runAgentWorkflow,
|
|
15204
15264
|
regexCase,
|
|
15205
15265
|
regex,
|
|
15206
15266
|
parsePolicy,
|
|
@@ -15210,7 +15270,7 @@ export {
|
|
|
15210
15270
|
mergePolicies,
|
|
15211
15271
|
matchPattern,
|
|
15212
15272
|
loadPolicy,
|
|
15213
|
-
loadAgentWorkflow,
|
|
15273
|
+
loadAgentWorkflow2 as loadAgentWorkflow,
|
|
15214
15274
|
llmGrade,
|
|
15215
15275
|
listWorkflowTools,
|
|
15216
15276
|
jsonSchema,
|
|
@@ -15242,18 +15302,22 @@ export {
|
|
|
15242
15302
|
definePlugin,
|
|
15243
15303
|
defineEvaluator,
|
|
15244
15304
|
defineAdapter,
|
|
15305
|
+
createSimulatedWorkflowEnvironment,
|
|
15245
15306
|
createSemanticValidator,
|
|
15246
15307
|
createPIIGuardrail,
|
|
15247
|
-
createModelClientTarget,
|
|
15308
|
+
createModelClientTarget2 as createModelClientTarget,
|
|
15248
15309
|
createInterceptor,
|
|
15249
15310
|
createIntentClassifier,
|
|
15250
15311
|
createInjectionGuardrail,
|
|
15251
15312
|
createGuardrails,
|
|
15252
15313
|
createGuardian,
|
|
15314
|
+
createDockerWorkflowEnvironmentFactory,
|
|
15315
|
+
createDockerWorkflowEnvironment,
|
|
15253
15316
|
createDefaultPolicy,
|
|
15254
15317
|
createDefaultActionValidator,
|
|
15255
15318
|
createCustomPatternGuardrail,
|
|
15256
15319
|
createContentFilterGuardrail,
|
|
15320
|
+
createAgentWorkflowSession2 as createAgentWorkflowSession,
|
|
15257
15321
|
containsCase,
|
|
15258
15322
|
contains,
|
|
15259
15323
|
calculateSuccessRate,
|
|
@@ -15263,7 +15327,9 @@ export {
|
|
|
15263
15327
|
anyOf,
|
|
15264
15328
|
allOf,
|
|
15265
15329
|
WorkflowPolicySchema,
|
|
15330
|
+
WorkflowEnvironmentInitializationError,
|
|
15266
15331
|
WORKFLOW_TOOL_IDS,
|
|
15332
|
+
WORKFLOW_SANDBOX_IMAGE,
|
|
15267
15333
|
TestCaseBuilder,
|
|
15268
15334
|
SemanticValidator,
|
|
15269
15335
|
ScenarioBuilder,
|
package/dist/types-only.d.ts
CHANGED
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
* } from '@artemiskit/sdk/types'
|
|
16
16
|
* ```
|
|
17
17
|
*/
|
|
18
|
-
export type { ArtemisKitConfig, RunOptions, RedTeamOptions, StressOptions, RunResult, RedTeamResult, StressResult, CaseStartEvent, CaseCompleteEvent, ProgressEvent, RedTeamMutationStartEvent, RedTeamMutationCompleteEvent, StressRequestCompleteEvent, CaseStartHandler, CaseCompleteHandler, ProgressHandler, RedTeamMutationStartHandler, RedTeamMutationCompleteHandler, StressRequestCompleteHandler, ArtemisKitEvents, ArtemisKitEventName, } from './types.js';
|
|
18
|
+
export type { ArtemisKitConfig, WorkflowRunOptions, RunOptions, RedTeamOptions, StressOptions, RunResult, RedTeamResult, StressResult, CaseStartEvent, CaseCompleteEvent, ProgressEvent, RedTeamMutationStartEvent, RedTeamMutationCompleteEvent, StressRequestCompleteEvent, CaseStartHandler, CaseCompleteHandler, ProgressHandler, RedTeamMutationStartHandler, RedTeamMutationCompleteHandler, StressRequestCompleteHandler, ArtemisKitEvents, ArtemisKitEventName, } from './types.js';
|
|
19
19
|
export type { Scenario, TestCase, Expected, Provider, ProviderConfig, ChatMessageType, Variables, } from '@artemiskit/core';
|
|
20
20
|
export type { CaseResult, CaseRedactionInfo, RunManifest, RunConfig, RunMetrics, RedTeamManifest, RedTeamConfig, RedTeamMetrics, RedTeamCaseResult, RedTeamStatus, RedTeamSeverity, StressManifest, StressConfig, StressMetrics, StressRequestResult, AnyManifest, ResolvedConfig, ConfigSource, ManifestRedactionInfo, CostEstimateInfo, GitInfo, ProvenanceInfo, } from '@artemiskit/core';
|
|
21
21
|
export type { ModelClient, ModelCapabilities, GenerateOptions, GenerateResult, TokenUsage, ChatMessage, FunctionDefinition, ToolDefinition, ToolCall, AdapterConfig, BaseAdapterConfig, OpenAIAdapterConfig, AzureOpenAIAdapterConfig, VercelAIAdapterConfig, AnthropicAdapterConfig, LangChainAdapterConfig, DeepAgentsAdapterConfig, ProviderType, } from '@artemiskit/core';
|
|
@@ -28,4 +28,5 @@ export type { ArtemisKitMatchers, MatcherResult } from './matchers/index.js';
|
|
|
28
28
|
export type { ProviderName, ExpectationType, AnyResult, ExtractRunCases, ExtractRedTeamCases, ExtractStressResults, ExtractManifest, DeepPartial, PartialScenario, RequireFields, OptionalFields, StrictAdapterConfig, } from './utils/index.js';
|
|
29
29
|
export type { AdapterContract, EvaluatorContract, StorageContract, AdapterFactory, EvaluatorFactory, StorageFactory, ArtemisKitPlugin, } from './contracts/index.js';
|
|
30
30
|
export type { ScenarioBuilder, TestCaseBuilder } from './builders/index.js';
|
|
31
|
+
export type { AgentWorkflow, WorkflowPolicy, WorkflowJson, WorkflowToolId, WorkflowResource, WorkflowToolDescriptor, SimulatedToolEvidence, SimulatedToolResult, AgentTarget, AgentTurnRequest, AgentTurnResult, AgentTargetCapabilities, AgentTargetFailure, AgentWorkflowExecution, AgentWorkflowReason, AgentWorkflowEvent, AgentWorkflowRecord, AgentWorkflowResult, AgentWorkflowSessionOptions, AgentWorkflowSession, WorkflowState, WorkflowEnvironment, WorkflowEnvironmentFactory, WorkflowEnvironmentCleanup, DockerWorkflowEnvironmentOptions, WorkflowEnvironmentInitializationError, } from '@artemiskit/core';
|
|
31
32
|
//# sourceMappingURL=types-only.d.ts.map
|
package/dist/types-only.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types-only.d.ts","sourceRoot":"","sources":["../src/types-only.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAMH,YAAY,EAEV,gBAAgB,EAChB,UAAU,EACV,cAAc,EACd,aAAa,EAEb,SAAS,EACT,aAAa,EACb,YAAY,EAEZ,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,yBAAyB,EACzB,4BAA4B,EAC5B,0BAA0B,EAE1B,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAE5B,gBAAgB,EAChB,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,cAAc,EACd,eAAe,EACf,SAAS,GACV,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,UAAU,EACV,iBAAiB,EAEjB,WAAW,EACX,SAAS,EACT,UAAU,EAEV,eAAe,EACf,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,eAAe,EAEf,cAAc,EACd,YAAY,EACZ,aAAa,EACb,mBAAmB,EAEnB,WAAW,EACX,cAAc,EACd,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,OAAO,EACP,cAAc,GACf,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,WAAW,EACX,iBAAiB,EAEjB,eAAe,EACf,cAAc,EACd,UAAU,EAEV,WAAW,EAEX,kBAAkB,EAClB,cAAc,EACd,QAAQ,EAER,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,YAAY,GACb,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMrF,YAAY,EACV,cAAc,EACd,aAAa,EACb,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EAEtB,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EACtB,eAAe,EACf,mBAAmB,EACnB,cAAc,EACd,uBAAuB,GACxB,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMxD,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAM7E,YAAY,EAEV,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,sBAAsB,EACtB,gBAAgB,EAChB,iBAAiB,EAEjB,YAAY,EACZ,iBAAiB,EACjB,eAAe,EACf,aAAa,EACb,SAAS,EACT,eAAe,EAEf,UAAU,EACV,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,eAAe,EACf,eAAe,EAEf,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EAEnB,oBAAoB,EACpB,cAAc,EAEd,YAAY,EACZ,OAAO,EACP,WAAW,EACX,kBAAkB,EAClB,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,eAAe,EAEf,eAAe,EACf,mBAAmB,EACnB,YAAY,EAEZ,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EAEpB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EAEpB,aAAa,EACb,0BAA0B,GAC3B,MAAM,YAAY,CAAC;AAMpB,YAAY,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAMpE,YAAY,EAEV,YAAY,EACZ,eAAe,EAEf,SAAS,EACT,eAAe,EACf,mBAAmB,EACnB,oBAAoB,EACpB,eAAe,EAEf,WAAW,EACX,eAAe,EACf,aAAa,EACb,cAAc,EACd,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,aAAa,CAAC;AAMrB,YAAY,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC"}
|
|
1
|
+
{"version":3,"file":"types-only.d.ts","sourceRoot":"","sources":["../src/types-only.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAMH,YAAY,EAEV,gBAAgB,EAChB,kBAAkB,EAClB,UAAU,EACV,cAAc,EACd,aAAa,EAEb,SAAS,EACT,aAAa,EACb,YAAY,EAEZ,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,yBAAyB,EACzB,4BAA4B,EAC5B,0BAA0B,EAE1B,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAE5B,gBAAgB,EAChB,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,cAAc,EACd,eAAe,EACf,SAAS,GACV,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,UAAU,EACV,iBAAiB,EAEjB,WAAW,EACX,SAAS,EACT,UAAU,EAEV,eAAe,EACf,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,eAAe,EAEf,cAAc,EACd,YAAY,EACZ,aAAa,EACb,mBAAmB,EAEnB,WAAW,EACX,cAAc,EACd,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,OAAO,EACP,cAAc,GACf,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,WAAW,EACX,iBAAiB,EAEjB,eAAe,EACf,cAAc,EACd,UAAU,EAEV,WAAW,EAEX,kBAAkB,EAClB,cAAc,EACd,QAAQ,EAER,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,YAAY,GACb,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMrF,YAAY,EACV,cAAc,EACd,aAAa,EACb,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EAEtB,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EACtB,eAAe,EACf,mBAAmB,EACnB,cAAc,EACd,uBAAuB,GACxB,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMxD,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAM7E,YAAY,EAEV,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,sBAAsB,EACtB,gBAAgB,EAChB,iBAAiB,EAEjB,YAAY,EACZ,iBAAiB,EACjB,eAAe,EACf,aAAa,EACb,SAAS,EACT,eAAe,EAEf,UAAU,EACV,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,eAAe,EACf,eAAe,EAEf,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EAEnB,oBAAoB,EACpB,cAAc,EAEd,YAAY,EACZ,OAAO,EACP,WAAW,EACX,kBAAkB,EAClB,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,eAAe,EAEf,eAAe,EACf,mBAAmB,EACnB,YAAY,EAEZ,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EAEpB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EAEpB,aAAa,EACb,0BAA0B,GAC3B,MAAM,YAAY,CAAC;AAMpB,YAAY,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAMpE,YAAY,EAEV,YAAY,EACZ,eAAe,EAEf,SAAS,EACT,eAAe,EACf,mBAAmB,EACnB,oBAAoB,EACpB,eAAe,EAEf,WAAW,EACX,eAAe,EACf,aAAa,EACb,cAAc,EACd,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,aAAa,CAAC;AAMrB,YAAY,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAEnE,YAAY,EACV,aAAa,EACb,cAAc,EACd,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,sBAAsB,EACtB,qBAAqB,EACrB,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,eAAe,EACf,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,2BAA2B,EAC3B,oBAAoB,EACpB,aAAa,EACb,mBAAmB,EACnB,0BAA0B,EAC1B,0BAA0B,EAC1B,gCAAgC,EAChC,sCAAsC,GACvC,MAAM,kBAAkB,CAAC"}
|
package/dist/types.d.ts
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* @artemiskit/sdk
|
|
3
3
|
* Type definitions for the programmatic SDK
|
|
4
4
|
*/
|
|
5
|
-
import type { AdapterConfig, CaseResult, ComparisonEligibility, ModelClient, RedTeamManifest, RunManifest, StorageConfig, StressManifest, StressRequestResult, ToolExecutor } from '@artemiskit/core';
|
|
5
|
+
import type { AdapterConfig, AgentTarget, AgentWorkflow, AgentWorkflowSessionOptions, CaseResult, ComparisonEligibility, ModelClient, RedTeamManifest, RunManifest, StorageConfig, StressManifest, StressRequestResult, ToolExecutor } from '@artemiskit/core';
|
|
6
6
|
import type { RedactionConfig } from '@artemiskit/core';
|
|
7
7
|
import type { Scenario } from '@artemiskit/core';
|
|
8
8
|
/**
|
|
@@ -380,4 +380,15 @@ export interface CompareResult {
|
|
|
380
380
|
/** Threshold used for regression detection */
|
|
381
381
|
threshold: number;
|
|
382
382
|
}
|
|
383
|
+
/** Controlled workflow execution. Text/state remain in memory unless the caller explicitly saves them. */
|
|
384
|
+
export interface WorkflowRunOptions extends Omit<AgentWorkflowSessionOptions, 'workflow' | 'target'> {
|
|
385
|
+
/** Workflow object or YAML path. File fixtures default to the workflow file's directory. */
|
|
386
|
+
workflow: AgentWorkflow | string;
|
|
387
|
+
/** Explicit host-owned client. Its transport retry behavior remains the caller's responsibility. */
|
|
388
|
+
client?: ModelClient;
|
|
389
|
+
/** Explicit custom target, mutually exclusive with client. */
|
|
390
|
+
target?: AgentTarget;
|
|
391
|
+
/** Trusted transport configuration; workflow provider/model and zero retries remain authoritative. */
|
|
392
|
+
providerConfig?: Partial<AdapterConfig>;
|
|
393
|
+
}
|
|
383
394
|
//# sourceMappingURL=types.d.ts.map
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EACV,aAAa,EACb,UAAU,EACV,qBAAqB,EACrB,WAAW,EACX,eAAe,EACf,WAAW,EACX,aAAa,EACb,cAAc,EACd,mBAAmB,EACnB,YAAY,EACb,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAMjD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,UAAU,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,OAAO,GAAG,SAAS,GAAG,UAAU,CAAC;IACxC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC3C,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;IAChD,QAAQ,EAAE,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAC;IAC1D,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,0BAA0B;IACzC,MAAM,EAAE,mBAAmB,CAAC;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;CACpB;AAMD,MAAM,MAAM,gBAAgB,GAAG,CAAC,KAAK,EAAE,cAAc,KAAK,IAAI,CAAC;AAC/D,MAAM,MAAM,mBAAmB,GAAG,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,CAAC;AACrE,MAAM,MAAM,eAAe,GAAG,CAAC,KAAK,EAAE,aAAa,KAAK,IAAI,CAAC;AAC7D,MAAM,MAAM,2BAA2B,GAAG,CAAC,KAAK,EAAE,yBAAyB,KAAK,IAAI,CAAC;AACrF,MAAM,MAAM,8BAA8B,GAAG,CAAC,KAAK,EAAE,4BAA4B,KAAK,IAAI,CAAC;AAC3F,MAAM,MAAM,4BAA4B,GAAG,CAAC,KAAK,EAAE,0BAA0B,KAAK,IAAI,CAAC;AAMvF;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,yCAAyC;IACzC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,2BAA2B;IAC3B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,sCAAsC;IACtC,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,iCAAiC;IACjC,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,+CAA+C;IAC/C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,gCAAgC;IAChC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iCAAiC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,2EAA2E;IAC3E,YAAY,CAAC,EAAE,YAAY,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,wCAAwC;IACxC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,mCAAmC;IACnC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gCAAgC;IAChC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+BAA+B;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gCAAgC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,kDAAkD;IAClD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B;AAMD;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,6BAA6B;IAC7B,QAAQ,EAAE,WAAW,CAAC;IACtB,8BAA8B;IAC9B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,+BAA+B;IAC/B,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,6BAA6B;IAC7B,QAAQ,EAAE,eAAe,CAAC;IAC1B,4DAA4D;IAC5D,OAAO,EAAE,OAAO,CAAC;IACjB,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,gCAAgC;IAChC,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,6BAA6B;IAC7B,QAAQ,EAAE,cAAc,CAAC;IACzB,qDAAqD;IACrD,OAAO,EAAE,OAAO,CAAC;IACjB,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,mCAAmC;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,kCAAkC;IAClC,YAAY,EAAE,MAAM,CAAC;CACtB;AAMD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,cAAc,CAAC;IAC1B,YAAY,EAAE,iBAAiB,CAAC;IAChC,QAAQ,EAAE,aAAa,CAAC;IACxB,oBAAoB,EAAE,yBAAyB,CAAC;IAChD,uBAAuB,EAAE,4BAA4B,CAAC;IACtD,qBAAqB,EAAE,0BAA0B,CAAC;CACnD;AAED,MAAM,MAAM,mBAAmB,GAAG,MAAM,gBAAgB,CAAC;AAMzD;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B;;;OAGG;IACH,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAE5B;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,qCAAqC;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,oBAAoB;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,sBAAsB;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,gBAAgB;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,oBAAoB;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,2BAA2B;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,qCAAqC;IACrC,KAAK,EAAE,OAAO,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,gCAAgC;IAChC,KAAK,EAAE,OAAO,CAAC;IACf,6CAA6C;IAC7C,SAAS,EAAE,kBAAkB,EAAE,CAAC;IAChC,wBAAwB;IACxB,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,0CAA0C;IAC1C,QAAQ,EAAE,iBAAiB,EAAE,CAAC;CAC/B;AAMD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;;OAGG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,qBAAqB;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,6BAA6B;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,6BAA6B;IAC7B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,+DAA+D;IAC/D,mBAAmB,EAAE,OAAO,CAAC;IAC7B,iFAAiF;IACjF,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,0DAA0D;IAC1D,WAAW,EAAE,KAAK,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,cAAc,EAAE,MAAM,CAAC;QACvB,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC,CAAC;IACH,0DAA0D;IAC1D,SAAS,EAAE,KAAK,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,cAAc,EAAE,MAAM,CAAC;QACvB,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC,CAAC;IACH,kCAAkC;IAClC,SAAS,EAAE,KAAK,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;IACH,2EAA2E;IAC3E,UAAU,EAAE,KAAK,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;IACH,+EAA+E;IAC/E,YAAY,EAAE,KAAK,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,2BAA2B;IAC3B,QAAQ,EAAE,UAAU,CAAC;IACrB,0BAA0B;IAC1B,OAAO,EAAE,UAAU,CAAC;IACpB,0BAA0B;IAC1B,UAAU,EAAE,iBAAiB,CAAC;IAC9B,oEAAoE;IACpE,WAAW,EAAE,qBAAqB,CAAC;IACnC,6EAA6E;IAC7E,aAAa,EAAE,OAAO,CAAC;IACvB,8CAA8C;IAC9C,SAAS,EAAE,MAAM,CAAC;CACnB"}
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EACV,aAAa,EACb,WAAW,EACX,aAAa,EACb,2BAA2B,EAC3B,UAAU,EACV,qBAAqB,EACrB,WAAW,EACX,eAAe,EACf,WAAW,EACX,aAAa,EACb,cAAc,EACd,mBAAmB,EACnB,YAAY,EACb,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAMjD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,UAAU,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,OAAO,GAAG,SAAS,GAAG,UAAU,CAAC;IACxC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC3C,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;IAChD,QAAQ,EAAE,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAC;IAC1D,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,0BAA0B;IACzC,MAAM,EAAE,mBAAmB,CAAC;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;CACpB;AAMD,MAAM,MAAM,gBAAgB,GAAG,CAAC,KAAK,EAAE,cAAc,KAAK,IAAI,CAAC;AAC/D,MAAM,MAAM,mBAAmB,GAAG,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,CAAC;AACrE,MAAM,MAAM,eAAe,GAAG,CAAC,KAAK,EAAE,aAAa,KAAK,IAAI,CAAC;AAC7D,MAAM,MAAM,2BAA2B,GAAG,CAAC,KAAK,EAAE,yBAAyB,KAAK,IAAI,CAAC;AACrF,MAAM,MAAM,8BAA8B,GAAG,CAAC,KAAK,EAAE,4BAA4B,KAAK,IAAI,CAAC;AAC3F,MAAM,MAAM,4BAA4B,GAAG,CAAC,KAAK,EAAE,0BAA0B,KAAK,IAAI,CAAC;AAMvF;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,yCAAyC;IACzC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,2BAA2B;IAC3B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,sCAAsC;IACtC,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,iCAAiC;IACjC,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,+CAA+C;IAC/C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,gCAAgC;IAChC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iCAAiC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,2EAA2E;IAC3E,YAAY,CAAC,EAAE,YAAY,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,wCAAwC;IACxC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,mCAAmC;IACnC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gCAAgC;IAChC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+BAA+B;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gCAAgC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,kDAAkD;IAClD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B;AAMD;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,6BAA6B;IAC7B,QAAQ,EAAE,WAAW,CAAC;IACtB,8BAA8B;IAC9B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,+BAA+B;IAC/B,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,6BAA6B;IAC7B,QAAQ,EAAE,eAAe,CAAC;IAC1B,4DAA4D;IAC5D,OAAO,EAAE,OAAO,CAAC;IACjB,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,gCAAgC;IAChC,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,6BAA6B;IAC7B,QAAQ,EAAE,cAAc,CAAC;IACzB,qDAAqD;IACrD,OAAO,EAAE,OAAO,CAAC;IACjB,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,mCAAmC;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,kCAAkC;IAClC,YAAY,EAAE,MAAM,CAAC;CACtB;AAMD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,cAAc,CAAC;IAC1B,YAAY,EAAE,iBAAiB,CAAC;IAChC,QAAQ,EAAE,aAAa,CAAC;IACxB,oBAAoB,EAAE,yBAAyB,CAAC;IAChD,uBAAuB,EAAE,4BAA4B,CAAC;IACtD,qBAAqB,EAAE,0BAA0B,CAAC;CACnD;AAED,MAAM,MAAM,mBAAmB,GAAG,MAAM,gBAAgB,CAAC;AAMzD;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B;;;OAGG;IACH,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAE5B;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,qCAAqC;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,oBAAoB;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,sBAAsB;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,gBAAgB;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,oBAAoB;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,2BAA2B;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,qCAAqC;IACrC,KAAK,EAAE,OAAO,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,gCAAgC;IAChC,KAAK,EAAE,OAAO,CAAC;IACf,6CAA6C;IAC7C,SAAS,EAAE,kBAAkB,EAAE,CAAC;IAChC,wBAAwB;IACxB,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,0CAA0C;IAC1C,QAAQ,EAAE,iBAAiB,EAAE,CAAC;CAC/B;AAMD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;;OAGG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,qBAAqB;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,6BAA6B;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,6BAA6B;IAC7B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,+DAA+D;IAC/D,mBAAmB,EAAE,OAAO,CAAC;IAC7B,iFAAiF;IACjF,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,0DAA0D;IAC1D,WAAW,EAAE,KAAK,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,cAAc,EAAE,MAAM,CAAC;QACvB,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC,CAAC;IACH,0DAA0D;IAC1D,SAAS,EAAE,KAAK,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,cAAc,EAAE,MAAM,CAAC;QACvB,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC,CAAC;IACH,kCAAkC;IAClC,SAAS,EAAE,KAAK,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;IACH,2EAA2E;IAC3E,UAAU,EAAE,KAAK,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;IACH,+EAA+E;IAC/E,YAAY,EAAE,KAAK,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,2BAA2B;IAC3B,QAAQ,EAAE,UAAU,CAAC;IACrB,0BAA0B;IAC1B,OAAO,EAAE,UAAU,CAAC;IACpB,0BAA0B;IAC1B,UAAU,EAAE,iBAAiB,CAAC;IAC9B,oEAAoE;IACpE,WAAW,EAAE,qBAAqB,CAAC;IACnC,6EAA6E;IAC7E,aAAa,EAAE,OAAO,CAAC;IACvB,8CAA8C;IAC9C,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,0GAA0G;AAC1G,MAAM,WAAW,kBACf,SAAQ,IAAI,CAAC,2BAA2B,EAAE,UAAU,GAAG,QAAQ,CAAC;IAChE,4FAA4F;IAC5F,QAAQ,EAAE,aAAa,GAAG,MAAM,CAAC;IACjC,oGAAoG;IACpG,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,8DAA8D;IAC9D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,sGAAsG;IACtG,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;CACzC"}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import { type AgentWorkflowSession } from '@artemiskit/core';
|
|
2
|
+
import type { ArtemisKitConfig, WorkflowRunOptions } from './types.js';
|
|
3
|
+
export declare function prepareWorkflowSession(options: WorkflowRunOptions, config: ArtemisKitConfig): Promise<AgentWorkflowSession>;
|
|
4
|
+
//# sourceMappingURL=workflow.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"workflow.d.ts","sourceRoot":"","sources":["../src/workflow.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,KAAK,oBAAoB,EAO1B,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAEpE,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,kBAAkB,EAC3B,MAAM,EAAE,gBAAgB,GACvB,OAAO,CAAC,oBAAoB,CAAC,CAkC/B"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@artemiskit/sdk",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.0",
|
|
4
4
|
"description": "Programmatic SDK for ArtemisKit LLM evaluation toolkit - integrate into your Node.js apps",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -71,8 +71,8 @@
|
|
|
71
71
|
"test:coverage": "bun test --coverage"
|
|
72
72
|
},
|
|
73
73
|
"dependencies": {
|
|
74
|
-
"@artemiskit/core": "0.6.
|
|
75
|
-
"@artemiskit/redteam": "0.3.
|
|
74
|
+
"@artemiskit/core": "0.6.1",
|
|
75
|
+
"@artemiskit/redteam": "0.3.9"
|
|
76
76
|
},
|
|
77
77
|
"devDependencies": {
|
|
78
78
|
"@types/bun": "^1.1.0",
|
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
import { afterEach, describe, expect, test } from 'bun:test';
|
|
2
|
+
import { mkdtemp, readdir, writeFile } from 'node:fs/promises';
|
|
3
|
+
import { tmpdir } from 'node:os';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
import { fileURLToPath } from 'node:url';
|
|
6
|
+
import {
|
|
7
|
+
type AgentTarget,
|
|
8
|
+
type AgentWorkflow,
|
|
9
|
+
type AgentWorkflowEvent,
|
|
10
|
+
createSimulatedWorkflowEnvironment,
|
|
11
|
+
validateAgentWorkflow,
|
|
12
|
+
} from '@artemiskit/core';
|
|
13
|
+
import { OpenAIAdapter } from '../../../adapters/openai/src/client';
|
|
14
|
+
import { ArtemisKit } from '../artemiskit';
|
|
15
|
+
import type { WorkflowRunOptions } from '../types-only';
|
|
16
|
+
|
|
17
|
+
const servers: ReturnType<typeof Bun.serve>[] = [];
|
|
18
|
+
afterEach(() => {
|
|
19
|
+
for (const server of servers.splice(0)) server.stop(true);
|
|
20
|
+
});
|
|
21
|
+
function workflow(): AgentWorkflow {
|
|
22
|
+
return validateAgentWorkflow({
|
|
23
|
+
version: '1',
|
|
24
|
+
kind: 'agent_workflow',
|
|
25
|
+
name: 'sdk-workflow',
|
|
26
|
+
target: {
|
|
27
|
+
provider: 'openai',
|
|
28
|
+
model: 'workflow-authoritative',
|
|
29
|
+
generation: { max_tokens: 256 },
|
|
30
|
+
},
|
|
31
|
+
environment: {
|
|
32
|
+
type: 'simulated',
|
|
33
|
+
policy: {
|
|
34
|
+
network: 'denied',
|
|
35
|
+
side_effects: 'denied',
|
|
36
|
+
permissions: { files: 'write' },
|
|
37
|
+
budgets: { max_actions: 10, max_tokens: 4096, timeout_ms: 1000 },
|
|
38
|
+
},
|
|
39
|
+
},
|
|
40
|
+
tools: ['write_file'],
|
|
41
|
+
workflow: {
|
|
42
|
+
system_instructions: 'PRIVATE-INSTRUCTIONS',
|
|
43
|
+
initial_state: { files: {} },
|
|
44
|
+
turns: [{ role: 'user', content: 'PRIVATE-PROMPT' }],
|
|
45
|
+
},
|
|
46
|
+
outcomes: { deterministic: [{ type: 'file', path: 'result.txt', exists: true }] },
|
|
47
|
+
evidence: { trace: 'summary', artifacts: 'checksums', redact: true },
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
function target(): AgentTarget {
|
|
51
|
+
return {
|
|
52
|
+
provider: 'custom',
|
|
53
|
+
capabilities: async () => ({ status: 'available', toolUse: true, transportCancellation: true }),
|
|
54
|
+
turn: async () => ({
|
|
55
|
+
status: 'completed',
|
|
56
|
+
id: 'fixture',
|
|
57
|
+
model: 'fixture',
|
|
58
|
+
message: { role: 'assistant', content: 'PRIVATE-ANSWER' },
|
|
59
|
+
tokens: { prompt: 2, completion: 1, total: 3 },
|
|
60
|
+
latencyMs: 1,
|
|
61
|
+
}),
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
describe('SDK native workflow session', () => {
|
|
66
|
+
test('configured adapter uses workflow identity and zero retries with real loopback multi-turn execution', async () => {
|
|
67
|
+
const bodies: Record<string, unknown>[] = [];
|
|
68
|
+
const server = Bun.serve({
|
|
69
|
+
hostname: '127.0.0.1',
|
|
70
|
+
port: 0,
|
|
71
|
+
async fetch(request) {
|
|
72
|
+
bodies.push(await request.json());
|
|
73
|
+
const first = bodies.length === 1;
|
|
74
|
+
return Response.json({
|
|
75
|
+
id: 'fixture',
|
|
76
|
+
model: 'observed-model',
|
|
77
|
+
choices: [
|
|
78
|
+
{
|
|
79
|
+
index: 0,
|
|
80
|
+
message: {
|
|
81
|
+
role: 'assistant',
|
|
82
|
+
content: first ? null : 'PRIVATE-ANSWER',
|
|
83
|
+
...(first
|
|
84
|
+
? {
|
|
85
|
+
tool_calls: [
|
|
86
|
+
{
|
|
87
|
+
id: 'private-id',
|
|
88
|
+
type: 'function',
|
|
89
|
+
function: {
|
|
90
|
+
name: 'write_file',
|
|
91
|
+
arguments: '{"path":"result.txt","content":"PRIVATE-STATE"}',
|
|
92
|
+
},
|
|
93
|
+
},
|
|
94
|
+
],
|
|
95
|
+
}
|
|
96
|
+
: {}),
|
|
97
|
+
},
|
|
98
|
+
finish_reason: first ? 'tool_calls' : 'stop',
|
|
99
|
+
},
|
|
100
|
+
],
|
|
101
|
+
usage: { prompt_tokens: 2, completion_tokens: 1, total_tokens: 3 },
|
|
102
|
+
});
|
|
103
|
+
},
|
|
104
|
+
});
|
|
105
|
+
servers.push(server);
|
|
106
|
+
// Legacy SDK tests mock the core factory globally; qualify the real public boundary in a fresh process.
|
|
107
|
+
const directory = await mkdtemp(join(tmpdir(), 'artemis-sdk-client-'));
|
|
108
|
+
const workflowPath = join(directory, 'workflow.json');
|
|
109
|
+
await writeFile(workflowPath, JSON.stringify(workflow()));
|
|
110
|
+
const child = Bun.spawn(
|
|
111
|
+
[
|
|
112
|
+
process.execPath,
|
|
113
|
+
'-e',
|
|
114
|
+
`
|
|
115
|
+
import { ArtemisKit } from ${JSON.stringify(fileURLToPath(new URL('../artemiskit.ts', import.meta.url)))};
|
|
116
|
+
import { registerAdapter } from '@artemiskit/core';
|
|
117
|
+
import { OpenAIAdapter } from '@artemiskit/adapter-openai';
|
|
118
|
+
import assert from 'node:assert/strict';
|
|
119
|
+
registerAdapter('openai', async (config) => {
|
|
120
|
+
assert.equal(config.provider, 'openai');
|
|
121
|
+
assert.equal(config.defaultModel, 'workflow-authoritative');
|
|
122
|
+
assert.equal(config.maxRetries, 0);
|
|
123
|
+
return new OpenAIAdapter(config);
|
|
124
|
+
});
|
|
125
|
+
const kit = new ArtemisKit({ provider: 'anthropic', model: 'wrong', providerConfig: {
|
|
126
|
+
provider: 'openai', defaultModel: 'wrong-config', apiKey: 'dummy-only',
|
|
127
|
+
baseUrl: ${JSON.stringify(`http://127.0.0.1:${server.port}/v1`)}, maxRetries: 9,
|
|
128
|
+
}});
|
|
129
|
+
const events = [];
|
|
130
|
+
const result = await kit.runWorkflow({ workflow: ${JSON.stringify(workflowPath)}, onEvent: (event) => events.push(event) });
|
|
131
|
+
registerAdapter('openai', async () => { throw new Error('SYNTHETIC_PRIVATE_CONSTRUCTION_DETAIL'); });
|
|
132
|
+
await assert.rejects(kit.createWorkflowSession({ workflow: ${JSON.stringify(workflowPath)} }),
|
|
133
|
+
(error) => error.code === 'PROVIDER_UNAVAILABLE' &&
|
|
134
|
+
error.message === 'Unable to initialize workflow target' && error.cause === undefined);
|
|
135
|
+
console.log(JSON.stringify({ result, events }));
|
|
136
|
+
`,
|
|
137
|
+
],
|
|
138
|
+
{
|
|
139
|
+
cwd: fileURLToPath(new URL('../', import.meta.url)),
|
|
140
|
+
stdin: 'ignore',
|
|
141
|
+
stdout: 'pipe',
|
|
142
|
+
stderr: 'pipe',
|
|
143
|
+
env: { PATH: process.env.PATH },
|
|
144
|
+
}
|
|
145
|
+
);
|
|
146
|
+
const timer = setTimeout(() => child.kill(), 10000);
|
|
147
|
+
const [exit, stdout, stderr] = await Promise.all([
|
|
148
|
+
child.exited,
|
|
149
|
+
new Response(child.stdout).text(),
|
|
150
|
+
new Response(child.stderr).text(),
|
|
151
|
+
]);
|
|
152
|
+
clearTimeout(timer);
|
|
153
|
+
expect({ exit, stderr }).toEqual({ exit: 0, stderr: '' });
|
|
154
|
+
const { result, events } = JSON.parse(stdout);
|
|
155
|
+
expect(bodies[0].model).toBe('workflow-authoritative');
|
|
156
|
+
expect(result.record.execution).toBe('completed');
|
|
157
|
+
expect(result.record.taskVerification).toBe('unavailable');
|
|
158
|
+
expect(result.record.budgets).toMatchObject({ modelRequests: 2, toolCalls: 1 });
|
|
159
|
+
expect(result.state?.files).toEqual({ 'result.txt': 'PRIVATE-STATE' });
|
|
160
|
+
expect(JSON.stringify(result.record)).not.toContain('PRIVATE-');
|
|
161
|
+
expect(events.some((event) => event.type === 'tool_completed')).toBe(true);
|
|
162
|
+
});
|
|
163
|
+
test('file fixture roots, session events and environment overrides use the same engine without persistence', async () => {
|
|
164
|
+
const directory = await mkdtemp(join(tmpdir(), 'artemis-sdk-workflow-'));
|
|
165
|
+
const scenario = workflow();
|
|
166
|
+
scenario.workflow.initial_state = 'fixture.json';
|
|
167
|
+
await writeFile(join(directory, 'fixture.json'), '{"files":{"initial.txt":"PRIVATE"}}');
|
|
168
|
+
await writeFile(join(directory, 'workflow.json'), JSON.stringify(scenario));
|
|
169
|
+
let factories = 0;
|
|
170
|
+
const options: WorkflowRunOptions = {
|
|
171
|
+
workflow: join(directory, 'workflow.json'),
|
|
172
|
+
target: target(),
|
|
173
|
+
environmentFactory: async (args) => {
|
|
174
|
+
factories++;
|
|
175
|
+
return createSimulatedWorkflowEnvironment(args);
|
|
176
|
+
},
|
|
177
|
+
};
|
|
178
|
+
const session = await new ArtemisKit().createWorkflowSession(options);
|
|
179
|
+
expect(session.state).toBe('idle');
|
|
180
|
+
expect(factories).toBe(0);
|
|
181
|
+
const stream = (async () => {
|
|
182
|
+
const events: AgentWorkflowEvent[] = [];
|
|
183
|
+
for await (const event of session.events()) events.push(event);
|
|
184
|
+
return events;
|
|
185
|
+
})();
|
|
186
|
+
const result = await session.run();
|
|
187
|
+
expect(session.state).toBe('completed');
|
|
188
|
+
expect(result.state?.files).toEqual({ 'initial.txt': 'PRIVATE' });
|
|
189
|
+
expect(factories).toBe(1);
|
|
190
|
+
expect((await stream).at(-1)?.type).toBe('finished');
|
|
191
|
+
expect((await readdir(directory)).sort()).toEqual(['fixture.json', 'workflow.json']);
|
|
192
|
+
});
|
|
193
|
+
test('supports explicit client, rejects client+target and custom unsupported capabilities', async () => {
|
|
194
|
+
const unsupported = target();
|
|
195
|
+
unsupported.capabilities = async () => ({
|
|
196
|
+
status: 'available',
|
|
197
|
+
toolUse: false,
|
|
198
|
+
transportCancellation: false,
|
|
199
|
+
});
|
|
200
|
+
const kit = new ArtemisKit();
|
|
201
|
+
const client = new OpenAIAdapter({
|
|
202
|
+
provider: 'openai',
|
|
203
|
+
apiKey: 'dummy-only',
|
|
204
|
+
baseUrl: 'http://127.0.0.1:1',
|
|
205
|
+
maxRetries: 0,
|
|
206
|
+
});
|
|
207
|
+
await expect(
|
|
208
|
+
kit.createWorkflowSession({ workflow: workflow(), client, target: unsupported })
|
|
209
|
+
).rejects.toThrow('client or target');
|
|
210
|
+
expect(
|
|
211
|
+
(await kit.runWorkflow({ workflow: workflow(), target: unsupported })).record.execution
|
|
212
|
+
).toBe('unsupported');
|
|
213
|
+
const aborted = new AbortController();
|
|
214
|
+
aborted.abort();
|
|
215
|
+
expect(
|
|
216
|
+
(await kit.runWorkflow({ workflow: workflow(), client, signal: aborted.signal })).record
|
|
217
|
+
.execution
|
|
218
|
+
).toBe('cancelled');
|
|
219
|
+
});
|
|
220
|
+
test('cancellation and all SDK engine options stay host controlled', async () => {
|
|
221
|
+
let resolveStarted!: () => void;
|
|
222
|
+
const started = new Promise<void>((resolve) => {
|
|
223
|
+
resolveStarted = resolve;
|
|
224
|
+
});
|
|
225
|
+
const custom = target();
|
|
226
|
+
custom.turn = async (_request, signal) => {
|
|
227
|
+
resolveStarted();
|
|
228
|
+
await new Promise<void>((resolve) =>
|
|
229
|
+
signal?.addEventListener('abort', () => resolve(), { once: true })
|
|
230
|
+
);
|
|
231
|
+
return { status: 'error', code: 'aborted' };
|
|
232
|
+
};
|
|
233
|
+
const session = await new ArtemisKit().createWorkflowSession({
|
|
234
|
+
workflow: workflow(),
|
|
235
|
+
target: custom,
|
|
236
|
+
cleanupTimeoutMs: 20,
|
|
237
|
+
});
|
|
238
|
+
const result = session.run();
|
|
239
|
+
await started;
|
|
240
|
+
session.cancel();
|
|
241
|
+
expect((await result).record.execution).toBe('cancelled');
|
|
242
|
+
expect(session.state).toBe('completed');
|
|
243
|
+
});
|
|
244
|
+
});
|
package/src/artemiskit.ts
CHANGED
|
@@ -41,6 +41,7 @@ import {
|
|
|
41
41
|
UnsafeResponseDetector,
|
|
42
42
|
} from '@artemiskit/redteam';
|
|
43
43
|
import { nanoid } from 'nanoid';
|
|
44
|
+
import { prepareWorkflowSession } from './workflow';
|
|
44
45
|
|
|
45
46
|
import type {
|
|
46
47
|
ArtemisKitConfig,
|
|
@@ -62,6 +63,7 @@ import type {
|
|
|
62
63
|
StressResult,
|
|
63
64
|
ValidateOptions,
|
|
64
65
|
ValidationResult,
|
|
66
|
+
WorkflowRunOptions,
|
|
65
67
|
} from './types';
|
|
66
68
|
|
|
67
69
|
/**
|
|
@@ -125,6 +127,16 @@ export class ArtemisKit {
|
|
|
125
127
|
}
|
|
126
128
|
}
|
|
127
129
|
|
|
130
|
+
/** Create a cancellable native session without executing model turns or persisting results. */
|
|
131
|
+
async createWorkflowSession(options: WorkflowRunOptions) {
|
|
132
|
+
return prepareWorkflowSession(options, this.config);
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** Execute a workflow; task verification is explicitly unavailable until outcome scoring. */
|
|
136
|
+
async runWorkflow(options: WorkflowRunOptions) {
|
|
137
|
+
return (await this.createWorkflowSession(options)).run();
|
|
138
|
+
}
|
|
139
|
+
|
|
128
140
|
// ==========================================================================
|
|
129
141
|
// Event Emitter Methods
|
|
130
142
|
// ==========================================================================
|
package/src/index.ts
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
// Main class
|
|
7
7
|
export { ArtemisKit } from './artemiskit';
|
|
8
8
|
|
|
9
|
-
// Workflow authoring
|
|
9
|
+
// Workflow authoring, controlled native execution and tool primitives.
|
|
10
10
|
export {
|
|
11
11
|
AgentWorkflowSchema,
|
|
12
12
|
WorkflowPolicySchema,
|
|
@@ -18,6 +18,13 @@ export {
|
|
|
18
18
|
getWorkflowTool,
|
|
19
19
|
executeSimulatedTool,
|
|
20
20
|
createModelClientTarget,
|
|
21
|
+
createAgentWorkflowSession,
|
|
22
|
+
runAgentWorkflow,
|
|
23
|
+
createSimulatedWorkflowEnvironment,
|
|
24
|
+
createDockerWorkflowEnvironment,
|
|
25
|
+
createDockerWorkflowEnvironmentFactory,
|
|
26
|
+
WORKFLOW_SANDBOX_IMAGE,
|
|
27
|
+
WorkflowEnvironmentInitializationError,
|
|
21
28
|
} from '@artemiskit/core';
|
|
22
29
|
export type {
|
|
23
30
|
AgentWorkflow,
|
|
@@ -33,12 +40,25 @@ export type {
|
|
|
33
40
|
AgentTurnResult,
|
|
34
41
|
AgentTargetCapabilities,
|
|
35
42
|
AgentTargetFailure,
|
|
43
|
+
AgentWorkflowExecution,
|
|
44
|
+
AgentWorkflowReason,
|
|
45
|
+
AgentWorkflowEvent,
|
|
46
|
+
AgentWorkflowRecord,
|
|
47
|
+
AgentWorkflowResult,
|
|
48
|
+
AgentWorkflowSessionOptions,
|
|
49
|
+
AgentWorkflowSession,
|
|
50
|
+
WorkflowState,
|
|
51
|
+
WorkflowEnvironment,
|
|
52
|
+
WorkflowEnvironmentFactory,
|
|
53
|
+
WorkflowEnvironmentCleanup,
|
|
54
|
+
DockerWorkflowEnvironmentOptions,
|
|
36
55
|
} from '@artemiskit/core';
|
|
37
56
|
|
|
38
57
|
// Types
|
|
39
58
|
export type {
|
|
40
59
|
// Configuration
|
|
41
60
|
ArtemisKitConfig,
|
|
61
|
+
WorkflowRunOptions,
|
|
42
62
|
RunOptions,
|
|
43
63
|
RedTeamOptions,
|
|
44
64
|
StressOptions,
|
package/src/types-only.ts
CHANGED
|
@@ -23,6 +23,7 @@
|
|
|
23
23
|
export type {
|
|
24
24
|
// Configuration
|
|
25
25
|
ArtemisKitConfig,
|
|
26
|
+
WorkflowRunOptions,
|
|
26
27
|
RunOptions,
|
|
27
28
|
RedTeamOptions,
|
|
28
29
|
StressOptions,
|
|
@@ -274,3 +275,32 @@ export type {
|
|
|
274
275
|
// ============================================================================
|
|
275
276
|
|
|
276
277
|
export type { ScenarioBuilder, TestCaseBuilder } from './builders';
|
|
278
|
+
|
|
279
|
+
export type {
|
|
280
|
+
AgentWorkflow,
|
|
281
|
+
WorkflowPolicy,
|
|
282
|
+
WorkflowJson,
|
|
283
|
+
WorkflowToolId,
|
|
284
|
+
WorkflowResource,
|
|
285
|
+
WorkflowToolDescriptor,
|
|
286
|
+
SimulatedToolEvidence,
|
|
287
|
+
SimulatedToolResult,
|
|
288
|
+
AgentTarget,
|
|
289
|
+
AgentTurnRequest,
|
|
290
|
+
AgentTurnResult,
|
|
291
|
+
AgentTargetCapabilities,
|
|
292
|
+
AgentTargetFailure,
|
|
293
|
+
AgentWorkflowExecution,
|
|
294
|
+
AgentWorkflowReason,
|
|
295
|
+
AgentWorkflowEvent,
|
|
296
|
+
AgentWorkflowRecord,
|
|
297
|
+
AgentWorkflowResult,
|
|
298
|
+
AgentWorkflowSessionOptions,
|
|
299
|
+
AgentWorkflowSession,
|
|
300
|
+
WorkflowState,
|
|
301
|
+
WorkflowEnvironment,
|
|
302
|
+
WorkflowEnvironmentFactory,
|
|
303
|
+
WorkflowEnvironmentCleanup,
|
|
304
|
+
DockerWorkflowEnvironmentOptions,
|
|
305
|
+
WorkflowEnvironmentInitializationError,
|
|
306
|
+
} from '@artemiskit/core';
|
package/src/types.ts
CHANGED
|
@@ -5,6 +5,9 @@
|
|
|
5
5
|
|
|
6
6
|
import type {
|
|
7
7
|
AdapterConfig,
|
|
8
|
+
AgentTarget,
|
|
9
|
+
AgentWorkflow,
|
|
10
|
+
AgentWorkflowSessionOptions,
|
|
8
11
|
CaseResult,
|
|
9
12
|
ComparisonEligibility,
|
|
10
13
|
ModelClient,
|
|
@@ -448,3 +451,16 @@ export interface CompareResult {
|
|
|
448
451
|
/** Threshold used for regression detection */
|
|
449
452
|
threshold: number;
|
|
450
453
|
}
|
|
454
|
+
|
|
455
|
+
/** Controlled workflow execution. Text/state remain in memory unless the caller explicitly saves them. */
|
|
456
|
+
export interface WorkflowRunOptions
|
|
457
|
+
extends Omit<AgentWorkflowSessionOptions, 'workflow' | 'target'> {
|
|
458
|
+
/** Workflow object or YAML path. File fixtures default to the workflow file's directory. */
|
|
459
|
+
workflow: AgentWorkflow | string;
|
|
460
|
+
/** Explicit host-owned client. Its transport retry behavior remains the caller's responsibility. */
|
|
461
|
+
client?: ModelClient;
|
|
462
|
+
/** Explicit custom target, mutually exclusive with client. */
|
|
463
|
+
target?: AgentTarget;
|
|
464
|
+
/** Trusted transport configuration; workflow provider/model and zero retries remain authoritative. */
|
|
465
|
+
providerConfig?: Partial<AdapterConfig>;
|
|
466
|
+
}
|
package/src/workflow.ts
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { dirname, resolve } from 'node:path';
|
|
2
|
+
import {
|
|
3
|
+
type AdapterConfig,
|
|
4
|
+
type AgentWorkflowSession,
|
|
5
|
+
ArtemisError,
|
|
6
|
+
createAdapter,
|
|
7
|
+
createAgentWorkflowSession,
|
|
8
|
+
createModelClientTarget,
|
|
9
|
+
loadAgentWorkflow,
|
|
10
|
+
validateAgentWorkflow,
|
|
11
|
+
} from '@artemiskit/core';
|
|
12
|
+
import type { ArtemisKitConfig, WorkflowRunOptions } from './types';
|
|
13
|
+
|
|
14
|
+
export async function prepareWorkflowSession(
|
|
15
|
+
options: WorkflowRunOptions,
|
|
16
|
+
config: ArtemisKitConfig
|
|
17
|
+
): Promise<AgentWorkflowSession> {
|
|
18
|
+
if (options.client && options.target) throw new Error('Choose a workflow client or target');
|
|
19
|
+
const file = typeof options.workflow === 'string' ? resolve(options.workflow) : undefined;
|
|
20
|
+
const workflow = file ? await loadAgentWorkflow(file) : validateAgentWorkflow(options.workflow);
|
|
21
|
+
let target = options.target;
|
|
22
|
+
if (!target) {
|
|
23
|
+
let client = options.client;
|
|
24
|
+
if (!client) {
|
|
25
|
+
try {
|
|
26
|
+
client = await createAdapter({
|
|
27
|
+
...config.providerConfig,
|
|
28
|
+
...options.providerConfig,
|
|
29
|
+
provider: workflow.target.provider,
|
|
30
|
+
defaultModel: workflow.target.model,
|
|
31
|
+
maxRetries: 0,
|
|
32
|
+
} as AdapterConfig);
|
|
33
|
+
} catch {
|
|
34
|
+
// Provider factories may put credentials or configuration values in their diagnostics.
|
|
35
|
+
throw new ArtemisError('Unable to initialize workflow target', 'PROVIDER_UNAVAILABLE');
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
target = createModelClientTarget(client);
|
|
39
|
+
}
|
|
40
|
+
return createAgentWorkflowSession({
|
|
41
|
+
workflow,
|
|
42
|
+
target,
|
|
43
|
+
fixtureRoot: options.fixtureRoot ?? (file ? dirname(file) : undefined),
|
|
44
|
+
environmentFactory: options.environmentFactory,
|
|
45
|
+
preflight: options.preflight,
|
|
46
|
+
preflightOnly: options.preflightOnly,
|
|
47
|
+
signal: options.signal,
|
|
48
|
+
cleanupTimeoutMs: options.cleanupTimeoutMs,
|
|
49
|
+
onEvent: options.onEvent,
|
|
50
|
+
});
|
|
51
|
+
}
|