@artemiskit/sdk 0.6.1 → 0.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/README.md +26 -2
- package/dist/artemiskit.d.ts +6 -1
- package/dist/artemiskit.d.ts.map +1 -1
- package/dist/index.d.ts +4 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +44 -9
- package/dist/types-only.d.ts +2 -1
- package/dist/types-only.d.ts.map +1 -1
- package/dist/workflow.d.ts.map +1 -1
- package/package.json +4 -3
- package/src/__tests__/workflow-recovery.test.ts +116 -0
- package/src/__tests__/workflow-report.test.ts +109 -0
- package/src/artemiskit.ts +16 -1
- package/src/index.ts +27 -0
- package/src/types-only.ts +17 -0
- package/src/workflow.ts +27 -10
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,24 @@
|
|
|
1
1
|
# @artemiskit/sdk
|
|
2
2
|
|
|
3
|
+
## 0.6.3
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- 5c35b04: Add deterministic offline workflow assessments in technical, executive and comprehensive HTML/Markdown views. Strict saved V1/V2/V3 records share one canonical report model, evidence links, explicit measurement denominators and compatible resume accounting. Expose `akit workflow report` and matching synchronous SDK functions with bounded input and no-overwrite exports. No provider calls, inferred pricing or task rescoring occur. Core receives a release-alignment patch identifying the coherent 0.6.4 milestone; its workflow record contract remains unchanged.
|
|
8
|
+
- Updated dependencies [5c35b04]
|
|
9
|
+
- @artemiskit/reports@0.5.0
|
|
10
|
+
- @artemiskit/core@0.6.4
|
|
11
|
+
- @artemiskit/redteam@0.3.12
|
|
12
|
+
|
|
13
|
+
## 0.6.2
|
|
14
|
+
|
|
15
|
+
### Patch Changes
|
|
16
|
+
|
|
17
|
+
- Add declared workflow faults and bounded retries, private local checkpoints with explicit pause/resume, and version 3 recovery evidence. Preserve workflow identity, tool authority, original deadlines and cumulative budgets across restart; refuse ambiguous pending operations. Expose the same recovery controls in CLI and SDK while retaining version 1/2 record compatibility.
|
|
18
|
+
- Updated dependencies
|
|
19
|
+
- @artemiskit/core@0.6.3
|
|
20
|
+
- @artemiskit/redteam@0.3.11
|
|
21
|
+
|
|
3
22
|
## 0.6.1
|
|
4
23
|
|
|
5
24
|
### Patch Changes
|
package/README.md
CHANGED
|
@@ -62,16 +62,23 @@ console.log(result.record); // bounded metadata with separate execution and task
|
|
|
62
62
|
// result.state and result.transcript contain sensitive working data.
|
|
63
63
|
```
|
|
64
64
|
|
|
65
|
-
`kit.runWorkflow(options)` creates and runs a
|
|
65
|
+
`kit.runWorkflow(options)` creates and runs a session in one call; fresh execution is the default. Supply a `client` or
|
|
66
66
|
`target` to use an existing trusted adapter or target. The same native controls apply through the
|
|
67
67
|
CLI and SDK: declared tools and permissions, cumulative budgets, explicit preflight, cancellation,
|
|
68
68
|
and cleanup evidence. Workflows use fresh simulated state or a disposable Docker filesystem.
|
|
69
69
|
The 0.6.2 milestone independently verifies declared artifacts, state, schemas, traces and policy.
|
|
70
70
|
Use explicit `semanticJudge` configuration for declared qualitative criteria. Target and judge usage
|
|
71
|
-
remain separate; `readWorkflowRecord` validates V1/V2 saved metadata without making model calls.
|
|
71
|
+
remain separate; `readWorkflowRecord` validates V1/V2/V3 saved metadata without making model calls.
|
|
72
72
|
See [workflow outcomes](https://github.com/code-sensei/artemiskit/blob/main/docs/workflow-outcomes.md)
|
|
73
73
|
for usage, denominators, compatibility and publication status.
|
|
74
74
|
|
|
75
|
+
For explicit private recovery, pass `checkpoint: { directory, mode: 'create', configurationId }`
|
|
76
|
+
and optionally `pauseAfterActions`. A later process uses the same authorized configuration and
|
|
77
|
+
`mode: 'resume'`. `session.pause()` requests a safe boundary; `session.cancel()` still cancels.
|
|
78
|
+
Checkpoints retain sensitive conversation and state with cumulative budgets and the original
|
|
79
|
+
deadline. A paused result is not task success. See the [0.6.3 recovery guide](https://github.com/code-sensei/artemiskit/blob/main/docs/workflow-recovery.md)
|
|
80
|
+
for complete examples, fault schedules, refusal conditions and qualification status.
|
|
81
|
+
|
|
75
82
|
The root and `/types` entry points expose workflow result/session/event types. Low-level
|
|
76
83
|
`createAgentWorkflowSession` and `runAgentWorkflow` are available for explicit target composition.
|
|
77
84
|
See the [execution guide](https://github.com/code-sensei/artemiskit/blob/main/docs/workflow-execution.md)
|
|
@@ -240,6 +247,23 @@ const outputResult = await guardian.validateOutput(llmResponse);
|
|
|
240
247
|
| `guardian` | Block violations and log |
|
|
241
248
|
| `hybrid` | Block critical, log others |
|
|
242
249
|
|
|
250
|
+
## Offline workflow reports
|
|
251
|
+
|
|
252
|
+
```ts
|
|
253
|
+
import { ArtemisKit, createWorkflowReport, generateWorkflowReport } from '@artemiskit/sdk';
|
|
254
|
+
|
|
255
|
+
const assessment = createWorkflowReport(savedRecord); // object, JSON string, or record array
|
|
256
|
+
const html = generateWorkflowReport(savedRecord, { view: 'comprehensive', format: 'html' });
|
|
257
|
+
const markdown = new ArtemisKit().generateWorkflowReport(savedRecord, {
|
|
258
|
+
view: 'executive', format: 'markdown',
|
|
259
|
+
});
|
|
260
|
+
```
|
|
261
|
+
|
|
262
|
+
These synchronous functions share `@artemiskit/reports` and never initialize providers, restore
|
|
263
|
+
checkpoints or rescore tasks. `renderWorkflowReportHTML` and `renderWorkflowReportMarkdown` render
|
|
264
|
+
an existing canonical model. Invalid/unavailable measurements, historical evidence and cumulative
|
|
265
|
+
resumed attempts keep their original meaning. See the [guide](../../docs/workflow-reports.md).
|
|
266
|
+
|
|
243
267
|
## License
|
|
244
268
|
|
|
245
269
|
Apache-2.0
|
package/dist/artemiskit.d.ts
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
* @artemiskit/sdk
|
|
3
3
|
* Main ArtemisKit class - programmatic API for LLM testing
|
|
4
4
|
*/
|
|
5
|
+
import { type GenerateWorkflowReportOptions } from '@artemiskit/reports';
|
|
5
6
|
import type { ArtemisKitConfig, ArtemisKitEventName, ArtemisKitEvents, CaseCompleteHandler, CaseStartHandler, CompareOptions, CompareResult, ProgressHandler, RedTeamMutationCompleteHandler, RedTeamMutationStartHandler, RedTeamOptions, RedTeamResult, RunOptions, RunResult, StressOptions, StressRequestCompleteHandler, StressResult, ValidateOptions, ValidationResult, WorkflowRunOptions } from './types.js';
|
|
6
7
|
/**
|
|
7
8
|
* ArtemisKit SDK - programmatic API for LLM evaluation testing
|
|
@@ -27,8 +28,12 @@ export declare class ArtemisKit {
|
|
|
27
28
|
constructor(config?: ArtemisKitConfig);
|
|
28
29
|
/** Create a cancellable native session without executing model turns or persisting results. */
|
|
29
30
|
createWorkflowSession(options: WorkflowRunOptions): Promise<import("@artemiskit/core").AgentWorkflowSession>;
|
|
30
|
-
/** Execute a workflow
|
|
31
|
+
/** Execute a workflow and independently evaluate its declared outcomes. */
|
|
31
32
|
runWorkflow(options: WorkflowRunOptions): Promise<import("@artemiskit/core").AgentWorkflowResult>;
|
|
33
|
+
/** Project saved evidence without running a workflow or loading provider configuration. */
|
|
34
|
+
createWorkflowReport(input: unknown): import("@artemiskit/reports").WorkflowReport;
|
|
35
|
+
/** Generate HTML or Markdown offline; task failures remain visible in a successful export. */
|
|
36
|
+
generateWorkflowReport(input: unknown, options?: GenerateWorkflowReportOptions): string;
|
|
32
37
|
/**
|
|
33
38
|
* Register an event handler
|
|
34
39
|
*/
|
package/dist/artemiskit.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"artemiskit.d.ts","sourceRoot":"","sources":["../src/artemiskit.ts"],"names":[],"mappings":"AAAA;;;GAGG;
|
|
1
|
+
{"version":3,"file":"artemiskit.d.ts","sourceRoot":"","sources":["../src/artemiskit.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAuCH,OAAO,EACL,KAAK,6BAA6B,EAGnC,MAAM,qBAAqB,CAAC;AAI7B,OAAO,KAAK,EACV,gBAAgB,EAChB,mBAAmB,EACnB,gBAAgB,EAChB,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,aAAa,EACb,eAAe,EACf,8BAA8B,EAC9B,2BAA2B,EAC3B,cAAc,EACd,aAAa,EACb,UAAU,EACV,SAAS,EACT,aAAa,EACb,4BAA4B,EAC5B,YAAY,EACZ,eAAe,EACf,gBAAgB,EAChB,kBAAkB,EACnB,MAAM,SAAS,CAAC;AAsBjB;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAmB;IACjC,OAAO,CAAC,aAAa,CAA6D;IAClF,OAAO,CAAC,OAAO,CAA+B;gBAElC,MAAM,GAAE,gBAAqB;IAmBzC,+FAA+F;IACzF,qBAAqB,CAAC,OAAO,EAAE,kBAAkB;IAIvD,2EAA2E;IACrE,WAAW,CAAC,OAAO,EAAE,kBAAkB;IAI7C,2FAA2F;IAC3F,oBAAoB,CAAC,KAAK,EAAE,OAAO;IAInC,8FAA8F;IAC9F,sBAAsB,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,6BAA6B;IAQ9E;;OAEG;IACH,EAAE,CAAC,CAAC,SAAS,mBAAmB,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,IAAI;IAQhG;;OAEG;IACH,GAAG,CAAC,CAAC,SAAS,mBAAmB,EAC/B,KAAK,EAAE,CAAC,EACR,OAAO,EAAE,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC,KAAK,IAAI,GAC5C,IAAI;IAQP;;OAEG;IACH,IAAI,CAAC,CAAC,SAAS,mBAAmB,EAChC,KAAK,EAAE,CAAC,EACR,OAAO,EAAE,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC,KAAK,IAAI,GAC5C,IAAI;IAQP;;OAEG;IACH,OAAO,CAAC,IAAI;IAiBZ;;OAEG;IACH,WAAW,CAAC,OAAO,EAAE,gBAAgB,GAAG,IAAI;IAI5C;;OAEG;IACH,cAAc,CAAC,OAAO,EAAE,mBAAmB,GAAG,IAAI;IAIlD;;OAEG;IACH,UAAU,CAAC,OAAO,EAAE,eAAe,GAAG,IAAI;IAI1C;;OAEG;IACH,sBAAsB,CAAC,OAAO,EAAE,2BAA2B,GAAG,IAAI;IAIlE;;OAEG;IACH,yBAAyB,CAAC,OAAO,EAAE,8BAA8B,GAAG,IAAI;IAIxE;;OAEG;IACH,uBAAuB,CAAC,OAAO,EAAE,4BAA4B,GAAG,IAAI;IAQpE;;OAEG;IACG,GAAG,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC;IA+DlD;;OAEG;IACG,OAAO,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;IAgN9D;;OAEG;IACG,MAAM,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC;IAwO3D;;;;;;;;;;;;;OAaG;IACG,QAAQ,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA2LnE;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACG,OAAO,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;IA2H9D;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAmBzB;;OAEG;IACH,OAAO,CAAC,YAAY;IA4EpB;;OAEG;IACH,OAAO,CAAC,YAAY;IAepB;;OAEG;YACW,YAAY;IAO1B;;OAEG;YACW,YAAY;IAgB1B;;OAEG;IACH,OAAO,CAAC,cAAc;IA6BtB;;OAEG;IACH,qBAAqB,IAAI,MAAM,EAAE;CAGlC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
* Programmatic SDK for ArtemisKit LLM evaluation toolkit
|
|
4
4
|
*/
|
|
5
5
|
export { ArtemisKit } from './artemiskit.js';
|
|
6
|
-
export { AgentWorkflowSchema, WorkflowPolicySchema, WORKFLOW_TOOL_IDS, validateAgentWorkflow, parseAgentWorkflow, loadAgentWorkflow, listWorkflowTools, getWorkflowTool, executeSimulatedTool, createModelClientTarget, createAgentWorkflowSession, runAgentWorkflow, evaluateWorkflowDeterministicOutcomes, evaluateWorkflowSemantics, isValidWorkflowJudgeOptions, decideWorkflowOutcome, validWorkflowAssertionCounts, readWorkflowRecord, createSimulatedWorkflowEnvironment, createDockerWorkflowEnvironment, createDockerWorkflowEnvironmentFactory, WORKFLOW_SANDBOX_IMAGE, WorkflowEnvironmentInitializationError, } from '@artemiskit/core';
|
|
7
|
-
export type { AgentWorkflow, WorkflowPolicy, WorkflowJson, WorkflowToolId, WorkflowResource, WorkflowToolDescriptor, SimulatedToolEvidence, SimulatedToolResult, AgentTarget, AgentTurnRequest, AgentTurnResult, AgentTargetCapabilities, AgentTargetFailure, AgentWorkflowExecution, AgentWorkflowReason, AgentWorkflowEvent, AgentWorkflowRecord, AgentWorkflowExecutionRecord, AgentWorkflowEvidence, WorkflowDeterministicStatus, WorkflowDeterministicReason, WorkflowDeterministicAssertion, WorkflowDeterministicSummary, WorkflowJudgeOptions, WorkflowSemanticOptions, WorkflowSemanticReason, WorkflowSemanticAssertion, WorkflowSemanticIdentity, WorkflowSemanticEvaluation, WorkflowOutcomeStatus, WorkflowAssertionCounts, WorkflowOutcomeDecision, WorkflowOutcomeAssessment, SavedWorkflowRecord, HistoricalWorkflowRecord, AgentWorkflowResult, AgentWorkflowSessionOptions, AgentWorkflowSession, WorkflowState, WorkflowEnvironment, WorkflowEnvironmentFactory, WorkflowEnvironmentCleanup, DockerWorkflowEnvironmentOptions, } from '@artemiskit/core';
|
|
6
|
+
export { AgentWorkflowSchema, WorkflowPolicySchema, WorkflowFaultsSchema, WorkflowRetrySchema, WORKFLOW_TOOL_IDS, validateAgentWorkflow, parseAgentWorkflow, loadAgentWorkflow, listWorkflowTools, getWorkflowTool, executeSimulatedTool, createModelClientTarget, createAgentWorkflowSession, runAgentWorkflow, evaluateWorkflowDeterministicOutcomes, evaluateWorkflowSemantics, isValidWorkflowJudgeOptions, decideWorkflowOutcome, validWorkflowAssertionCounts, readWorkflowRecord, createSimulatedWorkflowEnvironment, createDockerWorkflowEnvironment, createDockerWorkflowEnvironmentFactory, WORKFLOW_SANDBOX_IMAGE, WorkflowEnvironmentInitializationError, } from '@artemiskit/core';
|
|
7
|
+
export type { AgentWorkflow, WorkflowPolicy, WorkflowJson, WorkflowToolId, WorkflowResource, WorkflowToolDescriptor, SimulatedToolEvidence, SimulatedToolResult, AgentTarget, AgentTurnRequest, AgentTurnResult, AgentTargetCapabilities, AgentTargetFailure, AgentWorkflowExecution, AgentWorkflowReason, AgentWorkflowEvent, AgentWorkflowRecord, AgentWorkflowExecutionRecord, AgentWorkflowEvidence, WorkflowDeterministicStatus, WorkflowDeterministicReason, WorkflowDeterministicAssertion, WorkflowDeterministicSummary, WorkflowJudgeOptions, WorkflowSemanticOptions, WorkflowSemanticReason, WorkflowSemanticAssertion, WorkflowSemanticIdentity, WorkflowSemanticEvaluation, WorkflowOutcomeStatus, WorkflowAssertionCounts, WorkflowOutcomeDecision, WorkflowOutcomeAssessment, SavedWorkflowRecord, HistoricalWorkflowRecord, AgentWorkflowResult, AgentWorkflowSessionOptions, WorkflowCheckpointOptions, WorkflowRecoveryEvidence, WorkflowFault, WorkflowFaults, WorkflowFaultKind, WorkflowRetry, AgentWorkflowSession, WorkflowState, WorkflowEnvironment, WorkflowEnvironmentFactory, WorkflowEnvironmentCleanup, DockerWorkflowEnvironmentOptions, } from '@artemiskit/core';
|
|
8
8
|
export type { ArtemisKitConfig, WorkflowRunOptions, RunOptions, RedTeamOptions, StressOptions, RunResult, RedTeamResult, StressResult, CaseStartEvent, CaseCompleteEvent, ProgressEvent, RedTeamMutationStartEvent, RedTeamMutationCompleteEvent, StressRequestCompleteEvent, CaseStartHandler, CaseCompleteHandler, ProgressHandler, RedTeamMutationStartHandler, RedTeamMutationCompleteHandler, StressRequestCompleteHandler, ArtemisKitEvents, ArtemisKitEventName, ValidateOptions, ValidationResult, ValidationError, ValidationWarning, ScenarioValidation, CompareOptions, CompareResult, RunSummary, ComparisonDetails, } from './types.js';
|
|
9
9
|
export type { Scenario, TestCase, Expected, Provider, ProviderConfig, ChatMessageType, Variables, } from '@artemiskit/core';
|
|
10
10
|
export type { CaseResult, CaseRedactionInfo, RunManifest, RunConfig, RunMetrics, RedTeamManifest, RedTeamConfig, RedTeamMetrics, RedTeamCaseResult, RedTeamStatus, RedTeamSeverity, StressManifest, StressConfig, StressMetrics, StressRequestResult, AnyManifest, ResolvedConfig, ConfigSource, ManifestRedactionInfo, CostEstimateInfo, GitInfo, ProvenanceInfo, } from '@artemiskit/core';
|
|
@@ -22,4 +22,6 @@ export type { AdapterContract, EvaluatorContract, StorageContract, AdapterFactor
|
|
|
22
22
|
export { ScenarioBuilder, TestCaseBuilder, scenario, testCase, containsCase, exactCase, regexCase, jsonCase, gradedCase, exact, contains, notContains, regex, fuzzy, jsonSchema, llmGrade, similarity, inline, allOf, anyOf, } from './builders/index.js';
|
|
23
23
|
export { Guardian, createGuardian, normalizeGuardianMode, SemanticValidator, createSemanticValidator, GuardianInterceptor, GuardianBlockedError, createInterceptor, ActionValidator, createDefaultActionValidator, IntentClassifier, createIntentClassifier, detectInjection, createInjectionGuardrail, detectPII, createPIIGuardrail, filterContent, createContentFilterGuardrail, createGuardrails, matchPattern, createCustomPatternGuardrail, loadPolicy, parsePolicy, validatePolicy, createDefaultPolicy, mergePolicies, generatePolicyTemplate, PolicyLoadError, PolicyValidationError, CircuitBreaker, MetricsCollector, RateLimiter, } from './guardian/index.js';
|
|
24
24
|
export type { GuardianConfig, InterceptorConfig, ActionValidatorConfig, IntentClassifierConfig, GuardrailsConfig, RateLimiterConfig, CustomPatternOptions, GuardianMode, ViolationSeverity, ViolationAction, GuardrailType, Violation, GuardrailResult, PolicyRule, PolicyCondition, GuardianPolicy, CircuitBreakerConfig, RateLimitConfig, CostLimitConfig, ActionDefinition, ActionParameter, ParameterValidation, IntentClassification, IntentCategory, PIIDetection, PIIType, PIILocation, InjectionDetection, InjectionType, ContentFilterResult, ContentFlag, ContentCategory, GuardianMetrics, CircuitBreakerState, CostTracking, GuardianEventType, GuardianEvent, GuardianEventHandler, InterceptedRequest, InterceptedResponse, InterceptedToolCall, InterceptedAgentStep, FrameworkType, FrameworkIntegrationConfig, GuardianModeCanonical, GuardianModeLegacy, GuardianModeAll, ContentValidationConfig, PatternConfig, ValidationCategory, PatternCategory, SemanticValidationResult, SessionStorageType, SessionStorageConfig, SessionMessage, SessionMetrics, ConversationSession, TrustBuildingHeuristicConfig, EscalationHeuristicConfig, ContextManipulationHeuristicConfig, SplitPayloadHeuristicConfig, MultiTurnHeuristics, MultiTurnSemanticConfig, MultiTurnConfig, HeuristicResult, HeuristicResults, MultiTurnValidationResult, ValidateMessageOptions, } from './guardian/index.js';
|
|
25
|
+
export { createWorkflowReport, generateWorkflowReport, renderWorkflowReportHTML, renderWorkflowReportMarkdown, } from '@artemiskit/reports';
|
|
26
|
+
export type { WorkflowReport, WorkflowReportView, WorkflowReportFormat, WorkflowReportOptions, GenerateWorkflowReportOptions, WorkflowReportFinding, WorkflowReportEvidence, WorkflowReportRow, WorkflowReportSection, } from '@artemiskit/reports';
|
|
25
27
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAG1C,OAAO,EACL,mBAAmB,EACnB,oBAAoB,EACpB,iBAAiB,EACjB,qBAAqB,EACrB,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,eAAe,EACf,oBAAoB,EACpB,uBAAuB,EACvB,0BAA0B,EAC1B,gBAAgB,EAChB,qCAAqC,EACrC,yBAAyB,EACzB,2BAA2B,EAC3B,qBAAqB,EACrB,4BAA4B,EAC5B,kBAAkB,EAClB,kCAAkC,EAClC,+BAA+B,EAC/B,sCAAsC,EACtC,sBAAsB,EACtB,sCAAsC,GACvC,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,aAAa,EACb,cAAc,EACd,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,sBAAsB,EACtB,qBAAqB,EACrB,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,eAAe,EACf,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,kBAAkB,EAClB,mBAAmB,EACnB,4BAA4B,EAC5B,qBAAqB,EACrB,2BAA2B,EAC3B,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAC5B,oBAAoB,EACpB,uBAAuB,EACvB,sBAAsB,EACtB,yBAAyB,EACzB,wBAAwB,EACxB,0BAA0B,EAC1B,qBAAqB,EACrB,uBAAuB,EACvB,uBAAuB,EACvB,yBAAyB,EACzB,mBAAmB,EACnB,wBAAwB,EACxB,mBAAmB,EACnB,2BAA2B,EAC3B,oBAAoB,EACpB,aAAa,EACb,mBAAmB,EACnB,0BAA0B,EAC1B,0BAA0B,EAC1B,gCAAgC,GACjC,MAAM,kBAAkB,CAAC;AAG1B,YAAY,EAEV,gBAAgB,EAChB,kBAAkB,EAClB,UAAU,EACV,cAAc,EACd,aAAa,EAEb,SAAS,EACT,aAAa,EACb,YAAY,EAEZ,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,yBAAyB,EACzB,4BAA4B,EAC5B,0BAA0B,EAE1B,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAE5B,gBAAgB,EAChB,mBAAmB,EAEnB,eAAe,EACf,gBAAgB,EAChB,eAAe,EACf,iBAAiB,EACjB,kBAAkB,EAElB,cAAc,EACd,aAAa,EACb,UAAU,EACV,iBAAiB,GAClB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,cAAc,EACd,eAAe,EACf,SAAS,GACV,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,UAAU,EACV,iBAAiB,EAEjB,WAAW,EACX,SAAS,EACT,UAAU,EAEV,eAAe,EACf,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,eAAe,EAEf,cAAc,EACd,YAAY,EACZ,aAAa,EACb,mBAAmB,EAEnB,WAAW,EACX,cAAc,EACd,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,OAAO,EACP,cAAc,GACf,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,WAAW,EACX,iBAAiB,EAEjB,eAAe,EACf,cAAc,EACd,UAAU,EAEV,WAAW,EAEX,kBAAkB,EAClB,cAAc,EACd,QAAQ,EAER,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,YAAY,GACb,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMrF,YAAY,EACV,cAAc,EACd,aAAa,EACb,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EAEtB,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EACtB,eAAe,EACf,mBAAmB,EACnB,cAAc,EACd,uBAAuB,GACxB,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMxD,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAMtF,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAM7E,OAAO,EAAE,kBAAkB,EAAE,KAAK,kBAAkB,EAAE,KAAK,aAAa,EAAE,MAAM,YAAY,CAAC;AAM7F,OAAO,EAEL,WAAW,EACX,eAAe,EACf,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,oBAAoB,EAEpB,aAAa,EACb,MAAM,EAEN,cAAc,EACd,cAAc,EACd,aAAa,EACb,oBAAoB,GACrB,MAAM,SAAS,CAAC;AAEjB,YAAY,EAEV,YAAY,EACZ,eAAe,EAEf,SAAS,EACT,eAAe,EACf,mBAAmB,EACnB,oBAAoB,EACpB,eAAe,EAEf,WAAW,EACX,eAAe,EACf,aAAa,EACb,cAAc,EACd,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,OAAO,EAEL,aAAa,EAEb,eAAe,EAEf,aAAa,EAEb,YAAY,GACb,MAAM,aAAa,CAAC;AAErB,YAAY,EACV,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,aAAa,CAAC;AAMrB,OAAO,EAEL,eAAe,EACf,eAAe,EAEf,QAAQ,EACR,QAAQ,EAER,YAAY,EACZ,SAAS,EACT,SAAS,EACT,QAAQ,EACR,UAAU,EAEV,KAAK,EACL,QAAQ,EACR,WAAW,EACX,KAAK,EACL,KAAK,EACL,UAAU,EACV,QAAQ,EACR,UAAU,EACV,MAAM,EACN,KAAK,EACL,KAAK,GACN,MAAM,YAAY,CAAC;AAMpB,OAAO,EAEL,QAAQ,EACR,cAAc,EAEd,qBAAqB,EAErB,iBAAiB,EACjB,uBAAuB,EAEvB,mBAAmB,EACnB,oBAAoB,EACpB,iBAAiB,EAEjB,eAAe,EACf,4BAA4B,EAE5B,gBAAgB,EAChB,sBAAsB,EAEtB,eAAe,EACf,wBAAwB,EACxB,SAAS,EACT,kBAAkB,EAClB,aAAa,EACb,4BAA4B,EAC5B,gBAAgB,EAEhB,YAAY,EACZ,4BAA4B,EAE5B,UAAU,EACV,WAAW,EACX,cAAc,EACd,mBAAmB,EACnB,aAAa,EACb,sBAAsB,EACtB,eAAe,EACf,qBAAqB,EAErB,cAAc,EACd,gBAAgB,EAChB,WAAW,GACZ,MAAM,YAAY,CAAC;AAGpB,YAAY,EAEV,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,sBAAsB,EACtB,gBAAgB,EAChB,iBAAiB,EACjB,oBAAoB,EAEpB,YAAY,EACZ,iBAAiB,EACjB,eAAe,EACf,aAAa,EACb,SAAS,EACT,eAAe,EAEf,UAAU,EACV,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,eAAe,EACf,eAAe,EAEf,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EAEnB,oBAAoB,EACpB,cAAc,EAEd,YAAY,EACZ,OAAO,EACP,WAAW,EACX,kBAAkB,EAClB,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,eAAe,EAEf,eAAe,EACf,mBAAmB,EACnB,YAAY,EAEZ,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EAEpB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EAEpB,aAAa,EACb,0BAA0B,EAE1B,qBAAqB,EACrB,kBAAkB,EAClB,eAAe,EAEf,uBAAuB,EACvB,aAAa,EACb,kBAAkB,EAClB,eAAe,EACf,wBAAwB,EAExB,kBAAkB,EAClB,oBAAoB,EACpB,cAAc,EACd,cAAc,EACd,mBAAmB,EACnB,4BAA4B,EAC5B,yBAAyB,EACzB,kCAAkC,EAClC,2BAA2B,EAC3B,mBAAmB,EACnB,uBAAuB,EACvB,eAAe,EACf,eAAe,EACf,gBAAgB,EAChB,yBAAyB,EACzB,sBAAsB,GACvB,MAAM,YAAY,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAG1C,OAAO,EACL,mBAAmB,EACnB,oBAAoB,EACpB,oBAAoB,EACpB,mBAAmB,EACnB,iBAAiB,EACjB,qBAAqB,EACrB,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,eAAe,EACf,oBAAoB,EACpB,uBAAuB,EACvB,0BAA0B,EAC1B,gBAAgB,EAChB,qCAAqC,EACrC,yBAAyB,EACzB,2BAA2B,EAC3B,qBAAqB,EACrB,4BAA4B,EAC5B,kBAAkB,EAClB,kCAAkC,EAClC,+BAA+B,EAC/B,sCAAsC,EACtC,sBAAsB,EACtB,sCAAsC,GACvC,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,aAAa,EACb,cAAc,EACd,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,sBAAsB,EACtB,qBAAqB,EACrB,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,eAAe,EACf,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,kBAAkB,EAClB,mBAAmB,EACnB,4BAA4B,EAC5B,qBAAqB,EACrB,2BAA2B,EAC3B,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAC5B,oBAAoB,EACpB,uBAAuB,EACvB,sBAAsB,EACtB,yBAAyB,EACzB,wBAAwB,EACxB,0BAA0B,EAC1B,qBAAqB,EACrB,uBAAuB,EACvB,uBAAuB,EACvB,yBAAyB,EACzB,mBAAmB,EACnB,wBAAwB,EACxB,mBAAmB,EACnB,2BAA2B,EAC3B,yBAAyB,EACzB,wBAAwB,EACxB,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EACpB,aAAa,EACb,mBAAmB,EACnB,0BAA0B,EAC1B,0BAA0B,EAC1B,gCAAgC,GACjC,MAAM,kBAAkB,CAAC;AAG1B,YAAY,EAEV,gBAAgB,EAChB,kBAAkB,EAClB,UAAU,EACV,cAAc,EACd,aAAa,EAEb,SAAS,EACT,aAAa,EACb,YAAY,EAEZ,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,yBAAyB,EACzB,4BAA4B,EAC5B,0BAA0B,EAE1B,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAE5B,gBAAgB,EAChB,mBAAmB,EAEnB,eAAe,EACf,gBAAgB,EAChB,eAAe,EACf,iBAAiB,EACjB,kBAAkB,EAElB,cAAc,EACd,aAAa,EACb,UAAU,EACV,iBAAiB,GAClB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,cAAc,EACd,eAAe,EACf,SAAS,GACV,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,UAAU,EACV,iBAAiB,EAEjB,WAAW,EACX,SAAS,EACT,UAAU,EAEV,eAAe,EACf,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,eAAe,EAEf,cAAc,EACd,YAAY,EACZ,aAAa,EACb,mBAAmB,EAEnB,WAAW,EACX,cAAc,EACd,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,OAAO,EACP,cAAc,GACf,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,WAAW,EACX,iBAAiB,EAEjB,eAAe,EACf,cAAc,EACd,UAAU,EAEV,WAAW,EAEX,kBAAkB,EAClB,cAAc,EACd,QAAQ,EAER,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,YAAY,GACb,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMrF,YAAY,EACV,cAAc,EACd,aAAa,EACb,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EAEtB,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EACtB,eAAe,EACf,mBAAmB,EACnB,cAAc,EACd,uBAAuB,GACxB,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMxD,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAMtF,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAM7E,OAAO,EAAE,kBAAkB,EAAE,KAAK,kBAAkB,EAAE,KAAK,aAAa,EAAE,MAAM,YAAY,CAAC;AAM7F,OAAO,EAEL,WAAW,EACX,eAAe,EACf,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,oBAAoB,EAEpB,aAAa,EACb,MAAM,EAEN,cAAc,EACd,cAAc,EACd,aAAa,EACb,oBAAoB,GACrB,MAAM,SAAS,CAAC;AAEjB,YAAY,EAEV,YAAY,EACZ,eAAe,EAEf,SAAS,EACT,eAAe,EACf,mBAAmB,EACnB,oBAAoB,EACpB,eAAe,EAEf,WAAW,EACX,eAAe,EACf,aAAa,EACb,cAAc,EACd,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,OAAO,EAEL,aAAa,EAEb,eAAe,EAEf,aAAa,EAEb,YAAY,GACb,MAAM,aAAa,CAAC;AAErB,YAAY,EACV,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,aAAa,CAAC;AAMrB,OAAO,EAEL,eAAe,EACf,eAAe,EAEf,QAAQ,EACR,QAAQ,EAER,YAAY,EACZ,SAAS,EACT,SAAS,EACT,QAAQ,EACR,UAAU,EAEV,KAAK,EACL,QAAQ,EACR,WAAW,EACX,KAAK,EACL,KAAK,EACL,UAAU,EACV,QAAQ,EACR,UAAU,EACV,MAAM,EACN,KAAK,EACL,KAAK,GACN,MAAM,YAAY,CAAC;AAMpB,OAAO,EAEL,QAAQ,EACR,cAAc,EAEd,qBAAqB,EAErB,iBAAiB,EACjB,uBAAuB,EAEvB,mBAAmB,EACnB,oBAAoB,EACpB,iBAAiB,EAEjB,eAAe,EACf,4BAA4B,EAE5B,gBAAgB,EAChB,sBAAsB,EAEtB,eAAe,EACf,wBAAwB,EACxB,SAAS,EACT,kBAAkB,EAClB,aAAa,EACb,4BAA4B,EAC5B,gBAAgB,EAEhB,YAAY,EACZ,4BAA4B,EAE5B,UAAU,EACV,WAAW,EACX,cAAc,EACd,mBAAmB,EACnB,aAAa,EACb,sBAAsB,EACtB,eAAe,EACf,qBAAqB,EAErB,cAAc,EACd,gBAAgB,EAChB,WAAW,GACZ,MAAM,YAAY,CAAC;AAGpB,YAAY,EAEV,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,sBAAsB,EACtB,gBAAgB,EAChB,iBAAiB,EACjB,oBAAoB,EAEpB,YAAY,EACZ,iBAAiB,EACjB,eAAe,EACf,aAAa,EACb,SAAS,EACT,eAAe,EAEf,UAAU,EACV,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,eAAe,EACf,eAAe,EAEf,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EAEnB,oBAAoB,EACpB,cAAc,EAEd,YAAY,EACZ,OAAO,EACP,WAAW,EACX,kBAAkB,EAClB,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,eAAe,EAEf,eAAe,EACf,mBAAmB,EACnB,YAAY,EAEZ,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EAEpB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EAEpB,aAAa,EACb,0BAA0B,EAE1B,qBAAqB,EACrB,kBAAkB,EAClB,eAAe,EAEf,uBAAuB,EACvB,aAAa,EACb,kBAAkB,EAClB,eAAe,EACf,wBAAwB,EAExB,kBAAkB,EAClB,oBAAoB,EACpB,cAAc,EACd,cAAc,EACd,mBAAmB,EACnB,4BAA4B,EAC5B,yBAAyB,EACzB,kCAAkC,EAClC,2BAA2B,EAC3B,mBAAmB,EACnB,uBAAuB,EACvB,eAAe,EACf,eAAe,EACf,gBAAgB,EAChB,yBAAyB,EACzB,sBAAsB,GACvB,MAAM,YAAY,CAAC;AAGpB,OAAO,EACL,oBAAoB,EACpB,sBAAsB,EACtB,wBAAwB,EACxB,4BAA4B,GAC7B,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EACV,cAAc,EACd,kBAAkB,EAClB,oBAAoB,EACpB,qBAAqB,EACrB,6BAA6B,EAC7B,qBAAqB,EACrB,sBAAsB,EACtB,iBAAiB,EACjB,qBAAqB,GACtB,MAAM,qBAAqB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -7605,6 +7605,10 @@ import {
|
|
|
7605
7605
|
TypoMutation,
|
|
7606
7606
|
UnsafeResponseDetector
|
|
7607
7607
|
} from "@artemiskit/redteam";
|
|
7608
|
+
import {
|
|
7609
|
+
createWorkflowReport,
|
|
7610
|
+
generateWorkflowReport
|
|
7611
|
+
} from "@artemiskit/reports";
|
|
7608
7612
|
|
|
7609
7613
|
// ../../node_modules/nanoid/index.js
|
|
7610
7614
|
import { webcrypto as crypto } from "node:crypto";
|
|
@@ -7653,21 +7657,29 @@ async function prepareWorkflowSession(options, config) {
|
|
|
7653
7657
|
const workflow = file ? await loadAgentWorkflow(file) : validateAgentWorkflow(options.workflow);
|
|
7654
7658
|
let target = options.target;
|
|
7655
7659
|
if (!target) {
|
|
7656
|
-
|
|
7657
|
-
|
|
7658
|
-
|
|
7659
|
-
|
|
7660
|
+
if (options.client)
|
|
7661
|
+
target = createModelClientTarget(options.client);
|
|
7662
|
+
else {
|
|
7663
|
+
let initialized;
|
|
7664
|
+
const getTarget = () => {
|
|
7665
|
+
initialized ??= createAdapter({
|
|
7660
7666
|
...config.providerConfig,
|
|
7661
7667
|
...options.providerConfig,
|
|
7662
7668
|
provider: workflow.target.provider,
|
|
7663
7669
|
defaultModel: workflow.target.model,
|
|
7664
7670
|
maxRetries: 0
|
|
7671
|
+
}).then(createModelClientTarget).catch(() => {
|
|
7672
|
+
throw new ArtemisError("Unable to initialize workflow target", "PROVIDER_UNAVAILABLE");
|
|
7665
7673
|
});
|
|
7666
|
-
|
|
7667
|
-
|
|
7668
|
-
|
|
7674
|
+
return initialized;
|
|
7675
|
+
};
|
|
7676
|
+
target = options.checkpoint ? {
|
|
7677
|
+
provider: workflow.target.provider,
|
|
7678
|
+
capabilities: async (limits, signal) => (await getTarget()).capabilities(limits, signal),
|
|
7679
|
+
turn: async (request, signal) => (await getTarget()).turn(request, signal),
|
|
7680
|
+
drain: async (limits) => initialized ? (await initialized).drain?.(limits) ?? { pendingOperations: 0 } : { pendingOperations: 0 }
|
|
7681
|
+
} : await getTarget();
|
|
7669
7682
|
}
|
|
7670
|
-
target = createModelClientTarget(client);
|
|
7671
7683
|
}
|
|
7672
7684
|
return createAgentWorkflowSession({
|
|
7673
7685
|
workflow,
|
|
@@ -7679,7 +7691,9 @@ async function prepareWorkflowSession(options, config) {
|
|
|
7679
7691
|
signal: options.signal,
|
|
7680
7692
|
cleanupTimeoutMs: options.cleanupTimeoutMs,
|
|
7681
7693
|
semanticJudge: options.semanticJudge,
|
|
7682
|
-
onEvent: options.onEvent
|
|
7694
|
+
onEvent: options.onEvent,
|
|
7695
|
+
checkpoint: options.checkpoint,
|
|
7696
|
+
pauseAfterActions: options.pauseAfterActions
|
|
7683
7697
|
});
|
|
7684
7698
|
}
|
|
7685
7699
|
|
|
@@ -7723,6 +7737,12 @@ class ArtemisKit {
|
|
|
7723
7737
|
async runWorkflow(options) {
|
|
7724
7738
|
return (await this.createWorkflowSession(options)).run();
|
|
7725
7739
|
}
|
|
7740
|
+
createWorkflowReport(input) {
|
|
7741
|
+
return createWorkflowReport(input);
|
|
7742
|
+
}
|
|
7743
|
+
generateWorkflowReport(input, options) {
|
|
7744
|
+
return generateWorkflowReport(input, options);
|
|
7745
|
+
}
|
|
7726
7746
|
on(event, handler) {
|
|
7727
7747
|
if (!this.eventHandlers.has(event)) {
|
|
7728
7748
|
this.eventHandlers.set(event, new Set);
|
|
@@ -8507,6 +8527,8 @@ function percentile(sortedArray, p) {
|
|
|
8507
8527
|
import {
|
|
8508
8528
|
AgentWorkflowSchema,
|
|
8509
8529
|
WorkflowPolicySchema,
|
|
8530
|
+
WorkflowFaultsSchema,
|
|
8531
|
+
WorkflowRetrySchema,
|
|
8510
8532
|
WORKFLOW_TOOL_IDS,
|
|
8511
8533
|
validateAgentWorkflow as validateAgentWorkflow2,
|
|
8512
8534
|
parseAgentWorkflow,
|
|
@@ -15261,6 +15283,13 @@ class Guardian {
|
|
|
15261
15283
|
function createGuardian(config = {}) {
|
|
15262
15284
|
return new Guardian(config);
|
|
15263
15285
|
}
|
|
15286
|
+
// src/index.ts
|
|
15287
|
+
import {
|
|
15288
|
+
createWorkflowReport as createWorkflowReport2,
|
|
15289
|
+
generateWorkflowReport as generateWorkflowReport2,
|
|
15290
|
+
renderWorkflowReportHTML,
|
|
15291
|
+
renderWorkflowReportMarkdown
|
|
15292
|
+
} from "@artemiskit/reports";
|
|
15264
15293
|
export {
|
|
15265
15294
|
validatePolicy,
|
|
15266
15295
|
validateAgentWorkflow2 as validateAgentWorkflow,
|
|
@@ -15269,6 +15298,8 @@ export {
|
|
|
15269
15298
|
similarity,
|
|
15270
15299
|
scenario,
|
|
15271
15300
|
runAgentWorkflow,
|
|
15301
|
+
renderWorkflowReportMarkdown,
|
|
15302
|
+
renderWorkflowReportHTML,
|
|
15272
15303
|
regexCase,
|
|
15273
15304
|
regex,
|
|
15274
15305
|
readWorkflowRecord,
|
|
@@ -15300,6 +15331,7 @@ export {
|
|
|
15300
15331
|
getPassedCases,
|
|
15301
15332
|
getFailedCases,
|
|
15302
15333
|
getCasesByTag,
|
|
15334
|
+
generateWorkflowReport2 as generateWorkflowReport,
|
|
15303
15335
|
generatePolicyTemplate,
|
|
15304
15336
|
fuzzy,
|
|
15305
15337
|
filterContent,
|
|
@@ -15315,6 +15347,7 @@ export {
|
|
|
15315
15347
|
defineEvaluator,
|
|
15316
15348
|
defineAdapter,
|
|
15317
15349
|
decideWorkflowOutcome,
|
|
15350
|
+
createWorkflowReport2 as createWorkflowReport,
|
|
15318
15351
|
createSimulatedWorkflowEnvironment,
|
|
15319
15352
|
createSemanticValidator,
|
|
15320
15353
|
createPIIGuardrail,
|
|
@@ -15339,7 +15372,9 @@ export {
|
|
|
15339
15372
|
artemiskitMatchers,
|
|
15340
15373
|
anyOf,
|
|
15341
15374
|
allOf,
|
|
15375
|
+
WorkflowRetrySchema,
|
|
15342
15376
|
WorkflowPolicySchema,
|
|
15377
|
+
WorkflowFaultsSchema,
|
|
15343
15378
|
WorkflowEnvironmentInitializationError,
|
|
15344
15379
|
WORKFLOW_TOOL_IDS,
|
|
15345
15380
|
WORKFLOW_SANDBOX_IMAGE,
|
package/dist/types-only.d.ts
CHANGED
|
@@ -28,5 +28,6 @@ export type { ArtemisKitMatchers, MatcherResult } from './matchers/index.js';
|
|
|
28
28
|
export type { ProviderName, ExpectationType, AnyResult, ExtractRunCases, ExtractRedTeamCases, ExtractStressResults, ExtractManifest, DeepPartial, PartialScenario, RequireFields, OptionalFields, StrictAdapterConfig, } from './utils/index.js';
|
|
29
29
|
export type { AdapterContract, EvaluatorContract, StorageContract, AdapterFactory, EvaluatorFactory, StorageFactory, ArtemisKitPlugin, } from './contracts/index.js';
|
|
30
30
|
export type { ScenarioBuilder, TestCaseBuilder } from './builders/index.js';
|
|
31
|
-
export type { AgentWorkflow, WorkflowPolicy, WorkflowJson, WorkflowToolId, WorkflowResource, WorkflowToolDescriptor, SimulatedToolEvidence, SimulatedToolResult, AgentTarget, AgentTurnRequest, AgentTurnResult, AgentTargetCapabilities, AgentTargetFailure, AgentWorkflowExecution, AgentWorkflowReason, AgentWorkflowEvent, AgentWorkflowRecord, AgentWorkflowExecutionRecord, AgentWorkflowEvidence, WorkflowDeterministicStatus, WorkflowDeterministicReason, WorkflowDeterministicAssertion, WorkflowDeterministicSummary, WorkflowJudgeOptions, WorkflowSemanticOptions, WorkflowSemanticReason, WorkflowSemanticAssertion, WorkflowSemanticIdentity, WorkflowSemanticEvaluation, WorkflowOutcomeStatus, WorkflowAssertionCounts, WorkflowOutcomeDecision, WorkflowOutcomeAssessment, SavedWorkflowRecord, HistoricalWorkflowRecord, AgentWorkflowResult, AgentWorkflowSessionOptions, AgentWorkflowSession, WorkflowState, WorkflowEnvironment, WorkflowEnvironmentFactory, WorkflowEnvironmentCleanup, DockerWorkflowEnvironmentOptions, WorkflowEnvironmentInitializationError, } from '@artemiskit/core';
|
|
31
|
+
export type { AgentWorkflow, WorkflowPolicy, WorkflowJson, WorkflowToolId, WorkflowResource, WorkflowToolDescriptor, SimulatedToolEvidence, SimulatedToolResult, AgentTarget, AgentTurnRequest, AgentTurnResult, AgentTargetCapabilities, AgentTargetFailure, AgentWorkflowExecution, AgentWorkflowReason, AgentWorkflowEvent, AgentWorkflowRecord, AgentWorkflowExecutionRecord, AgentWorkflowEvidence, WorkflowDeterministicStatus, WorkflowDeterministicReason, WorkflowDeterministicAssertion, WorkflowDeterministicSummary, WorkflowJudgeOptions, WorkflowSemanticOptions, WorkflowSemanticReason, WorkflowSemanticAssertion, WorkflowSemanticIdentity, WorkflowSemanticEvaluation, WorkflowOutcomeStatus, WorkflowAssertionCounts, WorkflowOutcomeDecision, WorkflowOutcomeAssessment, SavedWorkflowRecord, HistoricalWorkflowRecord, AgentWorkflowResult, AgentWorkflowSessionOptions, WorkflowCheckpointOptions, WorkflowRecoveryEvidence, WorkflowFault, WorkflowFaults, WorkflowFaultKind, WorkflowRetry, AgentWorkflowSession, WorkflowState, WorkflowEnvironment, WorkflowEnvironmentFactory, WorkflowEnvironmentCleanup, DockerWorkflowEnvironmentOptions, WorkflowEnvironmentInitializationError, } from '@artemiskit/core';
|
|
32
|
+
export type { WorkflowReport, WorkflowReportView, WorkflowReportFormat, WorkflowReportOptions, GenerateWorkflowReportOptions, WorkflowReportFinding, WorkflowReportEvidence, WorkflowReportRow, WorkflowReportSection, } from '@artemiskit/reports';
|
|
32
33
|
//# sourceMappingURL=types-only.d.ts.map
|
package/dist/types-only.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types-only.d.ts","sourceRoot":"","sources":["../src/types-only.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAMH,YAAY,EAEV,gBAAgB,EAChB,kBAAkB,EAClB,UAAU,EACV,cAAc,EACd,aAAa,EAEb,SAAS,EACT,aAAa,EACb,YAAY,EAEZ,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,yBAAyB,EACzB,4BAA4B,EAC5B,0BAA0B,EAE1B,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAE5B,gBAAgB,EAChB,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,cAAc,EACd,eAAe,EACf,SAAS,GACV,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,UAAU,EACV,iBAAiB,EAEjB,WAAW,EACX,SAAS,EACT,UAAU,EAEV,eAAe,EACf,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,eAAe,EAEf,cAAc,EACd,YAAY,EACZ,aAAa,EACb,mBAAmB,EAEnB,WAAW,EACX,cAAc,EACd,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,OAAO,EACP,cAAc,GACf,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,WAAW,EACX,iBAAiB,EAEjB,eAAe,EACf,cAAc,EACd,UAAU,EAEV,WAAW,EAEX,kBAAkB,EAClB,cAAc,EACd,QAAQ,EAER,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,YAAY,GACb,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMrF,YAAY,EACV,cAAc,EACd,aAAa,EACb,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EAEtB,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EACtB,eAAe,EACf,mBAAmB,EACnB,cAAc,EACd,uBAAuB,GACxB,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMxD,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAM7E,YAAY,EAEV,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,sBAAsB,EACtB,gBAAgB,EAChB,iBAAiB,EAEjB,YAAY,EACZ,iBAAiB,EACjB,eAAe,EACf,aAAa,EACb,SAAS,EACT,eAAe,EAEf,UAAU,EACV,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,eAAe,EACf,eAAe,EAEf,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EAEnB,oBAAoB,EACpB,cAAc,EAEd,YAAY,EACZ,OAAO,EACP,WAAW,EACX,kBAAkB,EAClB,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,eAAe,EAEf,eAAe,EACf,mBAAmB,EACnB,YAAY,EAEZ,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EAEpB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EAEpB,aAAa,EACb,0BAA0B,GAC3B,MAAM,YAAY,CAAC;AAMpB,YAAY,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAMpE,YAAY,EAEV,YAAY,EACZ,eAAe,EAEf,SAAS,EACT,eAAe,EACf,mBAAmB,EACnB,oBAAoB,EACpB,eAAe,EAEf,WAAW,EACX,eAAe,EACf,aAAa,EACb,cAAc,EACd,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,aAAa,CAAC;AAMrB,YAAY,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAEnE,YAAY,EACV,aAAa,EACb,cAAc,EACd,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,sBAAsB,EACtB,qBAAqB,EACrB,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,eAAe,EACf,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,kBAAkB,EAClB,mBAAmB,EACnB,4BAA4B,EAC5B,qBAAqB,EACrB,2BAA2B,EAC3B,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAC5B,oBAAoB,EACpB,uBAAuB,EACvB,sBAAsB,EACtB,yBAAyB,EACzB,wBAAwB,EACxB,0BAA0B,EAC1B,qBAAqB,EACrB,uBAAuB,EACvB,uBAAuB,EACvB,yBAAyB,EACzB,mBAAmB,EACnB,wBAAwB,EACxB,mBAAmB,EACnB,2BAA2B,EAC3B,oBAAoB,EACpB,aAAa,EACb,mBAAmB,EACnB,0BAA0B,EAC1B,0BAA0B,EAC1B,gCAAgC,EAChC,sCAAsC,GACvC,MAAM,kBAAkB,CAAC"}
|
|
1
|
+
{"version":3,"file":"types-only.d.ts","sourceRoot":"","sources":["../src/types-only.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAMH,YAAY,EAEV,gBAAgB,EAChB,kBAAkB,EAClB,UAAU,EACV,cAAc,EACd,aAAa,EAEb,SAAS,EACT,aAAa,EACb,YAAY,EAEZ,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,yBAAyB,EACzB,4BAA4B,EAC5B,0BAA0B,EAE1B,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAE5B,gBAAgB,EAChB,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,cAAc,EACd,eAAe,EACf,SAAS,GACV,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,UAAU,EACV,iBAAiB,EAEjB,WAAW,EACX,SAAS,EACT,UAAU,EAEV,eAAe,EACf,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,eAAe,EAEf,cAAc,EACd,YAAY,EACZ,aAAa,EACb,mBAAmB,EAEnB,WAAW,EACX,cAAc,EACd,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,OAAO,EACP,cAAc,GACf,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,WAAW,EACX,iBAAiB,EAEjB,eAAe,EACf,cAAc,EACd,UAAU,EAEV,WAAW,EAEX,kBAAkB,EAClB,cAAc,EACd,QAAQ,EAER,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,YAAY,GACb,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMrF,YAAY,EACV,cAAc,EACd,aAAa,EACb,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EAEtB,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EACtB,eAAe,EACf,mBAAmB,EACnB,cAAc,EACd,uBAAuB,GACxB,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMxD,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAM7E,YAAY,EAEV,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,sBAAsB,EACtB,gBAAgB,EAChB,iBAAiB,EAEjB,YAAY,EACZ,iBAAiB,EACjB,eAAe,EACf,aAAa,EACb,SAAS,EACT,eAAe,EAEf,UAAU,EACV,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,eAAe,EACf,eAAe,EAEf,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EAEnB,oBAAoB,EACpB,cAAc,EAEd,YAAY,EACZ,OAAO,EACP,WAAW,EACX,kBAAkB,EAClB,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,eAAe,EAEf,eAAe,EACf,mBAAmB,EACnB,YAAY,EAEZ,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EAEpB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EAEpB,aAAa,EACb,0BAA0B,GAC3B,MAAM,YAAY,CAAC;AAMpB,YAAY,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAMpE,YAAY,EAEV,YAAY,EACZ,eAAe,EAEf,SAAS,EACT,eAAe,EACf,mBAAmB,EACnB,oBAAoB,EACpB,eAAe,EAEf,WAAW,EACX,eAAe,EACf,aAAa,EACb,cAAc,EACd,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,aAAa,CAAC;AAMrB,YAAY,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAEnE,YAAY,EACV,aAAa,EACb,cAAc,EACd,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,sBAAsB,EACtB,qBAAqB,EACrB,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,eAAe,EACf,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,kBAAkB,EAClB,mBAAmB,EACnB,4BAA4B,EAC5B,qBAAqB,EACrB,2BAA2B,EAC3B,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAC5B,oBAAoB,EACpB,uBAAuB,EACvB,sBAAsB,EACtB,yBAAyB,EACzB,wBAAwB,EACxB,0BAA0B,EAC1B,qBAAqB,EACrB,uBAAuB,EACvB,uBAAuB,EACvB,yBAAyB,EACzB,mBAAmB,EACnB,wBAAwB,EACxB,mBAAmB,EACnB,2BAA2B,EAC3B,yBAAyB,EACzB,wBAAwB,EACxB,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EACpB,aAAa,EACb,mBAAmB,EACnB,0BAA0B,EAC1B,0BAA0B,EAC1B,gCAAgC,EAChC,sCAAsC,GACvC,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,cAAc,EACd,kBAAkB,EAClB,oBAAoB,EACpB,qBAAqB,EACrB,6BAA6B,EAC7B,qBAAqB,EACrB,sBAAsB,EACtB,iBAAiB,EACjB,qBAAqB,GACtB,MAAM,qBAAqB,CAAC"}
|
package/dist/workflow.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"workflow.d.ts","sourceRoot":"","sources":["../src/workflow.ts"],"names":[],"mappings":"AACA,OAAO,
|
|
1
|
+
{"version":3,"file":"workflow.d.ts","sourceRoot":"","sources":["../src/workflow.ts"],"names":[],"mappings":"AACA,OAAO,EAGL,KAAK,oBAAoB,EAO1B,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAEpE,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,kBAAkB,EAC3B,MAAM,EAAE,gBAAgB,GACvB,OAAO,CAAC,oBAAoB,CAAC,CAmD/B"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@artemiskit/sdk",
|
|
3
|
-
"version": "0.6.
|
|
3
|
+
"version": "0.6.3",
|
|
4
4
|
"description": "Programmatic SDK for ArtemisKit LLM evaluation toolkit - integrate into your Node.js apps",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -71,8 +71,9 @@
|
|
|
71
71
|
"test:coverage": "bun test --coverage"
|
|
72
72
|
},
|
|
73
73
|
"dependencies": {
|
|
74
|
-
"@artemiskit/core": "0.6.
|
|
75
|
-
"@artemiskit/redteam": "0.3.
|
|
74
|
+
"@artemiskit/core": "0.6.4",
|
|
75
|
+
"@artemiskit/redteam": "0.3.12",
|
|
76
|
+
"@artemiskit/reports": "0.5.0"
|
|
76
77
|
},
|
|
77
78
|
"devDependencies": {
|
|
78
79
|
"@types/bun": "^1.1.0",
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import { describe, expect, test } from 'bun:test';
|
|
2
|
+
import { mkdtemp } from 'node:fs/promises';
|
|
3
|
+
import { tmpdir } from 'node:os';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
import { type AgentTarget, validateAgentWorkflow } from '@artemiskit/core';
|
|
6
|
+
import { ArtemisKit } from '../artemiskit';
|
|
7
|
+
import { readWorkflowRecord } from '../index';
|
|
8
|
+
import type { WorkflowCheckpointOptions } from '../types-only';
|
|
9
|
+
|
|
10
|
+
describe('SDK workflow recovery options', () => {
|
|
11
|
+
test('forwards checkpoint and pause controls and preserves the same logical run', async () => {
|
|
12
|
+
const parent = await mkdtemp(join(tmpdir(), 'artemis-sdk-recovery-'));
|
|
13
|
+
const checkpoint: WorkflowCheckpointOptions = {
|
|
14
|
+
directory: join(parent, 'private'),
|
|
15
|
+
mode: 'create',
|
|
16
|
+
configurationId: 'sdk-fixture-v1',
|
|
17
|
+
};
|
|
18
|
+
const workflow = validateAgentWorkflow({
|
|
19
|
+
version: '1',
|
|
20
|
+
kind: 'agent_workflow',
|
|
21
|
+
name: 'sdk-recovery',
|
|
22
|
+
target: { provider: 'openai', model: 'fixture' },
|
|
23
|
+
environment: {
|
|
24
|
+
type: 'simulated',
|
|
25
|
+
policy: {
|
|
26
|
+
network: 'denied',
|
|
27
|
+
side_effects: 'denied',
|
|
28
|
+
permissions: { files: 'write' },
|
|
29
|
+
budgets: {
|
|
30
|
+
max_actions: 6,
|
|
31
|
+
max_model_requests: 2,
|
|
32
|
+
max_tool_calls: 2,
|
|
33
|
+
max_tokens: 6,
|
|
34
|
+
timeout_ms: 20000,
|
|
35
|
+
},
|
|
36
|
+
},
|
|
37
|
+
},
|
|
38
|
+
tools: ['write_file'],
|
|
39
|
+
workflow: {
|
|
40
|
+
system_instructions: 'Write two files.',
|
|
41
|
+
initial_state: { files: {} },
|
|
42
|
+
turns: [{ role: 'user', content: 'PRIVATE-WORKFLOW' }],
|
|
43
|
+
},
|
|
44
|
+
outcomes: {
|
|
45
|
+
deterministic: [{ type: 'file', path: 'two.txt', exists: true, equals: 'PRIVATE-FILE' }],
|
|
46
|
+
},
|
|
47
|
+
evidence: { trace: 'summary', artifacts: 'checksums', redact: true },
|
|
48
|
+
});
|
|
49
|
+
let calls = 0;
|
|
50
|
+
let capabilities = 0;
|
|
51
|
+
const target: AgentTarget = {
|
|
52
|
+
provider: 'openai',
|
|
53
|
+
capabilities: async () => {
|
|
54
|
+
capabilities++;
|
|
55
|
+
return { status: 'available', toolUse: true, transportCancellation: true };
|
|
56
|
+
},
|
|
57
|
+
turn: async (request) => {
|
|
58
|
+
calls++;
|
|
59
|
+
const continued = request.messages.some((message) => message.role === 'tool');
|
|
60
|
+
return {
|
|
61
|
+
status: 'completed',
|
|
62
|
+
id: `answer-${calls}`,
|
|
63
|
+
model: 'fixture',
|
|
64
|
+
latencyMs: 1,
|
|
65
|
+
tokens: { prompt: 2, completion: 1, total: 3 },
|
|
66
|
+
message: {
|
|
67
|
+
role: 'assistant',
|
|
68
|
+
content: continued ? 'Done.' : '',
|
|
69
|
+
...(continued
|
|
70
|
+
? {}
|
|
71
|
+
: {
|
|
72
|
+
tool_calls: ['one', 'two'].map((name) => ({
|
|
73
|
+
id: name,
|
|
74
|
+
type: 'function' as const,
|
|
75
|
+
function: {
|
|
76
|
+
name: 'write_file',
|
|
77
|
+
arguments: JSON.stringify({ path: `${name}.txt`, content: 'PRIVATE-FILE' }),
|
|
78
|
+
},
|
|
79
|
+
})),
|
|
80
|
+
}),
|
|
81
|
+
},
|
|
82
|
+
};
|
|
83
|
+
},
|
|
84
|
+
};
|
|
85
|
+
const kit = new ArtemisKit();
|
|
86
|
+
const paused = await kit.runWorkflow({ workflow, target, checkpoint, pauseAfterActions: 2 });
|
|
87
|
+
expect(paused.record.reason).toBe('checkpoint_paused');
|
|
88
|
+
expect(paused.record.budgets.actions).toBe(2);
|
|
89
|
+
expect(calls).toBe(1);
|
|
90
|
+
expect(readWorkflowRecord(paused.record)).toEqual(paused.record);
|
|
91
|
+
const bad = await kit.runWorkflow({
|
|
92
|
+
workflow,
|
|
93
|
+
target,
|
|
94
|
+
checkpoint: { ...checkpoint, mode: 'resume', configurationId: 'changed' },
|
|
95
|
+
});
|
|
96
|
+
expect(bad.record.execution).not.toBe('completed');
|
|
97
|
+
expect(capabilities).toBe(1);
|
|
98
|
+
expect(calls).toBe(1);
|
|
99
|
+
const session = await kit.createWorkflowSession({
|
|
100
|
+
workflow,
|
|
101
|
+
target,
|
|
102
|
+
checkpoint: { ...checkpoint, mode: 'resume' },
|
|
103
|
+
});
|
|
104
|
+
expect(typeof session.pause).toBe('function');
|
|
105
|
+
const resumed = await session.run();
|
|
106
|
+
expect(resumed.record.taskVerification).toBe('passed');
|
|
107
|
+
expect(resumed.record.budgets).toMatchObject({ actions: 4, toolCalls: 2, modelRequests: 2 });
|
|
108
|
+
expect(resumed.record.usage.reported.total).toBe(6);
|
|
109
|
+
expect(calls).toBe(2);
|
|
110
|
+
expect(resumed.record.recovery?.runId).toBe(paused.record.recovery?.runId);
|
|
111
|
+
expect(resumed.record.recovery?.attempts).toBe(2);
|
|
112
|
+
expect(JSON.stringify(resumed.record)).not.toContain('PRIVATE-');
|
|
113
|
+
expect(JSON.stringify(resumed.record)).not.toContain(checkpoint.directory);
|
|
114
|
+
expect(readWorkflowRecord(resumed.record)).toEqual(resumed.record);
|
|
115
|
+
});
|
|
116
|
+
});
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import { describe, expect, test } from 'bun:test';
|
|
2
|
+
import { createAgentWorkflowSession, validateAgentWorkflow } from '@artemiskit/core';
|
|
3
|
+
import {
|
|
4
|
+
ArtemisKit,
|
|
5
|
+
createWorkflowReport,
|
|
6
|
+
generateWorkflowReport,
|
|
7
|
+
renderWorkflowReportHTML,
|
|
8
|
+
renderWorkflowReportMarkdown,
|
|
9
|
+
} from '../index';
|
|
10
|
+
|
|
11
|
+
async function evidence() {
|
|
12
|
+
const workflow = validateAgentWorkflow({
|
|
13
|
+
version: '1',
|
|
14
|
+
kind: 'agent_workflow',
|
|
15
|
+
name: 'report-only',
|
|
16
|
+
target: { provider: 'openai', model: 'fixture' },
|
|
17
|
+
environment: {
|
|
18
|
+
type: 'simulated',
|
|
19
|
+
policy: {
|
|
20
|
+
network: 'denied',
|
|
21
|
+
side_effects: 'denied',
|
|
22
|
+
permissions: { workflow_state: 'read' },
|
|
23
|
+
budgets: { max_actions: 2, timeout_ms: 1000 },
|
|
24
|
+
},
|
|
25
|
+
},
|
|
26
|
+
tools: ['get_workflow_state'],
|
|
27
|
+
workflow: {
|
|
28
|
+
system_instructions: 'Fixture',
|
|
29
|
+
initial_state: { workflow_state: { ready: true } },
|
|
30
|
+
turns: [{ role: 'user', content: 'Do not persist private prose.' }],
|
|
31
|
+
},
|
|
32
|
+
outcomes: { deterministic: [{ type: 'workflow_state', path: 'ready', equals: true }] },
|
|
33
|
+
evidence: { trace: 'summary', artifacts: 'checksums', redact: true },
|
|
34
|
+
});
|
|
35
|
+
return (
|
|
36
|
+
await createAgentWorkflowSession({
|
|
37
|
+
workflow,
|
|
38
|
+
target: {
|
|
39
|
+
provider: 'openai',
|
|
40
|
+
capabilities: async () => ({
|
|
41
|
+
status: 'available',
|
|
42
|
+
toolUse: true,
|
|
43
|
+
transportCancellation: true,
|
|
44
|
+
}),
|
|
45
|
+
turn: async () => ({
|
|
46
|
+
status: 'completed',
|
|
47
|
+
id: 'fixture',
|
|
48
|
+
model: 'fixture',
|
|
49
|
+
message: { role: 'assistant', content: 'Do not persist private prose.' },
|
|
50
|
+
tokens: { prompt: 1, completion: 1, total: 2 },
|
|
51
|
+
latencyMs: 1,
|
|
52
|
+
}),
|
|
53
|
+
},
|
|
54
|
+
}).run()
|
|
55
|
+
).record;
|
|
56
|
+
}
|
|
57
|
+
describe('SDK offline workflow reports', () => {
|
|
58
|
+
test('top-level and class surfaces share canonical model and both deterministic renderers', async () => {
|
|
59
|
+
const record = await evidence();
|
|
60
|
+
expect(record.taskVerification).toBe('passed');
|
|
61
|
+
const kit = new ArtemisKit({
|
|
62
|
+
provider: 'openai',
|
|
63
|
+
providerConfig: { apiKey: 'not-a-credential' },
|
|
64
|
+
});
|
|
65
|
+
const fetch = globalThis.fetch;
|
|
66
|
+
globalThis.fetch = (() => {
|
|
67
|
+
throw new Error('Reporting attempted network');
|
|
68
|
+
}) as typeof fetch;
|
|
69
|
+
try {
|
|
70
|
+
const before = JSON.stringify(record);
|
|
71
|
+
const model = createWorkflowReport(record);
|
|
72
|
+
expect(kit.createWorkflowReport(record)).toEqual(model);
|
|
73
|
+
for (const view of ['technical', 'executive', 'comprehensive'] as const) {
|
|
74
|
+
for (const format of ['html', 'markdown'] as const) {
|
|
75
|
+
const text = generateWorkflowReport(record, { view, format });
|
|
76
|
+
expect(kit.generateWorkflowReport(record, { view, format })).toBe(text);
|
|
77
|
+
expect(text).toBe(
|
|
78
|
+
format === 'html'
|
|
79
|
+
? renderWorkflowReportHTML(model, { view })
|
|
80
|
+
: renderWorkflowReportMarkdown(model, { view })
|
|
81
|
+
);
|
|
82
|
+
expect(text).not.toContain('private prose');
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
expect(JSON.stringify(record)).toBe(before);
|
|
86
|
+
expect(() => kit.createWorkflowReport({ schemaVersion: '99' })).toThrow();
|
|
87
|
+
expect(() => generateWorkflowReport(record, { format: 'pdf' } as never)).toThrow(
|
|
88
|
+
'Invalid workflow report options'
|
|
89
|
+
);
|
|
90
|
+
for (const options of [{ format: null }, { view: null }, { unexpected: true }]) {
|
|
91
|
+
expect(() => generateWorkflowReport(record, options as never)).toThrow(
|
|
92
|
+
'Invalid workflow report options'
|
|
93
|
+
);
|
|
94
|
+
}
|
|
95
|
+
let accessed = false;
|
|
96
|
+
expect(() =>
|
|
97
|
+
generateWorkflowReport(record, {
|
|
98
|
+
get view() {
|
|
99
|
+
accessed = true;
|
|
100
|
+
throw new Error('sensitive');
|
|
101
|
+
},
|
|
102
|
+
} as never)
|
|
103
|
+
).toThrow('Invalid workflow report options');
|
|
104
|
+
expect(accessed).toBe(false);
|
|
105
|
+
} finally {
|
|
106
|
+
globalThis.fetch = fetch;
|
|
107
|
+
}
|
|
108
|
+
});
|
|
109
|
+
});
|
package/src/artemiskit.ts
CHANGED
|
@@ -40,6 +40,11 @@ import {
|
|
|
40
40
|
TypoMutation,
|
|
41
41
|
UnsafeResponseDetector,
|
|
42
42
|
} from '@artemiskit/redteam';
|
|
43
|
+
import {
|
|
44
|
+
type GenerateWorkflowReportOptions,
|
|
45
|
+
createWorkflowReport,
|
|
46
|
+
generateWorkflowReport,
|
|
47
|
+
} from '@artemiskit/reports';
|
|
43
48
|
import { nanoid } from 'nanoid';
|
|
44
49
|
import { prepareWorkflowSession } from './workflow';
|
|
45
50
|
|
|
@@ -132,11 +137,21 @@ export class ArtemisKit {
|
|
|
132
137
|
return prepareWorkflowSession(options, this.config);
|
|
133
138
|
}
|
|
134
139
|
|
|
135
|
-
/** Execute a workflow
|
|
140
|
+
/** Execute a workflow and independently evaluate its declared outcomes. */
|
|
136
141
|
async runWorkflow(options: WorkflowRunOptions) {
|
|
137
142
|
return (await this.createWorkflowSession(options)).run();
|
|
138
143
|
}
|
|
139
144
|
|
|
145
|
+
/** Project saved evidence without running a workflow or loading provider configuration. */
|
|
146
|
+
createWorkflowReport(input: unknown) {
|
|
147
|
+
return createWorkflowReport(input);
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/** Generate HTML or Markdown offline; task failures remain visible in a successful export. */
|
|
151
|
+
generateWorkflowReport(input: unknown, options?: GenerateWorkflowReportOptions) {
|
|
152
|
+
return generateWorkflowReport(input, options);
|
|
153
|
+
}
|
|
154
|
+
|
|
140
155
|
// ==========================================================================
|
|
141
156
|
// Event Emitter Methods
|
|
142
157
|
// ==========================================================================
|
package/src/index.ts
CHANGED
|
@@ -10,6 +10,8 @@ export { ArtemisKit } from './artemiskit';
|
|
|
10
10
|
export {
|
|
11
11
|
AgentWorkflowSchema,
|
|
12
12
|
WorkflowPolicySchema,
|
|
13
|
+
WorkflowFaultsSchema,
|
|
14
|
+
WorkflowRetrySchema,
|
|
13
15
|
WORKFLOW_TOOL_IDS,
|
|
14
16
|
validateAgentWorkflow,
|
|
15
17
|
parseAgentWorkflow,
|
|
@@ -70,6 +72,12 @@ export type {
|
|
|
70
72
|
HistoricalWorkflowRecord,
|
|
71
73
|
AgentWorkflowResult,
|
|
72
74
|
AgentWorkflowSessionOptions,
|
|
75
|
+
WorkflowCheckpointOptions,
|
|
76
|
+
WorkflowRecoveryEvidence,
|
|
77
|
+
WorkflowFault,
|
|
78
|
+
WorkflowFaults,
|
|
79
|
+
WorkflowFaultKind,
|
|
80
|
+
WorkflowRetry,
|
|
73
81
|
AgentWorkflowSession,
|
|
74
82
|
WorkflowState,
|
|
75
83
|
WorkflowEnvironment,
|
|
@@ -479,3 +487,22 @@ export type {
|
|
|
479
487
|
MultiTurnValidationResult,
|
|
480
488
|
ValidateMessageOptions,
|
|
481
489
|
} from './guardian';
|
|
490
|
+
|
|
491
|
+
// Offline reports never initialize target or judge clients.
|
|
492
|
+
export {
|
|
493
|
+
createWorkflowReport,
|
|
494
|
+
generateWorkflowReport,
|
|
495
|
+
renderWorkflowReportHTML,
|
|
496
|
+
renderWorkflowReportMarkdown,
|
|
497
|
+
} from '@artemiskit/reports';
|
|
498
|
+
export type {
|
|
499
|
+
WorkflowReport,
|
|
500
|
+
WorkflowReportView,
|
|
501
|
+
WorkflowReportFormat,
|
|
502
|
+
WorkflowReportOptions,
|
|
503
|
+
GenerateWorkflowReportOptions,
|
|
504
|
+
WorkflowReportFinding,
|
|
505
|
+
WorkflowReportEvidence,
|
|
506
|
+
WorkflowReportRow,
|
|
507
|
+
WorkflowReportSection,
|
|
508
|
+
} from '@artemiskit/reports';
|
package/src/types-only.ts
CHANGED
|
@@ -314,6 +314,12 @@ export type {
|
|
|
314
314
|
HistoricalWorkflowRecord,
|
|
315
315
|
AgentWorkflowResult,
|
|
316
316
|
AgentWorkflowSessionOptions,
|
|
317
|
+
WorkflowCheckpointOptions,
|
|
318
|
+
WorkflowRecoveryEvidence,
|
|
319
|
+
WorkflowFault,
|
|
320
|
+
WorkflowFaults,
|
|
321
|
+
WorkflowFaultKind,
|
|
322
|
+
WorkflowRetry,
|
|
317
323
|
AgentWorkflowSession,
|
|
318
324
|
WorkflowState,
|
|
319
325
|
WorkflowEnvironment,
|
|
@@ -322,3 +328,14 @@ export type {
|
|
|
322
328
|
DockerWorkflowEnvironmentOptions,
|
|
323
329
|
WorkflowEnvironmentInitializationError,
|
|
324
330
|
} from '@artemiskit/core';
|
|
331
|
+
export type {
|
|
332
|
+
WorkflowReport,
|
|
333
|
+
WorkflowReportView,
|
|
334
|
+
WorkflowReportFormat,
|
|
335
|
+
WorkflowReportOptions,
|
|
336
|
+
GenerateWorkflowReportOptions,
|
|
337
|
+
WorkflowReportFinding,
|
|
338
|
+
WorkflowReportEvidence,
|
|
339
|
+
WorkflowReportRow,
|
|
340
|
+
WorkflowReportSection,
|
|
341
|
+
} from '@artemiskit/reports';
|
package/src/workflow.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { dirname, resolve } from 'node:path';
|
|
2
2
|
import {
|
|
3
3
|
type AdapterConfig,
|
|
4
|
+
type AgentTarget,
|
|
4
5
|
type AgentWorkflowSession,
|
|
5
6
|
ArtemisError,
|
|
6
7
|
createAdapter,
|
|
@@ -20,22 +21,36 @@ export async function prepareWorkflowSession(
|
|
|
20
21
|
const workflow = file ? await loadAgentWorkflow(file) : validateAgentWorkflow(options.workflow);
|
|
21
22
|
let target = options.target;
|
|
22
23
|
if (!target) {
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
24
|
+
if (options.client) target = createModelClientTarget(options.client);
|
|
25
|
+
else {
|
|
26
|
+
let initialized: Promise<AgentTarget> | undefined;
|
|
27
|
+
const getTarget = () => {
|
|
28
|
+
initialized ??= createAdapter({
|
|
27
29
|
...config.providerConfig,
|
|
28
30
|
...options.providerConfig,
|
|
29
31
|
provider: workflow.target.provider,
|
|
30
32
|
defaultModel: workflow.target.model,
|
|
31
33
|
maxRetries: 0,
|
|
32
|
-
} as AdapterConfig)
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
34
|
+
} as AdapterConfig)
|
|
35
|
+
.then(createModelClientTarget)
|
|
36
|
+
.catch(() => {
|
|
37
|
+
throw new ArtemisError('Unable to initialize workflow target', 'PROVIDER_UNAVAILABLE');
|
|
38
|
+
});
|
|
39
|
+
return initialized;
|
|
40
|
+
};
|
|
41
|
+
target = options.checkpoint
|
|
42
|
+
? {
|
|
43
|
+
provider: workflow.target.provider,
|
|
44
|
+
capabilities: async (limits, signal) =>
|
|
45
|
+
(await getTarget()).capabilities(limits, signal),
|
|
46
|
+
turn: async (request, signal) => (await getTarget()).turn(request, signal),
|
|
47
|
+
drain: async (limits) =>
|
|
48
|
+
initialized
|
|
49
|
+
? ((await initialized).drain?.(limits) ?? { pendingOperations: 0 })
|
|
50
|
+
: { pendingOperations: 0 },
|
|
51
|
+
}
|
|
52
|
+
: await getTarget();
|
|
37
53
|
}
|
|
38
|
-
target = createModelClientTarget(client);
|
|
39
54
|
}
|
|
40
55
|
return createAgentWorkflowSession({
|
|
41
56
|
workflow,
|
|
@@ -48,5 +63,7 @@ export async function prepareWorkflowSession(
|
|
|
48
63
|
cleanupTimeoutMs: options.cleanupTimeoutMs,
|
|
49
64
|
semanticJudge: options.semanticJudge,
|
|
50
65
|
onEvent: options.onEvent,
|
|
66
|
+
checkpoint: options.checkpoint,
|
|
67
|
+
pauseAfterActions: options.pauseAfterActions,
|
|
51
68
|
});
|
|
52
69
|
}
|