@artemiskit/sdk 0.6.2 → 0.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/README.md +17 -0
- package/dist/artemiskit.d.ts +6 -1
- package/dist/artemiskit.d.ts.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +21 -0
- package/dist/types-only.d.ts +1 -0
- package/dist/types-only.d.ts.map +1 -1
- package/package.json +4 -3
- package/src/__tests__/workflow-report.test.ts +109 -0
- package/src/artemiskit.ts +16 -1
- package/src/index.ts +19 -0
- package/src/types-only.ts +11 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,15 @@
|
|
|
1
1
|
# @artemiskit/sdk
|
|
2
2
|
|
|
3
|
+
## 0.6.3
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- 5c35b04: Add deterministic offline workflow assessments in technical, executive and comprehensive HTML/Markdown views. Strict saved V1/V2/V3 records share one canonical report model, evidence links, explicit measurement denominators and compatible resume accounting. Expose `akit workflow report` and matching synchronous SDK functions with bounded input and no-overwrite exports. No provider calls, inferred pricing or task rescoring occur. Core receives a release-alignment patch identifying the coherent 0.6.4 milestone; its workflow record contract remains unchanged.
|
|
8
|
+
- Updated dependencies [5c35b04]
|
|
9
|
+
- @artemiskit/reports@0.5.0
|
|
10
|
+
- @artemiskit/core@0.6.4
|
|
11
|
+
- @artemiskit/redteam@0.3.12
|
|
12
|
+
|
|
3
13
|
## 0.6.2
|
|
4
14
|
|
|
5
15
|
### Patch Changes
|
package/README.md
CHANGED
|
@@ -247,6 +247,23 @@ const outputResult = await guardian.validateOutput(llmResponse);
|
|
|
247
247
|
| `guardian` | Block violations and log |
|
|
248
248
|
| `hybrid` | Block critical, log others |
|
|
249
249
|
|
|
250
|
+
## Offline workflow reports
|
|
251
|
+
|
|
252
|
+
```ts
|
|
253
|
+
import { ArtemisKit, createWorkflowReport, generateWorkflowReport } from '@artemiskit/sdk';
|
|
254
|
+
|
|
255
|
+
const assessment = createWorkflowReport(savedRecord); // object, JSON string, or record array
|
|
256
|
+
const html = generateWorkflowReport(savedRecord, { view: 'comprehensive', format: 'html' });
|
|
257
|
+
const markdown = new ArtemisKit().generateWorkflowReport(savedRecord, {
|
|
258
|
+
view: 'executive', format: 'markdown',
|
|
259
|
+
});
|
|
260
|
+
```
|
|
261
|
+
|
|
262
|
+
These synchronous functions share `@artemiskit/reports` and never initialize providers, restore
|
|
263
|
+
checkpoints or rescore tasks. `renderWorkflowReportHTML` and `renderWorkflowReportMarkdown` render
|
|
264
|
+
an existing canonical model. Invalid/unavailable measurements, historical evidence and cumulative
|
|
265
|
+
resumed attempts keep their original meaning. See the [guide](../../docs/workflow-reports.md).
|
|
266
|
+
|
|
250
267
|
## License
|
|
251
268
|
|
|
252
269
|
Apache-2.0
|
package/dist/artemiskit.d.ts
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
* @artemiskit/sdk
|
|
3
3
|
* Main ArtemisKit class - programmatic API for LLM testing
|
|
4
4
|
*/
|
|
5
|
+
import { type GenerateWorkflowReportOptions } from '@artemiskit/reports';
|
|
5
6
|
import type { ArtemisKitConfig, ArtemisKitEventName, ArtemisKitEvents, CaseCompleteHandler, CaseStartHandler, CompareOptions, CompareResult, ProgressHandler, RedTeamMutationCompleteHandler, RedTeamMutationStartHandler, RedTeamOptions, RedTeamResult, RunOptions, RunResult, StressOptions, StressRequestCompleteHandler, StressResult, ValidateOptions, ValidationResult, WorkflowRunOptions } from './types.js';
|
|
6
7
|
/**
|
|
7
8
|
* ArtemisKit SDK - programmatic API for LLM evaluation testing
|
|
@@ -27,8 +28,12 @@ export declare class ArtemisKit {
|
|
|
27
28
|
constructor(config?: ArtemisKitConfig);
|
|
28
29
|
/** Create a cancellable native session without executing model turns or persisting results. */
|
|
29
30
|
createWorkflowSession(options: WorkflowRunOptions): Promise<import("@artemiskit/core").AgentWorkflowSession>;
|
|
30
|
-
/** Execute a workflow
|
|
31
|
+
/** Execute a workflow and independently evaluate its declared outcomes. */
|
|
31
32
|
runWorkflow(options: WorkflowRunOptions): Promise<import("@artemiskit/core").AgentWorkflowResult>;
|
|
33
|
+
/** Project saved evidence without running a workflow or loading provider configuration. */
|
|
34
|
+
createWorkflowReport(input: unknown): import("@artemiskit/reports").WorkflowReport;
|
|
35
|
+
/** Generate HTML or Markdown offline; task failures remain visible in a successful export. */
|
|
36
|
+
generateWorkflowReport(input: unknown, options?: GenerateWorkflowReportOptions): string;
|
|
32
37
|
/**
|
|
33
38
|
* Register an event handler
|
|
34
39
|
*/
|
package/dist/artemiskit.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"artemiskit.d.ts","sourceRoot":"","sources":["../src/artemiskit.ts"],"names":[],"mappings":"AAAA;;;GAGG;
|
|
1
|
+
{"version":3,"file":"artemiskit.d.ts","sourceRoot":"","sources":["../src/artemiskit.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAuCH,OAAO,EACL,KAAK,6BAA6B,EAGnC,MAAM,qBAAqB,CAAC;AAI7B,OAAO,KAAK,EACV,gBAAgB,EAChB,mBAAmB,EACnB,gBAAgB,EAChB,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,aAAa,EACb,eAAe,EACf,8BAA8B,EAC9B,2BAA2B,EAC3B,cAAc,EACd,aAAa,EACb,UAAU,EACV,SAAS,EACT,aAAa,EACb,4BAA4B,EAC5B,YAAY,EACZ,eAAe,EACf,gBAAgB,EAChB,kBAAkB,EACnB,MAAM,SAAS,CAAC;AAsBjB;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAmB;IACjC,OAAO,CAAC,aAAa,CAA6D;IAClF,OAAO,CAAC,OAAO,CAA+B;gBAElC,MAAM,GAAE,gBAAqB;IAmBzC,+FAA+F;IACzF,qBAAqB,CAAC,OAAO,EAAE,kBAAkB;IAIvD,2EAA2E;IACrE,WAAW,CAAC,OAAO,EAAE,kBAAkB;IAI7C,2FAA2F;IAC3F,oBAAoB,CAAC,KAAK,EAAE,OAAO;IAInC,8FAA8F;IAC9F,sBAAsB,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,6BAA6B;IAQ9E;;OAEG;IACH,EAAE,CAAC,CAAC,SAAS,mBAAmB,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,IAAI;IAQhG;;OAEG;IACH,GAAG,CAAC,CAAC,SAAS,mBAAmB,EAC/B,KAAK,EAAE,CAAC,EACR,OAAO,EAAE,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC,KAAK,IAAI,GAC5C,IAAI;IAQP;;OAEG;IACH,IAAI,CAAC,CAAC,SAAS,mBAAmB,EAChC,KAAK,EAAE,CAAC,EACR,OAAO,EAAE,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC,KAAK,IAAI,GAC5C,IAAI;IAQP;;OAEG;IACH,OAAO,CAAC,IAAI;IAiBZ;;OAEG;IACH,WAAW,CAAC,OAAO,EAAE,gBAAgB,GAAG,IAAI;IAI5C;;OAEG;IACH,cAAc,CAAC,OAAO,EAAE,mBAAmB,GAAG,IAAI;IAIlD;;OAEG;IACH,UAAU,CAAC,OAAO,EAAE,eAAe,GAAG,IAAI;IAI1C;;OAEG;IACH,sBAAsB,CAAC,OAAO,EAAE,2BAA2B,GAAG,IAAI;IAIlE;;OAEG;IACH,yBAAyB,CAAC,OAAO,EAAE,8BAA8B,GAAG,IAAI;IAIxE;;OAEG;IACH,uBAAuB,CAAC,OAAO,EAAE,4BAA4B,GAAG,IAAI;IAQpE;;OAEG;IACG,GAAG,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC;IA+DlD;;OAEG;IACG,OAAO,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;IAgN9D;;OAEG;IACG,MAAM,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC;IAwO3D;;;;;;;;;;;;;OAaG;IACG,QAAQ,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA2LnE;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACG,OAAO,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;IA2H9D;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAmBzB;;OAEG;IACH,OAAO,CAAC,YAAY;IA4EpB;;OAEG;IACH,OAAO,CAAC,YAAY;IAepB;;OAEG;YACW,YAAY;IAO1B;;OAEG;YACW,YAAY;IAgB1B;;OAEG;IACH,OAAO,CAAC,cAAc;IA6BtB;;OAEG;IACH,qBAAqB,IAAI,MAAM,EAAE;CAGlC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -22,4 +22,6 @@ export type { AdapterContract, EvaluatorContract, StorageContract, AdapterFactor
|
|
|
22
22
|
export { ScenarioBuilder, TestCaseBuilder, scenario, testCase, containsCase, exactCase, regexCase, jsonCase, gradedCase, exact, contains, notContains, regex, fuzzy, jsonSchema, llmGrade, similarity, inline, allOf, anyOf, } from './builders/index.js';
|
|
23
23
|
export { Guardian, createGuardian, normalizeGuardianMode, SemanticValidator, createSemanticValidator, GuardianInterceptor, GuardianBlockedError, createInterceptor, ActionValidator, createDefaultActionValidator, IntentClassifier, createIntentClassifier, detectInjection, createInjectionGuardrail, detectPII, createPIIGuardrail, filterContent, createContentFilterGuardrail, createGuardrails, matchPattern, createCustomPatternGuardrail, loadPolicy, parsePolicy, validatePolicy, createDefaultPolicy, mergePolicies, generatePolicyTemplate, PolicyLoadError, PolicyValidationError, CircuitBreaker, MetricsCollector, RateLimiter, } from './guardian/index.js';
|
|
24
24
|
export type { GuardianConfig, InterceptorConfig, ActionValidatorConfig, IntentClassifierConfig, GuardrailsConfig, RateLimiterConfig, CustomPatternOptions, GuardianMode, ViolationSeverity, ViolationAction, GuardrailType, Violation, GuardrailResult, PolicyRule, PolicyCondition, GuardianPolicy, CircuitBreakerConfig, RateLimitConfig, CostLimitConfig, ActionDefinition, ActionParameter, ParameterValidation, IntentClassification, IntentCategory, PIIDetection, PIIType, PIILocation, InjectionDetection, InjectionType, ContentFilterResult, ContentFlag, ContentCategory, GuardianMetrics, CircuitBreakerState, CostTracking, GuardianEventType, GuardianEvent, GuardianEventHandler, InterceptedRequest, InterceptedResponse, InterceptedToolCall, InterceptedAgentStep, FrameworkType, FrameworkIntegrationConfig, GuardianModeCanonical, GuardianModeLegacy, GuardianModeAll, ContentValidationConfig, PatternConfig, ValidationCategory, PatternCategory, SemanticValidationResult, SessionStorageType, SessionStorageConfig, SessionMessage, SessionMetrics, ConversationSession, TrustBuildingHeuristicConfig, EscalationHeuristicConfig, ContextManipulationHeuristicConfig, SplitPayloadHeuristicConfig, MultiTurnHeuristics, MultiTurnSemanticConfig, MultiTurnConfig, HeuristicResult, HeuristicResults, MultiTurnValidationResult, ValidateMessageOptions, } from './guardian/index.js';
|
|
25
|
+
export { createWorkflowReport, generateWorkflowReport, renderWorkflowReportHTML, renderWorkflowReportMarkdown, } from '@artemiskit/reports';
|
|
26
|
+
export type { WorkflowReport, WorkflowReportView, WorkflowReportFormat, WorkflowReportOptions, GenerateWorkflowReportOptions, WorkflowReportFinding, WorkflowReportEvidence, WorkflowReportRow, WorkflowReportSection, } from '@artemiskit/reports';
|
|
25
27
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAG1C,OAAO,EACL,mBAAmB,EACnB,oBAAoB,EACpB,oBAAoB,EACpB,mBAAmB,EACnB,iBAAiB,EACjB,qBAAqB,EACrB,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,eAAe,EACf,oBAAoB,EACpB,uBAAuB,EACvB,0BAA0B,EAC1B,gBAAgB,EAChB,qCAAqC,EACrC,yBAAyB,EACzB,2BAA2B,EAC3B,qBAAqB,EACrB,4BAA4B,EAC5B,kBAAkB,EAClB,kCAAkC,EAClC,+BAA+B,EAC/B,sCAAsC,EACtC,sBAAsB,EACtB,sCAAsC,GACvC,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,aAAa,EACb,cAAc,EACd,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,sBAAsB,EACtB,qBAAqB,EACrB,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,eAAe,EACf,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,kBAAkB,EAClB,mBAAmB,EACnB,4BAA4B,EAC5B,qBAAqB,EACrB,2BAA2B,EAC3B,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAC5B,oBAAoB,EACpB,uBAAuB,EACvB,sBAAsB,EACtB,yBAAyB,EACzB,wBAAwB,EACxB,0BAA0B,EAC1B,qBAAqB,EACrB,uBAAuB,EACvB,uBAAuB,EACvB,yBAAyB,EACzB,mBAAmB,EACnB,wBAAwB,EACxB,mBAAmB,EACnB,2BAA2B,EAC3B,yBAAyB,EACzB,wBAAwB,EACxB,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EACpB,aAAa,EACb,mBAAmB,EACnB,0BAA0B,EAC1B,0BAA0B,EAC1B,gCAAgC,GACjC,MAAM,kBAAkB,CAAC;AAG1B,YAAY,EAEV,gBAAgB,EAChB,kBAAkB,EAClB,UAAU,EACV,cAAc,EACd,aAAa,EAEb,SAAS,EACT,aAAa,EACb,YAAY,EAEZ,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,yBAAyB,EACzB,4BAA4B,EAC5B,0BAA0B,EAE1B,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAE5B,gBAAgB,EAChB,mBAAmB,EAEnB,eAAe,EACf,gBAAgB,EAChB,eAAe,EACf,iBAAiB,EACjB,kBAAkB,EAElB,cAAc,EACd,aAAa,EACb,UAAU,EACV,iBAAiB,GAClB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,cAAc,EACd,eAAe,EACf,SAAS,GACV,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,UAAU,EACV,iBAAiB,EAEjB,WAAW,EACX,SAAS,EACT,UAAU,EAEV,eAAe,EACf,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,eAAe,EAEf,cAAc,EACd,YAAY,EACZ,aAAa,EACb,mBAAmB,EAEnB,WAAW,EACX,cAAc,EACd,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,OAAO,EACP,cAAc,GACf,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,WAAW,EACX,iBAAiB,EAEjB,eAAe,EACf,cAAc,EACd,UAAU,EAEV,WAAW,EAEX,kBAAkB,EAClB,cAAc,EACd,QAAQ,EAER,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,YAAY,GACb,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMrF,YAAY,EACV,cAAc,EACd,aAAa,EACb,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EAEtB,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EACtB,eAAe,EACf,mBAAmB,EACnB,cAAc,EACd,uBAAuB,GACxB,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMxD,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAMtF,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAM7E,OAAO,EAAE,kBAAkB,EAAE,KAAK,kBAAkB,EAAE,KAAK,aAAa,EAAE,MAAM,YAAY,CAAC;AAM7F,OAAO,EAEL,WAAW,EACX,eAAe,EACf,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,oBAAoB,EAEpB,aAAa,EACb,MAAM,EAEN,cAAc,EACd,cAAc,EACd,aAAa,EACb,oBAAoB,GACrB,MAAM,SAAS,CAAC;AAEjB,YAAY,EAEV,YAAY,EACZ,eAAe,EAEf,SAAS,EACT,eAAe,EACf,mBAAmB,EACnB,oBAAoB,EACpB,eAAe,EAEf,WAAW,EACX,eAAe,EACf,aAAa,EACb,cAAc,EACd,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,OAAO,EAEL,aAAa,EAEb,eAAe,EAEf,aAAa,EAEb,YAAY,GACb,MAAM,aAAa,CAAC;AAErB,YAAY,EACV,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,aAAa,CAAC;AAMrB,OAAO,EAEL,eAAe,EACf,eAAe,EAEf,QAAQ,EACR,QAAQ,EAER,YAAY,EACZ,SAAS,EACT,SAAS,EACT,QAAQ,EACR,UAAU,EAEV,KAAK,EACL,QAAQ,EACR,WAAW,EACX,KAAK,EACL,KAAK,EACL,UAAU,EACV,QAAQ,EACR,UAAU,EACV,MAAM,EACN,KAAK,EACL,KAAK,GACN,MAAM,YAAY,CAAC;AAMpB,OAAO,EAEL,QAAQ,EACR,cAAc,EAEd,qBAAqB,EAErB,iBAAiB,EACjB,uBAAuB,EAEvB,mBAAmB,EACnB,oBAAoB,EACpB,iBAAiB,EAEjB,eAAe,EACf,4BAA4B,EAE5B,gBAAgB,EAChB,sBAAsB,EAEtB,eAAe,EACf,wBAAwB,EACxB,SAAS,EACT,kBAAkB,EAClB,aAAa,EACb,4BAA4B,EAC5B,gBAAgB,EAEhB,YAAY,EACZ,4BAA4B,EAE5B,UAAU,EACV,WAAW,EACX,cAAc,EACd,mBAAmB,EACnB,aAAa,EACb,sBAAsB,EACtB,eAAe,EACf,qBAAqB,EAErB,cAAc,EACd,gBAAgB,EAChB,WAAW,GACZ,MAAM,YAAY,CAAC;AAGpB,YAAY,EAEV,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,sBAAsB,EACtB,gBAAgB,EAChB,iBAAiB,EACjB,oBAAoB,EAEpB,YAAY,EACZ,iBAAiB,EACjB,eAAe,EACf,aAAa,EACb,SAAS,EACT,eAAe,EAEf,UAAU,EACV,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,eAAe,EACf,eAAe,EAEf,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EAEnB,oBAAoB,EACpB,cAAc,EAEd,YAAY,EACZ,OAAO,EACP,WAAW,EACX,kBAAkB,EAClB,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,eAAe,EAEf,eAAe,EACf,mBAAmB,EACnB,YAAY,EAEZ,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EAEpB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EAEpB,aAAa,EACb,0BAA0B,EAE1B,qBAAqB,EACrB,kBAAkB,EAClB,eAAe,EAEf,uBAAuB,EACvB,aAAa,EACb,kBAAkB,EAClB,eAAe,EACf,wBAAwB,EAExB,kBAAkB,EAClB,oBAAoB,EACpB,cAAc,EACd,cAAc,EACd,mBAAmB,EACnB,4BAA4B,EAC5B,yBAAyB,EACzB,kCAAkC,EAClC,2BAA2B,EAC3B,mBAAmB,EACnB,uBAAuB,EACvB,eAAe,EACf,eAAe,EACf,gBAAgB,EAChB,yBAAyB,EACzB,sBAAsB,GACvB,MAAM,YAAY,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAG1C,OAAO,EACL,mBAAmB,EACnB,oBAAoB,EACpB,oBAAoB,EACpB,mBAAmB,EACnB,iBAAiB,EACjB,qBAAqB,EACrB,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,eAAe,EACf,oBAAoB,EACpB,uBAAuB,EACvB,0BAA0B,EAC1B,gBAAgB,EAChB,qCAAqC,EACrC,yBAAyB,EACzB,2BAA2B,EAC3B,qBAAqB,EACrB,4BAA4B,EAC5B,kBAAkB,EAClB,kCAAkC,EAClC,+BAA+B,EAC/B,sCAAsC,EACtC,sBAAsB,EACtB,sCAAsC,GACvC,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,aAAa,EACb,cAAc,EACd,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,sBAAsB,EACtB,qBAAqB,EACrB,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,eAAe,EACf,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,kBAAkB,EAClB,mBAAmB,EACnB,4BAA4B,EAC5B,qBAAqB,EACrB,2BAA2B,EAC3B,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAC5B,oBAAoB,EACpB,uBAAuB,EACvB,sBAAsB,EACtB,yBAAyB,EACzB,wBAAwB,EACxB,0BAA0B,EAC1B,qBAAqB,EACrB,uBAAuB,EACvB,uBAAuB,EACvB,yBAAyB,EACzB,mBAAmB,EACnB,wBAAwB,EACxB,mBAAmB,EACnB,2BAA2B,EAC3B,yBAAyB,EACzB,wBAAwB,EACxB,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EACpB,aAAa,EACb,mBAAmB,EACnB,0BAA0B,EAC1B,0BAA0B,EAC1B,gCAAgC,GACjC,MAAM,kBAAkB,CAAC;AAG1B,YAAY,EAEV,gBAAgB,EAChB,kBAAkB,EAClB,UAAU,EACV,cAAc,EACd,aAAa,EAEb,SAAS,EACT,aAAa,EACb,YAAY,EAEZ,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,yBAAyB,EACzB,4BAA4B,EAC5B,0BAA0B,EAE1B,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAE5B,gBAAgB,EAChB,mBAAmB,EAEnB,eAAe,EACf,gBAAgB,EAChB,eAAe,EACf,iBAAiB,EACjB,kBAAkB,EAElB,cAAc,EACd,aAAa,EACb,UAAU,EACV,iBAAiB,GAClB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,cAAc,EACd,eAAe,EACf,SAAS,GACV,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,UAAU,EACV,iBAAiB,EAEjB,WAAW,EACX,SAAS,EACT,UAAU,EAEV,eAAe,EACf,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,eAAe,EAEf,cAAc,EACd,YAAY,EACZ,aAAa,EACb,mBAAmB,EAEnB,WAAW,EACX,cAAc,EACd,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,OAAO,EACP,cAAc,GACf,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,WAAW,EACX,iBAAiB,EAEjB,eAAe,EACf,cAAc,EACd,UAAU,EAEV,WAAW,EAEX,kBAAkB,EAClB,cAAc,EACd,QAAQ,EAER,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,YAAY,GACb,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMrF,YAAY,EACV,cAAc,EACd,aAAa,EACb,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EAEtB,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EACtB,eAAe,EACf,mBAAmB,EACnB,cAAc,EACd,uBAAuB,GACxB,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMxD,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAMtF,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAM7E,OAAO,EAAE,kBAAkB,EAAE,KAAK,kBAAkB,EAAE,KAAK,aAAa,EAAE,MAAM,YAAY,CAAC;AAM7F,OAAO,EAEL,WAAW,EACX,eAAe,EACf,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,oBAAoB,EAEpB,aAAa,EACb,MAAM,EAEN,cAAc,EACd,cAAc,EACd,aAAa,EACb,oBAAoB,GACrB,MAAM,SAAS,CAAC;AAEjB,YAAY,EAEV,YAAY,EACZ,eAAe,EAEf,SAAS,EACT,eAAe,EACf,mBAAmB,EACnB,oBAAoB,EACpB,eAAe,EAEf,WAAW,EACX,eAAe,EACf,aAAa,EACb,cAAc,EACd,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,OAAO,EAEL,aAAa,EAEb,eAAe,EAEf,aAAa,EAEb,YAAY,GACb,MAAM,aAAa,CAAC;AAErB,YAAY,EACV,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,aAAa,CAAC;AAMrB,OAAO,EAEL,eAAe,EACf,eAAe,EAEf,QAAQ,EACR,QAAQ,EAER,YAAY,EACZ,SAAS,EACT,SAAS,EACT,QAAQ,EACR,UAAU,EAEV,KAAK,EACL,QAAQ,EACR,WAAW,EACX,KAAK,EACL,KAAK,EACL,UAAU,EACV,QAAQ,EACR,UAAU,EACV,MAAM,EACN,KAAK,EACL,KAAK,GACN,MAAM,YAAY,CAAC;AAMpB,OAAO,EAEL,QAAQ,EACR,cAAc,EAEd,qBAAqB,EAErB,iBAAiB,EACjB,uBAAuB,EAEvB,mBAAmB,EACnB,oBAAoB,EACpB,iBAAiB,EAEjB,eAAe,EACf,4BAA4B,EAE5B,gBAAgB,EAChB,sBAAsB,EAEtB,eAAe,EACf,wBAAwB,EACxB,SAAS,EACT,kBAAkB,EAClB,aAAa,EACb,4BAA4B,EAC5B,gBAAgB,EAEhB,YAAY,EACZ,4BAA4B,EAE5B,UAAU,EACV,WAAW,EACX,cAAc,EACd,mBAAmB,EACnB,aAAa,EACb,sBAAsB,EACtB,eAAe,EACf,qBAAqB,EAErB,cAAc,EACd,gBAAgB,EAChB,WAAW,GACZ,MAAM,YAAY,CAAC;AAGpB,YAAY,EAEV,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,sBAAsB,EACtB,gBAAgB,EAChB,iBAAiB,EACjB,oBAAoB,EAEpB,YAAY,EACZ,iBAAiB,EACjB,eAAe,EACf,aAAa,EACb,SAAS,EACT,eAAe,EAEf,UAAU,EACV,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,eAAe,EACf,eAAe,EAEf,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EAEnB,oBAAoB,EACpB,cAAc,EAEd,YAAY,EACZ,OAAO,EACP,WAAW,EACX,kBAAkB,EAClB,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,eAAe,EAEf,eAAe,EACf,mBAAmB,EACnB,YAAY,EAEZ,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EAEpB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EAEpB,aAAa,EACb,0BAA0B,EAE1B,qBAAqB,EACrB,kBAAkB,EAClB,eAAe,EAEf,uBAAuB,EACvB,aAAa,EACb,kBAAkB,EAClB,eAAe,EACf,wBAAwB,EAExB,kBAAkB,EAClB,oBAAoB,EACpB,cAAc,EACd,cAAc,EACd,mBAAmB,EACnB,4BAA4B,EAC5B,yBAAyB,EACzB,kCAAkC,EAClC,2BAA2B,EAC3B,mBAAmB,EACnB,uBAAuB,EACvB,eAAe,EACf,eAAe,EACf,gBAAgB,EAChB,yBAAyB,EACzB,sBAAsB,GACvB,MAAM,YAAY,CAAC;AAGpB,OAAO,EACL,oBAAoB,EACpB,sBAAsB,EACtB,wBAAwB,EACxB,4BAA4B,GAC7B,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EACV,cAAc,EACd,kBAAkB,EAClB,oBAAoB,EACpB,qBAAqB,EACrB,6BAA6B,EAC7B,qBAAqB,EACrB,sBAAsB,EACtB,iBAAiB,EACjB,qBAAqB,GACtB,MAAM,qBAAqB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -7605,6 +7605,10 @@ import {
|
|
|
7605
7605
|
TypoMutation,
|
|
7606
7606
|
UnsafeResponseDetector
|
|
7607
7607
|
} from "@artemiskit/redteam";
|
|
7608
|
+
import {
|
|
7609
|
+
createWorkflowReport,
|
|
7610
|
+
generateWorkflowReport
|
|
7611
|
+
} from "@artemiskit/reports";
|
|
7608
7612
|
|
|
7609
7613
|
// ../../node_modules/nanoid/index.js
|
|
7610
7614
|
import { webcrypto as crypto } from "node:crypto";
|
|
@@ -7733,6 +7737,12 @@ class ArtemisKit {
|
|
|
7733
7737
|
async runWorkflow(options) {
|
|
7734
7738
|
return (await this.createWorkflowSession(options)).run();
|
|
7735
7739
|
}
|
|
7740
|
+
createWorkflowReport(input) {
|
|
7741
|
+
return createWorkflowReport(input);
|
|
7742
|
+
}
|
|
7743
|
+
generateWorkflowReport(input, options) {
|
|
7744
|
+
return generateWorkflowReport(input, options);
|
|
7745
|
+
}
|
|
7736
7746
|
on(event, handler) {
|
|
7737
7747
|
if (!this.eventHandlers.has(event)) {
|
|
7738
7748
|
this.eventHandlers.set(event, new Set);
|
|
@@ -15273,6 +15283,13 @@ class Guardian {
|
|
|
15273
15283
|
function createGuardian(config = {}) {
|
|
15274
15284
|
return new Guardian(config);
|
|
15275
15285
|
}
|
|
15286
|
+
// src/index.ts
|
|
15287
|
+
import {
|
|
15288
|
+
createWorkflowReport as createWorkflowReport2,
|
|
15289
|
+
generateWorkflowReport as generateWorkflowReport2,
|
|
15290
|
+
renderWorkflowReportHTML,
|
|
15291
|
+
renderWorkflowReportMarkdown
|
|
15292
|
+
} from "@artemiskit/reports";
|
|
15276
15293
|
export {
|
|
15277
15294
|
validatePolicy,
|
|
15278
15295
|
validateAgentWorkflow2 as validateAgentWorkflow,
|
|
@@ -15281,6 +15298,8 @@ export {
|
|
|
15281
15298
|
similarity,
|
|
15282
15299
|
scenario,
|
|
15283
15300
|
runAgentWorkflow,
|
|
15301
|
+
renderWorkflowReportMarkdown,
|
|
15302
|
+
renderWorkflowReportHTML,
|
|
15284
15303
|
regexCase,
|
|
15285
15304
|
regex,
|
|
15286
15305
|
readWorkflowRecord,
|
|
@@ -15312,6 +15331,7 @@ export {
|
|
|
15312
15331
|
getPassedCases,
|
|
15313
15332
|
getFailedCases,
|
|
15314
15333
|
getCasesByTag,
|
|
15334
|
+
generateWorkflowReport2 as generateWorkflowReport,
|
|
15315
15335
|
generatePolicyTemplate,
|
|
15316
15336
|
fuzzy,
|
|
15317
15337
|
filterContent,
|
|
@@ -15327,6 +15347,7 @@ export {
|
|
|
15327
15347
|
defineEvaluator,
|
|
15328
15348
|
defineAdapter,
|
|
15329
15349
|
decideWorkflowOutcome,
|
|
15350
|
+
createWorkflowReport2 as createWorkflowReport,
|
|
15330
15351
|
createSimulatedWorkflowEnvironment,
|
|
15331
15352
|
createSemanticValidator,
|
|
15332
15353
|
createPIIGuardrail,
|
package/dist/types-only.d.ts
CHANGED
|
@@ -29,4 +29,5 @@ export type { ProviderName, ExpectationType, AnyResult, ExtractRunCases, Extract
|
|
|
29
29
|
export type { AdapterContract, EvaluatorContract, StorageContract, AdapterFactory, EvaluatorFactory, StorageFactory, ArtemisKitPlugin, } from './contracts/index.js';
|
|
30
30
|
export type { ScenarioBuilder, TestCaseBuilder } from './builders/index.js';
|
|
31
31
|
export type { AgentWorkflow, WorkflowPolicy, WorkflowJson, WorkflowToolId, WorkflowResource, WorkflowToolDescriptor, SimulatedToolEvidence, SimulatedToolResult, AgentTarget, AgentTurnRequest, AgentTurnResult, AgentTargetCapabilities, AgentTargetFailure, AgentWorkflowExecution, AgentWorkflowReason, AgentWorkflowEvent, AgentWorkflowRecord, AgentWorkflowExecutionRecord, AgentWorkflowEvidence, WorkflowDeterministicStatus, WorkflowDeterministicReason, WorkflowDeterministicAssertion, WorkflowDeterministicSummary, WorkflowJudgeOptions, WorkflowSemanticOptions, WorkflowSemanticReason, WorkflowSemanticAssertion, WorkflowSemanticIdentity, WorkflowSemanticEvaluation, WorkflowOutcomeStatus, WorkflowAssertionCounts, WorkflowOutcomeDecision, WorkflowOutcomeAssessment, SavedWorkflowRecord, HistoricalWorkflowRecord, AgentWorkflowResult, AgentWorkflowSessionOptions, WorkflowCheckpointOptions, WorkflowRecoveryEvidence, WorkflowFault, WorkflowFaults, WorkflowFaultKind, WorkflowRetry, AgentWorkflowSession, WorkflowState, WorkflowEnvironment, WorkflowEnvironmentFactory, WorkflowEnvironmentCleanup, DockerWorkflowEnvironmentOptions, WorkflowEnvironmentInitializationError, } from '@artemiskit/core';
|
|
32
|
+
export type { WorkflowReport, WorkflowReportView, WorkflowReportFormat, WorkflowReportOptions, GenerateWorkflowReportOptions, WorkflowReportFinding, WorkflowReportEvidence, WorkflowReportRow, WorkflowReportSection, } from '@artemiskit/reports';
|
|
32
33
|
//# sourceMappingURL=types-only.d.ts.map
|
package/dist/types-only.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types-only.d.ts","sourceRoot":"","sources":["../src/types-only.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAMH,YAAY,EAEV,gBAAgB,EAChB,kBAAkB,EAClB,UAAU,EACV,cAAc,EACd,aAAa,EAEb,SAAS,EACT,aAAa,EACb,YAAY,EAEZ,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,yBAAyB,EACzB,4BAA4B,EAC5B,0BAA0B,EAE1B,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAE5B,gBAAgB,EAChB,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,cAAc,EACd,eAAe,EACf,SAAS,GACV,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,UAAU,EACV,iBAAiB,EAEjB,WAAW,EACX,SAAS,EACT,UAAU,EAEV,eAAe,EACf,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,eAAe,EAEf,cAAc,EACd,YAAY,EACZ,aAAa,EACb,mBAAmB,EAEnB,WAAW,EACX,cAAc,EACd,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,OAAO,EACP,cAAc,GACf,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,WAAW,EACX,iBAAiB,EAEjB,eAAe,EACf,cAAc,EACd,UAAU,EAEV,WAAW,EAEX,kBAAkB,EAClB,cAAc,EACd,QAAQ,EAER,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,YAAY,GACb,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMrF,YAAY,EACV,cAAc,EACd,aAAa,EACb,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EAEtB,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EACtB,eAAe,EACf,mBAAmB,EACnB,cAAc,EACd,uBAAuB,GACxB,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMxD,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAM7E,YAAY,EAEV,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,sBAAsB,EACtB,gBAAgB,EAChB,iBAAiB,EAEjB,YAAY,EACZ,iBAAiB,EACjB,eAAe,EACf,aAAa,EACb,SAAS,EACT,eAAe,EAEf,UAAU,EACV,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,eAAe,EACf,eAAe,EAEf,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EAEnB,oBAAoB,EACpB,cAAc,EAEd,YAAY,EACZ,OAAO,EACP,WAAW,EACX,kBAAkB,EAClB,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,eAAe,EAEf,eAAe,EACf,mBAAmB,EACnB,YAAY,EAEZ,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EAEpB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EAEpB,aAAa,EACb,0BAA0B,GAC3B,MAAM,YAAY,CAAC;AAMpB,YAAY,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAMpE,YAAY,EAEV,YAAY,EACZ,eAAe,EAEf,SAAS,EACT,eAAe,EACf,mBAAmB,EACnB,oBAAoB,EACpB,eAAe,EAEf,WAAW,EACX,eAAe,EACf,aAAa,EACb,cAAc,EACd,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,aAAa,CAAC;AAMrB,YAAY,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAEnE,YAAY,EACV,aAAa,EACb,cAAc,EACd,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,sBAAsB,EACtB,qBAAqB,EACrB,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,eAAe,EACf,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,kBAAkB,EAClB,mBAAmB,EACnB,4BAA4B,EAC5B,qBAAqB,EACrB,2BAA2B,EAC3B,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAC5B,oBAAoB,EACpB,uBAAuB,EACvB,sBAAsB,EACtB,yBAAyB,EACzB,wBAAwB,EACxB,0BAA0B,EAC1B,qBAAqB,EACrB,uBAAuB,EACvB,uBAAuB,EACvB,yBAAyB,EACzB,mBAAmB,EACnB,wBAAwB,EACxB,mBAAmB,EACnB,2BAA2B,EAC3B,yBAAyB,EACzB,wBAAwB,EACxB,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EACpB,aAAa,EACb,mBAAmB,EACnB,0BAA0B,EAC1B,0BAA0B,EAC1B,gCAAgC,EAChC,sCAAsC,GACvC,MAAM,kBAAkB,CAAC"}
|
|
1
|
+
{"version":3,"file":"types-only.d.ts","sourceRoot":"","sources":["../src/types-only.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAMH,YAAY,EAEV,gBAAgB,EAChB,kBAAkB,EAClB,UAAU,EACV,cAAc,EACd,aAAa,EAEb,SAAS,EACT,aAAa,EACb,YAAY,EAEZ,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,yBAAyB,EACzB,4BAA4B,EAC5B,0BAA0B,EAE1B,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAE5B,gBAAgB,EAChB,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,cAAc,EACd,eAAe,EACf,SAAS,GACV,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,UAAU,EACV,iBAAiB,EAEjB,WAAW,EACX,SAAS,EACT,UAAU,EAEV,eAAe,EACf,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,eAAe,EAEf,cAAc,EACd,YAAY,EACZ,aAAa,EACb,mBAAmB,EAEnB,WAAW,EACX,cAAc,EACd,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,OAAO,EACP,cAAc,GACf,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,WAAW,EACX,iBAAiB,EAEjB,eAAe,EACf,cAAc,EACd,UAAU,EAEV,WAAW,EAEX,kBAAkB,EAClB,cAAc,EACd,QAAQ,EAER,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,YAAY,GACb,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMrF,YAAY,EACV,cAAc,EACd,aAAa,EACb,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EAEtB,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EACtB,eAAe,EACf,mBAAmB,EACnB,cAAc,EACd,uBAAuB,GACxB,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMxD,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAM7E,YAAY,EAEV,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,sBAAsB,EACtB,gBAAgB,EAChB,iBAAiB,EAEjB,YAAY,EACZ,iBAAiB,EACjB,eAAe,EACf,aAAa,EACb,SAAS,EACT,eAAe,EAEf,UAAU,EACV,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,eAAe,EACf,eAAe,EAEf,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EAEnB,oBAAoB,EACpB,cAAc,EAEd,YAAY,EACZ,OAAO,EACP,WAAW,EACX,kBAAkB,EAClB,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,eAAe,EAEf,eAAe,EACf,mBAAmB,EACnB,YAAY,EAEZ,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EAEpB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EAEpB,aAAa,EACb,0BAA0B,GAC3B,MAAM,YAAY,CAAC;AAMpB,YAAY,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAMpE,YAAY,EAEV,YAAY,EACZ,eAAe,EAEf,SAAS,EACT,eAAe,EACf,mBAAmB,EACnB,oBAAoB,EACpB,eAAe,EAEf,WAAW,EACX,eAAe,EACf,aAAa,EACb,cAAc,EACd,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,aAAa,CAAC;AAMrB,YAAY,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAEnE,YAAY,EACV,aAAa,EACb,cAAc,EACd,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,sBAAsB,EACtB,qBAAqB,EACrB,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,eAAe,EACf,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,kBAAkB,EAClB,mBAAmB,EACnB,4BAA4B,EAC5B,qBAAqB,EACrB,2BAA2B,EAC3B,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAC5B,oBAAoB,EACpB,uBAAuB,EACvB,sBAAsB,EACtB,yBAAyB,EACzB,wBAAwB,EACxB,0BAA0B,EAC1B,qBAAqB,EACrB,uBAAuB,EACvB,uBAAuB,EACvB,yBAAyB,EACzB,mBAAmB,EACnB,wBAAwB,EACxB,mBAAmB,EACnB,2BAA2B,EAC3B,yBAAyB,EACzB,wBAAwB,EACxB,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EACpB,aAAa,EACb,mBAAmB,EACnB,0BAA0B,EAC1B,0BAA0B,EAC1B,gCAAgC,EAChC,sCAAsC,GACvC,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,cAAc,EACd,kBAAkB,EAClB,oBAAoB,EACpB,qBAAqB,EACrB,6BAA6B,EAC7B,qBAAqB,EACrB,sBAAsB,EACtB,iBAAiB,EACjB,qBAAqB,GACtB,MAAM,qBAAqB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@artemiskit/sdk",
|
|
3
|
-
"version": "0.6.
|
|
3
|
+
"version": "0.6.3",
|
|
4
4
|
"description": "Programmatic SDK for ArtemisKit LLM evaluation toolkit - integrate into your Node.js apps",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -71,8 +71,9 @@
|
|
|
71
71
|
"test:coverage": "bun test --coverage"
|
|
72
72
|
},
|
|
73
73
|
"dependencies": {
|
|
74
|
-
"@artemiskit/core": "0.6.
|
|
75
|
-
"@artemiskit/redteam": "0.3.
|
|
74
|
+
"@artemiskit/core": "0.6.4",
|
|
75
|
+
"@artemiskit/redteam": "0.3.12",
|
|
76
|
+
"@artemiskit/reports": "0.5.0"
|
|
76
77
|
},
|
|
77
78
|
"devDependencies": {
|
|
78
79
|
"@types/bun": "^1.1.0",
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import { describe, expect, test } from 'bun:test';
|
|
2
|
+
import { createAgentWorkflowSession, validateAgentWorkflow } from '@artemiskit/core';
|
|
3
|
+
import {
|
|
4
|
+
ArtemisKit,
|
|
5
|
+
createWorkflowReport,
|
|
6
|
+
generateWorkflowReport,
|
|
7
|
+
renderWorkflowReportHTML,
|
|
8
|
+
renderWorkflowReportMarkdown,
|
|
9
|
+
} from '../index';
|
|
10
|
+
|
|
11
|
+
async function evidence() {
|
|
12
|
+
const workflow = validateAgentWorkflow({
|
|
13
|
+
version: '1',
|
|
14
|
+
kind: 'agent_workflow',
|
|
15
|
+
name: 'report-only',
|
|
16
|
+
target: { provider: 'openai', model: 'fixture' },
|
|
17
|
+
environment: {
|
|
18
|
+
type: 'simulated',
|
|
19
|
+
policy: {
|
|
20
|
+
network: 'denied',
|
|
21
|
+
side_effects: 'denied',
|
|
22
|
+
permissions: { workflow_state: 'read' },
|
|
23
|
+
budgets: { max_actions: 2, timeout_ms: 1000 },
|
|
24
|
+
},
|
|
25
|
+
},
|
|
26
|
+
tools: ['get_workflow_state'],
|
|
27
|
+
workflow: {
|
|
28
|
+
system_instructions: 'Fixture',
|
|
29
|
+
initial_state: { workflow_state: { ready: true } },
|
|
30
|
+
turns: [{ role: 'user', content: 'Do not persist private prose.' }],
|
|
31
|
+
},
|
|
32
|
+
outcomes: { deterministic: [{ type: 'workflow_state', path: 'ready', equals: true }] },
|
|
33
|
+
evidence: { trace: 'summary', artifacts: 'checksums', redact: true },
|
|
34
|
+
});
|
|
35
|
+
return (
|
|
36
|
+
await createAgentWorkflowSession({
|
|
37
|
+
workflow,
|
|
38
|
+
target: {
|
|
39
|
+
provider: 'openai',
|
|
40
|
+
capabilities: async () => ({
|
|
41
|
+
status: 'available',
|
|
42
|
+
toolUse: true,
|
|
43
|
+
transportCancellation: true,
|
|
44
|
+
}),
|
|
45
|
+
turn: async () => ({
|
|
46
|
+
status: 'completed',
|
|
47
|
+
id: 'fixture',
|
|
48
|
+
model: 'fixture',
|
|
49
|
+
message: { role: 'assistant', content: 'Do not persist private prose.' },
|
|
50
|
+
tokens: { prompt: 1, completion: 1, total: 2 },
|
|
51
|
+
latencyMs: 1,
|
|
52
|
+
}),
|
|
53
|
+
},
|
|
54
|
+
}).run()
|
|
55
|
+
).record;
|
|
56
|
+
}
|
|
57
|
+
describe('SDK offline workflow reports', () => {
|
|
58
|
+
test('top-level and class surfaces share canonical model and both deterministic renderers', async () => {
|
|
59
|
+
const record = await evidence();
|
|
60
|
+
expect(record.taskVerification).toBe('passed');
|
|
61
|
+
const kit = new ArtemisKit({
|
|
62
|
+
provider: 'openai',
|
|
63
|
+
providerConfig: { apiKey: 'not-a-credential' },
|
|
64
|
+
});
|
|
65
|
+
const fetch = globalThis.fetch;
|
|
66
|
+
globalThis.fetch = (() => {
|
|
67
|
+
throw new Error('Reporting attempted network');
|
|
68
|
+
}) as typeof fetch;
|
|
69
|
+
try {
|
|
70
|
+
const before = JSON.stringify(record);
|
|
71
|
+
const model = createWorkflowReport(record);
|
|
72
|
+
expect(kit.createWorkflowReport(record)).toEqual(model);
|
|
73
|
+
for (const view of ['technical', 'executive', 'comprehensive'] as const) {
|
|
74
|
+
for (const format of ['html', 'markdown'] as const) {
|
|
75
|
+
const text = generateWorkflowReport(record, { view, format });
|
|
76
|
+
expect(kit.generateWorkflowReport(record, { view, format })).toBe(text);
|
|
77
|
+
expect(text).toBe(
|
|
78
|
+
format === 'html'
|
|
79
|
+
? renderWorkflowReportHTML(model, { view })
|
|
80
|
+
: renderWorkflowReportMarkdown(model, { view })
|
|
81
|
+
);
|
|
82
|
+
expect(text).not.toContain('private prose');
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
expect(JSON.stringify(record)).toBe(before);
|
|
86
|
+
expect(() => kit.createWorkflowReport({ schemaVersion: '99' })).toThrow();
|
|
87
|
+
expect(() => generateWorkflowReport(record, { format: 'pdf' } as never)).toThrow(
|
|
88
|
+
'Invalid workflow report options'
|
|
89
|
+
);
|
|
90
|
+
for (const options of [{ format: null }, { view: null }, { unexpected: true }]) {
|
|
91
|
+
expect(() => generateWorkflowReport(record, options as never)).toThrow(
|
|
92
|
+
'Invalid workflow report options'
|
|
93
|
+
);
|
|
94
|
+
}
|
|
95
|
+
let accessed = false;
|
|
96
|
+
expect(() =>
|
|
97
|
+
generateWorkflowReport(record, {
|
|
98
|
+
get view() {
|
|
99
|
+
accessed = true;
|
|
100
|
+
throw new Error('sensitive');
|
|
101
|
+
},
|
|
102
|
+
} as never)
|
|
103
|
+
).toThrow('Invalid workflow report options');
|
|
104
|
+
expect(accessed).toBe(false);
|
|
105
|
+
} finally {
|
|
106
|
+
globalThis.fetch = fetch;
|
|
107
|
+
}
|
|
108
|
+
});
|
|
109
|
+
});
|
package/src/artemiskit.ts
CHANGED
|
@@ -40,6 +40,11 @@ import {
|
|
|
40
40
|
TypoMutation,
|
|
41
41
|
UnsafeResponseDetector,
|
|
42
42
|
} from '@artemiskit/redteam';
|
|
43
|
+
import {
|
|
44
|
+
type GenerateWorkflowReportOptions,
|
|
45
|
+
createWorkflowReport,
|
|
46
|
+
generateWorkflowReport,
|
|
47
|
+
} from '@artemiskit/reports';
|
|
43
48
|
import { nanoid } from 'nanoid';
|
|
44
49
|
import { prepareWorkflowSession } from './workflow';
|
|
45
50
|
|
|
@@ -132,11 +137,21 @@ export class ArtemisKit {
|
|
|
132
137
|
return prepareWorkflowSession(options, this.config);
|
|
133
138
|
}
|
|
134
139
|
|
|
135
|
-
/** Execute a workflow
|
|
140
|
+
/** Execute a workflow and independently evaluate its declared outcomes. */
|
|
136
141
|
async runWorkflow(options: WorkflowRunOptions) {
|
|
137
142
|
return (await this.createWorkflowSession(options)).run();
|
|
138
143
|
}
|
|
139
144
|
|
|
145
|
+
/** Project saved evidence without running a workflow or loading provider configuration. */
|
|
146
|
+
createWorkflowReport(input: unknown) {
|
|
147
|
+
return createWorkflowReport(input);
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/** Generate HTML or Markdown offline; task failures remain visible in a successful export. */
|
|
151
|
+
generateWorkflowReport(input: unknown, options?: GenerateWorkflowReportOptions) {
|
|
152
|
+
return generateWorkflowReport(input, options);
|
|
153
|
+
}
|
|
154
|
+
|
|
140
155
|
// ==========================================================================
|
|
141
156
|
// Event Emitter Methods
|
|
142
157
|
// ==========================================================================
|
package/src/index.ts
CHANGED
|
@@ -487,3 +487,22 @@ export type {
|
|
|
487
487
|
MultiTurnValidationResult,
|
|
488
488
|
ValidateMessageOptions,
|
|
489
489
|
} from './guardian';
|
|
490
|
+
|
|
491
|
+
// Offline reports never initialize target or judge clients.
|
|
492
|
+
export {
|
|
493
|
+
createWorkflowReport,
|
|
494
|
+
generateWorkflowReport,
|
|
495
|
+
renderWorkflowReportHTML,
|
|
496
|
+
renderWorkflowReportMarkdown,
|
|
497
|
+
} from '@artemiskit/reports';
|
|
498
|
+
export type {
|
|
499
|
+
WorkflowReport,
|
|
500
|
+
WorkflowReportView,
|
|
501
|
+
WorkflowReportFormat,
|
|
502
|
+
WorkflowReportOptions,
|
|
503
|
+
GenerateWorkflowReportOptions,
|
|
504
|
+
WorkflowReportFinding,
|
|
505
|
+
WorkflowReportEvidence,
|
|
506
|
+
WorkflowReportRow,
|
|
507
|
+
WorkflowReportSection,
|
|
508
|
+
} from '@artemiskit/reports';
|
package/src/types-only.ts
CHANGED
|
@@ -328,3 +328,14 @@ export type {
|
|
|
328
328
|
DockerWorkflowEnvironmentOptions,
|
|
329
329
|
WorkflowEnvironmentInitializationError,
|
|
330
330
|
} from '@artemiskit/core';
|
|
331
|
+
export type {
|
|
332
|
+
WorkflowReport,
|
|
333
|
+
WorkflowReportView,
|
|
334
|
+
WorkflowReportFormat,
|
|
335
|
+
WorkflowReportOptions,
|
|
336
|
+
GenerateWorkflowReportOptions,
|
|
337
|
+
WorkflowReportFinding,
|
|
338
|
+
WorkflowReportEvidence,
|
|
339
|
+
WorkflowReportRow,
|
|
340
|
+
WorkflowReportSection,
|
|
341
|
+
} from '@artemiskit/reports';
|