@artemiskit/core 0.1.5 → 0.2.0

package/dist/events/types.d.ts

@@ -0,0 +1,177 @@
+/**
+ * Event types for observability integration
+ *
+ * These events are emitted by the core runner and can be consumed by
+ * the @artemiskit/observability package for metrics, traces, and hooks.
+ */
+import type { TokenUsage } from '../adapters/types';
+/**
+ * Emitted when a test case starts execution
+ */
+export interface TestStartEvent {
+    type: 'test:start';
+    timestamp: number;
+    testId: string;
+    testName?: string;
+    scenarioName: string;
+    tags: string[];
+}
+/**
+ * Emitted when a test case completes (success or failure)
+ */
+export interface TestCompleteEvent {
+    type: 'test:complete';
+    timestamp: number;
+    testId: string;
+    testName?: string;
+    scenarioName: string;
+    passed: boolean;
+    score: number;
+    durationMs: number;
+    evaluatorType: string;
+    reason?: string;
+    error?: string;
+    tags: string[];
+    tokens: TokenUsage;
+}
+/**
+ * Emitted when a scenario starts execution
+ */
+export interface ScenarioStartEvent {
+    type: 'scenario:start';
+    timestamp: number;
+    scenarioName: string;
+    testCount: number;
+    provider: string;
+    model?: string;
+}
+/**
+ * Emitted when a scenario completes
+ */
+export interface ScenarioCompleteEvent {
+    type: 'scenario:complete';
+    timestamp: number;
+    scenarioName: string;
+    durationMs: number;
+    passed: number;
+    failed: number;
+    skipped: number;
+    passRate: number;
+    totalTokens: number;
+    provider: string;
+    model?: string;
+}
+/**
+ * Emitted before a provider/LLM API call
+ */
+export interface ProviderCallStartEvent {
+    type: 'provider:call:start';
+    timestamp: number;
+    provider: string;
+    model: string;
+    testId: string;
+}
+/**
+ * Emitted after a provider/LLM API call completes
+ */
+export interface ProviderCallCompleteEvent {
+    type: 'provider:call:complete';
+    timestamp: number;
+    provider: string;
+    model: string;
+    testId: string;
+    latencyMs: number;
+    tokens: TokenUsage;
+    success: boolean;
+    error?: string;
+    finishReason?: string;
+}
+/**
+ * Emitted before an evaluator runs
+ */
+export interface EvaluatorStartEvent {
+    type: 'evaluator:start';
+    timestamp: number;
+    evaluatorType: string;
+    testId: string;
+    expected: unknown;
+}
+/**
+ * Emitted after an evaluator completes
+ */
+export interface EvaluatorCompleteEvent {
+    type: 'evaluator:complete';
+    timestamp: number;
+    evaluatorType: string;
+    testId: string;
+    passed: boolean;
+    score: number;
+    durationMs: number;
+    reason?: string;
+}
+/**
+ * Emitted when a run starts (can include multiple scenarios)
+ */
+export interface RunStartEvent {
+    type: 'run:start';
+    timestamp: number;
+    runId: string;
+    project: string;
+    command: 'run' | 'stress' | 'redteam';
+    scenarioCount: number;
+}
+/**
+ * Emitted when a run completes
+ */
+export interface RunCompleteEvent {
+    type: 'run:complete';
+    timestamp: number;
+    runId: string;
+    project: string;
+    command: 'run' | 'stress' | 'redteam';
+    durationMs: number;
+    totalTests: number;
+    passed: number;
+    failed: number;
+    success: boolean;
+}
+/**
+ * Emitted when an error occurs
+ */
+export interface ErrorEvent {
+    type: 'error';
+    timestamp: number;
+    error: Error;
+    source: 'test' | 'scenario' | 'provider' | 'evaluator' | 'runner';
+    context?: {
+        testId?: string;
+        scenarioName?: string;
+        provider?: string;
+        evaluatorType?: string;
+    };
+}
+/**
+ * All possible event types
+ */
+export type ArtemisEvent = TestStartEvent | TestCompleteEvent | ScenarioStartEvent | ScenarioCompleteEvent | ProviderCallStartEvent | ProviderCallCompleteEvent | EvaluatorStartEvent | EvaluatorCompleteEvent | RunStartEvent | RunCompleteEvent | ErrorEvent;
+/**
+ * Event type string literals
+ */
+export type ArtemisEventType = ArtemisEvent['type'];
+/**
+ * Map of event types to their payloads
+ */
+export interface ArtemisEventMap {
+    'test:start': TestStartEvent;
+    'test:complete': TestCompleteEvent;
+    'scenario:start': ScenarioStartEvent;
+    'scenario:complete': ScenarioCompleteEvent;
+    'provider:call:start': ProviderCallStartEvent;
+    'provider:call:complete': ProviderCallCompleteEvent;
+    'evaluator:start': EvaluatorStartEvent;
+    'evaluator:complete': EvaluatorCompleteEvent;
+    'run:start': RunStartEvent;
+    'run:complete': RunCompleteEvent;
+    error: ErrorEvent;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/events/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/events/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAMpD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,YAAY,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,EAAE,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,eAAe,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,MAAM,EAAE,UAAU,CAAC;CACpB;AAMD;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,gBAAgB,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,IAAI,EAAE,mBAAmB,CAAC;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAMD;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,qBAAqB,CAAC;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,IAAI,EAAE,wBAAwB,CAAC;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,UAAU,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAMD;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,iBAAiB,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,oBAAoB,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAMD;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,WAAW,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,KAAK,GAAG,QAAQ,GAAG,SAAS,CAAC;IACtC,aAAa,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,cAAc,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,KAAK,GAAG,QAAQ,GAAG,SAAS,CAAC;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;CAClB;AAMD;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,OAAO,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,KAAK,CAAC;IACb,MAAM,EAAE,MAAM,GAAG,UAAU,GAAG,UAAU,GAAG,WAAW,GAAG,QAAQ,CAAC;IAClE,OAAO,CAAC,EAAE;QACR,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,aAAa,CAAC,EAAE,MAAM,CAAC;KACxB,CAAC;CACH;AAMD;;GAEG;AACH,MAAM,MAAM,YAAY,GACpB,cAAc,GACd,iBAAiB,GACjB,kBAAkB,GAClB,qBAAqB,GACrB,sBAAsB,GACtB,yBAAyB,GACzB,mBAAmB,GACnB,sBAAsB,GACtB,aAAa,GACb,gBAAgB,GAChB,UAAU,CAAC;AAEf;;GAEG;AACH,MAAM,MAAM,gBAAgB,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;AAEpD;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,YAAY,EAAE,cAAc,CAAC;IAC7B,eAAe,EAAE,iBAAiB,CAAC;IACnC,gBAAgB,EAAE,kBAAkB,CAAC;IACrC,mBAAmB,EAAE,qBAAqB,CAAC;IAC3C,qBAAqB,EAAE,sBAAsB,CAAC;IAC9C,wBAAwB,EAAE,yBAAyB,CAAC;IACpD,iBAAiB,EAAE,mBAAmB,CAAC;IACvC,oBAAoB,EAAE,sBAAsB,CAAC;IAC7C,WAAW,EAAE,aAAa,CAAC;IAC3B,cAAc,EAAE,gBAAgB,CAAC;IACjC,KAAK,EAAE,UAAU,CAAC;CACnB"}

package/dist/index.d.ts

@@ -11,4 +11,5 @@ export * from './artifacts';
 export * from './provenance';
 export * from './utils';
 export * from './redaction';
+export * from './cost';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,cAAc,YAAY,CAAC;AAG3B,cAAc,YAAY,CAAC;AAG3B,cAAc,cAAc,CAAC;AAG7B,cAAc,UAAU,CAAC;AAGzB,cAAc,WAAW,CAAC;AAG1B,cAAc,aAAa,CAAC;AAG5B,cAAc,cAAc,CAAC;AAG7B,cAAc,SAAS,CAAC;AAGxB,cAAc,aAAa,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,cAAc,YAAY,CAAC;AAG3B,cAAc,YAAY,CAAC;AAG3B,cAAc,cAAc,CAAC;AAG7B,cAAc,UAAU,CAAC;AAGzB,cAAc,WAAW,CAAC;AAG1B,cAAc,aAAa,CAAC;AAG5B,cAAc,cAAc,CAAC;AAG7B,cAAc,SAAS,CAAC;AAGxB,cAAc,aAAa,CAAC;AAG5B,cAAc,QAAQ,CAAC"}