@artemiskit/sdk 0.3.0 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +225 -0
- package/README.md +44 -1
- package/dist/artemiskit.d.ts +149 -0
- package/dist/artemiskit.d.ts.map +1 -0
- package/dist/builders/index.d.ts +350 -0
- package/dist/builders/index.d.ts.map +1 -0
- package/dist/builders/index.js +398 -0
- package/dist/contracts/index.d.ts +253 -0
- package/dist/contracts/index.d.ts.map +1 -0
- package/dist/contracts/index.js +99 -0
- package/dist/guardian/action-validator.d.ts +92 -0
- package/dist/guardian/action-validator.d.ts.map +1 -0
- package/dist/guardian/circuit-breaker.d.ts +211 -0
- package/dist/guardian/circuit-breaker.d.ts.map +1 -0
- package/dist/guardian/guardian.d.ts +213 -0
- package/dist/guardian/guardian.d.ts.map +1 -0
- package/dist/guardian/guardrails.d.ts +107 -0
- package/dist/guardian/guardrails.d.ts.map +1 -0
- package/dist/guardian/index.d.ts +35 -0
- package/dist/guardian/index.d.ts.map +1 -0
- package/dist/guardian/intent-classifier.d.ts +81 -0
- package/dist/guardian/intent-classifier.d.ts.map +1 -0
- package/dist/guardian/interceptor.d.ts +141 -0
- package/dist/guardian/interceptor.d.ts.map +1 -0
- package/dist/guardian/policy.d.ts +52 -0
- package/dist/guardian/policy.d.ts.map +1 -0
- package/dist/guardian/semantic-validator.d.ts +61 -0
- package/dist/guardian/semantic-validator.d.ts.map +1 -0
- package/dist/guardian/types.d.ts +653 -0
- package/dist/guardian/types.d.ts.map +1 -0
- package/dist/index.d.ts +23 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +24847 -13653
- package/dist/matchers/core.d.ts +81 -0
- package/dist/matchers/core.d.ts.map +1 -0
- package/dist/matchers/index.d.ts +6 -0
- package/dist/matchers/index.d.ts.map +1 -0
- package/dist/matchers/jest.d.ts +46 -0
- package/dist/matchers/jest.d.ts.map +1 -0
- package/dist/matchers/vitest.d.ts +24 -0
- package/dist/matchers/vitest.d.ts.map +1 -0
- package/dist/types-only.d.ts +31 -0
- package/dist/types-only.d.ts.map +1 -0
- package/dist/types-only.js +1 -0
- package/dist/types.d.ts +377 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/utils/index.d.ts +112 -0
- package/dist/utils/index.d.ts.map +1 -0
- package/dist/utils/index.js +104 -0
- package/package.json +19 -4
- package/src/__tests__/builders.test.ts +445 -0
- package/src/__tests__/circuit-breaker.test.ts +704 -0
- package/src/__tests__/compare.test.ts +469 -0
- package/src/__tests__/contracts.test.ts +354 -0
- package/src/__tests__/guardian-modes.test.ts +215 -0
- package/src/__tests__/guardrails.test.ts +404 -0
- package/src/__tests__/semantic-validator.test.ts +421 -0
- package/src/__tests__/utils.test.ts +324 -0
- package/src/__tests__/validate.test.ts +307 -0
- package/src/artemiskit.ts +478 -5
- package/src/builders/index.ts +698 -0
- package/src/contracts/index.ts +357 -0
- package/src/guardian/circuit-breaker.ts +74 -28
- package/src/guardian/guardian.ts +307 -20
- package/src/guardian/guardrails.ts +143 -1
- package/src/guardian/index.ts +35 -1
- package/src/guardian/interceptor.ts +87 -11
- package/src/guardian/semantic-validator.ts +399 -0
- package/src/guardian/types.ts +413 -2
- package/src/index.ts +258 -14
- package/src/types-only.ts +276 -0
- package/src/types.ts +183 -0
- package/src/utils/index.ts +225 -0
- package/tsconfig.json +2 -1
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @artemiskit/sdk
|
|
3
|
+
* Programmatic SDK for ArtemisKit LLM evaluation toolkit
|
|
4
|
+
*/
|
|
5
|
+
export { ArtemisKit } from './artemiskit';
|
|
6
|
+
export type { ArtemisKitConfig, RunOptions, RedTeamOptions, StressOptions, RunResult, RedTeamResult, StressResult, CaseStartEvent, CaseCompleteEvent, ProgressEvent, RedTeamMutationStartEvent, RedTeamMutationCompleteEvent, StressRequestCompleteEvent, CaseStartHandler, CaseCompleteHandler, ProgressHandler, RedTeamMutationStartHandler, RedTeamMutationCompleteHandler, StressRequestCompleteHandler, ArtemisKitEvents, ArtemisKitEventName, ValidateOptions, ValidationResult, ValidationError, ValidationWarning, ScenarioValidation, CompareOptions, CompareResult, RunSummary, ComparisonDetails, } from './types';
|
|
7
|
+
export type { Scenario, TestCase, Expected, Provider, ProviderConfig, ChatMessageType, Variables, } from '@artemiskit/core';
|
|
8
|
+
export type { CaseResult, CaseRedactionInfo, RunManifest, RunConfig, RunMetrics, RedTeamManifest, RedTeamConfig, RedTeamMetrics, RedTeamCaseResult, RedTeamStatus, RedTeamSeverity, StressManifest, StressConfig, StressMetrics, StressRequestResult, AnyManifest, ResolvedConfig, ConfigSource, ManifestRedactionInfo, CostEstimateInfo, GitInfo, ProvenanceInfo, } from '@artemiskit/core';
|
|
9
|
+
export type { ModelClient, ModelCapabilities, GenerateOptions, GenerateResult, TokenUsage, ChatMessage, FunctionDefinition, ToolDefinition, ToolCall, AdapterConfig, BaseAdapterConfig, OpenAIAdapterConfig, AzureOpenAIAdapterConfig, VercelAIAdapterConfig, AnthropicAdapterConfig, LangChainAdapterConfig, DeepAgentsAdapterConfig, ProviderType, } from '@artemiskit/core';
|
|
10
|
+
export type { Evaluator, EvaluatorContext, EvaluatorResult } from '@artemiskit/core';
|
|
11
|
+
export type { StorageAdapter, StorageConfig, RunListItem, ListOptions, ComparisonResult, BaselineMetadata, BaselineStorageAdapter, CaseResultRecord, CaseResultStatus, CaseResultQueryOptions, MetricsSnapshot, MetricsTrendOptions, TrendDataPoint, AnalyticsStorageAdapter, } from '@artemiskit/core';
|
|
12
|
+
export type { RedactionConfig } from '@artemiskit/core';
|
|
13
|
+
export { isRunManifest, isRedTeamManifest, isStressManifest } from '@artemiskit/core';
|
|
14
|
+
export type { Severity, SeverityInfo, CvssScore } from '@artemiskit/redteam';
|
|
15
|
+
export { artemiskitMatchers, type ArtemisKitMatchers, type MatcherResult } from './matchers';
|
|
16
|
+
export { isRunResult, isRedTeamResult, isStressResult, isRunManifestType, isRedTeamManifestType, isStressManifestType, assertDefined, assert, getFailedCases, getPassedCases, getCasesByTag, calculateSuccessRate, } from './utils';
|
|
17
|
+
export type { ProviderName, ExpectationType, AnyResult, ExtractRunCases, ExtractRedTeamCases, ExtractStressResults, ExtractManifest, DeepPartial, PartialScenario, RequireFields, OptionalFields, StrictAdapterConfig, } from './utils';
|
|
18
|
+
export { defineAdapter, defineEvaluator, defineStorage, definePlugin, } from './contracts';
|
|
19
|
+
export type { AdapterContract, EvaluatorContract, StorageContract, AdapterFactory, EvaluatorFactory, StorageFactory, ArtemisKitPlugin, } from './contracts';
|
|
20
|
+
export { ScenarioBuilder, TestCaseBuilder, scenario, testCase, containsCase, exactCase, regexCase, jsonCase, gradedCase, exact, contains, notContains, regex, fuzzy, jsonSchema, llmGrade, similarity, inline, allOf, anyOf, } from './builders';
|
|
21
|
+
export { Guardian, createGuardian, normalizeGuardianMode, SemanticValidator, createSemanticValidator, GuardianInterceptor, GuardianBlockedError, createInterceptor, ActionValidator, createDefaultActionValidator, IntentClassifier, createIntentClassifier, detectInjection, createInjectionGuardrail, detectPII, createPIIGuardrail, filterContent, createContentFilterGuardrail, createGuardrails, matchPattern, createCustomPatternGuardrail, loadPolicy, parsePolicy, validatePolicy, createDefaultPolicy, mergePolicies, generatePolicyTemplate, PolicyLoadError, PolicyValidationError, CircuitBreaker, MetricsCollector, RateLimiter, } from './guardian';
|
|
22
|
+
export type { GuardianConfig, InterceptorConfig, ActionValidatorConfig, IntentClassifierConfig, GuardrailsConfig, RateLimiterConfig, CustomPatternOptions, GuardianMode, ViolationSeverity, ViolationAction, GuardrailType, Violation, GuardrailResult, PolicyRule, PolicyCondition, GuardianPolicy, CircuitBreakerConfig, RateLimitConfig, CostLimitConfig, ActionDefinition, ActionParameter, ParameterValidation, IntentClassification, IntentCategory, PIIDetection, PIIType, PIILocation, InjectionDetection, InjectionType, ContentFilterResult, ContentFlag, ContentCategory, GuardianMetrics, CircuitBreakerState, CostTracking, GuardianEventType, GuardianEvent, GuardianEventHandler, InterceptedRequest, InterceptedResponse, InterceptedToolCall, InterceptedAgentStep, FrameworkType, FrameworkIntegrationConfig, GuardianModeCanonical, GuardianModeLegacy, GuardianModeAll, ContentValidationConfig, PatternConfig, ValidationCategory, PatternCategory, SemanticValidationResult, SessionStorageType, SessionStorageConfig, SessionMessage, SessionMetrics, ConversationSession, TrustBuildingHeuristicConfig, EscalationHeuristicConfig, ContextManipulationHeuristicConfig, SplitPayloadHeuristicConfig, MultiTurnHeuristics, MultiTurnSemanticConfig, MultiTurnConfig, HeuristicResult, HeuristicResults, MultiTurnValidationResult, ValidateMessageOptions, } from './guardian';
|
|
23
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAG1C,YAAY,EAEV,gBAAgB,EAChB,UAAU,EACV,cAAc,EACd,aAAa,EAEb,SAAS,EACT,aAAa,EACb,YAAY,EAEZ,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,yBAAyB,EACzB,4BAA4B,EAC5B,0BAA0B,EAE1B,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAE5B,gBAAgB,EAChB,mBAAmB,EAEnB,eAAe,EACf,gBAAgB,EAChB,eAAe,EACf,iBAAiB,EACjB,kBAAkB,EAElB,cAAc,EACd,aAAa,EACb,UAAU,EACV,iBAAiB,GAClB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,cAAc,EACd,eAAe,EACf,SAAS,GACV,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,UAAU,EACV,iBAAiB,EAEjB,WAAW,EACX,SAAS,EACT,UAAU,EAEV,eAAe,EACf,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,eAAe,EAEf,cAAc,EACd,YAAY,EACZ,aAAa,EACb,mBAAmB,EAEnB,WAAW,EACX,cAAc,EACd,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,OAAO,EACP,cAAc,GACf,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,WAAW,EACX,iBAAiB,EAEjB,eAAe,EACf,cAAc,EACd,UAAU,EAEV,WAAW,EAEX,kBAAkB,EAClB,cAAc,EACd,QAAQ,EAER,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,YAAY,GACb,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMrF,YAAY,EACV,cAAc,EACd,aAAa,EACb,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EAEtB,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EACtB,eAAe,EACf,mBAAmB,EACnB,cAAc,EACd,uBAAuB,GACxB,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMxD,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAMtF,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAM7E,OAAO,EAAE,kBAAkB,EAAE,KAAK,kBAAkB,EAAE,KAAK,aAAa,EAAE,MAAM,YAAY,CAAC;AAM7F,OAAO,EAEL,WAAW,EACX,eAAe,EACf,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,oBAAoB,EAEpB,aAAa,EACb,MAAM,EAEN,cAAc,EACd,cAAc,EACd,aAAa,EACb,oBAAoB,GACrB,MAAM,SAAS,CAAC;AAEjB,YAAY,EAEV,YAAY,EACZ,eAAe,EAEf,SAAS,EACT,eAAe,EACf,mBAAmB,EACnB,oBAAoB,EACpB,eAAe,EAEf,WAAW,EACX,eAAe,EACf,aAAa,EACb,cAAc,EACd,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,OAAO,EAEL,aAAa,EAEb,eAAe,EAEf,aAAa,EAEb,YAAY,GACb,MAAM,aAAa,CAAC;AAErB,YAAY,EACV,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,aAAa,CAAC;AAMrB,OAAO,EAEL,eAAe,EACf,eAAe,EAEf,QAAQ,EACR,QAAQ,EAER,YAAY,EACZ,SAAS,EACT,SAAS,EACT,QAAQ,EACR,UAAU,EAEV,KAAK,EACL,QAAQ,EACR,WAAW,EACX,KAAK,EACL,KAAK,EACL,UAAU,EACV,QAAQ,EACR,UAAU,EACV,MAAM,EACN,KAAK,EACL,KAAK,GACN,MAAM,YAAY,CAAC;AAMpB,OAAO,EAEL,QAAQ,EACR,cAAc,EAEd,qBAAqB,EAErB,iBAAiB,EACjB,uBAAuB,EAEvB,mBAAmB,EACnB,oBAAoB,EACpB,iBAAiB,EAEjB,eAAe,EACf,4BAA4B,EAE5B,gBAAgB,EAChB,sBAAsB,EAEtB,eAAe,EACf,wBAAwB,EACxB,SAAS,EACT,kBAAkB,EAClB,aAAa,EACb,4BAA4B,EAC5B,gBAAgB,EAEhB,YAAY,EACZ,4BAA4B,EAE5B,UAAU,EACV,WAAW,EACX,cAAc,EACd,mBAAmB,EACnB,aAAa,EACb,sBAAsB,EACtB,eAAe,EACf,qBAAqB,EAErB,cAAc,EACd,gBAAgB,EAChB,WAAW,GACZ,MAAM,YAAY,CAAC;AAGpB,YAAY,EAEV,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,sBAAsB,EACtB,gBAAgB,EAChB,iBAAiB,EACjB,oBAAoB,EAEpB,YAAY,EACZ,iBAAiB,EACjB,eAAe,EACf,aAAa,EACb,SAAS,EACT,eAAe,EAEf,UAAU,EACV,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,eAAe,EACf,eAAe,EAEf,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EAEnB,oBAAoB,EACpB,cAAc,EAEd,YAAY,EACZ,OAAO,EACP,WAAW,EACX,kBAAkB,EAClB,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,eAAe,EAEf,eAAe,EACf,mBAAmB,EACnB,YAAY,EAEZ,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EAEpB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EAEpB,aAAa,EACb,0BAA0B,EAE1B,qBAAqB,EACrB,kBAAkB,EAClB,eAAe,EAEf,uBAAuB,EACvB,aAAa,EACb,kBAAkB,EAClB,eAAe,EACf,wBAAwB,EAExB,kBAAkB,EAClB,oBAAoB,EACpB,cAAc,EACd,cAAc,EACd,mBAAmB,EACnB,4BAA4B,EAC5B,yBAAyB,EACzB,kCAAkC,EAClC,2BAA2B,EAC3B,mBAAmB,EACnB,uBAAuB,EACvB,eAAe,EACf,eAAe,EACf,gBAAgB,EAChB,yBAAyB,EACzB,sBAAsB,GACvB,MAAM,YAAY,CAAC"}
|