@artemiskit/sdk 0.3.0

package/dist/matchers/vitest.js

@@ -0,0 +1,257 @@
+// @bun
+var __create = Object.create;
+var __getProtoOf = Object.getPrototypeOf;
+var __defProp = Object.defineProperty;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+function __accessProp(key) {
+  return this[key];
+}
+var __toESMCache_node;
+var __toESMCache_esm;
+var __toESM = (mod, isNodeMode, target) => {
+  var canCache = mod != null && typeof mod === "object";
+  if (canCache) {
+    var cache = isNodeMode ? __toESMCache_node ??= new WeakMap : __toESMCache_esm ??= new WeakMap;
+    var cached = cache.get(mod);
+    if (cached)
+      return cached;
+  }
+  target = mod != null ? __create(__getProtoOf(mod)) : {};
+  const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
+  for (let key of __getOwnPropNames(mod))
+    if (!__hasOwnProp.call(to, key))
+      __defProp(to, key, {
+        get: __accessProp.bind(mod, key),
+        enumerable: true
+      });
+  if (canCache)
+    cache.set(mod, to);
+  return to;
+};
+var __commonJS = (cb, mod) => () => (mod || cb((mod = { exports: {} }).exports, mod), mod.exports);
+var __returnValue = (v) => v;
+function __exportSetter(name, newValue) {
+  this[name] = __returnValue.bind(null, newValue);
+}
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, {
+      get: all[name],
+      enumerable: true,
+      configurable: true,
+      set: __exportSetter.bind(all, name)
+    });
+};
+var __require = import.meta.require;
+
+// src/matchers/core.ts
+function formatManifestSummary(manifest) {
+  const { metrics } = manifest;
+  return `
+  Scenario: ${manifest.config.scenario}
+  Total Cases: ${metrics.total_cases}
+  Passed: ${metrics.passed_cases}
+  Failed: ${metrics.failed_cases}
+  Success Rate: ${(metrics.success_rate * 100).toFixed(1)}%
+`;
+}
+function formatFailedCases(cases) {
+  const failed = cases.filter((c) => !c.ok);
+  if (failed.length === 0)
+    return "";
+  return `
+  Failed Cases:
+${failed.slice(0, 5).map((c) => `    - ${c.name ?? c.id}: ${c.reason ?? "No reason provided"}`).join(`
+`)}${failed.length > 5 ? `
+    ... and ${failed.length - 5} more` : ""}
+`;
+}
+function toPassAllCases(result) {
+  const pass = result.success;
+  return {
+    pass,
+    message: () => pass ? `Expected test run to fail, but all ${result.manifest.metrics.total_cases} cases passed` : `Expected test run to pass, but ${result.manifest.metrics.failed_cases} out of ${result.manifest.metrics.total_cases} cases failed
+${formatManifestSummary(result.manifest)}${formatFailedCases(result.cases)}`
+  };
+}
+function toHaveSuccessRate(result, expectedRate) {
+  const actualRate = result.manifest.metrics.success_rate;
+  const pass = actualRate >= expectedRate;
+  return {
+    pass,
+    message: () => pass ? `Expected success rate to be less than ${(expectedRate * 100).toFixed(1)}%, but got ${(actualRate * 100).toFixed(1)}%` : `Expected success rate to be at least ${(expectedRate * 100).toFixed(1)}%, but got ${(actualRate * 100).toFixed(1)}%
+${formatManifestSummary(result.manifest)}${formatFailedCases(result.cases)}`
+  };
+}
+function toPassCasesWithTag(result, tag) {
+  const taggedCases = result.cases.filter((c) => c.tags.includes(tag));
+  const failedTaggedCases = taggedCases.filter((c) => !c.ok);
+  const pass = failedTaggedCases.length === 0;
+  return {
+    pass,
+    message: () => pass ? `Expected cases with tag "${tag}" to fail, but all ${taggedCases.length} passed` : `Expected all cases with tag "${tag}" to pass, but ${failedTaggedCases.length} out of ${taggedCases.length} failed
+${formatFailedCases(failedTaggedCases)}`
+  };
+}
+function toHaveMedianLatencyBelow(result, maxLatencyMs) {
+  const actualLatency = result.manifest.metrics.median_latency_ms;
+  const pass = actualLatency <= maxLatencyMs;
+  return {
+    pass,
+    message: () => pass ? `Expected median latency to be above ${maxLatencyMs}ms, but got ${actualLatency}ms` : `Expected median latency to be at most ${maxLatencyMs}ms, but got ${actualLatency}ms`
+  };
+}
+function toHaveP95LatencyBelow(result, maxLatencyMs) {
+  const actualLatency = result.manifest.metrics.p95_latency_ms;
+  const pass = actualLatency <= maxLatencyMs;
+  return {
+    pass,
+    message: () => pass ? `Expected P95 latency to be above ${maxLatencyMs}ms, but got ${actualLatency}ms` : `Expected P95 latency to be at most ${maxLatencyMs}ms, but got ${actualLatency}ms`
+  };
+}
+function toHaveDefenseRate(result, expectedRate) {
+  const actualRate = result.defenseRate;
+  const pass = actualRate >= expectedRate;
+  return {
+    pass,
+    message: () => pass ? `Expected defense rate to be less than ${(expectedRate * 100).toFixed(1)}%, but got ${(actualRate * 100).toFixed(1)}%` : `Expected defense rate to be at least ${(expectedRate * 100).toFixed(1)}%, but got ${(actualRate * 100).toFixed(1)}%
+
+  Unsafe responses: ${result.unsafeCount}
+  Total tests: ${result.manifest.metrics.total_tests}`
+  };
+}
+function toHaveNoCriticalVulnerabilities(result) {
+  const criticalCount = result.manifest.metrics.by_severity.critical;
+  const pass = criticalCount === 0;
+  return {
+    pass,
+    message: () => pass ? "Expected critical vulnerabilities, but none were found" : `Expected no critical vulnerabilities, but found ${criticalCount}`
+  };
+}
+function toHaveNoHighSeverityVulnerabilities(result) {
+  const highCount = result.manifest.metrics.by_severity.high;
+  const criticalCount = result.manifest.metrics.by_severity.critical;
+  const totalHighSeverity = highCount + criticalCount;
+  const pass = totalHighSeverity === 0;
+  return {
+    pass,
+    message: () => pass ? "Expected high/critical vulnerabilities, but none were found" : `Expected no high/critical vulnerabilities, but found ${totalHighSeverity} (${highCount} high, ${criticalCount} critical)`
+  };
+}
+function toPassRedTeam(result) {
+  const pass = result.success;
+  const { metrics } = result.manifest;
+  return {
+    pass,
+    message: () => pass ? `Expected red team test to fail, but passed with ${(result.defenseRate * 100).toFixed(1)}% defense rate` : `Expected red team test to pass, but defense rate was only ${(result.defenseRate * 100).toFixed(1)}%
+
+  Total tests: ${metrics.total_tests}
+  Safe: ${metrics.safe_responses}
+  Blocked: ${metrics.blocked_responses}
+  Unsafe: ${metrics.unsafe_responses}
+  Errors: ${metrics.error_responses}`
+  };
+}
+function toHaveStressSuccessRate(result, expectedRate) {
+  const actualRate = result.successRate;
+  const pass = actualRate >= expectedRate;
+  return {
+    pass,
+    message: () => pass ? `Expected stress test success rate to be less than ${(expectedRate * 100).toFixed(1)}%, but got ${(actualRate * 100).toFixed(1)}%` : `Expected stress test success rate to be at least ${(expectedRate * 100).toFixed(1)}%, but got ${(actualRate * 100).toFixed(1)}%`
+  };
+}
+function toAchieveRPS(result, targetRPS) {
+  const actualRPS = result.rps;
+  const pass = actualRPS >= targetRPS;
+  return {
+    pass,
+    message: () => pass ? `Expected RPS to be less than ${targetRPS}, but achieved ${actualRPS.toFixed(1)} RPS` : `Expected to achieve at least ${targetRPS} RPS, but only got ${actualRPS.toFixed(1)} RPS`
+  };
+}
+function toHaveStressP95LatencyBelow(result, maxLatencyMs) {
+  const actualLatency = result.p95LatencyMs;
+  const pass = actualLatency <= maxLatencyMs;
+  return {
+    pass,
+    message: () => pass ? `Expected P95 latency to be above ${maxLatencyMs}ms, but got ${actualLatency}ms` : `Expected P95 latency to be at most ${maxLatencyMs}ms, but got ${actualLatency}ms`
+  };
+}
+function toPassStressTest(result) {
+  const pass = result.success;
+  const { metrics } = result.manifest;
+  return {
+    pass,
+    message: () => pass ? `Expected stress test to fail, but passed with ${(result.successRate * 100).toFixed(1)}% success rate` : `Expected stress test to pass, but success rate was only ${(result.successRate * 100).toFixed(1)}%
+
+  Total requests: ${metrics.total_requests}
+  Successful: ${metrics.successful_requests}
+  Failed: ${metrics.failed_requests}
+  RPS: ${metrics.requests_per_second.toFixed(1)}
+  P95 Latency: ${metrics.p95_latency_ms}ms`
+  };
+}
+var artemiskitMatchers = {
+  toPassAllCases,
+  toHaveSuccessRate,
+  toPassCasesWithTag,
+  toHaveMedianLatencyBelow,
+  toHaveP95LatencyBelow,
+  toHaveDefenseRate,
+  toHaveNoCriticalVulnerabilities,
+  toHaveNoHighSeverityVulnerabilities,
+  toPassRedTeam,
+  toHaveStressSuccessRate,
+  toAchieveRPS,
+  toHaveStressP95LatencyBelow,
+  toPassStressTest
+};
+
+// src/matchers/vitest.ts
+var vitestMatchers = {
+  toPassAllCases(received) {
+    return toPassAllCases(received);
+  },
+  toHaveSuccessRate(received, expectedRate) {
+    return toHaveSuccessRate(received, expectedRate);
+  },
+  toPassCasesWithTag(received, tag) {
+    return toPassCasesWithTag(received, tag);
+  },
+  toHaveMedianLatencyBelow(received, maxLatencyMs) {
+    return toHaveMedianLatencyBelow(received, maxLatencyMs);
+  },
+  toHaveP95LatencyBelow(received, maxLatencyMs) {
+    return toHaveP95LatencyBelow(received, maxLatencyMs);
+  },
+  toHaveDefenseRate(received, expectedRate) {
+    return toHaveDefenseRate(received, expectedRate);
+  },
+  toHaveNoCriticalVulnerabilities(received) {
+    return toHaveNoCriticalVulnerabilities(received);
+  },
+  toHaveNoHighSeverityVulnerabilities(received) {
+    return toHaveNoHighSeverityVulnerabilities(received);
+  },
+  toPassRedTeam(received) {
+    return toPassRedTeam(received);
+  },
+  toHaveStressSuccessRate(received, expectedRate) {
+    return toHaveStressSuccessRate(received, expectedRate);
+  },
+  toAchieveRPS(received, targetRPS) {
+    return toAchieveRPS(received, targetRPS);
+  },
+  toHaveStressP95LatencyBelow(received, maxLatencyMs) {
+    return toHaveStressP95LatencyBelow(received, maxLatencyMs);
+  },
+  toPassStressTest(received) {
+    return toPassStressTest(received);
+  }
+};
+if (typeof expect !== "undefined" && typeof expect.extend === "function") {
+  expect.extend(vitestMatchers);
+}
+export {
+  vitestMatchers
+};

package/package.json

@@ -0,0 +1,78 @@
+{
+  "name": "@artemiskit/sdk",
+  "version": "0.3.0",
+  "description": "Programmatic SDK for ArtemisKit LLM evaluation toolkit - integrate into your Node.js apps",
+  "type": "module",
+  "license": "Apache-2.0",
+  "author": "code-sensei",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/code-sensei/artemiskit.git",
+    "directory": "packages/sdk"
+  },
+  "bugs": {
+    "url": "https://github.com/code-sensei/artemiskit/issues"
+  },
+  "homepage": "https://artemiskit.vercel.app",
+  "keywords": [
+    "llm",
+    "testing",
+    "sdk",
+    "ai",
+    "agents",
+    "evaluation",
+    "openai",
+    "anthropic",
+    "vitest",
+    "jest",
+    "red-team",
+    "security"
+  ],
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    },
+    "./matchers": {
+      "import": "./dist/matchers/index.js",
+      "types": "./dist/matchers/index.d.ts"
+    },
+    "./vitest": {
+      "import": "./dist/matchers/vitest.js",
+      "types": "./dist/matchers/vitest.d.ts"
+    },
+    "./jest": {
+      "import": "./dist/matchers/jest.js",
+      "types": "./dist/matchers/jest.d.ts"
+    }
+  },
+  "scripts": {
+    "build": "tsc --emitDeclarationOnly && bun build ./src/index.ts ./src/matchers/index.ts ./src/matchers/vitest.ts ./src/matchers/jest.ts --outdir ./dist --target bun",
+    "typecheck": "tsc --noEmit",
+    "clean": "rm -rf dist",
+    "test": "bun test",
+    "test:coverage": "bun test --coverage"
+  },
+  "dependencies": {
+    "@artemiskit/core": "workspace:*",
+    "@artemiskit/redteam": "workspace:*"
+  },
+  "devDependencies": {
+    "@types/bun": "^1.1.0",
+    "typescript": "^5.3.0"
+  },
+  "peerDependencies": {
+    "vitest": ">=1.0.0",
+    "@jest/expect": ">=29.0.0"
+  },
+  "peerDependenciesMeta": {
+    "vitest": {
+      "optional": true
+    },
+    "@jest/expect": {
+      "optional": true
+    }
+  }
+}