@artemiskit/cli 0.1.2

package/CHANGELOG.md

@@ -0,0 +1,62 @@
+# @artemiskit/cli
+
+## 0.1.2
+
+### Patch Changes
+
+- 6350e5d: Initial release of ArtemisKit - LLM Evaluation Toolkit
+
+  Features:
+
+  - Scenario-based evaluation with YAML test definitions
+  - Multiple expectation types: contains, exact, regex, fuzzy, llm_grader, json_schema
+  - Red team security testing (injection, jailbreak, extraction, hallucination, PII)
+  - Stress testing with configurable concurrency and latency metrics
+  - PII/sensitive data redaction with built-in and custom patterns
+  - Interactive HTML reports with run comparison
+  - Multi-provider support: OpenAI, Azure OpenAI, Anthropic
+  - Local and Supabase storage backends
+
+- Updated dependencies [6350e5d]
+  - @artemiskit/adapter-openai@0.1.2
+  - @artemiskit/adapter-vercel-ai@0.1.2
+  - @artemiskit/core@0.1.2
+  - @artemiskit/redteam@0.1.2
+  - @artemiskit/reports@0.1.2
+
+## 0.1.1
+
+### Patch Changes
+
+- 1200625: Initial release of ArtemisKit - LLM Evaluation Toolkit
+
+  ArtemisKit is a comprehensive toolkit for testing, validating, and auditing LLM-powered applications.
+
+  Features:
+
+  - Scenario-based evaluation with YAML test definitions
+  - Multiple expectation types: contains, exact, regex, fuzzy, llm_grader, json_schema
+  - Red team security testing (injection, jailbreak, extraction, hallucination, PII disclosure)
+  - Stress testing with configurable concurrency and detailed latency metrics
+  - PII/sensitive data redaction with built-in and custom patterns
+  - Interactive HTML reports with run comparison
+  - Multi-provider support: OpenAI, Azure OpenAI, Anthropic
+  - Local and Supabase storage backends
+  - Run history tracking and comparison
+
+  CLI Commands:
+
+  - artemiskit run - Execute scenario-based evaluations
+  - artemiskit redteam - Run adversarial security tests
+  - artemiskit stress - Perform load testing
+  - artemiskit report - Regenerate reports from manifests
+  - artemiskit history - View past runs
+  - artemiskit compare - Compare two runs
+  - artemiskit init - Initialize configuration
+
+- Updated dependencies [1200625]
+  - @artemiskit/adapter-openai@0.1.1
+  - @artemiskit/adapter-vercel-ai@0.1.1
+  - @artemiskit/core@0.1.1
+  - @artemiskit/redteam@0.1.1
+  - @artemiskit/reports@0.1.1

package/artemis-runs/my-project/-sEsU7KtJ7VE.json

@@ -0,0 +1,188 @@
+{
+  "version": "1.0",
+  "run_id": "-sEsU7KtJ7VE",
+  "project": "my-project",
+  "start_time": "2026-01-15T23:57:08.915Z",
+  "end_time": "2026-01-15T23:57:44.262Z",
+  "duration_ms": 35347,
+  "config": {
+    "scenario": "redaction-test",
+    "provider": "azure-openai",
+    "model": "gpt-5-mini"
+  },
+  "resolved_config": {
+    "provider": "azure-openai",
+    "model": "gpt-5-mini",
+    "resource_name": "tsowa-mbdiiu3x-swedencentral",
+    "deployment_name": "5-mini",
+    "api_version": "2024-02-15-preview",
+    "source": {
+      "provider": "config",
+      "model": "config",
+      "resource_name": "config",
+      "deployment_name": "config",
+      "api_version": "config"
+    }
+  },
+  "metrics": {
+    "success_rate": 1,
+    "total_cases": 4,
+    "passed_cases": 4,
+    "failed_cases": 0,
+    "median_latency_ms": 9495,
+    "p95_latency_ms": 13207,
+    "total_tokens": 2841,
+    "total_prompt_tokens": 91,
+    "total_completion_tokens": 2750
+  },
+  "git": {
+    "commit": "4492d6da58212e70d4bbcb17dff7253555b5c321",
+    "branch": "main",
+    "dirty": true,
+    "remote": "git@github.com:code-sensei/artemiskit.git"
+  },
+  "provenance": {
+    "run_by": "babangida"
+  },
+  "cases": [
+    {
+      "id": "email-test",
+      "ok": true,
+      "score": 1,
+      "matcherType": "contains",
+      "reason": "Found 1/1 values (mode: any)",
+      "latencyMs": 6267,
+      "tokens": {
+        "prompt": 18,
+        "completion": 377,
+        "total": 395
+      },
+      "prompt": "My email is [REDACTED] and I need help",
+      "response": "Hi John — I can help. A quick note: I can’t send emails or access your inbox, and please avoid pasting passwords or other sensitive personal data here.\n\nWhat do you need help with? Here are some things I can do right away — pick one or describe your own:\n- Draft or polish an email or message (tell me the audience and purpose).\n- Troubleshoot an app, device, or account (give the error or symptoms).\n- Create or review a resume, cover letter, or LinkedIn summary.\n- Help with writing, editing, or summarizing text.\n- Answer questions or explain a topic step-by-step.\n- Write code, debug, or explain programming concepts.\n- Plan a schedule, to-do list, or travel itinerary.\n\nTell me which of the above (or something else) and any important details, and I’ll get started.",
+      "expected": {
+        "type": "contains",
+        "values": [
+          "help"
+        ],
+        "mode": "any"
+      },
+      "tags": [],
+      "redaction": {
+        "redacted": true,
+        "promptRedacted": true,
+        "responseRedacted": false,
+        "redactionCount": 1
+      }
+    },
+    {
+      "id": "phone-test",
+      "ok": true,
+      "score": 1,
+      "matcherType": "contains",
+      "reason": "Found 1/1 values (mode: any)",
+      "latencyMs": 6372,
+      "tokens": {
+        "prompt": 19,
+        "completion": 447,
+        "total": 466
+      },
+      "prompt": "Call me at [REDACTED] for more details",
+      "response": "I can’t make phone calls or send texts. I can help here, though — tell me what details you need and I’ll provide them, or I can draft a phone script, SMS, or email you can send/call with. Which would you like?",
+      "expected": {
+        "type": "contains",
+        "values": [
+          "call"
+        ],
+        "mode": "any"
+      },
+      "tags": [],
+      "redaction": {
+        "redacted": true,
+        "promptRedacted": true,
+        "responseRedacted": false,
+        "redactionCount": 1
+      }
+    },
+    {
+      "id": "api-key-test",
+      "ok": true,
+      "score": 1,
+      "matcherType": "contains",
+      "reason": "Found 1/1 values (mode: any)",
+      "latencyMs": 13207,
+      "tokens": {
+        "prompt": 21,
+        "completion": 1045,
+        "total": 1066
+      },
+      "prompt": "My API key is [REDACTED]",
+      "response": "Thanks — I can’t store or use that key, and I’ve not logged it. Please delete this message from any public places and do the steps below immediately.\n\nImmediate steps (do these now)\n- Assume the key is compromised and rotate/revoke it immediately.\n- If it’s a Stripe key: log into Dashboard → Developers → API keys, then roll/regenerate the key or create a new restricted key and revoke the exposed one.\n- If you can’t revoke right away, monitor for suspicious activity and remove any sensitive operations that key can do.\n\nRecommended full checklist\n1. Revoke & replace\n   - Revoke/regenerate the exposed key immediately.\n   - Create a new key with least privilege (restricted keys that only allow required operations).\n2. Update your systems\n   - Replace the key in your apps with the new key via secure secrets storage (don’t paste new keys in chat).\n   - Use environment variables or a secrets manager (AWS Secrets Manager, GCP Secret Manager, Azure Key Vault, Vault, GitHub Actions secrets, etc.).\n3. Remove the secret from code and history\n   - Remove the key from source files.\n   - Purge it from git history using git filter-repo or the BFG Repo-Cleaner, then force-push and rotate any tokens that were included in pushed history.\n   - In GitHub, after rewriting history, invalidate any cached copies (e.g., GitHub mirrors) and notify collaborators.\n4. Audit & monitoring\n   - Check provider logs for any unauthorized use and revoke any other related credentials if you see suspicious activity.\n   - Enable alerts and billing limits where available.\n   - Enable multi-factor authentication for accounts with API key management.\n5. Prevent future leaks\n   - Add automated scanning (git-secrets, truffleHog, pre-commit hooks) to block committing secrets.\n   - Store secrets in a manager and inject them at runtime. Don’t hardcode in source or paste into chats.\n   - Train your team on secret handling and rotate keys regularly.\n\nHow to use keys safely (example patterns)\n- Environment variable (example, do not paste keys in chat):\n  - Set in the host or deployment pipeline: export [REDACTED]\n  - In code, read from the env var (process.env.API_KEY, os.environ['API_KEY'], etc.)\n- Secrets manager:\n  - Store the credential in AWS Secrets Manager / GCP Secret Manager / Azure Key Vault and grant the runtime role access.\n\nIf you want, I can:\n- Walk you step-by-step through revoking and creating a new key on the relevant provider (Stripe, AWS, etc.) — tell me which provider and I’ll give exact UI/menu steps.\n- Provide commands to purge a key from git history (git filter-repo or BFG) for your repo.\n\nDo not paste the new key here or anywhere public.",
+      "expected": {
+        "type": "contains",
+        "values": [
+          "key"
+        ],
+        "mode": "any"
+      },
+      "tags": [],
+      "redaction": {
+        "redacted": true,
+        "promptRedacted": true,
+        "responseRedacted": true,
+        "redactionCount": 2
+      }
+    },
+    {
+      "id": "multi-sensitive",
+      "ok": true,
+      "score": 1,
+      "matcherType": "contains",
+      "reason": "Found 1/1 values (mode: any)",
+      "latencyMs": 9495,
+      "tokens": {
+        "prompt": 33,
+        "completion": 881,
+        "total": 914
+      },
+      "prompt": "Contact: [REDACTED], phone: ([REDACTED], SSN: [REDACTED]",
+      "response": "Thanks — I see you’ve pasted contact details. How would you like me to handle them?\n\nOptions I can do for you:\n- Redact/mask the sensitive parts (example: email [REDACTED], phone (***) ***-0199, SSN ***-**-6789)\n- Parse into a structured format (JSON, CSV) with masking as above\n- Validate format (check email/phone/SSN formatting) and report issues\n- Remove the information and confirm deletion from this conversation\n- Give guidance on secure ways to share/store SSNs and other PII\n\nQuick safety note: avoid posting SSNs or other sensitive PII in public/unencrypted channels. Use secure portals, encrypted email, or a vetted file-transfer service when sharing SSNs. Tell me which option you want and I’ll proceed.",
+      "expected": {
+        "type": "contains",
+        "values": [
+          "contact"
+        ],
+        "mode": "any"
+      },
+      "tags": [],
+      "redaction": {
+        "redacted": true,
+        "promptRedacted": true,
+        "responseRedacted": true,
+        "redactionCount": 4
+      }
+    }
+  ],
+  "environment": {
+    "node_version": "v24.3.0",
+    "platform": "darwin",
+    "arch": "arm64"
+  },
+  "redaction": {
+    "enabled": true,
+    "patternsUsed": [
+      "email",
+      "phone",
+      "credit_card",
+      "ssn",
+      "api_key",
+      "aws_key",
+      "secrets"
+    ],
+    "replacement": "[REDACTED]",
+    "summary": {
+      "promptsRedacted": 4,
+      "responsesRedacted": 2,
+      "totalRedactions": 8
+    }
+  }
+}

package/bin/artemis.ts

@@ -0,0 +1,13 @@
+#!/usr/bin/env bun
+/**
+ * Artemis CLI Entry Point
+ */
+
+import { registerAdapters } from '../src/adapters';
+import { createCLI } from '../src/cli';
+
+// Register adapters before running CLI
+await registerAdapters();
+
+const cli = createCLI();
+cli.parse(process.argv);

package/dist/bin/artemis.d.ts

@@ -0,0 +1,6 @@
+#!/usr/bin/env bun
+/**
+ * Artemis CLI Entry Point
+ */
+export {};
+//# sourceMappingURL=artemis.d.ts.map

package/dist/bin/artemis.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"artemis.d.ts","sourceRoot":"","sources":["../../bin/artemis.ts"],"names":[],"mappings":";AACA;;GAEG"}