@artemiskit/core 0.2.0 → 0.2.2

package/CHANGELOG.md

@@ -1,5 +1,47 @@
 # @artemiskit/core
 
+## 0.2.2
+
+### Patch Changes
+
+- d5ca7c6: Add baseline command and CI mode for regression detection
+
+  ### New Features
+
+  - **Baseline Command**: New `akit baseline` command with `set`, `list`, `get`, `remove` subcommands
+
+    - Lookup by run ID (default) or scenario name (`--scenario` flag)
+    - Store and manage baseline metrics for regression comparison
+
+  - **CI Mode**: New `--ci` flag for machine-readable output
+
+    - Outputs environment variable format for easy parsing
+    - Auto-detects CI environments (GitHub Actions, GitLab CI, etc.)
+    - Suppresses colors and spinners
+
+  - **Summary Formats**: New `--summary` flag with `json`, `text`, `security` formats
+
+    - JSON summary for pipeline parsing
+    - Security summary for compliance reporting
+
+  - **Regression Detection**: New `--baseline` and `--threshold` flags
+    - Compare runs against saved baselines
+    - Configurable regression threshold (default 5%)
+    - Exit code 1 on regression detection
+
+## 0.2.1
+
+### Patch Changes
+
+- fix: improve LLM grader compatibility with reasoning models
+
+  - Remove temperature parameter from LLM grader (reasoning models like o1, o3, gpt-5-mini only support temperature=1)
+  - Increase maxTokens from 200 to 1000 to accommodate reasoning models that use tokens for internal thinking
+  - Improve grader prompt for stricter JSON-only output format
+  - Add fallback parsing for malformed JSON responses
+  - Add markdown code block stripping from grader responses
+  - Add `modelFamily` configuration option to Azure OpenAI provider for correct parameter detection when deployment names differ from model names
+
 ## 0.2.0
 
 ### Minor Changes

package/dist/adapters/types.d.ts

@@ -131,6 +131,11 @@ export interface AzureOpenAIAdapterConfig extends BaseAdapterConfig {
     apiVersion: string;
     /** Optional separate deployment name for embedding models */
     embeddingDeploymentName?: string;
+    /**
+     * Model family for parameter detection (e.g., 'gpt-5-mini' when deployment is '5-mini')
+     * Used to determine which API parameters to use (max_tokens vs max_completion_tokens)
+     */
+    modelFamily?: string;
 }
 /**
  * Vercel AI SDK configuration

package/dist/adapters/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/adapters/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,QAAQ,GAAG,MAAM,GAAG,WAAW,GAAG,UAAU,GAAG,MAAM,CAAC;IAC5D,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,aAAa,CAAC,EAAE;QACd,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,UAAU,CAAC,EAAE,QAAQ,EAAE,CAAC;CACzB;AAED,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,UAAU,CAAC;IACjB,QAAQ,EAAE;QACR,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,GAAG,WAAW,EAAE,CAAC;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,CAAC,EAAE,kBAAkB,EAAE,CAAC;IACjC,KAAK,CAAC,EAAE,cAAc,EAAE,CAAC;IACzB,cAAc,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,GAAG,aAAa,CAAA;KAAE,CAAC;IAClD,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACrC;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,UAAU,CAAC;IACjB,QAAQ,EAAE,kBAAkB,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,UAAU,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,eAAe,GAAG,YAAY,GAAG,gBAAgB,CAAC;IACrF,YAAY,CAAC,EAAE;QACb,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IACvB,GAAG,CAAC,EAAE,OAAO,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,OAAO,CAAC;IACnB,eAAe,EAAE,OAAO,CAAC;IACzB,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAE1B,QAAQ,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IAE5D,MAAM,CAAC,CAAC,OAAO,EAAE,eAAe,EAAE,OAAO,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IAE3F,KAAK,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAExD,YAAY,IAAI,OAAO,CAAC,iBAAiB,CAAC,CAAC;IAE3C,KAAK,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,MAAM,YAAY,GACpB,QAAQ,GACR,cAAc,GACd,WAAW,GACX,WAAW,GACX,QAAQ,GACR,SAAS,GACT,QAAQ,GACR,aAAa,GACb,QAAQ,GACR,QAAQ,CAAC;AAEb;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,YAAY,CAAC;IACvB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAoB,SAAQ,iBAAiB;IAC5D,QAAQ,EAAE,QAAQ,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,wBAAyB,SAAQ,iBAAiB;IACjE,QAAQ,EAAE,cAAc,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,6DAA6D;IAC7D,uBAAuB,CAAC,EAAE,MAAM,CAAC;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,qBAAsB,SAAQ,iBAAiB;IAC9D,QAAQ,EAAE,WAAW,CAAC;IACtB,kBAAkB,EAAE,QAAQ,GAAG,OAAO,GAAG,WAAW,GAAG,QAAQ,GAAG,SAAS,CAAC;IAC5E,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC1C;AAED;;GAEG;AACH,MAAM,WAAW,sBAAuB,SAAQ,iBAAiB;IAC/D,QAAQ,EAAE,WAAW,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,MAAM,aAAa,GACrB,mBAAmB,GACnB,wBAAwB,GACxB,qBAAqB,GACrB,sBAAsB,GACtB,iBAAiB,CAAC"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/adapters/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,QAAQ,GAAG,MAAM,GAAG,WAAW,GAAG,UAAU,GAAG,MAAM,CAAC;IAC5D,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,aAAa,CAAC,EAAE;QACd,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,UAAU,CAAC,EAAE,QAAQ,EAAE,CAAC;CACzB;AAED,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,UAAU,CAAC;IACjB,QAAQ,EAAE;QACR,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,GAAG,WAAW,EAAE,CAAC;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,CAAC,EAAE,kBAAkB,EAAE,CAAC;IACjC,KAAK,CAAC,EAAE,cAAc,EAAE,CAAC;IACzB,cAAc,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,GAAG,aAAa,CAAA;KAAE,CAAC;IAClD,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACrC;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,UAAU,CAAC;IACjB,QAAQ,EAAE,kBAAkB,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,UAAU,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,eAAe,GAAG,YAAY,GAAG,gBAAgB,CAAC;IACrF,YAAY,CAAC,EAAE;QACb,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IACvB,GAAG,CAAC,EAAE,OAAO,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,OAAO,CAAC;IACnB,eAAe,EAAE,OAAO,CAAC;IACzB,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAE1B,QAAQ,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IAE5D,MAAM,CAAC,CAAC,OAAO,EAAE,eAAe,EAAE,OAAO,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IAE3F,KAAK,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAExD,YAAY,IAAI,OAAO,CAAC,iBAAiB,CAAC,CAAC;IAE3C,KAAK,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,MAAM,YAAY,GACpB,QAAQ,GACR,cAAc,GACd,WAAW,GACX,WAAW,GACX,QAAQ,GACR,SAAS,GACT,QAAQ,GACR,aAAa,GACb,QAAQ,GACR,QAAQ,CAAC;AAEb;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,YAAY,CAAC;IACvB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAoB,SAAQ,iBAAiB;IAC5D,QAAQ,EAAE,QAAQ,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,wBAAyB,SAAQ,iBAAiB;IACjE,QAAQ,EAAE,cAAc,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,6DAA6D;IAC7D,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAsB,SAAQ,iBAAiB;IAC9D,QAAQ,EAAE,WAAW,CAAC;IACtB,kBAAkB,EAAE,QAAQ,GAAG,OAAO,GAAG,WAAW,GAAG,QAAQ,GAAG,SAAS,CAAC;IAC5E,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC1C;AAED;;GAEG;AACH,MAAM,WAAW,sBAAuB,SAAQ,iBAAiB;IAC/D,QAAQ,EAAE,WAAW,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,MAAM,aAAa,GACrB,mBAAmB,GACnB,wBAAwB,GACxB,qBAAqB,GACrB,sBAAsB,GACtB,iBAAiB,CAAC"}

package/dist/cost/pricing.d.ts

@@ -3,6 +3,7 @@
  *
  * Pricing is per 1,000 tokens (1K tokens) in USD
  * Data is updated periodically - always verify with provider's official pricing
+ * Last comprehensive update: January 2026
  */
 export interface ModelPricing {
     /** Price per 1K prompt/input tokens in USD */
@@ -33,7 +34,7 @@ export interface CostEstimate {
 export declare const MODEL_PRICING: Record<string, ModelPricing>;
 /**
  * Default pricing for unknown models
- * Uses conservative estimates
+ * Uses conservative estimates based on mid-tier model pricing
  */
 export declare const DEFAULT_PRICING: ModelPricing;
 /**

package/dist/cost/pricing.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pricing.d.ts","sourceRoot":"","sources":["../../src/cost/pricing.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,MAAM,WAAW,YAAY;IAC3B,8CAA8C;IAC9C,WAAW,EAAE,MAAM,CAAC;IACpB,mDAAmD;IACnD,eAAe,EAAE,MAAM,CAAC;IACxB,wBAAwB;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,YAAY;IAC3B,kCAAkC;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,6BAA6B;IAC7B,aAAa,EAAE,MAAM,CAAC;IACtB,iCAAiC;IACjC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,iCAAiC;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,mBAAmB;IACnB,OAAO,EAAE,YAAY,CAAC;CACvB;AAED;;;GAGG;AACH,eAAO,MAAM,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAqHtD,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,eAAe,EAAE,YAK7B,CAAC;AAEF;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,YAAY,CAkD3D;AAED;;;;;;GAMG;AACH,wBAAgB,YAAY,CAC1B,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,MAAM,EACxB,KAAK,EAAE,MAAM,GACZ,YAAY,CAcd;AAED;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAQlD;AAED;;GAEG;AACH,wBAAgB,eAAe,IAAI,KAAK,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,YAAY,CAAA;CAAE,CAAC,CAKjF"}
+{"version":3,"file":"pricing.d.ts","sourceRoot":"","sources":["../../src/cost/pricing.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,MAAM,WAAW,YAAY;IAC3B,8CAA8C;IAC9C,WAAW,EAAE,MAAM,CAAC;IACpB,mDAAmD;IACnD,eAAe,EAAE,MAAM,CAAC;IACxB,wBAAwB;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,YAAY;IAC3B,kCAAkC;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,6BAA6B;IAC7B,aAAa,EAAE,MAAM,CAAC;IACtB,iCAAiC;IACjC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,iCAAiC;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,mBAAmB;IACnB,OAAO,EAAE,YAAY,CAAC;CACvB;AAED;;;GAGG;AACH,eAAO,MAAM,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAuNtD,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,eAAe,EAAE,YAK7B,CAAC;AAEF;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,YAAY,CAgI3D;AAED;;;;;;GAMG;AACH,wBAAgB,YAAY,CAC1B,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,MAAM,EACxB,KAAK,EAAE,MAAM,GACZ,YAAY,CAcd;AAED;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAQlD;AAED;;GAEG;AACH,wBAAgB,eAAe,IAAI,KAAK,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,YAAY,CAAA;CAAE,CAAC,CAKjF"}

package/dist/evaluators/llm-grader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"llm-grader.d.ts","sourceRoot":"","sources":["../../src/evaluators/llm-grader.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,KAAK,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAmB5E,qBAAa,kBAAmB,YAAW,SAAS;IAClD,QAAQ,CAAC,IAAI,gBAAgB;IAEvB,QAAQ,CACZ,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,QAAQ,EAClB,OAAO,CAAC,EAAE,gBAAgB,GACzB,OAAO,CAAC,eAAe,CAAC;IA8C3B,OAAO,CAAC,mBAAmB;CAsB5B"}
+{"version":3,"file":"llm-grader.d.ts","sourceRoot":"","sources":["../../src/evaluators/llm-grader.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,KAAK,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAwB5E,qBAAa,kBAAmB,YAAW,SAAS;IAClD,QAAQ,CAAC,IAAI,gBAAgB;IAEvB,QAAQ,CACZ,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,QAAQ,EAClB,OAAO,CAAC,EAAE,gBAAgB,GACzB,OAAO,CAAC,eAAe,CAAC;IAgD3B,OAAO,CAAC,mBAAmB;CA+C5B"}

package/dist/index.js

@@ -11564,8 +11564,7 @@ class LLMGraderEvaluator {
       const result = await context.client.generate({
         prompt,
         model: expected.model,
-        temperature: 0,
-        maxTokens: 200
+        maxTokens: 1000
       });
       const parsed = this.parseGraderResponse(result.text);
       const passed = parsed.score >= expected.threshold;
@@ -11590,9 +11589,17 @@ class LLMGraderEvaluator {
     }
   }
   parseGraderResponse(text) {
-    const jsonMatch = text.match(/\{[\s\S]*?\}/);
+    const cleanedText = text.replace(/```json\s*/gi, "").replace(/```\s*/g, "").trim();
+    const jsonMatch = cleanedText.match(/\{[\s\S]*?\}/);
     if (!jsonMatch) {
-      throw new Error("No JSON found in grader response");
+      const scoreMatch = cleanedText.match(/(?:score[:\s]*)?(\d+\.?\d*)/i);
+      if (scoreMatch) {
+        const score = Number(scoreMatch[1]);
+        if (!Number.isNaN(score) && score >= 0 && score <= 1) {
+          return { score, reason: cleanedText };
+        }
+      }
+      throw new Error(`No JSON found in grader response: ${text.substring(0, 100)}...`);
     }
     try {
       const parsed = JSON.parse(jsonMatch[0]);
@@ -11605,26 +11612,39 @@ class LLMGraderEvaluator {
         reason: parsed.reason
       };
     } catch (error) {
+      const scoreMatch = jsonMatch[0].match(/"score"[:\s]*(\d+\.?\d*)/i);
+      if (scoreMatch) {
+        const score = Number(scoreMatch[1]);
+        if (!Number.isNaN(score) && score >= 0 && score <= 1) {
+          const reasonMatch = jsonMatch[0].match(/"reason"[:\s]*"([^"]+)"/i);
+          return { score, reason: reasonMatch?.[1] };
+        }
+      }
       throw new Error(`Failed to parse grader response: ${error.message}`);
     }
   }
 }
-var GRADER_PROMPT = `You are an evaluator grading an AI response based on a rubric.
+var GRADER_PROMPT = `You are a strict JSON-only evaluator. You grade AI responses based on rubrics.
 
-## RUBRIC
+RUBRIC:
 {{rubric}}
 
-## RESPONSE TO EVALUATE
+RESPONSE TO EVALUATE:
 {{response}}
 
-## INSTRUCTIONS
-Score the response from 0.0 to 1.0 based on the rubric.
-Be objective and consistent in your scoring.
+TASK: Score the response from 0.0 to 1.0 based on the rubric above.
 
-Respond with ONLY a JSON object in this exact format:
-{"score": <number between 0 and 1>, "reason": "<brief explanation of score>"}
+OUTPUT FORMAT: You MUST respond with ONLY this exact JSON structure, nothing else:
+{"score":0.0,"reason":"explanation"}
+
+RULES:
+- Output ONLY valid JSON, no markdown, no code blocks, no extra text
+- "score" must be a number between 0.0 and 1.0
+- "reason" must be a brief string explaining the score
+- Do NOT wrap in \`\`\`json or any formatting
+- Your entire response must be parseable by JSON.parse()
 
-Do not include any other text, markdown, or formatting.`;
+JSON OUTPUT:`;
 
 // src/evaluators/not-contains.ts
 class NotContainsEvaluator {
@@ -13487,6 +13507,7 @@ var ProviderConfigSchema = exports_external.object({
   deploymentName: exports_external.string().optional(),
   apiVersion: exports_external.string().optional(),
   embeddingDeploymentName: exports_external.string().optional(),
+  modelFamily: exports_external.string().optional(),
   underlyingProvider: exports_external.enum(["openai", "azure", "anthropic", "google", "mistral"]).optional()
 }).optional();
 var BaseExpectedSchema = exports_external.discriminatedUnion("type", [
@@ -14493,8 +14514,10 @@ function getScenario(manifest) {
 
 class LocalStorageAdapter {
   basePath;
+  baselinesPath;
   constructor(basePath = "./artemis-runs") {
     this.basePath = resolve2(basePath);
+    this.baselinesPath = join2(this.basePath, ".artemis", "baselines.json");
   }
   async save(manifest) {
     const dir = join2(this.basePath, manifest.project);
@@ -14613,6 +14636,89 @@ class LocalStorageAdapter {
       return [];
     }
   }
+  async loadBaselinesFile() {
+    try {
+      const content = await readFile2(this.baselinesPath, "utf-8");
+      return JSON.parse(content);
+    } catch {
+      return { version: "1.0", baselines: {} };
+    }
+  }
+  async saveBaselinesFile(data) {
+    const dir = join2(this.basePath, ".artemis");
+    await mkdir(dir, { recursive: true });
+    await writeFile(this.baselinesPath, JSON.stringify(data, null, 2));
+  }
+  async setBaseline(scenario, runId, tag) {
+    const manifest = await this.loadRun(runId);
+    const scenarioName = scenario || getScenario(manifest);
+    const baseline = {
+      scenario: scenarioName,
+      runId,
+      createdAt: new Date().toISOString(),
+      metrics: {
+        successRate: manifest.metrics.success_rate,
+        medianLatencyMs: manifest.metrics.median_latency_ms,
+        totalTokens: manifest.metrics.total_tokens,
+        passedCases: manifest.metrics.passed_cases,
+        failedCases: manifest.metrics.failed_cases,
+        totalCases: manifest.metrics.total_cases
+      },
+      tag
+    };
+    const data = await this.loadBaselinesFile();
+    data.baselines[scenarioName] = baseline;
+    await this.saveBaselinesFile(data);
+    return baseline;
+  }
+  async getBaseline(scenario) {
+    const data = await this.loadBaselinesFile();
+    return data.baselines[scenario] || null;
+  }
+  async getBaselineByRunId(runId) {
+    const data = await this.loadBaselinesFile();
+    const baselines = Object.values(data.baselines);
+    return baselines.find((b) => b.runId === runId) || null;
+  }
+  async listBaselines() {
+    const data = await this.loadBaselinesFile();
+    return Object.values(data.baselines).sort((a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime());
+  }
+  async removeBaseline(scenario) {
+    const data = await this.loadBaselinesFile();
+    if (data.baselines[scenario]) {
+      delete data.baselines[scenario];
+      await this.saveBaselinesFile(data);
+      return true;
+    }
+    return false;
+  }
+  async removeBaselineByRunId(runId) {
+    const data = await this.loadBaselinesFile();
+    const entry = Object.entries(data.baselines).find(([_, b]) => b.runId === runId);
+    if (entry) {
+      delete data.baselines[entry[0]];
+      await this.saveBaselinesFile(data);
+      return true;
+    }
+    return false;
+  }
+  async compareToBaseline(runId, regressionThreshold = 0.05) {
+    const currentManifest = await this.loadRun(runId);
+    const scenario = getScenario(currentManifest);
+    const baseline = await this.getBaseline(scenario);
+    if (!baseline) {
+      return null;
+    }
+    const comparison = await this.compare(baseline.runId, runId);
+    const hasRegression = comparison.delta.successRate < -regressionThreshold;
+    return {
+      baseline,
+      comparison,
+      hasRegression,
+      regressionThreshold
+    };
+  }
 }
 
 // ../../node_modules/.bun/tslib@2.8.1/node_modules/tslib/modules/index.js
@@ -24303,116 +24409,184 @@ class Logger {
 var logger = new Logger("artemis");
 // src/cost/pricing.ts
 var MODEL_PRICING = {
-  "gpt-4": {
-    promptPer1K: 0.03,
-    completionPer1K: 0.06,
-    lastUpdated: "2024-01"
+  "gpt-5": {
+    promptPer1K: 0.00125,
+    completionPer1K: 0.01,
+    lastUpdated: "2026-01",
+    notes: "400K context window"
   },
-  "gpt-4-32k": {
-    promptPer1K: 0.06,
-    completionPer1K: 0.12,
-    lastUpdated: "2024-01"
+  "gpt-5.1": {
+    promptPer1K: 0.00125,
+    completionPer1K: 0.01,
+    lastUpdated: "2026-01"
   },
-  "gpt-4-turbo": {
-    promptPer1K: 0.01,
-    completionPer1K: 0.03,
-    lastUpdated: "2024-01"
+  "gpt-5.2": {
+    promptPer1K: 0.00175,
+    completionPer1K: 0.014,
+    lastUpdated: "2026-01"
   },
-  "gpt-4-turbo-preview": {
-    promptPer1K: 0.01,
-    completionPer1K: 0.03,
-    lastUpdated: "2024-01"
+  "gpt-5-mini": {
+    promptPer1K: 0.00025,
+    completionPer1K: 0.002,
+    lastUpdated: "2026-01"
+  },
+  "gpt-5-nano": {
+    promptPer1K: 0.00005,
+    completionPer1K: 0.0004,
+    lastUpdated: "2026-01"
+  },
+  "gpt-4.1": {
+    promptPer1K: 0.002,
+    completionPer1K: 0.008,
+    lastUpdated: "2026-01",
+    notes: "1M context window"
+  },
+  "gpt-4.1-mini": {
+    promptPer1K: 0.0004,
+    completionPer1K: 0.0016,
+    lastUpdated: "2026-01"
+  },
+  "gpt-4.1-nano": {
+    promptPer1K: 0.0001,
+    completionPer1K: 0.0004,
+    lastUpdated: "2026-01"
   },
   "gpt-4o": {
-    promptPer1K: 0.005,
-    completionPer1K: 0.015,
-    lastUpdated: "2024-05"
+    promptPer1K: 0.0025,
+    completionPer1K: 0.01,
+    lastUpdated: "2026-01",
+    notes: "128K context window"
   },
   "gpt-4o-mini": {
     promptPer1K: 0.00015,
     completionPer1K: 0.0006,
-    lastUpdated: "2024-07"
+    lastUpdated: "2026-01",
+    notes: "128K context window"
+  },
+  o1: {
+    promptPer1K: 0.015,
+    completionPer1K: 0.06,
+    lastUpdated: "2026-01",
+    notes: "Reasoning model - internal thinking tokens billed as output"
+  },
+  o3: {
+    promptPer1K: 0.002,
+    completionPer1K: 0.008,
+    lastUpdated: "2026-01"
+  },
+  "o3-mini": {
+    promptPer1K: 0.0011,
+    completionPer1K: 0.0044,
+    lastUpdated: "2026-01"
+  },
+  "o4-mini": {
+    promptPer1K: 0.0011,
+    completionPer1K: 0.0044,
+    lastUpdated: "2026-01"
+  },
+  "gpt-4-turbo": {
+    promptPer1K: 0.01,
+    completionPer1K: 0.03,
+    lastUpdated: "2026-01"
+  },
+  "gpt-4": {
+    promptPer1K: 0.03,
+    completionPer1K: 0.06,
+    lastUpdated: "2026-01"
   },
   "gpt-3.5-turbo": {
     promptPer1K: 0.0005,
     completionPer1K: 0.0015,
-    lastUpdated: "2024-01"
+    lastUpdated: "2026-01"
   },
-  "gpt-3.5-turbo-16k": {
+  "claude-opus-4.5": {
+    promptPer1K: 0.005,
+    completionPer1K: 0.025,
+    lastUpdated: "2026-01",
+    notes: "Most capable Claude model"
+  },
+  "claude-sonnet-4.5": {
     promptPer1K: 0.003,
-    completionPer1K: 0.004,
-    lastUpdated: "2024-01"
+    completionPer1K: 0.015,
+    lastUpdated: "2026-01",
+    notes: "Balanced performance and cost"
+  },
+  "claude-haiku-4.5": {
+    promptPer1K: 0.001,
+    completionPer1K: 0.005,
+    lastUpdated: "2026-01",
+    notes: "Fastest Claude model"
+  },
+  "claude-opus-4": {
+    promptPer1K: 0.015,
+    completionPer1K: 0.075,
+    lastUpdated: "2026-01"
   },
-  "claude-3-opus-20240229": {
+  "claude-opus-4.1": {
     promptPer1K: 0.015,
     completionPer1K: 0.075,
-    lastUpdated: "2024-03"
+    lastUpdated: "2026-01"
   },
-  "claude-3-sonnet-20240229": {
+  "claude-sonnet-4": {
     promptPer1K: 0.003,
     completionPer1K: 0.015,
-    lastUpdated: "2024-03"
+    lastUpdated: "2026-01"
   },
-  "claude-3-haiku-20240307": {
-    promptPer1K: 0.00025,
-    completionPer1K: 0.00125,
-    lastUpdated: "2024-03"
+  "claude-sonnet-3.7": {
+    promptPer1K: 0.003,
+    completionPer1K: 0.015,
+    lastUpdated: "2026-01"
   },
-  "claude-3-5-sonnet-20240620": {
+  "claude-3-7-sonnet": {
     promptPer1K: 0.003,
     completionPer1K: 0.015,
-    lastUpdated: "2024-06"
+    lastUpdated: "2026-01"
   },
   "claude-3-5-sonnet-20241022": {
     promptPer1K: 0.003,
     completionPer1K: 0.015,
-    lastUpdated: "2024-10"
+    lastUpdated: "2026-01"
   },
   "claude-3-5-haiku-20241022": {
     promptPer1K: 0.0008,
     completionPer1K: 0.004,
-    lastUpdated: "2024-10"
+    lastUpdated: "2026-01"
+  },
+  "claude-haiku-3.5": {
+    promptPer1K: 0.0008,
+    completionPer1K: 0.004,
+    lastUpdated: "2026-01"
   },
   "claude-3-opus": {
     promptPer1K: 0.015,
     completionPer1K: 0.075,
-    lastUpdated: "2024-03"
+    lastUpdated: "2026-01"
   },
   "claude-3-sonnet": {
     promptPer1K: 0.003,
     completionPer1K: 0.015,
-    lastUpdated: "2024-03"
+    lastUpdated: "2026-01"
   },
   "claude-3-haiku": {
     promptPer1K: 0.00025,
     completionPer1K: 0.00125,
-    lastUpdated: "2024-03"
+    lastUpdated: "2026-01"
   },
   "claude-3.5-sonnet": {
     promptPer1K: 0.003,
     completionPer1K: 0.015,
-    lastUpdated: "2024-10"
+    lastUpdated: "2026-01"
   },
   "claude-3.5-haiku": {
     promptPer1K: 0.0008,
     completionPer1K: 0.004,
-    lastUpdated: "2024-10"
-  },
-  "claude-2": {
-    promptPer1K: 0.008,
-    completionPer1K: 0.024,
-    lastUpdated: "2024-01"
-  },
-  "claude-instant-1": {
-    promptPer1K: 0.0008,
-    completionPer1K: 0.0024,
-    lastUpdated: "2024-01"
+    lastUpdated: "2026-01"
   }
 };
 var DEFAULT_PRICING = {
-  promptPer1K: 0.01,
-  completionPer1K: 0.03,
-  lastUpdated: "2024-01",
+  promptPer1K: 0.003,
+  completionPer1K: 0.015,
+  lastUpdated: "2026-01",
   notes: "Default pricing - verify with provider"
 };
 function getModelPricing(model) {
@@ -24425,12 +24599,48 @@ function getModelPricing(model) {
       return pricing;
     }
   }
+  if (lowerModel.includes("gpt-5.2")) {
+    return MODEL_PRICING["gpt-5.2"];
+  }
+  if (lowerModel.includes("gpt-5.1")) {
+    return MODEL_PRICING["gpt-5.1"];
+  }
+  if (lowerModel.includes("gpt-5-mini")) {
+    return MODEL_PRICING["gpt-5-mini"];
+  }
+  if (lowerModel.includes("gpt-5-nano")) {
+    return MODEL_PRICING["gpt-5-nano"];
+  }
+  if (lowerModel.includes("gpt-5")) {
+    return MODEL_PRICING["gpt-5"];
+  }
+  if (lowerModel.includes("gpt-4.1-mini")) {
+    return MODEL_PRICING["gpt-4.1-mini"];
+  }
+  if (lowerModel.includes("gpt-4.1-nano")) {
+    return MODEL_PRICING["gpt-4.1-nano"];
+  }
+  if (lowerModel.includes("gpt-4.1")) {
+    return MODEL_PRICING["gpt-4.1"];
+  }
   if (lowerModel.includes("gpt-4o-mini")) {
     return MODEL_PRICING["gpt-4o-mini"];
   }
   if (lowerModel.includes("gpt-4o")) {
     return MODEL_PRICING["gpt-4o"];
   }
+  if (lowerModel.includes("o4-mini")) {
+    return MODEL_PRICING["o4-mini"];
+  }
+  if (lowerModel.includes("o3-mini")) {
+    return MODEL_PRICING["o3-mini"];
+  }
+  if (lowerModel.includes("o3")) {
+    return MODEL_PRICING.o3;
+  }
+  if (lowerModel.includes("o1")) {
+    return MODEL_PRICING.o1;
+  }
   if (lowerModel.includes("gpt-4-turbo")) {
     return MODEL_PRICING["gpt-4-turbo"];
   }
@@ -24440,6 +24650,27 @@ function getModelPricing(model) {
   if (lowerModel.includes("gpt-3.5")) {
     return MODEL_PRICING["gpt-3.5-turbo"];
   }
+  if (lowerModel.includes("opus-4.5") || lowerModel.includes("opus-4-5")) {
+    return MODEL_PRICING["claude-opus-4.5"];
+  }
+  if (lowerModel.includes("sonnet-4.5") || lowerModel.includes("sonnet-4-5")) {
+    return MODEL_PRICING["claude-sonnet-4.5"];
+  }
+  if (lowerModel.includes("haiku-4.5") || lowerModel.includes("haiku-4-5")) {
+    return MODEL_PRICING["claude-haiku-4.5"];
+  }
+  if (lowerModel.includes("opus-4.1") || lowerModel.includes("opus-4-1")) {
+    return MODEL_PRICING["claude-opus-4.1"];
+  }
+  if (lowerModel.includes("opus-4")) {
+    return MODEL_PRICING["claude-opus-4"];
+  }
+  if (lowerModel.includes("sonnet-4")) {
+    return MODEL_PRICING["claude-sonnet-4"];
+  }
+  if (lowerModel.includes("sonnet-3.7") || lowerModel.includes("sonnet-3-7")) {
+    return MODEL_PRICING["claude-sonnet-3.7"];
+  }
   if (lowerModel.includes("claude-3-5-sonnet") || lowerModel.includes("claude-3.5-sonnet")) {
     return MODEL_PRICING["claude-3.5-sonnet"];
   }
@@ -24456,7 +24687,7 @@ function getModelPricing(model) {
     return MODEL_PRICING["claude-3-haiku"];
   }
   if (lowerModel.includes("claude")) {
-    return MODEL_PRICING["claude-2"];
+    return MODEL_PRICING["claude-sonnet-4.5"];
   }
   return DEFAULT_PRICING;
 }