@evalgate/sdk 2.2.3 → 2.2.4

package/CHANGELOG.md

@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [2.2.3] - 2026-03-03
 
+### Breaking
+
+- **`PaginatedIterator` API changed from cursor-based to offset-based** — the constructor signature changed from `(cursor) => { items, nextCursor, hasMore }` to `(offset, limit) => { data, hasMore }`. If you were using `PaginatedIterator` directly with a cursor-based fetcher, update your callback to accept `(offset: number, limit: number)` and return `{ data: T[], hasMore: boolean }`. The `autoPaginate` and `autoPaginateGenerator` helpers also use the new offset-based signature. Cursor encoding/decoding utilities (`encodeCursor`, `decodeCursor`) remain available for server-side cursor generation.
+- **`RequestCache` removed from public exports** — `RequestCache` was an internal HTTP cache with a method-specific API (`set(method, url, data, ttl, params)`) that did not match general-purpose cache expectations. It is no longer exported from the package entry point. If you were importing it directly, use your own cache implementation or rely on the SDK's built-in automatic caching. `CacheTTL` constants remain exported for advanced configuration.
+
 ### Fixed
 
 - **`RequestCache.set` missing default TTL** — entries stored without an explicit TTL were immediately stale on next read. Default is now `CacheTTL.MEDIUM`; callers that omit `ttl` get a live cache entry instead of a cache miss every time.

package/README.md

@@ -157,6 +157,42 @@ Every failure prints a clear next step:
 | `npx evalgate diff --base last --head last` | Compare last two runs |
 | `npx evalgate diff --format github` | GitHub Step Summary with regressions |
 
+### Compare — Side-by-Side Result Diff
+
+**Important:** `evalgate compare` compares **result files**, not models.
+You run each model/config separately (via `evalgate run --write-results`),
+then compare the saved JSON artifacts. Nothing is re-executed.
+
+```bash
+# The primary interface — two result files:
+evalgate compare --base .evalgate/runs/gpt4o-run.json --head .evalgate/runs/claude-run.json
+
+# Optional labels for the output table (cosmetic, not identifiers):
+evalgate compare --base gpt4o.json --head claude.json --labels "GPT-4o" "Claude 3.5"
+
+# N-way compare (3+ files):
+evalgate compare --runs run-a.json run-b.json run-c.json
+
+# Machine-readable:
+evalgate compare --base a.json --head b.json --format json
+```
+
+| Command | Description |
+|---------|-------------|
+| `evalgate compare --base <file> --head <file>` | Compare two run result JSON files |
+| `evalgate compare --runs <f1> <f2> [f3...]` | N-way comparison across multiple runs |
+| `--labels <l1> <l2>` | Optional human-readable labels for output |
+| `--sort-by <key>` | Sort specs by: `name` (default), `score`, `duration` |
+| `--format json` | Machine-readable JSON output |
+
+**Workflow:**
+```
+evalgate run --write-results   # saves .evalgate/runs/run-<id>.json
+# change model/config/prompt
+evalgate run --write-results   # saves another run file
+evalgate compare --base <first>.json --head <second>.json
+```
+
 ### Legacy Regression Gate (local, no account needed)
 
 | Command | Description |
@@ -389,7 +425,8 @@ console.log(hasNoToxicity("Have a great day!"));          // true
 console.log(hasValidCodeSyntax("function f() {}", "js")); // true
 
 // Async — LLM-backed, context-aware
-console.log(await hasSentimentAsync("subtle irony...", "negative")); // true
+const { matches, confidence } = await hasSentimentAsync("subtle irony...", "negative");
+console.log(matches, confidence);                                     // true, 0.85
 console.log(await hasNoToxicityAsync("sarcastic attack text"));       // false
 ```
 

package/dist/assertions.d.ts

@@ -132,8 +132,16 @@ export declare class Expectation {
      */
     toContainCode(language?: string, message?: string): AssertionResult;
     /**
-     * Assert value is professional tone (no profanity)
-     * @example expect(output).toBeProfessional()
+     * Blocklist check for 7 common profane words. Does NOT analyze tone,
+     * formality, or professional communication quality. For actual tone
+     * analysis, use an LLM-backed assertion.
+     * @see hasSentimentAsync for LLM-based tone checking
+     * @example expect(output).toHaveNoProfanity()
+     */
+    toHaveNoProfanity(message?: string): AssertionResult;
+    /**
+     * @deprecated Use {@link toHaveNoProfanity} instead. This method only
+     * checks for 7 profane words — it does not analyze professional tone.
      */
     toBeProfessional(message?: string): AssertionResult;
     /**
@@ -200,7 +208,63 @@ export declare function hasPII(text: string): boolean;
  * {@link hasSentimentAsync} with an LLM provider for context-aware accuracy.
  */
 export declare function hasSentiment(text: string, expected: "positive" | "negative" | "neutral"): boolean;
+/**
+ * Lexicon-based sentiment check with confidence score.
+ * Returns the detected sentiment, a confidence score (0–1), and whether
+ * it matches the expected sentiment.
+ *
+ * Confidence is derived from the magnitude of the word-count difference
+ * relative to the total sentiment-bearing words found.
+ *
+ * @example
+ * ```ts
+ * const { sentiment, confidence, matches } = hasSentimentWithScore(
+ *   "This product is absolutely amazing and wonderful!",
+ *   "positive",
+ * );
+ * // sentiment: "positive", confidence: ~0.9, matches: true
+ * ```
+ */
+export declare function hasSentimentWithScore(text: string, expected: "positive" | "negative" | "neutral"): {
+    sentiment: "positive" | "negative" | "neutral";
+    confidence: number;
+    matches: boolean;
+};
 export declare function similarTo(text1: string, text2: string, threshold?: number): boolean;
+/**
+ * Measure consistency across multiple outputs for the same input.
+ * **Fast and approximate** — uses word-overlap (Jaccard) across all pairs.
+ * Returns a score from 0 (completely inconsistent) to 1 (identical).
+ *
+ * @param outputs - Array of LLM outputs to compare (minimum 2)
+ * @param threshold - Optional minimum consistency score to return true (default 0.7)
+ * @returns `{ score, consistent }` where `consistent` is `score >= threshold`
+ *
+ * @example
+ * ```ts
+ * const { score, consistent } = hasConsistency([
+ *   "The capital of France is Paris.",
+ *   "Paris is the capital of France.",
+ *   "France's capital city is Paris.",
+ * ]);
+ * // score ≈ 0.6-0.8, consistent = true at default threshold
+ * ```
+ */
+export declare function hasConsistency(outputs: string[], threshold?: number): {
+    score: number;
+    consistent: boolean;
+};
+/**
+ * LLM-backed consistency check. **Slow and accurate** — asks the LLM to
+ * judge whether multiple outputs convey the same meaning, catching
+ * paraphrased contradictions that word-overlap misses.
+ *
+ * @returns A score from 0 to 1 where 1 = perfectly consistent.
+ */
+export declare function hasConsistencyAsync(outputs: string[], config?: AssertionLLMConfig): Promise<{
+    score: number;
+    consistent: boolean;
+}>;
 export declare function withinRange(value: number, min: number, max: number): boolean;
 export declare function isValidEmail(email: string): boolean;
 export declare function isValidURL(url: string): boolean;
@@ -229,6 +293,23 @@ export declare function containsLanguage(text: string, language: string): boolea
  * paraphrasing. Use {@link hasFactualAccuracyAsync} for semantic accuracy.
  */
 export declare function hasFactualAccuracy(text: string, facts: string[]): boolean;
+/**
+ * Check if a measured duration is within the allowed limit.
+ * @param durationMs - The actual elapsed time in milliseconds
+ * @param maxMs - Maximum allowed duration in milliseconds
+ */
+export declare function respondedWithinDuration(durationMs: number, maxMs: number): boolean;
+/**
+ * Check if elapsed time since a start timestamp is within the allowed limit.
+ * @param startTime - Timestamp from Date.now() captured before the operation
+ * @param maxMs - Maximum allowed duration in milliseconds
+ */
+export declare function respondedWithinTimeSince(startTime: number, maxMs: number): boolean;
+/**
+ * @deprecated Use {@link respondedWithinDuration} (takes measured duration)
+ * or {@link respondedWithinTimeSince} (takes start timestamp) instead.
+ * This function takes a start timestamp, not a duration — the name is misleading.
+ */
 export declare function respondedWithinTime(startTime: number, maxMs: number): boolean;
 /**
  * Blocklist-based toxicity check (~80 terms across 9 categories).
@@ -244,17 +325,66 @@ export interface AssertionLLMConfig {
     provider: "openai" | "anthropic";
     apiKey: string;
     model?: string;
+    /** Embedding model for toSemanticallyContain (default: text-embedding-3-small). OpenAI only. */
+    embeddingModel?: string;
     baseUrl?: string;
+    /** Maximum time in ms to wait for an LLM response. Default: 30 000 (30s). */
+    timeoutMs?: number;
 }
 export declare function configureAssertions(config: AssertionLLMConfig): void;
 export declare function getAssertionConfig(): AssertionLLMConfig | null;
+/**
+ * Result object from {@link hasSentimentAsync}.
+ *
+ * Implements `Symbol.toPrimitive` so that legacy callers using
+ * `if (await hasSentimentAsync(...))` get the correct `matches` boolean
+ * instead of an always-truthy object. A one-time deprecation warning is
+ * emitted when boolean coercion is detected.
+ *
+ * **Migration:** Destructure the result instead of using it as a boolean.
+ * ```ts
+ * // ❌ Deprecated (works but warns):
+ * if (await hasSentimentAsync(text, "positive")) { ... }
+ *
+ * // ✅ New pattern:
+ * const { matches } = await hasSentimentAsync(text, "positive");
+ * if (matches) { ... }
+ * ```
+ */
+export interface SentimentAsyncResult {
+    sentiment: "positive" | "negative" | "neutral";
+    confidence: number;
+    matches: boolean;
+    [Symbol.toPrimitive]: (hint: string) => boolean | number | string;
+}
+/** @internal Reset the one-time deprecation flag. For testing only. */
+export declare function resetSentimentDeprecationWarning(): void;
 /**
  * LLM-backed sentiment check. **Slow and accurate** — uses an LLM to
- * classify sentiment with full context awareness. Requires
- * {@link configureAssertions} or an inline `config` argument.
+ * classify sentiment with full context awareness and return a confidence score.
+ * Requires {@link configureAssertions} or an inline `config` argument.
  * Falls back gracefully with a clear error if no API key is configured.
+ *
+ * Returns `{ sentiment, confidence, matches }` — the async layer now provides
+ * the same rich return shape as {@link hasSentimentWithScore}, but powered by
+ * an LLM instead of keyword counting. The `confidence` field is the LLM's
+ * self-reported confidence (0–1), not a lexical heuristic.
+ *
+ * The returned object implements `Symbol.toPrimitive` so that legacy code
+ * using `if (await hasSentimentAsync(...))` still works correctly (coerces
+ * to `matches`), but a deprecation warning is emitted. Migrate to
+ * destructuring: `const { matches } = await hasSentimentAsync(...)`.
+ *
+ * @example
+ * ```ts
+ * const { sentiment, confidence, matches } = await hasSentimentAsync(
+ *   "This product is revolutionary but overpriced",
+ *   "negative",
+ * );
+ * // sentiment: "negative", confidence: 0.7, matches: true
+ * ```
  */
-export declare function hasSentimentAsync(text: string, expected: "positive" | "negative" | "neutral", config?: AssertionLLMConfig): Promise<boolean>;
+export declare function hasSentimentAsync(text: string, expected: "positive" | "negative" | "neutral", config?: AssertionLLMConfig): Promise<SentimentAsyncResult>;
 /**
  * LLM-backed toxicity check. **Slow and accurate** — context-aware, handles
  * sarcasm, implicit threats, and culturally specific harmful content that
@@ -269,4 +399,54 @@ export declare function hasFactualAccuracyAsync(text: string, facts: string[], c
  * claims even when they are paraphrased or contradict facts indirectly.
  */
 export declare function hasNoHallucinationsAsync(text: string, groundTruth: string[], config?: AssertionLLMConfig): Promise<boolean>;
+/**
+ * Embedding-based semantic containment check. Uses OpenAI embeddings and
+ * cosine similarity to determine whether the text semantically contains
+ * the given concept — no LLM prompt, no "does this text contain X" trick.
+ *
+ * This is **real semantic containment**: embed both strings, compute cosine
+ * similarity, and compare against a threshold. "The city of lights" will
+ * have high similarity to "Paris" because their embeddings are close in
+ * vector space.
+ *
+ * Requires `provider: "openai"` in the config. For Anthropic or other
+ * providers without an embedding API, use {@link toSemanticallyContainLLM}.
+ *
+ * @param text - The text to check
+ * @param phrase - The semantic concept to look for
+ * @param config - LLM config (must be OpenAI with embedding support)
+ * @param threshold - Cosine similarity threshold (default: 0.4). Lower values
+ *   are more permissive. Typical ranges: 0.3–0.5 for concept containment,
+ *   0.6–0.8 for paraphrase detection, 0.9+ for near-duplicates.
+ * @returns `{ contains, similarity }` — whether the threshold was met and the raw score
+ *
+ * @example
+ * ```ts
+ * const { contains, similarity } = await toSemanticallyContain(
+ *   "The city of lights is beautiful in spring",
+ *   "Paris",
+ *   { provider: "openai", apiKey: process.env.OPENAI_API_KEY },
+ * );
+ * // contains: true, similarity: ~0.52
+ * ```
+ */
+export declare function toSemanticallyContain(text: string, phrase: string, config?: AssertionLLMConfig, threshold?: number): Promise<{
+    contains: boolean;
+    similarity: number;
+}>;
+/**
+ * LLM-prompt-based semantic containment check. Uses an LLM prompt to ask
+ * whether the text conveys a concept. This is a **fallback** for providers
+ * that don't offer an embedding API (e.g., Anthropic).
+ *
+ * Note: This is functionally similar to `followsInstructions` — the LLM is
+ * being asked to judge containment, not compute vector similarity. For
+ * real embedding-based semantic containment, use {@link toSemanticallyContain}.
+ *
+ * @param text - The text to check
+ * @param phrase - The semantic concept to look for
+ * @param config - Optional LLM config
+ * @returns true if the LLM judges the text contains the concept
+ */
+export declare function toSemanticallyContainLLM(text: string, phrase: string, config?: AssertionLLMConfig): Promise<boolean>;
 export declare function hasValidCodeSyntax(code: string, language: string): boolean;