@evalgate/sdk 2.2.2 → 2.2.4

package/dist/assertions.js

@@ -26,7 +26,10 @@ exports.containsJSON = containsJSON;
 exports.notContainsPII = notContainsPII;
 exports.hasPII = hasPII;
 exports.hasSentiment = hasSentiment;
+exports.hasSentimentWithScore = hasSentimentWithScore;
 exports.similarTo = similarTo;
+exports.hasConsistency = hasConsistency;
+exports.hasConsistencyAsync = hasConsistencyAsync;
 exports.withinRange = withinRange;
 exports.isValidEmail = isValidEmail;
 exports.isValidURL = isValidURL;
@@ -35,19 +38,37 @@ exports.matchesSchema = matchesSchema;
 exports.hasReadabilityScore = hasReadabilityScore;
 exports.containsLanguage = containsLanguage;
 exports.hasFactualAccuracy = hasFactualAccuracy;
+exports.respondedWithinDuration = respondedWithinDuration;
+exports.respondedWithinTimeSince = respondedWithinTimeSince;
 exports.respondedWithinTime = respondedWithinTime;
 exports.hasNoToxicity = hasNoToxicity;
 exports.followsInstructions = followsInstructions;
 exports.containsAllRequiredFields = containsAllRequiredFields;
 exports.configureAssertions = configureAssertions;
 exports.getAssertionConfig = getAssertionConfig;
+exports.resetSentimentDeprecationWarning = resetSentimentDeprecationWarning;
 exports.hasSentimentAsync = hasSentimentAsync;
 exports.hasNoToxicityAsync = hasNoToxicityAsync;
 exports.containsLanguageAsync = containsLanguageAsync;
 exports.hasValidCodeSyntaxAsync = hasValidCodeSyntaxAsync;
 exports.hasFactualAccuracyAsync = hasFactualAccuracyAsync;
 exports.hasNoHallucinationsAsync = hasNoHallucinationsAsync;
+exports.toSemanticallyContain = toSemanticallyContain;
+exports.toSemanticallyContainLLM = toSemanticallyContainLLM;
 exports.hasValidCodeSyntax = hasValidCodeSyntax;
+/**
+ * Test if a term appears in text as a whole word (word-boundary match)
+ * or as a phrase (for multi-word terms). Single words use \b regex to
+ * avoid false positives like "hell" matching "hello".
+ */
+function textContainsTerm(lowerText, term) {
+    if (term.includes(" ")) {
+        // Multi-word phrases: substring match is correct
+        return lowerText.includes(term);
+    }
+    // Single words: word-boundary match
+    return new RegExp(`\\b${term.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\b`, "i").test(lowerText);
+}
 class AssertionError extends Error {
     constructor(message, expected, actual) {
         super(message);
@@ -234,9 +255,10 @@ class Expectation {
         let parsedJson = null;
         try {
             parsedJson = JSON.parse(String(this.value));
-            const requiredKeys = Object.keys(schema);
-            const actualKeys = Object.keys(parsedJson);
-            passed = requiredKeys.every((key) => actualKeys.includes(key));
+            const entries = Object.entries(schema);
+            passed = entries.every(([key, expectedValue]) => parsedJson !== null &&
+                key in parsedJson &&
+                JSON.stringify(parsedJson[key]) === JSON.stringify(expectedValue));
         }
         catch (_e) {
             passed = false;
@@ -436,43 +458,64 @@ class Expectation {
         };
     }
     /**
-     * Assert value contains code block
+     * Assert value contains code block or raw code
      * @example expect(output).toContainCode()
+     * @example expect(output).toContainCode('typescript')
      */
-    toContainCode(message) {
+    toContainCode(language, message) {
         const text = String(this.value);
-        const hasCodeBlock = /```[\s\S]*?```/.test(text) || /<code>[\s\S]*?<\/code>/.test(text);
+        const hasMarkdownBlock = language
+            ? new RegExp(`\`\`\`${language}[\\s\\S]*?\`\`\``).test(text)
+            : /```[\s\S]*?```/.test(text);
+        const hasHtmlBlock = /<code>[\s\S]*?<\/code>/.test(text);
+        const hasRawCode = /\bfunction\s+\w+\s*\(/.test(text) ||
+            /\b(?:const|let|var)\s+\w+\s*=/.test(text) ||
+            /\bclass\s+\w+/.test(text) ||
+            /=>\s*[{(]/.test(text) ||
+            /\bimport\s+.*\bfrom\b/.test(text) ||
+            /\bexport\s+(?:default\s+)?(?:function|class|const)/.test(text) ||
+            /\breturn\s+.+;/.test(text);
+        const hasCodeBlock = hasMarkdownBlock || hasHtmlBlock || hasRawCode;
         return {
             name: "toContainCode",
             passed: hasCodeBlock,
-            expected: "code block",
+            expected: language ? `code block (${language})` : "code block",
             actual: text,
-            message: message ||
-                (hasCodeBlock ? "Contains code block" : "No code block found"),
+            message: message || (hasCodeBlock ? "Contains code" : "No code found"),
         };
     }
     /**
-     * Assert value is professional tone (no profanity)
-     * @example expect(output).toBeProfessional()
+     * Blocklist check for 7 common profane words. Does NOT analyze tone,
+     * formality, or professional communication quality. For actual tone
+     * analysis, use an LLM-backed assertion.
+     * @see hasSentimentAsync for LLM-based tone checking
+     * @example expect(output).toHaveNoProfanity()
      */
-    toBeProfessional(message) {
+    toHaveNoProfanity(message) {
         const text = String(this.value).toLowerCase();
         const profanity = ["damn", "hell", "shit", "fuck", "ass", "bitch", "crap"];
-        const foundProfanity = profanity.filter((word) => text.includes(word));
+        const foundProfanity = profanity.filter((word) => textContainsTerm(text, word));
         const passed = foundProfanity.length === 0;
         return {
-            name: "toBeProfessional",
+            name: "toHaveNoProfanity",
             passed,
-            expected: "professional tone",
+            expected: "no profanity",
             actual: foundProfanity.length > 0
                 ? `Found: ${foundProfanity.join(", ")}`
-                : "professional",
+                : "clean",
             message: message ||
                 (passed
-                    ? "Professional tone"
-                    : `Unprofessional language: ${foundProfanity.join(", ")}`),
+                    ? "No profanity found"
+                    : `Profanity detected: ${foundProfanity.join(", ")}`),
         };
     }
+    /**
+     * @deprecated Use {@link toHaveNoProfanity} instead. This method only
+     * checks for 7 profane words — it does not analyze professional tone.
+     */
+    toBeProfessional(message) {
+        return this.toHaveNoProfanity(message);
+    }
     /**
      * Assert value has proper grammar (basic checks)
      * @example expect(output).toHaveProperGrammar()
@@ -690,6 +733,120 @@ function hasSentiment(text, expected) {
         return negativeCount > positiveCount;
     return positiveCount === negativeCount; // neutral
 }
+/**
+ * Lexicon-based sentiment check with confidence score.
+ * Returns the detected sentiment, a confidence score (0–1), and whether
+ * it matches the expected sentiment.
+ *
+ * Confidence is derived from the magnitude of the word-count difference
+ * relative to the total sentiment-bearing words found.
+ *
+ * @example
+ * ```ts
+ * const { sentiment, confidence, matches } = hasSentimentWithScore(
+ *   "This product is absolutely amazing and wonderful!",
+ *   "positive",
+ * );
+ * // sentiment: "positive", confidence: ~0.9, matches: true
+ * ```
+ */
+function hasSentimentWithScore(text, expected) {
+    const lower = text.toLowerCase();
+    const positiveWords = [
+        "good",
+        "great",
+        "excellent",
+        "amazing",
+        "wonderful",
+        "fantastic",
+        "love",
+        "best",
+        "happy",
+        "helpful",
+        "awesome",
+        "superb",
+        "outstanding",
+        "brilliant",
+        "perfect",
+        "delightful",
+        "joyful",
+        "pleased",
+        "glad",
+        "terrific",
+        "fabulous",
+        "exceptional",
+        "impressive",
+        "magnificent",
+        "marvelous",
+        "splendid",
+        "positive",
+        "enjoy",
+        "enjoyed",
+        "like",
+        "liked",
+        "beautiful",
+        "innovative",
+        "inspiring",
+        "effective",
+        "useful",
+        "valuable",
+    ];
+    const negativeWords = [
+        "bad",
+        "terrible",
+        "awful",
+        "horrible",
+        "worst",
+        "hate",
+        "poor",
+        "disappointing",
+        "sad",
+        "useless",
+        "dreadful",
+        "miserable",
+        "angry",
+        "frustrated",
+        "broken",
+        "failed",
+        "pathetic",
+        "stupid",
+        "disgusting",
+        "unacceptable",
+        "wrong",
+        "error",
+        "fail",
+        "problem",
+        "negative",
+        "dislike",
+        "annoying",
+        "irritating",
+        "offensive",
+        "regret",
+        "disappointment",
+        "inadequate",
+        "mediocre",
+        "flawed",
+        "unreliable",
+    ];
+    const positiveCount = positiveWords.filter((w) => lower.includes(w)).length;
+    const negativeCount = negativeWords.filter((w) => lower.includes(w)).length;
+    const total = positiveCount + negativeCount;
+    let sentiment;
+    let confidence;
+    if (positiveCount > negativeCount) {
+        sentiment = "positive";
+        confidence = total > 0 ? (positiveCount - negativeCount) / total : 0;
+    }
+    else if (negativeCount > positiveCount) {
+        sentiment = "negative";
+        confidence = total > 0 ? (negativeCount - positiveCount) / total : 0;
+    }
+    else {
+        sentiment = "neutral";
+        confidence = total === 0 ? 1 : 0; // high confidence neutral when no words found
+    }
+    return { sentiment, confidence, matches: sentiment === expected };
+}
 function similarTo(text1, text2, threshold = 0.8) {
     // Simple similarity check - in a real app, you'd use a proper string similarity algorithm
     const words1 = new Set(text1.toLowerCase().split(/\s+/));
@@ -698,6 +855,74 @@ function similarTo(text1, text2, threshold = 0.8) {
     const union = new Set([...words1, ...words2]);
     return intersection.size / union.size >= threshold;
 }
+/**
+ * Compute pairwise Jaccard similarity between word sets.
+ * Returns the mean of all C(n,2) pair similarities.
+ */
+function meanPairwiseJaccard(texts) {
+    if (texts.length < 2)
+        return 1;
+    const wordSets = texts.map((t) => new Set(t.toLowerCase().split(/\s+/).filter(Boolean)));
+    let sum = 0;
+    let count = 0;
+    for (let i = 0; i < wordSets.length; i++) {
+        for (let j = i + 1; j < wordSets.length; j++) {
+            const a = wordSets[i];
+            const b = wordSets[j];
+            const intersection = new Set([...a].filter((w) => b.has(w)));
+            const union = new Set([...a, ...b]);
+            sum += union.size > 0 ? intersection.size / union.size : 1;
+            count++;
+        }
+    }
+    return count > 0 ? sum / count : 1;
+}
+/**
+ * Measure consistency across multiple outputs for the same input.
+ * **Fast and approximate** — uses word-overlap (Jaccard) across all pairs.
+ * Returns a score from 0 (completely inconsistent) to 1 (identical).
+ *
+ * @param outputs - Array of LLM outputs to compare (minimum 2)
+ * @param threshold - Optional minimum consistency score to return true (default 0.7)
+ * @returns `{ score, consistent }` where `consistent` is `score >= threshold`
+ *
+ * @example
+ * ```ts
+ * const { score, consistent } = hasConsistency([
+ *   "The capital of France is Paris.",
+ *   "Paris is the capital of France.",
+ *   "France's capital city is Paris.",
+ * ]);
+ * // score ≈ 0.6-0.8, consistent = true at default threshold
+ * ```
+ */
+function hasConsistency(outputs, threshold = 0.7) {
+    if (outputs.length < 2) {
+        return { score: 1, consistent: true };
+    }
+    const score = meanPairwiseJaccard(outputs);
+    return { score, consistent: score >= threshold };
+}
+/**
+ * LLM-backed consistency check. **Slow and accurate** — asks the LLM to
+ * judge whether multiple outputs convey the same meaning, catching
+ * paraphrased contradictions that word-overlap misses.
+ *
+ * @returns A score from 0 to 1 where 1 = perfectly consistent.
+ */
+async function hasConsistencyAsync(outputs, config) {
+    if (outputs.length < 2) {
+        return { score: 1, consistent: true };
+    }
+    const numbered = outputs.map((o, i) => `Output ${i + 1}: "${o}"`).join("\n");
+    const prompt = `Rate the semantic consistency of the following ${outputs.length} outputs on a scale from 0 to 100, where 100 means they all convey exactly the same meaning and 0 means they completely contradict each other. Reply with ONLY a number.\n\n${numbered}`;
+    const result = await callAssertionLLM(prompt, config);
+    const parsed = parseInt(result.replace(/[^0-9]/g, ""), 10);
+    const score = Number.isNaN(parsed)
+        ? 0
+        : Math.min(100, Math.max(0, parsed)) / 100;
+    return { score, consistent: score >= 0.7 };
+}
 function withinRange(value, min, max) {
     return value >= min && value <= max;
 }
@@ -719,7 +944,7 @@ function isValidURL(url) {
  * facts but cannot detect paraphrased fabrications. Use
  * {@link hasNoHallucinationsAsync} for semantic accuracy.
  */
-function hasNoHallucinations(text, groundTruth) {
+function hasNoHallucinations(text, groundTruth = []) {
     const lower = text.toLowerCase();
     return groundTruth.every((truth) => lower.includes(truth.toLowerCase()));
 }
@@ -739,12 +964,14 @@ function matchesSchema(value, schema) {
     return Object.keys(schema).every((key) => key in obj);
 }
 function hasReadabilityScore(text, minScore) {
+    const threshold = typeof minScore === "number" ? minScore : (minScore.min ?? 0);
+    const maxThreshold = typeof minScore === "object" ? minScore.max : undefined;
     const wordList = text.trim().split(/\s+/).filter(Boolean);
     const words = wordList.length || 1;
     const sentences = text.split(/[.!?]+/).filter((s) => s.trim().length > 0).length || 1;
     const totalSyllables = wordList.reduce((sum, w) => sum + syllables(w), 0);
     const score = 206.835 - 1.015 * (words / sentences) - 84.6 * (totalSyllables / words);
-    return score >= minScore;
+    return (score >= threshold && (maxThreshold === undefined || score <= maxThreshold));
 }
 function syllables(word) {
     // Simple syllable counter
@@ -985,9 +1212,30 @@ function hasFactualAccuracy(text, facts) {
     const lower = text.toLowerCase();
     return facts.every((fact) => lower.includes(fact.toLowerCase()));
 }
-function respondedWithinTime(startTime, maxMs) {
+/**
+ * Check if a measured duration is within the allowed limit.
+ * @param durationMs - The actual elapsed time in milliseconds
+ * @param maxMs - Maximum allowed duration in milliseconds
+ */
+function respondedWithinDuration(durationMs, maxMs) {
+    return durationMs <= maxMs;
+}
+/**
+ * Check if elapsed time since a start timestamp is within the allowed limit.
+ * @param startTime - Timestamp from Date.now() captured before the operation
+ * @param maxMs - Maximum allowed duration in milliseconds
+ */
+function respondedWithinTimeSince(startTime, maxMs) {
     return Date.now() - startTime <= maxMs;
 }
+/**
+ * @deprecated Use {@link respondedWithinDuration} (takes measured duration)
+ * or {@link respondedWithinTimeSince} (takes start timestamp) instead.
+ * This function takes a start timestamp, not a duration — the name is misleading.
+ */
+function respondedWithinTime(startTime, maxMs) {
+    return respondedWithinTimeSince(startTime, maxMs);
+}
 /**
  * Blocklist-based toxicity check (~80 terms across 9 categories).
  * **Fast and approximate** — catches explicit harmful language but has
@@ -1151,10 +1399,13 @@ function hasNoToxicity(text) {
         "motherfucker",
         "fucktard",
     ];
-    return !toxicTerms.some((term) => lower.includes(term));
+    return !toxicTerms.some((term) => textContainsTerm(lower, term));
 }
 function followsInstructions(text, instructions) {
-    return instructions.every((instruction) => {
+    const instructionList = Array.isArray(instructions)
+        ? instructions
+        : [instructions];
+    return instructionList.every((instruction) => {
         if (instruction.startsWith("!")) {
             return !text.includes(instruction.slice(1));
         }
@@ -1164,6 +1415,7 @@ function followsInstructions(text, instructions) {
 function containsAllRequiredFields(obj, requiredFields) {
     return requiredFields.every((field) => obj && typeof obj === "object" && field in obj);
 }
+const DEFAULT_ASSERTION_TIMEOUT_MS = 30000;
 let _assertionLLMConfig = null;
 function configureAssertions(config) {
     _assertionLLMConfig = config;
@@ -1176,65 +1428,163 @@ async function callAssertionLLM(prompt, config) {
     if (!cfg) {
         throw new Error("No LLM config set. Call configureAssertions({ provider, apiKey }) first, or pass a config as the last argument.");
     }
-    if (cfg.provider === "openai") {
-        const baseUrl = cfg.baseUrl ?? "https://api.openai.com";
-        const model = cfg.model ?? "gpt-4o-mini";
-        const res = await fetch(`${baseUrl}/v1/chat/completions`, {
-            method: "POST",
-            headers: {
-                "Content-Type": "application/json",
-                Authorization: `Bearer ${cfg.apiKey}`,
-            },
-            body: JSON.stringify({
-                model,
-                messages: [{ role: "user", content: prompt }],
-                max_tokens: 10,
-                temperature: 0,
-            }),
-        });
-        if (!res.ok) {
-            throw new Error(`OpenAI API error ${res.status}: ${await res.text()}`);
+    const timeoutMs = cfg.timeoutMs ?? DEFAULT_ASSERTION_TIMEOUT_MS;
+    const ac = typeof AbortController !== "undefined" ? new AbortController() : null;
+    const fetchWithSignal = (url, init) => fetch(url, ac ? { ...init, signal: ac.signal } : init);
+    const llmCall = async () => {
+        if (cfg.provider === "openai") {
+            const baseUrl = cfg.baseUrl ?? "https://api.openai.com";
+            const model = cfg.model ?? "gpt-4o-mini";
+            const res = await fetchWithSignal(`${baseUrl}/v1/chat/completions`, {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    Authorization: `Bearer ${cfg.apiKey}`,
+                },
+                body: JSON.stringify({
+                    model,
+                    messages: [{ role: "user", content: prompt }],
+                    max_tokens: 60,
+                    temperature: 0,
+                }),
+            });
+            if (!res.ok) {
+                throw new Error(`OpenAI API error ${res.status}: ${await res.text()}`);
+            }
+            const data = (await res.json());
+            return data.choices[0]?.message?.content?.trim().toLowerCase() ?? "";
         }
-        const data = (await res.json());
-        return data.choices[0]?.message?.content?.trim().toLowerCase() ?? "";
-    }
-    if (cfg.provider === "anthropic") {
-        const baseUrl = cfg.baseUrl ?? "https://api.anthropic.com";
-        const model = cfg.model ?? "claude-3-haiku-20240307";
-        const res = await fetch(`${baseUrl}/v1/messages`, {
-            method: "POST",
-            headers: {
-                "Content-Type": "application/json",
-                "x-api-key": cfg.apiKey,
-                "anthropic-version": "2023-06-01",
-            },
-            body: JSON.stringify({
-                model,
-                max_tokens: 10,
-                messages: [{ role: "user", content: prompt }],
-            }),
-        });
-        if (!res.ok) {
-            throw new Error(`Anthropic API error ${res.status}: ${await res.text()}`);
+        if (cfg.provider === "anthropic") {
+            const baseUrl = cfg.baseUrl ?? "https://api.anthropic.com";
+            const model = cfg.model ?? "claude-3-haiku-20240307";
+            const res = await fetchWithSignal(`${baseUrl}/v1/messages`, {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    "x-api-key": cfg.apiKey,
+                    "anthropic-version": "2023-06-01",
+                },
+                body: JSON.stringify({
+                    model,
+                    max_tokens: 60,
+                    messages: [{ role: "user", content: prompt }],
+                }),
+            });
+            if (!res.ok) {
+                throw new Error(`Anthropic API error ${res.status}: ${await res.text()}`);
+            }
+            const data = (await res.json());
+            return data.content[0]?.text?.trim().toLowerCase() ?? "";
         }
-        const data = (await res.json());
-        return data.content[0]?.text?.trim().toLowerCase() ?? "";
+        throw new Error(`Unsupported provider: "${cfg.provider}". Use "openai" or "anthropic".`);
+    };
+    let timer;
+    const timeoutPromise = new Promise((_, reject) => {
+        timer = setTimeout(() => {
+            ac?.abort();
+            reject(new Error(`Assertion LLM call timed out after ${timeoutMs}ms`));
+        }, timeoutMs);
+    });
+    try {
+        return await Promise.race([llmCall(), timeoutPromise]);
     }
-    throw new Error(`Unsupported provider: "${cfg.provider}". Use "openai" or "anthropic".`);
+    finally {
+        clearTimeout(timer);
+    }
+}
+let _hasSentimentAsyncDeprecationWarned = false;
+/** @internal Reset the one-time deprecation flag. For testing only. */
+function resetSentimentDeprecationWarning() {
+    _hasSentimentAsyncDeprecationWarned = false;
+}
+function makeSentimentResult(sentiment, confidence, matches) {
+    return {
+        sentiment,
+        confidence,
+        matches,
+        [Symbol.toPrimitive](hint) {
+            if (!_hasSentimentAsyncDeprecationWarned) {
+                _hasSentimentAsyncDeprecationWarned = true;
+                console.warn("[evalgate] DEPRECATION: hasSentimentAsync() now returns { sentiment, confidence, matches }. " +
+                    "Using it as a boolean (e.g. `if (await hasSentimentAsync(...))`) is deprecated and will be " +
+                    "removed in the next major version. Migrate to: `const { matches } = await hasSentimentAsync(...)`");
+            }
+            if (hint === "number")
+                return matches ? 1 : 0;
+            if (hint === "string")
+                return `SentimentAsyncResult(${sentiment}, matches=${matches})`;
+            return matches;
+        },
+    };
 }
-// ============================================================================
-// LLM-BACKED ASYNC ASSERTION FUNCTIONS
-// ============================================================================
 /**
  * LLM-backed sentiment check. **Slow and accurate** — uses an LLM to
- * classify sentiment with full context awareness. Requires
- * {@link configureAssertions} or an inline `config` argument.
+ * classify sentiment with full context awareness and return a confidence score.
+ * Requires {@link configureAssertions} or an inline `config` argument.
  * Falls back gracefully with a clear error if no API key is configured.
+ *
+ * Returns `{ sentiment, confidence, matches }` — the async layer now provides
+ * the same rich return shape as {@link hasSentimentWithScore}, but powered by
+ * an LLM instead of keyword counting. The `confidence` field is the LLM's
+ * self-reported confidence (0–1), not a lexical heuristic.
+ *
+ * The returned object implements `Symbol.toPrimitive` so that legacy code
+ * using `if (await hasSentimentAsync(...))` still works correctly (coerces
+ * to `matches`), but a deprecation warning is emitted. Migrate to
+ * destructuring: `const { matches } = await hasSentimentAsync(...)`.
+ *
+ * @example
+ * ```ts
+ * const { sentiment, confidence, matches } = await hasSentimentAsync(
+ *   "This product is revolutionary but overpriced",
+ *   "negative",
+ * );
+ * // sentiment: "negative", confidence: 0.7, matches: true
+ * ```
  */
 async function hasSentimentAsync(text, expected, config) {
-    const prompt = `Classify the sentiment of the following text. Reply with exactly one word — positive, negative, or neutral — and nothing else.\n\nText: "${text}"`;
-    const result = await callAssertionLLM(prompt, config);
-    return result.replace(/[^a-z]/g, "") === expected;
+    const prompt = `Classify the sentiment of the following text as positive, negative, or neutral. Also rate your confidence from 0.0 to 1.0. Reply with ONLY a JSON object like {"sentiment":"positive","confidence":0.85} and nothing else.\n\nText: "${text}"`;
+    const raw = await callAssertionLLM(prompt, config);
+    // Parse structured response; fall back to keyword extraction if LLM doesn't return valid JSON
+    let sentiment = "neutral";
+    let confidence = 0.5;
+    try {
+        // Extract JSON from response (LLM may wrap in markdown code fences)
+        const jsonMatch = raw.match(/\{[^}]+\}/);
+        if (jsonMatch) {
+            const parsed = JSON.parse(jsonMatch[0]);
+            const s = String(parsed.sentiment ?? "").replace(/[^a-z]/g, "");
+            if (s === "positive" || s === "negative" || s === "neutral") {
+                sentiment = s;
+            }
+            const c = Number(parsed.confidence);
+            if (!Number.isNaN(c) && c >= 0 && c <= 1) {
+                confidence = c;
+            }
+        }
+        else {
+            // Fallback: old-style single-word response
+            const cleaned = raw.replace(/[^a-z]/g, "");
+            if (cleaned === "positive" ||
+                cleaned === "negative" ||
+                cleaned === "neutral") {
+                sentiment = cleaned;
+                confidence = 0.5; // no confidence info from single-word response
+            }
+        }
+    }
+    catch {
+        // JSON parse failed — try plain text extraction
+        const cleaned = raw.replace(/[^a-z]/g, "");
+        if (cleaned.includes("positive"))
+            sentiment = "positive";
+        else if (cleaned.includes("negative"))
+            sentiment = "negative";
+        else
+            sentiment = "neutral";
+        confidence = 0.5;
+    }
+    return makeSentimentResult(sentiment, confidence, sentiment === expected);
 }
 /**
  * LLM-backed toxicity check. **Slow and accurate** — context-aware, handles
@@ -1272,6 +1622,108 @@ async function hasNoHallucinationsAsync(text, groundTruth, config) {
     const result = await callAssertionLLM(prompt, config);
     return result.replace(/[^a-z]/g, "") === "yes";
 }
+/**
+ * Compute cosine similarity between two vectors.
+ */
+function cosineSimilarity(a, b) {
+    if (a.length !== b.length || a.length === 0)
+        return 0;
+    let dot = 0;
+    let normA = 0;
+    let normB = 0;
+    for (let i = 0; i < a.length; i++) {
+        dot += a[i] * b[i];
+        normA += a[i] * a[i];
+        normB += b[i] * b[i];
+    }
+    const denom = Math.sqrt(normA) * Math.sqrt(normB);
+    return denom === 0 ? 0 : dot / denom;
+}
+/**
+ * Fetch embeddings from OpenAI's embedding API.
+ * Returns an array of embedding vectors, one per input string.
+ */
+async function fetchEmbeddings(texts, config) {
+    if (config.provider !== "openai") {
+        throw new Error(`Embedding-based semantic containment requires provider "openai" (got "${config.provider}"). ` +
+            `Set provider to "openai" or use toSemanticallyContainLLM() for LLM-prompt fallback.`);
+    }
+    const baseUrl = config.baseUrl ?? "https://api.openai.com";
+    const model = config.embeddingModel ?? "text-embedding-3-small";
+    const res = await fetch(`${baseUrl}/v1/embeddings`, {
+        method: "POST",
+        headers: {
+            "Content-Type": "application/json",
+            Authorization: `Bearer ${config.apiKey}`,
+        },
+        body: JSON.stringify({ model, input: texts }),
+    });
+    if (!res.ok) {
+        throw new Error(`OpenAI Embeddings API error ${res.status}: ${await res.text()}`);
+    }
+    const data = (await res.json());
+    // Return embeddings sorted by input index
+    return data.data.sort((a, b) => a.index - b.index).map((d) => d.embedding);
+}
+/**
+ * Embedding-based semantic containment check. Uses OpenAI embeddings and
+ * cosine similarity to determine whether the text semantically contains
+ * the given concept — no LLM prompt, no "does this text contain X" trick.
+ *
+ * This is **real semantic containment**: embed both strings, compute cosine
+ * similarity, and compare against a threshold. "The city of lights" will
+ * have high similarity to "Paris" because their embeddings are close in
+ * vector space.
+ *
+ * Requires `provider: "openai"` in the config. For Anthropic or other
+ * providers without an embedding API, use {@link toSemanticallyContainLLM}.
+ *
+ * @param text - The text to check
+ * @param phrase - The semantic concept to look for
+ * @param config - LLM config (must be OpenAI with embedding support)
+ * @param threshold - Cosine similarity threshold (default: 0.4). Lower values
+ *   are more permissive. Typical ranges: 0.3–0.5 for concept containment,
+ *   0.6–0.8 for paraphrase detection, 0.9+ for near-duplicates.
+ * @returns `{ contains, similarity }` — whether the threshold was met and the raw score
+ *
+ * @example
+ * ```ts
+ * const { contains, similarity } = await toSemanticallyContain(
+ *   "The city of lights is beautiful in spring",
+ *   "Paris",
+ *   { provider: "openai", apiKey: process.env.OPENAI_API_KEY },
+ * );
+ * // contains: true, similarity: ~0.52
+ * ```
+ */
+async function toSemanticallyContain(text, phrase, config, threshold = 0.4) {
+    const cfg = config ?? _assertionLLMConfig;
+    if (!cfg) {
+        throw new Error("No LLM config set. Call configureAssertions({ provider, apiKey }) first, or pass a config argument.");
+    }
+    const [textEmbedding, phraseEmbedding] = await fetchEmbeddings([text, phrase], cfg);
+    const similarity = cosineSimilarity(textEmbedding, phraseEmbedding);
+    return { contains: similarity >= threshold, similarity };
+}
+/**
+ * LLM-prompt-based semantic containment check. Uses an LLM prompt to ask
+ * whether the text conveys a concept. This is a **fallback** for providers
+ * that don't offer an embedding API (e.g., Anthropic).
+ *
+ * Note: This is functionally similar to `followsInstructions` — the LLM is
+ * being asked to judge containment, not compute vector similarity. For
+ * real embedding-based semantic containment, use {@link toSemanticallyContain}.
+ *
+ * @param text - The text to check
+ * @param phrase - The semantic concept to look for
+ * @param config - Optional LLM config
+ * @returns true if the LLM judges the text contains the concept
+ */
+async function toSemanticallyContainLLM(text, phrase, config) {
+    const prompt = `Does the following text semantically contain or convey the concept "${phrase}"? The text does not need to use those exact words — paraphrases, synonyms, and implied references count. Reply with only "yes" or "no".\n\nText: "${text}"`;
+    const result = await callAssertionLLM(prompt, config);
+    return result.replace(/[^a-z]/g, "") === "yes";
+}
 function hasValidCodeSyntax(code, language) {
     const lang = language.toLowerCase();
     if (lang === "json") {