@eeacms/volto-eea-chatbot 3.0.0 → 4.0.0

package/src/halloumi/postprocessing.js

@@ -1,247 +0,0 @@
-/**
- * Gets the claim id from a subsegment.
- * @param subsegment A subsegment string of the form "<|r1|".
- * @returns The numeric claim id.
- */
-function getClaimIdFromSubsegment(subsegment) {
-  const textClaimId = subsegment.split('|')[1];
-  const integerClaimId = parseInt(textClaimId.split('r')[1]);
-  return integerClaimId;
-}
-
-/**
- * Gets the citations from a subsegment.
- * @param subsegment A subsegment string of the form "|s1|,|s2|,|s3|,|s4|".
- * @returns The list of numeric citations.
- */
-function getClaimCitationsFromSubsegment(subsegment) {
-  const citationSegments = subsegment.split(',');
-  const segments = [];
-  for (const citationSegment of citationSegments) {
-    const segment = citationSegment.replaceAll('|', '').replaceAll('s', '');
-    if (segment.includes('-')) {
-      const segmentRange = segment.split('-');
-      for (
-        let i = parseInt(segmentRange[0].trim());
-        i <= parseInt(segmentRange[1].trim());
-        i++
-      ) {
-        segments.push(i);
-      }
-    } else if (segment.includes('to')) {
-      const segmentRange = segment.split('to');
-      for (
-        let i = parseInt(segmentRange[0].trim());
-        i <= parseInt(segmentRange[1].trim());
-        i++
-      ) {
-        segments.push(i);
-      }
-    } else {
-      const segmentInt = parseInt(segment);
-      if (!isNaN(segmentInt)) {
-        segments.push(parseInt(segment));
-      }
-    }
-  }
-  return segments;
-}
-
-/**
- * Gets the support status from a subsegment.
- * @param subsegment A subsegment string of the form "|supported|" or "|unsupported|".
- * @returns True if the claim is supported, false otherwise.
- */
-function getSupportStatusFromSubsegment(subsegment) {
-  return subsegment.startsWith('|supported|');
-}
-
-/**
- * Gets a claim from a segment string.
- * @param segment A segment string containing all information for claim verification.
- * @returns The claim object with all relevant information.
- */
-function getClaimFromSegment(segment) {
-  const claim_segments = segment.split('><');
-  const claimId = getClaimIdFromSubsegment(claim_segments[0]);
-  // Strip trailing '>' which can occur in malformed segments without subclaims
-  const claimString = claim_segments[1]?.replace(/>$/, '') || '';
-
-  const subclaims = [];
-  let claimProgressIndex = 3; // Start at 3 to skip the claim id, claim string and the subclaims tag
-  for (let i = claimProgressIndex; i < claim_segments.length; i++) {
-    const subsegment = claim_segments[i];
-    if (subsegment.startsWith('end||subclaims')) {
-      claimProgressIndex = i + 1;
-      break;
-    } else {
-      subclaims.push(subsegment);
-    }
-  }
-
-  let cite_tag_index = -1;
-  let explanation_index = -1;
-  let label_index = -1;
-  for (let i = claimProgressIndex; i < claim_segments.length; i++) {
-    const subsegment = claim_segments[i];
-    if (subsegment.startsWith('|cite|')) {
-      cite_tag_index = i;
-    } else if (subsegment.startsWith('|explain|')) {
-      explanation_index = i + 1;
-    } else if (
-      subsegment.startsWith('|supported|') ||
-      subsegment.startsWith('|unsupported|')
-    ) {
-      label_index = i;
-    }
-  }
-
-  const segments =
-    cite_tag_index >= 0
-      ? getClaimCitationsFromSubsegment(claim_segments[cite_tag_index + 1])
-      : [];
-  const explanation =
-    explanation_index >= 0 ? claim_segments[explanation_index] : '';
-
-  // Default to true if label not found
-  const supported =
-    label_index >= 0
-      ? getSupportStatusFromSubsegment(claim_segments[label_index])
-      : true;
-
-  const claim = {
-    claimId,
-    claimString,
-    subclaims,
-    segments,
-    explanation,
-    supported,
-    probabilities: new Map(),
-  };
-
-  return claim;
-}
-
-/**
- * Gets all claims from a response.
- * @param response A string containing all claims and their information.
- * @returns A list of claim objects.
- */
-export function getClaimsFromResponse(response) {
-  // Example response: <|r1|><There is no information about the average lifespan of a giant squid in the deep waters of the Pacific Ocean in the provided document.><|subclaims|><The document contains information about the average lifespan of a giant squid.><The information about giant squid lifespan is related to the Pacific Ocean.><end||subclaims><|cite|><|s1 to s49|><end||cite><|explain|><Upon reviewing the entire document, there is no mention of giant squid or any related topic, including their average lifespan or the Pacific Ocean. The document is focused on international relations, diplomacy, and conflict resolution.><end||explain><|supported|><end||r><|r2|><The document is focused on international relations, diplomacy, and conflict resolution, and does not mention giant squid or any related topic.><|subclaims|><The document is focused on international relations, diplomacy, and conflict resolution.><The document does not mention giant squid or any related topic.><end||subclaims><|cite|><|s1|,|s2|,|s3|,|s4|><end||cite><|explain|><The first four sentences clearly establish the document's focus on international relations, diplomacy, and conflict resolution, and there is no mention of giant squid or any related topic throughout the document.><end||explain><|supported|><end||r><|r3|><The document mentions cats.><|subclaims|><The document makes some mention of cats.><end||subclaims><|cite|><None><end||cite><|explain|><There is no mention of cats anywhere in the document.><end||explain><|unsupported|><end||r>
-
-  // eslint-disable-next-line no-console
-  // console.log('getClaimsFromResponse', response);
-  let segments = response.split('<end||r>');
-  const claims = [];
-
-  for (let i = 0; i < segments.length; i++) {
-    const segment = segments[i];
-
-    if (segment.length === 0) {
-      continue;
-    }
-
-    // Skip segments that don't start with a valid claim marker <|r*|>
-    if (!segment.match(/^<\|r\d+\|>/)) {
-      continue;
-    }
-
-    // Check if this segment is well-formed (has subclaims within it)
-    // Well-formed: <|r1|><claim><|subclaims|>...<|supported|>...<end||r>
-    // Malformed:   <|r1|><claim><end||r><|subclaims|>...<|supported|>...<end||r>
-    // The malformed case puts verification data AFTER <end||r>, creating orphan segments
-    if (segment.includes('<|subclaims|>')) {
-      // Well-formed segment - process normally
-      const claim = getClaimFromSegment(segment);
-      claims.push(claim);
-    } else {
-      // Malformed segment - look for orphan data in the next segment
-      let fullSegment = segment;
-
-      // Look ahead for orphan segment (starts with <|subclaims|>)
-      if (i + 1 < segments.length) {
-        const nextSegment = segments[i + 1];
-        if (nextSegment.startsWith('<|subclaims|>')) {
-          // Merge: segment ends with '>', orphan starts with '<'
-          fullSegment = segment + nextSegment;
-          i++; // Skip the orphan in next iteration
-        }
-      }
-
-      const claim = getClaimFromSegment(fullSegment);
-      claims.push(claim);
-    }
-  }
-
-  return claims;
-}
-
-function exp(x) {
-  return Math.pow(Math.E, x);
-}
-
-function softmax(logits) {
-  const softmaxes = [];
-  const exp_values = [];
-  let total = 0;
-  for (let i = 0; i < logits.length; i++) {
-    const exponential = exp(logits[i]);
-    total += exponential;
-    exp_values.push(exponential);
-  }
-
-  for (let i = 0; i < exp_values.length; i++) {
-    softmaxes.push(exp_values[i] / total);
-  }
-  return softmaxes;
-}
-
-function getTokenProbabilitiesFromLogit(logit, tokenChoices) {
-  const tokenLogits = new Map();
-  let smallestLogit = 1000000;
-  for (const token of tokenChoices) {
-    const tokenLogit = logit.top_logprobs.find(
-      (logit) => logit.token === token,
-    );
-    if (tokenLogit !== undefined) {
-      smallestLogit = Math.min(smallestLogit, tokenLogit.logprob);
-      tokenLogits.set(token, tokenLogit.logprob);
-    }
-  }
-
-  // If any class tokens aren't present, set them below the smallest logit.
-  // Their true value is definitely <= the smallest logit value.
-  for (const token of tokenChoices) {
-    if (!tokenLogits.has(token)) {
-      tokenLogits.set(token, smallestLogit - 1e-6);
-    }
-  }
-
-  const classTokens = Array.from(tokenLogits.keys());
-  const logitValues = Array.from(tokenLogits.values());
-  const softmaxValues = softmax(logitValues);
-  const tokenProbabilities = new Map();
-  for (let i = 0; i < classTokens.length; i++) {
-    tokenProbabilities.set(classTokens[i], softmaxValues[i]);
-  }
-
-  return tokenProbabilities;
-}
-
-export function getTokenProbabilitiesFromLogits(logits, tokenChoices) {
-  const tokenProbabilities = [];
-  for (const logit of logits) {
-    const tokenPresent = tokenChoices.has(logit.token);
-    if (!tokenPresent) {
-      continue;
-    }
-
-    const tokenProbability = getTokenProbabilitiesFromLogit(
-      logit,
-      tokenChoices,
-    );
-    tokenProbabilities.push(tokenProbability);
-  }
-  return tokenProbabilities;
-}

package/src/halloumi/postprocessing.test.js

@@ -1,441 +0,0 @@
-import {
-  getClaimsFromResponse,
-  getTokenProbabilitiesFromLogits,
-} from './postprocessing';
-
-describe('getClaimsFromResponse', () => {
-  describe('well-formed responses', () => {
-    it('should parse a single well-formed supported claim', () => {
-      const response =
-        '<|r1|><The weather is sunny.><|subclaims|><The weather is being described.><end||subclaims><|cite|><|s1|><end||cite><|supported|><|explain|><The document states it is sunny.><end||explain><end||r>';
-
-      const claims = getClaimsFromResponse(response);
-
-      expect(claims).toHaveLength(1);
-      expect(claims[0].claimId).toBe(1);
-      expect(claims[0].claimString).toBe('The weather is sunny.');
-      expect(claims[0].subclaims).toEqual(['The weather is being described.']);
-      expect(claims[0].segments).toEqual([1]);
-      expect(claims[0].supported).toBe(true);
-      expect(claims[0].explanation).toBe('The document states it is sunny.');
-    });
-
-    it('should parse a single well-formed unsupported claim', () => {
-      const response =
-        '<|r1|><The document mentions cats.><|subclaims|><The document makes some mention of cats.><end||subclaims><|cite|><None><end||cite><|unsupported|><|explain|><There is no mention of cats.><end||explain><end||r>';
-
-      const claims = getClaimsFromResponse(response);
-
-      expect(claims).toHaveLength(1);
-      expect(claims[0].claimId).toBe(1);
-      expect(claims[0].supported).toBe(false);
-      expect(claims[0].segments).toEqual([]);
-    });
-
-    it('should parse multiple well-formed claims', () => {
-      const response =
-        '<|r1|><First claim.><|subclaims|><Sub1.><end||subclaims><|cite|><|s1|><end||cite><|supported|><|explain|><Explanation 1.><end||explain><end||r>' +
-        '<|r2|><Second claim.><|subclaims|><Sub2.><end||subclaims><|cite|><|s2|,|s3|><end||cite><|unsupported|><|explain|><Explanation 2.><end||explain><end||r>' +
-        '<|r3|><Third claim.><|subclaims|><Sub3.><end||subclaims><|cite|><|s4|><end||cite><|supported|><|explain|><Explanation 3.><end||explain><end||r>';
-
-      const claims = getClaimsFromResponse(response);
-
-      expect(claims).toHaveLength(3);
-      expect(claims[0].claimId).toBe(1);
-      expect(claims[0].supported).toBe(true);
-      expect(claims[1].claimId).toBe(2);
-      expect(claims[1].supported).toBe(false);
-      expect(claims[1].segments).toEqual([2, 3]);
-      expect(claims[2].claimId).toBe(3);
-      expect(claims[2].supported).toBe(true);
-    });
-
-    it('should parse multiple subclaims correctly', () => {
-      const response =
-        '<|r1|><Main claim.><|subclaims|><Subclaim one.><Subclaim two.><Subclaim three.><end||subclaims><|cite|><|s1|><end||cite><|supported|><|explain|><Explanation.><end||explain><end||r>';
-
-      const claims = getClaimsFromResponse(response);
-
-      expect(claims).toHaveLength(1);
-      expect(claims[0].subclaims).toEqual([
-        'Subclaim one.',
-        'Subclaim two.',
-        'Subclaim three.',
-      ]);
-    });
-  });
-
-  describe('citation parsing', () => {
-    it('should parse comma-separated citations', () => {
-      const response =
-        '<|r1|><Claim.><|subclaims|><Sub.><end||subclaims><|cite|><|s1|,|s2|,|s3|,|s4|><end||cite><|supported|><|explain|><Exp.><end||explain><end||r>';
-
-      const claims = getClaimsFromResponse(response);
-
-      expect(claims[0].segments).toEqual([1, 2, 3, 4]);
-    });
-
-    it('should parse citation ranges with dash', () => {
-      const response =
-        '<|r1|><Claim.><|subclaims|><Sub.><end||subclaims><|cite|><|s1-s5|><end||cite><|supported|><|explain|><Exp.><end||explain><end||r>';
-
-      const claims = getClaimsFromResponse(response);
-
-      expect(claims[0].segments).toEqual([1, 2, 3, 4, 5]);
-    });
-
-    it('should parse citation ranges with "to"', () => {
-      const response =
-        '<|r1|><Claim.><|subclaims|><Sub.><end||subclaims><|cite|><|s1 to s49|><end||cite><|supported|><|explain|><Exp.><end||explain><end||r>';
-
-      const claims = getClaimsFromResponse(response);
-
-      expect(claims[0].segments).toHaveLength(49);
-      expect(claims[0].segments[0]).toBe(1);
-      expect(claims[0].segments[48]).toBe(49);
-    });
-
-    it('should handle "None" citations', () => {
-      const response =
-        '<|r1|><Claim.><|subclaims|><Sub.><end||subclaims><|cite|><None><end||cite><|unsupported|><|explain|><Exp.><end||explain><end||r>';
-
-      const claims = getClaimsFromResponse(response);
-
-      expect(claims[0].segments).toEqual([]);
-    });
-
-    it('should parse mixed citation formats', () => {
-      const response =
-        '<|r1|><Claim.><|subclaims|><Sub.><end||subclaims><|cite|><|s1|,|s3-s5|,|s10|><end||cite><|supported|><|explain|><Exp.><end||explain><end||r>';
-
-      const claims = getClaimsFromResponse(response);
-
-      expect(claims[0].segments).toEqual([1, 3, 4, 5, 10]);
-    });
-  });
-
-  describe('malformed responses with orphan segments', () => {
-    it('should merge orphan segment with preceding claim', () => {
-      // Malformed: subclaims appear after <end||r>
-      const response =
-        '<|r1|><First claim with proper format.><|subclaims|><Sub1.><end||subclaims><|cite|><|s1|><end||cite><|supported|><|explain|><Exp1.><end||explain><end||r>' +
-        '<|r2|><Second claim without subclaims.><end||r>' +
-        '<|subclaims|><Sub2.><end||subclaims><|cite|><|s2|><end||cite><|supported|><|explain|><Exp2.><end||explain><end||r>';
-
-      const claims = getClaimsFromResponse(response);
-
-      expect(claims).toHaveLength(2);
-      expect(claims[0].claimId).toBe(1);
-      expect(claims[0].claimString).toBe('First claim with proper format.');
-      expect(claims[0].subclaims).toEqual(['Sub1.']);
-
-      expect(claims[1].claimId).toBe(2);
-      expect(claims[1].claimString).toBe('Second claim without subclaims.');
-      expect(claims[1].subclaims).toEqual(['Sub2.']);
-      expect(claims[1].segments).toEqual([2]);
-      expect(claims[1].supported).toBe(true);
-    });
-
-    it('should handle multiple consecutive malformed claims', () => {
-      const response =
-        '<|r1|><Claim 1.><end||r>' +
-        '<|subclaims|><Sub1.><end||subclaims><|cite|><|s1|><end||cite><|supported|><|explain|><Exp1.><end||explain><end||r>' +
-        '<|r2|><Claim 2.><end||r>' +
-        '<|subclaims|><Sub2.><end||subclaims><|cite|><|s2|><end||cite><|unsupported|><|explain|><Exp2.><end||explain><end||r>' +
-        '<|r3|><Claim 3.><end||r>' +
-        '<|subclaims|><Sub3.><end||subclaims><|cite|><|s3|><end||cite><|supported|><|explain|><Exp3.><end||explain><end||r>';
-
-      const claims = getClaimsFromResponse(response);
-
-      expect(claims).toHaveLength(3);
-      expect(claims[0].claimId).toBe(1);
-      expect(claims[0].subclaims).toEqual(['Sub1.']);
-      expect(claims[0].supported).toBe(true);
-
-      expect(claims[1].claimId).toBe(2);
-      expect(claims[1].subclaims).toEqual(['Sub2.']);
-      expect(claims[1].supported).toBe(false);
-
-      expect(claims[2].claimId).toBe(3);
-      expect(claims[2].subclaims).toEqual(['Sub3.']);
-      expect(claims[2].supported).toBe(true);
-    });
-
-    it('should skip orphan segments that are not associated with claims', () => {
-      const response =
-        '<|subclaims|><Orphan subclaim.><end||subclaims><|cite|><|s1|><end||cite><|supported|><|explain|><Orphan exp.><end||explain><end||r>' +
-        '<|r1|><Valid claim.><|subclaims|><Sub.><end||subclaims><|cite|><|s2|><end||cite><|supported|><|explain|><Exp.><end||explain><end||r>';
-
-      const claims = getClaimsFromResponse(response);
-
-      expect(claims).toHaveLength(1);
-      expect(claims[0].claimId).toBe(1);
-      expect(claims[0].claimString).toBe('Valid claim.');
-    });
-  });
-
-  describe('edge cases', () => {
-    it('should return empty array for empty response', () => {
-      const claims = getClaimsFromResponse('');
-      expect(claims).toEqual([]);
-    });
-
-    it('should return empty array for response with only end tags', () => {
-      const claims = getClaimsFromResponse('<end||r><end||r><end||r>');
-      expect(claims).toEqual([]);
-    });
-
-    it('should handle response without cite tag', () => {
-      const response =
-        '<|r1|><Claim.><|subclaims|><Sub.><end||subclaims><|supported|><|explain|><Exp.><end||explain><end||r>';
-
-      const claims = getClaimsFromResponse(response);
-
-      expect(claims).toHaveLength(1);
-      expect(claims[0].segments).toEqual([]);
-    });
-
-    it('should handle response without explain tag', () => {
-      const response =
-        '<|r1|><Claim.><|subclaims|><Sub.><end||subclaims><|cite|><|s1|><end||cite><|supported|><end||r>';
-
-      const claims = getClaimsFromResponse(response);
-
-      expect(claims).toHaveLength(1);
-      expect(claims[0].explanation).toBe('');
-    });
-
-    it('should handle response without supported/unsupported tag (defaults to true)', () => {
-      const response =
-        '<|r1|><Claim.><|subclaims|><Sub.><end||subclaims><|cite|><|s1|><end||cite><|explain|><Exp.><end||explain><end||r>';
-
-      const claims = getClaimsFromResponse(response);
-
-      expect(claims).toHaveLength(1);
-      expect(claims[0].supported).toBe(true);
-    });
-
-    it('should handle malformed claim without orphan (missing all verification data)', () => {
-      const response = '<|r1|><Claim without any verification data.><end||r>';
-
-      const claims = getClaimsFromResponse(response);
-
-      expect(claims).toHaveLength(1);
-      expect(claims[0].claimId).toBe(1);
-      expect(claims[0].claimString).toBe(
-        'Claim without any verification data.',
-      );
-      expect(claims[0].subclaims).toEqual([]);
-      expect(claims[0].segments).toEqual([]);
-      expect(claims[0].explanation).toBe('');
-      expect(claims[0].supported).toBe(true); // defaults to true
-    });
-
-    it('should handle claims with special characters in content', () => {
-      const response =
-        '<|r1|><CO₂ emissions ≈ 417 Mt (–30% vs. 1990).><|subclaims|><Emissions data with special chars: €, £, ¥.><end||subclaims><|cite|><|s1|><end||cite><|supported|><|explain|><Unicode: émissions, naïve.><end||explain><end||r>';
-
-      const claims = getClaimsFromResponse(response);
-
-      expect(claims).toHaveLength(1);
-      expect(claims[0].claimString).toBe(
-        'CO₂ emissions ≈ 417 Mt (–30% vs. 1990).',
-      );
-      expect(claims[0].subclaims).toEqual([
-        'Emissions data with special chars: €, £, ¥.',
-      ]);
-    });
-
-    it('should handle claims with markdown table content', () => {
-      const response =
-        '<|r1|><| Year | Emissions |><|subclaims|><Table data.><end||subclaims><|cite|><|s1|><end||cite><|supported|><|explain|><Contains table.><end||explain><end||r>';
-
-      const claims = getClaimsFromResponse(response);
-
-      expect(claims).toHaveLength(1);
-      expect(claims[0].claimString).toBe('| Year | Emissions |');
-    });
-  });
-
-  describe('real-world examples', () => {
-    it('should parse the example from the code comments', () => {
-      const response =
-        '<|r1|><There is no information about the average lifespan of a giant squid in the deep waters of the Pacific Ocean in the provided document.><|subclaims|><The document contains information about the average lifespan of a giant squid.><The information about giant squid lifespan is related to the Pacific Ocean.><end||subclaims><|cite|><|s1 to s49|><end||cite><|explain|><Upon reviewing the entire document, there is no mention of giant squid or any related topic, including their average lifespan or the Pacific Ocean. The document is focused on international relations, diplomacy, and conflict resolution.><end||explain><|supported|><end||r>' +
-        "<|r2|><The document is focused on international relations, diplomacy, and conflict resolution, and does not mention giant squid or any related topic.><|subclaims|><The document is focused on international relations, diplomacy, and conflict resolution.><The document does not mention giant squid or any related topic.><end||subclaims><|cite|><|s1|,|s2|,|s3|,|s4|><end||cite><|explain|><The first four sentences clearly establish the document's focus on international relations, diplomacy, and conflict resolution, and there is no mention of giant squid or any related topic throughout the document.><end||explain><|supported|><end||r>" +
-        '<|r3|><The document mentions cats.><|subclaims|><The document makes some mention of cats.><end||subclaims><|cite|><None><end||cite><|explain|><There is no mention of cats anywhere in the document.><end||explain><|unsupported|><end||r>';
-
-      const claims = getClaimsFromResponse(response);
-
-      expect(claims).toHaveLength(3);
-
-      // Claim 1
-      expect(claims[0].claimId).toBe(1);
-      expect(claims[0].subclaims).toHaveLength(2);
-      expect(claims[0].segments).toHaveLength(49);
-      expect(claims[0].supported).toBe(true);
-
-      // Claim 2
-      expect(claims[1].claimId).toBe(2);
-      expect(claims[1].segments).toEqual([1, 2, 3, 4]);
-      expect(claims[1].supported).toBe(true);
-
-      // Claim 3
-      expect(claims[2].claimId).toBe(3);
-      expect(claims[2].segments).toEqual([]);
-      expect(claims[2].supported).toBe(false);
-    });
-  });
-});
-
-describe('getTokenProbabilitiesFromLogits', () => {
-  const tokenChoices = new Set(['supported', 'unsupported']);
-
-  it('should extract probabilities for matching tokens', () => {
-    const logits = [
-      {
-        token: 'supported',
-        logprob: -0.1,
-        top_logprobs: [
-          { token: 'supported', logprob: -0.1 },
-          { token: 'unsupported', logprob: -2.5 },
-        ],
-      },
-    ];
-
-    const probabilities = getTokenProbabilitiesFromLogits(logits, tokenChoices);
-
-    expect(probabilities).toHaveLength(1);
-    expect(probabilities[0].get('supported')).toBeGreaterThan(0.5);
-    expect(probabilities[0].get('unsupported')).toBeLessThan(0.5);
-  });
-
-  it('should skip non-matching tokens', () => {
-    const logits = [
-      {
-        token: '<',
-        logprob: -0.001,
-        top_logprobs: [{ token: '<', logprob: -0.001 }],
-      },
-      {
-        token: '|',
-        logprob: -0.001,
-        top_logprobs: [{ token: '|', logprob: -0.001 }],
-      },
-      {
-        token: 'supported',
-        logprob: -0.1,
-        top_logprobs: [
-          { token: 'supported', logprob: -0.1 },
-          { token: 'unsupported', logprob: -2.5 },
-        ],
-      },
-      {
-        token: '|',
-        logprob: -0.001,
-        top_logprobs: [{ token: '|', logprob: -0.001 }],
-      },
-    ];
-
-    const probabilities = getTokenProbabilitiesFromLogits(logits, tokenChoices);
-
-    expect(probabilities).toHaveLength(1);
-  });
-
-  it('should handle multiple supported/unsupported tokens', () => {
-    const logits = [
-      {
-        token: 'supported',
-        logprob: -0.1,
-        top_logprobs: [
-          { token: 'supported', logprob: -0.1 },
-          { token: 'unsupported', logprob: -3.0 },
-        ],
-      },
-      {
-        token: 'unsupported',
-        logprob: -0.2,
-        top_logprobs: [
-          { token: 'unsupported', logprob: -0.2 },
-          { token: 'supported', logprob: -2.8 },
-        ],
-      },
-      {
-        token: 'supported',
-        logprob: -0.5,
-        top_logprobs: [
-          { token: 'supported', logprob: -0.5 },
-          { token: 'unsupported', logprob: -1.5 },
-        ],
-      },
-    ];
-
-    const probabilities = getTokenProbabilitiesFromLogits(logits, tokenChoices);
-
-    expect(probabilities).toHaveLength(3);
-    // First token: strongly supported
-    expect(probabilities[0].get('supported')).toBeGreaterThan(0.9);
-    // Second token: strongly unsupported
-    expect(probabilities[1].get('unsupported')).toBeGreaterThan(0.9);
-    // Third token: moderately supported
-    expect(probabilities[2].get('supported')).toBeGreaterThan(0.5);
-  });
-
-  it('should return empty array for empty logits', () => {
-    const probabilities = getTokenProbabilitiesFromLogits([], tokenChoices);
-    expect(probabilities).toEqual([]);
-  });
-
-  it('should return empty array when no matching tokens', () => {
-    const logits = [
-      {
-        token: 'other',
-        logprob: -0.1,
-        top_logprobs: [{ token: 'other', logprob: -0.1 }],
-      },
-    ];
-
-    const probabilities = getTokenProbabilitiesFromLogits(logits, tokenChoices);
-    expect(probabilities).toEqual([]);
-  });
-
-  it('should handle missing token in top_logprobs', () => {
-    const logits = [
-      {
-        token: 'supported',
-        logprob: -0.1,
-        top_logprobs: [
-          { token: 'supported', logprob: -0.1 },
-          // 'unsupported' not in top_logprobs
-        ],
-      },
-    ];
-
-    const probabilities = getTokenProbabilitiesFromLogits(logits, tokenChoices);
-
-    expect(probabilities).toHaveLength(1);
-    // Should still have both keys
-    expect(probabilities[0].has('supported')).toBe(true);
-    expect(probabilities[0].has('unsupported')).toBe(true);
-    // Supported should have high probability since unsupported is estimated lower
-    expect(probabilities[0].get('supported')).toBeGreaterThan(0.5);
-  });
-
-  it('should apply softmax correctly', () => {
-    const logits = [
-      {
-        token: 'supported',
-        logprob: 0, // e^0 = 1
-        top_logprobs: [
-          { token: 'supported', logprob: 0 },
-          { token: 'unsupported', logprob: 0 }, // e^0 = 1
-        ],
-      },
-    ];
-
-    const probabilities = getTokenProbabilitiesFromLogits(logits, tokenChoices);
-
-    // With equal logprobs, softmax should give 0.5 each
-    expect(probabilities[0].get('supported')).toBeCloseTo(0.5, 5);
-    expect(probabilities[0].get('unsupported')).toBeCloseTo(0.5, 5);
-  });
-});