@eeacms/volto-eea-chatbot 3.0.1 → 4.0.0

package/src/halloumi/markdown-splitter.js

@@ -1,172 +0,0 @@
-import nlp from 'compromise';
-
-const MIN_SENTENCE_LENGTH = 15;
-
-/**
- * Splits markdown text into segments meaningful for fact-checking.
- *
- * Strategy:
- * 1. Split into markdown blocks (paragraphs, list items, table rows, headers)
- * 2. For prose blocks, further split into sentences using Intl.Segmenter
- * 3. Merge short fragments (< MIN_SENTENCE_LENGTH) into the next sentence
- *
- * Structural blocks (table rows, headers, list items) are kept as-is since
- * they are already atomic units.
- *
- * @param {string} text Markdown text to split.
- * @returns {string[]} Array of segment strings.
- */
-export function splitMarkdown(text) {
-  const blocks = splitIntoBlocks(text);
-  const segments = [];
-
-  for (const block of blocks) {
-    if (isStructuralBlock(block)) {
-      if (block.trim().length > 0) {
-        segments.push(block);
-      }
-    } else {
-      segments.push(...splitProse(block));
-    }
-  }
-
-  return segments;
-}
-
-/**
- * Splits markdown text into structural blocks.
- * Separates: headers, table rows, list items, horizontal rules, and prose.
- * Prose lines within the same paragraph are joined together.
- */
-function splitIntoBlocks(text) {
-  const lines = text.split('\n');
-  const blocks = [];
-  let currentProse = '';
-
-  const flushProse = () => {
-    if (currentProse) {
-      blocks.push(currentProse);
-      currentProse = '';
-    }
-  };
-
-  for (const line of lines) {
-    const trimmed = line.trimStart();
-
-    if (isTableRow(trimmed)) {
-      flushProse();
-      blocks.push(line);
-    } else if (isTableSeparator(trimmed)) {
-      flushProse();
-      // Skip separator rows — not verifiable content
-    } else if (isHeader(trimmed)) {
-      flushProse();
-      blocks.push(line);
-    } else if (isHorizontalRule(trimmed)) {
-      flushProse();
-      // Skip horizontal rules
-    } else if (isListItem(trimmed)) {
-      flushProse();
-      blocks.push(line);
-    } else if (trimmed === '') {
-      flushProse();
-    } else {
-      // Prose continuation — join into a single paragraph
-      currentProse += (currentProse ? ' ' : '') + line.trim();
-    }
-  }
-
-  flushProse();
-  return blocks;
-}
-
-function isTableRow(line) {
-  return /^\|.*\|/.test(line) && !isTableSeparator(line);
-}
-
-function isTableSeparator(line) {
-  return /^\|[\s\-:|]+\|$/.test(line);
-}
-
-function isHeader(line) {
-  return /^#{1,6}\s/.test(line);
-}
-
-function isHorizontalRule(line) {
-  return /^(\*{3,}|-{3,}|_{3,})\s*$/.test(line);
-}
-
-function isListItem(line) {
-  return /^(\d+\.\s+|[-*+]\s+)/.test(line);
-}
-
-/**
- * Returns true if a block is structural (table row, header, list item)
- * and should not be further split into sentences.
- */
-function isStructuralBlock(block) {
-  const trimmed = block.trimStart();
-  return isTableRow(trimmed) || isHeader(trimmed) || isListItem(trimmed);
-}
-
-/**
- * Splits a prose paragraph into sentences using sbd (sentence boundary detection),
- * merging short fragments that aren't independently verifiable.
- * sbd handles abbreviations (Dr., Mr., U.S., Ph.D.) correctly.
- */
-export function splitProse(text, maxSentences = 0) {
-  // const segments = segment('en', text);
-  const doc = nlp(text);
-  const initialSentences = doc.sentences().out('array');
-
-  // Find each sentence's position in the original text
-  const positions = getPositions(text, initialSentences);
-
-  // Merge short sentences (< MIN_SENTENCE_LENGTH) into the next sentence
-  const merged = [];
-  const mergedPositions = [];
-  let pendingStart = null;
-  for (let i = 0; i < initialSentences.length; i++) {
-    if (pendingStart === null) {
-      pendingStart = positions[i].start;
-    }
-    if (
-      initialSentences[i].replaceAll('\n', '').length < MIN_SENTENCE_LENGTH &&
-      i < initialSentences.length - 1
-    ) {
-      // Too short — will be merged with the next sentence
-      continue;
-    }
-    const s = text.slice(pendingStart, positions[i].end);
-    merged.push(s);
-    mergedPositions.push({ start: pendingStart, end: positions[i].end });
-    pendingStart = null;
-  }
-
-  if (maxSentences && merged.length > maxSentences) {
-    // Merge groups by slicing original text to preserve separators
-    const groupSize = Math.ceil(merged.length / maxSentences);
-    const groupedSentences = [];
-    for (let i = 0; i < merged.length; i += groupSize) {
-      const groupStart = mergedPositions[i].start;
-      const groupEnd =
-        mergedPositions[Math.min(i + groupSize, merged.length) - 1].end;
-      groupedSentences.push(text.slice(groupStart, groupEnd));
-    }
-    return groupedSentences;
-  }
-
-  return merged;
-}
-
-function getPositions(text, sentences) {
-  const positions = [];
-  let searchFrom = 0;
-  for (const sentence of sentences) {
-    const start = text.indexOf(sentence, searchFrom);
-    const end = start + sentence.length;
-    positions.push({ start, end });
-    searchFrom = end;
-  }
-  return positions;
-}

package/src/halloumi/markdown-splitter.test.js

@@ -1,133 +0,0 @@
-import { splitMarkdown } from './markdown-splitter';
-
-const DUMMY_LLM_RESPONSE = `## European species most at risk
-
-Below is a concise, evidence-based snapshot of the taxonomic groups and flagship species. The list combines information on trend magnitude and risk status.
-
-| Taxonomic group | Species at risk | Main drivers | Signal |
-|-----------------|-----------------|--------------|--------|
-| **Birds** | Skylark (*Alauda arvensis*), European turtle-dove | Intensive agriculture, pesticide use | Farmland-bird index down 27% (1990-2019) |
-| **Fish** | Houting (*Coregonus oxyrhynchus*), Danube salmon | River regulation, dam construction | Both listed as critically endangered |
-| **Butterflies** | Large blue (*Phengaris arion*), Adonis blue | Loss of semi-natural grasslands | Grassland-butterfly index down 25% |
-
-### Key take-aways
-
-1. **Agricultural intensification** is the single biggest pressure across taxa.
-2. **Habitat loss and degradation** underlie the perilous status of many specialists.
-3. **Chemical stressors** affect both insects and the birds that depend on them.
-4. **Climate change** amplifies existing pressures such as drought stress on amphibians.
-
----
-
-### Conservation priorities
-
-- **Protect and restore semi-natural habitats** including high-diversity grasslands and river floodplains.
-- **Implement agri-environment schemes** that limit pesticide use and maintain field margins.
-- **Focus monitoring on flagship species** to gauge the effectiveness of policy actions.
-
-If you need a deeper dive into a particular taxon, just let me know!`;
-
-describe('splitMarkdown', () => {
-  it('should split the dummy LLM response into meaningful segments', () => {
-    const segments = splitMarkdown(DUMMY_LLM_RESPONSE);
-
-    // Headers are separate segments
-    expect(segments).toContain('## European species most at risk');
-    expect(segments).toContain('### Key take-aways');
-    expect(segments).toContain('### Conservation priorities');
-
-    // Table rows are individual segments (separator skipped)
-    const tableRows = segments.filter((s) => s.startsWith('|'));
-    expect(tableRows.length).toBe(4); // header row + 3 data rows
-
-    // Numbered list items are separate
-    const numberedItems = segments.filter((s) => /^\d+\./.test(s.trimStart()));
-    expect(numberedItems.length).toBe(4);
-
-    // Bullet list items are separate
-    const bulletItems = segments.filter((s) => /^-\s/.test(s.trimStart()));
-    expect(bulletItems.length).toBe(3);
-
-    // Prose paragraphs are split into sentences
-    expect(segments).toContainEqual(
-      expect.stringContaining('evidence-based snapshot'),
-    );
-    expect(segments).toContainEqual(expect.stringContaining('trend magnitude'));
-
-    // Horizontal rules are skipped
-    expect(segments).not.toContainEqual(expect.stringMatching(/^-{3,}$/));
-
-    // No empty segments
-    expect(segments.every((s) => s.trim().length > 0)).toBe(true);
-  });
-
-  it('should handle plain prose without markdown', () => {
-    const text =
-      'This is sentence one. This is sentence two. This is sentence three.';
-    const segments = splitMarkdown(text);
-    expect(segments).toEqual([
-      'This is sentence one.',
-      'This is sentence two.',
-      'This is sentence three.',
-    ]);
-  });
-
-  it('should handle headers followed by prose', () => {
-    const text = '## My Header\n\nSome prose paragraph here.';
-    const segments = splitMarkdown(text);
-    expect(segments).toEqual(['## My Header', 'Some prose paragraph here.']);
-  });
-
-  it('should skip table separator rows', () => {
-    const text = '| A | B |\n|---|---|\n| 1 | 2 |';
-    const segments = splitMarkdown(text);
-    expect(segments).toEqual(['| A | B |', '| 1 | 2 |']);
-  });
-
-  it('should handle numbered lists', () => {
-    const text =
-      '1. First item with enough text.\n2. Second item also long enough.';
-    const segments = splitMarkdown(text);
-    expect(segments).toEqual([
-      '1. First item with enough text.',
-      '2. Second item also long enough.',
-    ]);
-  });
-
-  it('should handle bullet lists', () => {
-    const text =
-      '- First bullet point.\n- Second bullet point.\n- Third bullet point.';
-    const segments = splitMarkdown(text);
-    expect(segments).toEqual([
-      '- First bullet point.',
-      '- Second bullet point.',
-      '- Third bullet point.',
-    ]);
-  });
-
-  it('should skip horizontal rules', () => {
-    const text = 'Before the rule.\n\n---\n\nAfter the rule.';
-    const segments = splitMarkdown(text);
-    expect(segments).toEqual(['Before the rule.', 'After the rule.']);
-  });
-
-  it('should return an empty array for empty string', () => {
-    expect(splitMarkdown('')).toEqual([]);
-  });
-
-  it('should merge short prose fragments', () => {
-    const text = 'Hi. This is a longer sentence here.';
-    const segments = splitMarkdown(text);
-    expect(segments).toEqual(['Hi. This is a longer sentence here.']);
-  });
-
-  it('should join multi-line prose into a single paragraph before splitting', () => {
-    const text =
-      'This is the first line of a paragraph.\nThis is the second line of the same paragraph.';
-    const segments = splitMarkdown(text);
-    // Should be joined and then split by Intl.Segmenter
-    expect(segments.length).toBeGreaterThanOrEqual(1);
-    expect(segments.join('')).toContain('first line');
-    expect(segments.join('')).toContain('second line');
-  });
-});

package/src/halloumi/postprocessing.js

@@ -1,247 +0,0 @@
-/**
- * Gets the claim id from a subsegment.
- * @param subsegment A subsegment string of the form "<|r1|".
- * @returns The numeric claim id.
- */
-function getClaimIdFromSubsegment(subsegment) {
-  const textClaimId = subsegment.split('|')[1];
-  const integerClaimId = parseInt(textClaimId.split('r')[1]);
-  return integerClaimId;
-}
-
-/**
- * Gets the citations from a subsegment.
- * @param subsegment A subsegment string of the form "|s1|,|s2|,|s3|,|s4|".
- * @returns The list of numeric citations.
- */
-function getClaimCitationsFromSubsegment(subsegment) {
-  const citationSegments = subsegment.split(',');
-  const segments = [];
-  for (const citationSegment of citationSegments) {
-    const segment = citationSegment.replaceAll('|', '').replaceAll('s', '');
-    if (segment.includes('-')) {
-      const segmentRange = segment.split('-');
-      for (
-        let i = parseInt(segmentRange[0].trim());
-        i <= parseInt(segmentRange[1].trim());
-        i++
-      ) {
-        segments.push(i);
-      }
-    } else if (segment.includes('to')) {
-      const segmentRange = segment.split('to');
-      for (
-        let i = parseInt(segmentRange[0].trim());
-        i <= parseInt(segmentRange[1].trim());
-        i++
-      ) {
-        segments.push(i);
-      }
-    } else {
-      const segmentInt = parseInt(segment);
-      if (!isNaN(segmentInt)) {
-        segments.push(parseInt(segment));
-      }
-    }
-  }
-  return segments;
-}
-
-/**
- * Gets the support status from a subsegment.
- * @param subsegment A subsegment string of the form "|supported|" or "|unsupported|".
- * @returns True if the claim is supported, false otherwise.
- */
-function getSupportStatusFromSubsegment(subsegment) {
-  return subsegment.startsWith('|supported|');
-}
-
-/**
- * Gets a claim from a segment string.
- * @param segment A segment string containing all information for claim verification.
- * @returns The claim object with all relevant information.
- */
-function getClaimFromSegment(segment) {
-  const claim_segments = segment.split('><');
-  const claimId = getClaimIdFromSubsegment(claim_segments[0]);
-  // Strip trailing '>' which can occur in malformed segments without subclaims
-  const claimString = claim_segments[1]?.replace(/>$/, '') || '';
-
-  const subclaims = [];
-  let claimProgressIndex = 3; // Start at 3 to skip the claim id, claim string and the subclaims tag
-  for (let i = claimProgressIndex; i < claim_segments.length; i++) {
-    const subsegment = claim_segments[i];
-    if (subsegment.startsWith('end||subclaims')) {
-      claimProgressIndex = i + 1;
-      break;
-    } else {
-      subclaims.push(subsegment);
-    }
-  }
-
-  let cite_tag_index = -1;
-  let explanation_index = -1;
-  let label_index = -1;
-  for (let i = claimProgressIndex; i < claim_segments.length; i++) {
-    const subsegment = claim_segments[i];
-    if (subsegment.startsWith('|cite|')) {
-      cite_tag_index = i;
-    } else if (subsegment.startsWith('|explain|')) {
-      explanation_index = i + 1;
-    } else if (
-      subsegment.startsWith('|supported|') ||
-      subsegment.startsWith('|unsupported|')
-    ) {
-      label_index = i;
-    }
-  }
-
-  const segments =
-    cite_tag_index >= 0
-      ? getClaimCitationsFromSubsegment(claim_segments[cite_tag_index + 1])
-      : [];
-  const explanation =
-    explanation_index >= 0 ? claim_segments[explanation_index] : '';
-
-  // Default to true if label not found
-  const supported =
-    label_index >= 0
-      ? getSupportStatusFromSubsegment(claim_segments[label_index])
-      : true;
-
-  const claim = {
-    claimId,
-    claimString,
-    subclaims,
-    segments,
-    explanation,
-    supported,
-    probabilities: new Map(),
-  };
-
-  return claim;
-}
-
-/**
- * Gets all claims from a response.
- * @param response A string containing all claims and their information.
- * @returns A list of claim objects.
- */
-export function getClaimsFromResponse(response) {
-  // Example response: <|r1|><There is no information about the average lifespan of a giant squid in the deep waters of the Pacific Ocean in the provided document.><|subclaims|><The document contains information about the average lifespan of a giant squid.><The information about giant squid lifespan is related to the Pacific Ocean.><end||subclaims><|cite|><|s1 to s49|><end||cite><|explain|><Upon reviewing the entire document, there is no mention of giant squid or any related topic, including their average lifespan or the Pacific Ocean. The document is focused on international relations, diplomacy, and conflict resolution.><end||explain><|supported|><end||r><|r2|><The document is focused on international relations, diplomacy, and conflict resolution, and does not mention giant squid or any related topic.><|subclaims|><The document is focused on international relations, diplomacy, and conflict resolution.><The document does not mention giant squid or any related topic.><end||subclaims><|cite|><|s1|,|s2|,|s3|,|s4|><end||cite><|explain|><The first four sentences clearly establish the document's focus on international relations, diplomacy, and conflict resolution, and there is no mention of giant squid or any related topic throughout the document.><end||explain><|supported|><end||r><|r3|><The document mentions cats.><|subclaims|><The document makes some mention of cats.><end||subclaims><|cite|><None><end||cite><|explain|><There is no mention of cats anywhere in the document.><end||explain><|unsupported|><end||r>
-
-  // eslint-disable-next-line no-console
-  // console.log('getClaimsFromResponse', response);
-  let segments = response.split('<end||r>');
-  const claims = [];
-
-  for (let i = 0; i < segments.length; i++) {
-    const segment = segments[i];
-
-    if (segment.length === 0) {
-      continue;
-    }
-
-    // Skip segments that don't start with a valid claim marker <|r*|>
-    if (!segment.match(/^<\|r\d+\|>/)) {
-      continue;
-    }
-
-    // Check if this segment is well-formed (has subclaims within it)
-    // Well-formed: <|r1|><claim><|subclaims|>...<|supported|>...<end||r>
-    // Malformed:   <|r1|><claim><end||r><|subclaims|>...<|supported|>...<end||r>
-    // The malformed case puts verification data AFTER <end||r>, creating orphan segments
-    if (segment.includes('<|subclaims|>')) {
-      // Well-formed segment - process normally
-      const claim = getClaimFromSegment(segment);
-      claims.push(claim);
-    } else {
-      // Malformed segment - look for orphan data in the next segment
-      let fullSegment = segment;
-
-      // Look ahead for orphan segment (starts with <|subclaims|>)
-      if (i + 1 < segments.length) {
-        const nextSegment = segments[i + 1];
-        if (nextSegment.startsWith('<|subclaims|>')) {
-          // Merge: segment ends with '>', orphan starts with '<'
-          fullSegment = segment + nextSegment;
-          i++; // Skip the orphan in next iteration
-        }
-      }
-
-      const claim = getClaimFromSegment(fullSegment);
-      claims.push(claim);
-    }
-  }
-
-  return claims;
-}
-
-function exp(x) {
-  return Math.pow(Math.E, x);
-}
-
-function softmax(logits) {
-  const softmaxes = [];
-  const exp_values = [];
-  let total = 0;
-  for (let i = 0; i < logits.length; i++) {
-    const exponential = exp(logits[i]);
-    total += exponential;
-    exp_values.push(exponential);
-  }
-
-  for (let i = 0; i < exp_values.length; i++) {
-    softmaxes.push(exp_values[i] / total);
-  }
-  return softmaxes;
-}
-
-function getTokenProbabilitiesFromLogit(logit, tokenChoices) {
-  const tokenLogits = new Map();
-  let smallestLogit = 1000000;
-  for (const token of tokenChoices) {
-    const tokenLogit = logit.top_logprobs.find(
-      (logit) => logit.token === token,
-    );
-    if (tokenLogit !== undefined) {
-      smallestLogit = Math.min(smallestLogit, tokenLogit.logprob);
-      tokenLogits.set(token, tokenLogit.logprob);
-    }
-  }
-
-  // If any class tokens aren't present, set them below the smallest logit.
-  // Their true value is definitely <= the smallest logit value.
-  for (const token of tokenChoices) {
-    if (!tokenLogits.has(token)) {
-      tokenLogits.set(token, smallestLogit - 1e-6);
-    }
-  }
-
-  const classTokens = Array.from(tokenLogits.keys());
-  const logitValues = Array.from(tokenLogits.values());
-  const softmaxValues = softmax(logitValues);
-  const tokenProbabilities = new Map();
-  for (let i = 0; i < classTokens.length; i++) {
-    tokenProbabilities.set(classTokens[i], softmaxValues[i]);
-  }
-
-  return tokenProbabilities;
-}
-
-export function getTokenProbabilitiesFromLogits(logits, tokenChoices) {
-  const tokenProbabilities = [];
-  for (const logit of logits) {
-    const tokenPresent = tokenChoices.has(logit.token);
-    if (!tokenPresent) {
-      continue;
-    }
-
-    const tokenProbability = getTokenProbabilitiesFromLogit(
-      logit,
-      tokenChoices,
-    );
-    tokenProbabilities.push(tokenProbability);
-  }
-  return tokenProbabilities;
-}