@eeacms/volto-eea-chatbot 3.0.0 → 4.0.0

package/src/halloumi/preprocessing.js

@@ -1,117 +0,0 @@
-const MAX_CONTEXT_SEGMENTS_PER_CHUNK = 100;
-
-/**
- * Annotate a set of sentences with a given annotation character.
- * @param sentences A list of sentences to annotate.
- * @param annotationChar The character to use for annotation.
- * @returns The annotated string with annotation characters + sentence number.
- */
-export function annotate(sentences, annotationChar, excludeIndices) {
-  return sentences
-    .map((sentence, i) => {
-      const id = i + 1;
-      if (excludeIndices && excludeIndices.has(id)) {
-        return '';
-      }
-      return `<|${annotationChar}${id}|><${sentence}><end||${annotationChar}>`;
-    })
-    .join('');
-}
-
-/**
- * Annotates a chunk of indexed sentences with their global IDs.
- */
-function annotateChunk(chunk, annotationChar) {
-  return chunk
-    .map(
-      ({ sentence, globalId }) =>
-        `<|${annotationChar}${globalId}|><${sentence}><end||${annotationChar}>`,
-    )
-    .join('');
-}
-
-export function getOffsets(originalString, sentences) {
-  const offsets = new Map();
-  let stringProgressPointer = 0;
-  let sentenceId = 1;
-  for (const sentence of sentences) {
-    const stringToSearch = originalString.slice(stringProgressPointer);
-    const startOffset =
-      stringToSearch.indexOf(sentence) + stringProgressPointer;
-    const endOffset = startOffset + sentence.length;
-    stringProgressPointer = endOffset;
-    offsets.set(sentenceId, { startOffset: startOffset, endOffset: endOffset });
-    sentenceId++;
-  }
-  return offsets;
-}
-
-/**
- * Creates multiple HallOumi prompts by chunking context segments.
- * Each chunk uses global segment IDs (s5, s42, ...) so no local-to-global
- * mapping is needed when merging results.
- *
- * @returns {{ prompts: Array<{prompt, responseOffsets}> }}
- */
-export function createChunkedHalloumiPrompts({
-  indexedContextSentences,
-  responseSentences,
-  responseOffsets,
-  request = 'Make one or more claims about information in the documents.',
-  excludeResponseIndices,
-}) {
-  // Build response annotation (same for all chunks)
-  const annotatedResponseSentences = annotate(
-    responseSentences || [],
-    'r',
-    excludeResponseIndices,
-  );
-  const annotatedRequest = `<|request|><${request}><end||request>`;
-  const annotatedResponse = `<|response|>${annotatedResponseSentences}<end||response>`;
-
-  // Group sentences by source
-  const sourceGroups = [];
-  let currentSourceId = null;
-  for (const s of indexedContextSentences) {
-    if (s.sourceId !== currentSourceId) {
-      sourceGroups.push([]);
-      currentSourceId = s.sourceId;
-    }
-    sourceGroups[sourceGroups.length - 1].push(s);
-  }
-
-  // Pack whole sources into chunks (first-fit decreasing bin packing)
-  // Sort by size descending so large sources get placed first
-  const sorted = [...sourceGroups].sort((a, b) => b.length - a.length);
-  const chunks = [];
-  const chunkSizes = [];
-  for (const group of sorted) {
-    let placed = false;
-    for (let c = 0; c < chunks.length; c++) {
-      if (chunkSizes[c] + group.length <= MAX_CONTEXT_SEGMENTS_PER_CHUNK) {
-        chunks[c].push(...group);
-        chunkSizes[c] += group.length;
-        placed = true;
-        break;
-      }
-    }
-    if (!placed) {
-      chunks.push([...group]);
-      chunkSizes.push(group.length);
-    }
-  }
-  if (chunks.length === 0) chunks.push([]);
-
-  // Build one prompt per chunk with global segment IDs
-  const prompts = chunks.map((chunk) => {
-    const annotatedContext = `<|context|>${annotateChunk(
-      chunk,
-      's',
-    )}<end||context>`;
-    const prompt = `${annotatedContext}${annotatedRequest}${annotatedResponse}`;
-
-    return { prompt, responseOffsets };
-  });
-
-  return { prompts };
-}

package/src/halloumi/preprocessing.test.js

@@ -1,184 +0,0 @@
-import {
-  createChunkedHalloumiPrompts,
-  annotate,
-  getOffsets,
-} from './preprocessing';
-
-describe('annotate', () => {
-  it('should annotate multiple sentences correctly', () => {
-    const sentences = ['Sentence one.', 'Sentence two.'];
-    const annotationChar = 's';
-    const expected =
-      '<|s1|><Sentence one.><end||s><|s2|><Sentence two.><end||s>';
-    expect(annotate(sentences, annotationChar)).toEqual(expected);
-  });
-
-  it('should handle an empty array of sentences', () => {
-    const sentences = [];
-    const annotationChar = 's';
-    expect(annotate(sentences, annotationChar)).toEqual('');
-  });
-
-  it('should annotate a single sentence', () => {
-    const sentences = ['Single sentence.'];
-    const annotationChar = 'r';
-    const expected = '<|r1|><Single sentence.><end||r>';
-    expect(annotate(sentences, annotationChar)).toEqual(expected);
-  });
-
-  it('should use different annotation characters', () => {
-    const sentences = ['Hello.'];
-    const annotationChar = 'x';
-    const expected = '<|x1|><Hello.><end||x>';
-    expect(annotate(sentences, annotationChar)).toEqual(expected);
-  });
-});
-
-describe('getOffsets', () => {
-  it('should calculate correct offsets for multiple sentences', () => {
-    const originalString = 'First sentence. Second sentence. Third sentence.';
-    const sentences = [
-      'First sentence. ',
-      'Second sentence. ',
-      'Third sentence.',
-    ];
-    const expected = new Map([
-      [1, { startOffset: 0, endOffset: 16 }],
-      [2, { startOffset: 16, endOffset: 33 }],
-      [3, { startOffset: 33, endOffset: 48 }],
-    ]);
-    expect(getOffsets(originalString, sentences)).toEqual(expected);
-  });
-
-  it('should handle empty original string and sentences', () => {
-    const originalString = '';
-    const sentences = [];
-    expect(getOffsets(originalString, sentences)).toEqual(new Map());
-  });
-
-  it('should handle original string matching sentences exactly', () => {
-    const originalString = 'Hello world.';
-    const sentences = ['Hello world.'];
-    const expected = new Map([[1, { startOffset: 0, endOffset: 12 }]]);
-    expect(getOffsets(originalString, sentences)).toEqual(expected);
-  });
-
-  it('should handle original string with leading/trailing spaces', () => {
-    const originalString = '  Sentence one.  Sentence two.  ';
-    const sentences = ['Sentence one.  ', 'Sentence two.  '];
-    const expected = new Map([
-      [1, { startOffset: 2, endOffset: 17 }],
-      [2, { startOffset: 17, endOffset: 32 }],
-    ]);
-    expect(getOffsets(originalString, sentences)).toEqual(expected);
-  });
-
-  it('should handle sentences with special characters', () => {
-    const originalString = 'Sentence with !@#$%^&*() special characters.';
-    const sentences = ['Sentence with !@#$%^&*() special characters.'];
-    const expected = new Map([[1, { startOffset: 0, endOffset: 44 }]]);
-    expect(getOffsets(originalString, sentences)).toEqual(expected);
-  });
-});
-
-describe('createChunkedHalloumiPrompts', () => {
-  it('should create a single chunk for small input', () => {
-    const indexedContextSentences = [
-      { sentence: 'Context one.', sourceId: 1, globalId: 1 },
-      { sentence: 'Context two.', sourceId: 1, globalId: 2 },
-    ];
-    const responseSentences = ['Response one.', 'Response two.'];
-    const responseOffsets = new Map([
-      [1, { startOffset: 0, endOffset: 13 }],
-      [2, { startOffset: 14, endOffset: 28 }],
-    ]);
-
-    const { prompts } = createChunkedHalloumiPrompts({
-      indexedContextSentences,
-      responseSentences,
-      responseOffsets,
-    });
-
-    expect(prompts).toHaveLength(1);
-    expect(prompts[0].prompt).toContain('<|s1|><Context one.><end||s>');
-    expect(prompts[0].prompt).toContain('<|s2|><Context two.><end||s>');
-    expect(prompts[0].prompt).toContain('<|r1|><Response one.><end||r>');
-    expect(prompts[0].prompt).toContain('<|r2|><Response two.><end||r>');
-  });
-
-  it('should handle empty context', () => {
-    const responseSentences = ['Response.'];
-    const responseOffsets = new Map([[1, { startOffset: 0, endOffset: 9 }]]);
-
-    const { prompts } = createChunkedHalloumiPrompts({
-      indexedContextSentences: [],
-      responseSentences,
-      responseOffsets,
-    });
-
-    expect(prompts).toHaveLength(1);
-    expect(prompts[0].prompt).toContain('<|context|><end||context>');
-  });
-
-  it('should exclude response sentences based on excludeResponseIndices', () => {
-    const indexedContextSentences = [
-      { sentence: 'Context.', sourceId: 1, globalId: 1 },
-    ];
-    const responseSentences = ['Keep this.', 'Skip this.', 'Keep too.'];
-    const responseOffsets = new Map([
-      [1, { startOffset: 0, endOffset: 10 }],
-      [2, { startOffset: 11, endOffset: 21 }],
-      [3, { startOffset: 22, endOffset: 31 }],
-    ]);
-    const excludeResponseIndices = new Set([2]);
-
-    const { prompts } = createChunkedHalloumiPrompts({
-      indexedContextSentences,
-      responseSentences,
-      responseOffsets,
-      excludeResponseIndices,
-    });
-
-    expect(prompts[0].prompt).toContain('<|r1|><Keep this.><end||r>');
-    expect(prompts[0].prompt).not.toContain('Skip this.');
-    expect(prompts[0].prompt).toContain('<|r3|><Keep too.><end||r>');
-  });
-
-  it('should keep sources together with bin packing', () => {
-    // 3 sources: 60 + 60 + 30 = 150 sentences
-    // Should pack into 2 chunks: (60+30) and (60) or similar
-    const indexed = [];
-    let gid = 1;
-    // Source 1: 60 sentences
-    for (let i = 0; i < 60; i++) {
-      indexed.push({ sentence: `S1-${i}`, sourceId: 1, globalId: gid++ });
-    }
-    // Source 2: 60 sentences
-    for (let i = 0; i < 60; i++) {
-      indexed.push({ sentence: `S2-${i}`, sourceId: 2, globalId: gid++ });
-    }
-    // Source 3: 30 sentences
-    for (let i = 0; i < 30; i++) {
-      indexed.push({ sentence: `S3-${i}`, sourceId: 3, globalId: gid++ });
-    }
-
-    const { prompts } = createChunkedHalloumiPrompts({
-      indexedContextSentences: indexed,
-      responseSentences: ['Claim.'],
-      responseOffsets: new Map([[1, { startOffset: 0, endOffset: 6 }]]),
-    });
-
-    expect(prompts).toHaveLength(2);
-
-    // Each source should be entirely within one chunk
-    for (const prompt of prompts) {
-      const s1Count = (prompt.prompt.match(/S1-/g) || []).length;
-      const s2Count = (prompt.prompt.match(/S2-/g) || []).length;
-      const s3Count = (prompt.prompt.match(/S3-/g) || []).length;
-
-      if (s1Count > 0) expect(s1Count).toBe(60);
-      if (s2Count > 0) expect(s2Count).toBe(60);
-      if (s3Count > 0) expect(s3Count).toBe(30);
-    }
-  });
-});