@eeacms/volto-eea-chatbot 3.0.0 → 4.0.0

package/src/halloumi/filtering.test.js

@@ -1,242 +0,0 @@
-import fetch from 'node-fetch';
-
-import {
-  parseExcludeIndices,
-  callLLM,
-  excludeClaimSentences,
-  excludeContextSentences,
-} from './filtering';
-jest.mock('node-fetch');
-
-describe('parseExcludeIndices', () => {
-  it('parses single indices', () => {
-    const result = parseExcludeIndices('1,3,5', 6);
-    expect(result).toEqual(new Set([1, 3, 5]));
-  });
-
-  it('returns empty set for NONE', () => {
-    const result = parseExcludeIndices('NONE', 10);
-    expect(result).toEqual(new Set());
-  });
-
-  it('returns empty set for none (lowercase)', () => {
-    const result = parseExcludeIndices('  none  ', 10);
-    expect(result).toEqual(new Set());
-  });
-
-  it('ignores indices below 1', () => {
-    const result = parseExcludeIndices('0, 1, 3', 5);
-    expect(result).toEqual(new Set([1, 3]));
-  });
-
-  it('ignores indices above maxIndex', () => {
-    const result = parseExcludeIndices('1, 3, 99', 5);
-    expect(result).toEqual(new Set([1, 3]));
-  });
-
-  it('handles whitespace variations', () => {
-    const result = parseExcludeIndices('  1 ,  5 , 7  ', 10);
-    expect(result).toEqual(new Set([1, 5, 7]));
-  });
-
-  it('extracts numbers even from unexpected formats', () => {
-    // Parser uses match(/\d+/g) so it extracts all numbers
-    const result = parseExcludeIndices('1-3, 5', 10);
-    expect(result).toEqual(new Set([1, 3, 5]));
-  });
-
-  it('returns empty set for empty string with no numbers', () => {
-    const result = parseExcludeIndices('no numbers here', 10);
-    expect(result).toEqual(new Set());
-  });
-});
-
-describe('callLLM', () => {
-  beforeEach(() => {
-    fetch.mockReset();
-  });
-
-  afterEach(() => {
-    fetch.mockReset();
-  });
-
-  it('makes POST request with correct headers', async () => {
-    const mockResponse = { choices: [{ message: { content: 'result' } }] };
-    fetch.mockResolvedValue({
-      json: () => Promise.resolve(mockResponse),
-    });
-
-    await callLLM('https://api.example.com', 'my-api-key', { model: 'gpt-4' });
-
-    expect(fetch).toHaveBeenCalledWith(
-      'https://api.example.com',
-      expect.objectContaining({
-        method: 'POST',
-        headers: expect.objectContaining({
-          'Content-Type': 'application/json',
-          Authorization: 'Bearer my-api-key',
-        }),
-      }),
-    );
-  });
-
-  it('omits Authorization header when no apiKey', async () => {
-    fetch.mockResolvedValue({
-      json: () => Promise.resolve({}),
-    });
-
-    await callLLM('https://api.example.com', null, { model: 'gpt-4' });
-
-    const headers = fetch.mock.calls[0][1].headers;
-    expect(headers['Authorization']).toBeUndefined();
-  });
-
-  it('adds X-Forwarded-For header when ip is provided', async () => {
-    fetch.mockResolvedValue({
-      json: () => Promise.resolve({}),
-    });
-
-    await callLLM('https://api.example.com', null, {}, { ip: '1.2.3.4' });
-
-    const headers = fetch.mock.calls[0][1].headers;
-    expect(headers['X-Forwarded-For']).toBe('1.2.3.4');
-  });
-
-  it('returns parsed JSON response', async () => {
-    const mockResponse = { choices: [{ message: { content: 'hello' } }] };
-    fetch.mockResolvedValue({
-      json: () => Promise.resolve(mockResponse),
-    });
-
-    const result = await callLLM('https://api.example.com', null, {});
-    expect(result).toEqual(mockResponse);
-  });
-});
-
-describe('excludeClaimSentences', () => {
-  beforeEach(() => {
-    fetch.mockReset();
-  });
-
-  afterEach(() => {
-    fetch.mockReset();
-  });
-
-  it('returns empty set for empty sentences array', async () => {
-    const result = await excludeClaimSentences([]);
-    expect(result).toEqual(new Set());
-    expect(fetch).not.toHaveBeenCalled();
-  });
-
-  it('calls LLM and returns excluded sentence indices', async () => {
-    fetch.mockResolvedValue({
-      json: () =>
-        Promise.resolve({
-          choices: [{ message: { content: '1,3' } }],
-        }),
-    });
-
-    const sentences = ['Sentence one.', 'Sentence two.', 'Sentence three.'];
-    const result = await excludeClaimSentences(sentences);
-
-    expect(result).toEqual(new Set([1, 3]));
-    expect(fetch).toHaveBeenCalledTimes(1);
-  });
-
-  it('returns empty set when LLM returns NONE', async () => {
-    fetch.mockResolvedValue({
-      json: () =>
-        Promise.resolve({
-          choices: [{ message: { content: 'NONE' } }],
-        }),
-    });
-
-    const sentences = ['Some factual sentence.', 'Another factual sentence.'];
-    const result = await excludeClaimSentences(sentences);
-
-    expect(result).toEqual(new Set());
-  });
-
-  it('returns empty set on LLM error', async () => {
-    fetch.mockRejectedValue(new Error('Network error'));
-
-    const sentences = ['Sentence one.'];
-    const result = await excludeClaimSentences(sentences);
-
-    expect(result).toEqual(new Set());
-  });
-
-  it('passes ip option to LLM call', async () => {
-    fetch.mockResolvedValue({
-      json: () =>
-        Promise.resolve({ choices: [{ message: { content: 'NONE' } }] }),
-    });
-
-    await excludeClaimSentences(['test sentence'], { ip: '10.0.0.1' });
-
-    const headers = fetch.mock.calls[0][1].headers;
-    expect(headers['X-Forwarded-For']).toBe('10.0.0.1');
-  });
-});
-
-describe('excludeContextSentences', () => {
-  const MIN_SENTENCES = 75;
-
-  beforeEach(() => {
-    fetch.mockReset();
-  });
-
-  afterEach(() => {
-    fetch.mockReset();
-  });
-
-  it('returns empty set when context sentences <= minimum threshold', async () => {
-    const shortContext = Array(10).fill('Short sentence.');
-    const claims = ['A claim.'];
-    const result = await excludeContextSentences(shortContext, claims);
-
-    expect(result).toEqual(new Set());
-    expect(fetch).not.toHaveBeenCalled();
-  });
-
-  it('calls LLM when context exceeds minimum threshold', async () => {
-    fetch.mockResolvedValue({
-      json: () =>
-        Promise.resolve({
-          choices: [{ message: { content: '1,2' } }],
-        }),
-    });
-
-    const longContext = Array(MIN_SENTENCES + 1).fill('Context sentence.');
-    const claims = ['A claim.'];
-    const result = await excludeContextSentences(longContext, claims);
-
-    expect(result).toEqual(new Set([1, 2]));
-    expect(fetch).toHaveBeenCalledTimes(1);
-  });
-
-  it('returns empty set on LLM error', async () => {
-    fetch.mockRejectedValue(new Error('timeout'));
-
-    const longContext = Array(MIN_SENTENCES + 1).fill('Context sentence.');
-    const claims = ['A claim.'];
-    const result = await excludeContextSentences(longContext, claims);
-
-    expect(result).toEqual(new Set());
-  });
-
-  it('returns empty set when LLM returns NONE for context', async () => {
-    fetch.mockResolvedValue({
-      json: () =>
-        Promise.resolve({
-          choices: [{ message: { content: 'NONE' } }],
-        }),
-    });
-
-    const longContext = Array(MIN_SENTENCES + 1).fill('Context sentence.');
-    const claims = ['A claim.'];
-    const result = await excludeContextSentences(longContext, claims);
-
-    expect(result).toEqual(new Set());
-  });
-});

package/src/halloumi/generative.js

@@ -1,324 +0,0 @@
-import debug from 'debug';
-import fs from 'fs';
-import {
-  getClaimsFromResponse,
-  getTokenProbabilitiesFromLogits,
-} from './postprocessing';
-import { createChunkedHalloumiPrompts, getOffsets } from './preprocessing';
-import { splitMarkdown, splitProse } from './markdown-splitter';
-import { callLLM, excludeClaimSentences } from './filtering';
-
-const log = debug('halloumi');
-
-const tokenChoices = new Set(['supported', 'unsupported']);
-
-function sigmoid(x) {
-  return 1 / (1 + Math.exp(-x));
-}
-
-export function applyPlattScaling(platt, probability) {
-  probability = Math.min(Math.max(probability, 1e-6), 1 - 1e-6);
-  const log_prob = Math.log(probability / (1 - probability));
-  return sigmoid(-1 * (platt.a * log_prob + platt.b));
-}
-
-/**
- * Merges claims from multiple chunked HallOumi responses.
- * For each response sentence (claimId), combines segment citations
- * and takes the max supported score.
- */
-function mergeChunkClaims(chunkResults) {
-  const claimMap = new Map();
-
-  for (const claims of chunkResults) {
-    for (const claim of claims) {
-      if (!claimMap.has(claim.claimId)) {
-        claimMap.set(claim.claimId, { ...claim });
-      } else {
-        const existing = claimMap.get(claim.claimId);
-        existing.segments.push(...claim.segments);
-        // Keep the result with the higher supported score
-        const existingScore = existing.probabilities.get('supported') || 0;
-        const newScore = claim.probabilities.get('supported') || 0;
-        if (newScore > existingScore) {
-          existing.probabilities = claim.probabilities;
-          existing.explanation = claim.explanation;
-          existing.supported = claim.supported;
-        }
-      }
-    }
-  }
-
-  return Array.from(claimMap.values());
-}
-
-export async function getVerifyClaimResponse(
-  model,
-  sources,
-  answer,
-  { ip } = {},
-) {
-  const emptyResponse = {
-    claims: [],
-    segments: {},
-  };
-  if (!sources?.length || !answer) {
-    return { ...emptyResponse, reason: 'Context is empty' };
-  }
-
-  // Split sentences
-  const responseSentences = splitMarkdown(answer);
-  const responseOffsets = getOffsets(answer, responseSentences);
-
-  // Filter claims and context in parallel
-  const [excludeResponseIndices] = await Promise.all([
-    excludeClaimSentences(responseSentences, { ip }),
-  ]);
-
-  const contextSentences = [];
-  const indexedContextSentences = sources.reduce((acc, text, sourceIdx) => {
-    const sentences = splitProse(text, 50).map((sentence, sentenceIdx) => {
-      const globalId = acc.length + sentenceIdx + 1;
-      contextSentences.push(sentence);
-      return {
-        sentence,
-        sourceId: sourceIdx + 1,
-        globalId,
-      };
-    });
-    acc.push(...sentences);
-    return acc;
-  }, []);
-  const joinedContext = sources.join('');
-  const contextOffsets = getOffsets(joinedContext, contextSentences);
-
-  if (excludeResponseIndices.size === responseSentences.length) {
-    log('All response sentences excluded');
-    return {
-      ...emptyResponse,
-      empty: 'Claims in the document could not be verified',
-    };
-  }
-  log('Excluded response indices', excludeResponseIndices);
-  const { prompts } = createChunkedHalloumiPrompts({
-    indexedContextSentences,
-    responseSentences,
-    responseOffsets,
-    request: null,
-    excludeResponseIndices,
-  });
-
-  log(`Split into ${prompts.length} chunk(s)`);
-
-  // Run all chunks in parallel
-  const chunkResults = await Promise.all(
-    prompts.map((chunkPrompt, i) => {
-      log(`Chunk ${i + 1} request`);
-      return halloumiGenerativeAPI(model, chunkPrompt, { ip });
-    }),
-  );
-
-  // Merge raw claims across chunks
-  const rawClaims = mergeChunkClaims(chunkResults);
-
-  const mergedPrompt = {
-    contextOffsets,
-    responseOffsets,
-    joinedContext,
-  };
-  const converted = convertGenerativesClaimToVerifyClaimResponse(
-    rawClaims,
-    mergedPrompt,
-  );
-
-  if (excludeResponseIndices.size > 0) {
-    for (const idx of excludeResponseIndices) {
-      if (responseOffsets.has(idx)) {
-        const offsets = responseOffsets.get(idx);
-        converted.claims.push({
-          claimId: idx,
-          claimString: responseSentences[idx - 1],
-          startOffset: offsets.startOffset,
-          endOffset: offsets.endOffset,
-          skipped: true,
-          score: null,
-        });
-      }
-    }
-    converted.claims.sort((a, b) => a.startOffset - b.startOffset);
-  }
-
-  const result = {
-    ...converted,
-    rawClaims,
-    ...(prompts.length === 1
-      ? { halloumiPrompt: prompts[0] }
-      : { halloumiPrompts: prompts }),
-  };
-
-  return result;
-}
-
-/**
- * Fetches a response from the LLM.
- *
- * @param {object} model The model configuration.
- * @param {object} prompt The prompt to send to the LLM.
- * @returns {Promise<object>} The JSON response from the LLM.
- *
- * Environment Variables:
- * - `MOCK_HALLOUMI_FILE_PATH`: If set, the function reads the LLM response from the specified file path instead of making an API call.
- * - `DUMP_HALLOUMI_REQ_FILE_PATH`: If set, the LLM request (URL and parameters) is dumped to the specified file path.
- * - `DUMP_HALLOUMI_FILE_PATH`: If set, the LLM response is dumped to the specified file path.
- */
-async function getLLMResponse(model, prompt, { ip } = {}) {
-  let jsonData;
-
-  if (process.env.MOCK_HALLOUMI_FILE_PATH) {
-    const filePath = process.env.MOCK_HALLOUMI_FILE_PATH;
-    const fileContent = fs.readFileSync(filePath, 'utf-8');
-    jsonData = JSON.parse(fileContent);
-    await new Promise((resolve) => setTimeout(resolve, 2000));
-    return jsonData;
-  }
-
-  const data = {
-    messages: [{ role: 'user', content: prompt.prompt }],
-    temperature: 0.0,
-    model: model.name,
-    logprobs: true,
-    top_logprobs: 3,
-  };
-
-  if (process.env.DUMP_HALLOUMI_REQ_FILE_PATH) {
-    const filePath = process.env.DUMP_HALLOUMI_REQ_FILE_PATH;
-    fs.writeFileSync(
-      filePath,
-      JSON.stringify({ url: model.apiUrl, body: data }, null, 2),
-    );
-    log(`Dumped halloumi request: ${filePath}`);
-  }
-
-  jsonData = await callLLM(model.apiUrl, model.apiKey, data, { ip });
-
-  if (process.env.DUMP_HALLOUMI_FILE_PATH) {
-    const filePath = process.env.DUMP_HALLOUMI_FILE_PATH;
-    fs.writeFileSync(filePath, JSON.stringify(jsonData, null, 2));
-    log(`Dumped halloumi response: ${filePath}`);
-  }
-
-  return jsonData;
-}
-
-/**
- * Gets all claims from a response.
- * @param response A string containing all claims and their information.
- * @returns A list of claim objects.
- */
-export async function halloumiGenerativeAPI(model, prompt, { ip } = {}) {
-  const jsonData = await getLLMResponse(model, prompt, { ip });
-
-  // Todo: restore log
-  // log('Generative response', jsonData);
-
-  const finishReason = jsonData.choices?.[0]?.finish_reason;
-  if (finishReason === 'length') {
-    throw new Error('HallOumi response truncated (finish_reason: length)');
-  }
-
-  // Todo: restore log
-  // log('Logprobs', jsonData.choices[0].logprobs.content);
-
-  const logits = jsonData.choices[0].logprobs.content;
-  const tokenProbabilities = getTokenProbabilitiesFromLogits(
-    logits,
-    tokenChoices,
-  );
-  const parsedResponse = getClaimsFromResponse(
-    jsonData.choices[0].message.content,
-  );
-
-  if (parsedResponse.length !== tokenProbabilities.length) {
-    log(
-      'Warning: token probabilities (%d) and claims (%d) do not match — using available probabilities, defaulting remainder to 0.5',
-      tokenProbabilities.length,
-      parsedResponse.length,
-    );
-  }
-
-  const defaultScoreMap = new Map([
-    ['supported', 0.5],
-    ['unsupported', 0.5],
-  ]);
-
-  for (let i = 0; i < parsedResponse.length; i++) {
-    const scoreMap = tokenProbabilities[i] ?? new Map(defaultScoreMap);
-    if (model.plattScaling) {
-      const platt = model.plattScaling;
-      const unsupportedScore = applyPlattScaling(
-        platt,
-        scoreMap.get('unsupported'),
-      );
-      const supportedScore = 1 - unsupportedScore;
-      scoreMap.set('supported', supportedScore);
-      scoreMap.set('unsupported', unsupportedScore);
-    }
-    parsedResponse[i].probabilities = scoreMap;
-  }
-
-  return parsedResponse;
-}
-
-export function convertGenerativesClaimToVerifyClaimResponse(
-  generativeClaims,
-  prompt,
-) {
-  const segments = {};
-  const claims = [];
-
-  for (const offset of prompt.contextOffsets) {
-    const id = offset[0].toString();
-    segments[id] = {
-      id,
-      startOffset: offset[1].startOffset,
-      endOffset: offset[1].endOffset,
-      text: prompt.joinedContext.slice(
-        offset[1].startOffset,
-        offset[1].endOffset,
-      ),
-    };
-  }
-
-  for (const generativeClaim of generativeClaims) {
-    const segmentIds = [];
-    for (const seg of generativeClaim.segments) {
-      if (!seg) continue;
-      segmentIds.push(seg.toString());
-    }
-
-    const claimId = generativeClaim.claimId;
-    if (!prompt.responseOffsets.has(claimId)) {
-      throw new Error(`Claim ${claimId} not found in response offsets.`);
-    }
-
-    const claimResponseWindow = prompt.responseOffsets.get(claimId);
-    const score = generativeClaim.probabilities.get('supported');
-    const claim = {
-      claimId,
-      claimString: generativeClaim.claimString,
-      startOffset: claimResponseWindow.startOffset,
-      endOffset: claimResponseWindow.endOffset,
-      rationale: generativeClaim.explanation,
-      segmentIds,
-      score,
-    };
-    claims.push(claim);
-  }
-
-  const response = {
-    claims,
-    segments,
-  };
-
-  return response;
-}