@eeacms/volto-eea-chatbot 3.0.1 → 4.1.0

package/src/halloumi/generative.js

@@ -1,324 +0,0 @@
-import debug from 'debug';
-import fs from 'fs';
-import {
-  getClaimsFromResponse,
-  getTokenProbabilitiesFromLogits,
-} from './postprocessing';
-import { createChunkedHalloumiPrompts, getOffsets } from './preprocessing';
-import { splitMarkdown, splitProse } from './markdown-splitter';
-import { callLLM, excludeClaimSentences } from './filtering';
-
-const log = debug('halloumi');
-
-const tokenChoices = new Set(['supported', 'unsupported']);
-
-function sigmoid(x) {
-  return 1 / (1 + Math.exp(-x));
-}
-
-export function applyPlattScaling(platt, probability) {
-  probability = Math.min(Math.max(probability, 1e-6), 1 - 1e-6);
-  const log_prob = Math.log(probability / (1 - probability));
-  return sigmoid(-1 * (platt.a * log_prob + platt.b));
-}
-
-/**
- * Merges claims from multiple chunked HallOumi responses.
- * For each response sentence (claimId), combines segment citations
- * and takes the max supported score.
- */
-function mergeChunkClaims(chunkResults) {
-  const claimMap = new Map();
-
-  for (const claims of chunkResults) {
-    for (const claim of claims) {
-      if (!claimMap.has(claim.claimId)) {
-        claimMap.set(claim.claimId, { ...claim });
-      } else {
-        const existing = claimMap.get(claim.claimId);
-        existing.segments.push(...claim.segments);
-        // Keep the result with the higher supported score
-        const existingScore = existing.probabilities.get('supported') || 0;
-        const newScore = claim.probabilities.get('supported') || 0;
-        if (newScore > existingScore) {
-          existing.probabilities = claim.probabilities;
-          existing.explanation = claim.explanation;
-          existing.supported = claim.supported;
-        }
-      }
-    }
-  }
-
-  return Array.from(claimMap.values());
-}
-
-export async function getVerifyClaimResponse(
-  model,
-  sources,
-  answer,
-  { ip } = {},
-) {
-  const emptyResponse = {
-    claims: [],
-    segments: {},
-  };
-  if (!sources?.length || !answer) {
-    return { ...emptyResponse, reason: 'Context is empty' };
-  }
-
-  // Split sentences
-  const responseSentences = splitMarkdown(answer);
-  const responseOffsets = getOffsets(answer, responseSentences);
-
-  // Filter claims and context in parallel
-  const [excludeResponseIndices] = await Promise.all([
-    excludeClaimSentences(responseSentences, { ip }),
-  ]);
-
-  const contextSentences = [];
-  const indexedContextSentences = sources.reduce((acc, text, sourceIdx) => {
-    const sentences = splitProse(text, 50).map((sentence, sentenceIdx) => {
-      const globalId = acc.length + sentenceIdx + 1;
-      contextSentences.push(sentence);
-      return {
-        sentence,
-        sourceId: sourceIdx + 1,
-        globalId,
-      };
-    });
-    acc.push(...sentences);
-    return acc;
-  }, []);
-  const joinedContext = sources.join('');
-  const contextOffsets = getOffsets(joinedContext, contextSentences);
-
-  if (excludeResponseIndices.size === responseSentences.length) {
-    log('All response sentences excluded');
-    return {
-      ...emptyResponse,
-      empty: 'Claims in the document could not be verified',
-    };
-  }
-  log('Excluded response indices', excludeResponseIndices);
-  const { prompts } = createChunkedHalloumiPrompts({
-    indexedContextSentences,
-    responseSentences,
-    responseOffsets,
-    request: null,
-    excludeResponseIndices,
-  });
-
-  log(`Split into ${prompts.length} chunk(s)`);
-
-  // Run all chunks in parallel
-  const chunkResults = await Promise.all(
-    prompts.map((chunkPrompt, i) => {
-      log(`Chunk ${i + 1} request`);
-      return halloumiGenerativeAPI(model, chunkPrompt, { ip });
-    }),
-  );
-
-  // Merge raw claims across chunks
-  const rawClaims = mergeChunkClaims(chunkResults);
-
-  const mergedPrompt = {
-    contextOffsets,
-    responseOffsets,
-    joinedContext,
-  };
-  const converted = convertGenerativesClaimToVerifyClaimResponse(
-    rawClaims,
-    mergedPrompt,
-  );
-
-  if (excludeResponseIndices.size > 0) {
-    for (const idx of excludeResponseIndices) {
-      if (responseOffsets.has(idx)) {
-        const offsets = responseOffsets.get(idx);
-        converted.claims.push({
-          claimId: idx,
-          claimString: responseSentences[idx - 1],
-          startOffset: offsets.startOffset,
-          endOffset: offsets.endOffset,
-          skipped: true,
-          score: null,
-        });
-      }
-    }
-    converted.claims.sort((a, b) => a.startOffset - b.startOffset);
-  }
-
-  const result = {
-    ...converted,
-    rawClaims,
-    ...(prompts.length === 1
-      ? { halloumiPrompt: prompts[0] }
-      : { halloumiPrompts: prompts }),
-  };
-
-  return result;
-}
-
-/**
- * Fetches a response from the LLM.
- *
- * @param {object} model The model configuration.
- * @param {object} prompt The prompt to send to the LLM.
- * @returns {Promise<object>} The JSON response from the LLM.
- *
- * Environment Variables:
- * - `MOCK_HALLOUMI_FILE_PATH`: If set, the function reads the LLM response from the specified file path instead of making an API call.
- * - `DUMP_HALLOUMI_REQ_FILE_PATH`: If set, the LLM request (URL and parameters) is dumped to the specified file path.
- * - `DUMP_HALLOUMI_FILE_PATH`: If set, the LLM response is dumped to the specified file path.
- */
-async function getLLMResponse(model, prompt, { ip } = {}) {
-  let jsonData;
-
-  if (process.env.MOCK_HALLOUMI_FILE_PATH) {
-    const filePath = process.env.MOCK_HALLOUMI_FILE_PATH;
-    const fileContent = fs.readFileSync(filePath, 'utf-8');
-    jsonData = JSON.parse(fileContent);
-    await new Promise((resolve) => setTimeout(resolve, 2000));
-    return jsonData;
-  }
-
-  const data = {
-    messages: [{ role: 'user', content: prompt.prompt }],
-    temperature: 0.0,
-    model: model.name,
-    logprobs: true,
-    top_logprobs: 3,
-  };
-
-  if (process.env.DUMP_HALLOUMI_REQ_FILE_PATH) {
-    const filePath = process.env.DUMP_HALLOUMI_REQ_FILE_PATH;
-    fs.writeFileSync(
-      filePath,
-      JSON.stringify({ url: model.apiUrl, body: data }, null, 2),
-    );
-    log(`Dumped halloumi request: ${filePath}`);
-  }
-
-  jsonData = await callLLM(model.apiUrl, model.apiKey, data, { ip });
-
-  if (process.env.DUMP_HALLOUMI_FILE_PATH) {
-    const filePath = process.env.DUMP_HALLOUMI_FILE_PATH;
-    fs.writeFileSync(filePath, JSON.stringify(jsonData, null, 2));
-    log(`Dumped halloumi response: ${filePath}`);
-  }
-
-  return jsonData;
-}
-
-/**
- * Gets all claims from a response.
- * @param response A string containing all claims and their information.
- * @returns A list of claim objects.
- */
-export async function halloumiGenerativeAPI(model, prompt, { ip } = {}) {
-  const jsonData = await getLLMResponse(model, prompt, { ip });
-
-  // Todo: restore log
-  // log('Generative response', jsonData);
-
-  const finishReason = jsonData.choices?.[0]?.finish_reason;
-  if (finishReason === 'length') {
-    throw new Error('HallOumi response truncated (finish_reason: length)');
-  }
-
-  // Todo: restore log
-  // log('Logprobs', jsonData.choices[0].logprobs.content);
-
-  const logits = jsonData.choices[0].logprobs.content;
-  const tokenProbabilities = getTokenProbabilitiesFromLogits(
-    logits,
-    tokenChoices,
-  );
-  const parsedResponse = getClaimsFromResponse(
-    jsonData.choices[0].message.content,
-  );
-
-  if (parsedResponse.length !== tokenProbabilities.length) {
-    log(
-      'Warning: token probabilities (%d) and claims (%d) do not match — using available probabilities, defaulting remainder to 0.5',
-      tokenProbabilities.length,
-      parsedResponse.length,
-    );
-  }
-
-  const defaultScoreMap = new Map([
-    ['supported', 0.5],
-    ['unsupported', 0.5],
-  ]);
-
-  for (let i = 0; i < parsedResponse.length; i++) {
-    const scoreMap = tokenProbabilities[i] ?? new Map(defaultScoreMap);
-    if (model.plattScaling) {
-      const platt = model.plattScaling;
-      const unsupportedScore = applyPlattScaling(
-        platt,
-        scoreMap.get('unsupported'),
-      );
-      const supportedScore = 1 - unsupportedScore;
-      scoreMap.set('supported', supportedScore);
-      scoreMap.set('unsupported', unsupportedScore);
-    }
-    parsedResponse[i].probabilities = scoreMap;
-  }
-
-  return parsedResponse;
-}
-
-export function convertGenerativesClaimToVerifyClaimResponse(
-  generativeClaims,
-  prompt,
-) {
-  const segments = {};
-  const claims = [];
-
-  for (const offset of prompt.contextOffsets) {
-    const id = offset[0].toString();
-    segments[id] = {
-      id,
-      startOffset: offset[1].startOffset,
-      endOffset: offset[1].endOffset,
-      text: prompt.joinedContext.slice(
-        offset[1].startOffset,
-        offset[1].endOffset,
-      ),
-    };
-  }
-
-  for (const generativeClaim of generativeClaims) {
-    const segmentIds = [];
-    for (const seg of generativeClaim.segments) {
-      if (!seg) continue;
-      segmentIds.push(seg.toString());
-    }
-
-    const claimId = generativeClaim.claimId;
-    if (!prompt.responseOffsets.has(claimId)) {
-      throw new Error(`Claim ${claimId} not found in response offsets.`);
-    }
-
-    const claimResponseWindow = prompt.responseOffsets.get(claimId);
-    const score = generativeClaim.probabilities.get('supported');
-    const claim = {
-      claimId,
-      claimString: generativeClaim.claimString,
-      startOffset: claimResponseWindow.startOffset,
-      endOffset: claimResponseWindow.endOffset,
-      rationale: generativeClaim.explanation,
-      segmentIds,
-      score,
-    };
-    claims.push(claim);
-  }
-
-  const response = {
-    claims,
-    segments,
-  };
-
-  return response;
-}

package/src/halloumi/generative.test.js

@@ -1,386 +0,0 @@
-import {
-  halloumiGenerativeAPI,
-  convertGenerativesClaimToVerifyClaimResponse,
-  applyPlattScaling,
-  getVerifyClaimResponse,
-} from './generative';
-import path from 'path';
-
-jest.mock('node-fetch');
-
-describe('halloumiGenerativeAPI reads from mock file', () => {
-  const originalEnv = process.env;
-
-  beforeEach(() => {
-    jest.resetModules(); // Most important - reset modules between test runs
-    process.env = {
-      ...originalEnv,
-      MOCK_HALLOUMI_FILE_PATH: path.join(__dirname, '../dummy/qa-raw-3.json'),
-    };
-  });
-
-  afterEach(() => {
-    process.env = originalEnv; // Restore original env
-    jest.restoreAllMocks(); // Restore all mocks
-  });
-
-  it('should read from the mock file when MOCK_LLM_CALL is true', async () => {
-    const model = { name: 'test-model', apiUrl: 'http://test.com' };
-    const prompt = {
-      prompt: 'test-prompt',
-      contextOffsets: new Map([[1, { startOffset: 0, endOffset: 10 }]]),
-      responseOffsets: new Map([[1, { startOffset: 0, endOffset: 20 }]]),
-    };
-
-    // We are testing halloumiGenerativeAPI which internally calls getLLMResponse
-    // and getLLMResponse uses the MOCK_LLM_CALL env variable.
-    const response = await halloumiGenerativeAPI(model, prompt);
-
-    expect(response[0].claimString).toEqual(
-      '**France – total waste generation (latest available data)**  \n',
-    );
-    expect(response[0].segments).toEqual([
-      38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48,
-    ]);
-  });
-});
-
-describe('applyPlattScaling', () => {
-  it('returns calibrated probability', () => {
-    const platt = { a: -0.5764, b: 0.1665 };
-    const result = applyPlattScaling(platt, 0.5);
-    expect(result).toBeGreaterThan(0);
-    expect(result).toBeLessThan(1);
-  });
-
-  it('clamps very low probability', () => {
-    const platt = { a: -1, b: 0 };
-    const result = applyPlattScaling(platt, 0);
-    expect(result).toBeGreaterThan(0);
-    expect(result).toBeLessThan(1);
-  });
-
-  it('clamps very high probability', () => {
-    const platt = { a: -1, b: 0 };
-    const result = applyPlattScaling(platt, 1);
-    expect(result).toBeGreaterThan(0);
-    expect(result).toBeLessThan(1);
-  });
-
-  it('returns 0.5 when platt params are zero', () => {
-    const platt = { a: 0, b: 0 };
-    const result = applyPlattScaling(platt, 0.5);
-    expect(result).toBeCloseTo(0.5, 1);
-  });
-});
-
-describe('getVerifyClaimResponse', () => {
-  it('returns empty response when sources is empty', async () => {
-    const result = await getVerifyClaimResponse({}, [], 'claims');
-    expect(result).toEqual({
-      claims: [],
-      segments: {},
-      reason: 'Context is empty',
-    });
-  });
-
-  it('returns empty response when sources is null', async () => {
-    const result = await getVerifyClaimResponse({}, null, 'claims');
-    expect(result).toEqual({
-      claims: [],
-      segments: {},
-      reason: 'Context is empty',
-    });
-  });
-
-  it('returns empty response when claims is falsy', async () => {
-    const result = await getVerifyClaimResponse({}, ['source'], null);
-    expect(result).toEqual({
-      claims: [],
-      segments: {},
-      reason: 'Context is empty',
-    });
-  });
-});
-
-describe('halloumiGenerativeAPI with plattScaling', () => {
-  const originalEnv = process.env;
-
-  beforeEach(() => {
-    jest.resetModules();
-    process.env = {
-      ...originalEnv,
-      MOCK_HALLOUMI_FILE_PATH: path.join(__dirname, '../dummy/qa-raw-3.json'),
-    };
-  });
-
-  afterEach(() => {
-    process.env = originalEnv;
-    jest.restoreAllMocks();
-  });
-
-  it('applies platt scaling when model has plattScaling config', async () => {
-    const model = {
-      name: 'test-model',
-      apiUrl: 'http://test.com',
-      plattScaling: { a: -0.5764, b: 0.1665 },
-    };
-    const prompt = {
-      prompt: 'test-prompt',
-      contextOffsets: new Map([[1, { startOffset: 0, endOffset: 10 }]]),
-      responseOffsets: new Map([[1, { startOffset: 0, endOffset: 20 }]]),
-    };
-
-    const response = await halloumiGenerativeAPI(model, prompt);
-
-    // With platt scaling, probabilities should be calibrated
-    expect(response[0].probabilities).toBeDefined();
-    const supported = response[0].probabilities.get('supported');
-    const unsupported = response[0].probabilities.get('unsupported');
-    expect(supported + unsupported).toBeCloseTo(1, 5);
-  });
-});
-
-describe('halloumiGenerativeAPI via real fetch', () => {
-  const originalEnv = process.env;
-
-  beforeEach(() => {
-    jest.resetModules();
-    // Remove mock file path to exercise the fetch path
-    process.env = { ...originalEnv };
-    delete process.env.MOCK_HALLOUMI_FILE_PATH;
-  });
-
-  afterEach(() => {
-    process.env = originalEnv;
-    jest.restoreAllMocks();
-  });
-
-  it('calls fetch with correct parameters and auth header', async () => {
-    // Mock postprocessing to control output and avoid format issues
-    jest.doMock('./postprocessing', () => ({
-      getClaimsFromResponse: jest.fn(() => [
-        {
-          claimId: 1,
-          claimString: 'Test claim',
-          subclaims: [],
-          segments: [1],
-          explanation: 'ok',
-          supported: true,
-        },
-      ]),
-      getTokenProbabilitiesFromLogits: jest.fn(() => [
-        new Map([
-          ['supported', 0.9],
-          ['unsupported', 0.1],
-        ]),
-      ]),
-    }));
-
-    const { halloumiGenerativeAPI } = require('./generative');
-    const nodeFetch = require('node-fetch');
-
-    const mockResponse = {
-      choices: [
-        {
-          message: { content: 'mock content' },
-          logprobs: { content: [] },
-        },
-      ],
-    };
-
-    nodeFetch.mockResolvedValueOnce({
-      json: () => Promise.resolve(mockResponse),
-    });
-
-    const model = {
-      name: 'test-model',
-      apiUrl: 'http://test.com/api',
-      apiKey: 'test-key', //betterleaks:allow
-    };
-    const prompt = {
-      prompt: 'test prompt',
-      contextOffsets: new Map([[1, { startOffset: 0, endOffset: 10 }]]),
-      responseOffsets: new Map([[1, { startOffset: 0, endOffset: 20 }]]),
-    };
-
-    const response = await halloumiGenerativeAPI(model, prompt);
-    expect(response).toBeDefined();
-    expect(Array.isArray(response)).toBe(true);
-    expect(response[0].probabilities.get('supported')).toBe(0.9);
-
-    // Verify fetch was called with auth header
-    expect(nodeFetch).toHaveBeenCalledWith(
-      'http://test.com/api',
-      expect.objectContaining({
-        method: 'POST',
-        headers: expect.objectContaining({
-          Authorization: 'Bearer test-key',
-        }),
-      }),
-    );
-  });
-
-  it('calls fetch without auth header when no apiKey', async () => {
-    jest.doMock('./postprocessing', () => ({
-      getClaimsFromResponse: jest.fn(() => [
-        {
-          claimId: 1,
-          claimString: 'Test',
-          subclaims: [],
-          segments: [],
-          explanation: 'ok',
-          supported: true,
-        },
-      ]),
-      getTokenProbabilitiesFromLogits: jest.fn(() => [
-        new Map([
-          ['supported', 0.8],
-          ['unsupported', 0.2],
-        ]),
-      ]),
-    }));
-
-    const { halloumiGenerativeAPI } = require('./generative');
-    const nodeFetch = require('node-fetch');
-
-    nodeFetch.mockResolvedValueOnce({
-      json: () =>
-        Promise.resolve({
-          choices: [
-            {
-              message: { content: 'mock' },
-              logprobs: { content: [] },
-            },
-          ],
-        }),
-    });
-
-    const model = { name: 'test-model', apiUrl: 'http://test.com/api' };
-    const prompt = {
-      prompt: 'test',
-      contextOffsets: new Map(),
-      responseOffsets: new Map(),
-    };
-
-    await halloumiGenerativeAPI(model, prompt);
-
-    const callHeaders = nodeFetch.mock.calls[0][1].headers;
-    expect(callHeaders.Authorization).toBeUndefined();
-  });
-
-  it('defaults to 0.5 when token probabilities and claims do not match', async () => {
-    jest.doMock('./postprocessing', () => ({
-      getClaimsFromResponse: jest.fn(() => [
-        { claimId: 1, claimString: 'Claim 1' },
-        { claimId: 2, claimString: 'Claim 2' },
-      ]),
-      getTokenProbabilitiesFromLogits: jest.fn(() => [
-        new Map([['supported', 0.9]]),
-      ]),
-    }));
-
-    const { halloumiGenerativeAPI } = require('./generative');
-    const nodeFetch = require('node-fetch');
-
-    nodeFetch.mockResolvedValueOnce({
-      json: () =>
-        Promise.resolve({
-          choices: [
-            {
-              message: { content: 'mock' },
-              logprobs: { content: [] },
-            },
-          ],
-        }),
-    });
-
-    const model = { name: 'test-model', apiUrl: 'http://test.com' };
-    const prompt = {
-      prompt: 'test',
-      contextOffsets: new Map(),
-      responseOffsets: new Map(),
-    };
-
-    const result = await halloumiGenerativeAPI(model, prompt);
-    // First claim gets the available probability
-    expect(result[0].probabilities.get('supported')).toBe(0.9);
-    // Second claim defaults to 0.5
-    expect(result[1].probabilities.get('supported')).toBe(0.5);
-  });
-});
-
-describe('convertGenerativesClaimToVerifyClaimResponse', () => {
-  it('should correctly convert generative claims to verify claim response', () => {
-    const generativeClaims = [
-      {
-        claimId: 1,
-        claimString: 'Test claim string',
-        subclaims: ['subclaim1', 'subclaim2'],
-        segments: [1, 2, 3],
-        explanation: 'Test explanation',
-        supported: true,
-        probabilities: new Map([
-          ['supported', 0.9],
-          ['unsupported', 0.1],
-        ]),
-      },
-    ];
-
-    const prompt = {
-      contextOffsets: new Map([[1, { startOffset: 0, endOffset: 10 }]]),
-      responseOffsets: new Map([[1, { startOffset: 100, endOffset: 120 }]]),
-      joinedContext: 'Test conte',
-    };
-
-    const result = convertGenerativesClaimToVerifyClaimResponse(
-      generativeClaims,
-      prompt,
-    );
-
-    expect(result).toEqual({
-      claims: [
-        {
-          claimId: 1,
-          claimString: 'Test claim string',
-          startOffset: 100,
-          endOffset: 120,
-          rationale: 'Test explanation',
-          segmentIds: ['1', '2', '3'],
-          score: 0.9,
-        },
-      ],
-      segments: {
-        1: {
-          id: '1',
-          startOffset: 0,
-          endOffset: 10,
-          text: 'Test conte',
-        },
-      },
-    });
-  });
-
-  it('throws when claim ID is not found in response offsets', () => {
-    const generativeClaims = [
-      {
-        claimId: 999,
-        claimString: 'Unknown claim',
-        subclaims: [],
-        segments: [],
-        explanation: 'Missing',
-        supported: false,
-        probabilities: new Map([['supported', 0.1]]),
-      },
-    ];
-
-    const prompt = {
-      contextOffsets: new Map(),
-      responseOffsets: new Map(),
-    };
-
-    expect(() =>
-      convertGenerativesClaimToVerifyClaimResponse(generativeClaims, prompt),
-    ).toThrow('Claim 999 not found in response offsets');
-  });
-});