@eeacms/volto-eea-chatbot 1.0.9

package/src/halloumi/generative.js

@@ -0,0 +1,219 @@
+import debug from 'debug';
+import fetch from 'node-fetch';
+import fs from 'fs';
+import {
+  getClaimsFromResponse,
+  getTokenProbabilitiesFromLogits,
+} from './postprocessing';
+import { createHalloumiPrompt } from './preprocessing';
+
+// const CONTEXT_SEPARTOR = '\n---\n';
+
+const log = debug('halloumi');
+
+function sigmoid(x) {
+  return 1 / (1 + Math.exp(-x));
+}
+
+export function applyPlattScaling(platt, probability) {
+  probability = Math.min(Math.max(probability, 1e-6), 1 - 1e-6);
+  const log_prob = Math.log(probability / (1 - probability));
+  return sigmoid(-1 * (platt.a * log_prob + platt.b));
+}
+
+export async function getVerifyClaimResponse(
+  model,
+  sources,
+  claims,
+  maxContextSegments = 0,
+) {
+  // const contextSeparator = CONTEXT_SEPARTOR;
+  // const joinedContext = sources.join(contextSeparator);
+
+  if (!sources?.length || !claims) {
+    const response = {
+      claims: [],
+      segments: {},
+    };
+    return response;
+  }
+
+  const prompt = createHalloumiPrompt({
+    sources,
+    response: claims,
+    maxContextSegments,
+    request: undefined,
+  });
+
+  log('Halloumi prompt', JSON.stringify(prompt, null, 2));
+
+  const rawClaims = await halloumiGenerativeAPI(model, prompt);
+  log('Raw claims', rawClaims);
+  const result = {
+    ...convertGenerativesClaimToVerifyClaimResponse(rawClaims, prompt),
+    rawClaims,
+    halloumiPrompt: prompt,
+  };
+
+  return result;
+}
+
+const tokenChoices = new Set(['supported', 'unsupported']);
+
+/**
+ * Fetches a response from the LLM.
+ *
+ * @param {object} model The model configuration.
+ * @param {object} prompt The prompt to send to the LLM.
+ * @returns {Promise<object>} The JSON response from the LLM.
+ *
+ * Environment Variables:
+ * - `MOCK_HALLOUMI_FILE_PATH`: If set, the function reads the LLM response from the specified file path instead of making an API call.
+ * - `DUMP_HALLOUMI_REQ_FILE_PATH`: If set, the LLM request (URL and parameters) is dumped to the specified file path.
+ * - `DUMP_HALLOUMI_FILE_PATH`: If set, the LLM response is dumped to the specified file path.
+ */
+async function getLLMResponse(model, prompt) {
+  let jsonData;
+
+  if (process.env.MOCK_HALLOUMI_FILE_PATH) {
+    const filePath = process.env.MOCK_HALLOUMI_FILE_PATH;
+    const fileContent = fs.readFileSync(filePath, 'utf-8');
+    jsonData = JSON.parse(fileContent);
+    return jsonData;
+  }
+
+  const data = {
+    messages: [{ role: 'user', content: prompt.prompt }],
+    temperature: 0.0,
+    model: model.name,
+    logprobs: true,
+    top_logprobs: 3,
+  };
+  const headers = {
+    'Content-Type': 'application/json',
+    accept: 'application/json',
+  };
+  if (model.apiKey) {
+    headers['Authorization'] = `Bearer ${model.apiKey}`;
+  }
+
+  const params = {
+    method: 'POST',
+    headers: headers,
+    body: JSON.stringify(data),
+  };
+  if (process.env.DUMP_HALLOUMI_REQ_FILE_PATH) {
+    const filePath = process.env.DUMP_HALLOUMI_REQ_FILE_PATH;
+    fs.writeFileSync(
+      filePath,
+      JSON.stringify(
+        { url: model.apiUrl, params: { ...params, body: data } },
+        null,
+        2,
+      ),
+    );
+    log(`Dumped halloumi response: ${filePath}`);
+  }
+
+  const response = await fetch(model.apiUrl, params);
+  jsonData = await response.json();
+
+  if (process.env.DUMP_HALLOUMI_FILE_PATH) {
+    const filePath = process.env.DUMP_HALLOUMI_FILE_PATH;
+    fs.writeFileSync(filePath, JSON.stringify(jsonData, null, 2));
+    log(`Dumped halloumi response: ${filePath}`);
+  }
+
+  return jsonData;
+}
+
+/**
+ * Gets all claims from a response.
+ * @param response A string containing all claims and their information.
+ * @returns A list of claim objects.
+ */
+export async function halloumiGenerativeAPI(model, prompt) {
+  const jsonData = await getLLMResponse(model, prompt);
+
+  log('Generative response', jsonData);
+  log('Logprobs', jsonData.choices[0].logprobs.content);
+
+  const logits = jsonData.choices[0].logprobs.content;
+  const tokenProbabilities = getTokenProbabilitiesFromLogits(
+    logits,
+    tokenChoices,
+  );
+  const parsedResponse = getClaimsFromResponse(
+    jsonData.choices[0].message.content,
+  );
+
+  if (parsedResponse.length !== tokenProbabilities.length) {
+    throw new Error('Token probabilities and claims do not match.');
+  }
+
+  for (let i = 0; i < parsedResponse.length; i++) {
+    const scoreMap = tokenProbabilities[i];
+    if (model.plattScaling) {
+      const platt = model.plattScaling;
+      const unsupportedScore = applyPlattScaling(
+        platt,
+        scoreMap.get('unsupported'),
+      );
+      const supportedScore = 1 - unsupportedScore;
+      scoreMap.set('supported', supportedScore);
+      scoreMap.set('unsupported', unsupportedScore);
+    }
+    parsedResponse[i].probabilities = scoreMap;
+  }
+
+  return parsedResponse;
+}
+
+export function convertGenerativesClaimToVerifyClaimResponse(
+  generativeClaims,
+  prompt,
+) {
+  const segments = {};
+  const claims = [];
+
+  for (const offset of prompt.contextOffsets) {
+    const id = offset[0].toString();
+    segments[id] = {
+      id,
+      startOffset: offset[1].startOffset,
+      endOffset: offset[1].endOffset,
+    };
+  }
+
+  for (const generativeClaim of generativeClaims) {
+    const segmentIds = [];
+    for (const seg of generativeClaim.segments) {
+      segmentIds.push(seg.toString());
+    }
+
+    const claimId = generativeClaim.claimId;
+    if (!prompt.responseOffsets.has(claimId)) {
+      throw new Error(`Claim ${claimId} not found in response offsets.`);
+    }
+
+    const claimResponseWindow = prompt.responseOffsets.get(claimId);
+    const score = generativeClaim.probabilities.get('supported');
+    const claim = {
+      claimId,
+      claimString: generativeClaim.claimString,
+      startOffset: claimResponseWindow.startOffset,
+      endOffset: claimResponseWindow.endOffset,
+      rationale: generativeClaim.explanation,
+      segmentIds,
+      score,
+    };
+    claims.push(claim);
+  }
+
+  const response = {
+    claims,
+    segments,
+  };
+
+  return response;
+}

package/src/halloumi/generative.test.js

@@ -0,0 +1,88 @@
+import {
+  halloumiGenerativeAPI,
+  convertGenerativesClaimToVerifyClaimResponse,
+} from './generative';
+import path from 'path';
+
+describe('halloumiGenerativeAPI reads from mock file', () => {
+  const originalEnv = process.env;
+
+  beforeEach(() => {
+    jest.resetModules(); // Most important - reset modules between test runs
+    process.env = {
+      ...originalEnv,
+      MOCK_HALLOUMI_FILE_PATH: path.join(__dirname, '../dummy/qa-raw-3.json'),
+    };
+  });
+
+  afterEach(() => {
+    process.env = originalEnv; // Restore original env
+    jest.restoreAllMocks(); // Restore all mocks
+  });
+
+  it('should read from the mock file when MOCK_LLM_CALL is true', async () => {
+    const model = { name: 'test-model', apiUrl: 'http://test.com' };
+    const prompt = {
+      prompt: 'test-prompt',
+      contextOffsets: new Map([[1, { startOffset: 0, endOffset: 10 }]]),
+      responseOffsets: new Map([[1, { startOffset: 0, endOffset: 20 }]]),
+    };
+
+    // We are testing halloumiGenerativeAPI which internally calls getLLMResponse
+    // and getLLMResponse uses the MOCK_LLM_CALL env variable.
+    const response = await halloumiGenerativeAPI(model, prompt);
+
+    expect(response[0].claimString).toEqual(
+      '**France – total waste generation (latest available data)**  \n',
+    );
+    expect(response[0].segments).toEqual([
+      38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48,
+    ]);
+  });
+});
+
+describe('convertGenerativesClaimToVerifyClaimResponse', () => {
+  it('should correctly convert generative claims to verify claim response', () => {
+    const generativeClaims = [
+      {
+        claimId: 1,
+        claimString: 'Test claim string',
+        subclaims: ['subclaim1', 'subclaim2'],
+        segments: [1, 2, 3],
+        explanation: 'Test explanation',
+        supported: true,
+        probabilities: new Map([
+          ['supported', 0.9],
+          ['unsupported', 0.1],
+        ]),
+      },
+    ];
+
+    const prompt = {
+      contextOffsets: new Map([[1, { startOffset: 0, endOffset: 10 }]]),
+      responseOffsets: new Map([[1, { startOffset: 100, endOffset: 120 }]]),
+    };
+
+    const result = convertGenerativesClaimToVerifyClaimResponse(
+      generativeClaims,
+      prompt,
+    );
+
+    expect(result).toEqual({
+      claims: [
+        {
+          claimId: 1,
+          claimString: 'Test claim string',
+          startOffset: 100,
+          endOffset: 120,
+          rationale: 'Test explanation',
+          segmentIds: ['1', '2', '3'],
+          score: 0.9,
+        },
+      ],
+      segments: {
+        1: { id: '1', startOffset: 0, endOffset: 10 },
+      },
+    });
+  });
+});

package/src/halloumi/middleware.js

@@ -0,0 +1,70 @@
+import debug from 'debug';
+import { getVerifyClaimResponse } from './generative';
+
+const log = debug('halloumi');
+
+const MSG_INVALID_CONFIGURATION =
+  'Invalid configuration: missing LLMGW_TOKEN or LLMGW_URL';
+
+const LLMGW_URL = process.env.LLMGW_URL;
+const LLMGW_TOKEN = process.env.LLMGW_TOKEN;
+
+const generativeModel = {
+  name: 'Inhouse-LLM/HallOumi-8B',
+  apiUrl: `${LLMGW_URL}/chat/completions`,
+  plattScaling: {
+    a: -0.5764390035379638,
+    b: 0.16648741572432335,
+  },
+  isEmbeddingModel: false,
+};
+
+const classifyModel = {
+  name: 'Inhouse-LLM/HallOumi-8B-classifier',
+  apiUrl: `${LLMGW_URL}/v1/embeddings`,
+  isEmbeddingModel: true,
+  plattScaling: {
+    a: -0.9468640744087437,
+    b: -0.07379217647931409,
+  },
+};
+
+export default async function middleware(req, res, next) {
+  const path = req.url.replace('/_ha/', '/');
+
+  if (!(LLMGW_TOKEN && LLMGW_URL)) {
+    res.send({
+      error: MSG_INVALID_CONFIGURATION,
+    });
+    return;
+  }
+
+  const model = {
+    ...(path === '/classify' ? classifyModel : generativeModel),
+    apiKey: LLMGW_TOKEN,
+  };
+  const body = req.body;
+
+  log('Halloumi body', body);
+  const { sources, answer, maxContextSegments = 0 } = body;
+
+  res.set('Content-Type', 'application/json');
+
+  try {
+    const resp = await getVerifyClaimResponse(
+      model,
+      // TODO: map with citation id
+      sources,
+      answer,
+      maxContextSegments,
+    );
+    log('Halloumi response', resp);
+    res.send(resp);
+  } catch (error) {
+    res.status(500).send({
+      error: `Halloumi error: ${error}`,
+      traceback: error.stack || null,
+    });
+    // throw error;
+  }
+}

package/src/halloumi/postprocessing.js

@@ -0,0 +1,273 @@
+// /**
+//  * Represents a claim object with all relevant information.
+//  */
+// export interface GenerativeClaim {
+//     claimId: number;
+//     claimString: string;
+//     subclaims: string[];
+//     segments: number[];
+//     explanation: string;
+//     supported: boolean;
+//     probabilities: Map<string, number>;
+// }
+//
+// export interface OpenAITokenLogProb {
+//     token: string;
+//     bytes: number[];
+//     logprob: number;
+// }
+//
+// export interface OpenAILogProb {
+//     token: string;
+//     bytes: number[];
+//     logprob: number;
+//     top_logprobs: OpenAITokenLogProb[];
+// }
+//
+/**
+ * Gets the claim id from a subsegment.
+ * @param subsegment A subsegment string of the form "<|r1|".
+ * @returns The numeric claim id.
+ */
+function getClaimIdFromSubsegment(subsegment) {
+  const textClaimId = subsegment.split('|')[1];
+  const integerClaimId = parseInt(textClaimId.split('r')[1]);
+  return integerClaimId;
+}
+
+/**
+ * Gets the citations from a subsegment.
+ * @param subsegment A subsegment string of the form "|s1|,|s2|,|s3|,|s4|".
+ * @returns The list of numeric citations.
+ */
+function getClaimCitationsFromSubsegment(subsegment) {
+  const citationSegments = subsegment.split(',');
+  const segments = [];
+  for (const citationSegment of citationSegments) {
+    const segment = citationSegment.replaceAll('|', '').replaceAll('s', '');
+    if (segment.includes('-')) {
+      const segmentRange = segment.split('-');
+      for (
+        let i = parseInt(segmentRange[0].trim());
+        i <= parseInt(segmentRange[1].trim());
+        i++
+      ) {
+        segments.push(i);
+      }
+    } else if (segment.includes('to')) {
+      const segmentRange = segment.split('to');
+      for (
+        let i = parseInt(segmentRange[0].trim());
+        i <= parseInt(segmentRange[1].trim());
+        i++
+      ) {
+        segments.push(i);
+      }
+    } else {
+      const segmentInt = parseInt(segment);
+      if (!isNaN(segmentInt)) {
+        segments.push(parseInt(segment));
+      }
+    }
+  }
+  return segments;
+}
+
+/**
+ * Gets the support status from a subsegment.
+ * @param subsegment A subsegment string of the form "|supported|" or "|unsupported|".
+ * @returns True if the claim is supported, false otherwise.
+ */
+function getSupportStatusFromSubsegment(subsegment) {
+  return subsegment.startsWith('|supported|');
+}
+
+/**
+ * Gets a claim from a segment string.
+ * @param segment A segment string containing all information for claim verification.
+ * @returns The claim object with all relevant information.
+ */
+function getClaimFromSegment(segment) {
+  const claim_segments = segment.split('><');
+  const claimId = getClaimIdFromSubsegment(claim_segments[0]);
+  // Strip trailing '>' which can occur in malformed segments without subclaims
+  const claimString = claim_segments[1]?.replace(/>$/, '') || '';
+
+  const subclaims = [];
+  let claimProgressIndex = 3; // Start at 3 to skip the claim id, claim string and the subclaims tag
+  for (let i = claimProgressIndex; i < claim_segments.length; i++) {
+    const subsegment = claim_segments[i];
+    if (subsegment.startsWith('end||subclaims')) {
+      claimProgressIndex = i + 1;
+      break;
+    } else {
+      subclaims.push(subsegment);
+    }
+  }
+
+  let cite_tag_index = -1;
+  let explanation_index = -1;
+  let label_index = -1;
+  for (let i = claimProgressIndex; i < claim_segments.length; i++) {
+    const subsegment = claim_segments[i];
+    if (subsegment.startsWith('|cite|')) {
+      cite_tag_index = i;
+    } else if (subsegment.startsWith('|explain|')) {
+      explanation_index = i + 1;
+    } else if (
+      subsegment.startsWith('|supported|') ||
+      subsegment.startsWith('|unsupported|')
+    ) {
+      label_index = i;
+    }
+  }
+
+  const segments =
+    cite_tag_index >= 0
+      ? getClaimCitationsFromSubsegment(claim_segments[cite_tag_index + 1])
+      : [];
+  const explanation =
+    explanation_index >= 0 ? claim_segments[explanation_index] : '';
+
+  // Default to true if label not found
+  const supported =
+    label_index >= 0
+      ? getSupportStatusFromSubsegment(claim_segments[label_index])
+      : true;
+
+  const claim = {
+    claimId,
+    claimString,
+    subclaims,
+    segments,
+    explanation,
+    supported,
+    probabilities: new Map(),
+  };
+
+  return claim;
+}
+
+/**
+ * Gets all claims from a response.
+ * @param response A string containing all claims and their information.
+ * @returns A list of claim objects.
+ */
+export function getClaimsFromResponse(response) {
+  // Example response: <|r1|><There is no information about the average lifespan of a giant squid in the deep waters of the Pacific Ocean in the provided document.><|subclaims|><The document contains information about the average lifespan of a giant squid.><The information about giant squid lifespan is related to the Pacific Ocean.><end||subclaims><|cite|><|s1 to s49|><end||cite><|explain|><Upon reviewing the entire document, there is no mention of giant squid or any related topic, including their average lifespan or the Pacific Ocean. The document is focused on international relations, diplomacy, and conflict resolution.><end||explain><|supported|><end||r><|r2|><The document is focused on international relations, diplomacy, and conflict resolution, and does not mention giant squid or any related topic.><|subclaims|><The document is focused on international relations, diplomacy, and conflict resolution.><The document does not mention giant squid or any related topic.><end||subclaims><|cite|><|s1|,|s2|,|s3|,|s4|><end||cite><|explain|><The first four sentences clearly establish the document's focus on international relations, diplomacy, and conflict resolution, and there is no mention of giant squid or any related topic throughout the document.><end||explain><|supported|><end||r><|r3|><The document mentions cats.><|subclaims|><The document makes some mention of cats.><end||subclaims><|cite|><None><end||cite><|explain|><There is no mention of cats anywhere in the document.><end||explain><|unsupported|><end||r>
+
+  // eslint-disable-next-line no-console
+  // console.log('getClaimsFromResponse', response);
+  let segments = response.split('<end||r>');
+  const claims = [];
+
+  for (let i = 0; i < segments.length; i++) {
+    const segment = segments[i];
+
+    if (segment.length === 0) {
+      continue;
+    }
+
+    // Skip segments that don't start with a valid claim marker <|r*|>
+    if (!segment.match(/^<\|r\d+\|>/)) {
+      continue;
+    }
+
+    // Check if this segment is well-formed (has subclaims within it)
+    // Well-formed: <|r1|><claim><|subclaims|>...<|supported|>...<end||r>
+    // Malformed:   <|r1|><claim><end||r><|subclaims|>...<|supported|>...<end||r>
+    // The malformed case puts verification data AFTER <end||r>, creating orphan segments
+    if (segment.includes('<|subclaims|>')) {
+      // Well-formed segment - process normally
+      const claim = getClaimFromSegment(segment);
+      claims.push(claim);
+    } else {
+      // Malformed segment - look for orphan data in the next segment
+      let fullSegment = segment;
+
+      // Look ahead for orphan segment (starts with <|subclaims|>)
+      if (i + 1 < segments.length) {
+        const nextSegment = segments[i + 1];
+        if (nextSegment.startsWith('<|subclaims|>')) {
+          // Merge: segment ends with '>', orphan starts with '<'
+          fullSegment = segment + nextSegment;
+          i++; // Skip the orphan in next iteration
+        }
+      }
+
+      const claim = getClaimFromSegment(fullSegment);
+      claims.push(claim);
+    }
+  }
+
+  return claims;
+}
+
+function exp(x) {
+  return Math.pow(Math.E, x);
+}
+
+function softmax(logits) {
+  const softmaxes = [];
+  const exp_values = [];
+  let total = 0;
+  for (let i = 0; i < logits.length; i++) {
+    const exponential = exp(logits[i]);
+    total += exponential;
+    exp_values.push(exponential);
+  }
+
+  for (let i = 0; i < exp_values.length; i++) {
+    softmaxes.push(exp_values[i] / total);
+  }
+  return softmaxes;
+}
+
+function getTokenProbabilitiesFromLogit(logit, tokenChoices) {
+  const tokenLogits = new Map();
+  let smallestLogit = 1000000;
+  for (const token of tokenChoices) {
+    const tokenLogit = logit.top_logprobs.find(
+      (logit) => logit.token === token,
+    );
+    if (tokenLogit !== undefined) {
+      smallestLogit = Math.min(smallestLogit, tokenLogit.logprob);
+      tokenLogits.set(token, tokenLogit.logprob);
+    }
+  }
+
+  // If any class tokens aren't present, set them below the smallest logit.
+  // Their true value is definitely <= the smallest logit value.
+  for (const token of tokenChoices) {
+    if (!tokenLogits.has(token)) {
+      tokenLogits.set(token, smallestLogit - 1e-6);
+    }
+  }
+
+  const classTokens = Array.from(tokenLogits.keys());
+  const logitValues = Array.from(tokenLogits.values());
+  const softmaxValues = softmax(logitValues);
+  const tokenProbabilities = new Map();
+  for (let i = 0; i < classTokens.length; i++) {
+    tokenProbabilities.set(classTokens[i], softmaxValues[i]);
+  }
+
+  return tokenProbabilities;
+}
+
+export function getTokenProbabilitiesFromLogits(logits, tokenChoices) {
+  const tokenProbabilities = [];
+  for (const logit of logits) {
+    const tokenPresent = tokenChoices.has(logit.token);
+    if (!tokenPresent) {
+      continue;
+    }
+
+    const tokenProbability = getTokenProbabilitiesFromLogit(
+      logit,
+      tokenChoices,
+    );
+    tokenProbabilities.push(tokenProbability);
+  }
+  return tokenProbabilities;
+}