@eeacms/volto-eea-chatbot 4.1.3 → 4.1.4

package/src/ChatBlock/services/chunkEvidence.ts

@@ -0,0 +1,251 @@
+/**
+ * Resolve the real document text behind a chat answer.
+ *
+ * The Onyx v3 stream ships documents as search *blurbs* (~600 chars) — it has no
+ * `content` field at all (see `SearchDoc` in Onyx `context/search/models.py`).
+ * Sending blurbs to the fact-checker makes supported claims look hallucinated,
+ * because the answer was written from full chunk text, not from those snippets.
+ *
+ * Onyx already exposes that text:
+ *
+ *     GET /api/document/chunk-info?document_id=<id>&chunk_id=<n>
+ *     -> { "content": "<chunk text>", "num_tokens": 512 }
+ *
+ * and the stream already carries both parameters for every document
+ * (`document_id` + `chunk_ind`). This module goes through the Volto `/_da`
+ * proxy so no extra credentials or CORS setup are needed.
+ *
+ * Notes:
+ * - One chunk is ~512 tokens (`DOC_EMBEDDING_CONTEXT_SIZE`). Onyx builds the
+ *   answer from the matched chunk *plus its neighbourhood* (±2 for
+ *   `INCLUDE_ADJACENT_SECTIONS`, ±5 for `FULL_DOCUMENT`), so the centre chunk
+ *   alone is not enough — `CHUNK_EVIDENCE_WINDOW` defaults to 2.
+ * - A gateway in front of Onyx returns 429 under concurrency; one retry after a
+ *   short delay clears it.
+ * - 404 simply means "no such chunk" (past the end of the document) and ends the
+ *   expansion for that document.
+ */
+
+export const CHUNK_EVIDENCE_WINDOW = 2;
+/**
+ * Safety bound for the window. Onyx itself never expands past ±5
+ * (`FULL_DOC_NUM_CHUNKS_AROUND`), and every extra step costs two more chunk
+ * requests per document, so a nonsensical CMS value is clamped rather than
+ * turned into a request storm against a rate-limited gateway.
+ */
+export const MAX_CHUNK_EVIDENCE_WINDOW = 20;
+const DEFAULT_CONCURRENCY = 4;
+const RETRY_DELAY_MS = 1000;
+
+export interface ChunkEvidence {
+  /** Joined chunk text, empty string when nothing could be resolved. */
+  text: string;
+  /** How many chunks were fetched. */
+  chunks: number;
+  /**
+   * `chunk`   – real Onyx chunk text, safe to verify against.
+   * `snippet` – only the search blurb is available; absence of a claim in it is
+   *             NOT evidence of hallucination.
+   */
+  kind: 'chunk' | 'snippet';
+}
+
+export type ChunkEvidenceMap = Record<string, ChunkEvidence>;
+
+type ChunkInfoBody = {
+  content?: string;
+  num_tokens?: number;
+  // Error envelopes: Onyx uses `detail`, the gateway in front of it uses `error`.
+  detail?: string;
+  error?: string;
+};
+
+type FetchLike = (url: string) => Promise<{
+  status: number;
+  json: () => Promise<ChunkInfoBody>;
+}>;
+
+const RATE_LIMIT_PATTERN = /rate limit|slow down/i;
+
+const sleep = (ms: number) =>
+  new Promise((resolve) => {
+    setTimeout(resolve, ms);
+  });
+
+/** Proxy path handled by `src/middleware.js` (`/_da/...` -> Onyx `/api/...`). */
+export function chunkInfoUrl(documentId: string, chunkInd: number): string {
+  return `/_da/document/chunk-info?document_id=${encodeURIComponent(
+    documentId,
+  )}&chunk_id=${chunkInd}`;
+}
+
+/**
+ * Fetch a single chunk. Returns null for "no such chunk" and for any
+ * transport/HTTP failure, retrying once on 429.
+ *
+ * The response body is authoritative, not the status: the Volto proxy pipes
+ * Onyx bodies through without the upstream status, so a 404 arrives as
+ * `200 + {"detail": "Chunk not found"}` and a gateway throttle as
+ * `200 + {"error": "Rate limit exceeded…"}`. A chunk with no `content` is never
+ * usable evidence, whatever the status says.
+ */
+export async function fetchChunk(
+  documentId: string,
+  chunkInd: number,
+  {
+    fetchImpl,
+    retryDelay = RETRY_DELAY_MS,
+  }: { fetchImpl?: FetchLike; retryDelay?: number } = {},
+): Promise<string | null> {
+  const doFetch: FetchLike = fetchImpl || ((url: string) => fetch(url));
+
+  const attempt = async (): Promise<{
+    status: number;
+    text: string;
+  } | null> => {
+    try {
+      const response = await doFetch(chunkInfoUrl(documentId, chunkInd));
+      if (response.status === 429) {
+        return { status: 429, text: '' };
+      }
+      if (response.status !== 200) {
+        return { status: response.status, text: '' };
+      }
+      const body = await response.json();
+      if (typeof body?.content === 'string' && body.content !== '') {
+        return { status: 200, text: body.content };
+      }
+      const message = String(body?.error || body?.detail || '');
+      return {
+        status: RATE_LIMIT_PATTERN.test(message) ? 429 : 404,
+        text: '',
+      };
+    } catch {
+      return null;
+    }
+  };
+
+  let result = await attempt();
+  if (result && result.status === 429) {
+    await sleep(retryDelay);
+    result = await attempt();
+  }
+  if (!result || result.status !== 200) {
+    return null;
+  }
+  return result.text;
+}
+
+/**
+ * Normalise a configured window: whole number between 0 and
+ * `MAX_CHUNK_EVIDENCE_WINDOW`, falling back to the default for junk input.
+ * The value comes from a Volto block field, so it can be anything an editor typed.
+ */
+export function normalizeWindow(window: number): number {
+  const value = Number(window);
+  if (!Number.isFinite(value)) {
+    return CHUNK_EVIDENCE_WINDOW;
+  }
+  return Math.max(0, Math.min(Math.floor(value), MAX_CHUNK_EVIDENCE_WINDOW));
+}
+
+/**
+ * Fetch the matched chunk plus `window` chunks on each side, stopping at the
+ * first missing chunk above the centre (the end of the document).
+ */
+export async function fetchDocumentEvidence(
+  documentId: string,
+  centerChunkInd: number,
+  {
+    window = CHUNK_EVIDENCE_WINDOW,
+    fetchImpl,
+    retryDelay,
+  }: { window?: number; fetchImpl?: FetchLike; retryDelay?: number } = {},
+): Promise<ChunkEvidence> {
+  const center = Math.max(0, Number(centerChunkInd) || 0);
+  const win = normalizeWindow(window);
+  const lower = Math.max(0, center - win);
+  const upper = center + win;
+
+  const inds: number[] = [];
+  for (let ind = lower; ind <= upper; ind++) {
+    inds.push(ind);
+  }
+
+  const texts: string[] = [];
+  for (const ind of inds) {
+    const content = await fetchChunk(documentId, ind, {
+      fetchImpl,
+      retryDelay,
+    });
+    if (content === null) {
+      // Past the end of the document — everything above is missing too.
+      if (ind > center) {
+        break;
+      }
+      continue;
+    }
+    texts.push(content);
+  }
+
+  return {
+    text: texts.join('\n'),
+    chunks: texts.length,
+    kind: texts.length > 0 ? 'chunk' : 'snippet',
+  };
+}
+
+/**
+ * Resolve evidence for a set of documents, a few at a time.
+ *
+ * Accepts anything carrying `document_id` + `chunk_ind` (the Onyx documents, or
+ * the `halloumiSource`-shaped objects built for the fact-checker).
+ */
+export async function fetchEvidence(
+  docs: any[],
+  {
+    window = CHUNK_EVIDENCE_WINDOW,
+    concurrency = DEFAULT_CONCURRENCY,
+    fetchImpl,
+    retryDelay,
+  }: {
+    window?: number;
+    concurrency?: number;
+    fetchImpl?: FetchLike;
+    retryDelay?: number;
+  } = {},
+): Promise<ChunkEvidenceMap> {
+  const unique: any[] = [];
+  const seen = new Set<string>();
+  (docs || []).forEach((doc: any) => {
+    const documentId = doc?.document_id;
+    if (!documentId || seen.has(documentId)) {
+      return;
+    }
+    seen.add(documentId);
+    unique.push(doc);
+  });
+
+  const evidence: ChunkEvidenceMap = {};
+  let cursor = 0;
+
+  const worker = async () => {
+    while (cursor < unique.length) {
+      const current = cursor++;
+      const doc = unique[current];
+      evidence[doc.document_id] = await fetchDocumentEvidence(
+        doc.document_id,
+        doc.chunk_ind,
+        { window, fetchImpl, retryDelay },
+      );
+    }
+  };
+
+  const workers = Array.from(
+    { length: Math.max(1, Math.min(concurrency, unique.length)) },
+    worker,
+  );
+  await Promise.all(workers);
+
+  return evidence;
+}

package/src/ChatBlock/style.less

@@ -813,19 +813,47 @@ mark {
     text-transform: uppercase;
   }
 
-  .score-progress-bar {
-    overflow: hidden;
+  // Three discrete positions instead of a proportional fill: a per-claim score is
+  // a verdict, not a probability. The old bar made not_enough_info read as
+  // "40% verified", which was never measured.
+  .verdict-scale {
+    display: flex;
     width: 100%;
+    gap: 4px;
+  }
+
+  .verdict-step {
     height: 6px;
+    flex: 1;
     border-radius: 3px;
     background-color: @gray-200;
+    // An inset ring rather than a border: the green of `not_enough_info` is about
+    // 2.8:1 against white, so without an outline the middle position - the one that
+    // needs pointing out most - reads as empty. No layout shift, unlike a border.
+    box-shadow: inset 0 0 0 1px @gray-300;
   }
 
-  .score-progress-fill {
-    height: 100%;
-    border-radius: 3px;
-    background-color: var(--claim-color);
-    transition: width 0.3s ease;
+  .verdict-scale[data-verdict='contradicted'] .verdict-step:nth-child(1) {
+    background-color: @red-500;
+  }
+
+  // Not red: sources saying nothing is not the same finding as sources saying
+  // something else. green-500 rather than the chip's green-300 because a solid bar
+  // on white needs more weight than a pale background does (green-300 is 1.7:1).
+  .verdict-scale[data-verdict='not_enough_info'] .verdict-step:nth-child(2) {
+    background-color: @green-500;
+  }
+
+  .verdict-scale[data-verdict='supported'] .verdict-step:nth-child(3) {
+    background-color: @green-700;
+  }
+
+  // Checked over search snippets only: no position can be claimed.
+  .verdict-scale[data-verdict='unknown'] .verdict-step {
+    height: 4px;
+    border: 1px dashed @gray-400;
+    background-color: transparent;
+    box-shadow: none; // the dashed outline already carries the shape
   }
 
   .rationale-section {

package/src/ChatBlock/types/interfaces.ts

@@ -153,6 +153,12 @@ export interface ChatMessageProps {
   persona?: number;
   maxContextSegments?: number;
   batchSize?: number;
+  /**
+   * How many Onyx chunks to pull around the matched one when resolving real
+   * document text for the fact-checker (0 = centre chunk only, 5 = the widest
+   * expansion Onyx itself uses). See `services/chunkEvidence.ts`.
+   */
+  chunkEvidenceWindow?: number;
   isLastMessage?: boolean;
   className?: string;
   chatWindowRef?: React.RefObject<HTMLDivElement>;

package/src/ChatBlock/utils/index.tsx

@@ -59,13 +59,63 @@ export const useCopyToClipboard = (text: string): [boolean, () => void] => {
 };
 
 /**
- * Map a 0-1 claim score to a categorical label.
- * Scores: 1.0 = supported, 0.4 = not_enough_info, 0.0 = contradicted.
+ * The fact-checker collapses its three verdicts into a per-claim score and does
+ * not send the verdict itself (`verdict_scores` in `rag_facts_check/server.py`:
+ * supported 1.0, not_enough_info 0.4, contradicted 0.0). The thresholds below are
+ * therefore part of the contract — keep them in step with the backend.
  */
-export function scoreToLabel(score: number): string {
-  if (score >= 0.8) return 'High';
-  if (score >= 0.1) return 'Low';
-  return 'Failed';
+export type ClaimVerdict = 'supported' | 'not_enough_info' | 'contradicted';
+
+export function scoreToVerdict(score: number): ClaimVerdict {
+  if (score >= 0.8) return 'supported';
+  if (score >= 0.1) return 'not_enough_info';
+  return 'contradicted';
+}
+
+/**
+ * Modal header copy. Names the outcome, never the process: "Verified Claim"
+ * reads as "this is true" even when the check contradicted the claim, and the
+ * old High/Low/Failed badge read as confidence levels for what are three
+ * discrete verdicts.
+ */
+const VERDICT_HEADER: Record<ClaimVerdict, string> = {
+  supported: 'Supported by the sources',
+  not_enough_info: 'Not confirmed by the sources',
+  contradicted: 'Contradicted by the sources',
+};
+
+const VERDICT_BADGE: Record<ClaimVerdict, string> = {
+  supported: 'Supported',
+  not_enough_info: 'Not confirmed',
+  contradicted: 'Contradicted',
+};
+
+export function claimHeaderLabel(
+  score: number,
+  contextLimited = false,
+): string {
+  // Checked over search snippets only: the supporting text may simply have been
+  // missing, so the verdict is not trustworthy enough to name.
+  if (contextLimited) return "Couldn't be checked";
+  return VERDICT_HEADER[scoreToVerdict(score)];
+}
+
+export function claimBadgeLabel(score: number, contextLimited = false): string {
+  if (contextLimited) return 'Unverified';
+  return VERDICT_BADGE[scoreToVerdict(score)];
+}
+
+/**
+ * Which of the three positions the verdict indicator marks, or `unknown` when the
+ * check ran over snippets only and no verdict can be named.
+ */
+export type ClaimScaleState = ClaimVerdict | 'unknown';
+
+export function claimScaleState(
+  score: number,
+  contextLimited = false,
+): ClaimScaleState {
+  return contextLimited ? 'unknown' : scoreToVerdict(score);
 }
 
 export function convertToPercentage(

package/src/halloumi/README.md

@@ -14,6 +14,97 @@ Frontend (useQualityMarkers.js)
 
 The rag-fact-checker's `/halloumi/generate` endpoint is a drop-in replacement for the original Halloumi API — it accepts the same request format (`{answer, sources}`) and returns a compatible response (`{claims, segments}`), so all existing frontend components (`HalloumiFeedback`, `useQualityMarkers`, `ClaimSegments`) work without changes.
 
+## Multi-evidence claims
+
+A claim can be supported by several evidence quotes (the fact-checker returns up to
+3 per claim), so `claim.segmentIds` is a list and each segment carries offsets into
+the **joined** sources string:
+
+```json
+{
+  "claims": [{ "segmentIds": ["0", "1", "2"], "score": 1.0 }],
+  "segments": {
+    "0": { "id": 0, "startOffset": 120, "endOffset": 190 },
+    "1": { "id": 1, "startOffset": 402, "endOffset": 455 },
+    "2": { "id": 2, "startOffset": 980, "endOffset": 1030 }
+  }
+}
+```
+
+`ClaimSegments` groups those segments per source (one tab per cited document, one
+"Jump to citation" chip per segment) and `RenderClaimView` highlights them inside the
+source text. Segments are half-open `[start, end)`; overlapping ones are clipped so
+the source text is never rendered twice.
+
+## Source text: Onyx chunks, not search blurbs
+
+The Onyx v3 chat stream carries documents as `SearchDoc` objects, which have **no
+`content` field** — only a ~600-character `blurb`. Sending blurbs makes supported
+claims look hallucinated, because the answer was written from full chunk text.
+
+`ChatBlock/services/chunkEvidence.ts` therefore resolves the real text before the
+fact-check runs, through the existing `/_da` proxy:
+
+```
+GET /_da/document/chunk-info?document_id=<id>&chunk_id=<n>
+  -> Onyx GET /api/document/chunk-info -> { "content": "<chunk text>", "num_tokens": 512 }
+```
+
+- Both parameters are already present on every streamed document (`document_id`,
+  `chunk_ind`), so no extra data is needed from Onyx.
+- The matched chunk is fetched together with `chunkEvidenceWindow` chunks on each
+  side (default **2**, i.e. what Onyx's `INCLUDE_ADJACENT_SECTIONS` expansion uses).
+  The centre chunk alone is not enough — the claims usually sit in its
+  neighbourhood. Onyx's widest expansion is ±5; raise the value to match it.
+- The window is also a block setting — **Evidence window** in the chat block sidebar —
+  so it can be tuned per block without a deploy. Values outside 0–20 are clamped
+  (`MAX_CHUNK_EVIDENCE_WINDOW`): each extra step costs two more chunk requests per
+  document against a rate-limited gateway.
+- A gateway in front of Onyx returns **429** under concurrency, so requests run
+  through a small pool with one retry after 1s. **404** means "no such chunk" and
+  simply ends the expansion for that document.
+- `useChunkEvidence` gates the fact-check call on `evidenceSettled`, so a check is
+  never fired on blurbs in the same commit that started the fetch. If resolution
+  fails, verification still runs on blurbs rather than never running.
+
+Each source sent to the fact-checker carries `kind`:
+
+| `kind`    | meaning                                                               |
+| --------- | --------------------------------------------------------------------- |
+| `chunk`   | real Onyx chunk text — a claim missing from it is meaningful evidence |
+| `snippet` | only the search blurb is available — a missing claim proves nothing   |
+
+The fact-checker answers with a `context_quality` block describing what it was
+actually given:
+
+```json
+{
+  "level": "partial",
+  "sources": 4,
+  "chunk_sources": 3,
+  "snippet_sources": 1,
+  "note": "1 of 4 sources were search snippets, not full document text…"
+}
+```
+
+| `level`   | meaning                                                     |
+| --------- | ----------------------------------------------------------- |
+| `full`    | every source was `chunk` text — the score is a real verdict |
+| `partial` | at least one source was a `snippet`                         |
+| `unknown` | the backend reported no level (older deployment)            |
+| `none`    | no sources reached the checker                              |
+
+In `partial` mode each `not_enough_info` claim additionally carries
+`context_limited: true`. Both signals are rendered: `HalloumiFeedback` shows a
+"Partial context" notice under the score, and `ClaimModal` labels such a claim
+`Unverified / Partial context` instead of `Low / Verification`. The numeric score is
+deliberately left alone — the label carries the caveat, the number stays comparable
+across runs.
+
+`halloumiContext` (used for segment highlighting) and `halloumiSource.text` (used by
+the fact-checker for offsets) are always the same nbsp-cleaned string. Changing one
+without the other breaks highlighting.
+
 ## Configuration
 
 | Environment Variable   | Default                 | Description                              |

package/src/index.js

@@ -57,5 +57,6 @@ export * from './ChatBlock/packets';
 export { default as UserActionsToolbar } from './ChatBlock/components/UserActionsToolbar';
 export { default as FeedbackModal } from './ChatBlock/components/FeedbackModal';
 export { default as ChatMessageFeedback } from './ChatBlock/components/ChatMessageFeedback';
+export { isAISearchInputRestricted } from './AISearchInput';
 
 export default applyConfig;

package/src/middleware.js

@@ -18,6 +18,9 @@ const ALLOWED_PROXY_PATHS = [
   { path: '/chat/send-message', methods: ['POST'] },
   { path: '/chat/send-chat-message', methods: ['POST'] },
   { path: '/chat/create-chat-message-feedback', methods: ['POST'] },
+  // Real document text behind an answer, used to give the fact-checker the same
+  // text the answer generator read (see ChatBlock/services/chunkEvidence.ts).
+  { pathPattern: /^\/document\/chunk-info$/, methods: ['GET'] },
 ];
 
 /**
@@ -259,6 +262,12 @@ async function send_onyx_request(
       log(`Dumped LLM response to: ${filePath}`);
     }
 
+    // Propagate the upstream status. Without this, Express defaults to 200 and
+    // every Onyx error (404 chunk not found, 429 rate limit, 401 revoked key)
+    // reaches the browser as a successful response, which makes the evidence
+    // client unable to tell "missing" from "found".
+    res.status(response.status);
+
     if (!api_key) {
       if (response.headers.get('transfer-encoding') === 'chunked') {
         res.set('Content-Type', 'text/event-stream');

package/src/middleware.test.js

@@ -174,6 +174,62 @@ describe('src/middleware', () => {
     expect(lastCall[1].body).toBeUndefined();
   });
 
+  // Express defaults to 200 when the proxy never sets a status, so every Onyx
+  // error reached the browser as a success. The evidence client cannot tell a
+  // missing chunk from a found one (or a gateway throttle from a response) unless
+  // the upstream status survives the proxy.
+  it('propagates an upstream 404 instead of flattening it to 200', async () => {
+    process.env.ONYX_API_KEY = 'test-key'; //betterleaks:allow
+    process.env.ONYX_URL = 'http://localhost:3000';
+    req.method = 'GET';
+    req.url = '/_da/document/chunk-info?document_id=doc-1&chunk_id=9';
+    req.body = null;
+    nodeFetch.mockResolvedValueOnce({
+      status: 404,
+      headers: {
+        get: jest.fn().mockReturnValue('application/json'),
+        raw: jest.fn().mockReturnValue({}),
+      },
+      body: { pipe: jest.fn() },
+    });
+
+    await middleware(req, res, next);
+
+    expect(res.status).toHaveBeenCalledWith(404);
+  });
+
+  it('propagates an upstream 429 so the client can retry it', async () => {
+    process.env.ONYX_API_KEY = 'test-key'; //betterleaks:allow
+    process.env.ONYX_URL = 'http://localhost:3000';
+    req.method = 'GET';
+    req.url = '/_da/document/chunk-info?document_id=doc-1&chunk_id=0';
+    req.body = null;
+    nodeFetch.mockResolvedValueOnce({
+      status: 429,
+      headers: {
+        get: jest.fn().mockReturnValue('application/json'),
+        raw: jest.fn().mockReturnValue({}),
+      },
+      body: { pipe: jest.fn() },
+    });
+
+    await middleware(req, res, next);
+
+    expect(res.status).toHaveBeenCalledWith(429);
+  });
+
+  it('answers 200 for a successful proxied chunk', async () => {
+    process.env.ONYX_API_KEY = 'test-key'; //betterleaks:allow
+    process.env.ONYX_URL = 'http://localhost:3000';
+    req.method = 'GET';
+    req.url = '/_da/document/chunk-info?document_id=doc-1&chunk_id=0';
+    req.body = null;
+
+    await middleware(req, res, next);
+
+    expect(res.status).toHaveBeenCalledWith(200);
+  });
+
   it('uses mock create-chat-session when MOCK_LLM_FILE_PATH is set', async () => {
     process.env.ONYX_API_KEY = 'test-key'; //betterleaks:allow
     process.env.ONYX_URL = 'http://localhost:3000';