@eeacms/volto-eea-chatbot 4.1.3 → 4.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +4 -0
- package/CHANGELOG.md +25 -3
- package/README.md +1 -14
- package/artifacts/I18N-SPEC.md +257 -0
- package/package.json +1 -1
- package/src/AISearchInput/index.js +20 -6
- package/src/ChatBlock/chat/AIMessage.tsx +71 -21
- package/src/ChatBlock/chat/ChatWindow.tsx +3 -0
- package/src/ChatBlock/components/HalloumiFeedback.jsx +25 -2
- package/src/ChatBlock/components/markdown/ClaimModal.jsx +34 -12
- package/src/ChatBlock/components/markdown/ClaimSegments.jsx +40 -8
- package/src/ChatBlock/components/markdown/RenderClaimView.jsx +56 -13
- package/src/ChatBlock/components/markdown/colors.js +31 -28
- package/src/ChatBlock/components/markdown/colors.less +27 -9
- package/src/ChatBlock/components/markdown/colors.test.js +86 -56
- package/src/ChatBlock/hooks/index.ts +1 -0
- package/src/ChatBlock/hooks/useChunkEvidence.ts +80 -0
- package/src/ChatBlock/schema.jsx +12 -0
- package/src/ChatBlock/services/chunkEvidence.ts +251 -0
- package/src/ChatBlock/style.less +65 -11
- package/src/ChatBlock/types/interfaces.ts +6 -0
- package/src/ChatBlock/utils/index.tsx +56 -6
- package/src/halloumi/README.md +91 -0
- package/src/index.js +1 -0
- package/src/middleware.js +9 -0
- package/src/middleware.test.js +56 -0
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
import { Modal, ModalContent, ModalHeader } from 'semantic-ui-react';
|
|
2
2
|
import cx from 'classnames';
|
|
3
|
-
import {
|
|
3
|
+
import {
|
|
4
|
+
claimBadgeLabel,
|
|
5
|
+
claimHeaderLabel,
|
|
6
|
+
claimScaleState,
|
|
7
|
+
} from '@eeacms/volto-eea-chatbot/ChatBlock/utils';
|
|
4
8
|
import SVGIcon from '@eeacms/volto-eea-chatbot/ChatBlock/components/Icon';
|
|
5
|
-
import {
|
|
9
|
+
import { getVerdictBgColor } from './colors';
|
|
6
10
|
import { ClaimSegments } from './ClaimSegments';
|
|
7
11
|
|
|
8
12
|
import BotIcon from '@eeacms/volto-eea-chatbot/icons/bot.svg';
|
|
@@ -34,10 +38,16 @@ export function ClaimModal({ claim, markers, text, citedSources }) {
|
|
|
34
38
|
|
|
35
39
|
return (
|
|
36
40
|
<Modal
|
|
37
|
-
className={cx(
|
|
41
|
+
className={cx(
|
|
42
|
+
'claim-modal',
|
|
43
|
+
getVerdictBgColor(claim.score, claim.context_limited),
|
|
44
|
+
)}
|
|
38
45
|
trigger={
|
|
39
46
|
<span
|
|
40
|
-
className={cx(
|
|
47
|
+
className={cx(
|
|
48
|
+
'claim',
|
|
49
|
+
getVerdictBgColor(claim.score, claim.context_limited),
|
|
50
|
+
)}
|
|
41
51
|
>
|
|
42
52
|
{text}
|
|
43
53
|
</span>
|
|
@@ -49,7 +59,9 @@ export function ClaimModal({ claim, markers, text, citedSources }) {
|
|
|
49
59
|
<div className="circle assistant">
|
|
50
60
|
<SVGIcon name={BotIcon} size="20" color="white" />
|
|
51
61
|
</div>
|
|
52
|
-
<span className="claim-label">
|
|
62
|
+
<span className="claim-label">
|
|
63
|
+
{claimHeaderLabel(claim.score, claim.context_limited)}
|
|
64
|
+
</span>
|
|
53
65
|
</div>
|
|
54
66
|
<blockquote className="claim-quote">
|
|
55
67
|
“
|
|
@@ -70,20 +82,30 @@ export function ClaimModal({ claim, markers, text, citedSources }) {
|
|
|
70
82
|
<div className="score-badge-section">
|
|
71
83
|
<div className="score-badge">
|
|
72
84
|
<span className="score-percentage">
|
|
73
|
-
{
|
|
85
|
+
{claimBadgeLabel(claim.score, claim.context_limited)}
|
|
74
86
|
</span>
|
|
75
|
-
<span className="score-label">
|
|
87
|
+
<span className="score-label">Fact check</span>
|
|
76
88
|
</div>
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
89
|
+
{/* Three discrete positions, not a fill. The per-claim score is a
|
|
90
|
+
verdict, not a probability: the old proportional bar made
|
|
91
|
+
not_enough_info look like "40% verified", which was never
|
|
92
|
+
measured. Decorative — the badge text carries the verdict. */}
|
|
93
|
+
<div
|
|
94
|
+
className="verdict-scale"
|
|
95
|
+
data-verdict={claimScaleState(claim.score, claim.context_limited)}
|
|
96
|
+
aria-hidden="true"
|
|
97
|
+
>
|
|
98
|
+
<span className="verdict-step" />
|
|
99
|
+
<span className="verdict-step" />
|
|
100
|
+
<span className="verdict-step" />
|
|
82
101
|
</div>
|
|
83
102
|
</div>
|
|
84
103
|
<div className="rationale-section">
|
|
85
104
|
<h5 className="rationale-header">Rationale</h5>
|
|
86
105
|
<p className="claim-rationale">{claim.rationale}</p>
|
|
106
|
+
{/* No snippet caveat here: the header already says "Couldn't be
|
|
107
|
+
checked" and the badge says "Unverified". The explanation lives
|
|
108
|
+
once, at answer level, in HalloumiFeedback. */}
|
|
87
109
|
</div>
|
|
88
110
|
</div>
|
|
89
111
|
|
|
@@ -27,18 +27,50 @@ export function ClaimSegments({ segmentIds, segments, citedSources }) {
|
|
|
27
27
|
.map((segment) => {
|
|
28
28
|
const startOffset = Math.max(0, segment.startOffset); // sometimes startOffset comes as -1
|
|
29
29
|
const endOffset = segment.endOffset;
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
30
|
+
|
|
31
|
+
// A claim can cite several passages, possibly in different documents.
|
|
32
|
+
// Attribute by containment instead of requiring the segment to fit
|
|
33
|
+
// entirely inside one source — a segment that straddles a boundary
|
|
34
|
+
// would otherwise disappear from the modal without any warning.
|
|
35
|
+
const source =
|
|
36
|
+
citedSources.find(
|
|
37
|
+
(source) =>
|
|
38
|
+
startOffset >= source.startIndex &&
|
|
39
|
+
startOffset < source.startIndex + source.halloumiContext.length,
|
|
40
|
+
) ||
|
|
41
|
+
citedSources.find(
|
|
42
|
+
(source) =>
|
|
43
|
+
endOffset > source.startIndex &&
|
|
44
|
+
endOffset <= source.startIndex + source.halloumiContext.length,
|
|
45
|
+
);
|
|
46
|
+
|
|
47
|
+
if (!source) {
|
|
48
|
+
// eslint-disable-next-line no-console
|
|
49
|
+
console.warn(
|
|
50
|
+
`Could not attribute segment ${segment.id} to a source`,
|
|
51
|
+
{ startOffset, endOffset },
|
|
52
|
+
citedSources,
|
|
53
|
+
);
|
|
54
|
+
return null;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// Clamp to the attributed source so the snippet never renders text that
|
|
58
|
+
// belongs to the neighbouring document.
|
|
59
|
+
const clippedStart = Math.max(startOffset, source.startIndex);
|
|
60
|
+
const clippedEnd = Math.min(
|
|
61
|
+
endOffset,
|
|
62
|
+
source.startIndex + source.halloumiContext.length,
|
|
35
63
|
);
|
|
64
|
+
|
|
36
65
|
return {
|
|
37
66
|
...segment,
|
|
38
|
-
|
|
39
|
-
|
|
67
|
+
startOffset: clippedStart,
|
|
68
|
+
endOffset: clippedEnd,
|
|
69
|
+
text: joinedSources.slice(clippedStart, clippedEnd),
|
|
70
|
+
source_id: source.id,
|
|
40
71
|
};
|
|
41
|
-
})
|
|
72
|
+
})
|
|
73
|
+
.filter((snippet) => !!snippet && snippet.endOffset > snippet.startOffset);
|
|
42
74
|
|
|
43
75
|
const sourcesWithSnippets = citedSources
|
|
44
76
|
.map((source) => ({
|
|
@@ -1,5 +1,19 @@
|
|
|
1
1
|
import React from 'react';
|
|
2
2
|
|
|
3
|
+
/**
|
|
4
|
+
* Render a source document with the evidence segments of a claim highlighted.
|
|
5
|
+
*
|
|
6
|
+
* Segment offsets are half-open ranges `[startOffset, endOffset)` expressed in
|
|
7
|
+
* the coordinates of the *joined* sources string, so they are shifted by
|
|
8
|
+
* `sourceStartIndex` before slicing this source's text.
|
|
9
|
+
*
|
|
10
|
+
* A claim can be supported by several evidence quotes, and those quotes can
|
|
11
|
+
* overlap (two passages covering the same sentence, for example). Because the
|
|
12
|
+
* renderer splits the text on segment boundaries, overlapping segments would
|
|
13
|
+
* emit the same characters twice — segments are therefore clipped to what is
|
|
14
|
+
* still uncovered, and a fully covered segment keeps its id on the segment that
|
|
15
|
+
* covers it so its "Jump to citation" chip still scrolls somewhere sensible.
|
|
16
|
+
*/
|
|
3
17
|
export const RenderClaimView = (props) => {
|
|
4
18
|
const {
|
|
5
19
|
value,
|
|
@@ -10,16 +24,36 @@ export const RenderClaimView = (props) => {
|
|
|
10
24
|
sourceStartIndex = 0,
|
|
11
25
|
} = props;
|
|
12
26
|
|
|
13
|
-
const sortedSegments = [...segments]
|
|
14
|
-
(
|
|
15
|
-
|
|
27
|
+
const sortedSegments = [...segments]
|
|
28
|
+
.filter(
|
|
29
|
+
(segment) =>
|
|
30
|
+
Number.isFinite(segment.startOffset) &&
|
|
31
|
+
Number.isFinite(segment.endOffset),
|
|
32
|
+
)
|
|
33
|
+
.sort((a, b) => a.startOffset - b.startOffset);
|
|
16
34
|
|
|
17
35
|
const parts = [];
|
|
18
36
|
let lastIndex = 0;
|
|
37
|
+
let lastSegmentPart = null;
|
|
19
38
|
|
|
20
39
|
sortedSegments.forEach((segment) => {
|
|
21
|
-
const segmentStart =
|
|
22
|
-
|
|
40
|
+
const segmentStart = Math.max(
|
|
41
|
+
0,
|
|
42
|
+
segment.startOffset - sourceStartIndex,
|
|
43
|
+
lastIndex,
|
|
44
|
+
);
|
|
45
|
+
const segmentEnd = Math.min(
|
|
46
|
+
value.length,
|
|
47
|
+
segment.endOffset - sourceStartIndex,
|
|
48
|
+
);
|
|
49
|
+
|
|
50
|
+
if (segmentEnd <= segmentStart) {
|
|
51
|
+
// Nothing left of this segment — it is covered by the previous one.
|
|
52
|
+
if (lastSegmentPart) {
|
|
53
|
+
lastSegmentPart.ids.push(segment.id);
|
|
54
|
+
}
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
23
57
|
|
|
24
58
|
// Add the text part before the current segment
|
|
25
59
|
if (segmentStart > lastIndex) {
|
|
@@ -30,12 +64,13 @@ export const RenderClaimView = (props) => {
|
|
|
30
64
|
}
|
|
31
65
|
|
|
32
66
|
// Add the segment part
|
|
33
|
-
|
|
67
|
+
const part = {
|
|
34
68
|
type: 'segment',
|
|
35
|
-
|
|
69
|
+
ids: [segment.id],
|
|
36
70
|
content: value.slice(segmentStart, segmentEnd),
|
|
37
|
-
}
|
|
38
|
-
|
|
71
|
+
};
|
|
72
|
+
parts.push(part);
|
|
73
|
+
lastSegmentPart = part;
|
|
39
74
|
lastIndex = segmentEnd;
|
|
40
75
|
});
|
|
41
76
|
|
|
@@ -55,12 +90,16 @@ export const RenderClaimView = (props) => {
|
|
|
55
90
|
const content = part.content.split('\n');
|
|
56
91
|
|
|
57
92
|
if (part.type === 'segment') {
|
|
58
|
-
const isSelectedSegment = part.
|
|
93
|
+
const isSelectedSegment = part.ids.includes(visibleSegmentId);
|
|
59
94
|
return (
|
|
60
|
-
<React.Fragment key={part.
|
|
95
|
+
<React.Fragment key={part.ids.join('-') || index}>
|
|
61
96
|
<span
|
|
62
97
|
ref={(el) => {
|
|
63
|
-
if (el)
|
|
98
|
+
if (el) {
|
|
99
|
+
part.ids.forEach((id) => {
|
|
100
|
+
spanRefs.current[id] = el;
|
|
101
|
+
});
|
|
102
|
+
}
|
|
64
103
|
}}
|
|
65
104
|
className={`citation-segment ${
|
|
66
105
|
isSelectedSegment ? 'active' : ''
|
|
@@ -68,7 +107,11 @@ export const RenderClaimView = (props) => {
|
|
|
68
107
|
>
|
|
69
108
|
<mark className="citation-highlight">
|
|
70
109
|
{part.content.trim()}
|
|
71
|
-
|
|
110
|
+
{part.ids.map((id) => (
|
|
111
|
+
<sup key={id} className="citation-ref">
|
|
112
|
+
{id}
|
|
113
|
+
</sup>
|
|
114
|
+
))}
|
|
72
115
|
</mark>
|
|
73
116
|
{!endline && <> </>}
|
|
74
117
|
</span>
|
|
@@ -1,29 +1,32 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
return
|
|
1
|
+
/**
|
|
2
|
+
* Colour for a verdict: one per claim (`ClaimModal`) and one for the
|
|
3
|
+
* answer-level summary (`HalloumiFeedback`).
|
|
4
|
+
*
|
|
5
|
+
* A per-claim score is a verdict (supported 1.0, not_enough_info 0.4,
|
|
6
|
+
* contradicted 0.0 - see `scoreToVerdict`), not a measurement on a 0-1 scale, so
|
|
7
|
+
* the eight-step red-to-green gradient that used to live in this file implied a
|
|
8
|
+
* confidence level that was never computed, and it put `not_enough_info` in the
|
|
9
|
+
* red family, which reads as "the sources contradict this" when they actually
|
|
10
|
+
* said nothing.
|
|
11
|
+
*
|
|
12
|
+
* Three verdicts, three hues: red for a conflict, yellow for silence, green for
|
|
13
|
+
* support. Hue alone is not enough - for red-green colour blindness the two ends
|
|
14
|
+
* of that ramp are the same colour - so each hue also gets its own underline
|
|
15
|
+
* style (see `.claim` in `ChatBlock/style.less`) and the modal names the verdict
|
|
16
|
+
* in words.
|
|
17
|
+
*
|
|
18
|
+
* A claim whose sources were only partly loaded (`contextLimited`) is yellow
|
|
19
|
+
* whatever score it got: the header already refuses to name a verdict for it
|
|
20
|
+
* ("Couldn't be checked"), so colouring it green or red would claim more than the
|
|
21
|
+
* check established.
|
|
22
|
+
*
|
|
23
|
+
* The class names must exist in `colors.less`: the `claim-*` utilities come from
|
|
24
|
+
* an explicit list of `.generate-colors()` calls, and an unlisted class renders
|
|
25
|
+
* unstyled, silently. `colors.test.js` asserts the two stay in step.
|
|
26
|
+
*/
|
|
27
|
+
export function getVerdictBgColor(score, contextLimited = false) {
|
|
28
|
+
if (contextLimited) return 'claim-yellow-500';
|
|
29
|
+
if (score >= 0.8) return 'claim-green-500';
|
|
30
|
+
if (score >= 0.1) return 'claim-yellow-500';
|
|
31
|
+
return 'claim-red-500';
|
|
29
32
|
}
|
|
@@ -16,7 +16,6 @@
|
|
|
16
16
|
@green-400: #4ade80;
|
|
17
17
|
@green-500: #22c55e;
|
|
18
18
|
@green-100: #dcfce7; //
|
|
19
|
-
@green-700: #166534; //
|
|
20
19
|
|
|
21
20
|
@yellow-500: #eab308; //
|
|
22
21
|
@yellow-400: #facc15; //
|
|
@@ -30,7 +29,7 @@
|
|
|
30
29
|
|
|
31
30
|
@lime-500: #84cc16; //
|
|
32
31
|
|
|
33
|
-
.generate-colors(@namespace, @name, @color, @lighter-value: 20, @darker-value: 20) {
|
|
32
|
+
.generate-colors(@namespace, @name, @color, @lighter-value: 20, @darker-value: 20, @stroke-value: 0) {
|
|
34
33
|
@var-name: ~'--@{namespace}-color';
|
|
35
34
|
|
|
36
35
|
.@{namespace}-@{name} {
|
|
@@ -38,15 +37,34 @@
|
|
|
38
37
|
--@{namespace}-color: @color;
|
|
39
38
|
--@{namespace}-lighter-color: lighten(@color, @lighter-value);
|
|
40
39
|
--@{namespace}-darker-color: darken(@color, @darker-value);
|
|
40
|
+
// The colour for thin strokes - the claim underline, the verdict scale. A hue
|
|
41
|
+
// that is a fine fill is too light as a 2px line on white: yellow-500 is
|
|
42
|
+
// 1.92:1 there and green-500 2.28:1. Each call below darkens its stroke just
|
|
43
|
+
// enough to clear 3:1 (WCAG 1.4.11, non-text contrast) without moving the
|
|
44
|
+
// hue: red-500 3.76:1 untouched, green-500 -8 -> 3.32:1, yellow-500 -12 ->
|
|
45
|
+
// 3.37:1.
|
|
46
|
+
--@{namespace}-stroke-color: darken(@color, @stroke-value);
|
|
41
47
|
}
|
|
42
48
|
}
|
|
43
49
|
|
|
50
|
+
// Only the classes `getVerdictBgColor` emits are generated - an unlisted class
|
|
51
|
+
// renders unstyled, silently, with no colour and no error. `colors.test.js`
|
|
52
|
+
// asserts the two lists stay in step.
|
|
53
|
+
//
|
|
54
|
+
// The three verdicts get three hues, not a ladder: red = contradicted, yellow =
|
|
55
|
+
// not_enough_info, green = supported. Two greens were tried and failed - they
|
|
56
|
+
// read as one colour, and for red-green colour blindness red and green read as
|
|
57
|
+
// one colour too, which is why the underline *style* also changes per verdict
|
|
58
|
+
// (see `.claim` in `../style.less`).
|
|
59
|
+
//
|
|
60
|
+
// lighter/darker are set so the chip's text (`--claim-darker-color`) on the
|
|
61
|
+
// chip's background (`--claim-lighter-color`) clears WCAG AA: 5.51:1 red,
|
|
62
|
+
// 4.81:1 yellow, 5.55:1 green.
|
|
44
63
|
.generate-colors(claim, gray-500, @gray-500, 45, 20);
|
|
45
64
|
.generate-colors(claim, red-500, @red-500, 35, 20);
|
|
46
|
-
.
|
|
47
|
-
|
|
48
|
-
.
|
|
49
|
-
|
|
50
|
-
.generate-colors(claim,
|
|
51
|
-
.generate-colors(claim, green-
|
|
52
|
-
.generate-colors(claim, green-200, @green-200, 10, 45);
|
|
65
|
+
// not_enough_info: the sources said nothing. Not red - that means they said
|
|
66
|
+
// something else - and not gray, which is reserved for the no-verdict-at-all
|
|
67
|
+
// state. It sits alongside the yellow citation highlights, which are fills
|
|
68
|
+
// (@yellow-100/200) while a claim is a stroke, so the two do not read alike.
|
|
69
|
+
.generate-colors(claim, yellow-500, @yellow-500, 45, 20, 12);
|
|
70
|
+
.generate-colors(claim, green-500, @green-500, 45, 20, 8);
|
|
@@ -1,69 +1,99 @@
|
|
|
1
|
-
import
|
|
1
|
+
import fs from 'fs';
|
|
2
|
+
import path from 'path';
|
|
3
|
+
import { scoreToVerdict } from '../../utils';
|
|
4
|
+
import { getVerdictBgColor } from './colors';
|
|
2
5
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
});
|
|
6
|
+
const VERDICT_SCORES = [
|
|
7
|
+
['supported', 1.0],
|
|
8
|
+
['not_enough_info', 0.4],
|
|
9
|
+
['contradicted', 0.0],
|
|
10
|
+
];
|
|
9
11
|
|
|
10
|
-
|
|
11
|
-
expect(getSupportedTextColor(0.5)).toBe('text-green-500');
|
|
12
|
-
expect(getSupportedTextColor(0.75)).toBe('text-green-500');
|
|
13
|
-
expect(getSupportedTextColor(1)).toBe('text-green-500');
|
|
14
|
-
});
|
|
12
|
+
const hueOf = (cls) => cls.match(/claim-([a-z]+)-/)[1];
|
|
15
13
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
expect(
|
|
14
|
+
describe('getVerdictBgColor', () => {
|
|
15
|
+
it('gives each verdict one colour band', () => {
|
|
16
|
+
expect(getVerdictBgColor(1.0)).toBe('claim-green-500');
|
|
17
|
+
expect(getVerdictBgColor(0.8)).toBe('claim-green-500');
|
|
18
|
+
expect(getVerdictBgColor(0.4)).toBe('claim-yellow-500');
|
|
19
|
+
expect(getVerdictBgColor(0.1)).toBe('claim-yellow-500');
|
|
20
|
+
expect(getVerdictBgColor(0.0)).toBe('claim-red-500');
|
|
19
21
|
});
|
|
20
|
-
});
|
|
21
|
-
|
|
22
|
-
describe('getSupportedBgColor', () => {
|
|
23
|
-
it('should return correct background colors based on score', () => {
|
|
24
|
-
// Test for scores in the range [0, 0.125)
|
|
25
|
-
expect(getSupportedBgColor(0)).toBe('bg-red-500');
|
|
26
|
-
expect(getSupportedBgColor(0.1)).toBe('bg-red-500');
|
|
27
|
-
expect(getSupportedBgColor(0.124)).toBe('bg-red-500');
|
|
28
|
-
|
|
29
|
-
// Test for scores in the range [0.125, 0.25)
|
|
30
|
-
expect(getSupportedBgColor(0.125)).toBe('bg-red-400');
|
|
31
|
-
expect(getSupportedBgColor(0.2)).toBe('bg-red-400');
|
|
32
|
-
expect(getSupportedBgColor(0.249)).toBe('bg-red-400');
|
|
33
22
|
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
expect(
|
|
23
|
+
it('uses three different hue families, one per verdict', () => {
|
|
24
|
+
// A reader who cannot separate red from green must still tell the verdicts
|
|
25
|
+
// apart, so no two of them may share a hue family. The green ladder this
|
|
26
|
+
// replaced (green-700 / green-300 / red-500) failed that: two of the three
|
|
27
|
+
// verdicts were "green".
|
|
28
|
+
const hues = VERDICT_SCORES.map(([, score]) =>
|
|
29
|
+
hueOf(getVerdictBgColor(score)),
|
|
30
|
+
);
|
|
31
|
+
expect(new Set(hues)).toEqual(new Set(['green', 'yellow', 'red']));
|
|
32
|
+
});
|
|
43
33
|
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
expect(
|
|
47
|
-
|
|
34
|
+
it('keeps not_enough_info out of the red family', () => {
|
|
35
|
+
// red reads as "the sources contradict this"; they said nothing
|
|
36
|
+
expect(hueOf(getVerdictBgColor(0.4))).not.toBe('red');
|
|
37
|
+
});
|
|
48
38
|
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
expect(
|
|
39
|
+
it('marks claims that could not be checked yellow, whatever the score', () => {
|
|
40
|
+
// a claim checked against only part of its sources gets no verdict in the
|
|
41
|
+
// header ("Couldn't be checked"), so the colour must not claim one either
|
|
42
|
+
expect(getVerdictBgColor(1.0, true)).toBe('claim-yellow-500');
|
|
43
|
+
expect(getVerdictBgColor(0.0, true)).toBe('claim-yellow-500');
|
|
44
|
+
});
|
|
53
45
|
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
46
|
+
it('colours verdicts exactly where scoreToVerdict names them', () => {
|
|
47
|
+
// the colour bands and the wording in the modal are computed separately from
|
|
48
|
+
// the same thresholds; if they drift, the chip says one thing and the header
|
|
49
|
+
// another
|
|
50
|
+
const hueForVerdict = {
|
|
51
|
+
'claim-green-500': 'supported',
|
|
52
|
+
'claim-yellow-500': 'not_enough_info',
|
|
53
|
+
'claim-red-500': 'contradicted',
|
|
54
|
+
};
|
|
55
|
+
for (const score of [0, 0.05, 0.09, 0.1, 0.4, 0.79, 0.8, 1]) {
|
|
56
|
+
expect(hueForVerdict[getVerdictBgColor(score)]).toBe(
|
|
57
|
+
scoreToVerdict(score),
|
|
58
|
+
);
|
|
59
|
+
}
|
|
60
|
+
});
|
|
58
61
|
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
62
|
+
it('only emits classes that colors.less actually generates', () => {
|
|
63
|
+
// an unlisted .generate-colors() call means the class renders unstyled, which
|
|
64
|
+
// fails silently - no colour, no error
|
|
65
|
+
const less = fs.readFileSync(path.join(__dirname, 'colors.less'), 'utf-8');
|
|
66
|
+
for (const cls of [
|
|
67
|
+
getVerdictBgColor(1.0),
|
|
68
|
+
getVerdictBgColor(0.4),
|
|
69
|
+
getVerdictBgColor(0.0),
|
|
70
|
+
]) {
|
|
71
|
+
expect(less).toContain(
|
|
72
|
+
`.generate-colors(claim, ${cls.replace('claim-', '')},`,
|
|
73
|
+
);
|
|
74
|
+
}
|
|
63
75
|
});
|
|
64
76
|
|
|
65
|
-
it('
|
|
66
|
-
|
|
67
|
-
|
|
77
|
+
it('gives each verdict class its own underline style in style.less', () => {
|
|
78
|
+
// colour is not the only signal: each hue also has a line style (wavy /
|
|
79
|
+
// dotted / dashed). Renaming a class here without renaming the LESS selector
|
|
80
|
+
// would drop the shape distinction silently, so assert both lists agree.
|
|
81
|
+
const less = fs.readFileSync(
|
|
82
|
+
path.join(__dirname, '..', '..', 'style.less'),
|
|
83
|
+
'utf-8',
|
|
84
|
+
);
|
|
85
|
+
const styles = {};
|
|
86
|
+
for (const cls of [
|
|
87
|
+
getVerdictBgColor(1.0),
|
|
88
|
+
getVerdictBgColor(0.4),
|
|
89
|
+
getVerdictBgColor(0.0),
|
|
90
|
+
]) {
|
|
91
|
+
const rule = less.match(
|
|
92
|
+
new RegExp(`&\\.${cls} \\{\\s*text-decoration-style: ([a-z]+);`),
|
|
93
|
+
);
|
|
94
|
+
expect(rule).not.toBeNull();
|
|
95
|
+
styles[cls] = rule[1];
|
|
96
|
+
}
|
|
97
|
+
expect(new Set(Object.values(styles)).size).toBe(3);
|
|
68
98
|
});
|
|
69
99
|
});
|
|
@@ -5,3 +5,4 @@ export { useMarked } from './useMarked';
|
|
|
5
5
|
export { useQualityMarkers } from './useQualityMarkers';
|
|
6
6
|
export { useDeepCompareMemoize } from './useDeepCompareMemoize';
|
|
7
7
|
export { useScrollonStream } from './useScrollonStream';
|
|
8
|
+
export { useChunkEvidence } from './useChunkEvidence';
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import { useEffect, useMemo, useRef, useState } from 'react';
|
|
2
|
+
|
|
3
|
+
import {
|
|
4
|
+
CHUNK_EVIDENCE_WINDOW,
|
|
5
|
+
fetchEvidence,
|
|
6
|
+
} from '@eeacms/volto-eea-chatbot/ChatBlock/services/chunkEvidence';
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Resolve real Onyx chunk text for the documents about to be sent to the
|
|
10
|
+
* fact-checker.
|
|
11
|
+
*
|
|
12
|
+
* Only runs when `enabled` (i.e. quality control is being requested), and
|
|
13
|
+
* re-runs only when the set of `document_id` + `chunk_ind` values changes.
|
|
14
|
+
*
|
|
15
|
+
* `evidenceSettled` is what callers must gate the fact-check call on. It is
|
|
16
|
+
* false for a key that has not been resolved yet, so the check cannot fire with
|
|
17
|
+
* blurbs in the same commit that started the evidence fetch. It also becomes
|
|
18
|
+
* true when a fetch fails — falling back to blurbs is better than never
|
|
19
|
+
* verifying at all.
|
|
20
|
+
*/
|
|
21
|
+
export function useChunkEvidence(
|
|
22
|
+
docs: any[],
|
|
23
|
+
{
|
|
24
|
+
enabled = true,
|
|
25
|
+
window = CHUNK_EVIDENCE_WINDOW,
|
|
26
|
+
}: { enabled?: boolean; window?: number } = {},
|
|
27
|
+
) {
|
|
28
|
+
const [evidence, setEvidence] = useState<Record<string, any>>({});
|
|
29
|
+
const [isFetchingEvidence, setIsFetchingEvidence] = useState(false);
|
|
30
|
+
const [settledKey, setSettledKey] = useState('');
|
|
31
|
+
|
|
32
|
+
const docsRef = useRef(docs);
|
|
33
|
+
docsRef.current = docs;
|
|
34
|
+
|
|
35
|
+
const key = useMemo(
|
|
36
|
+
() =>
|
|
37
|
+
(docs || [])
|
|
38
|
+
.map((doc: any) => `${doc?.document_id}@@${doc?.chunk_ind ?? 0}`)
|
|
39
|
+
.filter((part: string) => !part.startsWith('undefined'))
|
|
40
|
+
.join('|'),
|
|
41
|
+
[docs],
|
|
42
|
+
);
|
|
43
|
+
|
|
44
|
+
useEffect(() => {
|
|
45
|
+
if (!enabled || !key) {
|
|
46
|
+
setIsFetchingEvidence(false);
|
|
47
|
+
return;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
let cancelled = false;
|
|
51
|
+
setIsFetchingEvidence(true);
|
|
52
|
+
|
|
53
|
+
fetchEvidence(docsRef.current, { window })
|
|
54
|
+
.then((result) => {
|
|
55
|
+
if (cancelled) {
|
|
56
|
+
return;
|
|
57
|
+
}
|
|
58
|
+
setEvidence(result);
|
|
59
|
+
setIsFetchingEvidence(false);
|
|
60
|
+
setSettledKey(key);
|
|
61
|
+
})
|
|
62
|
+
.catch(() => {
|
|
63
|
+
// Fall back to whatever the stream gave us (blurbs).
|
|
64
|
+
if (!cancelled) {
|
|
65
|
+
setIsFetchingEvidence(false);
|
|
66
|
+
setSettledKey(key);
|
|
67
|
+
}
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
return () => {
|
|
71
|
+
cancelled = true;
|
|
72
|
+
};
|
|
73
|
+
}, [key, enabled, window]);
|
|
74
|
+
|
|
75
|
+
return {
|
|
76
|
+
evidence,
|
|
77
|
+
isFetchingEvidence,
|
|
78
|
+
evidenceSettled: key === '' || settledKey === key,
|
|
79
|
+
};
|
|
80
|
+
}
|
package/src/ChatBlock/schema.jsx
CHANGED
|
@@ -95,6 +95,7 @@ export function ChatBlockSchema({ assistants, data }) {
|
|
|
95
95
|
'batchSize',
|
|
96
96
|
'noSupportDocumentsMessage',
|
|
97
97
|
'qualityCheckContext',
|
|
98
|
+
'chunkEvidenceWindow',
|
|
98
99
|
'qualityCheckStages',
|
|
99
100
|
]
|
|
100
101
|
: []),
|
|
@@ -214,6 +215,17 @@ export function ChatBlockSchema({ assistants, data }) {
|
|
|
214
215
|
['all', 'All documents passed to LLM'],
|
|
215
216
|
],
|
|
216
217
|
},
|
|
218
|
+
chunkEvidenceWindow: {
|
|
219
|
+
title: 'Evidence window (chunks around each source)',
|
|
220
|
+
type: 'number',
|
|
221
|
+
default: 2,
|
|
222
|
+
description:
|
|
223
|
+
'How many Onyx chunks to fetch on each side of the matched chunk before ' +
|
|
224
|
+
'fact-checking. 0 = centre chunk only (usually not enough), 2 = what Onyx ' +
|
|
225
|
+
'feeds the answer generator by default, 5 = Onyx at its widest. Each extra ' +
|
|
226
|
+
'step costs two more chunk requests per document, and values above 20 are ' +
|
|
227
|
+
'clamped.',
|
|
228
|
+
},
|
|
217
229
|
qualityCheckStages: {
|
|
218
230
|
title: 'Score ranges',
|
|
219
231
|
widget: 'object_list',
|