@konneal/engine 0.2.48 → 0.2.49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ask-OH5KPHZA.js +12 -0
- package/dist/{chunk-SONICKNY.js → chunk-4KA34NMU.js} +140 -5
- package/dist/{chunk-47VBMM7I.js → chunk-DMIOBGQU.js} +4 -4
- package/dist/{chunk-R2V3X6SQ.js → chunk-JWTIPQ4L.js} +1 -1
- package/dist/{chunk-4GJGBGJK.js → chunk-O6VZIDPW.js} +1 -1
- package/dist/{chunk-IUMHKBOY.js → chunk-VVLNUHW2.js} +2 -2
- package/dist/{chunk-V46XM2GU.js → chunk-Z24IAKB6.js} +9 -0
- package/dist/config.d.ts +9 -0
- package/dist/entailment.d.ts +5 -0
- package/dist/search-KNB2S2PS.js +12 -0
- package/dist/worker_mcp/src/index.js +2 -2
- package/dist/worker_public/src/config.js +1 -1
- package/dist/worker_public/src/index.js +8 -119
- package/dist/worker_public/src/requestScope.js +2 -2
- package/package.json +1 -1
- package/workers/worker_public/src/ask.ts +23 -2
- package/workers/worker_public/src/config.ts +9 -0
- package/workers/worker_public/src/entailment.ts +15 -0
- package/dist/ask-KL7XUELS.js +0 -12
- package/dist/search-NTQVSSAE.js +0 -12
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import {
|
|
2
|
+
handleAsk
|
|
3
|
+
} from "./chunk-4KA34NMU.js";
|
|
4
|
+
import "./chunk-VVLNUHW2.js";
|
|
5
|
+
import "./chunk-JWTIPQ4L.js";
|
|
6
|
+
import "./chunk-A3QHHUN5.js";
|
|
7
|
+
import "./chunk-O6VZIDPW.js";
|
|
8
|
+
import "./chunk-Z24IAKB6.js";
|
|
9
|
+
import "./chunk-3FYJM7LH.js";
|
|
10
|
+
export {
|
|
11
|
+
handleAsk
|
|
12
|
+
};
|
|
@@ -35,14 +35,14 @@ import {
|
|
|
35
35
|
syntheticUnderstanding,
|
|
36
36
|
telemetry,
|
|
37
37
|
understandQuery
|
|
38
|
-
} from "./chunk-
|
|
38
|
+
} from "./chunk-VVLNUHW2.js";
|
|
39
39
|
import {
|
|
40
40
|
corsHeaders,
|
|
41
41
|
err,
|
|
42
42
|
json,
|
|
43
43
|
readJson,
|
|
44
44
|
validateQuery
|
|
45
|
-
} from "./chunk-
|
|
45
|
+
} from "./chunk-JWTIPQ4L.js";
|
|
46
46
|
import {
|
|
47
47
|
canonicalRefusal,
|
|
48
48
|
refusalAnswer
|
|
@@ -52,17 +52,18 @@ import {
|
|
|
52
52
|
requestSalt,
|
|
53
53
|
resolveRequestScope,
|
|
54
54
|
standardKeysFrom
|
|
55
|
-
} from "./chunk-
|
|
55
|
+
} from "./chunk-O6VZIDPW.js";
|
|
56
56
|
import {
|
|
57
57
|
LIMITS,
|
|
58
58
|
MODELS,
|
|
59
|
+
THRESHOLDS,
|
|
59
60
|
answerEffort,
|
|
60
61
|
effortBudget,
|
|
61
62
|
num,
|
|
62
63
|
requestEffort,
|
|
63
64
|
roleModel,
|
|
64
65
|
sha256Hex
|
|
65
|
-
} from "./chunk-
|
|
66
|
+
} from "./chunk-Z24IAKB6.js";
|
|
66
67
|
import {
|
|
67
68
|
P
|
|
68
69
|
} from "./chunk-3FYJM7LH.js";
|
|
@@ -216,6 +217,125 @@ ${ctx}`
|
|
|
216
217
|
}
|
|
217
218
|
}
|
|
218
219
|
|
|
220
|
+
// workers/worker_public/prompts/faithfulness.md
|
|
221
|
+
var faithfulness_default = `You are a factuality judge. Given an answer and the retrieved passages it was based on, identify any claims in the answer that are NOT directly supported by the passages. Reply with ONLY a JSON object: {"score": 0.0-1.0, "ungrounded_claims": ["claim text", ...]} \u2014 score is the fraction of claims that ARE grounded in the passages; if every claim is supported, score is 1.0 and ungrounded_claims is [].
|
|
222
|
+
|
|
223
|
+
Passages prefixed [M] are this service's own machine-computed model data (typed blocks the answer was given) \u2014 a claim restating [M] content is grounded.
|
|
224
|
+
`;
|
|
225
|
+
|
|
226
|
+
// workers/worker_public/src/verdict-parse.ts
|
|
227
|
+
function coerceVerdict(obj) {
|
|
228
|
+
if (obj === null || typeof obj !== "object" || Array.isArray(obj)) return null;
|
|
229
|
+
const raw = obj.score;
|
|
230
|
+
const score = typeof raw === "number" ? raw : typeof raw === "string" && raw.trim() !== "" ? Number(raw) : NaN;
|
|
231
|
+
if (!Number.isFinite(score)) return null;
|
|
232
|
+
const claims = obj.ungrounded_claims;
|
|
233
|
+
return {
|
|
234
|
+
score: Math.max(0, Math.min(1, score)),
|
|
235
|
+
ungrounded_claims: Array.isArray(claims) ? claims.map(String).slice(0, 5) : []
|
|
236
|
+
};
|
|
237
|
+
}
|
|
238
|
+
function parseVerdict(text) {
|
|
239
|
+
const stripped = text.replace(/```[a-zA-Z]*\n?/g, "").replace(/```/g, "").trim();
|
|
240
|
+
try {
|
|
241
|
+
const whole = coerceVerdict(JSON.parse(stripped));
|
|
242
|
+
if (whole) return whole;
|
|
243
|
+
} catch {
|
|
244
|
+
}
|
|
245
|
+
let verdict = null;
|
|
246
|
+
let depth = 0;
|
|
247
|
+
let start = -1;
|
|
248
|
+
let inString = false;
|
|
249
|
+
let escaped = false;
|
|
250
|
+
for (let i = 0; i < stripped.length; i++) {
|
|
251
|
+
const ch = stripped[i];
|
|
252
|
+
if (escaped) {
|
|
253
|
+
escaped = false;
|
|
254
|
+
continue;
|
|
255
|
+
}
|
|
256
|
+
if (inString && ch === "\\") {
|
|
257
|
+
escaped = true;
|
|
258
|
+
continue;
|
|
259
|
+
}
|
|
260
|
+
if (ch === '"') {
|
|
261
|
+
inString = !inString;
|
|
262
|
+
continue;
|
|
263
|
+
}
|
|
264
|
+
if (inString) continue;
|
|
265
|
+
if (ch === "{") {
|
|
266
|
+
if (depth === 0) start = i;
|
|
267
|
+
depth++;
|
|
268
|
+
} else if (ch === "}" && depth > 0) {
|
|
269
|
+
depth--;
|
|
270
|
+
if (depth === 0 && start >= 0) {
|
|
271
|
+
try {
|
|
272
|
+
const v = coerceVerdict(JSON.parse(stripped.slice(start, i + 1)));
|
|
273
|
+
if (v) verdict = v;
|
|
274
|
+
} catch {
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
return verdict;
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
// workers/worker_public/src/faithfulness-context.ts
|
|
283
|
+
function buildJudgeContext(passages, machine = []) {
|
|
284
|
+
const context = passages.slice(0, 8).map((p, i) => {
|
|
285
|
+
const text = typeof p === "string" ? p : p.text;
|
|
286
|
+
const limit = typeof p !== "string" && p.table ? 2400 : 1400;
|
|
287
|
+
return `[${i + 1}] ${text.replace(/\s+/g, " ").slice(0, limit)}`;
|
|
288
|
+
}).join("\n");
|
|
289
|
+
const machineContext = machine.length ? "\n" + machine.slice(0, 6).map((m) => `[M] ${m.replace(/\s+/g, " ").slice(0, 400)}`).join("\n") : "";
|
|
290
|
+
return context + machineContext;
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
// workers/worker_public/src/faithfulness.ts
|
|
294
|
+
async function scoreFaithfulness(ai, model, answer, passages, machine = []) {
|
|
295
|
+
if (!answer || !passages.length) return null;
|
|
296
|
+
const context = buildJudgeContext(passages, machine);
|
|
297
|
+
const t0 = Date.now();
|
|
298
|
+
const timeout = new Promise((r) => setTimeout(() => {
|
|
299
|
+
console.log(`faithfulness: timeout (${Date.now() - t0}ms)`);
|
|
300
|
+
r(null);
|
|
301
|
+
}, 24e4));
|
|
302
|
+
const call = (async () => {
|
|
303
|
+
const res = await ai.run(model, {
|
|
304
|
+
messages: [
|
|
305
|
+
{
|
|
306
|
+
role: "system",
|
|
307
|
+
content: faithfulness_default.trimEnd()
|
|
308
|
+
},
|
|
309
|
+
{ role: "user", content: `Answer:
|
|
310
|
+
${answer.slice(0, 2e3)}
|
|
311
|
+
|
|
312
|
+
Passages:
|
|
313
|
+
${context}${machine.length ? buildJudgeContext([], machine) : ""}` }
|
|
314
|
+
],
|
|
315
|
+
max_tokens: 6144,
|
|
316
|
+
reasoning_effort: "low",
|
|
317
|
+
// DeepSeek-V4 card: temp 1.0 / top_p 1.0
|
|
318
|
+
temperature: 1,
|
|
319
|
+
top_p: 1
|
|
320
|
+
});
|
|
321
|
+
const text = typeof res?.response === "string" ? res.response : res?.choices?.[0]?.message?.content;
|
|
322
|
+
const verdict = parseVerdict(text ?? "");
|
|
323
|
+
if (!verdict) {
|
|
324
|
+
console.log(`faithfulness: no parse (${Date.now() - t0}ms, text ${(text ?? "").length} chars) raw=${JSON.stringify((text ?? "").replace(/\s+/g, " ").slice(0, 500))}`);
|
|
325
|
+
return null;
|
|
326
|
+
}
|
|
327
|
+
return verdict;
|
|
328
|
+
})();
|
|
329
|
+
return await Promise.race([call, timeout]);
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
// workers/worker_public/src/entailment.ts
|
|
333
|
+
function entailmentVerdict(score, supportedFloor, partialFloor) {
|
|
334
|
+
if (score >= supportedFloor) return { support: "supported", note: "" };
|
|
335
|
+
if (score >= partialFloor) return { support: "partial", note: "Partially grounded \u2014 some claims lack support in the cited passages; verify against the cited clauses." };
|
|
336
|
+
return { support: "unsupported", note: "WARNING: This answer's claims are largely unsupported by the retrieved passages. Verify against official publications." };
|
|
337
|
+
}
|
|
338
|
+
|
|
219
339
|
// workers/worker_public/src/anchors.ts
|
|
220
340
|
var normalize = (s) => s.replace(/[‘’‛′]/g, "'").replace(/[“”‟″]/g, '"').replace(/[–—−]/g, "-").replace(//g, "").replace(/[ ]/g, " ").replace(/\s+/g, " ").toLowerCase();
|
|
221
341
|
var ANCHOR = /\[[^\[\]\n]*\]/g;
|
|
@@ -2688,7 +2808,21 @@ Answer account questions from these records ONLY: name the record when you use i
|
|
|
2688
2808
|
}
|
|
2689
2809
|
completionBlocks.push(...await completeFigures(env.DB, answer, [...c2ns.blocks, ...completionBlocks], used));
|
|
2690
2810
|
const jsonQuality = answerQuality(finalCites.map((c) => c.quality));
|
|
2691
|
-
|
|
2811
|
+
let entailment = null;
|
|
2812
|
+
let entailmentNote = "";
|
|
2813
|
+
if (THRESHOLDS.entailmentGate && jsonQuality && used.length && !answer.includes(refusalAnswer())) {
|
|
2814
|
+
const raced = await Promise.race([
|
|
2815
|
+
scoreFaithfulness(env.AI, roleModel(env, "grader"), answer, used.map((h) => h.text)),
|
|
2816
|
+
new Promise((r) => setTimeout(() => r(null), THRESHOLDS.entailmentBudgetMs))
|
|
2817
|
+
]);
|
|
2818
|
+
if (raced) {
|
|
2819
|
+
const v = entailmentVerdict(raced.score, THRESHOLDS.entailmentSupportedFloor, THRESHOLDS.entailmentPartialFloor);
|
|
2820
|
+
entailment = { support: v.support, score: Math.round(raced.score * 100) / 100, ...raced.ungrounded_claims?.length ? { ungrounded: raced.ungrounded_claims.slice(0, 3) } : {} };
|
|
2821
|
+
entailmentNote = v.note;
|
|
2822
|
+
if (v.support !== "supported") console.log("entailment:", v.support, raced.score, raced.ungrounded_claims?.slice(0, 2));
|
|
2823
|
+
}
|
|
2824
|
+
}
|
|
2825
|
+
const out = { answer, citations: finalCites, ...jsonQuality ? { source_quality: jsonQuality, confidence_note: entailmentNote || qualityNote(jsonQuality), ...entailment ? { entailment } : {}, ...jsonQuality === "ocr" ? { experimental_sources: experimentalSourceLabels(finalCites) } : {} } : {}, model: MODELS.member, query_hash: queryHash, follow_ups: understanding?.follow_ups ?? [], blocks: [...c2ns.blocks, ...verdictBlock ? [verdictBlock] : [], ...conditionBlock ? [conditionBlock] : [], ...aggregationBlock ? [aggregationBlock] : [], ...completionBlocks], context_applied: ctxApplied, ...liveRecords ? { records: liveRecords } : {} };
|
|
2692
2826
|
const cacheable = !contextual && !declaredCtx && !answer.includes(refusalAnswer()) && finalAnchors.violations.length === 0;
|
|
2693
2827
|
if (cacheable) {
|
|
2694
2828
|
const ck = exactCacheKey(env.INDEX_VERSION, gen, ns, await sha256Hex(cacheKeyMaterial(q.query, q.lang, salt)));
|
|
@@ -2720,6 +2854,7 @@ function sseResponse(events, cors) {
|
|
|
2720
2854
|
|
|
2721
2855
|
export {
|
|
2722
2856
|
handleMemories,
|
|
2857
|
+
scoreFaithfulness,
|
|
2723
2858
|
checkQuoteAnchors,
|
|
2724
2859
|
standardForDocNumber,
|
|
2725
2860
|
bindModelNode,
|
|
@@ -7,24 +7,24 @@ import {
|
|
|
7
7
|
sessionFrom,
|
|
8
8
|
telemetry,
|
|
9
9
|
understandQuery
|
|
10
|
-
} from "./chunk-
|
|
10
|
+
} from "./chunk-VVLNUHW2.js";
|
|
11
11
|
import {
|
|
12
12
|
corsHeaders,
|
|
13
13
|
err,
|
|
14
14
|
json,
|
|
15
15
|
readJson,
|
|
16
16
|
validateQuery
|
|
17
|
-
} from "./chunk-
|
|
17
|
+
} from "./chunk-JWTIPQ4L.js";
|
|
18
18
|
import {
|
|
19
19
|
entitlementScope,
|
|
20
20
|
standardKeysFrom
|
|
21
|
-
} from "./chunk-
|
|
21
|
+
} from "./chunk-O6VZIDPW.js";
|
|
22
22
|
import {
|
|
23
23
|
LIMITS,
|
|
24
24
|
MODELS,
|
|
25
25
|
num,
|
|
26
26
|
sha256Hex
|
|
27
|
-
} from "./chunk-
|
|
27
|
+
} from "./chunk-Z24IAKB6.js";
|
|
28
28
|
|
|
29
29
|
// workers/worker_public/src/search.ts
|
|
30
30
|
async function handleSearch(env, ctx, req, tier, key) {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import {
|
|
2
2
|
bubbleConfirmPage,
|
|
3
3
|
isAllowedBubbleOrigin
|
|
4
|
-
} from "./chunk-
|
|
4
|
+
} from "./chunk-JWTIPQ4L.js";
|
|
5
5
|
import {
|
|
6
6
|
DATASETS,
|
|
7
7
|
LIMITS,
|
|
@@ -12,7 +12,7 @@ import {
|
|
|
12
12
|
processExpansion,
|
|
13
13
|
sha256Hex,
|
|
14
14
|
today
|
|
15
|
-
} from "./chunk-
|
|
15
|
+
} from "./chunk-Z24IAKB6.js";
|
|
16
16
|
import {
|
|
17
17
|
P
|
|
18
18
|
} from "./chunk-3FYJM7LH.js";
|
|
@@ -60,6 +60,15 @@ var THRESHOLDS = {
|
|
|
60
60
|
* pool below the primary dense lane; they must earn their window
|
|
61
61
|
* slot under the cross-encoder, not by graph membership alone. */
|
|
62
62
|
graphLaneDiscount: 0.75,
|
|
63
|
+
/** The entailment gate (TODO.new-era/10): the faithfulness scorer
|
|
64
|
+
* promoted from eval-only onto the serving path. The confidence line
|
|
65
|
+
* then states a measurement — claim support in the cited passages —
|
|
66
|
+
* not only the source rung. Fails open (a slow or unparseable check
|
|
67
|
+
* leaves the source-rung note unchanged). */
|
|
68
|
+
entailmentGate: true,
|
|
69
|
+
entailmentBudgetMs: 6e3,
|
|
70
|
+
entailmentSupportedFloor: 0.9,
|
|
71
|
+
entailmentPartialFloor: 0.5,
|
|
63
72
|
/** Concept-graph candidate discount — same rationale as the graph
|
|
64
73
|
* lane: definitional content enters discounted. */
|
|
65
74
|
conceptGraphDiscount: 0.75,
|
package/dist/config.d.ts
CHANGED
|
@@ -32,6 +32,15 @@ export declare const THRESHOLDS: {
|
|
|
32
32
|
* pool below the primary dense lane; they must earn their window
|
|
33
33
|
* slot under the cross-encoder, not by graph membership alone. */
|
|
34
34
|
readonly graphLaneDiscount: 0.75;
|
|
35
|
+
/** The entailment gate (TODO.new-era/10): the faithfulness scorer
|
|
36
|
+
* promoted from eval-only onto the serving path. The confidence line
|
|
37
|
+
* then states a measurement — claim support in the cited passages —
|
|
38
|
+
* not only the source rung. Fails open (a slow or unparseable check
|
|
39
|
+
* leaves the source-rung note unchanged). */
|
|
40
|
+
readonly entailmentGate: true;
|
|
41
|
+
readonly entailmentBudgetMs: 6000;
|
|
42
|
+
readonly entailmentSupportedFloor: 0.9;
|
|
43
|
+
readonly entailmentPartialFloor: 0.5;
|
|
35
44
|
/** Concept-graph candidate discount — same rationale as the graph
|
|
36
45
|
* lane: definitional content enters discounted. */
|
|
37
46
|
readonly conceptGraphDiscount: 0.75;
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import {
|
|
2
|
+
handleSearch
|
|
3
|
+
} from "./chunk-DMIOBGQU.js";
|
|
4
|
+
import "./chunk-VVLNUHW2.js";
|
|
5
|
+
import "./chunk-JWTIPQ4L.js";
|
|
6
|
+
import "./chunk-A3QHHUN5.js";
|
|
7
|
+
import "./chunk-O6VZIDPW.js";
|
|
8
|
+
import "./chunk-Z24IAKB6.js";
|
|
9
|
+
import "./chunk-3FYJM7LH.js";
|
|
10
|
+
export {
|
|
11
|
+
handleSearch
|
|
12
|
+
};
|
|
@@ -1,15 +1,16 @@
|
|
|
1
1
|
import {
|
|
2
2
|
handleSearch
|
|
3
|
-
} from "../../chunk-
|
|
3
|
+
} from "../../chunk-DMIOBGQU.js";
|
|
4
4
|
import {
|
|
5
5
|
bindModelNode,
|
|
6
6
|
checkQuoteAnchors,
|
|
7
7
|
handleAsk,
|
|
8
8
|
handleMemories,
|
|
9
9
|
modelGroundingBlock,
|
|
10
|
+
scoreFaithfulness,
|
|
10
11
|
scoreJudge,
|
|
11
12
|
standardForDocNumber
|
|
12
|
-
} from "../../chunk-
|
|
13
|
+
} from "../../chunk-4KA34NMU.js";
|
|
13
14
|
import {
|
|
14
15
|
buildMessages,
|
|
15
16
|
cfBlobs,
|
|
@@ -36,7 +37,7 @@ import {
|
|
|
36
37
|
sessionFrom,
|
|
37
38
|
telemetry,
|
|
38
39
|
understandQuery
|
|
39
|
-
} from "../../chunk-
|
|
40
|
+
} from "../../chunk-VVLNUHW2.js";
|
|
40
41
|
import {
|
|
41
42
|
authenticate,
|
|
42
43
|
corsHeaders,
|
|
@@ -45,14 +46,14 @@ import {
|
|
|
45
46
|
readJson,
|
|
46
47
|
validateQuery,
|
|
47
48
|
withCors
|
|
48
|
-
} from "../../chunk-
|
|
49
|
+
} from "../../chunk-JWTIPQ4L.js";
|
|
49
50
|
import {
|
|
50
51
|
canonicalRefusal
|
|
51
52
|
} from "../../chunk-A3QHHUN5.js";
|
|
52
53
|
import {
|
|
53
54
|
entitlementScope,
|
|
54
55
|
standardKeysFrom
|
|
55
|
-
} from "../../chunk-
|
|
56
|
+
} from "../../chunk-O6VZIDPW.js";
|
|
56
57
|
import {
|
|
57
58
|
LIMITS,
|
|
58
59
|
MODELS,
|
|
@@ -63,7 +64,7 @@ import {
|
|
|
63
64
|
roleModel,
|
|
64
65
|
sha256Hex,
|
|
65
66
|
today
|
|
66
|
-
} from "../../chunk-
|
|
67
|
+
} from "../../chunk-Z24IAKB6.js";
|
|
67
68
|
import {
|
|
68
69
|
P,
|
|
69
70
|
setProfile
|
|
@@ -333,118 +334,6 @@ async function handleGetShared(env, slug) {
|
|
|
333
334
|
});
|
|
334
335
|
}
|
|
335
336
|
|
|
336
|
-
// workers/worker_public/prompts/faithfulness.md
|
|
337
|
-
var faithfulness_default = `You are a factuality judge. Given an answer and the retrieved passages it was based on, identify any claims in the answer that are NOT directly supported by the passages. Reply with ONLY a JSON object: {"score": 0.0-1.0, "ungrounded_claims": ["claim text", ...]} \u2014 score is the fraction of claims that ARE grounded in the passages; if every claim is supported, score is 1.0 and ungrounded_claims is [].
|
|
338
|
-
|
|
339
|
-
Passages prefixed [M] are this service's own machine-computed model data (typed blocks the answer was given) \u2014 a claim restating [M] content is grounded.
|
|
340
|
-
`;
|
|
341
|
-
|
|
342
|
-
// workers/worker_public/src/verdict-parse.ts
|
|
343
|
-
function coerceVerdict(obj) {
|
|
344
|
-
if (obj === null || typeof obj !== "object" || Array.isArray(obj)) return null;
|
|
345
|
-
const raw = obj.score;
|
|
346
|
-
const score = typeof raw === "number" ? raw : typeof raw === "string" && raw.trim() !== "" ? Number(raw) : NaN;
|
|
347
|
-
if (!Number.isFinite(score)) return null;
|
|
348
|
-
const claims = obj.ungrounded_claims;
|
|
349
|
-
return {
|
|
350
|
-
score: Math.max(0, Math.min(1, score)),
|
|
351
|
-
ungrounded_claims: Array.isArray(claims) ? claims.map(String).slice(0, 5) : []
|
|
352
|
-
};
|
|
353
|
-
}
|
|
354
|
-
function parseVerdict(text) {
|
|
355
|
-
const stripped = text.replace(/```[a-zA-Z]*\n?/g, "").replace(/```/g, "").trim();
|
|
356
|
-
try {
|
|
357
|
-
const whole = coerceVerdict(JSON.parse(stripped));
|
|
358
|
-
if (whole) return whole;
|
|
359
|
-
} catch {
|
|
360
|
-
}
|
|
361
|
-
let verdict = null;
|
|
362
|
-
let depth = 0;
|
|
363
|
-
let start = -1;
|
|
364
|
-
let inString = false;
|
|
365
|
-
let escaped = false;
|
|
366
|
-
for (let i = 0; i < stripped.length; i++) {
|
|
367
|
-
const ch = stripped[i];
|
|
368
|
-
if (escaped) {
|
|
369
|
-
escaped = false;
|
|
370
|
-
continue;
|
|
371
|
-
}
|
|
372
|
-
if (inString && ch === "\\") {
|
|
373
|
-
escaped = true;
|
|
374
|
-
continue;
|
|
375
|
-
}
|
|
376
|
-
if (ch === '"') {
|
|
377
|
-
inString = !inString;
|
|
378
|
-
continue;
|
|
379
|
-
}
|
|
380
|
-
if (inString) continue;
|
|
381
|
-
if (ch === "{") {
|
|
382
|
-
if (depth === 0) start = i;
|
|
383
|
-
depth++;
|
|
384
|
-
} else if (ch === "}" && depth > 0) {
|
|
385
|
-
depth--;
|
|
386
|
-
if (depth === 0 && start >= 0) {
|
|
387
|
-
try {
|
|
388
|
-
const v = coerceVerdict(JSON.parse(stripped.slice(start, i + 1)));
|
|
389
|
-
if (v) verdict = v;
|
|
390
|
-
} catch {
|
|
391
|
-
}
|
|
392
|
-
}
|
|
393
|
-
}
|
|
394
|
-
}
|
|
395
|
-
return verdict;
|
|
396
|
-
}
|
|
397
|
-
|
|
398
|
-
// workers/worker_public/src/faithfulness-context.ts
|
|
399
|
-
function buildJudgeContext(passages, machine = []) {
|
|
400
|
-
const context = passages.slice(0, 8).map((p, i) => {
|
|
401
|
-
const text = typeof p === "string" ? p : p.text;
|
|
402
|
-
const limit = typeof p !== "string" && p.table ? 2400 : 1400;
|
|
403
|
-
return `[${i + 1}] ${text.replace(/\s+/g, " ").slice(0, limit)}`;
|
|
404
|
-
}).join("\n");
|
|
405
|
-
const machineContext = machine.length ? "\n" + machine.slice(0, 6).map((m) => `[M] ${m.replace(/\s+/g, " ").slice(0, 400)}`).join("\n") : "";
|
|
406
|
-
return context + machineContext;
|
|
407
|
-
}
|
|
408
|
-
|
|
409
|
-
// workers/worker_public/src/faithfulness.ts
|
|
410
|
-
async function scoreFaithfulness(ai, model, answer, passages, machine = []) {
|
|
411
|
-
if (!answer || !passages.length) return null;
|
|
412
|
-
const context = buildJudgeContext(passages, machine);
|
|
413
|
-
const t0 = Date.now();
|
|
414
|
-
const timeout = new Promise((r) => setTimeout(() => {
|
|
415
|
-
console.log(`faithfulness: timeout (${Date.now() - t0}ms)`);
|
|
416
|
-
r(null);
|
|
417
|
-
}, 24e4));
|
|
418
|
-
const call = (async () => {
|
|
419
|
-
const res = await ai.run(model, {
|
|
420
|
-
messages: [
|
|
421
|
-
{
|
|
422
|
-
role: "system",
|
|
423
|
-
content: faithfulness_default.trimEnd()
|
|
424
|
-
},
|
|
425
|
-
{ role: "user", content: `Answer:
|
|
426
|
-
${answer.slice(0, 2e3)}
|
|
427
|
-
|
|
428
|
-
Passages:
|
|
429
|
-
${context}${machine.length ? buildJudgeContext([], machine) : ""}` }
|
|
430
|
-
],
|
|
431
|
-
max_tokens: 6144,
|
|
432
|
-
reasoning_effort: "low",
|
|
433
|
-
// DeepSeek-V4 card: temp 1.0 / top_p 1.0
|
|
434
|
-
temperature: 1,
|
|
435
|
-
top_p: 1
|
|
436
|
-
});
|
|
437
|
-
const text = typeof res?.response === "string" ? res.response : res?.choices?.[0]?.message?.content;
|
|
438
|
-
const verdict = parseVerdict(text ?? "");
|
|
439
|
-
if (!verdict) {
|
|
440
|
-
console.log(`faithfulness: no parse (${Date.now() - t0}ms, text ${(text ?? "").length} chars) raw=${JSON.stringify((text ?? "").replace(/\s+/g, " ").slice(0, 500))}`);
|
|
441
|
-
return null;
|
|
442
|
-
}
|
|
443
|
-
return verdict;
|
|
444
|
-
})();
|
|
445
|
-
return await Promise.race([call, timeout]);
|
|
446
|
-
}
|
|
447
|
-
|
|
448
337
|
// workers/worker_public/prompts/enrichment.md
|
|
449
338
|
var enrichment_default = "You write a retrieval context for a passage from {{CORPUS_KIND}}. The context is prepended to the passage before embedding so a semantic search can locate the passage even when the query uses different vocabulary than the passage itself.\n\nWrite ONE concise sentence (at most 40 words) that situates the passage: name the publication by its exact {{PUBLISHER_NAME}} identifier (including part or annex when applicable) and what the passage covers \u2014 paraphrasing the topic in words DIFFERENT from the passage's own. Do not copy the passage verbatim, do not add facts that are not derivable from the passage or its header, do not answer or explain the content. Reply with the context sentence only \u2014 no quotes, no preamble.\n";
|
|
450
339
|
|
|
@@ -1058,7 +947,7 @@ async function handleMcp(env, ctx, req, tier, key) {
|
|
|
1058
947
|
// stream:false forces the JSON lane (anon defaults to SSE)
|
|
1059
948
|
body: JSON.stringify({ ...args, stream: false })
|
|
1060
949
|
});
|
|
1061
|
-
const res = name === "ask" ? await (await import("../../ask-
|
|
950
|
+
const res = name === "ask" ? await (await import("../../ask-OH5KPHZA.js")).handleAsk(env, ctx, inner, tier, key) : await (await import("../../search-KNB2S2PS.js")).handleSearch(env, ctx, inner, tier, key);
|
|
1062
951
|
return res.json().catch(() => ({ error: { message: "tool transport failed", status: res.status } }));
|
|
1063
952
|
});
|
|
1064
953
|
if (out.ok && "accepted" in out) return new Response(null, { status: 202 });
|
|
@@ -4,8 +4,8 @@ import {
|
|
|
4
4
|
requestSalt,
|
|
5
5
|
resolveRequestScope,
|
|
6
6
|
standardKeysFrom
|
|
7
|
-
} from "../../chunk-
|
|
8
|
-
import "../../chunk-
|
|
7
|
+
} from "../../chunk-O6VZIDPW.js";
|
|
8
|
+
import "../../chunk-Z24IAKB6.js";
|
|
9
9
|
import "../../chunk-3FYJM7LH.js";
|
|
10
10
|
export {
|
|
11
11
|
entitlementScope,
|
package/package.json
CHANGED
|
@@ -27,7 +27,7 @@
|
|
|
27
27
|
"playwright": "^1.62.1",
|
|
28
28
|
"yaml": "^2.9.1"
|
|
29
29
|
},
|
|
30
|
-
"version": "0.2.
|
|
30
|
+
"version": "0.2.49",
|
|
31
31
|
"description": "The Konneal engine: the publisher-agnostic build pipeline and API plane for standards intelligence (retrieval, answer contract, verdicts, evaluation).",
|
|
32
32
|
"license": "BSD-3-Clause",
|
|
33
33
|
"type": "module",
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
// echo — everything between "request validated" and "response written".
|
|
4
4
|
// index.ts routes here; this module owns the answer contract.
|
|
5
5
|
|
|
6
|
-
import { LIMITS, MODELS, num, sha256Hex, roleModel, answerEffort, requestEffort, effortBudget } from "./config";
|
|
6
|
+
import { LIMITS, MODELS, THRESHOLDS, num, sha256Hex, roleModel, answerEffort, requestEffort, effortBudget } from "./config";
|
|
7
7
|
import { portModelRunner } from "./env.ts";
|
|
8
8
|
import { refCodec } from "./codecs";
|
|
9
9
|
import { buildMessages, citations, retrieve, retrievalQuery, identityNote, splitHistory, listwiseRerank, refusalAnswer, Hit } from "./pipeline";
|
|
@@ -14,6 +14,8 @@ import { gradeRetrieval } from "./grader";
|
|
|
14
14
|
import summarizePrompt from "../prompts/summarize.md";
|
|
15
15
|
import { embed, generateOnce } from "./ai";
|
|
16
16
|
import { reflect } from "./reflect";
|
|
17
|
+
import { scoreFaithfulness } from "./faithfulness";
|
|
18
|
+
import { entailmentVerdict } from "./entailment";
|
|
17
19
|
import { checkQuoteAnchors, ANCHOR_CORRECTION_NOTE } from "./anchors";
|
|
18
20
|
import { canonicalRefusal } from "./refusal";
|
|
19
21
|
import { contractV2, tableRetyped } from "./refs";
|
|
@@ -1309,7 +1311,26 @@ async function handleAsk(
|
|
|
1309
1311
|
completionBlocks.push(...(await completeFigures(env.DB, answer, [...c2ns.blocks, ...completionBlocks], used)));
|
|
1310
1312
|
|
|
1311
1313
|
const jsonQuality = answerQuality(finalCites.map((c: any) => c.quality));
|
|
1312
|
-
|
|
1314
|
+
// the entailment gate (TODO.new-era/10): the eval battery's faithfulness
|
|
1315
|
+
// scorer, promoted onto the serving path — the confidence line states a
|
|
1316
|
+
// measurement (claim support in the cited passages), not only the source
|
|
1317
|
+
// rung. Budgeted and fail-open: a slow or unparseable check leaves the
|
|
1318
|
+
// rung note untouched; refusals and uncited answers have nothing to gate.
|
|
1319
|
+
let entailment: { support: string; score: number; ungrounded?: string[] } | null = null;
|
|
1320
|
+
let entailmentNote = "";
|
|
1321
|
+
if (THRESHOLDS.entailmentGate && jsonQuality && used.length && !answer.includes(refusalAnswer())) {
|
|
1322
|
+
const raced = await Promise.race([
|
|
1323
|
+
scoreFaithfulness(env.AI, roleModel(env, "grader"), answer, (used as Hit[]).map((h: Hit) => h.text)),
|
|
1324
|
+
new Promise<null>((r) => setTimeout(() => r(null), THRESHOLDS.entailmentBudgetMs)),
|
|
1325
|
+
]);
|
|
1326
|
+
if (raced) {
|
|
1327
|
+
const v = entailmentVerdict(raced.score, THRESHOLDS.entailmentSupportedFloor, THRESHOLDS.entailmentPartialFloor);
|
|
1328
|
+
entailment = { support: v.support, score: Math.round(raced.score * 100) / 100, ...(raced.ungrounded_claims?.length ? { ungrounded: raced.ungrounded_claims.slice(0, 3) } : {}) };
|
|
1329
|
+
entailmentNote = v.note;
|
|
1330
|
+
if (v.support !== "supported") console.log("entailment:", v.support, raced.score, raced.ungrounded_claims?.slice(0, 2));
|
|
1331
|
+
}
|
|
1332
|
+
}
|
|
1333
|
+
const out = { answer, citations: finalCites, ...(jsonQuality ? { source_quality: jsonQuality, confidence_note: entailmentNote || qualityNote(jsonQuality), ...(entailment ? { entailment } : {}), ...(jsonQuality === "ocr" ? { experimental_sources: experimentalSourceLabels(finalCites) } : {}) } : {}), model: MODELS.member, query_hash: queryHash, follow_ups: understanding?.follow_ups ?? [], blocks: [...c2ns.blocks, ...(verdictBlock ? [verdictBlock] : []), ...(conditionBlock ? [conditionBlock] : []), ...(aggregationBlock ? [aggregationBlock] : []), ...completionBlocks], context_applied: ctxApplied, ...(liveRecords ? { records: liveRecords } : {}) };
|
|
1313
1334
|
const cacheable = !contextual && !declaredCtx && !answer.includes(refusalAnswer()) && finalAnchors.violations.length === 0;
|
|
1314
1335
|
if (cacheable) {
|
|
1315
1336
|
const ck = exactCacheKey(env.INDEX_VERSION, gen, ns, await sha256Hex(cacheKeyMaterial(q.query, q.lang, salt)));
|
|
@@ -56,6 +56,15 @@ export const THRESHOLDS = {
|
|
|
56
56
|
* pool below the primary dense lane; they must earn their window
|
|
57
57
|
* slot under the cross-encoder, not by graph membership alone. */
|
|
58
58
|
graphLaneDiscount: 0.75,
|
|
59
|
+
/** The entailment gate (TODO.new-era/10): the faithfulness scorer
|
|
60
|
+
* promoted from eval-only onto the serving path. The confidence line
|
|
61
|
+
* then states a measurement — claim support in the cited passages —
|
|
62
|
+
* not only the source rung. Fails open (a slow or unparseable check
|
|
63
|
+
* leaves the source-rung note unchanged). */
|
|
64
|
+
entailmentGate: true,
|
|
65
|
+
entailmentBudgetMs: 6000,
|
|
66
|
+
entailmentSupportedFloor: 0.9,
|
|
67
|
+
entailmentPartialFloor: 0.5,
|
|
59
68
|
/** Concept-graph candidate discount — same rationale as the graph
|
|
60
69
|
* lane: definitional content enters discounted. */
|
|
61
70
|
conceptGraphDiscount: 0.75,
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
// The entailment gate's pure verdict mapper (TODO.new-era/10): what the
|
|
2
|
+
// confidence line says once the claims were measured against the
|
|
3
|
+
// passages they cite. Lives apart from faithfulness.ts so the unit
|
|
4
|
+
// suites (plain node, no esbuild .md loader) can exercise it.
|
|
5
|
+
|
|
6
|
+
export interface EntailmentVerdict {
|
|
7
|
+
support: "supported" | "partial" | "unsupported";
|
|
8
|
+
note: string;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export function entailmentVerdict(score: number, supportedFloor: number, partialFloor: number): EntailmentVerdict {
|
|
12
|
+
if (score >= supportedFloor) return { support: "supported", note: "" };
|
|
13
|
+
if (score >= partialFloor) return { support: "partial", note: "Partially grounded — some claims lack support in the cited passages; verify against the cited clauses." };
|
|
14
|
+
return { support: "unsupported", note: "WARNING: This answer's claims are largely unsupported by the retrieved passages. Verify against official publications." };
|
|
15
|
+
}
|
package/dist/ask-KL7XUELS.js
DELETED
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
import {
|
|
2
|
-
handleAsk
|
|
3
|
-
} from "./chunk-SONICKNY.js";
|
|
4
|
-
import "./chunk-IUMHKBOY.js";
|
|
5
|
-
import "./chunk-R2V3X6SQ.js";
|
|
6
|
-
import "./chunk-A3QHHUN5.js";
|
|
7
|
-
import "./chunk-4GJGBGJK.js";
|
|
8
|
-
import "./chunk-V46XM2GU.js";
|
|
9
|
-
import "./chunk-3FYJM7LH.js";
|
|
10
|
-
export {
|
|
11
|
-
handleAsk
|
|
12
|
-
};
|
package/dist/search-NTQVSSAE.js
DELETED
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
import {
|
|
2
|
-
handleSearch
|
|
3
|
-
} from "./chunk-47VBMM7I.js";
|
|
4
|
-
import "./chunk-IUMHKBOY.js";
|
|
5
|
-
import "./chunk-R2V3X6SQ.js";
|
|
6
|
-
import "./chunk-A3QHHUN5.js";
|
|
7
|
-
import "./chunk-4GJGBGJK.js";
|
|
8
|
-
import "./chunk-V46XM2GU.js";
|
|
9
|
-
import "./chunk-3FYJM7LH.js";
|
|
10
|
-
export {
|
|
11
|
-
handleSearch
|
|
12
|
-
};
|