@konneal/engine 0.2.25 → 0.2.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ask-YN63SADP.js +12 -0
- package/dist/{chunk-FYGJDJPN.js → chunk-2RKYO3OC.js} +2 -2
- package/dist/{chunk-HYI32HMI.js → chunk-3FYJM7LH.js} +3 -1
- package/dist/{chunk-RZ3UZVRR.js → chunk-A3QHHUN5.js} +1 -1
- package/dist/{chunk-BWSH7VBG.js → chunk-ADQEM7IS.js} +21 -8
- package/dist/{chunk-WQ2PYIWK.js → chunk-GAPT3WEN.js} +4 -4
- package/dist/{chunk-IOZAZA7Y.js → chunk-JGKSUSF5.js} +2 -2
- package/dist/{chunk-434NRPSS.js → chunk-OMXAE27N.js} +1 -1
- package/dist/{chunk-6MHR5JI4.js → chunk-WKG65DBA.js} +3 -3
- package/dist/faithfulness-context.d.ts +1 -0
- package/dist/faithfulness.d.ts +1 -1
- package/dist/profile.gen.d.ts +3 -1
- package/dist/profile2.gen.d.ts +3 -1
- package/dist/prompts/faithfulness.md +2 -0
- package/dist/search-MSFDEFQG.js +12 -0
- package/dist/worker_mcp/src/index.js +3 -3
- package/dist/worker_public/src/config.js +2 -2
- package/dist/worker_public/src/index.js +26 -15
- package/dist/worker_public/src/profile.js +1 -1
- package/dist/worker_public/src/refusal.js +2 -2
- package/dist/worker_public/src/requestScope.js +3 -3
- package/package.json +1 -1
- package/profile/sources.yaml +4 -1
- package/workers/worker_public/prompts/faithfulness.md +2 -0
- package/workers/worker_public/src/admin.ts +1 -1
- package/workers/worker_public/src/ask.ts +23 -2
- package/workers/worker_public/src/faithfulness-context.ts +16 -0
- package/workers/worker_public/src/faithfulness.ts +4 -5
- package/workers/worker_public/src/index.ts +9 -1
- package/workers/worker_public/src/profile.gen.ts +3 -1
- package/workers/worker_public/src/profile2.gen.ts +3 -1
- package/dist/ask-YE2L3Q5H.js +0 -12
- package/dist/search-I26YUVSP.js +0 -12
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import {
|
|
2
|
+
handleAsk
|
|
3
|
+
} from "./chunk-ADQEM7IS.js";
|
|
4
|
+
import "./chunk-WKG65DBA.js";
|
|
5
|
+
import "./chunk-2RKYO3OC.js";
|
|
6
|
+
import "./chunk-A3QHHUN5.js";
|
|
7
|
+
import "./chunk-JGKSUSF5.js";
|
|
8
|
+
import "./chunk-OMXAE27N.js";
|
|
9
|
+
import "./chunk-3FYJM7LH.js";
|
|
10
|
+
export {
|
|
11
|
+
handleAsk
|
|
12
|
+
};
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import {
|
|
2
2
|
LIMITS,
|
|
3
3
|
sha256Hex
|
|
4
|
-
} from "./chunk-
|
|
4
|
+
} from "./chunk-OMXAE27N.js";
|
|
5
5
|
import {
|
|
6
6
|
P
|
|
7
|
-
} from "./chunk-
|
|
7
|
+
} from "./chunk-3FYJM7LH.js";
|
|
8
8
|
|
|
9
9
|
// workers/worker_public/src/bubble.ts
|
|
10
10
|
function isAllowedBubbleOrigin(origin) {
|
|
@@ -27,23 +27,23 @@ import {
|
|
|
27
27
|
syntheticUnderstanding,
|
|
28
28
|
telemetry,
|
|
29
29
|
understandQuery
|
|
30
|
-
} from "./chunk-
|
|
30
|
+
} from "./chunk-WKG65DBA.js";
|
|
31
31
|
import {
|
|
32
32
|
corsHeaders,
|
|
33
33
|
err,
|
|
34
34
|
json,
|
|
35
35
|
readJson,
|
|
36
36
|
validateQuery
|
|
37
|
-
} from "./chunk-
|
|
37
|
+
} from "./chunk-2RKYO3OC.js";
|
|
38
38
|
import {
|
|
39
39
|
canonicalRefusal,
|
|
40
40
|
refusalAnswer
|
|
41
|
-
} from "./chunk-
|
|
41
|
+
} from "./chunk-A3QHHUN5.js";
|
|
42
42
|
import {
|
|
43
43
|
entitlementScope,
|
|
44
44
|
requestSalt,
|
|
45
45
|
resolveRequestScope
|
|
46
|
-
} from "./chunk-
|
|
46
|
+
} from "./chunk-JGKSUSF5.js";
|
|
47
47
|
import {
|
|
48
48
|
LIMITS,
|
|
49
49
|
MODELS,
|
|
@@ -53,10 +53,10 @@ import {
|
|
|
53
53
|
requestEffort,
|
|
54
54
|
roleModel,
|
|
55
55
|
sha256Hex
|
|
56
|
-
} from "./chunk-
|
|
56
|
+
} from "./chunk-OMXAE27N.js";
|
|
57
57
|
import {
|
|
58
58
|
P
|
|
59
|
-
} from "./chunk-
|
|
59
|
+
} from "./chunk-3FYJM7LH.js";
|
|
60
60
|
|
|
61
61
|
// workers/worker_public/src/internal_gateway.ts
|
|
62
62
|
async function retrieveInternal(service, auth, query) {
|
|
@@ -2014,6 +2014,19 @@ ${summary}` }] : [],
|
|
|
2014
2014
|
if (aggregationVerdict) console.log("aggregation engine:", aggregationVerdict.operation, aggregationVerdict.table, "\u2192", aggregationVerdict.value);
|
|
2015
2015
|
let boundaryNote = null;
|
|
2016
2016
|
let boundaryBoost;
|
|
2017
|
+
let modelEditionBoost;
|
|
2018
|
+
if (boundModel && !boundModel.gated) {
|
|
2019
|
+
const prefix = String(P().sources?.models?.standard_prefix ?? "");
|
|
2020
|
+
if (prefix && boundModel.standard.startsWith(prefix)) {
|
|
2021
|
+
const num3 = boundModel.standard.slice(prefix.length);
|
|
2022
|
+
const family = `${prefix.slice(prefix.lastIndexOf("-") + 1).toUpperCase()}-${num3}`;
|
|
2023
|
+
const row = await env.DB.prepare("SELECT edition FROM documents WHERE family = ?1 AND active = 1 ORDER BY edition DESC LIMIT 1").bind(family).first().catch(() => null);
|
|
2024
|
+
if (row?.edition) modelEditionBoost = String(row.edition);
|
|
2025
|
+
} else {
|
|
2026
|
+
modelEditionBoost = boundModel.standard;
|
|
2027
|
+
}
|
|
2028
|
+
}
|
|
2029
|
+
const lexicalBoost = [boundaryBoost, modelEditionBoost].filter(Boolean).join(" ") || void 0;
|
|
2017
2030
|
if (P().sources?.licensed?.length) {
|
|
2018
2031
|
const match = matchLicensedTopic(q.query, P().sources.licensed);
|
|
2019
2032
|
if (match && !(standardKeys?.has(match.entry.key) ?? false)) {
|
|
@@ -2065,7 +2078,7 @@ Answer account questions from these records ONLY: name the record when you use i
|
|
|
2065
2078
|
optimisticVec,
|
|
2066
2079
|
datasetScope: narrowed ? corpora : null,
|
|
2067
2080
|
standardKeys,
|
|
2068
|
-
lexicalBoost
|
|
2081
|
+
lexicalBoost
|
|
2069
2082
|
});
|
|
2070
2083
|
stageTiming["retrieve-core"] = Date.now() - tR;
|
|
2071
2084
|
console.log("stage: retrieve", Date.now() - tR, "ms");
|
|
@@ -2089,7 +2102,7 @@ Answer account questions from these records ONLY: name the record when you use i
|
|
|
2089
2102
|
if (grade === "weak" && understanding?.docidentifier) {
|
|
2090
2103
|
const broaden = `${understanding.standalone_query || q.query} ${understanding.docidentifier}`.trim();
|
|
2091
2104
|
const tc = Date.now();
|
|
2092
|
-
const second = await retrieve(env, q.query, { prev, understanding, queryOverride: broaden, federate, datasetScope: narrowed ? corpora : null, standardKeys, sealScope: declaredScoped ? docScope : null, lexicalBoost
|
|
2105
|
+
const second = await retrieve(env, q.query, { prev, understanding, queryOverride: broaden, federate, datasetScope: narrowed ? corpora : null, standardKeys, sealScope: declaredScoped ? docScope : null, lexicalBoost });
|
|
2093
2106
|
const grade2 = await gradeRetrieval(env.AI, roleModel(env, "grader"), q.query, second.hits.map((h) => h.text));
|
|
2094
2107
|
stageTiming.corrective = Date.now() - tc;
|
|
2095
2108
|
if (grade2 === "good") retrieved = second;
|
|
@@ -7,24 +7,24 @@ import {
|
|
|
7
7
|
sessionFrom,
|
|
8
8
|
telemetry,
|
|
9
9
|
understandQuery
|
|
10
|
-
} from "./chunk-
|
|
10
|
+
} from "./chunk-WKG65DBA.js";
|
|
11
11
|
import {
|
|
12
12
|
corsHeaders,
|
|
13
13
|
err,
|
|
14
14
|
json,
|
|
15
15
|
readJson,
|
|
16
16
|
validateQuery
|
|
17
|
-
} from "./chunk-
|
|
17
|
+
} from "./chunk-2RKYO3OC.js";
|
|
18
18
|
import {
|
|
19
19
|
entitlementScope,
|
|
20
20
|
standardKeysFrom
|
|
21
|
-
} from "./chunk-
|
|
21
|
+
} from "./chunk-JGKSUSF5.js";
|
|
22
22
|
import {
|
|
23
23
|
LIMITS,
|
|
24
24
|
MODELS,
|
|
25
25
|
num,
|
|
26
26
|
sha256Hex
|
|
27
|
-
} from "./chunk-
|
|
27
|
+
} from "./chunk-OMXAE27N.js";
|
|
28
28
|
|
|
29
29
|
// workers/worker_public/src/search.ts
|
|
30
30
|
async function handleSearch(env, ctx, req, tier, key) {
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import {
|
|
2
2
|
DATASETS,
|
|
3
3
|
datasetAllowed
|
|
4
|
-
} from "./chunk-
|
|
4
|
+
} from "./chunk-OMXAE27N.js";
|
|
5
5
|
import {
|
|
6
6
|
P
|
|
7
|
-
} from "./chunk-
|
|
7
|
+
} from "./chunk-3FYJM7LH.js";
|
|
8
8
|
|
|
9
9
|
// workers/worker_public/src/requestScope.ts
|
|
10
10
|
function licenseDeclared() {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import {
|
|
2
2
|
bubbleConfirmPage,
|
|
3
3
|
isAllowedBubbleOrigin
|
|
4
|
-
} from "./chunk-
|
|
4
|
+
} from "./chunk-2RKYO3OC.js";
|
|
5
5
|
import {
|
|
6
6
|
DATASETS,
|
|
7
7
|
LIMITS,
|
|
@@ -12,10 +12,10 @@ import {
|
|
|
12
12
|
processExpansion,
|
|
13
13
|
sha256Hex,
|
|
14
14
|
today
|
|
15
|
-
} from "./chunk-
|
|
15
|
+
} from "./chunk-OMXAE27N.js";
|
|
16
16
|
import {
|
|
17
17
|
P
|
|
18
|
-
} from "./chunk-
|
|
18
|
+
} from "./chunk-3FYJM7LH.js";
|
|
19
19
|
|
|
20
20
|
// workers/worker_public/src/ai.ts
|
|
21
21
|
var delay = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function buildJudgeContext(passages: string[], machine?: string[]): string;
|
package/dist/faithfulness.d.ts
CHANGED
|
@@ -3,4 +3,4 @@ export interface FaithfulnessResult {
|
|
|
3
3
|
score: number;
|
|
4
4
|
ungrounded_claims: string[];
|
|
5
5
|
}
|
|
6
|
-
export declare function scoreFaithfulness(ai: any, model: string, answer: string, passages: string[]): Promise<FaithfulnessResult | null>;
|
|
6
|
+
export declare function scoreFaithfulness(ai: any, model: string, answer: string, passages: string[], machine?: string[]): Promise<FaithfulnessResult | null>;
|
package/dist/profile.gen.d.ts
CHANGED
|
@@ -55,7 +55,9 @@ export declare const PROFILE: {
|
|
|
55
55
|
};
|
|
56
56
|
readonly bibliography: {};
|
|
57
57
|
readonly terminology: {};
|
|
58
|
-
readonly models: {
|
|
58
|
+
readonly models: {
|
|
59
|
+
readonly standard_prefix: "fx-r";
|
|
60
|
+
};
|
|
59
61
|
readonly licensed: readonly [{
|
|
60
62
|
readonly key: "std:fixture-ab-99";
|
|
61
63
|
readonly package: "fixture-ab-99";
|
package/dist/profile2.gen.d.ts
CHANGED
|
@@ -63,7 +63,9 @@ export declare const PROFILE: {
|
|
|
63
63
|
readonly corpora: {};
|
|
64
64
|
readonly bibliography: {};
|
|
65
65
|
readonly terminology: {};
|
|
66
|
-
readonly models: {
|
|
66
|
+
readonly models: {
|
|
67
|
+
readonly standard_prefix: "atlas-s";
|
|
68
|
+
};
|
|
67
69
|
};
|
|
68
70
|
readonly ui: {
|
|
69
71
|
readonly suggestions: readonly ["What does ATLAS 12 specify?", "Which errata are open?"];
|
|
@@ -1 +1,3 @@
|
|
|
1
1
|
You are a factuality judge. Given an answer and the retrieved passages it was based on, identify any claims in the answer that are NOT directly supported by the passages. Reply with ONLY a JSON object: {"score": 0.0-1.0, "ungrounded_claims": ["claim text", ...]} — score is the fraction of claims that ARE grounded in the passages; if every claim is supported, score is 1.0 and ungrounded_claims is [].
|
|
2
|
+
|
|
3
|
+
Passages prefixed [M] are this service's own machine-computed model data (typed blocks the answer was given) — a claim restating [M] content is grounded.
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import {
|
|
2
|
+
handleSearch
|
|
3
|
+
} from "./chunk-GAPT3WEN.js";
|
|
4
|
+
import "./chunk-WKG65DBA.js";
|
|
5
|
+
import "./chunk-2RKYO3OC.js";
|
|
6
|
+
import "./chunk-A3QHHUN5.js";
|
|
7
|
+
import "./chunk-JGKSUSF5.js";
|
|
8
|
+
import "./chunk-OMXAE27N.js";
|
|
9
|
+
import "./chunk-3FYJM7LH.js";
|
|
10
|
+
export {
|
|
11
|
+
handleSearch
|
|
12
|
+
};
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import {
|
|
2
2
|
authenticate
|
|
3
|
-
} from "../../chunk-
|
|
4
|
-
import "../../chunk-
|
|
3
|
+
} from "../../chunk-2RKYO3OC.js";
|
|
4
|
+
import "../../chunk-OMXAE27N.js";
|
|
5
5
|
import {
|
|
6
6
|
P,
|
|
7
7
|
setProfile
|
|
8
|
-
} from "../../chunk-
|
|
8
|
+
} from "../../chunk-3FYJM7LH.js";
|
|
9
9
|
|
|
10
10
|
// workers/worker_mcp/src/index.ts
|
|
11
11
|
var PROTOCOL_VERSION = "2025-06-18";
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import {
|
|
2
2
|
handleSearch
|
|
3
|
-
} from "../../chunk-
|
|
3
|
+
} from "../../chunk-GAPT3WEN.js";
|
|
4
4
|
import {
|
|
5
5
|
checkQuoteAnchors,
|
|
6
6
|
handleAsk,
|
|
7
7
|
handleMemories,
|
|
8
8
|
scoreJudge,
|
|
9
9
|
standardForDocNumber
|
|
10
|
-
} from "../../chunk-
|
|
10
|
+
} from "../../chunk-ADQEM7IS.js";
|
|
11
11
|
import {
|
|
12
12
|
buildMessages,
|
|
13
13
|
citations,
|
|
@@ -29,7 +29,7 @@ import {
|
|
|
29
29
|
sessionFrom,
|
|
30
30
|
telemetry,
|
|
31
31
|
understandQuery
|
|
32
|
-
} from "../../chunk-
|
|
32
|
+
} from "../../chunk-WKG65DBA.js";
|
|
33
33
|
import {
|
|
34
34
|
authenticate,
|
|
35
35
|
corsHeaders,
|
|
@@ -38,14 +38,14 @@ import {
|
|
|
38
38
|
readJson,
|
|
39
39
|
validateQuery,
|
|
40
40
|
withCors
|
|
41
|
-
} from "../../chunk-
|
|
41
|
+
} from "../../chunk-2RKYO3OC.js";
|
|
42
42
|
import {
|
|
43
43
|
canonicalRefusal
|
|
44
|
-
} from "../../chunk-
|
|
44
|
+
} from "../../chunk-A3QHHUN5.js";
|
|
45
45
|
import {
|
|
46
46
|
entitlementScope,
|
|
47
47
|
standardKeysFrom
|
|
48
|
-
} from "../../chunk-
|
|
48
|
+
} from "../../chunk-JGKSUSF5.js";
|
|
49
49
|
import {
|
|
50
50
|
LIMITS,
|
|
51
51
|
MODELS,
|
|
@@ -56,11 +56,11 @@ import {
|
|
|
56
56
|
roleModel,
|
|
57
57
|
sha256Hex,
|
|
58
58
|
today
|
|
59
|
-
} from "../../chunk-
|
|
59
|
+
} from "../../chunk-OMXAE27N.js";
|
|
60
60
|
import {
|
|
61
61
|
P,
|
|
62
62
|
setProfile
|
|
63
|
-
} from "../../chunk-
|
|
63
|
+
} from "../../chunk-3FYJM7LH.js";
|
|
64
64
|
|
|
65
65
|
// workers/worker_public/src/conversations.ts
|
|
66
66
|
var ID_RE = /^[a-zA-Z0-9_-]{8,64}$/;
|
|
@@ -327,7 +327,10 @@ async function handleGetShared(env, slug) {
|
|
|
327
327
|
}
|
|
328
328
|
|
|
329
329
|
// workers/worker_public/prompts/faithfulness.md
|
|
330
|
-
var faithfulness_default =
|
|
330
|
+
var faithfulness_default = `You are a factuality judge. Given an answer and the retrieved passages it was based on, identify any claims in the answer that are NOT directly supported by the passages. Reply with ONLY a JSON object: {"score": 0.0-1.0, "ungrounded_claims": ["claim text", ...]} \u2014 score is the fraction of claims that ARE grounded in the passages; if every claim is supported, score is 1.0 and ungrounded_claims is [].
|
|
331
|
+
|
|
332
|
+
Passages prefixed [M] are this service's own machine-computed model data (typed blocks the answer was given) \u2014 a claim restating [M] content is grounded.
|
|
333
|
+
`;
|
|
331
334
|
|
|
332
335
|
// workers/worker_public/src/verdict-parse.ts
|
|
333
336
|
function coerceVerdict(obj) {
|
|
@@ -385,10 +388,17 @@ function parseVerdict(text) {
|
|
|
385
388
|
return verdict;
|
|
386
389
|
}
|
|
387
390
|
|
|
391
|
+
// workers/worker_public/src/faithfulness-context.ts
|
|
392
|
+
function buildJudgeContext(passages, machine = []) {
|
|
393
|
+
const context = passages.slice(0, 8).map((p, i) => `[${i + 1}] ${p.replace(/\s+/g, " ").slice(0, 900)}`).join("\n");
|
|
394
|
+
const machineContext = machine.length ? "\n" + machine.slice(0, 6).map((m) => `[M] ${m.replace(/\s+/g, " ").slice(0, 400)}`).join("\n") : "";
|
|
395
|
+
return context + machineContext;
|
|
396
|
+
}
|
|
397
|
+
|
|
388
398
|
// workers/worker_public/src/faithfulness.ts
|
|
389
|
-
async function scoreFaithfulness(ai, model, answer, passages) {
|
|
399
|
+
async function scoreFaithfulness(ai, model, answer, passages, machine = []) {
|
|
390
400
|
if (!answer || !passages.length) return null;
|
|
391
|
-
const context = passages
|
|
401
|
+
const context = buildJudgeContext(passages, machine);
|
|
392
402
|
const t0 = Date.now();
|
|
393
403
|
const timeout = new Promise((r) => setTimeout(() => {
|
|
394
404
|
console.log(`faithfulness: timeout (${Date.now() - t0}ms)`);
|
|
@@ -405,7 +415,7 @@ async function scoreFaithfulness(ai, model, answer, passages) {
|
|
|
405
415
|
${answer.slice(0, 2e3)}
|
|
406
416
|
|
|
407
417
|
Passages:
|
|
408
|
-
${context}` }
|
|
418
|
+
${context}${machine.length ? buildJudgeContext([], machine) : ""}` }
|
|
409
419
|
],
|
|
410
420
|
max_tokens: 6144,
|
|
411
421
|
reasoning_effort: "low",
|
|
@@ -696,7 +706,7 @@ async function handleJudge(env, req) {
|
|
|
696
706
|
if (!question || !answer) return err(400, "invalid_input", "question and answer required");
|
|
697
707
|
const passagesText = passages.map((p, i) => `[${i + 1}] ${p}`).join("\n");
|
|
698
708
|
const [faith, relevancy, precision] = await Promise.all([
|
|
699
|
-
passages.length ? scoreFaithfulness(env.AI, roleModel(env, "grader"), answer, passages) : Promise.resolve(null),
|
|
709
|
+
passages.length ? scoreFaithfulness(env.AI, roleModel(env, "grader"), answer, passages, (Array.isArray(body?.blocks) ? body.blocks : []).map((b) => [b?.payload?.check, b?.payload?.meaning].filter((x) => typeof x === "string").join(" \u2014 ")).filter(Boolean)) : Promise.resolve(null),
|
|
700
710
|
scoreJudge(env.AI, roleModel(env, "grader"), relevancy_default, `Question: ${question}
|
|
701
711
|
|
|
702
712
|
Answer:
|
|
@@ -979,7 +989,7 @@ async function handleMcp(env, ctx, req, tier, key) {
|
|
|
979
989
|
// stream:false forces the JSON lane (anon defaults to SSE)
|
|
980
990
|
body: JSON.stringify({ ...args, stream: false })
|
|
981
991
|
});
|
|
982
|
-
const res = name === "ask" ? await (await import("../../ask-
|
|
992
|
+
const res = name === "ask" ? await (await import("../../ask-YN63SADP.js")).handleAsk(env, ctx, inner, tier, key) : await (await import("../../search-MSFDEFQG.js")).handleSearch(env, ctx, inner, tier, key);
|
|
983
993
|
return res.json().catch(() => ({ error: { message: "tool transport failed", status: res.status } }));
|
|
984
994
|
});
|
|
985
995
|
if (out.ok && "accepted" in out) return new Response(null, { status: 202 });
|
|
@@ -1448,7 +1458,8 @@ async function verifyRoute(c) {
|
|
|
1448
1458
|
{ name: "unit_references", deterministic: true, pass: refs.length === validRefs.length, detail: refs.length ? `${validRefs.length}/${refs.length} unit references resolve to served units` : "no unit references" },
|
|
1449
1459
|
{ name: "citations_present", deterministic: true, pass: new RegExp(`\\[[^\\]]*(${P().publisher.name})[^\\]]*\\]`).test(answer), detail: "normative claims should carry a passage citation" }
|
|
1450
1460
|
];
|
|
1451
|
-
const
|
|
1461
|
+
const machine = (Array.isArray(body?.blocks) ? body.blocks : []).filter((b) => b?.type === "verdict" && b?.payload).map((b) => [b.payload.check, b.payload.meaning, b.payload.definition, b.payload.violation_meaning].filter((x) => typeof x === "string" && x).join(" \u2014 ")).filter(Boolean);
|
|
1462
|
+
const faith = await scoreFaithfulness(env.AI, roleModel(env, "grader"), answer, retrieved.hits.map((h) => h.text), machine);
|
|
1452
1463
|
return json({
|
|
1453
1464
|
checks,
|
|
1454
1465
|
judged: faith ? { name: "faithfulness", deterministic: false, score: faith.score, ungrounded_claims: faith.ungrounded_claims.slice(0, 5) } : null,
|
|
@@ -4,9 +4,9 @@ import {
|
|
|
4
4
|
requestSalt,
|
|
5
5
|
resolveRequestScope,
|
|
6
6
|
standardKeysFrom
|
|
7
|
-
} from "../../chunk-
|
|
8
|
-
import "../../chunk-
|
|
9
|
-
import "../../chunk-
|
|
7
|
+
} from "../../chunk-JGKSUSF5.js";
|
|
8
|
+
import "../../chunk-OMXAE27N.js";
|
|
9
|
+
import "../../chunk-3FYJM7LH.js";
|
|
10
10
|
export {
|
|
11
11
|
entitlementScope,
|
|
12
12
|
licenseDeclared,
|
package/package.json
CHANGED
|
@@ -27,7 +27,7 @@
|
|
|
27
27
|
"playwright": "^1.62.1",
|
|
28
28
|
"yaml": "^2.9.1"
|
|
29
29
|
},
|
|
30
|
-
"version": "0.2.
|
|
30
|
+
"version": "0.2.26",
|
|
31
31
|
"description": "The Konneal engine: the publisher-agnostic build pipeline and API plane for standards intelligence (retrieval, answer contract, verdicts, evaluation).",
|
|
32
32
|
"license": "BSD-3-Clause",
|
|
33
33
|
"type": "module",
|
package/profile/sources.yaml
CHANGED
|
@@ -2,7 +2,10 @@ corpora:
|
|
|
2
2
|
clean: { repo: fixtures/corpus, note: the engine's fixture corpus }
|
|
3
3
|
bibliography: {}
|
|
4
4
|
terminology: {}
|
|
5
|
-
models:
|
|
5
|
+
models:
|
|
6
|
+
# the package-id prefix the model lanes read (the edition boost derives
|
|
7
|
+
# the graph family from it: <prefix><num> → family <LETTER>-<num>)
|
|
8
|
+
standard_prefix: "fx-r"
|
|
6
9
|
# the licensed-standards declaration (TODO.external-refs/08): the
|
|
7
10
|
# entitlement keys the deployment's index may carry (`standard_key` chunk
|
|
8
11
|
# metadata, from the packages' license_key facet) — this list is the
|
|
@@ -1 +1,3 @@
|
|
|
1
1
|
You are a factuality judge. Given an answer and the retrieved passages it was based on, identify any claims in the answer that are NOT directly supported by the passages. Reply with ONLY a JSON object: {"score": 0.0-1.0, "ungrounded_claims": ["claim text", ...]} — score is the fraction of claims that ARE grounded in the passages; if every claim is supported, score is 1.0 and ungrounded_claims is [].
|
|
2
|
+
|
|
3
|
+
Passages prefixed [M] are this service's own machine-computed model data (typed blocks the answer was given) — a claim restating [M] content is grounded.
|
|
@@ -342,7 +342,7 @@ export async function handleJudge(env: Env, req: Request): Promise<Response> {
|
|
|
342
342
|
// and this route was the one path the flip never reached — it judged
|
|
343
343
|
// on with the retired model and the scores collapsed
|
|
344
344
|
const [faith, relevancy, precision] = await Promise.all([
|
|
345
|
-
passages.length ? scoreFaithfulness(env.AI, roleModel(env, "grader"), answer, passages) : Promise.resolve(null),
|
|
345
|
+
passages.length ? scoreFaithfulness(env.AI, roleModel(env, "grader"), answer, passages, (Array.isArray(body?.blocks) ? body.blocks : []).map((b: any) => [b?.payload?.check, b?.payload?.meaning].filter((x: unknown) => typeof x === "string").join(" — ")).filter(Boolean)) : Promise.resolve(null),
|
|
346
346
|
scoreJudge(env.AI, roleModel(env, "grader"), relevancyPrompt, `Question: ${question}\n\nAnswer:\n${answer}`),
|
|
347
347
|
passages.length ? scoreJudge(env.AI, roleModel(env, "grader"), fill(precisionPrompt, promptVars()), `Question: ${question}\n\nPassages:\n${passagesText}`) : Promise.resolve(null),
|
|
348
348
|
]);
|
|
@@ -858,6 +858,27 @@ async function handleAsk(
|
|
|
858
858
|
// publication's own clause — surface for a question the document's
|
|
859
859
|
// vocabulary alone would miss
|
|
860
860
|
let boundaryBoost: string | undefined;
|
|
861
|
+
// the model-bound edition boost: the grounding pool tilts to the
|
|
862
|
+
// CURRENT edition's chunks (the superseded dirty-corpus editions
|
|
863
|
+
// otherwise outrank them on shared vocabulary and the answer's model
|
|
864
|
+
// values read as ungrounded against 2017 passages)
|
|
865
|
+
let modelEditionBoost: string | undefined;
|
|
866
|
+
if (boundModel && !boundModel.gated) {
|
|
867
|
+
const prefix = String(P().sources?.models?.standard_prefix ?? "");
|
|
868
|
+
if (prefix && boundModel.standard.startsWith(prefix)) {
|
|
869
|
+
// the profile's package prefix derives the graph family
|
|
870
|
+
// (<prefix><num> → family <LETTER>-<num>) and the boost is the
|
|
871
|
+
// family's ACTIVE edition
|
|
872
|
+
const num = boundModel.standard.slice(prefix.length);
|
|
873
|
+
const family = `${prefix.slice(prefix.lastIndexOf("-") + 1).toUpperCase()}-${num}`;
|
|
874
|
+
const row = await env.DB.prepare("SELECT edition FROM documents WHERE family = ?1 AND active = 1 ORDER BY edition DESC LIMIT 1")
|
|
875
|
+
.bind(family).first<any>().catch(() => null);
|
|
876
|
+
if (row?.edition) modelEditionBoost = String(row.edition);
|
|
877
|
+
} else {
|
|
878
|
+
modelEditionBoost = boundModel.standard; // the doc number is embedded in the id
|
|
879
|
+
}
|
|
880
|
+
}
|
|
881
|
+
const lexicalBoost = [boundaryBoost, modelEditionBoost].filter(Boolean).join(" ") || undefined;
|
|
861
882
|
if (P().sources?.licensed?.length) {
|
|
862
883
|
const match = matchLicensedTopic(q.query, P().sources.licensed);
|
|
863
884
|
if (match && !(standardKeys?.has(match.entry.key) ?? false)) {
|
|
@@ -929,7 +950,7 @@ async function handleAsk(
|
|
|
929
950
|
// publications there).
|
|
930
951
|
retrieved = await retrieve(env, q.query, { prev, understanding, federate, warmEmbed, graphDocNumbers,
|
|
931
952
|
sealScope: declaredScoped ? docScope : null, optimisticHits, optimisticVec,
|
|
932
|
-
datasetScope: narrowed ? corpora : null, standardKeys, lexicalBoost
|
|
953
|
+
datasetScope: narrowed ? corpora : null, standardKeys, lexicalBoost });
|
|
933
954
|
stageTiming["retrieve-core"] = Date.now() - tR;
|
|
934
955
|
console.log("stage: retrieve", Date.now() - tR, "ms");
|
|
935
956
|
// ── TTFT surgery: the two post-retrieval LLM calls run IN PARALLEL —
|
|
@@ -963,7 +984,7 @@ async function handleAsk(
|
|
|
963
984
|
if (grade === "weak" && understanding?.docidentifier) {
|
|
964
985
|
const broaden = `${understanding.standalone_query || q.query} ${understanding.docidentifier}`.trim();
|
|
965
986
|
const tc = Date.now();
|
|
966
|
-
const second = await retrieve(env, q.query, { prev, understanding, queryOverride: broaden, federate, datasetScope: narrowed ? corpora : null, standardKeys, sealScope: declaredScoped ? docScope : null, lexicalBoost
|
|
987
|
+
const second = await retrieve(env, q.query, { prev, understanding, queryOverride: broaden, federate, datasetScope: narrowed ? corpora : null, standardKeys, sealScope: declaredScoped ? docScope : null, lexicalBoost });
|
|
967
988
|
const grade2 = await gradeRetrieval(env.AI, roleModel(env, "grader"), q.query, second.hits.map((h: Hit) => h.text));
|
|
968
989
|
stageTiming.corrective = Date.now() - tc;
|
|
969
990
|
if (grade2 === "good") retrieved = second; // corrective retry must be strictly better
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
// The dependency-free core of the faithfulness grounding (node-loadable
|
|
2
|
+
// for unit tests — the verdict-parse law; faithfulness.ts imports the
|
|
3
|
+
// bundled prompt and cannot load under node). The [M] family is this
|
|
4
|
+
// service's own machine-computed model data: a machine-grounded answer
|
|
5
|
+
// quotes it legitimately, so the judge must see it.
|
|
6
|
+
|
|
7
|
+
export function buildJudgeContext(passages: string[], machine: string[] = []): string {
|
|
8
|
+
const context = passages
|
|
9
|
+
.slice(0, 8)
|
|
10
|
+
.map((p, i) => `[${i + 1}] ${p.replace(/\s+/g, " ").slice(0, 900)}`)
|
|
11
|
+
.join("\n");
|
|
12
|
+
const machineContext = machine.length
|
|
13
|
+
? "\n" + machine.slice(0, 6).map((m) => `[M] ${m.replace(/\s+/g, " ").slice(0, 400)}`).join("\n")
|
|
14
|
+
: "";
|
|
15
|
+
return context + machineContext;
|
|
16
|
+
}
|
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
// The prompt is data (prompts/faithfulness.md), bundled as text.
|
|
7
7
|
import faithfulnessPrompt from "../prompts/faithfulness.md";
|
|
8
8
|
import { parseVerdict } from "./verdict-parse";
|
|
9
|
+
import { buildJudgeContext } from "./faithfulness-context";
|
|
9
10
|
|
|
10
11
|
export type { Verdict } from "./verdict-parse";
|
|
11
12
|
|
|
@@ -19,12 +20,10 @@ export async function scoreFaithfulness(
|
|
|
19
20
|
model: string,
|
|
20
21
|
answer: string,
|
|
21
22
|
passages: string[],
|
|
23
|
+
machine: string[] = [],
|
|
22
24
|
): Promise<FaithfulnessResult | null> {
|
|
23
25
|
if (!answer || !passages.length) return null;
|
|
24
|
-
const context = passages
|
|
25
|
-
.slice(0, 8)
|
|
26
|
-
.map((p, i) => `[${i + 1}] ${p.replace(/\s+/g, " ").slice(0, 900)}`)
|
|
27
|
-
.join("\n");
|
|
26
|
+
const context = buildJudgeContext(passages, machine);
|
|
28
27
|
|
|
29
28
|
const t0 = Date.now();
|
|
30
29
|
const timeout = new Promise<null>((r) => setTimeout(() => { console.log(`faithfulness: timeout (${Date.now() - t0}ms)`); r(null); }, 240000));
|
|
@@ -35,7 +34,7 @@ export async function scoreFaithfulness(
|
|
|
35
34
|
role: "system",
|
|
36
35
|
content: faithfulnessPrompt.trimEnd(),
|
|
37
36
|
},
|
|
38
|
-
{ role: "user", content: `Answer:\n${answer.slice(0, 2000)}\n\nPassages:\n${context}` },
|
|
37
|
+
{ role: "user", content: `Answer:\n${answer.slice(0, 2000)}\n\nPassages:\n${context}${machine.length ? buildJudgeContext([], machine) : ""}` },
|
|
39
38
|
],
|
|
40
39
|
max_tokens: 6144,
|
|
41
40
|
reasoning_effort: "low",
|
|
@@ -238,7 +238,15 @@ async function verifyRoute(c: RouteContext): Promise<Response> {
|
|
|
238
238
|
{ name: "unit_references", deterministic: true, pass: refs.length === validRefs.length, detail: refs.length ? `${validRefs.length}/${refs.length} unit references resolve to served units` : "no unit references" },
|
|
239
239
|
{ name: "citations_present", deterministic: true, pass: new RegExp(`\\[[^\\]]*(${P().publisher.name})[^\\]]*\\]`).test(answer), detail: "normative claims should carry a passage citation" },
|
|
240
240
|
];
|
|
241
|
-
|
|
241
|
+
// the answer's typed blocks are the machine grounding (the computed
|
|
242
|
+
// verdict/aggregation values) — the judge sees them or every
|
|
243
|
+
// computed value reads as ungrounded
|
|
244
|
+
const machine = (Array.isArray(body?.blocks) ? body.blocks : [])
|
|
245
|
+
.filter((b: any) => b?.type === "verdict" && b?.payload)
|
|
246
|
+
.map((b: any) => [b.payload.check, b.payload.meaning, b.payload.definition, b.payload.violation_meaning]
|
|
247
|
+
.filter((x: unknown) => typeof x === "string" && x).join(" — "))
|
|
248
|
+
.filter(Boolean);
|
|
249
|
+
const faith = await scoreFaithfulness(env.AI, roleModel(env, "grader"), answer, retrieved.hits.map((h: Hit) => h.text), machine);
|
|
242
250
|
return json({
|
|
243
251
|
checks,
|
|
244
252
|
judged: faith ? { name: "faithfulness", deterministic: false, score: faith.score, ungrounded_claims: faith.ungrounded_claims.slice(0, 5) } : null,
|
package/dist/ask-YE2L3Q5H.js
DELETED
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
import {
|
|
2
|
-
handleAsk
|
|
3
|
-
} from "./chunk-BWSH7VBG.js";
|
|
4
|
-
import "./chunk-6MHR5JI4.js";
|
|
5
|
-
import "./chunk-FYGJDJPN.js";
|
|
6
|
-
import "./chunk-RZ3UZVRR.js";
|
|
7
|
-
import "./chunk-IOZAZA7Y.js";
|
|
8
|
-
import "./chunk-434NRPSS.js";
|
|
9
|
-
import "./chunk-HYI32HMI.js";
|
|
10
|
-
export {
|
|
11
|
-
handleAsk
|
|
12
|
-
};
|
package/dist/search-I26YUVSP.js
DELETED
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
import {
|
|
2
|
-
handleSearch
|
|
3
|
-
} from "./chunk-WQ2PYIWK.js";
|
|
4
|
-
import "./chunk-6MHR5JI4.js";
|
|
5
|
-
import "./chunk-FYGJDJPN.js";
|
|
6
|
-
import "./chunk-RZ3UZVRR.js";
|
|
7
|
-
import "./chunk-IOZAZA7Y.js";
|
|
8
|
-
import "./chunk-434NRPSS.js";
|
|
9
|
-
import "./chunk-HYI32HMI.js";
|
|
10
|
-
export {
|
|
11
|
-
handleSearch
|
|
12
|
-
};
|