@konneal/engine 0.1.4 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/admin.d.ts +1 -0
- package/dist/ask-47RNGK2R.js +12 -0
- package/dist/chunk-3OXSQH7Y.js +1852 -0
- package/dist/chunk-6GOSMLRH.js +2781 -0
- package/dist/{chunk-EHJEELVB.js → chunk-LNSDBEKS.js} +1 -1
- package/dist/{chunk-35ODH64W.js → chunk-Q327B27J.js} +33 -0
- package/dist/{chunk-OCNLV7Q7.js → chunk-Q6LI4T7M.js} +6 -1
- package/dist/{chunk-ROF3Q7UC.js → chunk-SN3ANQ3Y.js} +2 -2
- package/dist/chunk-VJZLVU3S.js +64 -0
- package/dist/{chunk-CAEHIVG5.js → chunk-WGXATDXY.js} +1 -1
- package/dist/codecs.d.ts +3 -3
- package/dist/completion.d.ts +1 -1
- package/dist/config.d.ts +9 -0
- package/dist/faithfulness.d.ts +1 -0
- package/dist/mcp-proto.d.ts +25 -0
- package/dist/mcp.d.ts +4 -0
- package/dist/openapi-surface.gen.d.ts +9 -0
- package/dist/openapi-types.d.ts +2141 -0
- package/dist/profile.gen.d.ts +1 -0
- package/dist/prompts/system.md +1 -0
- package/dist/quota.d.ts +4 -1
- package/dist/search-OMPBMZT4.js +11 -0
- package/dist/tablecontext.d.ts +7 -0
- package/dist/verdict-parse.d.ts +5 -0
- package/dist/worker_mcp/src/index.js +3 -3
- package/dist/worker_public/src/config.js +4 -2
- package/dist/worker_public/src/index.js +1193 -4932
- package/dist/worker_public/src/profile.js +1 -1
- package/dist/worker_public/src/refusal.js +2 -2
- package/dist/worker_public/src/requestScope.js +3 -3
- package/docs/spec-api.md +27 -13
- package/package.json +12 -3
- package/profile/prompts.yaml +3 -0
- package/workers/shared/router.ts +23 -16
- package/workers/worker_public/migrations/0014_usage_cache.sql +5 -0
- package/workers/worker_public/openapi.yaml +1169 -0
- package/workers/worker_public/prompts/system.md +1 -0
- package/workers/worker_public/schema.sql +3 -1
- package/workers/worker_public/src/admin.ts +51 -7
- package/workers/worker_public/src/ai.ts +10 -2
- package/workers/worker_public/src/ask.ts +94 -22
- package/workers/worker_public/src/codecs.ts +35 -10
- package/workers/worker_public/src/completion.ts +24 -1
- package/workers/worker_public/src/config.ts +10 -0
- package/workers/worker_public/src/faithfulness.ts +10 -17
- package/workers/worker_public/src/grader.ts +2 -2
- package/workers/worker_public/src/index.ts +106 -58
- package/workers/worker_public/src/lib/router.ts +1 -1
- package/workers/worker_public/src/mcp-proto.ts +71 -0
- package/workers/worker_public/src/mcp.ts +47 -0
- package/workers/worker_public/src/openapi-surface.gen.ts +318 -0
- package/workers/worker_public/src/pipeline.ts +4 -2
- package/workers/worker_public/src/profile.gen.ts +1 -0
- package/workers/worker_public/src/projects.ts +4 -2
- package/workers/worker_public/src/quota.ts +4 -2
- package/workers/worker_public/src/research.ts +55 -3
- package/workers/worker_public/src/tablecontext.ts +13 -2
- package/workers/worker_public/src/verdict-parse.ts +60 -0
|
@@ -0,0 +1,1852 @@
|
|
|
1
|
+
import {
|
|
2
|
+
requestSalt,
|
|
3
|
+
resolveRequestScope
|
|
4
|
+
} from "./chunk-LNSDBEKS.js";
|
|
5
|
+
import {
|
|
6
|
+
NO_CONTEXT,
|
|
7
|
+
appliedContext,
|
|
8
|
+
buildMessages,
|
|
9
|
+
checkQuota,
|
|
10
|
+
citations,
|
|
11
|
+
clientIp,
|
|
12
|
+
contextNote,
|
|
13
|
+
editionNote,
|
|
14
|
+
embed,
|
|
15
|
+
exchangeForLiveToken,
|
|
16
|
+
generateOnce,
|
|
17
|
+
graphExpand,
|
|
18
|
+
identityNote,
|
|
19
|
+
listwiseRerank,
|
|
20
|
+
liveDataConfig,
|
|
21
|
+
namedDocumentIn,
|
|
22
|
+
parseContext,
|
|
23
|
+
portModelRunner,
|
|
24
|
+
rawSessionToken,
|
|
25
|
+
resolveDocScope,
|
|
26
|
+
resolveLiveAccount,
|
|
27
|
+
retrievalQuery,
|
|
28
|
+
retrieve,
|
|
29
|
+
sessionFrom,
|
|
30
|
+
splitHistory,
|
|
31
|
+
syntheticUnderstanding,
|
|
32
|
+
telemetry,
|
|
33
|
+
understandQuery
|
|
34
|
+
} from "./chunk-6GOSMLRH.js";
|
|
35
|
+
import {
|
|
36
|
+
corsHeaders,
|
|
37
|
+
err,
|
|
38
|
+
json,
|
|
39
|
+
readJson,
|
|
40
|
+
validateQuery
|
|
41
|
+
} from "./chunk-SN3ANQ3Y.js";
|
|
42
|
+
import {
|
|
43
|
+
canonicalRefusal,
|
|
44
|
+
refusalAnswer
|
|
45
|
+
} from "./chunk-WGXATDXY.js";
|
|
46
|
+
import {
|
|
47
|
+
LIMITS,
|
|
48
|
+
MODELS,
|
|
49
|
+
answerEffort,
|
|
50
|
+
effortBudget,
|
|
51
|
+
num,
|
|
52
|
+
requestEffort,
|
|
53
|
+
roleModel,
|
|
54
|
+
sha256Hex
|
|
55
|
+
} from "./chunk-Q6LI4T7M.js";
|
|
56
|
+
import {
|
|
57
|
+
P
|
|
58
|
+
} from "./chunk-Q327B27J.js";
|
|
59
|
+
|
|
60
|
+
// workers/worker_public/src/internal_gateway.ts
|
|
61
|
+
async function retrieveInternal(service, auth, query) {
|
|
62
|
+
try {
|
|
63
|
+
const res = await service.fetch("https://internal/retrieve", {
|
|
64
|
+
method: "POST",
|
|
65
|
+
headers: { "content-type": "application/json", cookie: auth.cookie, authorization: auth.authorization },
|
|
66
|
+
body: JSON.stringify({ query })
|
|
67
|
+
});
|
|
68
|
+
if (!res.ok) return [];
|
|
69
|
+
const data = await res.json();
|
|
70
|
+
if (!Array.isArray(data?.hits)) return [];
|
|
71
|
+
return data.hits.map((h) => ({
|
|
72
|
+
id: h.id,
|
|
73
|
+
score: h.score,
|
|
74
|
+
metadata: h.metadata ?? {},
|
|
75
|
+
text: h.text ?? ""
|
|
76
|
+
}));
|
|
77
|
+
} catch {
|
|
78
|
+
return [];
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// workers/worker_public/prompts/grader.md
|
|
83
|
+
var grader_default = 'You grade retrieval quality for a legal-metrology Q&A system.\nGiven the question and the retrieved passage summaries, reply with ONLY:\n{"grade": "good"} \u2014 passages clearly contain the material to answer\n{"grade": "weak"} \u2014 passages are on the right publication/topic but lack the specific material (a broader or differently-worded retrieval might find it)\n{"grade": "bad"} \u2014 passages are unrelated to the question\n';
|
|
84
|
+
|
|
85
|
+
// workers/worker_public/src/grader.ts
|
|
86
|
+
async function gradeRetrieval(ai, model, query, passages) {
|
|
87
|
+
if (!passages.length) return "bad";
|
|
88
|
+
const summary = passages.slice(0, 8).map((p, i) => `[${i + 1}] ${p.replace(/\s+/g, " ").slice(0, 220)}`).join("\n");
|
|
89
|
+
const timeout = new Promise((r) => setTimeout(() => r(null), 6e3));
|
|
90
|
+
const call = (async () => {
|
|
91
|
+
const res = await ai.run(model, {
|
|
92
|
+
messages: [
|
|
93
|
+
{ role: "system", content: grader_default },
|
|
94
|
+
{ role: "user", content: `Question: ${query}
|
|
95
|
+
|
|
96
|
+
Passages:
|
|
97
|
+
${summary}` }
|
|
98
|
+
],
|
|
99
|
+
// reasoning shares this budget — starved budgets silently disable
|
|
100
|
+
// the CRAG corrective layer (default "good" fires). DeepSeek-V4's
|
|
101
|
+
// non-think mode is severely degraded (model card: HLE 8.1 vs 34.8),
|
|
102
|
+
// so the grader keeps reasoning on with real headroom plus the
|
|
103
|
+
// card's recommended sampling.
|
|
104
|
+
max_tokens: 6144,
|
|
105
|
+
reasoning_effort: "low",
|
|
106
|
+
temperature: 1,
|
|
107
|
+
top_p: 1
|
|
108
|
+
});
|
|
109
|
+
const text = typeof res?.response === "string" ? res.response : res?.choices?.[0]?.message?.content;
|
|
110
|
+
const m = (text ?? "").match(/"grade"\s*:\s*"(good|weak|bad)"/);
|
|
111
|
+
return m ? m[1] : null;
|
|
112
|
+
})();
|
|
113
|
+
try {
|
|
114
|
+
const got = await Promise.race([call, timeout]);
|
|
115
|
+
return got ?? "good";
|
|
116
|
+
} catch {
|
|
117
|
+
return "good";
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
async function scoreJudge(ai, model, systemPrompt, userPrompt) {
|
|
121
|
+
try {
|
|
122
|
+
const timeout = new Promise((r) => setTimeout(() => r(null), 15e3));
|
|
123
|
+
const call = (async () => {
|
|
124
|
+
const res = await ai.run(model, {
|
|
125
|
+
messages: [
|
|
126
|
+
{ role: "system", content: systemPrompt.trimEnd() },
|
|
127
|
+
{ role: "user", content: userPrompt }
|
|
128
|
+
],
|
|
129
|
+
max_tokens: 6144,
|
|
130
|
+
reasoning_effort: "low"
|
|
131
|
+
});
|
|
132
|
+
const text = typeof res?.response === "string" ? res.response : res?.choices?.[0]?.message?.content;
|
|
133
|
+
let score = null;
|
|
134
|
+
for (const m of (text ?? "").matchAll(/\{[^{}]*\}/g)) {
|
|
135
|
+
try {
|
|
136
|
+
const obj = JSON.parse(m[0]);
|
|
137
|
+
if (typeof obj.score === "number") score = obj.score;
|
|
138
|
+
} catch {
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
return score === null ? null : Math.max(0, Math.min(1, score));
|
|
142
|
+
})();
|
|
143
|
+
return await Promise.race([call, timeout]);
|
|
144
|
+
} catch {
|
|
145
|
+
return null;
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
// workers/worker_public/prompts/summarize.md
|
|
150
|
+
var summarize_default = "Summarize this conversation so a Q&A assistant can continue it with full continuity. Capture: documents and editions discussed, questions asked, answers given (key values and definitions), terminology established, unresolved threads. Under 150 words, plain text, no preamble. The conversation may be any length and in any language \u2014 summarize it in English.\n";
|
|
151
|
+
|
|
152
|
+
// workers/worker_public/prompts/reflect.md
|
|
153
|
+
var reflect_default = 'You are a factuality critic. Given a question, an answer, and the passages the answer was based on, determine if every factual claim in the answer is directly supported by the passages. Reply with ONLY: {"grounded": true} or {"grounded": false, "missing_info": "what is missing"}\n';
|
|
154
|
+
|
|
155
|
+
// workers/worker_public/src/reflect.ts
|
|
156
|
+
async function reflect(ai, model, question, answer, passages) {
|
|
157
|
+
if (!answer || !passages.length) return null;
|
|
158
|
+
const ctx = passages.slice(0, 8).map((p, i) => `[${i + 1}] ${p.replace(/\s+/g, " ").slice(0, 300)}`).join("\n");
|
|
159
|
+
const timeout = new Promise((r) => setTimeout(() => r(null), 6e3));
|
|
160
|
+
const call = (async () => {
|
|
161
|
+
const res = await ai.run(model, {
|
|
162
|
+
messages: [
|
|
163
|
+
{
|
|
164
|
+
role: "system",
|
|
165
|
+
content: reflect_default.trimEnd()
|
|
166
|
+
},
|
|
167
|
+
{
|
|
168
|
+
role: "user",
|
|
169
|
+
content: `Question: ${question}
|
|
170
|
+
|
|
171
|
+
Answer:
|
|
172
|
+
${answer.slice(0, 1500)}
|
|
173
|
+
|
|
174
|
+
Passages:
|
|
175
|
+
${ctx}`
|
|
176
|
+
}
|
|
177
|
+
],
|
|
178
|
+
// reasoning shares this budget — starved budgets silently disable
|
|
179
|
+
// the reflection layer (null = no retry ever fires); DeepSeek-V4
|
|
180
|
+
// card: keep reasoning on with headroom + temp 1.0 / top_p 1.0
|
|
181
|
+
max_tokens: 3072,
|
|
182
|
+
reasoning_effort: "low",
|
|
183
|
+
temperature: 1,
|
|
184
|
+
top_p: 1
|
|
185
|
+
});
|
|
186
|
+
const text = typeof res?.response === "string" ? res.response : res?.choices?.[0]?.message?.content;
|
|
187
|
+
const m = (text ?? "").match(/\{[\s\S]*\}/);
|
|
188
|
+
if (!m) return null;
|
|
189
|
+
try {
|
|
190
|
+
const raw = JSON.parse(m[0]);
|
|
191
|
+
return {
|
|
192
|
+
grounded: raw.grounded === true,
|
|
193
|
+
missing_info: typeof raw.missing_info === "string" ? raw.missing_info.slice(0, 200) : ""
|
|
194
|
+
};
|
|
195
|
+
} catch {
|
|
196
|
+
return null;
|
|
197
|
+
}
|
|
198
|
+
})();
|
|
199
|
+
try {
|
|
200
|
+
return await Promise.race([call, timeout]);
|
|
201
|
+
} catch {
|
|
202
|
+
return null;
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
// workers/worker_public/src/anchors.ts
|
|
207
|
+
var normalize = (s) => s.replace(/[‘’‛′]/g, "'").replace(/[“”‟″]/g, '"').replace(/[–—−]/g, "-").replace(//g, "").replace(/[ ]/g, " ").replace(/\s+/g, " ").toLowerCase();
|
|
208
|
+
var ANCHOR = /\[[^\[\]\n]*\]/g;
|
|
209
|
+
function checkQuoteAnchors(answer, passages) {
|
|
210
|
+
const flat = answer.replace(/[“”‟«»]/g, '"');
|
|
211
|
+
const hay = normalize(passages.join("\n\n"));
|
|
212
|
+
const violations = [];
|
|
213
|
+
let total = 0;
|
|
214
|
+
for (const anchor of flat.matchAll(ANCHOR)) {
|
|
215
|
+
for (const q of anchor[0].matchAll(/"([^"\n]+)"/g)) {
|
|
216
|
+
total++;
|
|
217
|
+
if (!hay.includes(normalize(q[1]))) violations.push(anchor[0]);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
return { total, violations };
|
|
221
|
+
}
|
|
222
|
+
var ANCHOR_CORRECTION_NOTE = "Correction notice: your draft quoted text that does not appear verbatim in the provided passages. Rewrite the answer \u2014 every quoted phrase must be an exact copy from a passage, or cite the clause without quoting.";
|
|
223
|
+
|
|
224
|
+
// workers/worker_public/src/refs.ts
|
|
225
|
+
var REF = /\[\[(u:[A-Za-z0-9_-]+)\]\]/g;
|
|
226
|
+
function availableUnitIds(hits) {
|
|
227
|
+
const ids = /* @__PURE__ */ new Set();
|
|
228
|
+
for (const h of hits) {
|
|
229
|
+
const u = h.metadata?.unit_id;
|
|
230
|
+
if (u) ids.add(u);
|
|
231
|
+
}
|
|
232
|
+
return ids;
|
|
233
|
+
}
|
|
234
|
+
function parseRefs(text) {
|
|
235
|
+
return [...text.matchAll(REF)].map((m) => m[1]);
|
|
236
|
+
}
|
|
237
|
+
function sanitizeRefs(text, available) {
|
|
238
|
+
const dropped = [];
|
|
239
|
+
const out = text.replace(REF, (full, id) => {
|
|
240
|
+
if (available.has(id)) return full;
|
|
241
|
+
dropped.push(id);
|
|
242
|
+
return "";
|
|
243
|
+
});
|
|
244
|
+
return { text: out, dropped };
|
|
245
|
+
}
|
|
246
|
+
async function resolveBlocks(db, refs) {
|
|
247
|
+
if (!refs.length) return [];
|
|
248
|
+
const uniq = [...new Set(refs)].slice(0, 12);
|
|
249
|
+
const blocks = [];
|
|
250
|
+
for (let i = 0; i < uniq.length; i += 20) {
|
|
251
|
+
const batch = uniq.slice(i, i + 20);
|
|
252
|
+
const placeholders = batch.map((_, n) => `?${n + 1}`).join(",");
|
|
253
|
+
try {
|
|
254
|
+
const res = await db.prepare(`SELECT unit_id, type, docidentifier, edition, payload FROM unit_payloads WHERE unit_id IN (${placeholders})`).bind(...batch).all();
|
|
255
|
+
for (const r of res.results) {
|
|
256
|
+
let payload = {};
|
|
257
|
+
try {
|
|
258
|
+
payload = JSON.parse(String(r.payload));
|
|
259
|
+
} catch {
|
|
260
|
+
continue;
|
|
261
|
+
}
|
|
262
|
+
blocks.push({
|
|
263
|
+
unit_id: String(r.unit_id),
|
|
264
|
+
type: String(r.type),
|
|
265
|
+
docidentifier: String(r.docidentifier ?? ""),
|
|
266
|
+
edition: r.edition ? String(r.edition) : void 0,
|
|
267
|
+
payload
|
|
268
|
+
});
|
|
269
|
+
}
|
|
270
|
+
} catch (e) {
|
|
271
|
+
console.log("resolveBlocks failed:", String(e).slice(0, 150));
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
return blocks;
|
|
275
|
+
}
|
|
276
|
+
async function contractV2(db, answer, usedHits) {
|
|
277
|
+
const available = availableUnitIds(usedHits);
|
|
278
|
+
const { text, dropped } = sanitizeRefs(answer, available);
|
|
279
|
+
if (dropped.length) console.log("refs: dropped", dropped.length, "invalid (not in passages)");
|
|
280
|
+
const refs = parseRefs(text);
|
|
281
|
+
const blocks = await resolveBlocks(db, refs);
|
|
282
|
+
return { text, blocks, dropped };
|
|
283
|
+
}
|
|
284
|
+
function tableRetyped(text, availableTable) {
|
|
285
|
+
if (!availableTable) return false;
|
|
286
|
+
return /(^|\n)\s*\|[^\n]+\|\s*(\n\s*\|[-: |]+\|\s*)?(\n|$)/.test(text) && (text.match(/\|/g) ?? []).length >= 6;
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
// workers/worker_public/src/completion.ts
|
|
290
|
+
async function completeTables(db, answer, used) {
|
|
291
|
+
const blocks = [];
|
|
292
|
+
try {
|
|
293
|
+
const answerNums = new Set((answer.match(/\d[\d ,.]{1,8}\d/g) ?? []).map((x) => x.replace(/[ ,.]/g, "")));
|
|
294
|
+
if (!answerNums.size) return blocks;
|
|
295
|
+
const fams = [...new Set(used.map((h) => h.metadata.docidentifier).filter(Boolean))].slice(0, 3);
|
|
296
|
+
for (const fam of fams) {
|
|
297
|
+
const base = String(fam).replace(/\s*\([A-Z]\)\s*$/, "").split(":")[0].trim();
|
|
298
|
+
const rows = await db.prepare("SELECT unit_id, payload FROM unit_payloads WHERE type = 'table' AND docidentifier LIKE ?1 LIMIT 8").bind(`%${base}%`).all();
|
|
299
|
+
for (const r of rows.results ?? []) {
|
|
300
|
+
const tableNums = new Set((String(r.payload).match(/\d[\d ,.]{1,8}\d/g) ?? []).map((x) => x.replace(/[ ,.]/g, "")));
|
|
301
|
+
let hits = 0;
|
|
302
|
+
for (const n of answerNums) if (tableNums.has(n)) hits++;
|
|
303
|
+
if (hits >= 1) {
|
|
304
|
+
const resolved = await resolveBlocks(db, [r.unit_id]);
|
|
305
|
+
blocks.push(...resolved);
|
|
306
|
+
console.log("contract D1 completion: table", r.unit_id, "in", base, "\u2014", hits, "matching values");
|
|
307
|
+
break;
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
if (blocks.length) break;
|
|
311
|
+
}
|
|
312
|
+
} catch {
|
|
313
|
+
}
|
|
314
|
+
return blocks;
|
|
315
|
+
}
|
|
316
|
+
async function completeFigures(db, answer, alreadyAttached, used = []) {
|
|
317
|
+
const unitForm = (answer.match(/u:fig[\w.-]*/g) ?? []).map((x) => x.replace(/[.,;:)]+$/, ""));
|
|
318
|
+
const bareForm = (answer.match(/\bfig-[\w.-]+\b/g) ?? []).map((x) => x.replace(/[.,;:)]+$/, ""));
|
|
319
|
+
let mentioned = [.../* @__PURE__ */ new Set([...unitForm, ...bareForm.map((x) => x.startsWith("u:") ? x : "u:" + x)])].slice(0, 6);
|
|
320
|
+
const proseNums = new Set(
|
|
321
|
+
(answer.match(/\bFig(?:ure|\.)s?\s*([0-9]{1,2}[a-z]?)/g) ?? []).map((x) => x.replace(/\bFig(?:ure|\.)s?\s*/i, "").toLowerCase())
|
|
322
|
+
);
|
|
323
|
+
if (proseNums.size) {
|
|
324
|
+
const fams = [...new Set(used.map((h) => h.metadata.docidentifier).filter(Boolean))].slice(0, 3);
|
|
325
|
+
for (const fam of fams) {
|
|
326
|
+
const base = String(fam).replace(/\s*\([A-Z]\)\s*$/, "").split(":")[0].trim();
|
|
327
|
+
const rows = await db.prepare("SELECT unit_id FROM unit_payloads WHERE type = 'figure' AND docidentifier LIKE ?1 LIMIT 24").bind(`%${base}%`).all();
|
|
328
|
+
for (const r of rows.results ?? []) {
|
|
329
|
+
const m = r.unit_id.match(/fig-?([0-9]{1,2}[a-z]?)/i);
|
|
330
|
+
if (m && proseNums.has(m[1].toLowerCase())) mentioned.push(r.unit_id);
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
mentioned = [...new Set(mentioned)].slice(0, 6);
|
|
334
|
+
}
|
|
335
|
+
const have = new Set(alreadyAttached.map((b) => b.unit_id));
|
|
336
|
+
const missing = mentioned.filter((id) => !have.has(id));
|
|
337
|
+
if (!missing.length) return [];
|
|
338
|
+
const figs = await resolveBlocks(db, missing);
|
|
339
|
+
if (figs.length) console.log("figure completion:", figs.map((b) => b.unit_id).join(", "), "attached from D1");
|
|
340
|
+
return figs;
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
// workers/worker_public/src/modelplane.ts
|
|
344
|
+
var NODE_RE = /(?:^|[\s("'`])\/(req|conf|term|constraint|characteristic|state-machine|dimension)\/([a-z0-9][a-z0-9_-]*(?:\/[a-z0-9][a-z0-9_-]*)?)(?=[\s)"'`,;:.]|$)/i;
|
|
345
|
+
function modelNodeRefIn(text) {
|
|
346
|
+
if (!text) return null;
|
|
347
|
+
const m = text.match(NODE_RE);
|
|
348
|
+
if (!m) return null;
|
|
349
|
+
const id = `/${m[1].toLowerCase()}/${m[2]}`;
|
|
350
|
+
return id.length <= 120 ? id : null;
|
|
351
|
+
}
|
|
352
|
+
function standardForDocNumber(docNumber) {
|
|
353
|
+
if (!docNumber) return null;
|
|
354
|
+
const models = P().sources?.models;
|
|
355
|
+
if (!models?.standards?.length || !models?.standard_prefix) return null;
|
|
356
|
+
return models.standards.includes(docNumber) ? `${models.standard_prefix}${docNumber}` : null;
|
|
357
|
+
}
|
|
358
|
+
async function fetchNode(env, standard, nodeId) {
|
|
359
|
+
try {
|
|
360
|
+
const row = await env.DB.prepare(
|
|
361
|
+
"SELECT standard, node_id, kind, name, clause_doc, clause_ref, content FROM model_nodes WHERE standard = ?1 AND node_id = ?2"
|
|
362
|
+
).bind(standard, nodeId).first();
|
|
363
|
+
if (!row) return null;
|
|
364
|
+
const content = JSON.parse(String(row.content));
|
|
365
|
+
const clause = row.clause_doc && row.clause_ref ? { doc: String(row.clause_doc), ref: String(row.clause_ref), urn: `${row.clause_doc}#clause-${row.clause_ref}` } : row.clause_doc ? { doc: String(row.clause_doc), ref: "", urn: String(row.clause_doc) } : null;
|
|
366
|
+
return {
|
|
367
|
+
standard: String(row.standard),
|
|
368
|
+
node_id: String(row.node_id),
|
|
369
|
+
kind: String(row.kind),
|
|
370
|
+
name: String(row.name ?? row.node_id),
|
|
371
|
+
clause,
|
|
372
|
+
content
|
|
373
|
+
};
|
|
374
|
+
} catch {
|
|
375
|
+
return null;
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
async function bindModelNode(env, opts) {
|
|
379
|
+
const nodeId = modelNodeRefIn(opts.label) ?? modelNodeRefIn(opts.query);
|
|
380
|
+
if (!nodeId) return null;
|
|
381
|
+
if (opts.standard) return fetchNode(env, opts.standard, nodeId);
|
|
382
|
+
try {
|
|
383
|
+
const rows = await env.DB.prepare("SELECT standard FROM model_nodes WHERE node_id = ?1 LIMIT 2").bind(nodeId).all();
|
|
384
|
+
const standards = (rows?.results ?? []).map((r) => String(r.standard));
|
|
385
|
+
if (standards.length === 1) return fetchNode(env, standards[0], nodeId);
|
|
386
|
+
return null;
|
|
387
|
+
} catch {
|
|
388
|
+
return null;
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
function clip(s, n = 500) {
|
|
392
|
+
const t = String(s ?? "").trim();
|
|
393
|
+
return t.length <= n ? t : t.slice(0, n - 1).trimEnd() + " \u2026";
|
|
394
|
+
}
|
|
395
|
+
function applicabilityText(app) {
|
|
396
|
+
if (!app || typeof app !== "object") return "";
|
|
397
|
+
const parts = [];
|
|
398
|
+
for (const [dim, cond] of Object.entries(app)) {
|
|
399
|
+
if (Array.isArray(cond)) parts.push(`${dim.replace(/_/g, " ")}: ${cond.join(", ")}`);
|
|
400
|
+
else if (cond && typeof cond === "object" && Array.isArray(cond.values)) {
|
|
401
|
+
parts.push(`${dim.replace(/_/g, " ")} (${cond.match ?? "any"}): ${cond.values.join(", ")}`);
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
return parts.join("; ");
|
|
405
|
+
}
|
|
406
|
+
function modelGroundingBlock(node) {
|
|
407
|
+
const c = node.content ?? {};
|
|
408
|
+
const lines = [];
|
|
409
|
+
lines.push(
|
|
410
|
+
P().prompts.vars.model_grounding_intro ?? "Model grounding \u2014 the model plane's own statement:"
|
|
411
|
+
);
|
|
412
|
+
lines.push(`Node: ${node.node_id} (${node.kind.replace(/_/g, " ")}) \u2014 ${node.name} [${node.standard}]`);
|
|
413
|
+
if (node.clause) lines.push(`Provenance: ${node.clause.urn}`);
|
|
414
|
+
if (c.statement) lines.push(`Statement: ${clip(c.statement)}`);
|
|
415
|
+
if (c.definition) lines.push(`Definition: ${clip(c.definition)}`);
|
|
416
|
+
if (c.purpose) lines.push(`Purpose: ${clip(c.purpose)}`);
|
|
417
|
+
const limit = c.limit ?? {};
|
|
418
|
+
if (limit.expression) lines.push(`Machine limit (the constraint the platform's verdict engine evaluates \u2014 quote it verbatim): ${limit.expression}`);
|
|
419
|
+
if (limit.accepts?.verdict) lines.push(`Machine limit: ${limit.accepts.verdict} ${limit.accepts.op} ${limit.accepts.limit} (the canonical acceptance chain)`);
|
|
420
|
+
if (c.check) lines.push(`Machine check: ${c.check}`);
|
|
421
|
+
if (c.derive) lines.push(`Derivation: ${c.derive}${Array.isArray(c.inputs) ? ` (inputs: ${c.inputs.join(", ")})` : ""}`);
|
|
422
|
+
const app = applicabilityText(c.applicability);
|
|
423
|
+
const scopeApp = applicabilityText(c.scope_applicability);
|
|
424
|
+
if (app || scopeApp) lines.push(`Applicability: ${[scopeApp, app].filter(Boolean).join("; ")}`);
|
|
425
|
+
if (Array.isArray(c.binds_to) && c.binds_to.length) lines.push(`Binds to: ${c.binds_to.join(", ")}`);
|
|
426
|
+
if (Array.isArray(c.targets) && c.targets.length) lines.push(`Verifies requirements: ${c.targets.join(", ")}`);
|
|
427
|
+
if (Array.isArray(c.preconditions) && c.preconditions.length) {
|
|
428
|
+
const pcs = c.preconditions.map((p) => `${p.id}: ${clip(p.check ?? (p.state ? `state = ${p.state}` : ""), 120)}`).join("; ");
|
|
429
|
+
lines.push(`Run-validity preconditions (a violation voids the run \u2014 invalid, never a fail): ${pcs}`);
|
|
430
|
+
}
|
|
431
|
+
if (c.acceptance_criteria?.description) lines.push(`Acceptance: ${clip(c.acceptance_criteria.description, 300)}`);
|
|
432
|
+
if (c.violation_meaning) lines.push(`Violation meaning (verbatim): ${clip(c.violation_meaning, 300)} \u2014 on violation: ${c.on_violation ?? "invalid"}`);
|
|
433
|
+
if (Array.isArray(c.values) && c.values.length) {
|
|
434
|
+
lines.push(`Values: ${c.values.map((v) => `${v.id}${v.implies?.length ? ` (implies ${v.implies.join(", ")})` : ""}`).join("; ")}`);
|
|
435
|
+
}
|
|
436
|
+
if (c.source_discrepancy) {
|
|
437
|
+
const sd = c.source_discrepancy;
|
|
438
|
+
lines.push(
|
|
439
|
+
`DECLARED SOURCE DISCREPANCY \u2014 the model and the text disagree; you MUST surface this and cite both: ${clip(sd.summary, 300)} Sources: ${(sd.sources ?? []).join(" and ")}. The model ${sd.resolution === "follows_clause_x" ? "follows one side" : "records the conflict without resolving it"}: ${clip(sd.rationale, 300)}`
|
|
440
|
+
);
|
|
441
|
+
}
|
|
442
|
+
lines.push(
|
|
443
|
+
"Rules for this answer: the machine facts (the constraint, the applicability, the acceptance, the provenance) come from THIS node \u2014 quote the machine limit verbatim, never invent one the node does not carry. If this model content and a prose passage disagree \u2014 including a passage from a different edition \u2014 say so explicitly and cite both (this node and the prose clause)."
|
|
444
|
+
);
|
|
445
|
+
return lines.join("\n");
|
|
446
|
+
}
|
|
447
|
+
function modelCitation(node) {
|
|
448
|
+
return {
|
|
449
|
+
doc_id: `model:${node.standard}`,
|
|
450
|
+
docidentifier: `${P().publisher.name} SMART model (${(() => {
|
|
451
|
+
const prefix = P().sources?.models?.standard_prefix ?? "";
|
|
452
|
+
const letter = prefix.replace(/^.*-/, "").toUpperCase();
|
|
453
|
+
return String(node.standard).replace(new RegExp(`^${prefix}`, "i"), `${letter} `);
|
|
454
|
+
})()})`,
|
|
455
|
+
edition: "",
|
|
456
|
+
language: "en",
|
|
457
|
+
clause_anchor: node.clause?.ref || "model",
|
|
458
|
+
clause_title: `${node.kind.replace(/_/g, " ")} \u2014 ${node.name} (${node.node_id})`,
|
|
459
|
+
status: "in-force",
|
|
460
|
+
corpus: "smart-model",
|
|
461
|
+
url: void 0,
|
|
462
|
+
snippet: `${node.node_id}${node.clause ? ` \xB7 ${node.clause.urn}` : ""}${node.content?.statement ? ` \u2014 ${clip(node.content.statement, 240)}` : ""}`,
|
|
463
|
+
score: 1
|
|
464
|
+
};
|
|
465
|
+
}
|
|
466
|
+
function modelEcho(node) {
|
|
467
|
+
return {
|
|
468
|
+
node_id: node.node_id.slice(0, 120),
|
|
469
|
+
kind: node.kind.slice(0, 40),
|
|
470
|
+
standard: node.standard.slice(0, 40),
|
|
471
|
+
...node.clause?.urn ? { clause: node.clause.urn.slice(0, 120) } : {}
|
|
472
|
+
};
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
// workers/worker_public/src/verdict.ts
|
|
476
|
+
function tokenize(src) {
|
|
477
|
+
const toks = [];
|
|
478
|
+
let i = 0;
|
|
479
|
+
const s = src.replace(/\s+/g, " ");
|
|
480
|
+
while (i < s.length) {
|
|
481
|
+
const c = s[i];
|
|
482
|
+
if (c === " ") {
|
|
483
|
+
i++;
|
|
484
|
+
continue;
|
|
485
|
+
}
|
|
486
|
+
if (/[0-9.]/.test(c)) {
|
|
487
|
+
const m = s.slice(i).match(/^[0-9]*\.?[0-9]+/);
|
|
488
|
+
toks.push({ t: "num", v: Number(m[0]) });
|
|
489
|
+
i += m[0].length;
|
|
490
|
+
continue;
|
|
491
|
+
}
|
|
492
|
+
if (/[A-Za-z_]/.test(c)) {
|
|
493
|
+
const m = s.slice(i).match(/^[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*/);
|
|
494
|
+
toks.push({ t: "id", v: m[0] });
|
|
495
|
+
i += m[0].length;
|
|
496
|
+
continue;
|
|
497
|
+
}
|
|
498
|
+
const two = s.slice(i, i + 2);
|
|
499
|
+
if ([">=", "<=", "==", "!="].includes(two)) {
|
|
500
|
+
toks.push({ t: "op", v: two });
|
|
501
|
+
i += 2;
|
|
502
|
+
continue;
|
|
503
|
+
}
|
|
504
|
+
if ("+-*/()<>".includes(c)) {
|
|
505
|
+
toks.push({ t: "op", v: c });
|
|
506
|
+
i++;
|
|
507
|
+
continue;
|
|
508
|
+
}
|
|
509
|
+
throw new Error(`bad char ${c}`);
|
|
510
|
+
}
|
|
511
|
+
return toks;
|
|
512
|
+
}
|
|
513
|
+
function parseAndEval(src, params) {
|
|
514
|
+
const toks = tokenize(src);
|
|
515
|
+
let p = 0;
|
|
516
|
+
const peek = () => toks[p];
|
|
517
|
+
const eat = (v) => {
|
|
518
|
+
const t = toks[p++];
|
|
519
|
+
if (v && (!t || t.t !== "op" || t.v !== v)) throw new Error(`expected ${v}`);
|
|
520
|
+
return t;
|
|
521
|
+
};
|
|
522
|
+
const isKw = (k) => {
|
|
523
|
+
const t = peek();
|
|
524
|
+
return t && t.t === "id" && t.v.toLowerCase() === k;
|
|
525
|
+
};
|
|
526
|
+
function or() {
|
|
527
|
+
let l = and();
|
|
528
|
+
while (isKw("or")) {
|
|
529
|
+
p++;
|
|
530
|
+
const r = and();
|
|
531
|
+
l = truthy(l) || truthy(r);
|
|
532
|
+
}
|
|
533
|
+
return l;
|
|
534
|
+
}
|
|
535
|
+
function and() {
|
|
536
|
+
let l = not();
|
|
537
|
+
while (isKw("and")) {
|
|
538
|
+
p++;
|
|
539
|
+
const r = not();
|
|
540
|
+
l = truthy(l) && truthy(r);
|
|
541
|
+
}
|
|
542
|
+
return l;
|
|
543
|
+
}
|
|
544
|
+
function not() {
|
|
545
|
+
if (isKw("not")) {
|
|
546
|
+
p++;
|
|
547
|
+
return !truthy(not());
|
|
548
|
+
}
|
|
549
|
+
return cmp();
|
|
550
|
+
}
|
|
551
|
+
function cmp() {
|
|
552
|
+
const l = add();
|
|
553
|
+
const t = peek();
|
|
554
|
+
if (t && t.t === "op" && [">=", "<=", ">", "<", "==", "!="].includes(t.v)) {
|
|
555
|
+
p++;
|
|
556
|
+
const r = add();
|
|
557
|
+
switch (t.v) {
|
|
558
|
+
case ">=":
|
|
559
|
+
return num2(l) >= num2(r);
|
|
560
|
+
case "<=":
|
|
561
|
+
return num2(l) <= num2(r);
|
|
562
|
+
case ">":
|
|
563
|
+
return num2(l) > num2(r);
|
|
564
|
+
case "<":
|
|
565
|
+
return num2(l) < num2(r);
|
|
566
|
+
case "==":
|
|
567
|
+
return num2(l) === num2(r);
|
|
568
|
+
default:
|
|
569
|
+
return num2(l) !== num2(r);
|
|
570
|
+
}
|
|
571
|
+
}
|
|
572
|
+
return l;
|
|
573
|
+
}
|
|
574
|
+
function add() {
|
|
575
|
+
let l = mul();
|
|
576
|
+
for (; ; ) {
|
|
577
|
+
const t = peek();
|
|
578
|
+
if (t && t.t === "op" && (t.v === "+" || t.v === "-")) {
|
|
579
|
+
p++;
|
|
580
|
+
const r = mul();
|
|
581
|
+
l = t.v === "+" ? num2(l) + num2(r) : num2(l) - num2(r);
|
|
582
|
+
} else return l;
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
function mul() {
|
|
586
|
+
let l = unary();
|
|
587
|
+
for (; ; ) {
|
|
588
|
+
const t = peek();
|
|
589
|
+
if (t && t.t === "op" && (t.v === "*" || t.v === "/")) {
|
|
590
|
+
p++;
|
|
591
|
+
const r = unary();
|
|
592
|
+
l = t.v === "*" ? num2(l) * num2(r) : num2(l) / num2(r);
|
|
593
|
+
} else return l;
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
function unary() {
|
|
597
|
+
const t = peek();
|
|
598
|
+
if (t && t.t === "op" && t.v === "-") {
|
|
599
|
+
p++;
|
|
600
|
+
return -num2(unary());
|
|
601
|
+
}
|
|
602
|
+
return atom();
|
|
603
|
+
}
|
|
604
|
+
function atom() {
|
|
605
|
+
const t = eat();
|
|
606
|
+
if (!t) throw new Error("unexpected end");
|
|
607
|
+
if (t.t === "num") return t.v;
|
|
608
|
+
if (t.t === "id") {
|
|
609
|
+
if (params[t.v] !== void 0) return params[t.v];
|
|
610
|
+
throw new Error(`missing ${t.v}`);
|
|
611
|
+
}
|
|
612
|
+
if (t.v === "(") {
|
|
613
|
+
const v = or();
|
|
614
|
+
eat(")");
|
|
615
|
+
return v;
|
|
616
|
+
}
|
|
617
|
+
throw new Error(`unexpected ${t.v}`);
|
|
618
|
+
}
|
|
619
|
+
const truthy = (v) => typeof v === "boolean" ? v : v !== 0;
|
|
620
|
+
const num2 = (v) => typeof v === "boolean" ? v ? 1 : 0 : v;
|
|
621
|
+
const out = or();
|
|
622
|
+
if (p !== toks.length) throw new Error("trailing tokens");
|
|
623
|
+
return out;
|
|
624
|
+
}
|
|
625
|
+
function oclBody(s) {
|
|
626
|
+
const m = String(s ?? "").match(/ocl\{([\s\S]*?)\}/);
|
|
627
|
+
return m ? m[1].trim() : null;
|
|
628
|
+
}
|
|
629
|
+
function extractChecks(content) {
|
|
630
|
+
if (!content || typeof content !== "object") return [];
|
|
631
|
+
const c = content;
|
|
632
|
+
const out = [];
|
|
633
|
+
const push = (e) => {
|
|
634
|
+
const b = e && oclBody(e);
|
|
635
|
+
if (b) out.push(b);
|
|
636
|
+
};
|
|
637
|
+
push(c.check);
|
|
638
|
+
push(c.limit?.expression);
|
|
639
|
+
push(c.acceptance_criteria?.limit && !c.acceptance_criteria.limit.expression?.includes("ocl{") ? null : c.acceptance_criteria?.limit?.expression);
|
|
640
|
+
const st = c.acceptance_criteria?.limit;
|
|
641
|
+
if (st?.expression && st.operator && st.threshold_expression) {
|
|
642
|
+
out.push(`${st.expression} ${st.operator} ${st.threshold_expression}`);
|
|
643
|
+
}
|
|
644
|
+
return [...new Set(out)];
|
|
645
|
+
}
|
|
646
|
+
function symbolsIn(checks) {
|
|
647
|
+
const ids = /* @__PURE__ */ new Set();
|
|
648
|
+
const KEYWORDS = /* @__PURE__ */ new Set(["and", "or", "not"]);
|
|
649
|
+
for (const chk of checks) {
|
|
650
|
+
try {
|
|
651
|
+
for (const t of tokenize(chk)) if (t.t === "id" && !KEYWORDS.has(t.v.toLowerCase())) ids.add(t.v);
|
|
652
|
+
} catch {
|
|
653
|
+
}
|
|
654
|
+
}
|
|
655
|
+
return [...ids];
|
|
656
|
+
}
|
|
657
|
+
function parseNumber(raw) {
|
|
658
|
+
let s = raw.replace(/[ ,]/g, "");
|
|
659
|
+
s = s.replace(/\.(\d{3})$/, "$1");
|
|
660
|
+
return Number(s.replace(/,(?=\d{3}\b)/g, ""));
|
|
661
|
+
}
|
|
662
|
+
function extractParams(query, symbols) {
|
|
663
|
+
const params = {};
|
|
664
|
+
for (const sym of symbols) {
|
|
665
|
+
const leaf = sym.split(".").pop() ?? sym;
|
|
666
|
+
const esc = leaf.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
667
|
+
const re = new RegExp(`\\b${esc}\\b\\D{0,14}?([0-9][0-9 ,.]*[0-9])`, "iu");
|
|
668
|
+
const m = query.match(re);
|
|
669
|
+
if (m) {
|
|
670
|
+
const v = parseNumber(m[1]);
|
|
671
|
+
if (Number.isFinite(v)) params[sym] = v;
|
|
672
|
+
}
|
|
673
|
+
}
|
|
674
|
+
return params;
|
|
675
|
+
}
|
|
676
|
+
function evaluate(content, query) {
|
|
677
|
+
const c = content && typeof content === "object" ? content : {};
|
|
678
|
+
const checks = extractChecks(content);
|
|
679
|
+
if (!checks.length) return null;
|
|
680
|
+
const symbols = symbolsIn(checks);
|
|
681
|
+
const params = extractParams(query, symbols);
|
|
682
|
+
const missing = symbols.filter((s) => params[s] === void 0);
|
|
683
|
+
const machine = checks.map((expression) => {
|
|
684
|
+
const values = {};
|
|
685
|
+
try {
|
|
686
|
+
for (const t of tokenize(expression)) if (t.t === "id" && params[t.v] !== void 0) values[t.v] = params[t.v];
|
|
687
|
+
} catch {
|
|
688
|
+
}
|
|
689
|
+
let result = null;
|
|
690
|
+
if (missing.length === 0) {
|
|
691
|
+
try {
|
|
692
|
+
result = !!parseAndEval(expression, params);
|
|
693
|
+
} catch {
|
|
694
|
+
result = null;
|
|
695
|
+
}
|
|
696
|
+
}
|
|
697
|
+
return { expression, symbolic: expression, values, result };
|
|
698
|
+
});
|
|
699
|
+
if (missing.length) {
|
|
700
|
+
return { verdict: "void", missing, checks: machine };
|
|
701
|
+
}
|
|
702
|
+
const failed = machine.some((m) => m.result === false);
|
|
703
|
+
const evaluable = machine.some((m) => m.result !== null);
|
|
704
|
+
if (!evaluable) return null;
|
|
705
|
+
return {
|
|
706
|
+
verdict: failed ? "fail" : "pass",
|
|
707
|
+
on_violation: failed ? String(c.on_violation ?? "invalid") : void 0,
|
|
708
|
+
violation_meaning: failed ? typeof c.violation_meaning === "string" ? c.violation_meaning : void 0 : void 0,
|
|
709
|
+
missing: [],
|
|
710
|
+
checks: machine
|
|
711
|
+
};
|
|
712
|
+
}
|
|
713
|
+
function verdictNote(v, node) {
|
|
714
|
+
const lines = [
|
|
715
|
+
`Machine verdict (deterministic evaluation of node ${node.node_id}${node.clause?.urn ? `, ${node.clause.urn}` : ""}) \u2014 the service EXECUTED the node's machine check against the values stated in the question:`
|
|
716
|
+
];
|
|
717
|
+
for (const c of v.checks) {
|
|
718
|
+
const vals = Object.entries(c.values).map(([k, n]) => `${k}=${n}`).join(", ");
|
|
719
|
+
lines.push(`- ${c.expression}${vals ? ` [${vals}]` : ""} \u2192 ${c.result === null ? "not evaluated" : c.result ? "holds" : "VIOLATED"}`);
|
|
720
|
+
}
|
|
721
|
+
if (v.verdict === "void") {
|
|
722
|
+
lines.push(`VERDICT: VOID \u2014 the question does not state: ${v.missing.join(", ")}. Say exactly what is missing; never assume values.`);
|
|
723
|
+
} else if (v.verdict === "pass") {
|
|
724
|
+
lines.push(`VERDICT: PASS \u2014 every machine check holds at the stated values. Present this verdict, the arithmetic above, and cite the node's clause.`);
|
|
725
|
+
} else {
|
|
726
|
+
lines.push(`VERDICT: ${String(v.on_violation ?? "FAIL").toUpperCase()} \u2014 a machine check is violated. Present this verdict, the arithmetic, the violation meaning verbatim, and cite the node's clause.`);
|
|
727
|
+
}
|
|
728
|
+
lines.push("This verdict is computed data \u2014 quote it faithfully; do not recompute, soften, or contradict it.");
|
|
729
|
+
return lines.join("\n");
|
|
730
|
+
}
|
|
731
|
+
|
|
732
|
+
// workers/worker_public/src/drafts.ts
|
|
733
|
+
var ACT_VERB = "(?:draft|prepare|pre-?fill|fill\\s+(?:in|out)|start|submit|file|lodge)";
|
|
734
|
+
var ACT_TARGET = "(?:new\\s+)?(?:certification\\s+|type[ -]evaluation\\s+|OIML[- ]CS\\s+)?application";
|
|
735
|
+
var INTENT_RES = [
|
|
736
|
+
new RegExp(`\\b${ACT_VERB}\\b[\\s\\S]{0,60}?\\b${ACT_TARGET}\\b`, "i"),
|
|
737
|
+
new RegExp(`\\b${ACT_TARGET}\\b[\\s\\S]{0,30}?\\b(?:draft|prepare|pre-?fill|for me)\\b`, "i")
|
|
738
|
+
];
|
|
739
|
+
function detectDraftIntent(query) {
|
|
740
|
+
if (/\b(?:status|where|progress|state)\b/i.test(query) && /\bapplication\b/i.test(query) && !INTENT_RES[0].test(query)) return null;
|
|
741
|
+
return INTENT_RES.some((re) => re.test(query)) ? "application_prefill" : null;
|
|
742
|
+
}
|
|
743
|
+
function decodeServiceRoles(token, platformClientId) {
|
|
744
|
+
try {
|
|
745
|
+
const payload = JSON.parse(atob(token.split(".")[1].replace(/-/g, "+").replace(/_/g, "/")));
|
|
746
|
+
const roles = payload?.service_roles?.[platformClientId];
|
|
747
|
+
return Array.isArray(roles) ? roles.filter((r) => typeof r === "string") : [];
|
|
748
|
+
} catch {
|
|
749
|
+
return [];
|
|
750
|
+
}
|
|
751
|
+
}
|
|
752
|
+
function roleLabel(role) {
|
|
753
|
+
if (role === "tl_operator") return "a test laboratory operator";
|
|
754
|
+
if (["ia_officer", "case_officer", "certification_officer", "signatory"].includes(role)) return "an issuing authority officer";
|
|
755
|
+
if (role === "viewer") return "a read-only viewer";
|
|
756
|
+
if (["cs_admin", "admin"].includes(role)) return "a scheme administrator";
|
|
757
|
+
return `the "${role}" role`;
|
|
758
|
+
}
|
|
759
|
+
var EXTRACTION_SYSTEM = `You extract the fields of a new OIML certification application from the user's own messages.
|
|
760
|
+
|
|
761
|
+
Rules:
|
|
762
|
+
- Output ONLY a JSON object \u2014 no prose, no code fence.
|
|
763
|
+
- Copy every value from the user's own words, and for each field give "source": the exact contiguous span of the user's message you copied it from.
|
|
764
|
+
- NEVER infer, complete, normalize away, or guess a value. If the user did not state it, omit the field entirely.
|
|
765
|
+
- "standard": the Recommendation the user named (e.g. "R 60" or "OIML R 60:2021") \u2014 a plain string, or omit when none was named.
|
|
766
|
+
- "scheme": only when the user named scheme A or scheme B explicitly.
|
|
767
|
+
|
|
768
|
+
Schema (every field optional):
|
|
769
|
+
{
|
|
770
|
+
"standard": "R 60",
|
|
771
|
+
"family_designation": { "value": "\u2026", "source": "\u2026" },
|
|
772
|
+
"group_label": { "value": "\u2026", "source": "\u2026" },
|
|
773
|
+
"model_designation": { "value": "\u2026", "source": "\u2026" },
|
|
774
|
+
"description": { "value": "\u2026", "source": "\u2026" },
|
|
775
|
+
"scheme": { "value": "A", "source": "\u2026" },
|
|
776
|
+
"samples": [ { "serial": "\u2026", "condition": "\u2026", "source": "\u2026" } ]
|
|
777
|
+
}`;
|
|
778
|
+
async function extractDraftFields(ai, model, userTurns) {
|
|
779
|
+
const transcript = userTurns.map((t, i) => `${i + 1}. ${t}`).join("\n").slice(0, 12e3);
|
|
780
|
+
try {
|
|
781
|
+
const res = await ai.run(model, {
|
|
782
|
+
messages: [
|
|
783
|
+
{ role: "system", content: EXTRACTION_SYSTEM },
|
|
784
|
+
{ role: "user", content: `The user's messages, oldest first:
|
|
785
|
+
${transcript}` }
|
|
786
|
+
],
|
|
787
|
+
max_tokens: 1200,
|
|
788
|
+
reasoning_effort: "low",
|
|
789
|
+
temperature: 0.1
|
|
790
|
+
});
|
|
791
|
+
const text = typeof res?.response === "string" ? res.response : res?.choices?.[0]?.message?.content;
|
|
792
|
+
if (typeof text !== "string") return null;
|
|
793
|
+
const start = text.indexOf("{");
|
|
794
|
+
const end = text.lastIndexOf("}");
|
|
795
|
+
if (start < 0 || end <= start) return null;
|
|
796
|
+
const parsed = JSON.parse(text.slice(start, end + 1));
|
|
797
|
+
return parsed && typeof parsed === "object" ? parsed : null;
|
|
798
|
+
} catch {
|
|
799
|
+
return null;
|
|
800
|
+
}
|
|
801
|
+
}
|
|
802
|
+
var norm = (s) => s.toLowerCase().replace(/[^a-z0-9-]+/g, " ").replace(/\s+/g, " ").trim();
|
|
803
|
+
var docNorm = (s) => norm(s).replace(/\s+/g, "");
|
|
804
|
+
var DROP_REASON = "not stated in your own words";
|
|
805
|
+
function traceabilityGuard(extraction, userTurns) {
|
|
806
|
+
const haystack = norm(userTurns.join("\n"));
|
|
807
|
+
const docHaystack = docNorm(userTurns.join(" "));
|
|
808
|
+
const kept = {};
|
|
809
|
+
const dropped = [];
|
|
810
|
+
const traced = (value, source) => {
|
|
811
|
+
if (typeof value !== "string" || !value.trim()) return null;
|
|
812
|
+
if (typeof source !== "string" || !source.trim()) return null;
|
|
813
|
+
const v = norm(value);
|
|
814
|
+
const s = norm(source);
|
|
815
|
+
if (!v || !s) return null;
|
|
816
|
+
return s.includes(v) && haystack.includes(s) ? value.trim() : null;
|
|
817
|
+
};
|
|
818
|
+
const scalar = (field) => {
|
|
819
|
+
const entry = extraction[field];
|
|
820
|
+
const ok = entry ? traced(entry.value, entry.source) : null;
|
|
821
|
+
if (ok) kept[field] = ok.slice(0, 300);
|
|
822
|
+
else if (entry && typeof entry.value === "string" && entry.value.trim()) {
|
|
823
|
+
dropped.push({ field, value: entry.value.trim().slice(0, 120), reason: DROP_REASON });
|
|
824
|
+
}
|
|
825
|
+
};
|
|
826
|
+
scalar("family_designation");
|
|
827
|
+
scalar("group_label");
|
|
828
|
+
scalar("model_designation");
|
|
829
|
+
scalar("description");
|
|
830
|
+
const scheme = extraction.scheme;
|
|
831
|
+
const schemeOk = scheme ? traced(scheme.value, scheme.source) : null;
|
|
832
|
+
if (schemeOk && /^[ab]$/i.test(schemeOk.trim())) kept.scheme = schemeOk.trim().toUpperCase();
|
|
833
|
+
else if (scheme && typeof scheme.value === "string" && scheme.value.trim()) {
|
|
834
|
+
dropped.push({ field: "scheme", value: scheme.value.trim().slice(0, 20), reason: DROP_REASON });
|
|
835
|
+
}
|
|
836
|
+
const samples = [];
|
|
837
|
+
(Array.isArray(extraction.samples) ? extraction.samples : []).forEach((s, i) => {
|
|
838
|
+
const ok = s ? traced(s.serial, s.source) : null;
|
|
839
|
+
if (ok) {
|
|
840
|
+
const sample = { serial: ok.slice(0, 80) };
|
|
841
|
+
if (typeof s?.condition === "string" && s.condition.trim()) sample.condition = s.condition.trim().slice(0, 20).toUpperCase();
|
|
842
|
+
samples.push(sample);
|
|
843
|
+
} else if (s && typeof s.serial === "string" && s.serial.trim()) {
|
|
844
|
+
dropped.push({ field: `samples[${i}].serial`, value: s.serial.trim().slice(0, 80), reason: DROP_REASON });
|
|
845
|
+
}
|
|
846
|
+
});
|
|
847
|
+
if (samples.length) kept.samples = samples;
|
|
848
|
+
if (typeof extraction.standard === "string" && extraction.standard.trim()) {
|
|
849
|
+
if (docHaystack.includes(docNorm(extraction.standard))) kept.standard = extraction.standard.trim().slice(0, 80);
|
|
850
|
+
else dropped.push({ field: "standard", value: extraction.standard.trim().slice(0, 80), reason: DROP_REASON });
|
|
851
|
+
}
|
|
852
|
+
return { kept, dropped };
|
|
853
|
+
}
|
|
854
|
+
async function resolveStandard(env, named) {
|
|
855
|
+
const m = named.match(/^urn:oiml:pub:([rdbge]):(\d{1,3})(?:-[0-9A-Za-z]+)?(?::(\d{4}))?$/i) ?? named.match(/^(?:OIML\s+)?([RDBGE])\s*(\d{1,3})(?:-[0-9A-Za-z]+)?(?:\s*:\s*(\d{4}))?$/i);
|
|
856
|
+
if (!m) return null;
|
|
857
|
+
const type = m[1].toUpperCase();
|
|
858
|
+
const num2 = String(Number(m[2]));
|
|
859
|
+
try {
|
|
860
|
+
const row = await env.DB.prepare(
|
|
861
|
+
"SELECT docidentifier, edition, status, derived_status FROM documents WHERE family = ?1 AND active = 1 ORDER BY (part IS NULL) DESC, edition DESC LIMIT 1"
|
|
862
|
+
).bind(`${type}-${num2}`).first();
|
|
863
|
+
if (!row) return null;
|
|
864
|
+
const edition = typeof row.edition === "string" ? row.edition : void 0;
|
|
865
|
+
return {
|
|
866
|
+
urn: `urn:oiml:pub:${type.toLowerCase()}:${num2}${edition ? `:${edition}` : ""}`,
|
|
867
|
+
label: typeof row.docidentifier === "string" ? row.docidentifier : `OIML ${type} ${num2}`,
|
|
868
|
+
...edition ? { edition } : {},
|
|
869
|
+
status: typeof row.derived_status === "string" ? row.derived_status : typeof row.status === "string" ? row.status : void 0
|
|
870
|
+
};
|
|
871
|
+
} catch {
|
|
872
|
+
return null;
|
|
873
|
+
}
|
|
874
|
+
}
|
|
875
|
+
var FIELD_LABELS = [
|
|
876
|
+
["family_designation", "the instrument family"],
|
|
877
|
+
["group_label", "the instrument group"],
|
|
878
|
+
["model_designation", "the model designation"],
|
|
879
|
+
["description", "the description"]
|
|
880
|
+
];
|
|
881
|
+
function refusal(reason, answer) {
|
|
882
|
+
return { status: "refused", reason, answer, citation: null };
|
|
883
|
+
}
|
|
884
|
+
async function prepareDraft(env, opts) {
|
|
885
|
+
if (!opts.member || opts.delegation.status === "unsigned") {
|
|
886
|
+
return refusal(
|
|
887
|
+
"sign_in_required",
|
|
888
|
+
"Preparing an act starts from your own account \u2014 sign in with your OIML SMART account and ask again. The draft would still be yours alone: it opens in the real form and only your own click commits it \u2014 I never hold a write credential."
|
|
889
|
+
);
|
|
890
|
+
}
|
|
891
|
+
if (opts.delegation.status === "not_configured") {
|
|
892
|
+
return refusal(
|
|
893
|
+
"not_configured",
|
|
894
|
+
"This deployment has not wired the live account link, so I cannot prepare acts here. I can still explain what the application asks for \u2014 just ask."
|
|
895
|
+
);
|
|
896
|
+
}
|
|
897
|
+
if (opts.delegation.status === "window_expired") {
|
|
898
|
+
return refusal(
|
|
899
|
+
"window_expired",
|
|
900
|
+
"Your live access window has lapsed \u2014 sign in again to refresh it, and I will prepare the draft. It stays a draft either way: only your own click in the real form commits it."
|
|
901
|
+
);
|
|
902
|
+
}
|
|
903
|
+
if (opts.delegation.status !== "ok") {
|
|
904
|
+
return refusal(
|
|
905
|
+
"exchange_refused",
|
|
906
|
+
"The live role check was refused, so I cannot prepare the draft honestly \u2014 the act needs your account's standing. Sign in afresh and ask again."
|
|
907
|
+
);
|
|
908
|
+
}
|
|
909
|
+
const roles = opts.platformClientId ? decodeServiceRoles(opts.delegation.token, opts.platformClientId) : [];
|
|
910
|
+
if (!roles.includes("applicant")) {
|
|
911
|
+
const primary = roles[0] ?? "unknown";
|
|
912
|
+
return refusal(
|
|
913
|
+
"role_refused",
|
|
914
|
+
`Your account's platform role \u2014 ${roleLabel(primary)} \u2014 can't prepare a new certification application: that act belongs to the applicant (the manufacturer's own account). Nothing was drafted. I can still walk you through what the application asks for \u2014 just ask.`
|
|
915
|
+
);
|
|
916
|
+
}
|
|
917
|
+
const userTurns = [
|
|
918
|
+
...opts.history.filter((h) => h.role === "user").map((h) => h.content),
|
|
919
|
+
opts.query
|
|
920
|
+
];
|
|
921
|
+
const extraction = await extractDraftFields(env.AI, opts.model, userTurns);
|
|
922
|
+
if (!extraction) {
|
|
923
|
+
return refusal(
|
|
924
|
+
"extraction_failed",
|
|
925
|
+
"I could not read your requirements reliably just now \u2014 nothing was drafted. Ask again in a moment, or start the application directly in the portal: every field there is yours either way."
|
|
926
|
+
);
|
|
927
|
+
}
|
|
928
|
+
const { kept, dropped } = traceabilityGuard(extraction, userTurns);
|
|
929
|
+
if (!kept.standard) {
|
|
930
|
+
const untraced = dropped.find((d) => d.field === "standard");
|
|
931
|
+
return refusal(
|
|
932
|
+
"standard_unresolved",
|
|
933
|
+
untraced ? `I can't anchor the draft: you haven't named the Recommendation in your own words (the ${untraced.value} reading isn't yours). Name it plainly \u2014 for example OIML R 60 \u2014 and I'll prepare the draft.` : "I can't anchor the draft: you haven't named the Recommendation. Name it plainly \u2014 for example OIML R 60 \u2014 and I'll prepare it."
|
|
934
|
+
);
|
|
935
|
+
}
|
|
936
|
+
const standard = await resolveStandard(env, kept.standard);
|
|
937
|
+
if (!standard) {
|
|
938
|
+
return refusal(
|
|
939
|
+
"standard_unresolved",
|
|
940
|
+
`I couldn't resolve ${kept.standard} as a publication in the corpus, so I can't anchor the draft. Name the Recommendation plainly \u2014 for example OIML R 60 \u2014 and I'll prepare it.`
|
|
941
|
+
);
|
|
942
|
+
}
|
|
943
|
+
const fields = {
|
|
944
|
+
standard_doc: standard.urn,
|
|
945
|
+
standard_label: standard.label,
|
|
946
|
+
...kept.family_designation ? { family_designation: kept.family_designation } : {},
|
|
947
|
+
...kept.group_label ? { group_label: kept.group_label } : {},
|
|
948
|
+
...kept.model_designation ? { model_designation: kept.model_designation } : {},
|
|
949
|
+
...kept.description ? { description: kept.description } : {},
|
|
950
|
+
...kept.samples?.length ? { samples: kept.samples } : {},
|
|
951
|
+
...kept.scheme ? { scheme: kept.scheme } : {}
|
|
952
|
+
};
|
|
953
|
+
const carries = [`the Recommendation: ${standard.label}`];
|
|
954
|
+
for (const [key, label] of FIELD_LABELS) {
|
|
955
|
+
const v = kept[key];
|
|
956
|
+
if (typeof v === "string") carries.push(`${label}: ${v}`);
|
|
957
|
+
}
|
|
958
|
+
if (kept.scheme) carries.push(`scheme ${kept.scheme}`);
|
|
959
|
+
if (kept.samples?.length) carries.push(`${kept.samples.length} sample${kept.samples.length === 1 ? "" : "s"}: ${kept.samples.map((s) => s.serial).join(", ")}`);
|
|
960
|
+
const notes = [
|
|
961
|
+
`The technical parameters (capacities, dimensions, classes) stay with you: the form derives what ${standard.label}'s model declares, and you confirm each value.`,
|
|
962
|
+
"Every field in the real form stays editable \u2014 the draft is a starting point, never a decision."
|
|
963
|
+
];
|
|
964
|
+
const draft = {
|
|
965
|
+
kind: "draft",
|
|
966
|
+
act: "application_prefill",
|
|
967
|
+
version: 1,
|
|
968
|
+
title: `New ${standard.label} application`,
|
|
969
|
+
prepared_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
970
|
+
requires_confirmation: true,
|
|
971
|
+
fields,
|
|
972
|
+
...dropped.length ? { dropped } : {},
|
|
973
|
+
notes
|
|
974
|
+
};
|
|
975
|
+
const answer = `I've prepared a draft for a new ${standard.label} application from your own words.
|
|
976
|
+
|
|
977
|
+
What the draft carries:
|
|
978
|
+
${carries.map((c) => `- ${c}`).join("\n")}
|
|
979
|
+
` + (dropped.length ? `
|
|
980
|
+
I left ${dropped.length === 1 ? "this" : "these"} out because you never stated ${dropped.length === 1 ? "it" : "them"} in your own words: ${dropped.map((d) => `${d.field.replace(/\[(\d+)\]/, " $1")} ("${d.value}")`).join("; ")}. Say them plainly and I'll add them.
|
|
981
|
+
` : "") + `
|
|
982
|
+
${notes[0]}
|
|
983
|
+
|
|
984
|
+
The draft opens in the real application form with every field editable \u2014 review it carefully. Nothing is submitted until you confirm it there yourself: I never hold a write credential; your own click is the only commit.`;
|
|
985
|
+
return {
|
|
986
|
+
status: "draft",
|
|
987
|
+
draft,
|
|
988
|
+
answer,
|
|
989
|
+
citation: { docidentifier: standard.label, ...standard.edition ? { edition: standard.edition } : {}, ...standard.status ? { status: standard.status } : {} }
|
|
990
|
+
};
|
|
991
|
+
}
|
|
992
|
+
|
|
993
|
+
// workers/worker_public/src/memories.ts
|
|
994
|
+
var MAX_FILES = 10;
|
|
995
|
+
var MAX_CONTENT = 8e3;
|
|
996
|
+
var MAX_NAME = 64;
|
|
997
|
+
var ID_RE = /^m:[a-f0-9]{16}$/;
|
|
998
|
+
var newId = () => "m:" + [...crypto.getRandomValues(new Uint8Array(8))].map((b) => b.toString(16).padStart(2, "0")).join("");
|
|
999
|
+
async function handleMemories(env, sub, req, route) {
|
|
1000
|
+
const { method, id } = route;
|
|
1001
|
+
if (method === "GET") {
|
|
1002
|
+
const rows = (await env.DB.prepare("SELECT id, name, content, enabled, updated_at FROM memories WHERE sub = ?1 ORDER BY updated_at DESC").bind(sub).all()).results ?? [];
|
|
1003
|
+
return json({ memories: rows });
|
|
1004
|
+
}
|
|
1005
|
+
if (method === "DELETE") {
|
|
1006
|
+
if (!id || !ID_RE.test(id)) return err(400, "invalid_input", "bad memory id");
|
|
1007
|
+
await env.DB.prepare("DELETE FROM memories WHERE id = ?1 AND sub = ?2").bind(id, sub).run();
|
|
1008
|
+
return json({ ok: true });
|
|
1009
|
+
}
|
|
1010
|
+
if (method === "POST") {
|
|
1011
|
+
const body = await req.json().catch(() => null);
|
|
1012
|
+
const name = typeof body?.name === "string" ? body.name.trim().slice(0, MAX_NAME) : "";
|
|
1013
|
+
const content = typeof body?.content === "string" ? body.content.slice(0, MAX_CONTENT) : "";
|
|
1014
|
+
if (!name || !content.trim()) return err(400, "invalid_input", "name and content are required");
|
|
1015
|
+
const now = Date.now();
|
|
1016
|
+
if (typeof body?.id === "string" && ID_RE.test(body.id)) {
|
|
1017
|
+
const r = await env.DB.prepare(
|
|
1018
|
+
"UPDATE memories SET name = ?1, content = ?2, updated_at = ?3 WHERE id = ?4 AND sub = ?5"
|
|
1019
|
+
).bind(name, content, now, body.id, sub).run();
|
|
1020
|
+
if (!r.meta?.changes) return err(404, "not_found", "no such memory");
|
|
1021
|
+
return json({ ok: true, id: body.id });
|
|
1022
|
+
}
|
|
1023
|
+
const count = Number((await env.DB.prepare("SELECT COUNT(*) AS n FROM memories WHERE sub = ?1").bind(sub).first())?.n ?? 0);
|
|
1024
|
+
if (count >= MAX_FILES) return err(400, "quota_exceeded", `at most ${MAX_FILES} memory files`);
|
|
1025
|
+
const mid = newId();
|
|
1026
|
+
await env.DB.prepare(
|
|
1027
|
+
"INSERT INTO memories (id, sub, name, content, enabled, created_at, updated_at) VALUES (?1, ?2, ?3, ?4, 1, ?5, ?5)"
|
|
1028
|
+
).bind(mid, sub, name, content, now).run();
|
|
1029
|
+
return json({ ok: true, id: mid });
|
|
1030
|
+
}
|
|
1031
|
+
return err(405, "method_not_allowed", "GET, POST or DELETE");
|
|
1032
|
+
}
|
|
1033
|
+
async function memoryNote(env, sub, ids) {
|
|
1034
|
+
const wanted = [...new Set(ids.filter((x) => typeof x === "string" && (ID_RE.test(x) || /^pf:[a-f0-9]{16}$/.test(x))))].slice(0, 4);
|
|
1035
|
+
if (!wanted.length) return [null, []];
|
|
1036
|
+
const ph = wanted.map((_, i) => `?${i + 2}`).join(",");
|
|
1037
|
+
const rows = [
|
|
1038
|
+
...(await env.DB.prepare(`SELECT id, name, content FROM memories WHERE sub = ?1 AND id IN (${ph})`).bind(sub, ...wanted).all()).results ?? [],
|
|
1039
|
+
...(await env.DB.prepare(
|
|
1040
|
+
`SELECT f.id, f.name, f.content FROM project_files f JOIN projects p ON p.id = f.project_id WHERE p.sub = ?1 AND f.id IN (${ph})`
|
|
1041
|
+
).bind(sub, ...wanted).all()).results ?? []
|
|
1042
|
+
];
|
|
1043
|
+
if (!rows.length) return [null, []];
|
|
1044
|
+
let budget = 4e3;
|
|
1045
|
+
const parts = [];
|
|
1046
|
+
const used = [];
|
|
1047
|
+
for (const r of rows) {
|
|
1048
|
+
if (budget <= 200) break;
|
|
1049
|
+
const body = String(r.content).slice(0, budget);
|
|
1050
|
+
budget -= body.length;
|
|
1051
|
+
parts.push(`### ${r.name}
|
|
1052
|
+
${body}`);
|
|
1053
|
+
used.push(String(r.id));
|
|
1054
|
+
}
|
|
1055
|
+
return [
|
|
1056
|
+
"The user's own memory files \u2014 their stated context. Treat as trusted user facts (their lab, instruments, preferences); blend with the passages, never contradict them silently:\n" + parts.join("\n\n"),
|
|
1057
|
+
used
|
|
1058
|
+
];
|
|
1059
|
+
}
|
|
1060
|
+
|
|
1061
|
+
// workers/worker_public/src/answercache.ts
|
|
1062
|
+
var CORPUS_GEN_KEY = "sys:corpus_gen";
|
|
1063
|
+
async function corpusGen(cache) {
|
|
1064
|
+
try {
|
|
1065
|
+
return await cache.get(CORPUS_GEN_KEY) ?? "0";
|
|
1066
|
+
} catch {
|
|
1067
|
+
return "0";
|
|
1068
|
+
}
|
|
1069
|
+
}
|
|
1070
|
+
function freshRequested(body) {
|
|
1071
|
+
const f = body?.fresh;
|
|
1072
|
+
return f === true || f === "true" || f === 1 || f === "1";
|
|
1073
|
+
}
|
|
1074
|
+
function cacheKeyMaterial(query, lang, salt) {
|
|
1075
|
+
return `${query.toLowerCase().replace(/\s+/g, " ").trim()}|${lang ?? ""}${salt ? "|" + salt : ""}`;
|
|
1076
|
+
}
|
|
1077
|
+
function exactCacheKey(indexVersion, gen, ns, queryHash) {
|
|
1078
|
+
return `a:${indexVersion}:g${gen}:${ns}:${queryHash}`;
|
|
1079
|
+
}
|
|
1080
|
+
function semanticCacheKey(indexVersion, gen, signature) {
|
|
1081
|
+
return `sc:${indexVersion}:g${gen}:${signature}`;
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1084
|
+
// workers/worker_public/src/ask.ts
|
|
1085
|
+
function userImageDataUrl(body) {
|
|
1086
|
+
const img = body?.image;
|
|
1087
|
+
if (img == null) return null;
|
|
1088
|
+
if (typeof img !== "string" || img.length > 6e6) return null;
|
|
1089
|
+
const m = img.match(/^data:image\/(png|jpe?g|webp|gif);base64,([A-Za-z0-9+/=]+)$/);
|
|
1090
|
+
if (!m || !m[2]) return null;
|
|
1091
|
+
return img;
|
|
1092
|
+
}
|
|
1093
|
+
async function cacheGet(env, gen, ns, query, lang, salt) {
|
|
1094
|
+
const key = exactCacheKey(env.INDEX_VERSION, gen, ns, await sha256Hex(cacheKeyMaterial(query, lang, salt)));
|
|
1095
|
+
const hit = await env.CACHE.get(key, "json");
|
|
1096
|
+
return hit ? { key, value: hit } : null;
|
|
1097
|
+
}
|
|
1098
|
+
function embedWarm(env, text) {
|
|
1099
|
+
return embed(portModelRunner(env), MODELS.embed, text).catch(() => null);
|
|
1100
|
+
}
|
|
1101
|
+
async function attachFigureImages(env, messages, usedHits, query) {
|
|
1102
|
+
const figIntent = /\b(fig(ure)?s?|diagram|drawing|graph|chart)\b/i.test(query);
|
|
1103
|
+
const topProseAnchor = usedHits.find((h) => !h.metadata.unit_id)?.metadata.clause_anchor;
|
|
1104
|
+
const figures = usedHits.filter((h) => h.metadata.unit_id && h.metadata.block === "figure").filter((h) => figIntent || !!h.metadata.clause_anchor && h.metadata.clause_anchor === topProseAnchor).slice(0, 1);
|
|
1105
|
+
if (!figures.length) return;
|
|
1106
|
+
const fig = figures[0];
|
|
1107
|
+
const figAnchor = fig.metadata.clause_anchor ?? "";
|
|
1108
|
+
const figTitle = fig.metadata.clause_title ?? "";
|
|
1109
|
+
const referencing = usedHits.filter((h) => !h.metadata.unit_id && h.metadata.docidentifier === fig.metadata.docidentifier && /\bfig(ure)?s?\.?\s*\d/i.test(h.text ?? "")).slice(0, 2).map((h) => `clause ${h.metadata.clause_anchor ?? ""}${h.metadata.clause_title ? ` (${h.metadata.clause_title})` : ""}: ${(h.text ?? "").slice(0, 400)}`);
|
|
1110
|
+
const parts = [];
|
|
1111
|
+
const names = [];
|
|
1112
|
+
let figCaption = "";
|
|
1113
|
+
for (const h of figures) {
|
|
1114
|
+
try {
|
|
1115
|
+
const row = await env.DB.prepare("SELECT payload FROM unit_payloads WHERE unit_id = ?1").bind(h.metadata.unit_id).first();
|
|
1116
|
+
const payload = row ? JSON.parse(String(row.payload)) : {};
|
|
1117
|
+
const uri = typeof payload.uri === "string" ? payload.uri : "";
|
|
1118
|
+
if (typeof payload.caption === "string" && payload.caption.trim()) figCaption = payload.caption.trim();
|
|
1119
|
+
const m = typeof uri === "string" ? uri.match(/^\/assets\/(.+)/) : null;
|
|
1120
|
+
if (!m) continue;
|
|
1121
|
+
const obj = await env.UNIT_ASSETS.get(m[1]);
|
|
1122
|
+
if (!obj) continue;
|
|
1123
|
+
const buf = new Uint8Array(await obj.arrayBuffer());
|
|
1124
|
+
const ext = m[1].split(".").pop()?.toLowerCase() ?? "png";
|
|
1125
|
+
const mime = ext === "svg" ? "image/svg+xml" : `image/${ext === "jpg" ? "jpeg" : ext}`;
|
|
1126
|
+
let binary = "";
|
|
1127
|
+
for (let i = 0; i < buf.length; i += 8192) binary += String.fromCharCode(...buf.subarray(i, i + 8192));
|
|
1128
|
+
parts.push({ type: "image_url", image_url: { url: `data:${mime};base64,${btoa(binary)}` } });
|
|
1129
|
+
names.push(h.metadata.unit_id);
|
|
1130
|
+
} catch {
|
|
1131
|
+
}
|
|
1132
|
+
}
|
|
1133
|
+
if (!parts.length) return;
|
|
1134
|
+
const context = [];
|
|
1135
|
+
if (figCaption) context.push(`Its caption reads: "${figCaption}".`);
|
|
1136
|
+
if (figAnchor) context.push(`It belongs to clause ${figAnchor}${figTitle ? ` (${figTitle})` : ""} of its publication.`);
|
|
1137
|
+
if (referencing.length) context.push(`The publication's prose references it from \u2014 ${referencing.join(" \u2014 and from \u2014 ")}.`);
|
|
1138
|
+
messages.push({
|
|
1139
|
+
role: "user",
|
|
1140
|
+
content: [
|
|
1141
|
+
{
|
|
1142
|
+
type: "text",
|
|
1143
|
+
text: `The original image of figure unit ${names.join(", ")} is attached; interpret the drawing directly when answering about this figure.` + (context.length ? ` To understand what the figure is doing: ${context.join(" ")}` : "")
|
|
1144
|
+
},
|
|
1145
|
+
...parts
|
|
1146
|
+
]
|
|
1147
|
+
});
|
|
1148
|
+
console.log("figure images attached:", names.join(", "));
|
|
1149
|
+
}
|
|
1150
|
+
async function generateStream(env, model, messages, effort) {
|
|
1151
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
1152
|
+
try {
|
|
1153
|
+
const res = await env.AI.run(model, {
|
|
1154
|
+
messages,
|
|
1155
|
+
stream: true,
|
|
1156
|
+
max_tokens: effortBudget(effort ?? answerEffort(env)),
|
|
1157
|
+
reasoning_effort: effort ?? answerEffort(env),
|
|
1158
|
+
temperature: 0.6,
|
|
1159
|
+
top_p: 0.95
|
|
1160
|
+
});
|
|
1161
|
+
if (res && typeof res.getReader === "function") return res;
|
|
1162
|
+
if (res && res.body && typeof res.body.getReader === "function") return res.body;
|
|
1163
|
+
} catch (e) {
|
|
1164
|
+
console.error("stream failed:", model, String(e).slice(0, 120));
|
|
1165
|
+
}
|
|
1166
|
+
}
|
|
1167
|
+
return null;
|
|
1168
|
+
}
|
|
1169
|
+
async function summarizeHistory(env, model, turns) {
|
|
1170
|
+
try {
|
|
1171
|
+
const convo = turns.map((t) => `${t.role === "user" ? "User" : "Assistant"}: ${t.content.slice(0, 1200)}`).join("\n").slice(0, 24e3);
|
|
1172
|
+
const res = await env.AI.run(model, {
|
|
1173
|
+
messages: [
|
|
1174
|
+
{
|
|
1175
|
+
role: "system",
|
|
1176
|
+
content: summarize_default.trimEnd()
|
|
1177
|
+
},
|
|
1178
|
+
{ role: "user", content: convo }
|
|
1179
|
+
],
|
|
1180
|
+
max_tokens: 2048,
|
|
1181
|
+
reasoning_effort: "low",
|
|
1182
|
+
// Qwen3 thinking-mode sampling (model card) — prevents the
|
|
1183
|
+
// repetition loops that eat the budget before the summary lands
|
|
1184
|
+
temperature: 0.6,
|
|
1185
|
+
top_p: 0.95,
|
|
1186
|
+
top_k: 20
|
|
1187
|
+
});
|
|
1188
|
+
const text = typeof res?.response === "string" ? res.response : res?.choices?.[0]?.message?.content;
|
|
1189
|
+
return typeof text === "string" && text.trim() ? text.trim().slice(0, 1200) : null;
|
|
1190
|
+
} catch {
|
|
1191
|
+
return null;
|
|
1192
|
+
}
|
|
1193
|
+
}
|
|
1194
|
+
async function* sseTokens(stream) {
|
|
1195
|
+
const reader = stream.getReader();
|
|
1196
|
+
const decoder = new TextDecoder();
|
|
1197
|
+
let buf = "";
|
|
1198
|
+
while (true) {
|
|
1199
|
+
const { done, value } = await reader.read();
|
|
1200
|
+
if (done) break;
|
|
1201
|
+
buf += decoder.decode(value, { stream: true });
|
|
1202
|
+
const lines = buf.split("\n");
|
|
1203
|
+
buf = lines.pop() ?? "";
|
|
1204
|
+
for (const line of lines) {
|
|
1205
|
+
const trimmed = line.trim();
|
|
1206
|
+
if (!trimmed.startsWith("data:")) continue;
|
|
1207
|
+
const payload = trimmed.slice(5).trim();
|
|
1208
|
+
if (!payload || payload === "[DONE]") continue;
|
|
1209
|
+
try {
|
|
1210
|
+
const evt = JSON.parse(payload);
|
|
1211
|
+
const tok = typeof evt?.response === "string" ? evt.response : evt?.choices?.[0]?.delta?.content;
|
|
1212
|
+
if (tok) yield tok;
|
|
1213
|
+
} catch {
|
|
1214
|
+
}
|
|
1215
|
+
}
|
|
1216
|
+
}
|
|
1217
|
+
}
|
|
1218
|
+
async function handleAsk(env, ctx, req, tier, key) {
|
|
1219
|
+
const tStart = Date.now();
|
|
1220
|
+
const telemetryMeta = () => ({ durationMs: Date.now() - tStart, keyId: key?.id ?? null });
|
|
1221
|
+
const stageTiming = {};
|
|
1222
|
+
let generateRetries = 0;
|
|
1223
|
+
const readAs = () => understanding ? {
|
|
1224
|
+
intent: understanding.intent,
|
|
1225
|
+
doc: understanding.docidentifier,
|
|
1226
|
+
edition: understanding.edition ?? null,
|
|
1227
|
+
term: understanding.term,
|
|
1228
|
+
terms: (understanding.defined_terms ?? []).slice(0, 4),
|
|
1229
|
+
lang: q?.lang ?? null
|
|
1230
|
+
} : void 0;
|
|
1231
|
+
const serverTiming = () => Object.entries(stageTiming).map(([k, v]) => `${k};dur=${v}`).concat([`generate-retries;desc=count;dur=${generateRetries ?? 0}`, `total;dur=${Date.now() - tStart}`]).join(", ");
|
|
1232
|
+
const body = await readJson(req);
|
|
1233
|
+
const q = validateQuery(body);
|
|
1234
|
+
if (!q) return err(400, "invalid_input", `query is required (1-${LIMITS.maxInputChars} chars)`);
|
|
1235
|
+
const declaredCtx = parseContext(body);
|
|
1236
|
+
const draftAct = P().publisher.features?.drafts ? detectDraftIntent(q.query) : null;
|
|
1237
|
+
const member = tier === "member" ? await sessionFrom(req, env) : null;
|
|
1238
|
+
const effort = requestEffort(env, member, body?.effort);
|
|
1239
|
+
const limit = tier === "key" ? key.day_limit : tier === "member" || member ? num(env, "MEMBER_DAY_ASK", 300) : num(env, "ANON_DAY_ASK", 20);
|
|
1240
|
+
const bucketId = tier === "key" ? `key:${key.id}` : member ? `sub:${member.sub}` : clientIp(req);
|
|
1241
|
+
const quota = await checkQuota(env, "ask", bucketId, limit, effort === "low" ? 1 : 2);
|
|
1242
|
+
if (!quota.ok) {
|
|
1243
|
+
return err(429, "quota_exceeded", `Daily question limit reached (${quota.limit}). Try again tomorrow.`);
|
|
1244
|
+
}
|
|
1245
|
+
const hardCap = num(env, "ANON_DAY_HARD_CAP", 5e3);
|
|
1246
|
+
if (tier === "anon" && quota.used > hardCap) {
|
|
1247
|
+
return err(503, "generation_disabled", "Generation is temporarily paused; search remains available.");
|
|
1248
|
+
}
|
|
1249
|
+
if (await env.CACHE.get("sys:generation") === "off") {
|
|
1250
|
+
return err(503, "generation_disabled", "Generation is temporarily paused; search remains available.");
|
|
1251
|
+
}
|
|
1252
|
+
const service = env.INTERNAL_SERVICE;
|
|
1253
|
+
const fedAuth = {
|
|
1254
|
+
cookie: req.headers.get("cookie") ?? "",
|
|
1255
|
+
authorization: req.headers.get("authorization") ?? ""
|
|
1256
|
+
};
|
|
1257
|
+
const scope = resolveRequestScope(body, member);
|
|
1258
|
+
if ("error" in scope) return err(400, "invalid_input", "datasets: at least one dataset must stay enabled");
|
|
1259
|
+
const { corpora, narrowed, isoOn } = scope;
|
|
1260
|
+
const [memNote, memoryUsed] = member && scope.memoryIds.length ? await memoryNote(env, member.sub, scope.memoryIds) : [null, []];
|
|
1261
|
+
const requestSaltStr = requestSalt(scope, memoryUsed);
|
|
1262
|
+
const salt = requestSaltStr ? `${requestSaltStr}|effort:${effort}` : `effort:${effort}`;
|
|
1263
|
+
const federate = member && service && isoOn ? (q2) => retrieveInternal(service, fedAuth, q2) : void 0;
|
|
1264
|
+
const ns = tier === "key" ? `k:${key.id}` : member ? `m:${member.sub}` : "anon";
|
|
1265
|
+
const model = member ? MODELS.member : MODELS.anon;
|
|
1266
|
+
const prev = typeof body?.prev === "string" ? body.prev.slice(0, 800) : void 0;
|
|
1267
|
+
const rawHistory = Array.isArray(body?.history) ? body.history : [];
|
|
1268
|
+
const history = rawHistory.filter((h) => (h?.role === "user" || h?.role === "assistant") && typeof h?.content === "string" && h.content.trim()).slice(-24).map((h) => ({ role: h.role, content: h.content.slice(0, 4e3) }));
|
|
1269
|
+
const contextual = history.length > 0;
|
|
1270
|
+
const userImage = body?.image != null ? userImageDataUrl(body) : null;
|
|
1271
|
+
if (body?.image != null && !userImage) {
|
|
1272
|
+
return err(400, "invalid_image", "image must be a data URL (data:image/png|jpeg|webp|gif;base64,\u2026) up to 6 MB");
|
|
1273
|
+
}
|
|
1274
|
+
const budget = num(env, "INPUT_TOKEN_BUDGET", LIMITS.inputTokenBudget);
|
|
1275
|
+
const { kept: keptHistory, overflow } = splitHistory(history, budget);
|
|
1276
|
+
const summary = overflow.length >= 2 ? await summarizeHistory(env, MODELS.understand, overflow) ?? void 0 : void 0;
|
|
1277
|
+
let retrieved;
|
|
1278
|
+
const fresh = freshRequested(body);
|
|
1279
|
+
const gen = await corpusGen(env.CACHE);
|
|
1280
|
+
const cached = fresh || contextual || declaredCtx || draftAct || userImage ? null : await cacheGet(env, gen, ns, q.query, q.lang, salt);
|
|
1281
|
+
const wantsStream = body?.stream === true || tier === "anon" && body?.stream !== false;
|
|
1282
|
+
if (cached) {
|
|
1283
|
+
telemetry(env, ctx, tier, "ask", null, true, (cached.value.answer ?? "").length, cached.value.query_hash, q.lang, "exact", telemetryMeta());
|
|
1284
|
+
const cctx = cached.value.context_applied ?? NO_CONTEXT;
|
|
1285
|
+
if (wantsStream) {
|
|
1286
|
+
return sseResponse([{ type: "citations", citations: cached.value.citations ?? [], quota, context_applied: cctx }, { type: "token", v: cached.value.answer ?? "" }, { type: "done", model: cached.value.model ?? MODELS.member, query_hash: cached.value.query_hash, served_from: "cache", context_applied: cctx }], corsHeaders(req));
|
|
1287
|
+
}
|
|
1288
|
+
return json({ ...cached.value, cached: true, quota, context_applied: cctx });
|
|
1289
|
+
}
|
|
1290
|
+
const warmQuery = retrievalQuery(q.query, prev);
|
|
1291
|
+
const warmEmbed = embedWarm(env, warmQuery);
|
|
1292
|
+
const conversationId = typeof body?.conversation_id === "string" && /^[A-Za-z0-9_-]{8,64}$/.test(body.conversation_id) ? body.conversation_id : null;
|
|
1293
|
+
let convEntities = [];
|
|
1294
|
+
if (conversationId) {
|
|
1295
|
+
try {
|
|
1296
|
+
const rows = await env.DB.prepare("SELECT entity, kind FROM conversation_entities WHERE conversation_id = ?1 LIMIT 12").bind(conversationId).all();
|
|
1297
|
+
convEntities = rows.results ?? [];
|
|
1298
|
+
if (convEntities.length) console.log("entity map:", convEntities.length, "entries");
|
|
1299
|
+
} catch {
|
|
1300
|
+
}
|
|
1301
|
+
}
|
|
1302
|
+
let understanding = null;
|
|
1303
|
+
const nodeScoped = !!modelNodeRefIn(q.query) || !!modelNodeRefIn(declaredCtx?.label);
|
|
1304
|
+
if (!cached && !nodeScoped && !contextual && !declaredCtx && !draftAct && !q.lang && !userImage && !fresh) {
|
|
1305
|
+
const wv0 = await warmEmbed ?? null;
|
|
1306
|
+
if (wv0) {
|
|
1307
|
+
const sc0 = await semanticCacheGet(env, gen, wv0, salt);
|
|
1308
|
+
if (sc0) {
|
|
1309
|
+
console.log("semantic cache hit (pre-understanding)");
|
|
1310
|
+
telemetry(env, ctx, tier, "ask", null, true, sc0.answer.length, sc0.query_hash, q.lang, "semantic", telemetryMeta());
|
|
1311
|
+
const cctx0 = sc0.context_applied ?? NO_CONTEXT;
|
|
1312
|
+
if (wantsStream) {
|
|
1313
|
+
return sseResponse([{ type: "citations", citations: sc0.citations ?? [], context_applied: cctx0 }, { type: "token", v: sc0.answer }, { type: "done", model: sc0.model, query_hash: sc0.query_hash, similar: true, served_from: "similar", context_applied: cctx0 }], corsHeaders(req));
|
|
1314
|
+
}
|
|
1315
|
+
return json({ ...sc0, similar: true, context_applied: cctx0, quota });
|
|
1316
|
+
}
|
|
1317
|
+
}
|
|
1318
|
+
}
|
|
1319
|
+
let optimisticVec = null;
|
|
1320
|
+
let optimisticHits = [];
|
|
1321
|
+
const t0 = Date.now();
|
|
1322
|
+
if (!cached) {
|
|
1323
|
+
const understandingP = understandQuery(portModelRunner(env), roleModel(env, "understand"), q.query, history, convEntities);
|
|
1324
|
+
try {
|
|
1325
|
+
optimisticVec = await warmEmbed ?? null;
|
|
1326
|
+
if (optimisticVec) {
|
|
1327
|
+
const ores = await env.VECTORIZE.query(optimisticVec, { topK: LIMITS.retrieveK, returnMetadata: "all" });
|
|
1328
|
+
optimisticHits = (ores.matches ?? []).map((m) => ({
|
|
1329
|
+
id: m.id,
|
|
1330
|
+
score: m.score,
|
|
1331
|
+
metadata: m.metadata,
|
|
1332
|
+
text: m.metadata?.chunk_text ?? ""
|
|
1333
|
+
}));
|
|
1334
|
+
}
|
|
1335
|
+
} catch {
|
|
1336
|
+
}
|
|
1337
|
+
understanding = await understandingP;
|
|
1338
|
+
stageTiming.understand = Date.now() - t0;
|
|
1339
|
+
console.log("stage: understand+optimistic", Date.now() - t0, "ms");
|
|
1340
|
+
}
|
|
1341
|
+
const docScope = declaredCtx && declaredCtx.kind !== "account" ? await resolveDocScope(env, declaredCtx) : null;
|
|
1342
|
+
const named = declaredCtx && declaredCtx.kind !== "account" ? namedDocumentIn(q.query) : null;
|
|
1343
|
+
let ctxApplied;
|
|
1344
|
+
let declaredScoped = false;
|
|
1345
|
+
if (!declaredCtx) {
|
|
1346
|
+
ctxApplied = NO_CONTEXT;
|
|
1347
|
+
const bare = understanding?.process_intent ? null : namedDocumentIn(q.query);
|
|
1348
|
+
if (bare && understanding?.doc_number !== bare.doc_number) {
|
|
1349
|
+
understanding = {
|
|
1350
|
+
...understanding ?? syntheticUnderstanding(bare),
|
|
1351
|
+
docidentifier: bare.label,
|
|
1352
|
+
doc_number: bare.doc_number,
|
|
1353
|
+
edition: bare.edition ?? understanding?.edition ?? null
|
|
1354
|
+
};
|
|
1355
|
+
console.log("question names", bare.label, "\u2014 scoping retrieval from the text");
|
|
1356
|
+
}
|
|
1357
|
+
} else if (declaredCtx.kind === "account") {
|
|
1358
|
+
ctxApplied = appliedContext(declaredCtx, null);
|
|
1359
|
+
} else if (docScope && (!named || named.doc_number === docScope.doc_number)) {
|
|
1360
|
+
if (understanding?.doc_number && understanding.doc_number !== docScope.doc_number) {
|
|
1361
|
+
console.log("context scope: understand's doc#" + understanding.doc_number, "is inferred, not named in the question \u2014 the declared", docScope.label, "scopes");
|
|
1362
|
+
}
|
|
1363
|
+
understanding = {
|
|
1364
|
+
...understanding ?? syntheticUnderstanding(docScope),
|
|
1365
|
+
docidentifier: docScope.label,
|
|
1366
|
+
doc_number: docScope.doc_number,
|
|
1367
|
+
edition: docScope.edition ?? understanding?.edition ?? null
|
|
1368
|
+
};
|
|
1369
|
+
ctxApplied = appliedContext(declaredCtx, docScope);
|
|
1370
|
+
declaredScoped = true;
|
|
1371
|
+
console.log("context scope:", docScope.label, `(${declaredCtx.kind})`);
|
|
1372
|
+
} else if (docScope && named) {
|
|
1373
|
+
if (understanding?.doc_number !== named.doc_number) {
|
|
1374
|
+
understanding = {
|
|
1375
|
+
...understanding ?? syntheticUnderstanding(named),
|
|
1376
|
+
docidentifier: named.label,
|
|
1377
|
+
doc_number: named.doc_number,
|
|
1378
|
+
edition: named.edition ?? null
|
|
1379
|
+
};
|
|
1380
|
+
}
|
|
1381
|
+
ctxApplied = appliedContext(declaredCtx, null, "question-document-wins");
|
|
1382
|
+
console.log("context scope: the question names", named.label, "\u2014 it wins over the declared", docScope.label);
|
|
1383
|
+
} else if (declaredCtx.doc) {
|
|
1384
|
+
ctxApplied = appliedContext(declaredCtx, null, "document-not-in-corpus");
|
|
1385
|
+
console.log("context scope:", declaredCtx.doc, "not in the corpus \u2014 the general corpus answers");
|
|
1386
|
+
} else {
|
|
1387
|
+
ctxApplied = appliedContext(declaredCtx, null);
|
|
1388
|
+
}
|
|
1389
|
+
if (conversationId && understanding) {
|
|
1390
|
+
const now = Date.now();
|
|
1391
|
+
const ents = [];
|
|
1392
|
+
if (understanding.docidentifier) ents.push([understanding.docidentifier, "document"]);
|
|
1393
|
+
for (const t of understanding.defined_terms) ents.push([t, "term"]);
|
|
1394
|
+
if (ents.length) {
|
|
1395
|
+
const upsert = (e, k) => env.DB.prepare("INSERT OR REPLACE INTO conversation_entities (conversation_id, entity, kind, ts) VALUES (?1, ?2, ?3, ?4)").bind(conversationId, e, k, now).run();
|
|
1396
|
+
ctx.waitUntil(Promise.allSettled(ents.map(([e, k]) => upsert(e, k))));
|
|
1397
|
+
}
|
|
1398
|
+
}
|
|
1399
|
+
console.log("understand:", understanding?.intent ?? "null", understanding?.doc_number ? `doc#${understanding.doc_number}${understanding.edition ? "@" + understanding.edition : ""}` : "nodoc", "|", q.query.slice(0, 50));
|
|
1400
|
+
const graphDocNumbers = await graphExpand(env, understanding);
|
|
1401
|
+
const eNote = await editionNote(env, understanding);
|
|
1402
|
+
if (understanding?.intent !== "conversational" && !nodeScoped && !contextual && !declaredCtx && !draftAct && !userImage && !fresh) {
|
|
1403
|
+
const warmVec = await warmEmbed ?? null;
|
|
1404
|
+
if (warmVec) {
|
|
1405
|
+
const sc = await semanticCacheGet(env, gen, warmVec, salt);
|
|
1406
|
+
if (sc) {
|
|
1407
|
+
console.log("semantic cache hit");
|
|
1408
|
+
telemetry(env, ctx, tier, "ask", null, true, sc.answer.length, sc.query_hash, q.lang, "semantic", telemetryMeta());
|
|
1409
|
+
const cctx = sc.context_applied ?? NO_CONTEXT;
|
|
1410
|
+
if (wantsStream) {
|
|
1411
|
+
return sseResponse([{ type: "citations", citations: sc.citations ?? [], context_applied: cctx }, { type: "token", v: sc.answer }, { type: "done", model: sc.model, query_hash: sc.query_hash, similar: true, served_from: "similar", context_applied: cctx }], corsHeaders(req));
|
|
1412
|
+
}
|
|
1413
|
+
return json({ ...sc, similar: true, context_applied: cctx, quota });
|
|
1414
|
+
}
|
|
1415
|
+
}
|
|
1416
|
+
}
|
|
1417
|
+
if (understanding?.intent === "conversational") {
|
|
1418
|
+
const queryHash2 = await sha256Hex(q.query);
|
|
1419
|
+
const messages2 = [
|
|
1420
|
+
{ role: "system", content: identityNote(!!member) },
|
|
1421
|
+
...summary ? [{ role: "system", content: `Earlier in this conversation (summarized for continuity):
|
|
1422
|
+
${summary}` }] : [],
|
|
1423
|
+
...keptHistory.slice(-6),
|
|
1424
|
+
{ role: "user", content: q.query }
|
|
1425
|
+
];
|
|
1426
|
+
if (wantsStream) {
|
|
1427
|
+
const stream = await generateStream(env, model, messages2, effort);
|
|
1428
|
+
if (stream) {
|
|
1429
|
+
const encoder = new TextEncoder();
|
|
1430
|
+
const sse = new ReadableStream({
|
|
1431
|
+
async start(controller) {
|
|
1432
|
+
const send = (obj) => controller.enqueue(encoder.encode(`data: ${JSON.stringify(obj)}
|
|
1433
|
+
|
|
1434
|
+
`));
|
|
1435
|
+
send({ type: "citations", citations: [], context_applied: NO_CONTEXT, quota });
|
|
1436
|
+
let full = "";
|
|
1437
|
+
try {
|
|
1438
|
+
for await (const tok of sseTokens(stream)) {
|
|
1439
|
+
full += tok;
|
|
1440
|
+
send({ type: "token", v: tok });
|
|
1441
|
+
}
|
|
1442
|
+
} catch {
|
|
1443
|
+
}
|
|
1444
|
+
send({ type: "done", model, query_hash: queryHash2, context_applied: NO_CONTEXT });
|
|
1445
|
+
telemetry(env, ctx, tier, "ask", model, true, full.length, queryHash2, q.lang, void 0, telemetryMeta());
|
|
1446
|
+
controller.close();
|
|
1447
|
+
}
|
|
1448
|
+
});
|
|
1449
|
+
return new Response(sse, {
|
|
1450
|
+
headers: { "content-type": "text/event-stream", "cache-control": "no-cache", "x-accel-buffering": "no", ...corsHeaders(req) }
|
|
1451
|
+
});
|
|
1452
|
+
}
|
|
1453
|
+
}
|
|
1454
|
+
let answer2 = await generateOnce(env, model, messages2, effort);
|
|
1455
|
+
if (answer2 === null) answer2 = await generateOnce(env, MODELS.fallback, messages2, effort);
|
|
1456
|
+
if (answer2 === null) {
|
|
1457
|
+
telemetry(env, ctx, tier, "ask", model, false, 0, queryHash2, q.lang, void 0, telemetryMeta());
|
|
1458
|
+
return err(502, "generation_failed", "The generation model is unavailable; please retry.");
|
|
1459
|
+
}
|
|
1460
|
+
telemetry(env, ctx, tier, "ask", model, true, answer2.length, queryHash2, q.lang, void 0, telemetryMeta());
|
|
1461
|
+
return json({ answer: answer2, citations: [], model, query_hash: queryHash2, follow_ups: [], context_applied: NO_CONTEXT, quota });
|
|
1462
|
+
}
|
|
1463
|
+
if (draftAct) {
|
|
1464
|
+
const draftCtxApplied = declaredCtx ? appliedContext(declaredCtx, null) : NO_CONTEXT;
|
|
1465
|
+
const queryHash2 = await sha256Hex(q.query);
|
|
1466
|
+
const liveCfg = liveDataConfig(env);
|
|
1467
|
+
const sessionRaw = rawSessionToken(req);
|
|
1468
|
+
let delegation;
|
|
1469
|
+
if (!member || !sessionRaw) delegation = { status: "unsigned" };
|
|
1470
|
+
else if (!liveCfg) delegation = { status: "not_configured" };
|
|
1471
|
+
else {
|
|
1472
|
+
const exchanged = await exchangeForLiveToken(env, sessionRaw);
|
|
1473
|
+
delegation = exchanged.ok ? { status: "ok", token: exchanged.token } : { status: exchanged.reason };
|
|
1474
|
+
}
|
|
1475
|
+
const verdict = await prepareDraft(env, {
|
|
1476
|
+
act: draftAct,
|
|
1477
|
+
query: q.query,
|
|
1478
|
+
history: keptHistory,
|
|
1479
|
+
member,
|
|
1480
|
+
delegation,
|
|
1481
|
+
platformClientId: liveCfg?.platformClientId,
|
|
1482
|
+
model: roleModel(env, "understand")
|
|
1483
|
+
});
|
|
1484
|
+
console.log("draft act:", draftAct, "\u2192", verdict.status === "draft" ? `draft (${Object.keys(verdict.draft.fields).length} fields)` : `refused (${verdict.reason})`);
|
|
1485
|
+
const citations2 = verdict.citation ? [{ ...verdict.citation, corpus: P().publisher.id }] : [];
|
|
1486
|
+
const draftPayload = verdict.status === "draft" ? verdict.draft : void 0;
|
|
1487
|
+
telemetry(env, ctx, tier, "ask", model, true, verdict.answer.length, queryHash2, q.lang, void 0, telemetryMeta());
|
|
1488
|
+
if (wantsStream) {
|
|
1489
|
+
return sseResponse(
|
|
1490
|
+
[
|
|
1491
|
+
{ type: "citations", citations: citations2, context_applied: draftCtxApplied, ...draftPayload ? { draft: draftPayload } : {}, quota },
|
|
1492
|
+
{ type: "token", v: verdict.answer },
|
|
1493
|
+
{ type: "done", model, query_hash: queryHash2, context_applied: draftCtxApplied }
|
|
1494
|
+
],
|
|
1495
|
+
corsHeaders(req)
|
|
1496
|
+
);
|
|
1497
|
+
}
|
|
1498
|
+
return json({ answer: verdict.answer, citations: citations2, model, query_hash: queryHash2, follow_ups: [], context_applied: draftCtxApplied, ...draftPayload ? { draft: draftPayload } : {}, quota });
|
|
1499
|
+
}
|
|
1500
|
+
let liveRecords;
|
|
1501
|
+
let accountNote;
|
|
1502
|
+
const modelDocHint = named ?? docScope ?? namedDocumentIn(q.query);
|
|
1503
|
+
const boundModel = P().publisher.features?.model_plane ? await bindModelNode(env, {
|
|
1504
|
+
label: declaredCtx?.label,
|
|
1505
|
+
query: q.query,
|
|
1506
|
+
standard: standardForDocNumber(modelDocHint?.doc_number)
|
|
1507
|
+
}) : null;
|
|
1508
|
+
if (boundModel) {
|
|
1509
|
+
ctxApplied = { ...ctxApplied, model: modelEcho(boundModel) };
|
|
1510
|
+
console.log("model plane: bound", boundModel.node_id, `[${boundModel.standard}]`, boundModel.clause?.urn ?? "no-clause");
|
|
1511
|
+
}
|
|
1512
|
+
const modelNote = boundModel ? modelGroundingBlock(boundModel) : void 0;
|
|
1513
|
+
const machineVerdict = boundModel ? evaluate(boundModel.content, q.query) : null;
|
|
1514
|
+
const machineNote = machineVerdict && boundModel ? verdictNote(machineVerdict, boundModel) : void 0;
|
|
1515
|
+
const verdictBlock = machineVerdict ? {
|
|
1516
|
+
unit_id: boundModel.node_id,
|
|
1517
|
+
type: "verdict",
|
|
1518
|
+
docidentifier: `${P().publisher.name} SMART model (${boundModel.standard})`,
|
|
1519
|
+
payload: {
|
|
1520
|
+
verdict: machineVerdict.verdict,
|
|
1521
|
+
on_violation: machineVerdict.on_violation,
|
|
1522
|
+
violation_meaning: machineVerdict.violation_meaning,
|
|
1523
|
+
missing: machineVerdict.missing,
|
|
1524
|
+
checks: machineVerdict.checks
|
|
1525
|
+
}
|
|
1526
|
+
} : null;
|
|
1527
|
+
if (machineVerdict) console.log("verdict engine:", boundModel.node_id, "\u2192", machineVerdict.verdict.toUpperCase(), machineVerdict.missing.length ? `(missing ${machineVerdict.missing.join(",")})` : "");
|
|
1528
|
+
try {
|
|
1529
|
+
const tR = Date.now();
|
|
1530
|
+
if (declaredCtx?.kind === "account") {
|
|
1531
|
+
const live = await resolveLiveAccount(env, rawSessionToken(req), member);
|
|
1532
|
+
if (live.status === "ok") {
|
|
1533
|
+
liveRecords = live.records;
|
|
1534
|
+
ctxApplied = appliedContext(declaredCtx, null, void 0, {
|
|
1535
|
+
read_at: live.readAt,
|
|
1536
|
+
stores: live.stores,
|
|
1537
|
+
records: live.records.length
|
|
1538
|
+
});
|
|
1539
|
+
const lines = live.records.map(
|
|
1540
|
+
(r) => `- ${r.label} [${[r.status, r.detail].filter(Boolean).join("; ")}] ${r.url}`
|
|
1541
|
+
);
|
|
1542
|
+
accountNote = `Live account data (read ${live.readAt} from ${P().prompts.vars.account_note_source ?? `the user's own ${P().publisher.product_name} account`} \u2014 exactly what they may see, never more):
|
|
1543
|
+
` + (lines.length ? lines.join("\n") : "(the account surfaces answered empty)") + `
|
|
1544
|
+
Answer account questions from these records ONLY: name the record when you use it, never invent one, and say honestly when they do not hold the answer. The corpus passages still ground the regulatory claims (the requirements, the procedures); the records are the user's own work.`;
|
|
1545
|
+
console.log("live data:", live.records.length, "records from", live.stores.join("+") || "none");
|
|
1546
|
+
} else {
|
|
1547
|
+
const note = live.reason === "sign_in_required" ? "sign-in-required" : live.reason === "window_expired" ? "live-window-expired" : "live-unavailable";
|
|
1548
|
+
ctxApplied = appliedContext(declaredCtx, null, note);
|
|
1549
|
+
accountNote = live.reason === "sign_in_required" ? "Context note: the user asked with the 'my account' context but is not signed in \u2014 the account data was NOT read; answer from the corpus and say so." : live.reason === "window_expired" ? "Context note: the user's live access window lapsed \u2014 the account data was NOT read; answer from the corpus, say the live read did not happen, and suggest signing in again to refresh it." : "Context note: the live account read was refused or unreachable \u2014 the account data was NOT read; answer from the corpus and say so honestly.";
|
|
1550
|
+
console.log("live data: not read \u2014", live.reason);
|
|
1551
|
+
}
|
|
1552
|
+
}
|
|
1553
|
+
retrieved = await retrieve(env, q.query, {
|
|
1554
|
+
prev,
|
|
1555
|
+
understanding,
|
|
1556
|
+
federate,
|
|
1557
|
+
warmEmbed,
|
|
1558
|
+
graphDocNumbers,
|
|
1559
|
+
sealScope: declaredScoped ? docScope : null,
|
|
1560
|
+
optimisticHits,
|
|
1561
|
+
optimisticVec,
|
|
1562
|
+
datasetScope: narrowed ? corpora : null
|
|
1563
|
+
});
|
|
1564
|
+
stageTiming["retrieve-core"] = Date.now() - tR;
|
|
1565
|
+
console.log("stage: retrieve", Date.now() - tR, "ms");
|
|
1566
|
+
const docScoped = !!understanding?.doc_number;
|
|
1567
|
+
const gradePromise = docScoped ? Promise.resolve("skipped-doc-scoped") : (() => {
|
|
1568
|
+
const tg = Date.now();
|
|
1569
|
+
return gradeRetrieval(env.AI, roleModel(env, "grader"), q.query, retrieved.hits.map((h) => h.text)).catch(() => null).finally(() => stageTiming["grade"] = Date.now() - tg);
|
|
1570
|
+
})();
|
|
1571
|
+
if (retrieved.hits.length >= 4 && (member || understanding?.complexity === "complex")) {
|
|
1572
|
+
const tl = Date.now();
|
|
1573
|
+
const reordered = await listwiseRerank(env, MODELS.listwise, understanding?.standalone_query || q.query, retrieved.hits);
|
|
1574
|
+
stageTiming.listwise = Date.now() - tl;
|
|
1575
|
+
if (reordered) {
|
|
1576
|
+
console.log("listwise: reordered", reordered[0]?.metadata?.docidentifier ?? "?", "to top");
|
|
1577
|
+
retrieved = { ...retrieved, hits: reordered };
|
|
1578
|
+
}
|
|
1579
|
+
}
|
|
1580
|
+
const grade = await gradePromise;
|
|
1581
|
+
stageTiming.retrieve = Date.now() - tR;
|
|
1582
|
+
console.log("stage: grade+listwise", Date.now() - tR, "ms since retrieve start | grade:", grade);
|
|
1583
|
+
if (grade === "weak" && understanding?.docidentifier) {
|
|
1584
|
+
const broaden = `${understanding.standalone_query || q.query} ${understanding.docidentifier}`.trim();
|
|
1585
|
+
const tc = Date.now();
|
|
1586
|
+
const second = await retrieve(env, q.query, { prev, understanding, queryOverride: broaden, federate, datasetScope: narrowed ? corpora : null });
|
|
1587
|
+
const grade2 = await gradeRetrieval(env.AI, roleModel(env, "grader"), q.query, second.hits.map((h) => h.text));
|
|
1588
|
+
stageTiming.corrective = Date.now() - tc;
|
|
1589
|
+
if (grade2 === "good") retrieved = second;
|
|
1590
|
+
}
|
|
1591
|
+
} catch (e) {
|
|
1592
|
+
console.log("ask: retrieval failed:", String(e).slice(0, 300));
|
|
1593
|
+
telemetry(env, ctx, tier, "ask", MODELS.embed, false, 0, await sha256Hex(q.query), q.lang, void 0, telemetryMeta());
|
|
1594
|
+
return err(503, "retrieval_unavailable", "Search is briefly busy \u2014 please retry in a moment.");
|
|
1595
|
+
}
|
|
1596
|
+
const { hits } = retrieved;
|
|
1597
|
+
if (hits.length === 0 && !liveRecords?.length && !boundModel) {
|
|
1598
|
+
const answer2 = refusalAnswer();
|
|
1599
|
+
const out2 = { answer: answer2, citations: [], model, query_hash: await sha256Hex(q.query), context_applied: ctxApplied };
|
|
1600
|
+
telemetry(env, ctx, tier, "ask", model, true, answer2.length, out2.query_hash, q.lang, void 0, telemetryMeta());
|
|
1601
|
+
return json({ ...out2, quota });
|
|
1602
|
+
}
|
|
1603
|
+
const processNote = understanding?.process_intent ? P().retrieval.process_note : void 0;
|
|
1604
|
+
const glossaryForNote = (() => {
|
|
1605
|
+
const g = retrieved.glossary ?? [];
|
|
1606
|
+
if (!g.length) return g;
|
|
1607
|
+
const dt = (understanding?.defined_terms ?? []).map((s) => s.toLowerCase());
|
|
1608
|
+
if (!dt.length) return g;
|
|
1609
|
+
const matched = g.filter((x) => dt.some((d) => x.term.toLowerCase().includes(d.split(" ")[0]) || d.includes(x.term.toLowerCase().split(" ")[0])));
|
|
1610
|
+
return matched.length ? matched : g;
|
|
1611
|
+
})();
|
|
1612
|
+
const vocabNote = glossaryForNote.length ? "Vocabulary binding \u2014 defined terms in the indexed corpus that may name this question's subject:\n" + glossaryForNote.map((g) => `- ${g.term} (${g.docidentifier}): ${g.definition}`).join("\n") + "\nIf the question describes a symptom or behavior in everyday words, OPEN the answer by naming the matching defined term, quote its definition, and cite its defining publication; keep using that term throughout. Match TIME SCALE carefully: change under a constant load over minutes/hours is creep; change over months/years of use is span stability or durability \u2014 do not call long-term drift creep." : void 0;
|
|
1613
|
+
const { messages, usedHits } = buildMessages(
|
|
1614
|
+
q.query,
|
|
1615
|
+
hits,
|
|
1616
|
+
q.lang,
|
|
1617
|
+
keptHistory,
|
|
1618
|
+
// stage-extracted graph facts (GraphRAG) ride the same note channel
|
|
1619
|
+
[processNote, eNote, contextNote(declaredCtx, docScope), accountNote, modelNote, vocabNote, memNote, machineNote, ...retrieved.notes ?? []].filter(Boolean).join("\n") || void 0,
|
|
1620
|
+
summary,
|
|
1621
|
+
budget
|
|
1622
|
+
);
|
|
1623
|
+
await attachFigureImages(env, messages, usedHits, q.query);
|
|
1624
|
+
if (userImage) {
|
|
1625
|
+
const last = messages[messages.length - 1];
|
|
1626
|
+
const note = "\n\n(The user attached an image with this question; interpret it directly when answering.)";
|
|
1627
|
+
if (Array.isArray(last.content)) {
|
|
1628
|
+
const textPart = last.content.find((p) => p.type === "text");
|
|
1629
|
+
if (textPart) textPart.text += note;
|
|
1630
|
+
last.content = [...last.content, { type: "image_url", image_url: { url: userImage } }];
|
|
1631
|
+
} else {
|
|
1632
|
+
last.content = [
|
|
1633
|
+
{ type: "text", text: last.content + note },
|
|
1634
|
+
{ type: "image_url", image_url: { url: userImage } }
|
|
1635
|
+
];
|
|
1636
|
+
}
|
|
1637
|
+
console.log("user image attached to generation");
|
|
1638
|
+
}
|
|
1639
|
+
const queryHash = await sha256Hex(q.query);
|
|
1640
|
+
const cites = boundModel ? [modelCitation(boundModel), ...citations(usedHits)] : citations(usedHits);
|
|
1641
|
+
if (wantsStream) {
|
|
1642
|
+
const stream = await generateStream(env, model, messages, effort);
|
|
1643
|
+
if (stream) {
|
|
1644
|
+
const encoder = new TextEncoder();
|
|
1645
|
+
const sse = new ReadableStream({
|
|
1646
|
+
async start(controller) {
|
|
1647
|
+
const send = (obj) => controller.enqueue(encoder.encode(`data: ${JSON.stringify(obj)}
|
|
1648
|
+
|
|
1649
|
+
`));
|
|
1650
|
+
send({ type: "citations", citations: cites, context_applied: ctxApplied, ...liveRecords ? { records: liveRecords } : {}, quota });
|
|
1651
|
+
let full = "";
|
|
1652
|
+
try {
|
|
1653
|
+
for await (const tok of sseTokens(stream)) {
|
|
1654
|
+
full += tok;
|
|
1655
|
+
send({ type: "token", v: tok });
|
|
1656
|
+
}
|
|
1657
|
+
} catch {
|
|
1658
|
+
}
|
|
1659
|
+
const canonical0 = canonicalRefusal(full);
|
|
1660
|
+
const c2 = canonical0.includes(refusalAnswer()) ? { text: canonical0, blocks: [], dropped: [] } : await contractV2(env.DB, canonical0, usedHits);
|
|
1661
|
+
send({
|
|
1662
|
+
type: "done",
|
|
1663
|
+
model,
|
|
1664
|
+
query_hash: queryHash,
|
|
1665
|
+
follow_ups: understanding?.follow_ups ?? [],
|
|
1666
|
+
blocks: verdictBlock ? [...c2.blocks, verdictBlock] : c2.blocks,
|
|
1667
|
+
context_applied: ctxApplied,
|
|
1668
|
+
read: readAs(),
|
|
1669
|
+
// the evidence view's ground truth: the exact passages this
|
|
1670
|
+
// answer was built from, compact — cache hits carry none,
|
|
1671
|
+
// because the cache stores the answer and never the passages
|
|
1672
|
+
passages: usedHits.slice(0, 8).map((h) => ({ d: h.metadata.docidentifier ?? "", a: h.metadata.clause_anchor ?? "", t: (h.text ?? "").slice(0, 600), ...h.metadata.table_selection ? { s: h.metadata.table_selection } : {} }))
|
|
1673
|
+
});
|
|
1674
|
+
telemetry(env, ctx, tier, "ask", model, true, c2.text.length, queryHash, q.lang, void 0, telemetryMeta());
|
|
1675
|
+
const canonical = c2.text;
|
|
1676
|
+
const streamedAnchors = checkQuoteAnchors(canonical, usedHits.map((h) => h.text));
|
|
1677
|
+
const streamedRetyped = tableRetyped(canonical, usedHits.some((h) => h.metadata.unit_id && h.metadata.block === "table"));
|
|
1678
|
+
const streamed = { total: streamedAnchors.total, violations: streamedRetyped ? ["table-retyped"] : streamedAnchors.violations };
|
|
1679
|
+
if (streamed.violations.length > 0) {
|
|
1680
|
+
console.log("anchors:", streamed.violations.length, "of", streamed.total, "unverified \u2014 not caching");
|
|
1681
|
+
}
|
|
1682
|
+
if (streamed.violations.length === 0 && canonical.length > 0 && !contextual && !declaredCtx && !canonical.includes(refusalAnswer())) {
|
|
1683
|
+
const wv = await warmEmbed ?? null;
|
|
1684
|
+
if (wv) semanticCachePut(env, ctx, gen, wv, salt, { answer: canonical, citations: cites, model, query_hash: queryHash });
|
|
1685
|
+
ctx.waitUntil(
|
|
1686
|
+
env.CACHE.put(exactCacheKey(env.INDEX_VERSION, gen, ns, await sha256Hex(cacheKeyMaterial(q.query, q.lang, salt))), JSON.stringify({ answer: canonical, citations: cites, model, query_hash: queryHash }), { expirationTtl: LIMITS.cacheTtlSec })
|
|
1687
|
+
);
|
|
1688
|
+
}
|
|
1689
|
+
controller.close();
|
|
1690
|
+
}
|
|
1691
|
+
});
|
|
1692
|
+
return new Response(sse, {
|
|
1693
|
+
headers: {
|
|
1694
|
+
"content-type": "text/event-stream",
|
|
1695
|
+
"cache-control": "no-cache",
|
|
1696
|
+
"x-accel-buffering": "no",
|
|
1697
|
+
...corsHeaders(req)
|
|
1698
|
+
}
|
|
1699
|
+
});
|
|
1700
|
+
}
|
|
1701
|
+
}
|
|
1702
|
+
const tGen = Date.now();
|
|
1703
|
+
let answer = await generateOnce(env, model, messages, effort);
|
|
1704
|
+
if (answer === null) {
|
|
1705
|
+
generateRetries += 1;
|
|
1706
|
+
const isFigureAttachMessage = (m) => Array.isArray(m.content) && m.content.some((part) => part?.type === "text" && /^The original image of figure unit /.test(part.text ?? ""));
|
|
1707
|
+
const flat = messages.filter((m) => !isFigureAttachMessage(m)).map(
|
|
1708
|
+
(m) => typeof m.content === "string" ? m : { ...m, content: m.content.filter((p) => p?.type === "text").map((p) => (p?.text ?? "").replace(/\n?\(The user attached an image with this question; interpret it directly when answering\.\)/, "")).join("\n") }
|
|
1709
|
+
);
|
|
1710
|
+
answer = await generateOnce(env, MODELS.fallback, flat, effort);
|
|
1711
|
+
}
|
|
1712
|
+
if (answer) answer = canonicalRefusal(answer);
|
|
1713
|
+
stageTiming.generate = Date.now() - tGen;
|
|
1714
|
+
let used = usedHits;
|
|
1715
|
+
if (answer && !answer.includes(refusalAnswer())) {
|
|
1716
|
+
const anchors = checkQuoteAnchors(answer, used.map((h) => h.text));
|
|
1717
|
+
const hasTableUnit = used.some((h) => h.metadata.unit_id && h.metadata.block === "table");
|
|
1718
|
+
const retyped = tableRetyped(answer, hasTableUnit);
|
|
1719
|
+
const unreferenced = (() => {
|
|
1720
|
+
if (!hasTableUnit || answer.includes("[[u:")) return false;
|
|
1721
|
+
const norm2 = (s) => (s.match(/\d[\d ,.]{1,8}\d/g) ?? []).map((x) => x.replace(/[ ,.]/g, ""));
|
|
1722
|
+
const nums = norm2(answer);
|
|
1723
|
+
if (nums.length < 2) return false;
|
|
1724
|
+
const tableNums = new Set(
|
|
1725
|
+
norm2(used.filter((h) => h.metadata.unit_id && h.metadata.block === "table").map((h) => h.text).join(" "))
|
|
1726
|
+
);
|
|
1727
|
+
return nums.filter((n) => tableNums.has(n)).length >= 2;
|
|
1728
|
+
})();
|
|
1729
|
+
if (anchors.violations.length > 0 || retyped || unreferenced) {
|
|
1730
|
+
console.log("contract check:", anchors.violations.length, "anchor violations; tableRetyped:", retyped, "; tableDataUnreferenced:", unreferenced, "\u2014 regenerating");
|
|
1731
|
+
const tableUnitId = unreferenced ? used.find((h) => h.metadata.unit_id && h.metadata.block === "table")?.metadata.unit_id : void 0;
|
|
1732
|
+
const note = retyped || unreferenced ? `Correction notice: your draft reproduced a table as markdown or presented a served table's data without its reference. Rewrite the answer: describe the table in prose, cite the clause, and write the reference token [[u:${tableUnitId ?? "<unit id>"}]] exactly where the table belongs. Do not render any table as markdown.` : ANCHOR_CORRECTION_NOTE;
|
|
1733
|
+
generateRetries += 1;
|
|
1734
|
+
const corrected = await generateOnce(env, model, [...messages, { role: "system", content: note }], effort);
|
|
1735
|
+
if (corrected) {
|
|
1736
|
+
const correctedAnswer = canonicalRefusal(corrected);
|
|
1737
|
+
const retryAnchors = checkQuoteAnchors(correctedAnswer, used.map((h) => h.text));
|
|
1738
|
+
const retryRetyped = tableRetyped(correctedAnswer, hasTableUnit);
|
|
1739
|
+
if (retryAnchors.violations.length < anchors.violations.length || !retryRetyped && retyped || unreferenced && correctedAnswer.includes("[[u:")) {
|
|
1740
|
+
answer = correctedAnswer;
|
|
1741
|
+
}
|
|
1742
|
+
}
|
|
1743
|
+
}
|
|
1744
|
+
}
|
|
1745
|
+
if (answer && !answer.includes(refusalAnswer())) {
|
|
1746
|
+
const reflection = await reflect(env.AI, MODELS.grader, q.query, answer, hits.map((h) => h.text));
|
|
1747
|
+
console.log("reflection:", reflection ? reflection.grounded ? "grounded" : "ungrounded" : "null");
|
|
1748
|
+
if (reflection && !reflection.grounded && reflection.missing_info) {
|
|
1749
|
+
const retryRetrieve = await retrieve(env, q.query, {
|
|
1750
|
+
prev,
|
|
1751
|
+
understanding: { ...understanding, standalone_query: `${understanding?.standalone_query || q.query} ${reflection.missing_info}` },
|
|
1752
|
+
sealScope: declaredScoped ? docScope : null,
|
|
1753
|
+
datasetScope: narrowed ? corpora : null
|
|
1754
|
+
});
|
|
1755
|
+
if (retryRetrieve.hits.length > 0) {
|
|
1756
|
+
const { messages: retryMessages, usedHits: retryUsed } = buildMessages(q.query, retryRetrieve.hits, q.lang, keptHistory, void 0, summary, budget);
|
|
1757
|
+
const retryAnswer = await generateOnce(env, model, retryMessages, effort);
|
|
1758
|
+
if (retryAnswer) {
|
|
1759
|
+
answer = canonicalRefusal(retryAnswer);
|
|
1760
|
+
used = retryUsed;
|
|
1761
|
+
}
|
|
1762
|
+
}
|
|
1763
|
+
}
|
|
1764
|
+
}
|
|
1765
|
+
if (answer === null) {
|
|
1766
|
+
telemetry(env, ctx, tier, "ask", model, false, 0, queryHash, q.lang, void 0, telemetryMeta());
|
|
1767
|
+
return err(502, "generation_failed", "The generation model is unavailable; please retry.");
|
|
1768
|
+
}
|
|
1769
|
+
const finalCites = boundModel ? [modelCitation(boundModel), ...citations(used)] : citations(used);
|
|
1770
|
+
const c2ns = answer.includes(refusalAnswer()) ? { text: answer, blocks: [], dropped: [] } : await contractV2(env.DB, answer, used);
|
|
1771
|
+
answer = c2ns.text;
|
|
1772
|
+
const finalAnchors = answer.includes(refusalAnswer()) ? { total: 0, violations: [] } : checkQuoteAnchors(answer, used.map((h) => h.text));
|
|
1773
|
+
if (finalAnchors.violations.length > 0) {
|
|
1774
|
+
console.log("anchors:", finalAnchors.violations.length, "of", finalAnchors.total, "unverified \u2014 not caching");
|
|
1775
|
+
}
|
|
1776
|
+
let completionBlocks = [];
|
|
1777
|
+
if (!answer.includes(refusalAnswer()) && !c2ns.blocks.some((b) => b.type === "table")) {
|
|
1778
|
+
completionBlocks = await completeTables(env.DB, answer, used);
|
|
1779
|
+
if (completionBlocks.length) console.log("contract completion:", completionBlocks.length, "table block(s) attached server-side");
|
|
1780
|
+
}
|
|
1781
|
+
completionBlocks.push(...await completeFigures(env.DB, answer, [...c2ns.blocks, ...completionBlocks], used));
|
|
1782
|
+
const out = { answer, citations: finalCites, model: MODELS.member, query_hash: queryHash, follow_ups: understanding?.follow_ups ?? [], blocks: [...c2ns.blocks, ...verdictBlock ? [verdictBlock] : [], ...completionBlocks], context_applied: ctxApplied, ...liveRecords ? { records: liveRecords } : {} };
|
|
1783
|
+
const cacheable = !contextual && !declaredCtx && !answer.includes(refusalAnswer()) && finalAnchors.violations.length === 0;
|
|
1784
|
+
if (cacheable) {
|
|
1785
|
+
const warmVec = await warmEmbed ?? null;
|
|
1786
|
+
if (warmVec) semanticCachePut(env, ctx, gen, warmVec, salt, out);
|
|
1787
|
+
}
|
|
1788
|
+
if (cacheable) {
|
|
1789
|
+
const ck = exactCacheKey(env.INDEX_VERSION, gen, ns, await sha256Hex(cacheKeyMaterial(q.query, q.lang, salt)));
|
|
1790
|
+
ctx.waitUntil(env.CACHE.put(ck, JSON.stringify(out), { expirationTtl: LIMITS.cacheTtlSec }));
|
|
1791
|
+
}
|
|
1792
|
+
telemetry(env, ctx, tier, "ask", model, true, answer.length, queryHash, q.lang, void 0, telemetryMeta());
|
|
1793
|
+
const contextOut = used.map((h) => ({
|
|
1794
|
+
doc_id: h.metadata.doc_id,
|
|
1795
|
+
clause_anchor: h.metadata.clause_anchor,
|
|
1796
|
+
text: h.text.slice(0, 1200),
|
|
1797
|
+
...h.metadata.table_selection ? { sel: h.metadata.table_selection } : {}
|
|
1798
|
+
}));
|
|
1799
|
+
return json({ ...out, context: contextOut, read: readAs(), quota }, 200, { ...corsHeaders(req), "server-timing": serverTiming() });
|
|
1800
|
+
}
|
|
1801
|
+
function sseResponse(events, cors) {
|
|
1802
|
+
const encoder = new TextEncoder();
|
|
1803
|
+
const body = events.map((e) => `data: ${JSON.stringify(e)}
|
|
1804
|
+
|
|
1805
|
+
`).join("");
|
|
1806
|
+
return new Response(encoder.encode(body), {
|
|
1807
|
+
headers: {
|
|
1808
|
+
"content-type": "text/event-stream",
|
|
1809
|
+
"cache-control": "no-cache",
|
|
1810
|
+
"x-accel-buffering": "no",
|
|
1811
|
+
...cors
|
|
1812
|
+
}
|
|
1813
|
+
});
|
|
1814
|
+
}
|
|
1815
|
+
function cosine(a, b) {
|
|
1816
|
+
let dot = 0;
|
|
1817
|
+
let na = 0;
|
|
1818
|
+
let nb = 0;
|
|
1819
|
+
for (let i = 0; i < a.length; i++) {
|
|
1820
|
+
dot += a[i] * b[i];
|
|
1821
|
+
na += a[i] * a[i];
|
|
1822
|
+
nb += b[i] * b[i];
|
|
1823
|
+
}
|
|
1824
|
+
return dot / (Math.sqrt(na) * Math.sqrt(nb) || 1);
|
|
1825
|
+
}
|
|
1826
|
+
function scSignature(v, salt) {
|
|
1827
|
+
return v.slice(0, 16).map((x) => x.toFixed(2)).join(",") + (salt ? `|s:${salt.length}:${salt.slice(0, 64)}` : "");
|
|
1828
|
+
}
|
|
1829
|
+
async function semanticCacheGet(env, gen, vec, salt) {
|
|
1830
|
+
try {
|
|
1831
|
+
const raw = await env.CACHE.get(semanticCacheKey(env.INDEX_VERSION, gen, scSignature(vec, salt)), "json");
|
|
1832
|
+
if (!raw?.v || !Array.isArray(raw.v) || raw.v.length !== vec.length) return null;
|
|
1833
|
+
if (cosine(raw.v, vec) < 0.97) return null;
|
|
1834
|
+
return raw;
|
|
1835
|
+
} catch {
|
|
1836
|
+
return null;
|
|
1837
|
+
}
|
|
1838
|
+
}
|
|
1839
|
+
function semanticCachePut(env, ctx, gen, vec, salt, payload) {
|
|
1840
|
+
const v = vec.map((x) => Number(x.toFixed(3)));
|
|
1841
|
+
ctx.waitUntil(
|
|
1842
|
+
env.CACHE.put(semanticCacheKey(env.INDEX_VERSION, gen, scSignature(vec, salt)), JSON.stringify({ v, ...payload }), { expirationTtl: LIMITS.cacheTtlSec })
|
|
1843
|
+
);
|
|
1844
|
+
}
|
|
1845
|
+
|
|
1846
|
+
export {
|
|
1847
|
+
handleMemories,
|
|
1848
|
+
checkQuoteAnchors,
|
|
1849
|
+
standardForDocNumber,
|
|
1850
|
+
scoreJudge,
|
|
1851
|
+
handleAsk
|
|
1852
|
+
};
|