@konneal/engine 0.2.47 → 0.2.49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ask-OH5KPHZA.js +12 -0
- package/dist/attachments.d.ts +31 -0
- package/dist/{chunk-SE67PTJ6.js → chunk-4KA34NMU.js} +140 -5
- package/dist/{chunk-AKAFLMIB.js → chunk-DMIOBGQU.js} +4 -4
- package/dist/{chunk-R2V3X6SQ.js → chunk-JWTIPQ4L.js} +1 -1
- package/dist/{chunk-4GJGBGJK.js → chunk-O6VZIDPW.js} +1 -1
- package/dist/{chunk-N2UOENDY.js → chunk-VVLNUHW2.js} +19 -2
- package/dist/{chunk-V46XM2GU.js → chunk-Z24IAKB6.js} +9 -0
- package/dist/config.d.ts +9 -0
- package/dist/entailment.d.ts +5 -0
- package/dist/env.d.ts +1 -0
- package/dist/openapi-surface.gen.d.ts +1 -1
- package/dist/openapi-types.d.ts +149 -0
- package/dist/search-KNB2S2PS.js +12 -0
- package/dist/worker_mcp/src/index.js +2 -2
- package/dist/worker_public/src/config.js +1 -1
- package/dist/worker_public/src/index.js +121 -119
- package/dist/worker_public/src/requestScope.js +2 -2
- package/package.json +1 -1
- package/workers/worker_public/migrations/0017_attachments.sql +12 -0
- package/workers/worker_public/openapi.yaml +69 -0
- package/workers/worker_public/schema.sql +10 -0
- package/workers/worker_public/src/ask.ts +23 -2
- package/workers/worker_public/src/attachments.ts +102 -0
- package/workers/worker_public/src/config.ts +9 -0
- package/workers/worker_public/src/entailment.ts +15 -0
- package/workers/worker_public/src/env.ts +1 -0
- package/workers/worker_public/src/index.ts +42 -0
- package/workers/worker_public/src/openapi-surface.gen.ts +24 -0
- package/workers/worker_public/src/ports/blobs.ts +1 -0
- package/workers/worker_public/src/ports/cloudflare/adapters.ts +3 -0
- package/dist/ask-6QPD3QA5.js +0 -12
- package/dist/search-QZDQVB57.js +0 -12
|
@@ -1,17 +1,19 @@
|
|
|
1
1
|
import {
|
|
2
2
|
handleSearch
|
|
3
|
-
} from "../../chunk-
|
|
3
|
+
} from "../../chunk-DMIOBGQU.js";
|
|
4
4
|
import {
|
|
5
5
|
bindModelNode,
|
|
6
6
|
checkQuoteAnchors,
|
|
7
7
|
handleAsk,
|
|
8
8
|
handleMemories,
|
|
9
9
|
modelGroundingBlock,
|
|
10
|
+
scoreFaithfulness,
|
|
10
11
|
scoreJudge,
|
|
11
12
|
standardForDocNumber
|
|
12
|
-
} from "../../chunk-
|
|
13
|
+
} from "../../chunk-4KA34NMU.js";
|
|
13
14
|
import {
|
|
14
15
|
buildMessages,
|
|
16
|
+
cfBlobs,
|
|
15
17
|
citations,
|
|
16
18
|
editionNote,
|
|
17
19
|
embed,
|
|
@@ -35,7 +37,7 @@ import {
|
|
|
35
37
|
sessionFrom,
|
|
36
38
|
telemetry,
|
|
37
39
|
understandQuery
|
|
38
|
-
} from "../../chunk-
|
|
40
|
+
} from "../../chunk-VVLNUHW2.js";
|
|
39
41
|
import {
|
|
40
42
|
authenticate,
|
|
41
43
|
corsHeaders,
|
|
@@ -44,14 +46,14 @@ import {
|
|
|
44
46
|
readJson,
|
|
45
47
|
validateQuery,
|
|
46
48
|
withCors
|
|
47
|
-
} from "../../chunk-
|
|
49
|
+
} from "../../chunk-JWTIPQ4L.js";
|
|
48
50
|
import {
|
|
49
51
|
canonicalRefusal
|
|
50
52
|
} from "../../chunk-A3QHHUN5.js";
|
|
51
53
|
import {
|
|
52
54
|
entitlementScope,
|
|
53
55
|
standardKeysFrom
|
|
54
|
-
} from "../../chunk-
|
|
56
|
+
} from "../../chunk-O6VZIDPW.js";
|
|
55
57
|
import {
|
|
56
58
|
LIMITS,
|
|
57
59
|
MODELS,
|
|
@@ -62,7 +64,7 @@ import {
|
|
|
62
64
|
roleModel,
|
|
63
65
|
sha256Hex,
|
|
64
66
|
today
|
|
65
|
-
} from "../../chunk-
|
|
67
|
+
} from "../../chunk-Z24IAKB6.js";
|
|
66
68
|
import {
|
|
67
69
|
P,
|
|
68
70
|
setProfile
|
|
@@ -332,118 +334,6 @@ async function handleGetShared(env, slug) {
|
|
|
332
334
|
});
|
|
333
335
|
}
|
|
334
336
|
|
|
335
|
-
// workers/worker_public/prompts/faithfulness.md
|
|
336
|
-
var faithfulness_default = `You are a factuality judge. Given an answer and the retrieved passages it was based on, identify any claims in the answer that are NOT directly supported by the passages. Reply with ONLY a JSON object: {"score": 0.0-1.0, "ungrounded_claims": ["claim text", ...]} \u2014 score is the fraction of claims that ARE grounded in the passages; if every claim is supported, score is 1.0 and ungrounded_claims is [].
|
|
337
|
-
|
|
338
|
-
Passages prefixed [M] are this service's own machine-computed model data (typed blocks the answer was given) \u2014 a claim restating [M] content is grounded.
|
|
339
|
-
`;
|
|
340
|
-
|
|
341
|
-
// workers/worker_public/src/verdict-parse.ts
|
|
342
|
-
function coerceVerdict(obj) {
|
|
343
|
-
if (obj === null || typeof obj !== "object" || Array.isArray(obj)) return null;
|
|
344
|
-
const raw = obj.score;
|
|
345
|
-
const score = typeof raw === "number" ? raw : typeof raw === "string" && raw.trim() !== "" ? Number(raw) : NaN;
|
|
346
|
-
if (!Number.isFinite(score)) return null;
|
|
347
|
-
const claims = obj.ungrounded_claims;
|
|
348
|
-
return {
|
|
349
|
-
score: Math.max(0, Math.min(1, score)),
|
|
350
|
-
ungrounded_claims: Array.isArray(claims) ? claims.map(String).slice(0, 5) : []
|
|
351
|
-
};
|
|
352
|
-
}
|
|
353
|
-
function parseVerdict(text) {
|
|
354
|
-
const stripped = text.replace(/```[a-zA-Z]*\n?/g, "").replace(/```/g, "").trim();
|
|
355
|
-
try {
|
|
356
|
-
const whole = coerceVerdict(JSON.parse(stripped));
|
|
357
|
-
if (whole) return whole;
|
|
358
|
-
} catch {
|
|
359
|
-
}
|
|
360
|
-
let verdict = null;
|
|
361
|
-
let depth = 0;
|
|
362
|
-
let start = -1;
|
|
363
|
-
let inString = false;
|
|
364
|
-
let escaped = false;
|
|
365
|
-
for (let i = 0; i < stripped.length; i++) {
|
|
366
|
-
const ch = stripped[i];
|
|
367
|
-
if (escaped) {
|
|
368
|
-
escaped = false;
|
|
369
|
-
continue;
|
|
370
|
-
}
|
|
371
|
-
if (inString && ch === "\\") {
|
|
372
|
-
escaped = true;
|
|
373
|
-
continue;
|
|
374
|
-
}
|
|
375
|
-
if (ch === '"') {
|
|
376
|
-
inString = !inString;
|
|
377
|
-
continue;
|
|
378
|
-
}
|
|
379
|
-
if (inString) continue;
|
|
380
|
-
if (ch === "{") {
|
|
381
|
-
if (depth === 0) start = i;
|
|
382
|
-
depth++;
|
|
383
|
-
} else if (ch === "}" && depth > 0) {
|
|
384
|
-
depth--;
|
|
385
|
-
if (depth === 0 && start >= 0) {
|
|
386
|
-
try {
|
|
387
|
-
const v = coerceVerdict(JSON.parse(stripped.slice(start, i + 1)));
|
|
388
|
-
if (v) verdict = v;
|
|
389
|
-
} catch {
|
|
390
|
-
}
|
|
391
|
-
}
|
|
392
|
-
}
|
|
393
|
-
}
|
|
394
|
-
return verdict;
|
|
395
|
-
}
|
|
396
|
-
|
|
397
|
-
// workers/worker_public/src/faithfulness-context.ts
|
|
398
|
-
function buildJudgeContext(passages, machine = []) {
|
|
399
|
-
const context = passages.slice(0, 8).map((p, i) => {
|
|
400
|
-
const text = typeof p === "string" ? p : p.text;
|
|
401
|
-
const limit = typeof p !== "string" && p.table ? 2400 : 1400;
|
|
402
|
-
return `[${i + 1}] ${text.replace(/\s+/g, " ").slice(0, limit)}`;
|
|
403
|
-
}).join("\n");
|
|
404
|
-
const machineContext = machine.length ? "\n" + machine.slice(0, 6).map((m) => `[M] ${m.replace(/\s+/g, " ").slice(0, 400)}`).join("\n") : "";
|
|
405
|
-
return context + machineContext;
|
|
406
|
-
}
|
|
407
|
-
|
|
408
|
-
// workers/worker_public/src/faithfulness.ts
|
|
409
|
-
async function scoreFaithfulness(ai, model, answer, passages, machine = []) {
|
|
410
|
-
if (!answer || !passages.length) return null;
|
|
411
|
-
const context = buildJudgeContext(passages, machine);
|
|
412
|
-
const t0 = Date.now();
|
|
413
|
-
const timeout = new Promise((r) => setTimeout(() => {
|
|
414
|
-
console.log(`faithfulness: timeout (${Date.now() - t0}ms)`);
|
|
415
|
-
r(null);
|
|
416
|
-
}, 24e4));
|
|
417
|
-
const call = (async () => {
|
|
418
|
-
const res = await ai.run(model, {
|
|
419
|
-
messages: [
|
|
420
|
-
{
|
|
421
|
-
role: "system",
|
|
422
|
-
content: faithfulness_default.trimEnd()
|
|
423
|
-
},
|
|
424
|
-
{ role: "user", content: `Answer:
|
|
425
|
-
${answer.slice(0, 2e3)}
|
|
426
|
-
|
|
427
|
-
Passages:
|
|
428
|
-
${context}${machine.length ? buildJudgeContext([], machine) : ""}` }
|
|
429
|
-
],
|
|
430
|
-
max_tokens: 6144,
|
|
431
|
-
reasoning_effort: "low",
|
|
432
|
-
// DeepSeek-V4 card: temp 1.0 / top_p 1.0
|
|
433
|
-
temperature: 1,
|
|
434
|
-
top_p: 1
|
|
435
|
-
});
|
|
436
|
-
const text = typeof res?.response === "string" ? res.response : res?.choices?.[0]?.message?.content;
|
|
437
|
-
const verdict = parseVerdict(text ?? "");
|
|
438
|
-
if (!verdict) {
|
|
439
|
-
console.log(`faithfulness: no parse (${Date.now() - t0}ms, text ${(text ?? "").length} chars) raw=${JSON.stringify((text ?? "").replace(/\s+/g, " ").slice(0, 500))}`);
|
|
440
|
-
return null;
|
|
441
|
-
}
|
|
442
|
-
return verdict;
|
|
443
|
-
})();
|
|
444
|
-
return await Promise.race([call, timeout]);
|
|
445
|
-
}
|
|
446
|
-
|
|
447
337
|
// workers/worker_public/prompts/enrichment.md
|
|
448
338
|
var enrichment_default = "You write a retrieval context for a passage from {{CORPUS_KIND}}. The context is prepended to the passage before embedding so a semantic search can locate the passage even when the query uses different vocabulary than the passage itself.\n\nWrite ONE concise sentence (at most 40 words) that situates the passage: name the publication by its exact {{PUBLISHER_NAME}} identifier (including part or annex when applicable) and what the passage covers \u2014 paraphrasing the topic in words DIFFERENT from the passage's own. Do not copy the passage verbatim, do not add facts that are not derivable from the passage or its header, do not answer or explain the content. Reply with the context sentence only \u2014 no quotes, no preamble.\n";
|
|
449
339
|
|
|
@@ -936,6 +826,64 @@ ${recent.map((h, n) => `[${n + 1}] ${h.metadata.docidentifier ?? ""} \xA7${h.met
|
|
|
936
826
|
return out;
|
|
937
827
|
}
|
|
938
828
|
|
|
829
|
+
// workers/worker_public/src/attachments.ts
|
|
830
|
+
var MAX_BYTES = 4e6;
|
|
831
|
+
var ALLOWED = {
|
|
832
|
+
"image/png": "png",
|
|
833
|
+
"image/jpeg": "jpg",
|
|
834
|
+
"image/webp": "webp",
|
|
835
|
+
"image/gif": "gif"
|
|
836
|
+
};
|
|
837
|
+
function sniffImage(head, totalBytes) {
|
|
838
|
+
if (totalBytes > MAX_BYTES || totalBytes < 12) return null;
|
|
839
|
+
if (head[0] === 137 && head[1] === 80 && head[2] === 78 && head[3] === 71) return "image/png";
|
|
840
|
+
if (head[0] === 255 && head[1] === 216) return "image/jpeg";
|
|
841
|
+
if (head.length >= 12 && head[8] === 87 && head[9] === 69 && head[10] === 66 && head[11] === 80) return "image/webp";
|
|
842
|
+
if (head[0] === 71 && head[1] === 73 && head[2] === 70) return "image/gif";
|
|
843
|
+
return null;
|
|
844
|
+
}
|
|
845
|
+
function parseDataUrl(dataUrl) {
|
|
846
|
+
const m = /^data:(image\/[a-z+]+);base64,([\s\S]+)$/.exec(dataUrl);
|
|
847
|
+
if (!m) return null;
|
|
848
|
+
const raw = atob(m[2]);
|
|
849
|
+
const bytes = new Uint8Array(raw.length);
|
|
850
|
+
for (let i = 0; i < raw.length; i++) bytes[i] = raw.charCodeAt(i);
|
|
851
|
+
const mime = ALLOWED[m[1]] ? m[1] : sniffImage(bytes.subarray(0, 12), bytes.length);
|
|
852
|
+
if (!mime || bytes.length > MAX_BYTES) return null;
|
|
853
|
+
return { mime, bytes };
|
|
854
|
+
}
|
|
855
|
+
async function uploadAttachment(deps, sub, dataUrl) {
|
|
856
|
+
const parsed = parseDataUrl(dataUrl);
|
|
857
|
+
if (!parsed) {
|
|
858
|
+
return new Response(JSON.stringify({ error: { code: "invalid_input", message: "A PNG, JPEG, WebP or GIF image of at most 4 MB is required" } }), { status: 400, headers: { "content-type": "application/json" } });
|
|
859
|
+
}
|
|
860
|
+
const id = crypto.randomUUID();
|
|
861
|
+
const key = `att/${sub}/${id}`;
|
|
862
|
+
await deps.blobs.put(key, parsed.bytes.buffer, parsed.mime);
|
|
863
|
+
await deps.store.prepare("INSERT INTO attachments (id, sub, mime, bytes, r2_key) VALUES (?1,?2,?3,?4,?5)").bind(id, sub, parsed.mime, parsed.bytes.length, key).run();
|
|
864
|
+
return { id };
|
|
865
|
+
}
|
|
866
|
+
async function ownedAttachment(deps, sub, id) {
|
|
867
|
+
const row = await deps.store.prepare("SELECT id, sub, mime, bytes, created_at FROM attachments WHERE id = ?1 AND sub = ?2").bind(id, sub).first();
|
|
868
|
+
return row ?? null;
|
|
869
|
+
}
|
|
870
|
+
async function readAttachment(deps, sub, id) {
|
|
871
|
+
const row = await ownedAttachment(deps, sub, id);
|
|
872
|
+
if (!row) return null;
|
|
873
|
+
const obj = await deps.blobs.get(`att/${row.sub}/${row.id}`);
|
|
874
|
+
if (!obj) return null;
|
|
875
|
+
return new Response(obj.body, {
|
|
876
|
+
headers: { "content-type": obj.contentType ?? row.mime, "cache-control": "private, max-age=86400" }
|
|
877
|
+
});
|
|
878
|
+
}
|
|
879
|
+
async function deleteAttachment(deps, sub, id) {
|
|
880
|
+
const row = await ownedAttachment(deps, sub, id);
|
|
881
|
+
if (!row) return false;
|
|
882
|
+
await deps.blobs.delete(`att/${sub}/${id}`);
|
|
883
|
+
await deps.store.prepare("DELETE FROM attachments WHERE id = ?1 AND sub = ?2").bind(id, sub).run();
|
|
884
|
+
return true;
|
|
885
|
+
}
|
|
886
|
+
|
|
939
887
|
// workers/worker_public/src/mcp-proto.ts
|
|
940
888
|
var PROTOCOL_VERSION = "2025-06-18";
|
|
941
889
|
var TOOLS = [
|
|
@@ -999,7 +947,7 @@ async function handleMcp(env, ctx, req, tier, key) {
|
|
|
999
947
|
// stream:false forces the JSON lane (anon defaults to SSE)
|
|
1000
948
|
body: JSON.stringify({ ...args, stream: false })
|
|
1001
949
|
});
|
|
1002
|
-
const res = name === "ask" ? await (await import("../../ask-
|
|
950
|
+
const res = name === "ask" ? await (await import("../../ask-OH5KPHZA.js")).handleAsk(env, ctx, inner, tier, key) : await (await import("../../search-KNB2S2PS.js")).handleSearch(env, ctx, inner, tier, key);
|
|
1003
951
|
return res.json().catch(() => ({ error: { message: "tool transport failed", status: res.status } }));
|
|
1004
952
|
});
|
|
1005
953
|
if (out.ok && "accepted" in out) return new Response(null, { status: 202 });
|
|
@@ -1149,6 +1097,26 @@ var OPENAPI_SURFACE = [
|
|
|
1149
1097
|
"pattern": "/api/shared/:slug",
|
|
1150
1098
|
"operationId": "getShared"
|
|
1151
1099
|
},
|
|
1100
|
+
{
|
|
1101
|
+
"method": "POST",
|
|
1102
|
+
"pattern": "/api/attachments",
|
|
1103
|
+
"operationId": "uploadAttachment"
|
|
1104
|
+
},
|
|
1105
|
+
{
|
|
1106
|
+
"method": "GET",
|
|
1107
|
+
"pattern": "/api/attachments",
|
|
1108
|
+
"operationId": "listAttachments"
|
|
1109
|
+
},
|
|
1110
|
+
{
|
|
1111
|
+
"method": "GET",
|
|
1112
|
+
"pattern": "/api/attachments/:id",
|
|
1113
|
+
"operationId": "getAttachment"
|
|
1114
|
+
},
|
|
1115
|
+
{
|
|
1116
|
+
"method": "DELETE",
|
|
1117
|
+
"pattern": "/api/attachments/:id",
|
|
1118
|
+
"operationId": "deleteAttachment"
|
|
1119
|
+
},
|
|
1152
1120
|
{
|
|
1153
1121
|
"method": "GET",
|
|
1154
1122
|
"pattern": "/api/memories",
|
|
@@ -1325,6 +1293,36 @@ function opWriteRefused(c, who) {
|
|
|
1325
1293
|
}
|
|
1326
1294
|
return null;
|
|
1327
1295
|
}
|
|
1296
|
+
async function attachmentsUploadRoute(c) {
|
|
1297
|
+
const session = await sessionFrom(c.req, c.env);
|
|
1298
|
+
if (!session) return withCors(err(401, "unauthorized", "Sign in to store attachments"), corsHeaders(c.req));
|
|
1299
|
+
const deps = { blobs: cfBlobs(c.env.CHAT_UPLOADS), store: c.env.DB };
|
|
1300
|
+
const body = await readJson(c.req);
|
|
1301
|
+
const out = await uploadAttachment(deps, session.sub, String(body?.data_url ?? ""));
|
|
1302
|
+
if (out instanceof Response) return withCors(out, corsHeaders(c.req));
|
|
1303
|
+
return withCors(json({ id: out.id }, 201), corsHeaders(c.req));
|
|
1304
|
+
}
|
|
1305
|
+
async function attachmentsListRoute(c) {
|
|
1306
|
+
const session = await sessionFrom(c.req, c.env);
|
|
1307
|
+
if (!session) return withCors(err(401, "unauthorized", "Sign in to list attachments"), corsHeaders(c.req));
|
|
1308
|
+
const rows = await c.env.DB.prepare("SELECT id, mime, bytes, created_at FROM attachments WHERE sub = ?1 ORDER BY created_at DESC LIMIT 200").bind(session.sub).all();
|
|
1309
|
+
return withCors(json({ attachments: rows.results ?? [] }), corsHeaders(c.req));
|
|
1310
|
+
}
|
|
1311
|
+
async function attachmentsGetRoute(c) {
|
|
1312
|
+
const session = await sessionFrom(c.req, c.env);
|
|
1313
|
+
if (!session) return err(404, "not_found", "No such attachment");
|
|
1314
|
+
const deps = { blobs: cfBlobs(c.env.CHAT_UPLOADS), store: c.env.DB };
|
|
1315
|
+
const out = await readAttachment(deps, session.sub, c.params.id);
|
|
1316
|
+
return out ?? err(404, "not_found", "No such attachment");
|
|
1317
|
+
}
|
|
1318
|
+
async function attachmentsDeleteRoute(c) {
|
|
1319
|
+
const session = await sessionFrom(c.req, c.env);
|
|
1320
|
+
if (!session) return withCors(err(401, "unauthorized", "Sign in to manage attachments"), corsHeaders(c.req));
|
|
1321
|
+
const deps = { blobs: cfBlobs(c.env.CHAT_UPLOADS), store: c.env.DB };
|
|
1322
|
+
const gone = await deleteAttachment(deps, session.sub, c.params.id);
|
|
1323
|
+
if (!gone) return withCors(err(404, "not_found", "No such attachment"), corsHeaders(c.req));
|
|
1324
|
+
return withCors(json({ ok: true }), corsHeaders(c.req));
|
|
1325
|
+
}
|
|
1328
1326
|
async function memoriesRoute(c) {
|
|
1329
1327
|
const who = await opIdentity(c, "Sign in to use memory files");
|
|
1330
1328
|
if (who instanceof Response) return withCors(who, corsHeaders(c.req));
|
|
@@ -1742,6 +1740,10 @@ var OPENAPI_HANDLERS = {
|
|
|
1742
1740
|
appendMessage: appendMessageRoute,
|
|
1743
1741
|
shareConversation: shareRoute,
|
|
1744
1742
|
getShared: getSharedRoute,
|
|
1743
|
+
uploadAttachment: attachmentsUploadRoute,
|
|
1744
|
+
listAttachments: attachmentsListRoute,
|
|
1745
|
+
getAttachment: attachmentsGetRoute,
|
|
1746
|
+
deleteAttachment: attachmentsDeleteRoute,
|
|
1745
1747
|
listMemories: memoriesRoute,
|
|
1746
1748
|
createMemory: memoriesRoute,
|
|
1747
1749
|
deleteMemory: memoriesRoute,
|
|
@@ -4,8 +4,8 @@ import {
|
|
|
4
4
|
requestSalt,
|
|
5
5
|
resolveRequestScope,
|
|
6
6
|
standardKeysFrom
|
|
7
|
-
} from "../../chunk-
|
|
8
|
-
import "../../chunk-
|
|
7
|
+
} from "../../chunk-O6VZIDPW.js";
|
|
8
|
+
import "../../chunk-Z24IAKB6.js";
|
|
9
9
|
import "../../chunk-3FYJM7LH.js";
|
|
10
10
|
export {
|
|
11
11
|
entitlementScope,
|
package/package.json
CHANGED
|
@@ -27,7 +27,7 @@
|
|
|
27
27
|
"playwright": "^1.62.1",
|
|
28
28
|
"yaml": "^2.9.1"
|
|
29
29
|
},
|
|
30
|
-
"version": "0.2.
|
|
30
|
+
"version": "0.2.49",
|
|
31
31
|
"description": "The Konneal engine: the publisher-agnostic build pipeline and API plane for standards intelligence (retrieval, answer contract, verdicts, evaluation).",
|
|
32
32
|
"license": "BSD-3-Clause",
|
|
33
33
|
"type": "module",
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
-- conversation attachments (TODO.new-era/8 Tier 1): a member's uploaded
|
|
2
|
+
-- images persist until they delete them; the R2 object key embeds the
|
|
3
|
+
-- owner's subject so the storage partition is the isolation
|
|
4
|
+
CREATE TABLE IF NOT EXISTS attachments (
|
|
5
|
+
id TEXT PRIMARY KEY,
|
|
6
|
+
sub TEXT NOT NULL,
|
|
7
|
+
mime TEXT NOT NULL,
|
|
8
|
+
bytes INTEGER NOT NULL,
|
|
9
|
+
r2_key TEXT NOT NULL,
|
|
10
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
11
|
+
);
|
|
12
|
+
CREATE INDEX IF NOT EXISTS idx_attachments_sub ON attachments (sub);
|
|
@@ -545,6 +545,75 @@ paths:
|
|
|
545
545
|
responses:
|
|
546
546
|
'200':
|
|
547
547
|
description: The shared conversation, without member identity.
|
|
548
|
+
/api/attachments:
|
|
549
|
+
post:
|
|
550
|
+
tags: [Attachments]
|
|
551
|
+
summary: Upload an image attachment
|
|
552
|
+
description: >-
|
|
553
|
+
Stores a member-uploaded image (PNG, JPEG, WebP or GIF, at most
|
|
554
|
+
4 MB) and returns its identifier. The attachment persists until
|
|
555
|
+
the member deletes it. The content is verified from its bytes,
|
|
556
|
+
never from a declared type. The body carries the image as a data
|
|
557
|
+
URL — the same shape the composer produces.
|
|
558
|
+
operationId: uploadAttachment
|
|
559
|
+
requestBody:
|
|
560
|
+
required: true
|
|
561
|
+
content:
|
|
562
|
+
application/json:
|
|
563
|
+
schema:
|
|
564
|
+
type: object
|
|
565
|
+
required: [data_url]
|
|
566
|
+
properties:
|
|
567
|
+
data_url:
|
|
568
|
+
type: string
|
|
569
|
+
description: The image as a data URL.
|
|
570
|
+
responses:
|
|
571
|
+
'201':
|
|
572
|
+
description: The attachment's identifier.
|
|
573
|
+
'400':
|
|
574
|
+
description: The content is not an allowed image, or exceeds the size cap.
|
|
575
|
+
get:
|
|
576
|
+
tags: [Attachments]
|
|
577
|
+
summary: List the member's attachments
|
|
578
|
+
operationId: listAttachments
|
|
579
|
+
responses:
|
|
580
|
+
'200':
|
|
581
|
+
description: The attachment identifiers, newest first.
|
|
582
|
+
/api/attachments/{id}:
|
|
583
|
+
get:
|
|
584
|
+
tags: [Attachments]
|
|
585
|
+
summary: Read one attachment's image
|
|
586
|
+
description: >-
|
|
587
|
+
Streams the stored image. The response is 404 for a foreign
|
|
588
|
+
identifier — ownership is checked on every fetch, and a file
|
|
589
|
+
belonging to another account is indistinguishable from a missing
|
|
590
|
+
one.
|
|
591
|
+
operationId: getAttachment
|
|
592
|
+
parameters:
|
|
593
|
+
- name: id
|
|
594
|
+
in: path
|
|
595
|
+
required: true
|
|
596
|
+
schema: { type: string }
|
|
597
|
+
responses:
|
|
598
|
+
'200':
|
|
599
|
+
description: The image bytes.
|
|
600
|
+
'404':
|
|
601
|
+
description: No such attachment owned by the caller.
|
|
602
|
+
delete:
|
|
603
|
+
tags: [Attachments]
|
|
604
|
+
summary: Delete an attachment
|
|
605
|
+
description: Removes the stored object and its record. Stored until deleted — and this deletes it.
|
|
606
|
+
operationId: deleteAttachment
|
|
607
|
+
parameters:
|
|
608
|
+
- name: id
|
|
609
|
+
in: path
|
|
610
|
+
required: true
|
|
611
|
+
schema: { type: string }
|
|
612
|
+
responses:
|
|
613
|
+
'200':
|
|
614
|
+
description: Deleted.
|
|
615
|
+
'404':
|
|
616
|
+
description: No such attachment owned by the caller.
|
|
548
617
|
/api/memories:
|
|
549
618
|
get:
|
|
550
619
|
tags: [Memory]
|
|
@@ -248,3 +248,13 @@ CREATE TABLE IF NOT EXISTS model_plane_meta (
|
|
|
248
248
|
node_count INTEGER NOT NULL,
|
|
249
249
|
indexed_at TEXT NOT NULL -- when the index landed (operational truth)
|
|
250
250
|
);
|
|
251
|
+
|
|
252
|
+
CREATE TABLE IF NOT EXISTS attachments (
|
|
253
|
+
id TEXT PRIMARY KEY,
|
|
254
|
+
sub TEXT NOT NULL,
|
|
255
|
+
mime TEXT NOT NULL,
|
|
256
|
+
bytes INTEGER NOT NULL,
|
|
257
|
+
r2_key TEXT NOT NULL,
|
|
258
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
259
|
+
);
|
|
260
|
+
CREATE INDEX IF NOT EXISTS idx_attachments_sub ON attachments (sub);
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
// echo — everything between "request validated" and "response written".
|
|
4
4
|
// index.ts routes here; this module owns the answer contract.
|
|
5
5
|
|
|
6
|
-
import { LIMITS, MODELS, num, sha256Hex, roleModel, answerEffort, requestEffort, effortBudget } from "./config";
|
|
6
|
+
import { LIMITS, MODELS, THRESHOLDS, num, sha256Hex, roleModel, answerEffort, requestEffort, effortBudget } from "./config";
|
|
7
7
|
import { portModelRunner } from "./env.ts";
|
|
8
8
|
import { refCodec } from "./codecs";
|
|
9
9
|
import { buildMessages, citations, retrieve, retrievalQuery, identityNote, splitHistory, listwiseRerank, refusalAnswer, Hit } from "./pipeline";
|
|
@@ -14,6 +14,8 @@ import { gradeRetrieval } from "./grader";
|
|
|
14
14
|
import summarizePrompt from "../prompts/summarize.md";
|
|
15
15
|
import { embed, generateOnce } from "./ai";
|
|
16
16
|
import { reflect } from "./reflect";
|
|
17
|
+
import { scoreFaithfulness } from "./faithfulness";
|
|
18
|
+
import { entailmentVerdict } from "./entailment";
|
|
17
19
|
import { checkQuoteAnchors, ANCHOR_CORRECTION_NOTE } from "./anchors";
|
|
18
20
|
import { canonicalRefusal } from "./refusal";
|
|
19
21
|
import { contractV2, tableRetyped } from "./refs";
|
|
@@ -1309,7 +1311,26 @@ async function handleAsk(
|
|
|
1309
1311
|
completionBlocks.push(...(await completeFigures(env.DB, answer, [...c2ns.blocks, ...completionBlocks], used)));
|
|
1310
1312
|
|
|
1311
1313
|
const jsonQuality = answerQuality(finalCites.map((c: any) => c.quality));
|
|
1312
|
-
|
|
1314
|
+
// the entailment gate (TODO.new-era/10): the eval battery's faithfulness
|
|
1315
|
+
// scorer, promoted onto the serving path — the confidence line states a
|
|
1316
|
+
// measurement (claim support in the cited passages), not only the source
|
|
1317
|
+
// rung. Budgeted and fail-open: a slow or unparseable check leaves the
|
|
1318
|
+
// rung note untouched; refusals and uncited answers have nothing to gate.
|
|
1319
|
+
let entailment: { support: string; score: number; ungrounded?: string[] } | null = null;
|
|
1320
|
+
let entailmentNote = "";
|
|
1321
|
+
if (THRESHOLDS.entailmentGate && jsonQuality && used.length && !answer.includes(refusalAnswer())) {
|
|
1322
|
+
const raced = await Promise.race([
|
|
1323
|
+
scoreFaithfulness(env.AI, roleModel(env, "grader"), answer, (used as Hit[]).map((h: Hit) => h.text)),
|
|
1324
|
+
new Promise<null>((r) => setTimeout(() => r(null), THRESHOLDS.entailmentBudgetMs)),
|
|
1325
|
+
]);
|
|
1326
|
+
if (raced) {
|
|
1327
|
+
const v = entailmentVerdict(raced.score, THRESHOLDS.entailmentSupportedFloor, THRESHOLDS.entailmentPartialFloor);
|
|
1328
|
+
entailment = { support: v.support, score: Math.round(raced.score * 100) / 100, ...(raced.ungrounded_claims?.length ? { ungrounded: raced.ungrounded_claims.slice(0, 3) } : {}) };
|
|
1329
|
+
entailmentNote = v.note;
|
|
1330
|
+
if (v.support !== "supported") console.log("entailment:", v.support, raced.score, raced.ungrounded_claims?.slice(0, 2));
|
|
1331
|
+
}
|
|
1332
|
+
}
|
|
1333
|
+
const out = { answer, citations: finalCites, ...(jsonQuality ? { source_quality: jsonQuality, confidence_note: entailmentNote || qualityNote(jsonQuality), ...(entailment ? { entailment } : {}), ...(jsonQuality === "ocr" ? { experimental_sources: experimentalSourceLabels(finalCites) } : {}) } : {}), model: MODELS.member, query_hash: queryHash, follow_ups: understanding?.follow_ups ?? [], blocks: [...c2ns.blocks, ...(verdictBlock ? [verdictBlock] : []), ...(conditionBlock ? [conditionBlock] : []), ...(aggregationBlock ? [aggregationBlock] : []), ...completionBlocks], context_applied: ctxApplied, ...(liveRecords ? { records: liveRecords } : {}) };
|
|
1313
1334
|
const cacheable = !contextual && !declaredCtx && !answer.includes(refusalAnswer()) && finalAnchors.violations.length === 0;
|
|
1314
1335
|
if (cacheable) {
|
|
1315
1336
|
const ck = exactCacheKey(env.INDEX_VERSION, gen, ns, await sha256Hex(cacheKeyMaterial(q.query, q.lang, salt)));
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
// Conversation attachments (TODO.new-era/8, Tier 1): a member's uploaded
|
|
2
|
+
// image persists until they delete it. The guarantees, in order:
|
|
3
|
+
//
|
|
4
|
+
// 1. NO CROSS-ACCOUNT READS — the R2 object key embeds the owner's
|
|
5
|
+
// subject (`att/{sub}/{id}`), and every route re-checks the D1
|
|
6
|
+
// row's ownership against the caller before touching storage. A
|
|
7
|
+
// logic slip cannot cross accounts, because the key-space itself
|
|
8
|
+
// is partitioned.
|
|
9
|
+
// 2. Stored until deleted — by the owner, or by the cascade when
|
|
10
|
+
// their conversation is deleted. No TTL quietly eats member files.
|
|
11
|
+
// 3. The content is verified, not the extension: the magic bytes
|
|
12
|
+
// decide what the file is.
|
|
13
|
+
//
|
|
14
|
+
// Storage rides the Blobs port and the row rides the StoreQuery port —
|
|
15
|
+
// the module never names a provider binding (the ports purity lint).
|
|
16
|
+
|
|
17
|
+
import type { Blobs } from "./ports/blobs.ts";
|
|
18
|
+
import type { StoreQuery } from "./ports/store.ts";
|
|
19
|
+
|
|
20
|
+
export interface AttachmentDeps {
|
|
21
|
+
blobs: Blobs;
|
|
22
|
+
store: StoreQuery;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
const MAX_BYTES = 4_000_000;
|
|
26
|
+
const ALLOWED: Record<string, string> = {
|
|
27
|
+
"image/png": "png",
|
|
28
|
+
"image/jpeg": "jpg",
|
|
29
|
+
"image/webp": "webp",
|
|
30
|
+
"image/gif": "gif",
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
export interface AttachmentRow {
|
|
34
|
+
id: string;
|
|
35
|
+
sub: string;
|
|
36
|
+
mime: string;
|
|
37
|
+
bytes: number;
|
|
38
|
+
created_at: string;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** Sniff the image type from the magic bytes — the declared type is
|
|
42
|
+
* never trusted. Returns null for anything that is not one of the four
|
|
43
|
+
* allowed images or that exceeds the cap. */
|
|
44
|
+
export function sniffImage(head: Uint8Array, totalBytes: number): string | null {
|
|
45
|
+
if (totalBytes > MAX_BYTES || totalBytes < 12) return null;
|
|
46
|
+
if (head[0] === 0x89 && head[1] === 0x50 && head[2] === 0x4e && head[3] === 0x47) return "image/png";
|
|
47
|
+
if (head[0] === 0xff && head[1] === 0xd8) return "image/jpeg";
|
|
48
|
+
if (head.length >= 12 && head[8] === 0x57 && head[9] === 0x45 && head[10] === 0x42 && head[11] === 0x50) return "image/webp";
|
|
49
|
+
if (head[0] === 0x47 && head[1] === 0x49 && head[2] === 0x46) return "image/gif";
|
|
50
|
+
return null;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** A data URL (the composer's current shape) → { mime, bytes } or null. */
|
|
54
|
+
export function parseDataUrl(dataUrl: string): { mime: string; bytes: Uint8Array } | null {
|
|
55
|
+
const m = /^data:(image\/[a-z+]+);base64,([\s\S]+)$/.exec(dataUrl);
|
|
56
|
+
if (!m) return null;
|
|
57
|
+
const raw = atob(m[2]);
|
|
58
|
+
const bytes = new Uint8Array(raw.length);
|
|
59
|
+
for (let i = 0; i < raw.length; i++) bytes[i] = raw.charCodeAt(i);
|
|
60
|
+
const mime = ALLOWED[m[1]] ? m[1] : sniffImage(bytes.subarray(0, 12), bytes.length);
|
|
61
|
+
if (!mime || bytes.length > MAX_BYTES) return null;
|
|
62
|
+
return { mime, bytes };
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export async function uploadAttachment(deps: AttachmentDeps, sub: string, dataUrl: string): Promise<{ id: string } | Response> {
|
|
66
|
+
const parsed = parseDataUrl(dataUrl);
|
|
67
|
+
if (!parsed) {
|
|
68
|
+
return new Response(JSON.stringify({ error: { code: "invalid_input", message: "A PNG, JPEG, WebP or GIF image of at most 4 MB is required" } }), { status: 400, headers: { "content-type": "application/json" } });
|
|
69
|
+
}
|
|
70
|
+
const id = crypto.randomUUID();
|
|
71
|
+
// the key embeds the owner — the storage partition IS the isolation
|
|
72
|
+
const key = `att/${sub}/${id}`;
|
|
73
|
+
await deps.blobs.put(key, parsed.bytes.buffer as ArrayBuffer, parsed.mime);
|
|
74
|
+
await deps.store.prepare("INSERT INTO attachments (id, sub, mime, bytes, r2_key) VALUES (?1,?2,?3,?4,?5)").bind(id, sub, parsed.mime, parsed.bytes.length, key).run();
|
|
75
|
+
return { id };
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** The ownership check every access runs first: the row must exist and
|
|
79
|
+
* belong to the caller. A foreign id is indistinguishable from a
|
|
80
|
+
* missing one (404 — never a confirmation of someone else's file). */
|
|
81
|
+
export async function ownedAttachment(deps: AttachmentDeps, sub: string, id: string): Promise<AttachmentRow | null> {
|
|
82
|
+
const row = await deps.store.prepare("SELECT id, sub, mime, bytes, created_at FROM attachments WHERE id = ?1 AND sub = ?2").bind(id, sub).first();
|
|
83
|
+
return (row as AttachmentRow) ?? null;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export async function readAttachment(deps: AttachmentDeps, sub: string, id: string): Promise<Response | null> {
|
|
87
|
+
const row = await ownedAttachment(deps, sub, id);
|
|
88
|
+
if (!row) return null;
|
|
89
|
+
const obj = await deps.blobs.get(`att/${row.sub}/${row.id}`);
|
|
90
|
+
if (!obj) return null;
|
|
91
|
+
return new Response(obj.body, {
|
|
92
|
+
headers: { "content-type": obj.contentType ?? row.mime, "cache-control": "private, max-age=86400" },
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
export async function deleteAttachment(deps: AttachmentDeps, sub: string, id: string): Promise<boolean> {
|
|
97
|
+
const row = await ownedAttachment(deps, sub, id);
|
|
98
|
+
if (!row) return false;
|
|
99
|
+
await deps.blobs.delete(`att/${sub}/${id}`);
|
|
100
|
+
await deps.store.prepare("DELETE FROM attachments WHERE id = ?1 AND sub = ?2").bind(id, sub).run();
|
|
101
|
+
return true;
|
|
102
|
+
}
|
|
@@ -56,6 +56,15 @@ export const THRESHOLDS = {
|
|
|
56
56
|
* pool below the primary dense lane; they must earn their window
|
|
57
57
|
* slot under the cross-encoder, not by graph membership alone. */
|
|
58
58
|
graphLaneDiscount: 0.75,
|
|
59
|
+
/** The entailment gate (TODO.new-era/10): the faithfulness scorer
|
|
60
|
+
* promoted from eval-only onto the serving path. The confidence line
|
|
61
|
+
* then states a measurement — claim support in the cited passages —
|
|
62
|
+
* not only the source rung. Fails open (a slow or unparseable check
|
|
63
|
+
* leaves the source-rung note unchanged). */
|
|
64
|
+
entailmentGate: true,
|
|
65
|
+
entailmentBudgetMs: 6000,
|
|
66
|
+
entailmentSupportedFloor: 0.9,
|
|
67
|
+
entailmentPartialFloor: 0.5,
|
|
59
68
|
/** Concept-graph candidate discount — same rationale as the graph
|
|
60
69
|
* lane: definitional content enters discounted. */
|
|
61
70
|
conceptGraphDiscount: 0.75,
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
// The entailment gate's pure verdict mapper (TODO.new-era/10): what the
|
|
2
|
+
// confidence line says once the claims were measured against the
|
|
3
|
+
// passages they cite. Lives apart from faithfulness.ts so the unit
|
|
4
|
+
// suites (plain node, no esbuild .md loader) can exercise it.
|
|
5
|
+
|
|
6
|
+
export interface EntailmentVerdict {
|
|
7
|
+
support: "supported" | "partial" | "unsupported";
|
|
8
|
+
note: string;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export function entailmentVerdict(score: number, supportedFloor: number, partialFloor: number): EntailmentVerdict {
|
|
12
|
+
if (score >= supportedFloor) return { support: "supported", note: "" };
|
|
13
|
+
if (score >= partialFloor) return { support: "partial", note: "Partially grounded — some claims lack support in the cited passages; verify against the cited clauses." };
|
|
14
|
+
return { support: "unsupported", note: "WARNING: This answer's claims are largely unsupported by the retrieved passages. Verify against official publications." };
|
|
15
|
+
}
|