@konneal/engine 0.2.1 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ask-YCDXMIZ4.js +12 -0
- package/dist/{chunk-LNSDBEKS.js → chunk-5MBWE7WD.js} +29 -4
- package/dist/{chunk-Q6LI4T7M.js → chunk-ADXV2DPK.js} +1 -1
- package/dist/{chunk-WGXATDXY.js → chunk-DBBGOOMZ.js} +1 -1
- package/dist/{chunk-SN3ANQ3Y.js → chunk-EFQALN2Z.js} +2 -2
- package/dist/{chunk-3OXSQH7Y.js → chunk-OGFH3RDM.js} +70 -27
- package/dist/{chunk-6GOSMLRH.js → chunk-RLT4W2VX.js} +28 -5
- package/dist/{chunk-VJZLVU3S.js → chunk-THSHLUOS.js} +9 -4
- package/dist/{chunk-Q327B27J.js → chunk-TJRTVJW5.js} +13 -2
- package/dist/modelplane.d.ts +44 -1
- package/dist/openapi-types.d.ts +16 -2
- package/dist/profile.gen.d.ts +9 -0
- package/dist/prompts/system.md +1 -0
- package/dist/requestScope.d.ts +25 -4
- package/dist/search-7TO2RWQE.js +12 -0
- package/dist/selfquery.d.ts +6 -0
- package/dist/worker_mcp/src/index.js +3 -3
- package/dist/worker_public/src/config.js +2 -2
- package/dist/worker_public/src/index.js +38 -15
- package/dist/worker_public/src/profile.js +1 -1
- package/dist/worker_public/src/refusal.js +2 -2
- package/dist/worker_public/src/requestScope.js +11 -5
- package/docs/spec-pipeline.md +1 -0
- package/package.json +1 -1
- package/profile/prompts.yaml +14 -0
- package/profile/sources.yaml +9 -0
- package/workers/shared/chunk.ts +7 -0
- package/workers/worker_internal/src/index.ts +3 -3
- package/workers/worker_public/openapi.yaml +28 -3
- package/workers/worker_public/prompts/system.md +1 -0
- package/workers/worker_public/src/ask.ts +50 -16
- package/workers/worker_public/src/index.ts +2 -1
- package/workers/worker_public/src/modelplane.ts +94 -4
- package/workers/worker_public/src/pipeline.ts +13 -5
- package/workers/worker_public/src/profile.gen.ts +13 -2
- package/workers/worker_public/src/requestScope.ts +49 -7
- package/workers/worker_public/src/search.ts +7 -1
- package/workers/worker_public/src/selfquery.ts +26 -0
- package/workers/worker_public/src/share.ts +27 -5
- package/workers/worker_public/src/stages/citationProbe.ts +6 -1
- package/workers/worker_public/src/stages/index.ts +2 -0
- package/workers/worker_public/src/stages/licenseScope.ts +23 -0
- package/workers/worker_public/src/stages/types.ts +11 -0
- package/dist/ask-47RNGK2R.js +0 -12
- package/dist/search-OMPBMZT4.js +0 -11
package/dist/profile.gen.d.ts
CHANGED
|
@@ -54,6 +54,13 @@ export declare const PROFILE: {
|
|
|
54
54
|
readonly bibliography: {};
|
|
55
55
|
readonly terminology: {};
|
|
56
56
|
readonly models: {};
|
|
57
|
+
readonly licensed: readonly [{
|
|
58
|
+
readonly key: "std:fixture-60068-2-30";
|
|
59
|
+
readonly package: "fixture-60068-2-30";
|
|
60
|
+
readonly doc_number: "60068-2-30";
|
|
61
|
+
readonly title: "FIXTURE environmental testing — damp heat, cyclic";
|
|
62
|
+
readonly edition: "2005";
|
|
63
|
+
}];
|
|
57
64
|
};
|
|
58
65
|
readonly ui: {
|
|
59
66
|
readonly suggestions: readonly ["What is in the fixture corpus?", "Which documents does the fixture publisher issue?"];
|
|
@@ -85,6 +92,8 @@ export declare const PROFILE: {
|
|
|
85
92
|
readonly docid_example: "FIXTURE 1-2";
|
|
86
93
|
readonly spelling_examples: "\"f1\", \"FIXTURE 1\"";
|
|
87
94
|
readonly process_vocab: "the fixture certification system framework";
|
|
95
|
+
readonly license_declare_pointer: "org admin → Settings → Standards licenses";
|
|
96
|
+
readonly license_posture: "Some indexed publications are LICENSED. When a license boundary note names the question's publication, obey it: answer at the citation level only — the standard's title and edition, and the invoking clause the public passages carry — never state, paraphrase or \"summarize from memory\" any procedure of the licensed text, and point at the declare flow the note names. Public content (the Recommendations' own models, applicability and references) stays fully answerable.";
|
|
88
97
|
};
|
|
89
98
|
};
|
|
90
99
|
};
|
package/dist/prompts/system.md
CHANGED
|
@@ -14,6 +14,7 @@ Synthesize practical answers from the passages: definitions, procedures and rule
|
|
|
14
14
|
MANDATORY: when the question asks how to do something (get certified, apply, comply, register, test) and the passages describe the governing system or procedure, ALWAYS answer with that procedure citing the governing documents. Refusing such a question because the passages do not name the specific publication is WRONG — the publication sets technical requirements; the HOW is governed by the certification-system documents in the passages.
|
|
15
15
|
If the passages cover only part of the question, answer the covered part fully, then state precisely what the indexed publications do not cover — do not pad with outside knowledge.
|
|
16
16
|
Refuse ONLY when no passage relates to the question's topic. Use exactly this sentence: {{REFUSAL_SENTENCE}} Then add one short line naming what you can answer instead, so the refusal redirects rather than dead-ends.
|
|
17
|
+
{{LICENSE_POSTURE}}
|
|
17
18
|
{{CORPUS_NOTES}}
|
|
18
19
|
Lead with the direct answer, then supporting detail; no preamble like 'Based on the passages'. Use short paragraphs or bullets for multi-part answers. Be concise and precise. Answer in the question's language{{LANG_CLAUSE}}.
|
|
19
20
|
- HARD RULE — typed units: passages whose header shows `unit u:xxxx (table)` contain a typed table. If your answer presents that table's data, you MUST write the token `[[u:xxxx]]` where the table belongs and MUST NOT render the table as markdown or reproduce more than ONE of its rows inline. Summarize the pattern in prose ("classes A–D with lower limits from 100 to 50 000"), cite the clause normally, and let `[[u:xxxx]]` stand for the full table — the interface renders it exactly from the source. The same rule applies to `unit u:xxxx (formula|figure|term)` objects.
|
package/dist/requestScope.d.ts
CHANGED
|
@@ -10,7 +10,25 @@ export interface RequestScope {
|
|
|
10
10
|
isoOn: boolean;
|
|
11
11
|
/** raw (validated) memory ids from the body — empty for anon */
|
|
12
12
|
memoryIds: string[];
|
|
13
|
+
/** raw (validated) license entitlement keys from the body — empty for
|
|
14
|
+
* anon and for requests that carry none (TODO.external-refs/08) */
|
|
15
|
+
standardKeys: Set<string>;
|
|
13
16
|
}
|
|
17
|
+
/** The deployment's declared licensed standards (profile sources.yaml
|
|
18
|
+
* `licensed:` — key/package/doc_number rows). Empty = the deployment
|
|
19
|
+
* serves public content only and the entitlement scope is inert. */
|
|
20
|
+
export declare function licenseDeclared(): boolean;
|
|
21
|
+
/** The request's entitlement set, VALIDATED against the declared
|
|
22
|
+
* whitelist: unknown keys drop (a forged key can never widen scope past
|
|
23
|
+
* the standards the deployment actually keys). Absent field = empty set
|
|
24
|
+
* — the fail-closed default for a deployment that declares licensed
|
|
25
|
+
* content. */
|
|
26
|
+
export declare function standardKeysFrom(body: any): Set<string>;
|
|
27
|
+
/** The RetrieveOptions value for the hard scope: null when the
|
|
28
|
+
* deployment declares no licensed content (inert — zero behavior
|
|
29
|
+
* change); otherwise the caller's validated set, EMPTY INCLUDED (the
|
|
30
|
+
* unentitled caller: licensed chunks hidden, citation metadata stays). */
|
|
31
|
+
export declare function entitlementScope(keys: Set<string>): Set<string> | null;
|
|
14
32
|
/** Validate + intersect. Returns { error } when the request explicitly
|
|
15
33
|
* disables every dataset (a user error, not a scope). The corpora a
|
|
16
34
|
* dataset searches travel WITH the declaration (profile datasets.yaml,
|
|
@@ -19,8 +37,11 @@ export declare function resolveRequestScope(body: any, member: unknown): Request
|
|
|
19
37
|
error: "empty-datasets";
|
|
20
38
|
};
|
|
21
39
|
/** The answer-cache salt: request-scoped context that materially changes
|
|
22
|
-
* the answer (dataset scope, memory selection
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
40
|
+
* the answer (dataset scope, memory selection, license entitlements —
|
|
41
|
+
* the licensed tier changes the grounding, so two callers asking the
|
|
42
|
+
* same question must never share an entry). Requests differing only in
|
|
43
|
+
* salt share query text — an unsalted key would serve a scoped (or
|
|
44
|
+
* memory-flavored, or licensed-tier) answer to a plain ask. Null =
|
|
45
|
+
* default scope, no memory, no entitlement effect: keys stay
|
|
46
|
+
* byte-identical to the pre-salt era. */
|
|
26
47
|
export declare function requestSalt(scope: RequestScope, memoryUsed: string[]): string | null;
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import {
|
|
2
|
+
handleSearch
|
|
3
|
+
} from "./chunk-THSHLUOS.js";
|
|
4
|
+
import "./chunk-RLT4W2VX.js";
|
|
5
|
+
import "./chunk-EFQALN2Z.js";
|
|
6
|
+
import "./chunk-DBBGOOMZ.js";
|
|
7
|
+
import "./chunk-5MBWE7WD.js";
|
|
8
|
+
import "./chunk-ADXV2DPK.js";
|
|
9
|
+
import "./chunk-TJRTVJW5.js";
|
|
10
|
+
export {
|
|
11
|
+
handleSearch
|
|
12
|
+
};
|
package/dist/selfquery.d.ts
CHANGED
|
@@ -5,3 +5,9 @@ export interface QueryFilters {
|
|
|
5
5
|
language?: string;
|
|
6
6
|
}
|
|
7
7
|
export declare function toVectorizeFilter(f: QueryFilters): Record<string, string> | undefined;
|
|
8
|
+
/** The one entitlement predicate: no key = public = always allowed; a key
|
|
9
|
+
* outside the caller's set = never allowed. `null`/undefined keys (the
|
|
10
|
+
* deployment declares no licensed content) disable the scope entirely. */
|
|
11
|
+
export declare function standardKeyAllowed(meta: {
|
|
12
|
+
standard_key?: string;
|
|
13
|
+
}, keys: ReadonlySet<string> | null | undefined): boolean;
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import {
|
|
2
2
|
authenticate
|
|
3
|
-
} from "../../chunk-
|
|
4
|
-
import "../../chunk-
|
|
3
|
+
} from "../../chunk-EFQALN2Z.js";
|
|
4
|
+
import "../../chunk-ADXV2DPK.js";
|
|
5
5
|
import {
|
|
6
6
|
P,
|
|
7
7
|
setProfile
|
|
8
|
-
} from "../../chunk-
|
|
8
|
+
} from "../../chunk-TJRTVJW5.js";
|
|
9
9
|
|
|
10
10
|
// workers/worker_mcp/src/index.ts
|
|
11
11
|
var PROTOCOL_VERSION = "2025-06-18";
|
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
import {
|
|
2
2
|
handleSearch
|
|
3
|
-
} from "../../chunk-
|
|
3
|
+
} from "../../chunk-THSHLUOS.js";
|
|
4
4
|
import {
|
|
5
5
|
checkQuoteAnchors,
|
|
6
6
|
handleAsk,
|
|
7
7
|
handleMemories,
|
|
8
8
|
scoreJudge,
|
|
9
9
|
standardForDocNumber
|
|
10
|
-
} from "../../chunk-
|
|
11
|
-
import "../../chunk-LNSDBEKS.js";
|
|
10
|
+
} from "../../chunk-OGFH3RDM.js";
|
|
12
11
|
import {
|
|
13
12
|
buildMessages,
|
|
14
13
|
citations,
|
|
@@ -30,7 +29,7 @@ import {
|
|
|
30
29
|
sessionFrom,
|
|
31
30
|
telemetry,
|
|
32
31
|
understandQuery
|
|
33
|
-
} from "../../chunk-
|
|
32
|
+
} from "../../chunk-RLT4W2VX.js";
|
|
34
33
|
import {
|
|
35
34
|
authenticate,
|
|
36
35
|
corsHeaders,
|
|
@@ -39,10 +38,14 @@ import {
|
|
|
39
38
|
readJson,
|
|
40
39
|
validateQuery,
|
|
41
40
|
withCors
|
|
42
|
-
} from "../../chunk-
|
|
41
|
+
} from "../../chunk-EFQALN2Z.js";
|
|
43
42
|
import {
|
|
44
43
|
canonicalRefusal
|
|
45
|
-
} from "../../chunk-
|
|
44
|
+
} from "../../chunk-DBBGOOMZ.js";
|
|
45
|
+
import {
|
|
46
|
+
entitlementScope,
|
|
47
|
+
standardKeysFrom
|
|
48
|
+
} from "../../chunk-5MBWE7WD.js";
|
|
46
49
|
import {
|
|
47
50
|
LIMITS,
|
|
48
51
|
MODELS,
|
|
@@ -53,11 +56,11 @@ import {
|
|
|
53
56
|
roleModel,
|
|
54
57
|
sha256Hex,
|
|
55
58
|
today
|
|
56
|
-
} from "../../chunk-
|
|
59
|
+
} from "../../chunk-ADXV2DPK.js";
|
|
57
60
|
import {
|
|
58
61
|
P,
|
|
59
62
|
setProfile
|
|
60
|
-
} from "../../chunk-
|
|
63
|
+
} from "../../chunk-TJRTVJW5.js";
|
|
61
64
|
|
|
62
65
|
// workers/worker_public/src/conversations.ts
|
|
63
66
|
var ID_RE = /^[a-zA-Z0-9_-]{8,64}$/;
|
|
@@ -282,11 +285,31 @@ async function handleShareConversation(env, ownerSub, title, messages) {
|
|
|
282
285
|
if (count >= 10) return err(429, "rate_limited", "Daily share limit reached");
|
|
283
286
|
await env.CACHE.put(`sh:${day}:${ownerSub.slice(0, 20)}`, String(count + 1), { expirationTtl: 9e4 });
|
|
284
287
|
const slug = makeSlug();
|
|
285
|
-
const cleanMessages = messages.slice(0, 50).map((m) =>
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
288
|
+
const cleanMessages = messages.slice(0, 50).map((m) => {
|
|
289
|
+
const citations2 = Array.isArray(m.citations) ? m.citations : typeof m.citations === "string" ? (() => {
|
|
290
|
+
try {
|
|
291
|
+
return JSON.parse(m.citations);
|
|
292
|
+
} catch {
|
|
293
|
+
return null;
|
|
294
|
+
}
|
|
295
|
+
})() : null;
|
|
296
|
+
const read = m.read && typeof m.read === "object" ? {
|
|
297
|
+
intent: String(m.read.intent ?? ""),
|
|
298
|
+
doc: m.read.doc ?? null,
|
|
299
|
+
edition: m.read.edition ?? null,
|
|
300
|
+
term: m.read.term ?? null,
|
|
301
|
+
terms: Array.isArray(m.read.terms) ? m.read.terms.slice(0, 4).map(String) : [],
|
|
302
|
+
lang: m.read.lang ?? null
|
|
303
|
+
} : void 0;
|
|
304
|
+
return {
|
|
305
|
+
role: m.role === "user" ? "user" : "assistant",
|
|
306
|
+
content: (m.content ?? "").slice(0, LIMITS.maxOutputTokens * 2),
|
|
307
|
+
citations: citations2,
|
|
308
|
+
...m.model ? { model: String(m.model).slice(0, 80) } : {},
|
|
309
|
+
...Array.isArray(m.blocks) ? { blocks: m.blocks.slice(0, 12) } : {},
|
|
310
|
+
...read ? { read } : {}
|
|
311
|
+
};
|
|
312
|
+
});
|
|
290
313
|
await env.DB.prepare(
|
|
291
314
|
"INSERT INTO shared_conversations (slug, owner_sub, title, messages, created_at) VALUES (?1,?2,?3,?4,?5)"
|
|
292
315
|
).bind(slug, ownerSub, title.slice(0, 120), JSON.stringify(cleanMessages), (/* @__PURE__ */ new Date()).toISOString()).run();
|
|
@@ -956,7 +979,7 @@ async function handleMcp(env, ctx, req, tier, key) {
|
|
|
956
979
|
// stream:false forces the JSON lane (anon defaults to SSE)
|
|
957
980
|
body: JSON.stringify({ ...args, stream: false })
|
|
958
981
|
});
|
|
959
|
-
const res = name === "ask" ? await (await import("../../ask-
|
|
982
|
+
const res = name === "ask" ? await (await import("../../ask-YCDXMIZ4.js")).handleAsk(env, ctx, inner, tier, key) : await (await import("../../search-7TO2RWQE.js")).handleSearch(env, ctx, inner, tier, key);
|
|
960
983
|
return res.json().catch(() => ({ error: { message: "tool transport failed", status: res.status } }));
|
|
961
984
|
});
|
|
962
985
|
if (out.ok && "accepted" in out) return new Response(null, { status: 202 });
|
|
@@ -1415,7 +1438,7 @@ async function verifyRoute(c) {
|
|
|
1415
1438
|
if (!answer || !query) return err(400, "invalid_input", "answer and query are required");
|
|
1416
1439
|
try {
|
|
1417
1440
|
const u = await understandQuery(env.AI, roleModel(env, "understand"), query, [], []);
|
|
1418
|
-
const retrieved = await retrieve(env, query, { understanding: u });
|
|
1441
|
+
const retrieved = await retrieve(env, query, { understanding: u, standardKeys: entitlementScope(standardKeysFrom(body)) });
|
|
1419
1442
|
const passages = retrieved.hits.map((h) => h.text);
|
|
1420
1443
|
const anchors = checkQuoteAnchors(answer, passages);
|
|
1421
1444
|
const refs = [...answer.matchAll(/\[\[u:([^\]]+)\]\]/g)].map((m) => m[1]);
|
|
@@ -1,10 +1,16 @@
|
|
|
1
1
|
import {
|
|
2
|
+
entitlementScope,
|
|
3
|
+
licenseDeclared,
|
|
2
4
|
requestSalt,
|
|
3
|
-
resolveRequestScope
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
import "../../chunk-
|
|
5
|
+
resolveRequestScope,
|
|
6
|
+
standardKeysFrom
|
|
7
|
+
} from "../../chunk-5MBWE7WD.js";
|
|
8
|
+
import "../../chunk-ADXV2DPK.js";
|
|
9
|
+
import "../../chunk-TJRTVJW5.js";
|
|
7
10
|
export {
|
|
11
|
+
entitlementScope,
|
|
12
|
+
licenseDeclared,
|
|
8
13
|
requestSalt,
|
|
9
|
-
resolveRequestScope
|
|
14
|
+
resolveRequestScope,
|
|
15
|
+
standardKeysFrom
|
|
10
16
|
};
|
package/docs/spec-pipeline.md
CHANGED
|
@@ -47,6 +47,7 @@ candidate lanes → pool open → pool-level merges → refinement → window as
|
|
|
47
47
|
| 9 | `lexical-union` | `lexicalHits.length` | blocking | appends `hits` | Full-corpus BM25 hits dense missed (dense metadata wins on id collision). |
|
|
48
48
|
| 10 | `federate` | `opts.federate` | additive (callback catches transport) | appends `hits` | Internal ISO/IEC passages at `federateDiscount`. |
|
|
49
49
|
| 11 | `seal` | `opts.sealScope` | blocking | filters `hits` | The declared context's hard cut: nothing outside the declared family reaches rerank. (The lexical lane is sealed at the SOURCE, in the prelude — not here.) |
|
|
50
|
+
| 11b | `license-scope` | `opts.standardKeys` | blocking | filters `hits` | The license entitlement hard cut (TODO.external-refs/08): a chunk carrying a `standard_key` outside the caller's set never reaches rerank; public chunks (no key) always pass. (The lexical lane is bound at the SOURCE, in the prelude — the lexical union AND the post-rerank lexical-rrf consume the bound list. The post-scope single-document fetches — section-descent, typed-pin parent, edition-cover — are doc-scoped, and a document's key is a per-document constant, so they cannot re-admit a dropped key; citation-probe filters its bibliography push by the same predicate.) |
|
|
50
51
|
| 12 | `overview-demote` | — | blocking | mutates `hits[].score` | Overview boilerplate demotion (`overviewDemotion`). |
|
|
51
52
|
| 13 | `family-boost` | boost only under `filter.doc_number`; sort always | blocking | mutates `hits[].score`, sorts | Family chunks decisively boosted for doc-scoped queries; the sort establishes the rerank-failure fallback order. |
|
|
52
53
|
| 14 | `rerank` | `hits.length > 1` | additive | `hits[].rerank_score`, sorts, family pin | Cross-encoder scores; vector order is the designed fallback. Post-rerank family pin for doc-scoped queries. |
|
package/package.json
CHANGED
|
@@ -36,7 +36,7 @@
|
|
|
36
36
|
"@konneal/engine": "github:konneal/engine#2d6c39b",
|
|
37
37
|
"@oimlsmart/oiml-pubid": "^1.2.1"
|
|
38
38
|
},
|
|
39
|
-
"version": "0.2.
|
|
39
|
+
"version": "0.2.3",
|
|
40
40
|
"description": "The Konneal engine: the publisher-agnostic build pipeline and API plane for standards intelligence (retrieval, answer contract, verdicts, evaluation).",
|
|
41
41
|
"license": "BSD-3-Clause",
|
|
42
42
|
"type": "module",
|
package/profile/prompts.yaml
CHANGED
|
@@ -17,3 +17,17 @@ vars:
|
|
|
17
17
|
docid_example: "FIXTURE 1-2"
|
|
18
18
|
spelling_examples: '"f1", "FIXTURE 1"'
|
|
19
19
|
process_vocab: "the fixture certification system framework"
|
|
20
|
+
# the license declare flow (TODO.external-refs/08): where an org admin
|
|
21
|
+
# declares a standards license — the boundary refusal's pointer
|
|
22
|
+
license_declare_pointer: "org admin → Settings → Standards licenses"
|
|
23
|
+
# the licensed-content posture (TODO.external-refs/08): how the answer
|
|
24
|
+
# model behaves when a question hits the license boundary (the
|
|
25
|
+
# per-question note rides the retrieval-note channel)
|
|
26
|
+
license_posture: >-
|
|
27
|
+
Some indexed publications are LICENSED. When a license boundary note
|
|
28
|
+
names the question's publication, obey it: answer at the citation
|
|
29
|
+
level only — the standard's title and edition, and the invoking clause
|
|
30
|
+
the public passages carry — never state, paraphrase or "summarize from
|
|
31
|
+
memory" any procedure of the licensed text, and point at the declare
|
|
32
|
+
flow the note names. Public content (the Recommendations' own models,
|
|
33
|
+
applicability and references) stays fully answerable.
|
package/profile/sources.yaml
CHANGED
|
@@ -3,3 +3,12 @@ corpora:
|
|
|
3
3
|
bibliography: {}
|
|
4
4
|
terminology: {}
|
|
5
5
|
models: {}
|
|
6
|
+
# the licensed-standards declaration (TODO.external-refs/08): the
|
|
7
|
+
# entitlement keys the deployment's index may carry (`standard_key` chunk
|
|
8
|
+
# metadata, from the packages' license_key facet) — this list is the
|
|
9
|
+
# whitelist the request's entitlement set validates against, and the
|
|
10
|
+
# package/doc-number mapping the ask path gates model bindings with.
|
|
11
|
+
# Absent or empty = the deployment serves public content only. The
|
|
12
|
+
# fixture declares one so the license scope rides every engine test.
|
|
13
|
+
licensed:
|
|
14
|
+
- { key: "std:fixture-60068-2-30", package: "fixture-60068-2-30", doc_number: "60068-2-30", title: "FIXTURE environmental testing — damp heat, cyclic", edition: "2005" }
|
package/workers/shared/chunk.ts
CHANGED
|
@@ -35,6 +35,12 @@ export interface ChunkMeta {
|
|
|
35
35
|
model_node?: string;
|
|
36
36
|
model_kind?: string;
|
|
37
37
|
standard?: string;
|
|
38
|
+
/** The license entitlement key (the package's `license_key` facet, e.g.
|
|
39
|
+
* `std:iec-60068-2-30`) — present ONLY when the chunk's content comes
|
|
40
|
+
* from a licensed package; public content carries none. The ask path's
|
|
41
|
+
* hard scope (stages/licenseScope.ts) drops chunks whose key is absent
|
|
42
|
+
* from the caller's entitlement set, before ranking. */
|
|
43
|
+
standard_key?: string;
|
|
38
44
|
/** section-summary unit (FABLE multi-granularity, arXiv:2601.18116): a
|
|
39
45
|
* depth-1 clause summary vector — a navigation node whose children
|
|
40
46
|
* (child_anchors CSV) are quotable leaf clauses. The corpus's real
|
|
@@ -83,6 +89,7 @@ export const CHUNK_META_FIELDS = [
|
|
|
83
89
|
"model_node",
|
|
84
90
|
"model_kind",
|
|
85
91
|
"standard",
|
|
92
|
+
"standard_key",
|
|
86
93
|
"section_summary",
|
|
87
94
|
"child_anchors",
|
|
88
95
|
"ctx",
|
|
@@ -85,7 +85,7 @@ async function adminSyncRoute(c: RouteContext): Promise<Response> {
|
|
|
85
85
|
if (Array.isArray(body?.upserts)) {
|
|
86
86
|
for (let i = 0; i < body.upserts.length; i += 100) {
|
|
87
87
|
const batch = body.upserts.slice(i, i + 100).filter((v: any) => v?.id && Array.isArray(v?.values) && v?.metadata);
|
|
88
|
-
if (batch.length) await env.
|
|
88
|
+
if (batch.length) await env.INTERNAL.upsert(batch);
|
|
89
89
|
out.upserted += batch.length;
|
|
90
90
|
}
|
|
91
91
|
}
|
|
@@ -94,14 +94,14 @@ async function adminSyncRoute(c: RouteContext): Promise<Response> {
|
|
|
94
94
|
for (let i = 0; i < todo.length; i += 16) {
|
|
95
95
|
const batch = todo.slice(i, i + 16);
|
|
96
96
|
const vectors = await Promise.all(batch.map((b: any) => embed(env.AI, b.text.slice(0, 6000))));
|
|
97
|
-
await env.
|
|
97
|
+
await env.INTERNAL.upsert(batch.map((b: any, j: number) => ({ id: b.id, values: vectors[j], metadata: b.metadata })));
|
|
98
98
|
out.embedded += batch.length;
|
|
99
99
|
}
|
|
100
100
|
}
|
|
101
101
|
if (Array.isArray(body?.deletes)) {
|
|
102
102
|
const ids = body.deletes.filter((x: any) => typeof x === "string");
|
|
103
103
|
for (let i = 0; i < ids.length; i += 100) {
|
|
104
|
-
await env.
|
|
104
|
+
await env.INTERNAL.deleteByIds(ids.slice(i, i + 100));
|
|
105
105
|
out.deleted += Math.min(100, ids.length - i);
|
|
106
106
|
}
|
|
107
107
|
}
|
|
@@ -102,7 +102,8 @@ paths:
|
|
|
102
102
|
Identical to /api/ask, with authentication by API key and a
|
|
103
103
|
per-key daily allowance. Streaming responses are available by
|
|
104
104
|
setting stream to true, which returns a server-sent event stream
|
|
105
|
-
whose
|
|
105
|
+
whose first event is the reading, then the citations, then the
|
|
106
|
+
answer tokens, and finally the completion marker.
|
|
106
107
|
operationId: askKeyed
|
|
107
108
|
security:
|
|
108
109
|
- bearerAuth: []
|
|
@@ -1103,8 +1104,10 @@ components:
|
|
|
1103
1104
|
type: boolean
|
|
1104
1105
|
default: false
|
|
1105
1106
|
description: >-
|
|
1106
|
-
When true, the response is a server-sent event stream
|
|
1107
|
-
|
|
1107
|
+
When true, the response is a server-sent event stream. The
|
|
1108
|
+
first event is the reading (how the service interpreted the
|
|
1109
|
+
question), then the citations, then the answer tokens, and
|
|
1110
|
+
finally the completion marker.
|
|
1108
1111
|
fresh:
|
|
1109
1112
|
type: boolean
|
|
1110
1113
|
default: false
|
|
@@ -1164,6 +1167,28 @@ components:
|
|
|
1164
1167
|
query_hash:
|
|
1165
1168
|
type: string
|
|
1166
1169
|
description: The hash of the question, used for feedback.
|
|
1170
|
+
read:
|
|
1171
|
+
type: object
|
|
1172
|
+
description: >-
|
|
1173
|
+
How the service read the question — the interpretation that
|
|
1174
|
+
steered retrieval. Present on fresh answers; absent on exact
|
|
1175
|
+
cache hits, which never re-ran understanding.
|
|
1176
|
+
properties:
|
|
1177
|
+
intent: { type: string }
|
|
1178
|
+
doc: { type: string, nullable: true }
|
|
1179
|
+
edition: { type: string, nullable: true }
|
|
1180
|
+
term: { type: string, nullable: true }
|
|
1181
|
+
terms:
|
|
1182
|
+
type: array
|
|
1183
|
+
items: { type: string }
|
|
1184
|
+
lang: { type: string, nullable: true }
|
|
1185
|
+
served_from:
|
|
1186
|
+
type: string
|
|
1187
|
+
enum: [cache, similar]
|
|
1188
|
+
description: >-
|
|
1189
|
+
Present when the answer was served from the answer cache for
|
|
1190
|
+
an identical or similar earlier question. The cache stores the
|
|
1191
|
+
answer and never the passages.
|
|
1167
1192
|
quota:
|
|
1168
1193
|
type: object
|
|
1169
1194
|
description: The remaining daily allowance.
|
|
@@ -14,6 +14,7 @@ Synthesize practical answers from the passages: definitions, procedures and rule
|
|
|
14
14
|
MANDATORY: when the question asks how to do something (get certified, apply, comply, register, test) and the passages describe the governing system or procedure, ALWAYS answer with that procedure citing the governing documents. Refusing such a question because the passages do not name the specific publication is WRONG — the publication sets technical requirements; the HOW is governed by the certification-system documents in the passages.
|
|
15
15
|
If the passages cover only part of the question, answer the covered part fully, then state precisely what the indexed publications do not cover — do not pad with outside knowledge.
|
|
16
16
|
Refuse ONLY when no passage relates to the question's topic. Use exactly this sentence: {{REFUSAL_SENTENCE}} Then add one short line naming what you can answer instead, so the refusal redirects rather than dead-ends.
|
|
17
|
+
{{LICENSE_POSTURE}}
|
|
17
18
|
{{CORPUS_NOTES}}
|
|
18
19
|
Lead with the direct answer, then supporting detail; no preamble like 'Based on the passages'. Use short paragraphs or bullets for multi-part answers. Be concise and precise. Answer in the question's language{{LANG_CLAUSE}}.
|
|
19
20
|
- HARD RULE — typed units: passages whose header shows `unit u:xxxx (table)` contain a typed table. If your answer presents that table's data, you MUST write the token `[[u:xxxx]]` where the table belongs and MUST NOT render the table as markdown or reproduce more than ONE of its rows inline. Summarize the pattern in prose ("classes A–D with lower limits from 100 to 50 000"), cite the clause normally, and let `[[u:xxxx]]` stand for the full table — the interface renders it exactly from the source. The same rule applies to `unit u:xxxx (formula|figure|term)` objects.
|
|
@@ -19,11 +19,11 @@ import { contractV2, tableRetyped } from "./refs";
|
|
|
19
19
|
import { completeTables, completeFigures } from "./completion";
|
|
20
20
|
import { NO_CONTEXT, appliedContext, contextNote, namedDocumentIn, parseContext, resolveDocScope, syntheticUnderstanding } from "./context";
|
|
21
21
|
import { exchangeForLiveToken, liveDataConfig, resolveLiveAccount, type LiveRecord } from "./livedata";
|
|
22
|
-
import { bindModelNode, modelCitation, modelEcho, modelGroundingBlock, modelNodeRefIn, standardForDocNumber } from "./modelplane";
|
|
22
|
+
import { bindModelNode, licenseBoundaryNote, licenseBoundaryRefusal, modelCitation, modelEcho, modelGroundingBlock, modelNodeRefIn, standardForDocNumber } from "./modelplane";
|
|
23
23
|
import { evaluate as machineEvaluate, verdictNote } from "./verdict";
|
|
24
24
|
import { detectDraftIntent, prepareDraft } from "./drafts";
|
|
25
25
|
import { memoryNote } from "./memories";
|
|
26
|
-
import { resolveRequestScope, requestSalt } from "./requestScope";
|
|
26
|
+
import { entitlementScope, resolveRequestScope, requestSalt } from "./requestScope";
|
|
27
27
|
import { rawSessionToken } from "./session";
|
|
28
28
|
import { cacheKeyMaterial, corpusGen, exactCacheKey, freshRequested, semanticCacheKey } from "./answercache";
|
|
29
29
|
import type { Env } from "./env";
|
|
@@ -308,6 +308,13 @@ async function handleAsk(
|
|
|
308
308
|
const scope = resolveRequestScope(body, member);
|
|
309
309
|
if ("error" in scope) return err(400, "invalid_input", "datasets: at least one dataset must stay enabled");
|
|
310
310
|
const { corpora, narrowed, isoOn } = scope;
|
|
311
|
+
// The license entitlement set (TODO.external-refs/08): request-scoped,
|
|
312
|
+
// validated against the profile's declared licensed list; the hard
|
|
313
|
+
// retrieval scope it feeds is fail-closed (an unentitled ask — empty
|
|
314
|
+
// set, the anon default — never sees licensed chunks, citation-level
|
|
315
|
+
// metadata stays). It salts the answer cache with the effort segment
|
|
316
|
+
// below.
|
|
317
|
+
const standardKeys = entitlementScope(scope.standardKeys);
|
|
311
318
|
// Personalized memory files (#171): member-scoped, selected per ask;
|
|
312
319
|
// the note rides buildMessages as a trusted-user-facts preamble, and
|
|
313
320
|
// the selections SALT the answer cache (requestScope.requestSalt).
|
|
@@ -551,9 +558,14 @@ async function handleAsk(
|
|
|
551
558
|
telemetry(env, ctx, tier, "ask", null, true, sc.answer.length, sc.query_hash, q.lang, "semantic", telemetryMeta());
|
|
552
559
|
const cctx = sc.context_applied ?? NO_CONTEXT;
|
|
553
560
|
if (wantsStream) {
|
|
554
|
-
return sseResponse([
|
|
561
|
+
return sseResponse([
|
|
562
|
+
...(readAs() ? [{ type: "read", read: readAs() }] : []),
|
|
563
|
+
{ type: "citations", citations: sc.citations ?? [], context_applied: cctx },
|
|
564
|
+
{ type: "token", v: sc.answer },
|
|
565
|
+
{ type: "done", model: sc.model, query_hash: sc.query_hash, similar: true, served_from: "similar", context_applied: cctx, read: readAs() },
|
|
566
|
+
], corsHeaders(req));
|
|
555
567
|
}
|
|
556
|
-
return json({ ...sc, similar: true, context_applied: cctx, quota, });
|
|
568
|
+
return json({ ...sc, similar: true, context_applied: cctx, read: readAs(), quota, });
|
|
557
569
|
}
|
|
558
570
|
}
|
|
559
571
|
}
|
|
@@ -579,6 +591,8 @@ async function handleAsk(
|
|
|
579
591
|
const sse = new ReadableStream({
|
|
580
592
|
async start(controller) {
|
|
581
593
|
const send = (obj: unknown) => controller.enqueue(encoder.encode(`data: ${JSON.stringify(obj)}\n\n`));
|
|
594
|
+
// the reading arrives first — a conversational turn is still a reading
|
|
595
|
+
if (readAs()) send({ type: "read", read: readAs() });
|
|
582
596
|
send({ type: "citations", citations: [], context_applied: NO_CONTEXT, quota, });
|
|
583
597
|
let full = "";
|
|
584
598
|
try {
|
|
@@ -589,7 +603,7 @@ async function handleAsk(
|
|
|
589
603
|
} catch {
|
|
590
604
|
// stream ended prematurely — deliver what we have
|
|
591
605
|
}
|
|
592
|
-
send({ type: "done", model, query_hash: queryHash, context_applied: NO_CONTEXT });
|
|
606
|
+
send({ type: "done", model, query_hash: queryHash, context_applied: NO_CONTEXT, read: readAs() });
|
|
593
607
|
telemetry(env, ctx, tier, "ask", model, true, full.length, queryHash, q.lang, undefined, telemetryMeta());
|
|
594
608
|
controller.close();
|
|
595
609
|
},
|
|
@@ -606,7 +620,7 @@ async function handleAsk(
|
|
|
606
620
|
return err(502, "generation_failed", "The generation model is unavailable; please retry.");
|
|
607
621
|
}
|
|
608
622
|
telemetry(env, ctx, tier, "ask", model, true, answer.length, queryHash, q.lang, undefined, telemetryMeta());
|
|
609
|
-
return json({ answer, citations: [], model, query_hash: queryHash, follow_ups: [], context_applied: NO_CONTEXT, quota, });
|
|
623
|
+
return json({ answer, citations: [], model, query_hash: queryHash, follow_ups: [], context_applied: NO_CONTEXT, read: readAs(), quota, });
|
|
610
624
|
}
|
|
611
625
|
|
|
612
626
|
// ── The draft act (TODO.ai-platform/04) — the assistant PREPARES, the
|
|
@@ -652,14 +666,15 @@ async function handleAsk(
|
|
|
652
666
|
if (wantsStream) {
|
|
653
667
|
return sseResponse(
|
|
654
668
|
[
|
|
669
|
+
...(readAs() ? [{ type: "read", read: readAs() }] : []),
|
|
655
670
|
{ type: "citations", citations, context_applied: draftCtxApplied, ...(draftPayload ? { draft: draftPayload } : {}), quota, },
|
|
656
671
|
{ type: "token", v: verdict.answer },
|
|
657
|
-
{ type: "done", model, query_hash: queryHash, context_applied: draftCtxApplied },
|
|
672
|
+
{ type: "done", model, query_hash: queryHash, context_applied: draftCtxApplied, read: readAs() },
|
|
658
673
|
],
|
|
659
674
|
corsHeaders(req),
|
|
660
675
|
);
|
|
661
676
|
}
|
|
662
|
-
return json({ answer: verdict.answer, citations, model, query_hash: queryHash, follow_ups: [], context_applied: draftCtxApplied, ...(draftPayload ? { draft: draftPayload } : {}), quota, });
|
|
677
|
+
return json({ answer: verdict.answer, citations, model, query_hash: queryHash, follow_ups: [], context_applied: draftCtxApplied, read: readAs(), ...(draftPayload ? { draft: draftPayload } : {}), quota, });
|
|
663
678
|
}
|
|
664
679
|
|
|
665
680
|
// (declared before the retrieval try: the account block, the refusal
|
|
@@ -684,18 +699,23 @@ async function handleAsk(
|
|
|
684
699
|
label: declaredCtx?.label,
|
|
685
700
|
query: q.query,
|
|
686
701
|
standard: standardForDocNumber(modelDocHint?.doc_number),
|
|
702
|
+
standardKeys,
|
|
687
703
|
})
|
|
688
704
|
: null;
|
|
689
705
|
if (boundModel) {
|
|
690
706
|
ctxApplied = { ...ctxApplied, model: modelEcho(boundModel) };
|
|
691
|
-
console.log("model plane: bound", boundModel.node_id, `[${boundModel.standard}]`, boundModel.clause?.urn ?? "no-clause");
|
|
707
|
+
console.log("model plane: bound", boundModel.node_id, `[${boundModel.standard}]`, boundModel.clause?.urn ?? "no-clause", boundModel.gated ? "(gated: license)" : "");
|
|
692
708
|
}
|
|
693
|
-
|
|
709
|
+
// A GATED binding (licensed package, unentitled caller) keeps its
|
|
710
|
+
// citation + echo — metadata the honesty posture keeps — and withholds
|
|
711
|
+
// the grounding block and the verdict engine: no licensed machine
|
|
712
|
+
// content enters the prompt by the binding lane either.
|
|
713
|
+
const modelNote = boundModel && !boundModel.gated ? modelGroundingBlock(boundModel) : undefined;
|
|
694
714
|
// ── the verdict engine (TODO.era3/01) ──
|
|
695
715
|
// the worker EXECUTES the bound node's machine checks against the
|
|
696
716
|
// question's stated values; the model narrates the computed verdict
|
|
697
717
|
// and the verdict BLOCK is server-built — data, never generated prose
|
|
698
|
-
const machineVerdict = boundModel ? machineEvaluate(boundModel.content, q.query) : null;
|
|
718
|
+
const machineVerdict = boundModel && !boundModel.gated ? machineEvaluate(boundModel.content, q.query) : null;
|
|
699
719
|
const machineNote = machineVerdict && boundModel ? verdictNote(machineVerdict, boundModel) : undefined;
|
|
700
720
|
const verdictBlock = machineVerdict
|
|
701
721
|
? {
|
|
@@ -768,7 +788,7 @@ async function handleAsk(
|
|
|
768
788
|
// publications there).
|
|
769
789
|
retrieved = await retrieve(env, q.query, { prev, understanding, federate, warmEmbed, graphDocNumbers,
|
|
770
790
|
sealScope: declaredScoped ? docScope : null, optimisticHits, optimisticVec,
|
|
771
|
-
datasetScope: narrowed ? corpora : null });
|
|
791
|
+
datasetScope: narrowed ? corpora : null, standardKeys });
|
|
772
792
|
stageTiming["retrieve-core"] = Date.now() - tR;
|
|
773
793
|
console.log("stage: retrieve", Date.now() - tR, "ms");
|
|
774
794
|
// ── TTFT surgery: the two post-retrieval LLM calls run IN PARALLEL —
|
|
@@ -802,7 +822,7 @@ async function handleAsk(
|
|
|
802
822
|
if (grade === "weak" && understanding?.docidentifier) {
|
|
803
823
|
const broaden = `${understanding.standalone_query || q.query} ${understanding.docidentifier}`.trim();
|
|
804
824
|
const tc = Date.now();
|
|
805
|
-
const second = await retrieve(env, q.query, { prev, understanding, queryOverride: broaden, federate, datasetScope: narrowed ? corpora : null });
|
|
825
|
+
const second = await retrieve(env, q.query, { prev, understanding, queryOverride: broaden, federate, datasetScope: narrowed ? corpora : null, standardKeys, sealScope: declaredScoped ? docScope : null });
|
|
806
826
|
const grade2 = await gradeRetrieval(env.AI, roleModel(env, "grader"), q.query, second.hits.map((h: Hit) => h.text));
|
|
807
827
|
stageTiming.corrective = Date.now() - tc;
|
|
808
828
|
if (grade2 === "good") retrieved = second; // corrective retry must be strictly better
|
|
@@ -813,8 +833,13 @@ async function handleAsk(
|
|
|
813
833
|
return err(503, "retrieval_unavailable", "Search is briefly busy — please retry in a moment.");
|
|
814
834
|
}
|
|
815
835
|
const { hits } = retrieved;
|
|
816
|
-
if (hits.length === 0 && !liveRecords?.length && !boundModel) {
|
|
817
|
-
|
|
836
|
+
if (hits.length === 0 && !liveRecords?.length && (!boundModel || boundModel.gated)) {
|
|
837
|
+
// the license boundary refusal (TODO.external-refs/08) outranks the
|
|
838
|
+
// plain "no information" sentence when the question's publication is
|
|
839
|
+
// licensed and unentitled: name the standard, state the boundary,
|
|
840
|
+
// point at the declare flow
|
|
841
|
+
const answer =
|
|
842
|
+
licenseBoundaryRefusal(modelDocHint?.doc_number ?? understanding?.doc_number ?? null, standardKeys) ?? refusalAnswer();
|
|
818
843
|
const out = { answer, citations: [], model, query_hash: await sha256Hex(q.query), context_applied: ctxApplied };
|
|
819
844
|
telemetry(env, ctx, tier, "ask", model, true, answer.length, out.query_hash, q.lang, undefined, telemetryMeta());
|
|
820
845
|
return json({ ...out, quota, });
|
|
@@ -823,6 +848,11 @@ async function handleAsk(
|
|
|
823
848
|
const processNote = understanding?.process_intent
|
|
824
849
|
? P().retrieval.process_note
|
|
825
850
|
: undefined;
|
|
851
|
+
// the license boundary (TODO.external-refs/08): the question's named or
|
|
852
|
+
// understood publication is licensed and the caller's set lacks the key
|
|
853
|
+
// — the refusal-class note rides the retrieval-note channel beside the
|
|
854
|
+
// other structured facts
|
|
855
|
+
const licenseNote = licenseBoundaryNote(modelDocHint?.doc_number ?? understanding?.doc_number ?? null, standardKeys);
|
|
826
856
|
// the vocabulary binding (L2): the corpus's defined-term candidates for
|
|
827
857
|
// the question's subject — the model adjudicates among them and uses
|
|
828
858
|
// the corpus term (with its defining publication) when it names the
|
|
@@ -851,7 +881,7 @@ async function handleAsk(
|
|
|
851
881
|
q.lang,
|
|
852
882
|
keptHistory,
|
|
853
883
|
// stage-extracted graph facts (GraphRAG) ride the same note channel
|
|
854
|
-
[processNote, eNote, contextNote(declaredCtx, docScope), accountNote, modelNote, vocabNote, memNote, machineNote, ...(retrieved.notes ?? [])].filter(Boolean).join("\n") || undefined,
|
|
884
|
+
[processNote, eNote, contextNote(declaredCtx, docScope), accountNote, modelNote, vocabNote, memNote, machineNote, licenseNote, ...(retrieved.notes ?? [])].filter(Boolean).join("\n") || undefined,
|
|
855
885
|
summary,
|
|
856
886
|
budget,
|
|
857
887
|
);
|
|
@@ -886,6 +916,9 @@ async function handleAsk(
|
|
|
886
916
|
const sse = new ReadableStream({
|
|
887
917
|
async start(controller) {
|
|
888
918
|
const send = (obj: unknown) => controller.enqueue(encoder.encode(`data: ${JSON.stringify(obj)}\n\n`));
|
|
919
|
+
// the reading arrives first: the interpretation that steered
|
|
920
|
+
// retrieval, before a single token of the answer
|
|
921
|
+
send({ type: "read", read: readAs() });
|
|
889
922
|
send({ type: "citations", citations: cites, context_applied: ctxApplied, ...(liveRecords ? { records: liveRecords } : {}), quota, });
|
|
890
923
|
let full = "";
|
|
891
924
|
try {
|
|
@@ -1035,6 +1068,7 @@ async function handleAsk(
|
|
|
1035
1068
|
understanding: { ...understanding, standalone_query: `${understanding?.standalone_query || q.query} ${reflection.missing_info}` } as any,
|
|
1036
1069
|
sealScope: declaredScoped ? docScope : null,
|
|
1037
1070
|
datasetScope: narrowed ? corpora : null,
|
|
1071
|
+
standardKeys,
|
|
1038
1072
|
});
|
|
1039
1073
|
if (retryRetrieve.hits.length > 0) {
|
|
1040
1074
|
const { messages: retryMessages, usedHits: retryUsed } = buildMessages(q.query, retryRetrieve.hits, q.lang, keptHistory, undefined, summary, budget);
|