@konneal/engine 0.2.2 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ask-YCDXMIZ4.js +12 -0
- package/dist/{chunk-LNSDBEKS.js → chunk-5MBWE7WD.js} +29 -4
- package/dist/{chunk-Q6LI4T7M.js → chunk-ADXV2DPK.js} +1 -1
- package/dist/{chunk-WGXATDXY.js → chunk-DBBGOOMZ.js} +1 -1
- package/dist/{chunk-SN3ANQ3Y.js → chunk-EFQALN2Z.js} +2 -2
- package/dist/{chunk-PVHSBXZD.js → chunk-OGFH3RDM.js} +56 -21
- package/dist/{chunk-6GOSMLRH.js → chunk-RLT4W2VX.js} +28 -5
- package/dist/{chunk-VJZLVU3S.js → chunk-THSHLUOS.js} +9 -4
- package/dist/{chunk-Q327B27J.js → chunk-TJRTVJW5.js} +13 -2
- package/dist/modelplane.d.ts +44 -1
- package/dist/profile.gen.d.ts +9 -0
- package/dist/prompts/system.md +1 -0
- package/dist/requestScope.d.ts +25 -4
- package/dist/search-7TO2RWQE.js +12 -0
- package/dist/selfquery.d.ts +6 -0
- package/dist/worker_mcp/src/index.js +3 -3
- package/dist/worker_public/src/config.js +2 -2
- package/dist/worker_public/src/index.js +13 -10
- package/dist/worker_public/src/profile.js +1 -1
- package/dist/worker_public/src/refusal.js +2 -2
- package/dist/worker_public/src/requestScope.js +11 -5
- package/docs/spec-pipeline.md +1 -0
- package/package.json +1 -1
- package/profile/prompts.yaml +14 -0
- package/profile/sources.yaml +9 -0
- package/workers/shared/chunk.ts +7 -0
- package/workers/worker_internal/src/index.ts +3 -3
- package/workers/worker_public/prompts/system.md +1 -0
- package/workers/worker_public/src/ask.ts +33 -10
- package/workers/worker_public/src/index.ts +2 -1
- package/workers/worker_public/src/modelplane.ts +94 -4
- package/workers/worker_public/src/pipeline.ts +13 -5
- package/workers/worker_public/src/profile.gen.ts +13 -2
- package/workers/worker_public/src/requestScope.ts +49 -7
- package/workers/worker_public/src/search.ts +7 -1
- package/workers/worker_public/src/selfquery.ts +26 -0
- package/workers/worker_public/src/stages/citationProbe.ts +6 -1
- package/workers/worker_public/src/stages/index.ts +2 -0
- package/workers/worker_public/src/stages/licenseScope.ts +23 -0
- package/workers/worker_public/src/stages/types.ts +11 -0
- package/dist/ask-VFSAF5WG.js +0 -12
- package/dist/search-OMPBMZT4.js +0 -11
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
import {
|
|
2
2
|
handleSearch
|
|
3
|
-
} from "../../chunk-
|
|
3
|
+
} from "../../chunk-THSHLUOS.js";
|
|
4
4
|
import {
|
|
5
5
|
checkQuoteAnchors,
|
|
6
6
|
handleAsk,
|
|
7
7
|
handleMemories,
|
|
8
8
|
scoreJudge,
|
|
9
9
|
standardForDocNumber
|
|
10
|
-
} from "../../chunk-
|
|
11
|
-
import "../../chunk-LNSDBEKS.js";
|
|
10
|
+
} from "../../chunk-OGFH3RDM.js";
|
|
12
11
|
import {
|
|
13
12
|
buildMessages,
|
|
14
13
|
citations,
|
|
@@ -30,7 +29,7 @@ import {
|
|
|
30
29
|
sessionFrom,
|
|
31
30
|
telemetry,
|
|
32
31
|
understandQuery
|
|
33
|
-
} from "../../chunk-
|
|
32
|
+
} from "../../chunk-RLT4W2VX.js";
|
|
34
33
|
import {
|
|
35
34
|
authenticate,
|
|
36
35
|
corsHeaders,
|
|
@@ -39,10 +38,14 @@ import {
|
|
|
39
38
|
readJson,
|
|
40
39
|
validateQuery,
|
|
41
40
|
withCors
|
|
42
|
-
} from "../../chunk-
|
|
41
|
+
} from "../../chunk-EFQALN2Z.js";
|
|
43
42
|
import {
|
|
44
43
|
canonicalRefusal
|
|
45
|
-
} from "../../chunk-
|
|
44
|
+
} from "../../chunk-DBBGOOMZ.js";
|
|
45
|
+
import {
|
|
46
|
+
entitlementScope,
|
|
47
|
+
standardKeysFrom
|
|
48
|
+
} from "../../chunk-5MBWE7WD.js";
|
|
46
49
|
import {
|
|
47
50
|
LIMITS,
|
|
48
51
|
MODELS,
|
|
@@ -53,11 +56,11 @@ import {
|
|
|
53
56
|
roleModel,
|
|
54
57
|
sha256Hex,
|
|
55
58
|
today
|
|
56
|
-
} from "../../chunk-
|
|
59
|
+
} from "../../chunk-ADXV2DPK.js";
|
|
57
60
|
import {
|
|
58
61
|
P,
|
|
59
62
|
setProfile
|
|
60
|
-
} from "../../chunk-
|
|
63
|
+
} from "../../chunk-TJRTVJW5.js";
|
|
61
64
|
|
|
62
65
|
// workers/worker_public/src/conversations.ts
|
|
63
66
|
var ID_RE = /^[a-zA-Z0-9_-]{8,64}$/;
|
|
@@ -976,7 +979,7 @@ async function handleMcp(env, ctx, req, tier, key) {
|
|
|
976
979
|
// stream:false forces the JSON lane (anon defaults to SSE)
|
|
977
980
|
body: JSON.stringify({ ...args, stream: false })
|
|
978
981
|
});
|
|
979
|
-
const res = name === "ask" ? await (await import("../../ask-
|
|
982
|
+
const res = name === "ask" ? await (await import("../../ask-YCDXMIZ4.js")).handleAsk(env, ctx, inner, tier, key) : await (await import("../../search-7TO2RWQE.js")).handleSearch(env, ctx, inner, tier, key);
|
|
980
983
|
return res.json().catch(() => ({ error: { message: "tool transport failed", status: res.status } }));
|
|
981
984
|
});
|
|
982
985
|
if (out.ok && "accepted" in out) return new Response(null, { status: 202 });
|
|
@@ -1435,7 +1438,7 @@ async function verifyRoute(c) {
|
|
|
1435
1438
|
if (!answer || !query) return err(400, "invalid_input", "answer and query are required");
|
|
1436
1439
|
try {
|
|
1437
1440
|
const u = await understandQuery(env.AI, roleModel(env, "understand"), query, [], []);
|
|
1438
|
-
const retrieved = await retrieve(env, query, { understanding: u });
|
|
1441
|
+
const retrieved = await retrieve(env, query, { understanding: u, standardKeys: entitlementScope(standardKeysFrom(body)) });
|
|
1439
1442
|
const passages = retrieved.hits.map((h) => h.text);
|
|
1440
1443
|
const anchors = checkQuoteAnchors(answer, passages);
|
|
1441
1444
|
const refs = [...answer.matchAll(/\[\[u:([^\]]+)\]\]/g)].map((m) => m[1]);
|
|
@@ -1,10 +1,16 @@
|
|
|
1
1
|
import {
|
|
2
|
+
entitlementScope,
|
|
3
|
+
licenseDeclared,
|
|
2
4
|
requestSalt,
|
|
3
|
-
resolveRequestScope
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
import "../../chunk-
|
|
5
|
+
resolveRequestScope,
|
|
6
|
+
standardKeysFrom
|
|
7
|
+
} from "../../chunk-5MBWE7WD.js";
|
|
8
|
+
import "../../chunk-ADXV2DPK.js";
|
|
9
|
+
import "../../chunk-TJRTVJW5.js";
|
|
7
10
|
export {
|
|
11
|
+
entitlementScope,
|
|
12
|
+
licenseDeclared,
|
|
8
13
|
requestSalt,
|
|
9
|
-
resolveRequestScope
|
|
14
|
+
resolveRequestScope,
|
|
15
|
+
standardKeysFrom
|
|
10
16
|
};
|
package/docs/spec-pipeline.md
CHANGED
|
@@ -47,6 +47,7 @@ candidate lanes → pool open → pool-level merges → refinement → window as
|
|
|
47
47
|
| 9 | `lexical-union` | `lexicalHits.length` | blocking | appends `hits` | Full-corpus BM25 hits dense missed (dense metadata wins on id collision). |
|
|
48
48
|
| 10 | `federate` | `opts.federate` | additive (callback catches transport) | appends `hits` | Internal ISO/IEC passages at `federateDiscount`. |
|
|
49
49
|
| 11 | `seal` | `opts.sealScope` | blocking | filters `hits` | The declared context's hard cut: nothing outside the declared family reaches rerank. (The lexical lane is sealed at the SOURCE, in the prelude — not here.) |
|
|
50
|
+
| 11b | `license-scope` | `opts.standardKeys` | blocking | filters `hits` | The license entitlement hard cut (TODO.external-refs/08): a chunk carrying a `standard_key` outside the caller's set never reaches rerank; public chunks (no key) always pass. (The lexical lane is bound at the SOURCE, in the prelude — the lexical union AND the post-rerank lexical-rrf consume the bound list. The post-scope single-document fetches — section-descent, typed-pin parent, edition-cover — are doc-scoped, and a document's key is a per-document constant, so they cannot re-admit a dropped key; citation-probe filters its bibliography push by the same predicate.) |
|
|
50
51
|
| 12 | `overview-demote` | — | blocking | mutates `hits[].score` | Overview boilerplate demotion (`overviewDemotion`). |
|
|
51
52
|
| 13 | `family-boost` | boost only under `filter.doc_number`; sort always | blocking | mutates `hits[].score`, sorts | Family chunks decisively boosted for doc-scoped queries; the sort establishes the rerank-failure fallback order. |
|
|
52
53
|
| 14 | `rerank` | `hits.length > 1` | additive | `hits[].rerank_score`, sorts, family pin | Cross-encoder scores; vector order is the designed fallback. Post-rerank family pin for doc-scoped queries. |
|
package/package.json
CHANGED
|
@@ -36,7 +36,7 @@
|
|
|
36
36
|
"@konneal/engine": "github:konneal/engine#2d6c39b",
|
|
37
37
|
"@oimlsmart/oiml-pubid": "^1.2.1"
|
|
38
38
|
},
|
|
39
|
-
"version": "0.2.
|
|
39
|
+
"version": "0.2.3",
|
|
40
40
|
"description": "The Konneal engine: the publisher-agnostic build pipeline and API plane for standards intelligence (retrieval, answer contract, verdicts, evaluation).",
|
|
41
41
|
"license": "BSD-3-Clause",
|
|
42
42
|
"type": "module",
|
package/profile/prompts.yaml
CHANGED
|
@@ -17,3 +17,17 @@ vars:
|
|
|
17
17
|
docid_example: "FIXTURE 1-2"
|
|
18
18
|
spelling_examples: '"f1", "FIXTURE 1"'
|
|
19
19
|
process_vocab: "the fixture certification system framework"
|
|
20
|
+
# the license declare flow (TODO.external-refs/08): where an org admin
|
|
21
|
+
# declares a standards license — the boundary refusal's pointer
|
|
22
|
+
license_declare_pointer: "org admin → Settings → Standards licenses"
|
|
23
|
+
# the licensed-content posture (TODO.external-refs/08): how the answer
|
|
24
|
+
# model behaves when a question hits the license boundary (the
|
|
25
|
+
# per-question note rides the retrieval-note channel)
|
|
26
|
+
license_posture: >-
|
|
27
|
+
Some indexed publications are LICENSED. When a license boundary note
|
|
28
|
+
names the question's publication, obey it: answer at the citation
|
|
29
|
+
level only — the standard's title and edition, and the invoking clause
|
|
30
|
+
the public passages carry — never state, paraphrase or "summarize from
|
|
31
|
+
memory" any procedure of the licensed text, and point at the declare
|
|
32
|
+
flow the note names. Public content (the Recommendations' own models,
|
|
33
|
+
applicability and references) stays fully answerable.
|
package/profile/sources.yaml
CHANGED
|
@@ -3,3 +3,12 @@ corpora:
|
|
|
3
3
|
bibliography: {}
|
|
4
4
|
terminology: {}
|
|
5
5
|
models: {}
|
|
6
|
+
# the licensed-standards declaration (TODO.external-refs/08): the
|
|
7
|
+
# entitlement keys the deployment's index may carry (`standard_key` chunk
|
|
8
|
+
# metadata, from the packages' license_key facet) — this list is the
|
|
9
|
+
# whitelist the request's entitlement set validates against, and the
|
|
10
|
+
# package/doc-number mapping the ask path gates model bindings with.
|
|
11
|
+
# Absent or empty = the deployment serves public content only. The
|
|
12
|
+
# fixture declares one so the license scope rides every engine test.
|
|
13
|
+
licensed:
|
|
14
|
+
- { key: "std:fixture-60068-2-30", package: "fixture-60068-2-30", doc_number: "60068-2-30", title: "FIXTURE environmental testing — damp heat, cyclic", edition: "2005" }
|
package/workers/shared/chunk.ts
CHANGED
|
@@ -35,6 +35,12 @@ export interface ChunkMeta {
|
|
|
35
35
|
model_node?: string;
|
|
36
36
|
model_kind?: string;
|
|
37
37
|
standard?: string;
|
|
38
|
+
/** The license entitlement key (the package's `license_key` facet, e.g.
|
|
39
|
+
* `std:iec-60068-2-30`) — present ONLY when the chunk's content comes
|
|
40
|
+
* from a licensed package; public content carries none. The ask path's
|
|
41
|
+
* hard scope (stages/licenseScope.ts) drops chunks whose key is absent
|
|
42
|
+
* from the caller's entitlement set, before ranking. */
|
|
43
|
+
standard_key?: string;
|
|
38
44
|
/** section-summary unit (FABLE multi-granularity, arXiv:2601.18116): a
|
|
39
45
|
* depth-1 clause summary vector — a navigation node whose children
|
|
40
46
|
* (child_anchors CSV) are quotable leaf clauses. The corpus's real
|
|
@@ -83,6 +89,7 @@ export const CHUNK_META_FIELDS = [
|
|
|
83
89
|
"model_node",
|
|
84
90
|
"model_kind",
|
|
85
91
|
"standard",
|
|
92
|
+
"standard_key",
|
|
86
93
|
"section_summary",
|
|
87
94
|
"child_anchors",
|
|
88
95
|
"ctx",
|
|
@@ -85,7 +85,7 @@ async function adminSyncRoute(c: RouteContext): Promise<Response> {
|
|
|
85
85
|
if (Array.isArray(body?.upserts)) {
|
|
86
86
|
for (let i = 0; i < body.upserts.length; i += 100) {
|
|
87
87
|
const batch = body.upserts.slice(i, i + 100).filter((v: any) => v?.id && Array.isArray(v?.values) && v?.metadata);
|
|
88
|
-
if (batch.length) await env.
|
|
88
|
+
if (batch.length) await env.INTERNAL.upsert(batch);
|
|
89
89
|
out.upserted += batch.length;
|
|
90
90
|
}
|
|
91
91
|
}
|
|
@@ -94,14 +94,14 @@ async function adminSyncRoute(c: RouteContext): Promise<Response> {
|
|
|
94
94
|
for (let i = 0; i < todo.length; i += 16) {
|
|
95
95
|
const batch = todo.slice(i, i + 16);
|
|
96
96
|
const vectors = await Promise.all(batch.map((b: any) => embed(env.AI, b.text.slice(0, 6000))));
|
|
97
|
-
await env.
|
|
97
|
+
await env.INTERNAL.upsert(batch.map((b: any, j: number) => ({ id: b.id, values: vectors[j], metadata: b.metadata })));
|
|
98
98
|
out.embedded += batch.length;
|
|
99
99
|
}
|
|
100
100
|
}
|
|
101
101
|
if (Array.isArray(body?.deletes)) {
|
|
102
102
|
const ids = body.deletes.filter((x: any) => typeof x === "string");
|
|
103
103
|
for (let i = 0; i < ids.length; i += 100) {
|
|
104
|
-
await env.
|
|
104
|
+
await env.INTERNAL.deleteByIds(ids.slice(i, i + 100));
|
|
105
105
|
out.deleted += Math.min(100, ids.length - i);
|
|
106
106
|
}
|
|
107
107
|
}
|
|
@@ -14,6 +14,7 @@ Synthesize practical answers from the passages: definitions, procedures and rule
|
|
|
14
14
|
MANDATORY: when the question asks how to do something (get certified, apply, comply, register, test) and the passages describe the governing system or procedure, ALWAYS answer with that procedure citing the governing documents. Refusing such a question because the passages do not name the specific publication is WRONG — the publication sets technical requirements; the HOW is governed by the certification-system documents in the passages.
|
|
15
15
|
If the passages cover only part of the question, answer the covered part fully, then state precisely what the indexed publications do not cover — do not pad with outside knowledge.
|
|
16
16
|
Refuse ONLY when no passage relates to the question's topic. Use exactly this sentence: {{REFUSAL_SENTENCE}} Then add one short line naming what you can answer instead, so the refusal redirects rather than dead-ends.
|
|
17
|
+
{{LICENSE_POSTURE}}
|
|
17
18
|
{{CORPUS_NOTES}}
|
|
18
19
|
Lead with the direct answer, then supporting detail; no preamble like 'Based on the passages'. Use short paragraphs or bullets for multi-part answers. Be concise and precise. Answer in the question's language{{LANG_CLAUSE}}.
|
|
19
20
|
- HARD RULE — typed units: passages whose header shows `unit u:xxxx (table)` contain a typed table. If your answer presents that table's data, you MUST write the token `[[u:xxxx]]` where the table belongs and MUST NOT render the table as markdown or reproduce more than ONE of its rows inline. Summarize the pattern in prose ("classes A–D with lower limits from 100 to 50 000"), cite the clause normally, and let `[[u:xxxx]]` stand for the full table — the interface renders it exactly from the source. The same rule applies to `unit u:xxxx (formula|figure|term)` objects.
|
|
@@ -19,11 +19,11 @@ import { contractV2, tableRetyped } from "./refs";
|
|
|
19
19
|
import { completeTables, completeFigures } from "./completion";
|
|
20
20
|
import { NO_CONTEXT, appliedContext, contextNote, namedDocumentIn, parseContext, resolveDocScope, syntheticUnderstanding } from "./context";
|
|
21
21
|
import { exchangeForLiveToken, liveDataConfig, resolveLiveAccount, type LiveRecord } from "./livedata";
|
|
22
|
-
import { bindModelNode, modelCitation, modelEcho, modelGroundingBlock, modelNodeRefIn, standardForDocNumber } from "./modelplane";
|
|
22
|
+
import { bindModelNode, licenseBoundaryNote, licenseBoundaryRefusal, modelCitation, modelEcho, modelGroundingBlock, modelNodeRefIn, standardForDocNumber } from "./modelplane";
|
|
23
23
|
import { evaluate as machineEvaluate, verdictNote } from "./verdict";
|
|
24
24
|
import { detectDraftIntent, prepareDraft } from "./drafts";
|
|
25
25
|
import { memoryNote } from "./memories";
|
|
26
|
-
import { resolveRequestScope, requestSalt } from "./requestScope";
|
|
26
|
+
import { entitlementScope, resolveRequestScope, requestSalt } from "./requestScope";
|
|
27
27
|
import { rawSessionToken } from "./session";
|
|
28
28
|
import { cacheKeyMaterial, corpusGen, exactCacheKey, freshRequested, semanticCacheKey } from "./answercache";
|
|
29
29
|
import type { Env } from "./env";
|
|
@@ -308,6 +308,13 @@ async function handleAsk(
|
|
|
308
308
|
const scope = resolveRequestScope(body, member);
|
|
309
309
|
if ("error" in scope) return err(400, "invalid_input", "datasets: at least one dataset must stay enabled");
|
|
310
310
|
const { corpora, narrowed, isoOn } = scope;
|
|
311
|
+
// The license entitlement set (TODO.external-refs/08): request-scoped,
|
|
312
|
+
// validated against the profile's declared licensed list; the hard
|
|
313
|
+
// retrieval scope it feeds is fail-closed (an unentitled ask — empty
|
|
314
|
+
// set, the anon default — never sees licensed chunks, citation-level
|
|
315
|
+
// metadata stays). It salts the answer cache with the effort segment
|
|
316
|
+
// below.
|
|
317
|
+
const standardKeys = entitlementScope(scope.standardKeys);
|
|
311
318
|
// Personalized memory files (#171): member-scoped, selected per ask;
|
|
312
319
|
// the note rides buildMessages as a trusted-user-facts preamble, and
|
|
313
320
|
// the selections SALT the answer cache (requestScope.requestSalt).
|
|
@@ -692,18 +699,23 @@ async function handleAsk(
|
|
|
692
699
|
label: declaredCtx?.label,
|
|
693
700
|
query: q.query,
|
|
694
701
|
standard: standardForDocNumber(modelDocHint?.doc_number),
|
|
702
|
+
standardKeys,
|
|
695
703
|
})
|
|
696
704
|
: null;
|
|
697
705
|
if (boundModel) {
|
|
698
706
|
ctxApplied = { ...ctxApplied, model: modelEcho(boundModel) };
|
|
699
|
-
console.log("model plane: bound", boundModel.node_id, `[${boundModel.standard}]`, boundModel.clause?.urn ?? "no-clause");
|
|
707
|
+
console.log("model plane: bound", boundModel.node_id, `[${boundModel.standard}]`, boundModel.clause?.urn ?? "no-clause", boundModel.gated ? "(gated: license)" : "");
|
|
700
708
|
}
|
|
701
|
-
|
|
709
|
+
// A GATED binding (licensed package, unentitled caller) keeps its
|
|
710
|
+
// citation + echo — metadata the honesty posture keeps — and withholds
|
|
711
|
+
// the grounding block and the verdict engine: no licensed machine
|
|
712
|
+
// content enters the prompt by the binding lane either.
|
|
713
|
+
const modelNote = boundModel && !boundModel.gated ? modelGroundingBlock(boundModel) : undefined;
|
|
702
714
|
// ── the verdict engine (TODO.era3/01) ──
|
|
703
715
|
// the worker EXECUTES the bound node's machine checks against the
|
|
704
716
|
// question's stated values; the model narrates the computed verdict
|
|
705
717
|
// and the verdict BLOCK is server-built — data, never generated prose
|
|
706
|
-
const machineVerdict = boundModel ? machineEvaluate(boundModel.content, q.query) : null;
|
|
718
|
+
const machineVerdict = boundModel && !boundModel.gated ? machineEvaluate(boundModel.content, q.query) : null;
|
|
707
719
|
const machineNote = machineVerdict && boundModel ? verdictNote(machineVerdict, boundModel) : undefined;
|
|
708
720
|
const verdictBlock = machineVerdict
|
|
709
721
|
? {
|
|
@@ -776,7 +788,7 @@ async function handleAsk(
|
|
|
776
788
|
// publications there).
|
|
777
789
|
retrieved = await retrieve(env, q.query, { prev, understanding, federate, warmEmbed, graphDocNumbers,
|
|
778
790
|
sealScope: declaredScoped ? docScope : null, optimisticHits, optimisticVec,
|
|
779
|
-
datasetScope: narrowed ? corpora : null });
|
|
791
|
+
datasetScope: narrowed ? corpora : null, standardKeys });
|
|
780
792
|
stageTiming["retrieve-core"] = Date.now() - tR;
|
|
781
793
|
console.log("stage: retrieve", Date.now() - tR, "ms");
|
|
782
794
|
// ── TTFT surgery: the two post-retrieval LLM calls run IN PARALLEL —
|
|
@@ -810,7 +822,7 @@ async function handleAsk(
|
|
|
810
822
|
if (grade === "weak" && understanding?.docidentifier) {
|
|
811
823
|
const broaden = `${understanding.standalone_query || q.query} ${understanding.docidentifier}`.trim();
|
|
812
824
|
const tc = Date.now();
|
|
813
|
-
const second = await retrieve(env, q.query, { prev, understanding, queryOverride: broaden, federate, datasetScope: narrowed ? corpora : null });
|
|
825
|
+
const second = await retrieve(env, q.query, { prev, understanding, queryOverride: broaden, federate, datasetScope: narrowed ? corpora : null, standardKeys, sealScope: declaredScoped ? docScope : null });
|
|
814
826
|
const grade2 = await gradeRetrieval(env.AI, roleModel(env, "grader"), q.query, second.hits.map((h: Hit) => h.text));
|
|
815
827
|
stageTiming.corrective = Date.now() - tc;
|
|
816
828
|
if (grade2 === "good") retrieved = second; // corrective retry must be strictly better
|
|
@@ -821,8 +833,13 @@ async function handleAsk(
|
|
|
821
833
|
return err(503, "retrieval_unavailable", "Search is briefly busy — please retry in a moment.");
|
|
822
834
|
}
|
|
823
835
|
const { hits } = retrieved;
|
|
824
|
-
if (hits.length === 0 && !liveRecords?.length && !boundModel) {
|
|
825
|
-
|
|
836
|
+
if (hits.length === 0 && !liveRecords?.length && (!boundModel || boundModel.gated)) {
|
|
837
|
+
// the license boundary refusal (TODO.external-refs/08) outranks the
|
|
838
|
+
// plain "no information" sentence when the question's publication is
|
|
839
|
+
// licensed and unentitled: name the standard, state the boundary,
|
|
840
|
+
// point at the declare flow
|
|
841
|
+
const answer =
|
|
842
|
+
licenseBoundaryRefusal(modelDocHint?.doc_number ?? understanding?.doc_number ?? null, standardKeys) ?? refusalAnswer();
|
|
826
843
|
const out = { answer, citations: [], model, query_hash: await sha256Hex(q.query), context_applied: ctxApplied };
|
|
827
844
|
telemetry(env, ctx, tier, "ask", model, true, answer.length, out.query_hash, q.lang, undefined, telemetryMeta());
|
|
828
845
|
return json({ ...out, quota, });
|
|
@@ -831,6 +848,11 @@ async function handleAsk(
|
|
|
831
848
|
const processNote = understanding?.process_intent
|
|
832
849
|
? P().retrieval.process_note
|
|
833
850
|
: undefined;
|
|
851
|
+
// the license boundary (TODO.external-refs/08): the question's named or
|
|
852
|
+
// understood publication is licensed and the caller's set lacks the key
|
|
853
|
+
// — the refusal-class note rides the retrieval-note channel beside the
|
|
854
|
+
// other structured facts
|
|
855
|
+
const licenseNote = licenseBoundaryNote(modelDocHint?.doc_number ?? understanding?.doc_number ?? null, standardKeys);
|
|
834
856
|
// the vocabulary binding (L2): the corpus's defined-term candidates for
|
|
835
857
|
// the question's subject — the model adjudicates among them and uses
|
|
836
858
|
// the corpus term (with its defining publication) when it names the
|
|
@@ -859,7 +881,7 @@ async function handleAsk(
|
|
|
859
881
|
q.lang,
|
|
860
882
|
keptHistory,
|
|
861
883
|
// stage-extracted graph facts (GraphRAG) ride the same note channel
|
|
862
|
-
[processNote, eNote, contextNote(declaredCtx, docScope), accountNote, modelNote, vocabNote, memNote, machineNote, ...(retrieved.notes ?? [])].filter(Boolean).join("\n") || undefined,
|
|
884
|
+
[processNote, eNote, contextNote(declaredCtx, docScope), accountNote, modelNote, vocabNote, memNote, machineNote, licenseNote, ...(retrieved.notes ?? [])].filter(Boolean).join("\n") || undefined,
|
|
863
885
|
summary,
|
|
864
886
|
budget,
|
|
865
887
|
);
|
|
@@ -1046,6 +1068,7 @@ async function handleAsk(
|
|
|
1046
1068
|
understanding: { ...understanding, standalone_query: `${understanding?.standalone_query || q.query} ${reflection.missing_info}` } as any,
|
|
1047
1069
|
sealScope: declaredScoped ? docScope : null,
|
|
1048
1070
|
datasetScope: narrowed ? corpora : null,
|
|
1071
|
+
standardKeys,
|
|
1049
1072
|
});
|
|
1050
1073
|
if (retryRetrieve.hits.length > 0) {
|
|
1051
1074
|
const { messages: retryMessages, usedHits: retryUsed } = buildMessages(q.query, retryRetrieve.hits, q.lang, keptHistory, undefined, summary, budget);
|
|
@@ -14,6 +14,7 @@ import { scoreFaithfulness } from "./faithfulness";
|
|
|
14
14
|
import { checkQuoteAnchors } from "./anchors";
|
|
15
15
|
import { namedDocumentIn } from "./context";
|
|
16
16
|
import { standardForDocNumber } from "./modelplane";
|
|
17
|
+
import { entitlementScope, standardKeysFrom } from "./requestScope";
|
|
17
18
|
import type { Env } from "./env";
|
|
18
19
|
export type { Env };
|
|
19
20
|
import { json, err, corsHeaders, withCors, readJson, authenticate, type ApiKey } from "./lib/http";
|
|
@@ -227,7 +228,7 @@ async function verifyRoute(c: RouteContext): Promise<Response> {
|
|
|
227
228
|
if (!answer || !query) return err(400, "invalid_input", "answer and query are required");
|
|
228
229
|
try {
|
|
229
230
|
const u = await understandQuery(env.AI, roleModel(env, "understand"), query, [], []);
|
|
230
|
-
const retrieved = await retrieve(env, query, { understanding: u });
|
|
231
|
+
const retrieved = await retrieve(env, query, { understanding: u, standardKeys: entitlementScope(standardKeysFrom(body)) });
|
|
231
232
|
const passages = retrieved.hits.map((h: Hit) => h.text);
|
|
232
233
|
const anchors = checkQuoteAnchors(answer, passages);
|
|
233
234
|
const refs = [...answer.matchAll(/\[\[u:([^\]]+)\]\]/g)].map((m) => m[1]);
|
|
@@ -52,6 +52,82 @@ export function standardForDocNumber(docNumber: string | undefined): string | nu
|
|
|
52
52
|
return (models.standards as string[]).includes(docNumber) ? `${models.standard_prefix}${docNumber}` : null;
|
|
53
53
|
}
|
|
54
54
|
|
|
55
|
+
// ── the license tier (TODO.external-refs/08) ─────────────────────────────
|
|
56
|
+
// The profile's `sources.licensed` rows ({key, package, doc_number,
|
|
57
|
+
// title, edition}) are the deployment's declared licensed standards —
|
|
58
|
+
// the same keys the chunk metadata carries (`standard_key`) and the
|
|
59
|
+
// entitlement set arrives with (requestScope.standardKeysFrom validates
|
|
60
|
+
// against exactly this list).
|
|
61
|
+
|
|
62
|
+
/** The license entry for a package id (the model-plane standard id, e.g.
|
|
63
|
+
* `iec-60068-2-30`), or null when the package is public content. */
|
|
64
|
+
export function licensedEntryForPackage(packageId: string | undefined | null):
|
|
65
|
+
| { key: string; package: string; doc_number?: string; title?: string; edition?: string }
|
|
66
|
+
| null {
|
|
67
|
+
if (!packageId) return null;
|
|
68
|
+
return (P().sources?.licensed ?? []).find((l: any) => l.package === packageId) ?? null;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** The license entry for a doc number (the question's named publication,
|
|
72
|
+
* e.g. "60068-2-30"), or null when unnamed/public. */
|
|
73
|
+
export function licensedEntryForDocNumber(docNumber: string | undefined | null):
|
|
74
|
+
| { key: string; package: string; doc_number?: string; title?: string; edition?: string }
|
|
75
|
+
| null {
|
|
76
|
+
if (!docNumber) return null;
|
|
77
|
+
return (P().sources?.licensed ?? []).find((l: any) => String(l.doc_number ?? "") === docNumber) ?? null;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** The per-question license boundary note: composed ONLY when the
|
|
81
|
+
* question's named/understood publication is licensed AND the caller's
|
|
82
|
+
* entitlement set does not carry its key. The honesty posture, made
|
|
83
|
+
* structural: name the standard, say the organization's license does
|
|
84
|
+
* not cover its text, keep every procedural claim out, point at the
|
|
85
|
+
* declare flow. Citation-level metadata (title, edition, the invoking
|
|
86
|
+
* clause the RECs publicly name) stays answerable from the public
|
|
87
|
+
* passages already in context — the note instructs exactly that. */
|
|
88
|
+
export function licenseBoundaryNote(
|
|
89
|
+
docNumber: string | undefined | null,
|
|
90
|
+
standardKeys: ReadonlySet<string> | null | undefined,
|
|
91
|
+
): string | undefined {
|
|
92
|
+
const entry = licensedEntryForDocNumber(docNumber);
|
|
93
|
+
if (!entry || (standardKeys && standardKeys.has(entry.key))) return undefined;
|
|
94
|
+
const pointer = P().prompts?.vars?.license_declare_pointer;
|
|
95
|
+
return (
|
|
96
|
+
`License boundary — the question is about ${licenseBoundaryName(entry)}, a licensed publication` +
|
|
97
|
+
` (entitlement key ${entry.key}). The caller's organization license does not cover its text, so no passage of it was retrieved` +
|
|
98
|
+
` and NONE of its procedural content (steps, parameters, severities, limits) may be stated, paraphrased or recalled from memory.` +
|
|
99
|
+
` You MAY answer at the citation level: name the standard and edition, and cite the invoking clause from the PUBLIC passages in context` +
|
|
100
|
+
` (the Recommendation's own applicability and normative references are public and stay answerable).` +
|
|
101
|
+
` Then say the organization's license does not cover the standard's text` +
|
|
102
|
+
(pointer ? ` and point to the declare flow: ${pointer}.` : ".")
|
|
103
|
+
);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function licenseBoundaryName(entry: { title?: string; edition?: string; package: string; doc_number?: string }): string {
|
|
107
|
+
const id = entry.doc_number ? ` ${entry.doc_number}` : ` ${entry.package}`;
|
|
108
|
+
return `${entry.title ?? "standard"}${entry.edition ? ` (${entry.edition})` : ""} —${id}`;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/** The deterministic boundary answer for the zero-passage case: the
|
|
112
|
+
* question's licensed publication has nothing to show an unentitled
|
|
113
|
+
* caller — name the standard, state the boundary, point at the declare
|
|
114
|
+
* flow. Undefined when the question is not the licensed case (the plain
|
|
115
|
+
* refusal applies). */
|
|
116
|
+
export function licenseBoundaryRefusal(
|
|
117
|
+
docNumber: string | undefined | null,
|
|
118
|
+
standardKeys: ReadonlySet<string> | null | undefined,
|
|
119
|
+
): string | undefined {
|
|
120
|
+
const entry = licensedEntryForDocNumber(docNumber);
|
|
121
|
+
if (!entry || (standardKeys && standardKeys.has(entry.key))) return undefined;
|
|
122
|
+
const pointer = P().prompts?.vars?.license_declare_pointer;
|
|
123
|
+
return (
|
|
124
|
+
`${licenseBoundaryName(entry)} is a licensed publication and your organization's license does not cover its text, ` +
|
|
125
|
+
`so I can't quote or summarize its procedure. I can answer at the citation level — the standard's title and edition, ` +
|
|
126
|
+
`and the clause your Recommendation invokes — and the public ${P().publisher.name} content in full.` +
|
|
127
|
+
(pointer ? ` To unlock the full text, an org admin can declare the license under ${pointer}.` : "")
|
|
128
|
+
);
|
|
129
|
+
}
|
|
130
|
+
|
|
55
131
|
export interface BoundModelNode {
|
|
56
132
|
standard: string;
|
|
57
133
|
node_id: string;
|
|
@@ -60,6 +136,12 @@ export interface BoundModelNode {
|
|
|
60
136
|
clause: { doc: string; ref: string; urn: string } | null;
|
|
61
137
|
/** The node's bundle projection (verbatim JSON). */
|
|
62
138
|
content: any;
|
|
139
|
+
/** True when the node's package is licensed and the caller's
|
|
140
|
+
* entitlement set lacks the key (TODO.external-refs/08): the citation
|
|
141
|
+
* and the echo stay (metadata), but the grounding block and the
|
|
142
|
+
* verdict engine are withheld — no licensed machine content enters
|
|
143
|
+
* the prompt. */
|
|
144
|
+
gated?: boolean;
|
|
63
145
|
}
|
|
64
146
|
|
|
65
147
|
async function fetchNode(env: any, standard: string, nodeId: string): Promise<BoundModelNode | null> {
|
|
@@ -97,18 +179,26 @@ async function fetchNode(env: any, standard: string, nodeId: string): Promise<Bo
|
|
|
97
179
|
* model-aware chip), then a node id the question names. The standard
|
|
98
180
|
* comes from the declared doc scope when it carries one; without a scope
|
|
99
181
|
* the node binds only when it exists in EXACTLY ONE indexed standard —
|
|
100
|
-
* ambiguity is refused honestly (retrieval still surfaces the chunks).
|
|
182
|
+
* ambiguity is refused honestly (retrieval still surfaces the chunks).
|
|
183
|
+
* A licensed package binds GATED for an unentitled caller (metadata
|
|
184
|
+
* only — the grounding block and the verdict engine are the ask path's
|
|
185
|
+
* to withhold). */
|
|
101
186
|
export async function bindModelNode(
|
|
102
187
|
env: any,
|
|
103
|
-
opts: { label?: string; query: string; standard?: string | null },
|
|
188
|
+
opts: { label?: string; query: string; standard?: string | null; standardKeys?: ReadonlySet<string> | null },
|
|
104
189
|
): Promise<BoundModelNode | null> {
|
|
105
190
|
const nodeId = modelNodeRefIn(opts.label) ?? modelNodeRefIn(opts.query);
|
|
106
191
|
if (!nodeId) return null;
|
|
107
|
-
|
|
192
|
+
const gate = (node: BoundModelNode | null): BoundModelNode | null => {
|
|
193
|
+
if (!node) return null;
|
|
194
|
+
const entry = licensedEntryForPackage(node.standard);
|
|
195
|
+
return entry && !(opts.standardKeys?.has(entry.key) ?? false) ? { ...node, gated: true, content: {} } : node;
|
|
196
|
+
};
|
|
197
|
+
if (opts.standard) return gate(await fetchNode(env, opts.standard, nodeId));
|
|
108
198
|
try {
|
|
109
199
|
const rows = await env.DB.prepare("SELECT standard FROM model_nodes WHERE node_id = ?1 LIMIT 2").bind(nodeId).all();
|
|
110
200
|
const standards = (rows?.results ?? []).map((r: any) => String(r.standard));
|
|
111
|
-
if (standards.length === 1) return fetchNode(env, standards[0]!, nodeId);
|
|
201
|
+
if (standards.length === 1) return gate(await fetchNode(env, standards[0]!, nodeId));
|
|
112
202
|
return null; // zero (not indexed) or ambiguous (several standards) — no silent pick
|
|
113
203
|
} catch {
|
|
114
204
|
return null;
|
|
@@ -32,7 +32,7 @@ export function promptVars(extra: Record<string, string> = {}): Record<string, s
|
|
|
32
32
|
export function fill(template: string, vars: Record<string, string>): string {
|
|
33
33
|
return template.replace(/\{\{(\w+)\}\}/g, (_m, k: string) => (k in vars ? vars[k] : ""));
|
|
34
34
|
}
|
|
35
|
-
import { QueryFilters, toVectorizeFilter } from "./selfquery";
|
|
35
|
+
import { QueryFilters, toVectorizeFilter, standardKeyAllowed } from "./selfquery";
|
|
36
36
|
import { lexicalPrefilter } from "./lexical";
|
|
37
37
|
import { positionOrder } from "./structural";
|
|
38
38
|
import { STAGES, runStages } from "./stages";
|
|
@@ -125,10 +125,18 @@ export async function retrieve(
|
|
|
125
125
|
// The declared context's seal binds the lexical lane at the SOURCE: the
|
|
126
126
|
// RRF fusion mixes the full-corpus lexical ranking straight into the
|
|
127
127
|
// final hits — past the pool-level seal — so under a seal the lexical
|
|
128
|
-
// lane is the FAMILY's lexical hits only.
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
128
|
+
// lane is the FAMILY's lexical hits only. The license entitlement scope
|
|
129
|
+
// binds the same lane the same way (it re-enters twice: lexical-union
|
|
130
|
+
// pre-rerank and lexical-rrf post-rerank — both consume this list).
|
|
131
|
+
const lexicalHits = (opts.sealScope || opts.standardKeys
|
|
132
|
+
? lexicalHits0.filter(
|
|
133
|
+
(h) =>
|
|
134
|
+
(!opts.sealScope ||
|
|
135
|
+
(h.metadata.doc_number === opts.sealScope!.doc_number &&
|
|
136
|
+
(!opts.sealScope!.edition || h.metadata.edition === opts.sealScope!.edition))) &&
|
|
137
|
+
standardKeyAllowed(h.metadata, opts.standardKeys),
|
|
138
|
+
)
|
|
139
|
+
: lexicalHits0);
|
|
132
140
|
if (lexicalHits.length) console.log("lexical prefilter:", lexicalHits.length, "hits");
|
|
133
141
|
|
|
134
142
|
const ctx: PipelineContext = {
|
|
@@ -77,7 +77,16 @@ export const PROFILE = {
|
|
|
77
77
|
},
|
|
78
78
|
"bibliography": {},
|
|
79
79
|
"terminology": {},
|
|
80
|
-
"models": {}
|
|
80
|
+
"models": {},
|
|
81
|
+
"licensed": [
|
|
82
|
+
{
|
|
83
|
+
"key": "std:fixture-60068-2-30",
|
|
84
|
+
"package": "fixture-60068-2-30",
|
|
85
|
+
"doc_number": "60068-2-30",
|
|
86
|
+
"title": "FIXTURE environmental testing — damp heat, cyclic",
|
|
87
|
+
"edition": "2005"
|
|
88
|
+
}
|
|
89
|
+
]
|
|
81
90
|
},
|
|
82
91
|
"ui": {
|
|
83
92
|
"suggestions": [
|
|
@@ -115,7 +124,9 @@ export const PROFILE = {
|
|
|
115
124
|
"parts_example": "FIXTURE 1-1, FIXTURE 1-A",
|
|
116
125
|
"docid_example": "FIXTURE 1-2",
|
|
117
126
|
"spelling_examples": "\"f1\", \"FIXTURE 1\"",
|
|
118
|
-
"process_vocab": "the fixture certification system framework"
|
|
127
|
+
"process_vocab": "the fixture certification system framework",
|
|
128
|
+
"license_declare_pointer": "org admin → Settings → Standards licenses",
|
|
129
|
+
"license_posture": "Some indexed publications are LICENSED. When a license boundary note names the question's publication, obey it: answer at the citation level only — the standard's title and edition, and the invoking clause the public passages carry — never state, paraphrase or \"summarize from memory\" any procedure of the licensed text, and point at the declare flow the note names. Public content (the Recommendations' own models, applicability and references) stays fully answerable."
|
|
119
130
|
}
|
|
120
131
|
}
|
|
121
132
|
} as const;
|