@konneal/engine 0.2.1 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ask-YCDXMIZ4.js +12 -0
- package/dist/{chunk-LNSDBEKS.js → chunk-5MBWE7WD.js} +29 -4
- package/dist/{chunk-Q6LI4T7M.js → chunk-ADXV2DPK.js} +1 -1
- package/dist/{chunk-WGXATDXY.js → chunk-DBBGOOMZ.js} +1 -1
- package/dist/{chunk-SN3ANQ3Y.js → chunk-EFQALN2Z.js} +2 -2
- package/dist/{chunk-3OXSQH7Y.js → chunk-OGFH3RDM.js} +70 -27
- package/dist/{chunk-6GOSMLRH.js → chunk-RLT4W2VX.js} +28 -5
- package/dist/{chunk-VJZLVU3S.js → chunk-THSHLUOS.js} +9 -4
- package/dist/{chunk-Q327B27J.js → chunk-TJRTVJW5.js} +13 -2
- package/dist/modelplane.d.ts +44 -1
- package/dist/openapi-types.d.ts +16 -2
- package/dist/profile.gen.d.ts +9 -0
- package/dist/prompts/system.md +1 -0
- package/dist/requestScope.d.ts +25 -4
- package/dist/search-7TO2RWQE.js +12 -0
- package/dist/selfquery.d.ts +6 -0
- package/dist/worker_mcp/src/index.js +3 -3
- package/dist/worker_public/src/config.js +2 -2
- package/dist/worker_public/src/index.js +38 -15
- package/dist/worker_public/src/profile.js +1 -1
- package/dist/worker_public/src/refusal.js +2 -2
- package/dist/worker_public/src/requestScope.js +11 -5
- package/docs/spec-pipeline.md +1 -0
- package/package.json +1 -1
- package/profile/prompts.yaml +14 -0
- package/profile/sources.yaml +9 -0
- package/workers/shared/chunk.ts +7 -0
- package/workers/worker_internal/src/index.ts +3 -3
- package/workers/worker_public/openapi.yaml +28 -3
- package/workers/worker_public/prompts/system.md +1 -0
- package/workers/worker_public/src/ask.ts +50 -16
- package/workers/worker_public/src/index.ts +2 -1
- package/workers/worker_public/src/modelplane.ts +94 -4
- package/workers/worker_public/src/pipeline.ts +13 -5
- package/workers/worker_public/src/profile.gen.ts +13 -2
- package/workers/worker_public/src/requestScope.ts +49 -7
- package/workers/worker_public/src/search.ts +7 -1
- package/workers/worker_public/src/selfquery.ts +26 -0
- package/workers/worker_public/src/share.ts +27 -5
- package/workers/worker_public/src/stages/citationProbe.ts +6 -1
- package/workers/worker_public/src/stages/index.ts +2 -0
- package/workers/worker_public/src/stages/licenseScope.ts +23 -0
- package/workers/worker_public/src/stages/types.ts +11 -0
- package/dist/ask-47RNGK2R.js +0 -12
- package/dist/search-OMPBMZT4.js +0 -11
|
@@ -14,6 +14,7 @@ import { scoreFaithfulness } from "./faithfulness";
|
|
|
14
14
|
import { checkQuoteAnchors } from "./anchors";
|
|
15
15
|
import { namedDocumentIn } from "./context";
|
|
16
16
|
import { standardForDocNumber } from "./modelplane";
|
|
17
|
+
import { entitlementScope, standardKeysFrom } from "./requestScope";
|
|
17
18
|
import type { Env } from "./env";
|
|
18
19
|
export type { Env };
|
|
19
20
|
import { json, err, corsHeaders, withCors, readJson, authenticate, type ApiKey } from "./lib/http";
|
|
@@ -227,7 +228,7 @@ async function verifyRoute(c: RouteContext): Promise<Response> {
|
|
|
227
228
|
if (!answer || !query) return err(400, "invalid_input", "answer and query are required");
|
|
228
229
|
try {
|
|
229
230
|
const u = await understandQuery(env.AI, roleModel(env, "understand"), query, [], []);
|
|
230
|
-
const retrieved = await retrieve(env, query, { understanding: u });
|
|
231
|
+
const retrieved = await retrieve(env, query, { understanding: u, standardKeys: entitlementScope(standardKeysFrom(body)) });
|
|
231
232
|
const passages = retrieved.hits.map((h: Hit) => h.text);
|
|
232
233
|
const anchors = checkQuoteAnchors(answer, passages);
|
|
233
234
|
const refs = [...answer.matchAll(/\[\[u:([^\]]+)\]\]/g)].map((m) => m[1]);
|
|
@@ -52,6 +52,82 @@ export function standardForDocNumber(docNumber: string | undefined): string | nu
|
|
|
52
52
|
return (models.standards as string[]).includes(docNumber) ? `${models.standard_prefix}${docNumber}` : null;
|
|
53
53
|
}
|
|
54
54
|
|
|
55
|
+
// ── the license tier (TODO.external-refs/08) ─────────────────────────────
|
|
56
|
+
// The profile's `sources.licensed` rows ({key, package, doc_number,
|
|
57
|
+
// title, edition}) are the deployment's declared licensed standards —
|
|
58
|
+
// the same keys the chunk metadata carries (`standard_key`) and the
|
|
59
|
+
// entitlement set arrives with (requestScope.standardKeysFrom validates
|
|
60
|
+
// against exactly this list).
|
|
61
|
+
|
|
62
|
+
/** The license entry for a package id (the model-plane standard id, e.g.
|
|
63
|
+
* `iec-60068-2-30`), or null when the package is public content. */
|
|
64
|
+
export function licensedEntryForPackage(packageId: string | undefined | null):
|
|
65
|
+
| { key: string; package: string; doc_number?: string; title?: string; edition?: string }
|
|
66
|
+
| null {
|
|
67
|
+
if (!packageId) return null;
|
|
68
|
+
return (P().sources?.licensed ?? []).find((l: any) => l.package === packageId) ?? null;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** The license entry for a doc number (the question's named publication,
|
|
72
|
+
* e.g. "60068-2-30"), or null when unnamed/public. */
|
|
73
|
+
export function licensedEntryForDocNumber(docNumber: string | undefined | null):
|
|
74
|
+
| { key: string; package: string; doc_number?: string; title?: string; edition?: string }
|
|
75
|
+
| null {
|
|
76
|
+
if (!docNumber) return null;
|
|
77
|
+
return (P().sources?.licensed ?? []).find((l: any) => String(l.doc_number ?? "") === docNumber) ?? null;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** The per-question license boundary note: composed ONLY when the
|
|
81
|
+
* question's named/understood publication is licensed AND the caller's
|
|
82
|
+
* entitlement set does not carry its key. The honesty posture, made
|
|
83
|
+
* structural: name the standard, say the organization's license does
|
|
84
|
+
* not cover its text, keep every procedural claim out, point at the
|
|
85
|
+
* declare flow. Citation-level metadata (title, edition, the invoking
|
|
86
|
+
* clause the RECs publicly name) stays answerable from the public
|
|
87
|
+
* passages already in context — the note instructs exactly that. */
|
|
88
|
+
export function licenseBoundaryNote(
|
|
89
|
+
docNumber: string | undefined | null,
|
|
90
|
+
standardKeys: ReadonlySet<string> | null | undefined,
|
|
91
|
+
): string | undefined {
|
|
92
|
+
const entry = licensedEntryForDocNumber(docNumber);
|
|
93
|
+
if (!entry || (standardKeys && standardKeys.has(entry.key))) return undefined;
|
|
94
|
+
const pointer = P().prompts?.vars?.license_declare_pointer;
|
|
95
|
+
return (
|
|
96
|
+
`License boundary — the question is about ${licenseBoundaryName(entry)}, a licensed publication` +
|
|
97
|
+
` (entitlement key ${entry.key}). The caller's organization license does not cover its text, so no passage of it was retrieved` +
|
|
98
|
+
` and NONE of its procedural content (steps, parameters, severities, limits) may be stated, paraphrased or recalled from memory.` +
|
|
99
|
+
` You MAY answer at the citation level: name the standard and edition, and cite the invoking clause from the PUBLIC passages in context` +
|
|
100
|
+
` (the Recommendation's own applicability and normative references are public and stay answerable).` +
|
|
101
|
+
` Then say the organization's license does not cover the standard's text` +
|
|
102
|
+
(pointer ? ` and point to the declare flow: ${pointer}.` : ".")
|
|
103
|
+
);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function licenseBoundaryName(entry: { title?: string; edition?: string; package: string; doc_number?: string }): string {
|
|
107
|
+
const id = entry.doc_number ? ` ${entry.doc_number}` : ` ${entry.package}`;
|
|
108
|
+
return `${entry.title ?? "standard"}${entry.edition ? ` (${entry.edition})` : ""} —${id}`;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/** The deterministic boundary answer for the zero-passage case: the
|
|
112
|
+
* question's licensed publication has nothing to show an unentitled
|
|
113
|
+
* caller — name the standard, state the boundary, point at the declare
|
|
114
|
+
* flow. Undefined when the question is not the licensed case (the plain
|
|
115
|
+
* refusal applies). */
|
|
116
|
+
export function licenseBoundaryRefusal(
|
|
117
|
+
docNumber: string | undefined | null,
|
|
118
|
+
standardKeys: ReadonlySet<string> | null | undefined,
|
|
119
|
+
): string | undefined {
|
|
120
|
+
const entry = licensedEntryForDocNumber(docNumber);
|
|
121
|
+
if (!entry || (standardKeys && standardKeys.has(entry.key))) return undefined;
|
|
122
|
+
const pointer = P().prompts?.vars?.license_declare_pointer;
|
|
123
|
+
return (
|
|
124
|
+
`${licenseBoundaryName(entry)} is a licensed publication and your organization's license does not cover its text, ` +
|
|
125
|
+
`so I can't quote or summarize its procedure. I can answer at the citation level — the standard's title and edition, ` +
|
|
126
|
+
`and the clause your Recommendation invokes — and the public ${P().publisher.name} content in full.` +
|
|
127
|
+
(pointer ? ` To unlock the full text, an org admin can declare the license under ${pointer}.` : "")
|
|
128
|
+
);
|
|
129
|
+
}
|
|
130
|
+
|
|
55
131
|
export interface BoundModelNode {
|
|
56
132
|
standard: string;
|
|
57
133
|
node_id: string;
|
|
@@ -60,6 +136,12 @@ export interface BoundModelNode {
|
|
|
60
136
|
clause: { doc: string; ref: string; urn: string } | null;
|
|
61
137
|
/** The node's bundle projection (verbatim JSON). */
|
|
62
138
|
content: any;
|
|
139
|
+
/** True when the node's package is licensed and the caller's
|
|
140
|
+
* entitlement set lacks the key (TODO.external-refs/08): the citation
|
|
141
|
+
* and the echo stay (metadata), but the grounding block and the
|
|
142
|
+
* verdict engine are withheld — no licensed machine content enters
|
|
143
|
+
* the prompt. */
|
|
144
|
+
gated?: boolean;
|
|
63
145
|
}
|
|
64
146
|
|
|
65
147
|
async function fetchNode(env: any, standard: string, nodeId: string): Promise<BoundModelNode | null> {
|
|
@@ -97,18 +179,26 @@ async function fetchNode(env: any, standard: string, nodeId: string): Promise<Bo
|
|
|
97
179
|
* model-aware chip), then a node id the question names. The standard
|
|
98
180
|
* comes from the declared doc scope when it carries one; without a scope
|
|
99
181
|
* the node binds only when it exists in EXACTLY ONE indexed standard —
|
|
100
|
-
* ambiguity is refused honestly (retrieval still surfaces the chunks).
|
|
182
|
+
* ambiguity is refused honestly (retrieval still surfaces the chunks).
|
|
183
|
+
* A licensed package binds GATED for an unentitled caller (metadata
|
|
184
|
+
* only — the grounding block and the verdict engine are the ask path's
|
|
185
|
+
* to withhold). */
|
|
101
186
|
export async function bindModelNode(
|
|
102
187
|
env: any,
|
|
103
|
-
opts: { label?: string; query: string; standard?: string | null },
|
|
188
|
+
opts: { label?: string; query: string; standard?: string | null; standardKeys?: ReadonlySet<string> | null },
|
|
104
189
|
): Promise<BoundModelNode | null> {
|
|
105
190
|
const nodeId = modelNodeRefIn(opts.label) ?? modelNodeRefIn(opts.query);
|
|
106
191
|
if (!nodeId) return null;
|
|
107
|
-
|
|
192
|
+
const gate = (node: BoundModelNode | null): BoundModelNode | null => {
|
|
193
|
+
if (!node) return null;
|
|
194
|
+
const entry = licensedEntryForPackage(node.standard);
|
|
195
|
+
return entry && !(opts.standardKeys?.has(entry.key) ?? false) ? { ...node, gated: true, content: {} } : node;
|
|
196
|
+
};
|
|
197
|
+
if (opts.standard) return gate(await fetchNode(env, opts.standard, nodeId));
|
|
108
198
|
try {
|
|
109
199
|
const rows = await env.DB.prepare("SELECT standard FROM model_nodes WHERE node_id = ?1 LIMIT 2").bind(nodeId).all();
|
|
110
200
|
const standards = (rows?.results ?? []).map((r: any) => String(r.standard));
|
|
111
|
-
if (standards.length === 1) return fetchNode(env, standards[0]!, nodeId);
|
|
201
|
+
if (standards.length === 1) return gate(await fetchNode(env, standards[0]!, nodeId));
|
|
112
202
|
return null; // zero (not indexed) or ambiguous (several standards) — no silent pick
|
|
113
203
|
} catch {
|
|
114
204
|
return null;
|
|
@@ -32,7 +32,7 @@ export function promptVars(extra: Record<string, string> = {}): Record<string, s
|
|
|
32
32
|
export function fill(template: string, vars: Record<string, string>): string {
|
|
33
33
|
return template.replace(/\{\{(\w+)\}\}/g, (_m, k: string) => (k in vars ? vars[k] : ""));
|
|
34
34
|
}
|
|
35
|
-
import { QueryFilters, toVectorizeFilter } from "./selfquery";
|
|
35
|
+
import { QueryFilters, toVectorizeFilter, standardKeyAllowed } from "./selfquery";
|
|
36
36
|
import { lexicalPrefilter } from "./lexical";
|
|
37
37
|
import { positionOrder } from "./structural";
|
|
38
38
|
import { STAGES, runStages } from "./stages";
|
|
@@ -125,10 +125,18 @@ export async function retrieve(
|
|
|
125
125
|
// The declared context's seal binds the lexical lane at the SOURCE: the
|
|
126
126
|
// RRF fusion mixes the full-corpus lexical ranking straight into the
|
|
127
127
|
// final hits — past the pool-level seal — so under a seal the lexical
|
|
128
|
-
// lane is the FAMILY's lexical hits only.
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
128
|
+
// lane is the FAMILY's lexical hits only. The license entitlement scope
|
|
129
|
+
// binds the same lane the same way (it re-enters twice: lexical-union
|
|
130
|
+
// pre-rerank and lexical-rrf post-rerank — both consume this list).
|
|
131
|
+
const lexicalHits = (opts.sealScope || opts.standardKeys
|
|
132
|
+
? lexicalHits0.filter(
|
|
133
|
+
(h) =>
|
|
134
|
+
(!opts.sealScope ||
|
|
135
|
+
(h.metadata.doc_number === opts.sealScope!.doc_number &&
|
|
136
|
+
(!opts.sealScope!.edition || h.metadata.edition === opts.sealScope!.edition))) &&
|
|
137
|
+
standardKeyAllowed(h.metadata, opts.standardKeys),
|
|
138
|
+
)
|
|
139
|
+
: lexicalHits0);
|
|
132
140
|
if (lexicalHits.length) console.log("lexical prefilter:", lexicalHits.length, "hits");
|
|
133
141
|
|
|
134
142
|
const ctx: PipelineContext = {
|
|
@@ -77,7 +77,16 @@ export const PROFILE = {
|
|
|
77
77
|
},
|
|
78
78
|
"bibliography": {},
|
|
79
79
|
"terminology": {},
|
|
80
|
-
"models": {}
|
|
80
|
+
"models": {},
|
|
81
|
+
"licensed": [
|
|
82
|
+
{
|
|
83
|
+
"key": "std:fixture-60068-2-30",
|
|
84
|
+
"package": "fixture-60068-2-30",
|
|
85
|
+
"doc_number": "60068-2-30",
|
|
86
|
+
"title": "FIXTURE environmental testing — damp heat, cyclic",
|
|
87
|
+
"edition": "2005"
|
|
88
|
+
}
|
|
89
|
+
]
|
|
81
90
|
},
|
|
82
91
|
"ui": {
|
|
83
92
|
"suggestions": [
|
|
@@ -115,7 +124,9 @@ export const PROFILE = {
|
|
|
115
124
|
"parts_example": "FIXTURE 1-1, FIXTURE 1-A",
|
|
116
125
|
"docid_example": "FIXTURE 1-2",
|
|
117
126
|
"spelling_examples": "\"f1\", \"FIXTURE 1\"",
|
|
118
|
-
"process_vocab": "the fixture certification system framework"
|
|
127
|
+
"process_vocab": "the fixture certification system framework",
|
|
128
|
+
"license_declare_pointer": "org admin → Settings → Standards licenses",
|
|
129
|
+
"license_posture": "Some indexed publications are LICENSED. When a license boundary note names the question's publication, obey it: answer at the citation level only — the standard's title and edition, and the invoking clause the public passages carry — never state, paraphrase or \"summarize from memory\" any procedure of the licensed text, and point at the declare flow the note names. Public content (the Recommendations' own models, applicability and references) stays fully answerable."
|
|
119
130
|
}
|
|
120
131
|
}
|
|
121
132
|
} as const;
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
// The per-request scope model (MECE: ask.ts orchestrates, this module
|
|
2
2
|
// owns the derivation): which DATASETS the request searches (the sidebar
|
|
3
|
-
// toggles, server-intersected with session permissions)
|
|
4
|
-
//
|
|
3
|
+
// toggles, server-intersected with session permissions), which memory
|
|
4
|
+
// files ride it, and which LICENSED standards the caller's organization
|
|
5
|
+
// is entitled to — plus the answer-cache SALT all three selections
|
|
5
6
|
// produce. Pure: D1 access stays with the caller (memoryNote); this
|
|
6
7
|
// module only derives.
|
|
7
8
|
import { DATASETS, datasetAllowed } from "./config.ts";
|
|
9
|
+
import { P } from "./profile.ts";
|
|
8
10
|
|
|
9
11
|
export interface RequestScope {
|
|
10
12
|
/** dataset ids the request may search */
|
|
@@ -18,6 +20,37 @@ export interface RequestScope {
|
|
|
18
20
|
isoOn: boolean;
|
|
19
21
|
/** raw (validated) memory ids from the body — empty for anon */
|
|
20
22
|
memoryIds: string[];
|
|
23
|
+
/** raw (validated) license entitlement keys from the body — empty for
|
|
24
|
+
* anon and for requests that carry none (TODO.external-refs/08) */
|
|
25
|
+
standardKeys: Set<string>;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** The deployment's declared licensed standards (profile sources.yaml
|
|
29
|
+
* `licensed:` — key/package/doc_number rows). Empty = the deployment
|
|
30
|
+
* serves public content only and the entitlement scope is inert. */
|
|
31
|
+
export function licenseDeclared(): boolean {
|
|
32
|
+
return (P().sources?.licensed?.length ?? 0) > 0;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** The request's entitlement set, VALIDATED against the declared
|
|
36
|
+
* whitelist: unknown keys drop (a forged key can never widen scope past
|
|
37
|
+
* the standards the deployment actually keys). Absent field = empty set
|
|
38
|
+
* — the fail-closed default for a deployment that declares licensed
|
|
39
|
+
* content. */
|
|
40
|
+
export function standardKeysFrom(body: any): Set<string> {
|
|
41
|
+
const declared = new Set<string>((P().sources?.licensed ?? []).map((l: any) => String(l.key)));
|
|
42
|
+
const raw = Array.isArray(body?.licensed_standards) ? body.licensed_standards : [];
|
|
43
|
+
return new Set(
|
|
44
|
+
raw.filter((x: unknown): x is string => typeof x === "string" && declared.has(x)),
|
|
45
|
+
);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** The RetrieveOptions value for the hard scope: null when the
|
|
49
|
+
* deployment declares no licensed content (inert — zero behavior
|
|
50
|
+
* change); otherwise the caller's validated set, EMPTY INCLUDED (the
|
|
51
|
+
* unentitled caller: licensed chunks hidden, citation metadata stays). */
|
|
52
|
+
export function entitlementScope(keys: Set<string>): Set<string> | null {
|
|
53
|
+
return licenseDeclared() ? keys : null;
|
|
21
54
|
}
|
|
22
55
|
|
|
23
56
|
/** Validate + intersect. Returns { error } when the request explicitly
|
|
@@ -54,18 +87,27 @@ export function resolveRequestScope(body: any, member: unknown): RequestScope |
|
|
|
54
87
|
// federation flag: any session-gated (federated) dataset in scope
|
|
55
88
|
isoOn: scopeIds.some((id) => DATASETS().find((x) => x.id === id)?.session === true),
|
|
56
89
|
memoryIds,
|
|
90
|
+
standardKeys: standardKeysFrom(body),
|
|
57
91
|
};
|
|
58
92
|
}
|
|
59
93
|
|
|
60
94
|
/** The answer-cache salt: request-scoped context that materially changes
|
|
61
|
-
* the answer (dataset scope, memory selection
|
|
62
|
-
*
|
|
63
|
-
*
|
|
64
|
-
*
|
|
95
|
+
* the answer (dataset scope, memory selection, license entitlements —
|
|
96
|
+
* the licensed tier changes the grounding, so two callers asking the
|
|
97
|
+
* same question must never share an entry). Requests differing only in
|
|
98
|
+
* salt share query text — an unsalted key would serve a scoped (or
|
|
99
|
+
* memory-flavored, or licensed-tier) answer to a plain ask. Null =
|
|
100
|
+
* default scope, no memory, no entitlement effect: keys stay
|
|
101
|
+
* byte-identical to the pre-salt era. */
|
|
65
102
|
export function requestSalt(scope: RequestScope, memoryUsed: string[]): string | null {
|
|
66
|
-
|
|
103
|
+
const licensed = licenseDeclared();
|
|
104
|
+
if (!scope.narrowed && !memoryUsed.length && !licensed) return null;
|
|
67
105
|
return JSON.stringify({
|
|
68
106
|
...(scope.narrowed ? { d: [...scope.corpora].sort() } : {}),
|
|
69
107
|
...(memoryUsed.length ? { m: [...memoryUsed].sort() } : {}),
|
|
108
|
+
// the entitlement set rides whenever the deployment keys content at
|
|
109
|
+
// all — an unentitled ask and an entitled ask of the same text are
|
|
110
|
+
// different answers even when the set is empty
|
|
111
|
+
...(licensed ? { s: [...scope.standardKeys].sort() } : {}),
|
|
70
112
|
});
|
|
71
113
|
}
|
|
@@ -4,6 +4,7 @@ import { portModelRunner } from "./env.ts";
|
|
|
4
4
|
import type { Background } from "./ports/runtime.ts";
|
|
5
5
|
import { retrieve } from "./pipeline";
|
|
6
6
|
import { understandQuery } from "./understand";
|
|
7
|
+
import { entitlementScope, standardKeysFrom } from "./requestScope";
|
|
7
8
|
import { sessionFrom } from "./auth";
|
|
8
9
|
import { err, json, corsHeaders, readJson, validateQuery, type ApiKey } from "./lib/http";
|
|
9
10
|
import { checkQuota, clientIp, telemetry } from "./quota";
|
|
@@ -32,9 +33,14 @@ export async function handleSearch(
|
|
|
32
33
|
|
|
33
34
|
const understanding = await understandQuery(portModelRunner(env), MODELS.understand, q.query, []);
|
|
34
35
|
const graphDocNumbers = await graphExpand(env, understanding);
|
|
36
|
+
// the license entitlement set rides the body exactly as it does on the
|
|
37
|
+
// ask path (TODO.external-refs/08): request-scoped, whitelist-validated,
|
|
38
|
+
// fail-closed — a retrieval with an empty set returns zero licensed
|
|
39
|
+
// chunks at the transport level
|
|
40
|
+
const standardKeys = entitlementScope(standardKeysFrom(body));
|
|
35
41
|
let retrieved;
|
|
36
42
|
try {
|
|
37
|
-
retrieved = await retrieve(env, q.query, { understanding, graphDocNumbers });
|
|
43
|
+
retrieved = await retrieve(env, q.query, { understanding, graphDocNumbers, standardKeys });
|
|
38
44
|
} catch {
|
|
39
45
|
return err(503, "retrieval_unavailable", "Search is briefly busy — please retry in a moment.");
|
|
40
46
|
}
|
|
@@ -23,3 +23,29 @@ export function toVectorizeFilter(f: QueryFilters): Record<string, string> | und
|
|
|
23
23
|
}
|
|
24
24
|
return undefined;
|
|
25
25
|
}
|
|
26
|
+
|
|
27
|
+
// ── the license entitlement scope (TODO.external-refs/08) ────────────────
|
|
28
|
+
// A chunk's `standard_key` metadata carries the licensed package's
|
|
29
|
+
// entitlement key; public content carries none. The scope is a HARD
|
|
30
|
+
// FILTER (a hit never reaches ranking when its key is outside the
|
|
31
|
+
// caller's set) — but it deliberately does NOT translate into a
|
|
32
|
+
// Vectorize metadata predicate: the wire has no "field missing OR in-set"
|
|
33
|
+
// operator, so a `standard_key $in […]` push-down would exclude every
|
|
34
|
+
// public chunk (they predate the field). The scope binds pool-level
|
|
35
|
+
// (stages/licenseScope.ts, before rerank) and at the lexical lane's
|
|
36
|
+
// source (pipeline.ts, the sealScope posture) instead; the single-doc
|
|
37
|
+
// post-seal fetches (section-descent, typed-pin parent, edition-cover)
|
|
38
|
+
// are doc-scoped and a document's key is a per-document constant, so
|
|
39
|
+
// they cannot re-admit a dropped key.
|
|
40
|
+
|
|
41
|
+
/** The one entitlement predicate: no key = public = always allowed; a key
|
|
42
|
+
* outside the caller's set = never allowed. `null`/undefined keys (the
|
|
43
|
+
* deployment declares no licensed content) disable the scope entirely. */
|
|
44
|
+
export function standardKeyAllowed(
|
|
45
|
+
meta: { standard_key?: string },
|
|
46
|
+
keys: ReadonlySet<string> | null | undefined,
|
|
47
|
+
): boolean {
|
|
48
|
+
if (!keys) return true;
|
|
49
|
+
const k = meta.standard_key;
|
|
50
|
+
return !k || keys.has(k);
|
|
51
|
+
}
|
|
@@ -26,11 +26,33 @@ export async function handleShareConversation(
|
|
|
26
26
|
await env.CACHE.put(`sh:${day}:${ownerSub.slice(0, 20)}`, String(count + 1), { expirationTtl: 90000 });
|
|
27
27
|
|
|
28
28
|
const slug = makeSlug();
|
|
29
|
-
const cleanMessages = messages.slice(0, 50).map((m: any) =>
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
29
|
+
const cleanMessages = messages.slice(0, 50).map((m: any) => {
|
|
30
|
+
const citations = Array.isArray(m.citations)
|
|
31
|
+
? m.citations
|
|
32
|
+
: typeof m.citations === "string"
|
|
33
|
+
? (() => { try { return JSON.parse(m.citations); } catch { return null; } })()
|
|
34
|
+
: null;
|
|
35
|
+
const read = m.read && typeof m.read === "object"
|
|
36
|
+
? {
|
|
37
|
+
intent: String(m.read.intent ?? ""),
|
|
38
|
+
doc: m.read.doc ?? null,
|
|
39
|
+
edition: m.read.edition ?? null,
|
|
40
|
+
term: m.read.term ?? null,
|
|
41
|
+
terms: Array.isArray(m.read.terms) ? m.read.terms.slice(0, 4).map(String) : [],
|
|
42
|
+
lang: m.read.lang ?? null,
|
|
43
|
+
}
|
|
44
|
+
: undefined;
|
|
45
|
+
// passages stay off the shared wire: the recipient verifies against the
|
|
46
|
+
// live corpus, never against a grounding they cannot re-derive
|
|
47
|
+
return {
|
|
48
|
+
role: m.role === "user" ? "user" : "assistant",
|
|
49
|
+
content: (m.content ?? "").slice(0, LIMITS.maxOutputTokens * 2),
|
|
50
|
+
citations,
|
|
51
|
+
...(m.model ? { model: String(m.model).slice(0, 80) } : {}),
|
|
52
|
+
...(Array.isArray(m.blocks) ? { blocks: m.blocks.slice(0, 12) } : {}),
|
|
53
|
+
...(read ? { read } : {}),
|
|
54
|
+
};
|
|
55
|
+
});
|
|
34
56
|
await env.DB.prepare(
|
|
35
57
|
"INSERT INTO shared_conversations (slug, owner_sub, title, messages, created_at) VALUES (?1,?2,?3,?4,?5)",
|
|
36
58
|
)
|
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
import type { Stage } from "./types.ts";
|
|
15
15
|
import { namedDocumentIn } from "../context.ts";
|
|
16
16
|
import { refCodec } from "../codecs.ts";
|
|
17
|
+
import { standardKeyAllowed } from "../selfquery.ts";
|
|
17
18
|
import type { Hit } from "../../../shared/chunk.ts";
|
|
18
19
|
|
|
19
20
|
const CITE_PATTERN = /\b(?:cite[sd]?|citing|referenc(?:e|es|ed|ing)|list[s]?|quote[sd]?)\b/i;
|
|
@@ -123,9 +124,13 @@ export const citationProbe: Stage = {
|
|
|
123
124
|
const [probes, citeRows] = (await c.lane["citation-probe"]) as [Hit[], CiteRow[]];
|
|
124
125
|
const seen = new Set(c.hits.map((m: any) => m.id));
|
|
125
126
|
let added = 0;
|
|
126
|
-
// only bibliography-shaped chunks (clause title or text mentions it)
|
|
127
|
+
// only bibliography-shaped chunks (clause title or text mentions it);
|
|
128
|
+
// the push rides the entitlement predicate — the probe runs past the
|
|
129
|
+
// pool-level license scope, and a licensed family's bibliography
|
|
130
|
+
// passages must obey the same hard scope as everything else
|
|
127
131
|
for (const h of probes) {
|
|
128
132
|
if (seen.has(h.id as any)) continue;
|
|
133
|
+
if (!standardKeyAllowed(h.metadata as any, c.opts.standardKeys)) continue;
|
|
129
134
|
const title = String((h.metadata as any)?.clause_title ?? "");
|
|
130
135
|
const text = String(h.text ?? "");
|
|
131
136
|
if (/bibliograph|normative reference/i.test(title + " " + text.slice(0, 300))) {
|
|
@@ -23,6 +23,7 @@ import { poolOpen } from "./poolOpen.ts";
|
|
|
23
23
|
import { lexicalUnion } from "./lexicalUnion.ts";
|
|
24
24
|
import { federate } from "./federate.ts";
|
|
25
25
|
import { seal } from "./seal.ts";
|
|
26
|
+
import { licenseScope } from "./licenseScope.ts";
|
|
26
27
|
import { corpusScope } from "./corpusScope.ts";
|
|
27
28
|
import { editionCover } from "./editionCover.ts";
|
|
28
29
|
import { stdRefNudge } from "./stdRefNudge.ts";
|
|
@@ -51,6 +52,7 @@ export const STAGES: Stage[] = [
|
|
|
51
52
|
lexicalUnion,
|
|
52
53
|
federate,
|
|
53
54
|
seal,
|
|
55
|
+
licenseScope,
|
|
54
56
|
overviewDemote,
|
|
55
57
|
familyBoost,
|
|
56
58
|
rerankStage,
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
// The license entitlement hard scope (TODO.external-refs/08) — pool-level,
|
|
2
|
+
// after every candidate lane has merged, BEFORE rerank + the top-N cut
|
|
3
|
+
// (the `seal` posture, never the post-rerank corpus-scope drop). A chunk
|
|
4
|
+
// whose `standard_key` lies outside the caller's entitlement set never
|
|
5
|
+
// competes for the window: the model never sees licensed text the caller
|
|
6
|
+
// cannot be shown, so no prompt-level leakage either. The predicate is
|
|
7
|
+
// selfquery.standardKeyAllowed; the lane audit that binds every query
|
|
8
|
+
// shape lives in its doc comment.
|
|
9
|
+
import { standardKeyAllowed } from "../selfquery.ts";
|
|
10
|
+
import type { Stage } from "./types.ts";
|
|
11
|
+
|
|
12
|
+
export const licenseScope: Stage = {
|
|
13
|
+
name: "license-scope",
|
|
14
|
+
when: (c) => !!c.opts.standardKeys,
|
|
15
|
+
run: (c) => {
|
|
16
|
+
const keys = c.opts.standardKeys!;
|
|
17
|
+
const before = c.hits.length;
|
|
18
|
+
c.hits = c.hits.filter((h) => standardKeyAllowed(h.metadata, keys));
|
|
19
|
+
if (c.hits.length !== before) {
|
|
20
|
+
console.log("license scope:", before, "→", c.hits.length, "candidates within the caller's entitlement set");
|
|
21
|
+
}
|
|
22
|
+
},
|
|
23
|
+
};
|
|
@@ -36,6 +36,17 @@ export interface RetrieveOptions {
|
|
|
36
36
|
* intersects the requested ids with session permissions before
|
|
37
37
|
* building this set. */
|
|
38
38
|
datasetScope?: Set<string> | null;
|
|
39
|
+
/** The license entitlement set (TODO.external-refs/08): the standard
|
|
40
|
+
* keys the caller's organization is entitled to, resolved
|
|
41
|
+
* request-scoped by the deployment (never a client-tunable filter —
|
|
42
|
+
* it arrives through the same trusted request context as the session,
|
|
43
|
+
* and the profile's declared licensed list is the validation
|
|
44
|
+
* whitelist). NON-NULL activates the hard scope: chunks carrying a
|
|
45
|
+
* `standard_key` outside the set never reach ranking (an EMPTY set is
|
|
46
|
+
* the unentitled caller — licensed content hidden, citation-level
|
|
47
|
+
* metadata stays). Null = the deployment declares no licensed
|
|
48
|
+
* content, the scope is inert. */
|
|
49
|
+
standardKeys?: Set<string> | null;
|
|
39
50
|
}
|
|
40
51
|
|
|
41
52
|
export interface GlossaryEntry {
|
package/dist/ask-47RNGK2R.js
DELETED
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
import {
|
|
2
|
-
handleAsk
|
|
3
|
-
} from "./chunk-3OXSQH7Y.js";
|
|
4
|
-
import "./chunk-LNSDBEKS.js";
|
|
5
|
-
import "./chunk-6GOSMLRH.js";
|
|
6
|
-
import "./chunk-SN3ANQ3Y.js";
|
|
7
|
-
import "./chunk-WGXATDXY.js";
|
|
8
|
-
import "./chunk-Q6LI4T7M.js";
|
|
9
|
-
import "./chunk-Q327B27J.js";
|
|
10
|
-
export {
|
|
11
|
-
handleAsk
|
|
12
|
-
};
|
package/dist/search-OMPBMZT4.js
DELETED