@mnmal-ai/substrate-nl 0.0.0-stage → 1.0.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +37 -2
- package/config.schema.json +22 -0
- package/default-config.json +1 -0
- package/dist/access.d.ts +18 -0
- package/dist/access.d.ts.map +1 -0
- package/dist/ask/encode.d.ts +11 -0
- package/dist/ask/encode.d.ts.map +1 -0
- package/dist/ask/events.d.ts +7 -0
- package/dist/ask/events.d.ts.map +1 -0
- package/dist/ask/run.d.ts +14 -0
- package/dist/ask/run.d.ts.map +1 -0
- package/dist/bootstrap-BpKUloR3.js +1821 -0
- package/dist/bootstrap.d.ts +18 -0
- package/dist/bootstrap.d.ts.map +1 -0
- package/dist/bootstrap.js +3 -0
- package/dist/complete/complete.d.ts +19 -0
- package/dist/complete/complete.d.ts.map +1 -0
- package/dist/complete.js +157 -0
- package/dist/index.d.ts +23 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +4 -0
- package/dist/judge/judge.d.ts +65 -0
- package/dist/judge/judge.d.ts.map +1 -0
- package/dist/judge/ollama.d.ts +20 -0
- package/dist/judge/ollama.d.ts.map +1 -0
- package/dist/own.d.ts +6 -0
- package/dist/own.d.ts.map +1 -0
- package/dist/pack/build.d.ts +17 -0
- package/dist/pack/build.d.ts.map +1 -0
- package/dist/pack/profile.d.ts +27 -0
- package/dist/pack/profile.d.ts.map +1 -0
- package/dist/pack/rulings.d.ts +9 -0
- package/dist/pack/rulings.d.ts.map +1 -0
- package/dist/pack/service.d.ts +28 -0
- package/dist/pack/service.d.ts.map +1 -0
- package/dist/pack/types.d.ts +21 -0
- package/dist/pack/types.d.ts.map +1 -0
- package/dist/pack/words.d.ts +7 -0
- package/dist/pack/words.d.ts.map +1 -0
- package/dist/parse/parse.d.ts +47 -0
- package/dist/parse/parse.d.ts.map +1 -0
- package/dist/parse/time.d.ts +13 -0
- package/dist/parse/time.d.ts.map +1 -0
- package/dist/plugin.d.ts +20 -0
- package/dist/plugin.d.ts.map +1 -0
- package/dist/routes.d.ts +16 -0
- package/dist/routes.d.ts.map +1 -0
- package/dist/rulings/schema.d.ts +20 -0
- package/dist/rulings/schema.d.ts.map +1 -0
- package/package.json +62 -3
|
@@ -0,0 +1,1821 @@
|
|
|
1
|
+
import { defineBootstrap, error400, jsonResponse, parseJsonBody } from "@mnmal-ai/substrate/sdk";
|
|
2
|
+
import { z } from "zod";
|
|
3
|
+
import { createHash } from "node:crypto";
|
|
4
|
+
|
|
5
|
+
//#region src/own.ts
|
|
6
|
+
/**
|
|
7
|
+
* A lookup into a plain-object record by a user-supplied key. Inherited
|
|
8
|
+
* names (`constructor`, `__proto__`, `toString`) are not entries.
|
|
9
|
+
*/
|
|
10
|
+
function own(rec, key) {
|
|
11
|
+
return rec !== void 0 && Object.hasOwn(rec, key) ? rec[key] : void 0;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
//#endregion
|
|
15
|
+
//#region src/pack/words.ts
|
|
16
|
+
/** `AgentMessage` -> ['agent', 'message']; `shortId` -> ['short', 'id']. */
|
|
17
|
+
function splitWords(name) {
|
|
18
|
+
return name.replace(/([a-z0-9])([A-Z])/g, "$1 $2").replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2").toLowerCase().split(/[\s_-]+/).filter((w) => w.length > 0);
|
|
19
|
+
}
|
|
20
|
+
/** English plural for a lower-case noun, by suffix rule. */
|
|
21
|
+
function pluralize(word) {
|
|
22
|
+
if (/(s|x|z|ch|sh)$/.test(word)) return `${word}es`;
|
|
23
|
+
if (/[^aeiou]y$/.test(word)) return `${word.slice(0, -1)}ies`;
|
|
24
|
+
return `${word}s`;
|
|
25
|
+
}
|
|
26
|
+
/** Sorted, de-duplicated. */
|
|
27
|
+
function uniqueSorted(xs) {
|
|
28
|
+
return [...new Set(xs)].sort();
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
//#endregion
|
|
32
|
+
//#region src/parse/time.ts
|
|
33
|
+
const WEEKDAYS = [
|
|
34
|
+
"sunday",
|
|
35
|
+
"monday",
|
|
36
|
+
"tuesday",
|
|
37
|
+
"wednesday",
|
|
38
|
+
"thursday",
|
|
39
|
+
"friday",
|
|
40
|
+
"saturday"
|
|
41
|
+
];
|
|
42
|
+
/**
|
|
43
|
+
* The range a time phrase names, relative to `asOf` and in its offset:
|
|
44
|
+
* today, yesterday, this/last week (weeks start Monday), this/last month,
|
|
45
|
+
* rolling "past <unit>" and "last N <units>", "since <weekday>". Null when
|
|
46
|
+
* the phrase is not a time phrase.
|
|
47
|
+
*/
|
|
48
|
+
function rangeFor(phrase, asOf) {
|
|
49
|
+
const m = /^(\d{4})-(\d{2})-(\d{2})T[\d:.]+(Z|[+-]\d{2}:\d{2})$/.exec(asOf);
|
|
50
|
+
if (m === null) throw new Error(`asOf must be ISO-8601 with an offset: ${asOf}`);
|
|
51
|
+
const [, y, mo, d, offset] = m;
|
|
52
|
+
const today = Date.UTC(Number(y), Number(mo) - 1, Number(d));
|
|
53
|
+
const day = 864e5;
|
|
54
|
+
const at = (utcMidnight) => `${new Date(utcMidnight).toISOString().slice(0, 10)}T00:00:00${offset === "Z" ? "Z" : offset}`;
|
|
55
|
+
const weekday = new Date(today).getUTCDay();
|
|
56
|
+
const monday = today - (weekday + 6) % 7 * day;
|
|
57
|
+
const monthStart = Date.UTC(Number(y), Number(mo) - 1, 1);
|
|
58
|
+
const nextMonth = Date.UTC(Number(y), Number(mo), 1);
|
|
59
|
+
const lastMonth = Date.UTC(Number(y), Number(mo) - 2, 1);
|
|
60
|
+
switch (phrase) {
|
|
61
|
+
case "today": return {
|
|
62
|
+
gte: at(today),
|
|
63
|
+
lt: at(today + day)
|
|
64
|
+
};
|
|
65
|
+
case "yesterday": return {
|
|
66
|
+
gte: at(today - day),
|
|
67
|
+
lt: at(today)
|
|
68
|
+
};
|
|
69
|
+
case "this week": return {
|
|
70
|
+
gte: at(monday),
|
|
71
|
+
lt: at(monday + 7 * day)
|
|
72
|
+
};
|
|
73
|
+
case "last week": return {
|
|
74
|
+
gte: at(monday - 7 * day),
|
|
75
|
+
lt: at(monday)
|
|
76
|
+
};
|
|
77
|
+
case "this month": return {
|
|
78
|
+
gte: at(monthStart),
|
|
79
|
+
lt: at(nextMonth)
|
|
80
|
+
};
|
|
81
|
+
case "last month": return {
|
|
82
|
+
gte: at(lastMonth),
|
|
83
|
+
lt: at(monthStart)
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
const rolling = /^(?:past|last) (\d+ )?(day|week|month|year)s?$/.exec(phrase);
|
|
87
|
+
if (rolling !== null && (rolling[1] !== void 0 || phrase.startsWith("past"))) {
|
|
88
|
+
const n = rolling[1] === void 0 ? 1 : Number(rolling[1]);
|
|
89
|
+
const unit = rolling[2];
|
|
90
|
+
const back = new Date(today);
|
|
91
|
+
if (unit === "day") back.setUTCDate(back.getUTCDate() - n);
|
|
92
|
+
if (unit === "week") back.setUTCDate(back.getUTCDate() - 7 * n);
|
|
93
|
+
if (unit === "month") back.setUTCMonth(back.getUTCMonth() - n);
|
|
94
|
+
if (unit === "year") back.setUTCFullYear(back.getUTCFullYear() - n);
|
|
95
|
+
return {
|
|
96
|
+
gte: at(back.getTime()),
|
|
97
|
+
lt: at(today + day)
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
const since = /^since (\w+)$/.exec(phrase);
|
|
101
|
+
if (since !== null) {
|
|
102
|
+
const target = WEEKDAYS.indexOf(since[1]);
|
|
103
|
+
if (target < 0) return null;
|
|
104
|
+
return {
|
|
105
|
+
gte: at(today - (weekday - target + 7) % 7 * day),
|
|
106
|
+
lt: at(today + day)
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
return null;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
//#endregion
|
|
113
|
+
//#region src/parse/parse.ts
|
|
114
|
+
/** Words that carry no query meaning on their own. */
|
|
115
|
+
const GLUE = new Set("a about again all am an and any anything are as at be been by can could current did do does for from get give go got have hey i i'm in is it it's let let's lets list me more my need needs next now of ok on or our please raised see show so some still talk tell that the their them there these this those to us want was we we've we'd what what's whats when where which who whose why will with would you your yo up thought handled judgement judgment make nice there's anymore stuff things few runs run lives wrote write written created made filed logged".split(/\s+/));
|
|
116
|
+
/** First word of an imperative edit, which substrate-nl refuses: it only reads. */
|
|
117
|
+
const MUTATIONS = new Set("close delete remove archive merge approve approved create add update set mark rename reopen assign ack deploy cut land".split(" "));
|
|
118
|
+
const LEADING_FILLER = new Set("please can could would you go ahead let let's lets hey ok yo so and then".split(" "));
|
|
119
|
+
const STATE_CUES = new Set("what what's whats show talk tell status is describe about see which who".split(" "));
|
|
120
|
+
const CLOSE_VERBS = new Set("close closed closing done finished completed resolved ship shipped drop dropped cancel cancelled abandoned".split(" "));
|
|
121
|
+
const UPDATE_VERBS = new Set("updated changed edited touched".split(" "));
|
|
122
|
+
/** Terminal status values: matching one is a question about the past. */
|
|
123
|
+
const TERMINAL = [
|
|
124
|
+
"closed",
|
|
125
|
+
"complete",
|
|
126
|
+
"completed",
|
|
127
|
+
"done",
|
|
128
|
+
"resolved",
|
|
129
|
+
"superseded"
|
|
130
|
+
];
|
|
131
|
+
/** Terminal statuses a closing verb selects: finished, not merely replaced. */
|
|
132
|
+
const CLOSED = [
|
|
133
|
+
"closed",
|
|
134
|
+
"complete",
|
|
135
|
+
"completed",
|
|
136
|
+
"done",
|
|
137
|
+
"resolved"
|
|
138
|
+
];
|
|
139
|
+
const HISTORY_WORDS = new Set("archived archive history historical ever previously past trend trends over-time all-time total".split(" "));
|
|
140
|
+
const LATEST = new Set("latest newest recent last".split(" "));
|
|
141
|
+
const OLDEST = new Set("oldest earliest first".split(" "));
|
|
142
|
+
const AUTHOR_VERBS = new Set("raised created written filed authored opened made sent logged".split(" "));
|
|
143
|
+
const BUILTIN_FIELD_ALIASES = { author: "createdById" };
|
|
144
|
+
/** `field:value`, `field:[a,b]`, `field:"two words"`; optionally wrapped in quotes. A clock time starts with a digit. */
|
|
145
|
+
const CLAUSE = /(['"]?)\b([A-Za-z][A-Za-z0-9]*):(\[[^\]]*\]|"[^"]*"|[^\s,'"]+)\1/g;
|
|
146
|
+
function parseClause(span, field, raw) {
|
|
147
|
+
const unquote = (v) => v.trim().replace(/^["']|["']$/g, "");
|
|
148
|
+
if (raw.startsWith("[")) return {
|
|
149
|
+
span,
|
|
150
|
+
field,
|
|
151
|
+
op: "in",
|
|
152
|
+
values: raw.slice(1, -1).split(",").map(unquote).filter((v) => v.length > 0)
|
|
153
|
+
};
|
|
154
|
+
if (raw.startsWith("\"")) return {
|
|
155
|
+
span,
|
|
156
|
+
field,
|
|
157
|
+
op: "eq",
|
|
158
|
+
values: [unquote(raw)]
|
|
159
|
+
};
|
|
160
|
+
const cmp = /^(>=|<=|>|<)(.+)$/.exec(raw);
|
|
161
|
+
if (cmp !== null) return {
|
|
162
|
+
span,
|
|
163
|
+
field,
|
|
164
|
+
op: {
|
|
165
|
+
">": "gt",
|
|
166
|
+
">=": "gte",
|
|
167
|
+
"<": "lt",
|
|
168
|
+
"<=": "lte"
|
|
169
|
+
}[cmp[1]],
|
|
170
|
+
values: [cmp[2]]
|
|
171
|
+
};
|
|
172
|
+
const range = /^(.+)\.\.(.+)$/.exec(raw);
|
|
173
|
+
if (range !== null) return {
|
|
174
|
+
span,
|
|
175
|
+
field,
|
|
176
|
+
op: "between",
|
|
177
|
+
values: [range[1], range[2]]
|
|
178
|
+
};
|
|
179
|
+
if (raw.startsWith("!")) return {
|
|
180
|
+
span,
|
|
181
|
+
field,
|
|
182
|
+
op: "neq",
|
|
183
|
+
values: [raw.slice(1)]
|
|
184
|
+
};
|
|
185
|
+
if (raw.startsWith("~")) return {
|
|
186
|
+
span,
|
|
187
|
+
field,
|
|
188
|
+
op: "contains",
|
|
189
|
+
values: [raw.slice(1)]
|
|
190
|
+
};
|
|
191
|
+
return {
|
|
192
|
+
span,
|
|
193
|
+
field,
|
|
194
|
+
op: "eq",
|
|
195
|
+
values: [raw]
|
|
196
|
+
};
|
|
197
|
+
}
|
|
198
|
+
function parseQuestion(question, pack, ctx) {
|
|
199
|
+
const rulings = ctx.rulings ?? {};
|
|
200
|
+
const clauses = [];
|
|
201
|
+
const tokens = (question.replace(CLAUSE, (span, _q, field, raw) => {
|
|
202
|
+
clauses.push(parseClause(span, field, raw));
|
|
203
|
+
return " ";
|
|
204
|
+
}).toLowerCase().replace(/[‘’]/g, "'").match(/#\d+|[a-z0-9@[][a-z0-9@._'\]-]*/g) ?? []).map((t) => t.replace(/[.'-]+$/, "").replace(/'s$/, "")).filter((t) => t.length > 0).map((text) => ({
|
|
205
|
+
text,
|
|
206
|
+
used: false
|
|
207
|
+
}));
|
|
208
|
+
const words = tokens.map((t) => t.text);
|
|
209
|
+
const evidence = [];
|
|
210
|
+
const lead = words.find((w) => !LEADING_FILLER.has(w));
|
|
211
|
+
if (lead !== void 0 && MUTATIONS.has(lead)) return {
|
|
212
|
+
status: "refuse",
|
|
213
|
+
candidates: [],
|
|
214
|
+
residual: [],
|
|
215
|
+
reason: `"${lead}" is an edit; substrate-nl answers questions and does not change data`
|
|
216
|
+
};
|
|
217
|
+
const values = [];
|
|
218
|
+
claimNgrams(tokens, 4, (key, at) => {
|
|
219
|
+
const alias = own(rulings.valueAliases, key);
|
|
220
|
+
if (alias === void 0) return false;
|
|
221
|
+
values.push({
|
|
222
|
+
span: key,
|
|
223
|
+
key: null,
|
|
224
|
+
alias,
|
|
225
|
+
at
|
|
226
|
+
});
|
|
227
|
+
return true;
|
|
228
|
+
});
|
|
229
|
+
const mentioned = /* @__PURE__ */ new Map();
|
|
230
|
+
for (const entry of buildTypeIndex(pack, rulings)) {
|
|
231
|
+
const n = entry.phrase.split(" ").length;
|
|
232
|
+
for (let i = 0; i + n <= tokens.length; i++) {
|
|
233
|
+
const slice = tokens.slice(i, i + n);
|
|
234
|
+
if (slice.some((t) => t.used) || slice.map((t) => t.text).join(" ") !== entry.phrase) continue;
|
|
235
|
+
for (const t of slice) t.used = true;
|
|
236
|
+
for (const type of entry.types) mentioned.set(type, {
|
|
237
|
+
span: entry.phrase,
|
|
238
|
+
plural: entry.plural
|
|
239
|
+
});
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
for (const [type, m] of mentioned) evidence.push({
|
|
243
|
+
span: m.span,
|
|
244
|
+
role: "type",
|
|
245
|
+
detail: type
|
|
246
|
+
});
|
|
247
|
+
const time = claimTime(tokens, ctx.asOf);
|
|
248
|
+
if (time !== null) evidence.push({
|
|
249
|
+
span: time.span,
|
|
250
|
+
role: "time",
|
|
251
|
+
detail: `${time.range.gte} .. ${time.range.lt}`
|
|
252
|
+
});
|
|
253
|
+
const ids = [];
|
|
254
|
+
let bareNumber = false;
|
|
255
|
+
tokens.forEach((t, i) => {
|
|
256
|
+
if (t.used) return;
|
|
257
|
+
if (/^#\d+$/.test(t.text)) {
|
|
258
|
+
ids.push(Number(t.text.slice(1)));
|
|
259
|
+
t.used = true;
|
|
260
|
+
evidence.push({
|
|
261
|
+
span: t.text,
|
|
262
|
+
role: "id",
|
|
263
|
+
detail: "shortId"
|
|
264
|
+
});
|
|
265
|
+
} else if (/^\d+$/.test(t.text)) {
|
|
266
|
+
if (tokens[i - 1]?.used === true && mentioned.size > 0) {
|
|
267
|
+
ids.push(Number(t.text));
|
|
268
|
+
t.used = true;
|
|
269
|
+
evidence.push({
|
|
270
|
+
span: t.text,
|
|
271
|
+
role: "id",
|
|
272
|
+
detail: "shortId"
|
|
273
|
+
});
|
|
274
|
+
} else bareNumber = true;
|
|
275
|
+
}
|
|
276
|
+
});
|
|
277
|
+
let order = null;
|
|
278
|
+
let count = false;
|
|
279
|
+
let allProjects = false;
|
|
280
|
+
let historyWord = false;
|
|
281
|
+
const singularMention = [...mentioned.values()].some((m) => !m.plural);
|
|
282
|
+
tokens.forEach((t, i) => {
|
|
283
|
+
if (t.used) return;
|
|
284
|
+
const next = tokens[i + 1];
|
|
285
|
+
if (LATEST.has(t.text) || t.text === "most" && /^recent(ly)?$/.test(next?.text ?? "") || t.text === "current" && singularMention) {
|
|
286
|
+
order = "desc";
|
|
287
|
+
t.used = true;
|
|
288
|
+
if (t.text === "most" && next !== void 0) next.used = true;
|
|
289
|
+
evidence.push({
|
|
290
|
+
span: t.text,
|
|
291
|
+
role: "order",
|
|
292
|
+
detail: "createdAt desc, limit 1"
|
|
293
|
+
});
|
|
294
|
+
} else if (OLDEST.has(t.text)) {
|
|
295
|
+
order = "asc";
|
|
296
|
+
t.used = true;
|
|
297
|
+
evidence.push({
|
|
298
|
+
span: t.text,
|
|
299
|
+
role: "order",
|
|
300
|
+
detail: "createdAt asc, limit 1"
|
|
301
|
+
});
|
|
302
|
+
} else if (t.text === "how" && next?.text === "many") {
|
|
303
|
+
count = true;
|
|
304
|
+
t.used = true;
|
|
305
|
+
next.used = true;
|
|
306
|
+
evidence.push({
|
|
307
|
+
span: "how many",
|
|
308
|
+
role: "count",
|
|
309
|
+
detail: "answer = count"
|
|
310
|
+
});
|
|
311
|
+
} else if ((t.text === "every" || t.text === "all") && /^projects?$/.test(next?.text ?? "")) {
|
|
312
|
+
allProjects = true;
|
|
313
|
+
t.used = true;
|
|
314
|
+
next.used = true;
|
|
315
|
+
evidence.push({
|
|
316
|
+
span: `${t.text} ${next?.text}`,
|
|
317
|
+
role: "scope",
|
|
318
|
+
detail: "no project scope"
|
|
319
|
+
});
|
|
320
|
+
} else if (t.text === "across") {
|
|
321
|
+
allProjects = true;
|
|
322
|
+
t.used = true;
|
|
323
|
+
} else if (HISTORY_WORDS.has(t.text)) {
|
|
324
|
+
historyWord = true;
|
|
325
|
+
t.used = true;
|
|
326
|
+
evidence.push({
|
|
327
|
+
span: t.text,
|
|
328
|
+
role: "history",
|
|
329
|
+
detail: "search the whole record"
|
|
330
|
+
});
|
|
331
|
+
}
|
|
332
|
+
});
|
|
333
|
+
let author = null;
|
|
334
|
+
tokens.forEach((t, i) => {
|
|
335
|
+
const prev = tokens[i - 1];
|
|
336
|
+
const next = tokens[i + 1];
|
|
337
|
+
if (t.text === "by" && prev !== void 0 && AUTHOR_VERBS.has(prev.text) && next !== void 0) {
|
|
338
|
+
author = next.text;
|
|
339
|
+
for (const x of [
|
|
340
|
+
prev,
|
|
341
|
+
t,
|
|
342
|
+
next
|
|
343
|
+
]) x.used = true;
|
|
344
|
+
} else if (t.text === "sent" && prev !== void 0 && !GLUE.has(prev.text) && !prev.used) {
|
|
345
|
+
author = prev.text;
|
|
346
|
+
prev.used = true;
|
|
347
|
+
t.used = true;
|
|
348
|
+
}
|
|
349
|
+
});
|
|
350
|
+
if (author !== null) evidence.push({
|
|
351
|
+
span: author,
|
|
352
|
+
role: "author",
|
|
353
|
+
detail: `createdById startsWith ${author}`
|
|
354
|
+
});
|
|
355
|
+
const fieldAliases = {
|
|
356
|
+
...BUILTIN_FIELD_ALIASES,
|
|
357
|
+
...rulings.fieldAliases ?? {}
|
|
358
|
+
};
|
|
359
|
+
let askField = null;
|
|
360
|
+
tokens.forEach((t, i) => {
|
|
361
|
+
if (t.used || tokens[i - 1]?.text !== "which") return;
|
|
362
|
+
for (const form of [t.text, t.text.replace(/s$/, "")]) {
|
|
363
|
+
const field = own(fieldAliases, form) ?? (fieldNamed(pack, form) ? form : null);
|
|
364
|
+
if (field === null) continue;
|
|
365
|
+
askField = {
|
|
366
|
+
field,
|
|
367
|
+
group: form !== t.text
|
|
368
|
+
};
|
|
369
|
+
t.used = true;
|
|
370
|
+
evidence.push({
|
|
371
|
+
span: t.text,
|
|
372
|
+
role: "field",
|
|
373
|
+
detail: askField.group ? `group by ${field}` : `answer = ${field}`
|
|
374
|
+
});
|
|
375
|
+
return;
|
|
376
|
+
}
|
|
377
|
+
});
|
|
378
|
+
const flagClaims = /* @__PURE__ */ new Map();
|
|
379
|
+
for (const type of mentioned.keys()) for (const [name, f] of Object.entries(own(pack.types, type)?.fields ?? {})) {
|
|
380
|
+
if (f.valueType !== "boolean" || name === "archived") continue;
|
|
381
|
+
const t = tokens.find((x) => !x.used && f.phrases.includes(x.text));
|
|
382
|
+
if (t === void 0) continue;
|
|
383
|
+
t.used = true;
|
|
384
|
+
flagClaims.set(type, [...flagClaims.get(type) ?? [], {
|
|
385
|
+
span: t.text,
|
|
386
|
+
role: "value",
|
|
387
|
+
detail: name
|
|
388
|
+
}]);
|
|
389
|
+
}
|
|
390
|
+
claimNgrams(tokens, 4, (key, at) => {
|
|
391
|
+
if (at.length === 1 && GLUE.has(key)) return false;
|
|
392
|
+
if (at.length === 1 && CLOSE_VERBS.has(key)) return false;
|
|
393
|
+
const found = own(pack.values, key) !== void 0 ? key : at.length === 1 && /[^s]s$/.test(key) && own(pack.values, key.slice(0, -1)) !== void 0 ? key.slice(0, -1) : null;
|
|
394
|
+
if (found === null) return false;
|
|
395
|
+
values.push({
|
|
396
|
+
span: key,
|
|
397
|
+
key: found,
|
|
398
|
+
at
|
|
399
|
+
});
|
|
400
|
+
return true;
|
|
401
|
+
});
|
|
402
|
+
const closeVerb = words.some((w) => CLOSE_VERBS.has(w));
|
|
403
|
+
const updateVerb = words.some((w) => UPDATE_VERBS.has(w));
|
|
404
|
+
for (const t of tokens) if (CLOSE_VERBS.has(t.text) || UPDATE_VERBS.has(t.text)) t.used = true;
|
|
405
|
+
let types = [...mentioned.keys()].sort();
|
|
406
|
+
if (ctx.forceType !== void 0 && own(pack.types, ctx.forceType) !== void 0) {
|
|
407
|
+
types = [ctx.forceType];
|
|
408
|
+
if (!mentioned.has(ctx.forceType)) evidence.push({
|
|
409
|
+
span: "",
|
|
410
|
+
role: "type",
|
|
411
|
+
detail: ctx.forceType
|
|
412
|
+
});
|
|
413
|
+
}
|
|
414
|
+
if (types.length === 0 && ids.length > 0 && ctx.defaultIdType !== void 0) types = [ctx.defaultIdType];
|
|
415
|
+
if (types.length === 0 && values.length > 0) {
|
|
416
|
+
types = typesCovering(pack, values);
|
|
417
|
+
const live = types.filter((t) => values.some((v) => v.key !== null && v.key === own(rulings.liveStatus, t)));
|
|
418
|
+
if (types.length > 1 && live.length > 0) types = live;
|
|
419
|
+
}
|
|
420
|
+
if (types.length === 0 && clauses.length > 0) types = typesWithFields(pack, clauses, rulings);
|
|
421
|
+
if (types.length === 0 && closeVerb) types = closableTypes(pack);
|
|
422
|
+
const named = mentioned.size > 0 || ctx.forceType !== void 0;
|
|
423
|
+
if (bareNumber && ids.length === 0 && !named) return {
|
|
424
|
+
status: "clarify",
|
|
425
|
+
candidates: [],
|
|
426
|
+
residual: [],
|
|
427
|
+
reason: "a bare number with no type"
|
|
428
|
+
};
|
|
429
|
+
if (ids.length > 0 && !named && !words.some((w) => STATE_CUES.has(w))) return {
|
|
430
|
+
status: "clarify",
|
|
431
|
+
candidates: [],
|
|
432
|
+
residual: [],
|
|
433
|
+
reason: "#N with no question about it: a Todo or a PR?"
|
|
434
|
+
};
|
|
435
|
+
if (types.length === 0) return {
|
|
436
|
+
status: "unparsed",
|
|
437
|
+
candidates: [],
|
|
438
|
+
evidence,
|
|
439
|
+
residual: contentWords(tokens),
|
|
440
|
+
reason: "no type, id or known value"
|
|
441
|
+
};
|
|
442
|
+
const candidates = [];
|
|
443
|
+
const ruledOut = [];
|
|
444
|
+
for (const type of types) {
|
|
445
|
+
const c = assemble(type, {
|
|
446
|
+
pack,
|
|
447
|
+
ctx,
|
|
448
|
+
tokens: tokens.map((t) => ({ ...t })),
|
|
449
|
+
evidence: evidence.filter((e) => e.role !== "type" || e.detail === type),
|
|
450
|
+
ids,
|
|
451
|
+
values,
|
|
452
|
+
clauses,
|
|
453
|
+
time,
|
|
454
|
+
order,
|
|
455
|
+
count,
|
|
456
|
+
allProjects,
|
|
457
|
+
historyWord,
|
|
458
|
+
flags: flagClaims.get(type) ?? [],
|
|
459
|
+
author,
|
|
460
|
+
askField,
|
|
461
|
+
closeVerb,
|
|
462
|
+
updateVerb
|
|
463
|
+
});
|
|
464
|
+
if ("reason" in c) ruledOut.push(`${type}: ${c.reason}`);
|
|
465
|
+
else candidates.push(c);
|
|
466
|
+
}
|
|
467
|
+
if (candidates.length === 0) {
|
|
468
|
+
const reason = ruledOut.length > 0 ? ruledOut.join("; ") : "no type fits every value";
|
|
469
|
+
return {
|
|
470
|
+
status: "unparsed",
|
|
471
|
+
candidates,
|
|
472
|
+
evidence,
|
|
473
|
+
residual: contentWords(tokens),
|
|
474
|
+
reason
|
|
475
|
+
};
|
|
476
|
+
}
|
|
477
|
+
if (candidates.length > 1) return {
|
|
478
|
+
status: "ambiguous",
|
|
479
|
+
candidates,
|
|
480
|
+
residual: [],
|
|
481
|
+
reason: `${candidates.length} readings`
|
|
482
|
+
};
|
|
483
|
+
const only = candidates[0];
|
|
484
|
+
if (only.residual.length > 0 && ids.length === 0) return {
|
|
485
|
+
status: "partial",
|
|
486
|
+
candidates,
|
|
487
|
+
residual: only.residual,
|
|
488
|
+
reason: `unexplained: ${only.residual.join(", ")}`
|
|
489
|
+
};
|
|
490
|
+
return {
|
|
491
|
+
status: "resolved",
|
|
492
|
+
candidates,
|
|
493
|
+
residual: only.residual
|
|
494
|
+
};
|
|
495
|
+
}
|
|
496
|
+
/** One reading of the question as `type`, or why there is none. */
|
|
497
|
+
function assemble(type, a) {
|
|
498
|
+
const { pack, ctx, tokens } = a;
|
|
499
|
+
const rulings = ctx.rulings ?? {};
|
|
500
|
+
const fields = own(pack.types, type)?.fields ?? {};
|
|
501
|
+
const words = tokens.map((t) => t.text);
|
|
502
|
+
const ev = [...a.evidence];
|
|
503
|
+
const where = [];
|
|
504
|
+
if (a.ids.length > 0) where.push({ shortId: a.ids.length === 1 ? { eq: a.ids[0] } : { in: a.ids } });
|
|
505
|
+
const byField = /* @__PURE__ */ new Map();
|
|
506
|
+
const usedFields = /* @__PURE__ */ new Set();
|
|
507
|
+
let terminal = false;
|
|
508
|
+
for (const v of a.ids.length > 0 ? [] : a.values) {
|
|
509
|
+
if (v.alias !== void 0) {
|
|
510
|
+
const hit = v.alias.find((x) => x.type === type);
|
|
511
|
+
if (hit === void 0) return { reason: `"${v.span}" does not apply to ${type}` };
|
|
512
|
+
const op = hit.in !== void 0 ? { in: hit.in } : hit.eq !== void 0 ? { eq: hit.eq } : hit.startsWith !== void 0 ? { startsWith: hit.startsWith } : { contains: hit.contains };
|
|
513
|
+
where.push({ [hit.field]: op });
|
|
514
|
+
usedFields.add(hit.field);
|
|
515
|
+
ev.push({
|
|
516
|
+
span: v.span,
|
|
517
|
+
role: "value",
|
|
518
|
+
detail: `${hit.field} ${JSON.stringify(op)}`
|
|
519
|
+
});
|
|
520
|
+
continue;
|
|
521
|
+
}
|
|
522
|
+
const refs = (own(a.pack.values, v.key) ?? []).filter((r) => r.type === type);
|
|
523
|
+
if (refs.length === 0) return { reason: `"${v.span}" is not a ${type} value` };
|
|
524
|
+
const field = pickField(refs.map((r) => r.field), fields, words);
|
|
525
|
+
const stored = refs.find((r) => r.field === field)?.value ?? v.key;
|
|
526
|
+
if (TERMINAL.includes(stored)) terminal = true;
|
|
527
|
+
byField.set(field, [...byField.get(field) ?? [], stored]);
|
|
528
|
+
usedFields.add(field);
|
|
529
|
+
ev.push({
|
|
530
|
+
span: v.span,
|
|
531
|
+
role: "value",
|
|
532
|
+
detail: `${field} = ${stored}`
|
|
533
|
+
});
|
|
534
|
+
}
|
|
535
|
+
for (const [field, vals] of [...byField].sort(([x], [y]) => x.localeCompare(y))) if (fields[field]?.kind === "list") for (const v of vals) where.push({ [field]: { has: v } });
|
|
536
|
+
else where.push({ [field]: vals.length === 1 ? { eq: vals[0] } : { in: vals } });
|
|
537
|
+
for (const c of a.clauses) {
|
|
538
|
+
const name = resolveField(c.field, fields, rulings);
|
|
539
|
+
const f = name === null ? void 0 : fields[name];
|
|
540
|
+
if (name === null || f === void 0) return { reason: `${type} has no field "${c.field}"` };
|
|
541
|
+
if (!f.operators.includes(c.op)) return { reason: `${name} does not accept ${c.op}` };
|
|
542
|
+
const coerced = c.values.map((v) => coerce(v, f));
|
|
543
|
+
if (coerced.some((v) => v === void 0)) return { reason: `"${c.span}" is not a ${f.valueType} value` };
|
|
544
|
+
const operand = c.op === "in" || c.op === "between" ? coerced : coerced[0];
|
|
545
|
+
where.push({ [name]: { [c.op]: operand } });
|
|
546
|
+
usedFields.add(name);
|
|
547
|
+
ev.push({
|
|
548
|
+
span: c.span,
|
|
549
|
+
role: "value",
|
|
550
|
+
detail: `${name} ${c.op} ${JSON.stringify(operand)} (inline)`
|
|
551
|
+
});
|
|
552
|
+
}
|
|
553
|
+
for (const field of usedFields) creditFieldWords(tokens, fields[field]);
|
|
554
|
+
if (a.author !== null) where.push({ createdById: { startsWith: a.author } });
|
|
555
|
+
for (const flag of a.flags) {
|
|
556
|
+
usedFields.add(flag.detail);
|
|
557
|
+
where.push({ [flag.detail]: { eq: true } });
|
|
558
|
+
ev.push({
|
|
559
|
+
...flag,
|
|
560
|
+
detail: `${flag.detail} = true`
|
|
561
|
+
});
|
|
562
|
+
}
|
|
563
|
+
for (const [name, f] of Object.entries(fields)) {
|
|
564
|
+
if (f.valueType !== "boolean" || name === "archived" || usedFields.has(name)) continue;
|
|
565
|
+
const t = tokens.find((x) => !x.used && f.phrases.includes(x.text));
|
|
566
|
+
if (t === void 0) continue;
|
|
567
|
+
t.used = true;
|
|
568
|
+
usedFields.add(name);
|
|
569
|
+
where.push({ [name]: { eq: true } });
|
|
570
|
+
ev.push({
|
|
571
|
+
span: t.text,
|
|
572
|
+
role: "value",
|
|
573
|
+
detail: `${name} = true`
|
|
574
|
+
});
|
|
575
|
+
}
|
|
576
|
+
const historical = a.historyWord || a.time !== null || a.closeVerb || terminal;
|
|
577
|
+
if (a.time !== null) {
|
|
578
|
+
const dateField = Object.entries(fields).find(([, f]) => f.valueType === "date" && f.kind !== "metadata" && creditFieldWords(tokens, f))?.[0] ?? (a.closeVerb || a.updateVerb ? "updatedAt" : own(rulings.timeField, type) ?? "createdAt");
|
|
579
|
+
where.push({ [dateField]: {
|
|
580
|
+
gte: a.time.range.gte,
|
|
581
|
+
lt: a.time.range.lt
|
|
582
|
+
} });
|
|
583
|
+
}
|
|
584
|
+
if (a.ids.length === 0) {
|
|
585
|
+
const statusField = fields.status;
|
|
586
|
+
const live = own(rulings.liveStatus, type);
|
|
587
|
+
if (a.closeVerb && statusField?.enum !== void 0 && !byField.has("status")) {
|
|
588
|
+
const closed = statusField.enum.filter((s) => CLOSED.includes(s));
|
|
589
|
+
if (closed.length > 0) where.push({ status: { in: closed } });
|
|
590
|
+
} else if (!historical && live !== void 0 && statusField?.enum?.includes(live) === true && !usedFields.has("status") && a.order === null) where.push({ status: { eq: live } });
|
|
591
|
+
if (!historical && fields.archived !== void 0) where.push({ archived: { eq: false } });
|
|
592
|
+
else if (historical) ev.push({
|
|
593
|
+
span: "(history)",
|
|
594
|
+
role: "history",
|
|
595
|
+
detail: "archived rows included"
|
|
596
|
+
});
|
|
597
|
+
if (fields.active?.valueType === "boolean" && !usedFields.has("active") && !words.includes("inactive")) where.push({ active: { eq: true } });
|
|
598
|
+
const scoped = fields.project !== void 0 && !(rulings.unscopedTypes ?? []).includes(type);
|
|
599
|
+
const groupedByProject = a.askField?.group === true && a.askField.field === "project";
|
|
600
|
+
if (scoped && !usedFields.has("project") && !a.allProjects && !groupedByProject && ctx.project !== void 0) {
|
|
601
|
+
where.push({ OR: [{ project: { eq: ctx.project } }, { project: { eq: null } }] });
|
|
602
|
+
ev.push({
|
|
603
|
+
span: "(context)",
|
|
604
|
+
role: "scope",
|
|
605
|
+
detail: `project = ${ctx.project} or none`
|
|
606
|
+
});
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
const params = {};
|
|
610
|
+
if (where.length === 1) params.where = where[0];
|
|
611
|
+
else if (where.length > 1) params.where = { AND: where };
|
|
612
|
+
if (a.order !== null) {
|
|
613
|
+
params.orderBy = [{ createdAt: a.order }];
|
|
614
|
+
params.limit = 1;
|
|
615
|
+
}
|
|
616
|
+
const answer = a.askField !== null ? a.askField.group ? `group:${a.askField.field}` : `field:${a.askField.field}` : a.count ? "count" : a.order !== null || a.ids.length === 1 ? "single" : "rows";
|
|
617
|
+
return {
|
|
618
|
+
type,
|
|
619
|
+
frame: { [`${pack.namespace}/${type}`]: { params } },
|
|
620
|
+
answer,
|
|
621
|
+
evidence: ev,
|
|
622
|
+
residual: contentWords(tokens)
|
|
623
|
+
};
|
|
624
|
+
}
|
|
625
|
+
/** A clause's field name: the field itself, case-insensitively, then the rulings' and built-in aliases. */
|
|
626
|
+
function resolveField(raw, fields, rulings) {
|
|
627
|
+
const lower = raw.toLowerCase();
|
|
628
|
+
const direct = Object.keys(fields).find((f) => f.toLowerCase() === lower);
|
|
629
|
+
if (direct !== void 0) return direct;
|
|
630
|
+
const aliased = own(rulings.fieldAliases, lower) ?? own(BUILTIN_FIELD_ALIASES, lower);
|
|
631
|
+
return aliased !== void 0 && Object.hasOwn(fields, aliased) ? aliased : null;
|
|
632
|
+
}
|
|
633
|
+
/** A clause value as the field's type, or undefined when it is not one. */
|
|
634
|
+
function coerce(raw, f) {
|
|
635
|
+
if (f.valueType === "number") {
|
|
636
|
+
const n = Number(raw);
|
|
637
|
+
return Number.isFinite(n) ? n : void 0;
|
|
638
|
+
}
|
|
639
|
+
if (f.valueType === "boolean") return raw === "true" ? true : raw === "false" ? false : void 0;
|
|
640
|
+
return raw;
|
|
641
|
+
}
|
|
642
|
+
/** Types that have every field the clauses name, sorted. */
|
|
643
|
+
function typesWithFields(pack, clauses, rulings) {
|
|
644
|
+
return Object.entries(pack.types).filter(([, t]) => clauses.every((c) => resolveField(c.field, t.fields, rulings) !== null)).map(([name]) => name).sort();
|
|
645
|
+
}
|
|
646
|
+
/** Marks unused tokens naming `field` as explained; true when any did. */
|
|
647
|
+
function creditFieldWords(tokens, field) {
|
|
648
|
+
if (field === void 0) return false;
|
|
649
|
+
let hit = false;
|
|
650
|
+
for (const phrase of field.phrases) {
|
|
651
|
+
const parts = phrase.split(" ");
|
|
652
|
+
const stem = (parts[parts.length - 1] ?? "").replace(/s$/, "");
|
|
653
|
+
for (let i = 0; i + parts.length <= tokens.length; i++) {
|
|
654
|
+
const slice = tokens.slice(i, i + parts.length);
|
|
655
|
+
if (slice.some((t) => t.used)) continue;
|
|
656
|
+
const text = slice.map((t) => t.text).join(" ");
|
|
657
|
+
const last = slice[slice.length - 1]?.text ?? "";
|
|
658
|
+
const head = slice.slice(0, -1).map((t) => t.text).join(" ");
|
|
659
|
+
const headWant = parts.slice(0, -1).join(" ");
|
|
660
|
+
if (text === phrase || head === headWant && stem.length >= 3 && last.startsWith(stem)) {
|
|
661
|
+
for (const t of slice) t.used = true;
|
|
662
|
+
hit = true;
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
}
|
|
666
|
+
return hit;
|
|
667
|
+
}
|
|
668
|
+
function contentWords(tokens) {
|
|
669
|
+
return tokens.filter((t) => !t.used && !GLUE.has(t.text) && !/^\d+$/.test(t.text)).map((t) => t.text);
|
|
670
|
+
}
|
|
671
|
+
/** Claims the longest n-grams (up to `max` words) for which `accept` returns true. */
|
|
672
|
+
function claimNgrams(tokens, max, accept) {
|
|
673
|
+
for (let n = max; n >= 1; n--) for (let i = 0; i + n <= tokens.length; i++) {
|
|
674
|
+
const slice = tokens.slice(i, i + n);
|
|
675
|
+
if (slice.some((t) => t.used)) continue;
|
|
676
|
+
const at = slice.map((_, j) => i + j);
|
|
677
|
+
if (!accept(slice.map((t) => t.text).join(" "), at)) continue;
|
|
678
|
+
for (const t of slice) t.used = true;
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
/** Type phrases: rulings aliases, then the pack's, plus each multi-word type's first word. Longest first. */
|
|
682
|
+
function buildTypeIndex(pack, rulings) {
|
|
683
|
+
const map = /* @__PURE__ */ new Map();
|
|
684
|
+
const add = (phrase, type, plural) => {
|
|
685
|
+
const e = map.get(phrase) ?? {
|
|
686
|
+
types: /* @__PURE__ */ new Set(),
|
|
687
|
+
plural
|
|
688
|
+
};
|
|
689
|
+
e.types.add(type);
|
|
690
|
+
map.set(phrase, e);
|
|
691
|
+
};
|
|
692
|
+
const aliased = new Set(Object.keys(rulings.typeAliases ?? {}));
|
|
693
|
+
for (const [phrase, type] of Object.entries(rulings.typeAliases ?? {})) add(phrase, type, phrase.endsWith("s"));
|
|
694
|
+
for (const [name, t] of Object.entries(pack.types)) {
|
|
695
|
+
const words = splitWords(name);
|
|
696
|
+
const singulars = /* @__PURE__ */ new Set([words.join(" "), words[words.length - 1] ?? ""]);
|
|
697
|
+
for (const p of [...t.phrases, name.toLowerCase()]) if (!aliased.has(p)) add(p, name, !singulars.has(p) && p !== name.toLowerCase());
|
|
698
|
+
if (words.length > 1) {
|
|
699
|
+
const first = words[0];
|
|
700
|
+
if (!aliased.has(first)) add(first, name, false);
|
|
701
|
+
if (!aliased.has(pluralize(first))) add(pluralize(first), name, true);
|
|
702
|
+
}
|
|
703
|
+
}
|
|
704
|
+
return [...map].map(([phrase, e]) => ({
|
|
705
|
+
phrase,
|
|
706
|
+
types: [...e.types].sort(),
|
|
707
|
+
plural: e.plural
|
|
708
|
+
})).sort((x, y) => y.phrase.split(" ").length - x.phrase.split(" ").length || x.phrase.localeCompare(y.phrase));
|
|
709
|
+
}
|
|
710
|
+
function fieldNamed(pack, word) {
|
|
711
|
+
return Object.values(pack.types).some((t) => Object.values(t.fields).some((f) => f.phrases.includes(word)));
|
|
712
|
+
}
|
|
713
|
+
/** Types every value can be read as, sorted. */
|
|
714
|
+
function typesCovering(pack, values) {
|
|
715
|
+
let acc = null;
|
|
716
|
+
for (const v of values) {
|
|
717
|
+
const types = new Set(v.alias !== void 0 ? v.alias.map((x) => x.type) : (own(pack.values, v.key) ?? []).map((r) => r.type));
|
|
718
|
+
acc = acc === null ? [...types] : acc.filter((t) => types.has(t));
|
|
719
|
+
}
|
|
720
|
+
return [...acc ?? []].sort();
|
|
721
|
+
}
|
|
722
|
+
/** Types whose `status` enum has a terminal value. */
|
|
723
|
+
function closableTypes(pack) {
|
|
724
|
+
return Object.entries(pack.types).filter(([, t]) => t.fields.status?.enum?.some((v) => CLOSED.includes(v)) === true).map(([name]) => name).sort();
|
|
725
|
+
}
|
|
726
|
+
/**
|
|
727
|
+
* The field a value reads as when it fits several on one type: a field named
|
|
728
|
+
* in the question first, then the most categorical (fewest distinct values,
|
|
729
|
+
* enum size for enums), then name order.
|
|
730
|
+
*/
|
|
731
|
+
function pickField(candidates, fields, words) {
|
|
732
|
+
const unique = [...new Set(candidates)];
|
|
733
|
+
const text = ` ${words.join(" ")} `;
|
|
734
|
+
const stem = (p) => p.replace(/s$/, "");
|
|
735
|
+
const named = unique.filter((f) => fields[f]?.phrases.some((p) => text.includes(` ${p} `) || stem(p).length >= 3 && words.some((w) => w !== p && w.startsWith(stem(p)))));
|
|
736
|
+
const pool = named.length > 0 ? named : unique;
|
|
737
|
+
const size = (f) => fields[f]?.enum?.length ?? fields[f]?.distinct ?? Number.MAX_SAFE_INTEGER;
|
|
738
|
+
return [...pool].sort((x, y) => size(x) - size(y) || x.localeCompare(y))[0];
|
|
739
|
+
}
|
|
740
|
+
function claimTime(tokens, asOf) {
|
|
741
|
+
let found = null;
|
|
742
|
+
claimNgrams(tokens, 3, (key) => {
|
|
743
|
+
if (found !== null) return false;
|
|
744
|
+
const range = rangeFor(key, asOf);
|
|
745
|
+
if (range === null) return false;
|
|
746
|
+
found = {
|
|
747
|
+
span: key,
|
|
748
|
+
range
|
|
749
|
+
};
|
|
750
|
+
return true;
|
|
751
|
+
});
|
|
752
|
+
return found;
|
|
753
|
+
}
|
|
754
|
+
|
|
755
|
+
//#endregion
|
|
756
|
+
//#region src/judge/judge.ts
|
|
757
|
+
/** String scalar fields a binding may target, as `Type.field`. */
|
|
758
|
+
function bindableFields(pack, types) {
|
|
759
|
+
const out = [];
|
|
760
|
+
for (const type of types) for (const [name, f] of Object.entries(pack.types[type]?.fields ?? {})) if (f.kind === "scalar" && f.valueType === "string" && f.indexed) out.push(`${type}.${name}`);
|
|
761
|
+
return out.sort();
|
|
762
|
+
}
|
|
763
|
+
/** Build the options, the prompt and the schema for one escalated parse. */
|
|
764
|
+
function buildJudgePrompt(question, result, pack, rulings = {}, rejected = []) {
|
|
765
|
+
const allTypes = Object.keys(pack.types).sort();
|
|
766
|
+
const words = [...new Set(result.residual)];
|
|
767
|
+
const fields = bindableFields(pack, result.candidates.length > 0 ? [...new Set(result.candidates.map((c) => c.type))] : allTypes);
|
|
768
|
+
const branch = (action, args = {}) => ({
|
|
769
|
+
type: "object",
|
|
770
|
+
properties: {
|
|
771
|
+
action: { const: action },
|
|
772
|
+
...args
|
|
773
|
+
},
|
|
774
|
+
required: ["action", ...Object.keys(args)]
|
|
775
|
+
});
|
|
776
|
+
const schema = { anyOf: [
|
|
777
|
+
...result.candidates.length > 0 ? [branch("pick", { candidate: {
|
|
778
|
+
type: "integer",
|
|
779
|
+
enum: result.candidates.map((_, i) => i)
|
|
780
|
+
} })] : [],
|
|
781
|
+
branch("type", { type: {
|
|
782
|
+
type: "string",
|
|
783
|
+
enum: allTypes
|
|
784
|
+
} }),
|
|
785
|
+
...words.length > 0 && fields.length > 0 ? [branch("bind", {
|
|
786
|
+
word: {
|
|
787
|
+
type: "string",
|
|
788
|
+
enum: words
|
|
789
|
+
},
|
|
790
|
+
field: {
|
|
791
|
+
type: "string",
|
|
792
|
+
enum: fields
|
|
793
|
+
},
|
|
794
|
+
op: {
|
|
795
|
+
type: "string",
|
|
796
|
+
enum: [
|
|
797
|
+
"eq",
|
|
798
|
+
"startsWith",
|
|
799
|
+
"contains"
|
|
800
|
+
]
|
|
801
|
+
}
|
|
802
|
+
})] : [],
|
|
803
|
+
branch("recall"),
|
|
804
|
+
branch("clarify"),
|
|
805
|
+
branch("refuse"),
|
|
806
|
+
branch("escalate")
|
|
807
|
+
] };
|
|
808
|
+
const ruled = (t) => Object.entries(rulings.typeAliases ?? {}).filter(([, type]) => type === t).map(([phrase]) => phrase);
|
|
809
|
+
const typeLines = allTypes.map((t) => {
|
|
810
|
+
const note = rulings.typeNotes?.[t];
|
|
811
|
+
const names = [...ruled(t), ...(pack.types[t]?.phrases ?? []).slice(0, 4)].join(", ");
|
|
812
|
+
return `- ${t}${note !== void 0 ? ` (${note})` : ""}: ${names}`;
|
|
813
|
+
});
|
|
814
|
+
const candidateLines = result.candidates.map((c, i) => `${i}. ${c.type} where ${JSON.stringify(Object.values(c.frame)[0]?.params.where ?? {})} (from: ${c.evidence.map((e) => `"${e.span}"=${e.detail}`).join("; ")})`);
|
|
815
|
+
return {
|
|
816
|
+
system: [
|
|
817
|
+
"You choose how a question about a data store should be answered. You do not write queries.",
|
|
818
|
+
"A deterministic parser read the question first and could not settle it. Choose ONE action:",
|
|
819
|
+
"- pick: one reading answers the WHOLE question as asked; leftover words are noise. If the",
|
|
820
|
+
" question needs rows of one type found through another, no single reading answers it.",
|
|
821
|
+
"- type: the question lists or counts rows of one type; the parser then builds the query from the",
|
|
822
|
+
" question's own words (dates, \"latest\", known values). Prefer this when a type clearly fits.",
|
|
823
|
+
"- bind: a leftover word is a value stored in a field: eq = the whole value, startsWith = a prefix",
|
|
824
|
+
" of a name or id, contains = a word inside a title or subject. Never bind a word that only",
|
|
825
|
+
" describes the question (e.g. \"status\", \"called\").",
|
|
826
|
+
"- recall: the question asks what is known ABOUT a topic (free-text subject words, not field values);",
|
|
827
|
+
" semantic search over the text answers it.",
|
|
828
|
+
"- clarify: RARE. Only when the question has two specific readings and nothing in it chooses one.",
|
|
829
|
+
" Missing detail is not ambiguity: if a type or a topic fits, choose type or recall.",
|
|
830
|
+
"- refuse: the question is not about this store at all, or asks to change data.",
|
|
831
|
+
"- escalate: none of these answers the question as asked (e.g. it needs two queries chained).",
|
|
832
|
+
"",
|
|
833
|
+
`Types in the "${pack.namespace}" store:`,
|
|
834
|
+
...typeLines
|
|
835
|
+
].join("\n"),
|
|
836
|
+
user: [
|
|
837
|
+
`Question: ${question}`,
|
|
838
|
+
`Parser: ${result.status}${result.reason !== void 0 ? ` (${result.reason})` : ""}`,
|
|
839
|
+
...result.evidence !== void 0 && result.evidence.length > 0 ? [`Understood so far: ${result.evidence.map((e) => `"${e.span}" = ${e.role} ${e.detail}`).join("; ")}`] : [],
|
|
840
|
+
...candidateLines.length > 0 ? ["Readings:", ...candidateLines] : [],
|
|
841
|
+
...words.length > 0 ? [`Leftover words: ${words.join(", ")}`] : [],
|
|
842
|
+
...rejected.map((r) => `Already tried and rejected: ${r}`)
|
|
843
|
+
].join("\n"),
|
|
844
|
+
schema
|
|
845
|
+
};
|
|
846
|
+
}
|
|
847
|
+
/** Check a reply against the options it was offered. */
|
|
848
|
+
function checkChoice(reply, prompt) {
|
|
849
|
+
if (typeof reply !== "object" || reply === null) return "reply is not an object";
|
|
850
|
+
const r = reply;
|
|
851
|
+
const branch = (prompt.schema.anyOf ?? []).find((b) => b.properties.action?.const === r.action);
|
|
852
|
+
if (branch === void 0) return `action ${JSON.stringify(r.action)} was not offered`;
|
|
853
|
+
const choice = { action: r.action };
|
|
854
|
+
for (const key of branch.required) {
|
|
855
|
+
if (key === "action") continue;
|
|
856
|
+
if (branch.properties[key]?.enum?.includes(r[key]) !== true) return `${String(r.action)} needs an offered ${key}`;
|
|
857
|
+
choice[key] = r[key];
|
|
858
|
+
}
|
|
859
|
+
return choice;
|
|
860
|
+
}
|
|
861
|
+
/** A binding becomes a one-question ruling, so the parser applies it like any other. */
|
|
862
|
+
function withBinding(rulings, choice) {
|
|
863
|
+
const [type, field] = choice.field.split(".");
|
|
864
|
+
const word = choice.word;
|
|
865
|
+
const alias = choice.op === "startsWith" ? {
|
|
866
|
+
type,
|
|
867
|
+
field,
|
|
868
|
+
startsWith: word
|
|
869
|
+
} : choice.op === "contains" ? {
|
|
870
|
+
type,
|
|
871
|
+
field,
|
|
872
|
+
contains: word
|
|
873
|
+
} : {
|
|
874
|
+
type,
|
|
875
|
+
field,
|
|
876
|
+
eq: word
|
|
877
|
+
};
|
|
878
|
+
return {
|
|
879
|
+
...rulings ?? {},
|
|
880
|
+
valueAliases: {
|
|
881
|
+
...rulings?.valueAliases ?? {},
|
|
882
|
+
[word]: [alias]
|
|
883
|
+
}
|
|
884
|
+
};
|
|
885
|
+
}
|
|
886
|
+
/** Why a binding cannot be applied, or null when it grounds. */
|
|
887
|
+
async function verifyBinding(choice, earlier, pack, opts) {
|
|
888
|
+
const [type, field] = choice.field.split(".");
|
|
889
|
+
const word = choice.word;
|
|
890
|
+
const def = own(own(pack.types, type)?.fields, field);
|
|
891
|
+
if (def === void 0) return `${type}.${field} is not a field`;
|
|
892
|
+
if (!def.operators.includes(choice.op)) return `${type}.${field} does not accept ${choice.op}`;
|
|
893
|
+
if (def.enum !== void 0 && choice.op === "eq") {
|
|
894
|
+
if (![...def.enum, ...Object.keys(def.aliases ?? {})].map((v) => v.toLowerCase()).includes(word.toLowerCase())) return `"${word}" is not a ${type}.${field} value`;
|
|
895
|
+
}
|
|
896
|
+
if (opts.exists === void 0) return null;
|
|
897
|
+
const predicate = (c) => ({ [c.field.split(".")[1]]: { [c.op]: c.word } });
|
|
898
|
+
const same = earlier.filter((c) => c.field.startsWith(`${type}.`));
|
|
899
|
+
const where = same.length === 0 ? predicate(choice) : { AND: [...same, choice].map(predicate) };
|
|
900
|
+
if (!await opts.exists(type, where)) return same.length === 0 ? `no ${type} has ${field} ${choice.op} "${word}"` : `no ${type} matches it together with ${same.map((c) => `"${c.word}"`).join(", ")}`;
|
|
901
|
+
return null;
|
|
902
|
+
}
|
|
903
|
+
/**
|
|
904
|
+
* Parse; while the parse is not resolved, ask the judge and apply its choice.
|
|
905
|
+
* A refused, clarified or resolved parse never reaches the model.
|
|
906
|
+
*/
|
|
907
|
+
async function parseWithJudge(question, pack, ctx, judge, opts = {}) {
|
|
908
|
+
const trail = [];
|
|
909
|
+
let context = ctx;
|
|
910
|
+
let result = parseQuestion(question, pack, context);
|
|
911
|
+
const rejected = [];
|
|
912
|
+
const bound = [];
|
|
913
|
+
for (let round = 0; round < (opts.rounds ?? 3); round++) {
|
|
914
|
+
if (result.status === "resolved") return {
|
|
915
|
+
status: "resolved",
|
|
916
|
+
candidate: result.candidates[0],
|
|
917
|
+
trail
|
|
918
|
+
};
|
|
919
|
+
if (result.status === "refuse" || result.status === "clarify") return {
|
|
920
|
+
status: result.status,
|
|
921
|
+
reason: result.reason ?? "",
|
|
922
|
+
trail
|
|
923
|
+
};
|
|
924
|
+
const step = { parse: {
|
|
925
|
+
status: result.status,
|
|
926
|
+
...result.reason !== void 0 ? { reason: result.reason } : {}
|
|
927
|
+
} };
|
|
928
|
+
trail.push(step);
|
|
929
|
+
const prompt = buildJudgePrompt(question, result, pack, context.rulings, rejected);
|
|
930
|
+
const checked = checkChoice(await judge.decide(prompt, opts.signal), prompt);
|
|
931
|
+
if (typeof checked === "string") {
|
|
932
|
+
step.rejected = checked;
|
|
933
|
+
break;
|
|
934
|
+
}
|
|
935
|
+
step.choice = checked;
|
|
936
|
+
switch (checked.action) {
|
|
937
|
+
case "pick": return {
|
|
938
|
+
status: "resolved",
|
|
939
|
+
candidate: result.candidates[checked.candidate],
|
|
940
|
+
trail
|
|
941
|
+
};
|
|
942
|
+
case "recall": return {
|
|
943
|
+
status: "recall",
|
|
944
|
+
query: question,
|
|
945
|
+
trail
|
|
946
|
+
};
|
|
947
|
+
case "clarify":
|
|
948
|
+
case "refuse": return {
|
|
949
|
+
status: checked.action,
|
|
950
|
+
reason: "judge",
|
|
951
|
+
trail
|
|
952
|
+
};
|
|
953
|
+
case "type":
|
|
954
|
+
context = {
|
|
955
|
+
...context,
|
|
956
|
+
forceType: checked.type
|
|
957
|
+
};
|
|
958
|
+
break;
|
|
959
|
+
case "escalate": return {
|
|
960
|
+
status: "escalated",
|
|
961
|
+
result,
|
|
962
|
+
trail
|
|
963
|
+
};
|
|
964
|
+
case "bind": {
|
|
965
|
+
const why = await verifyBinding(checked, bound, pack, opts);
|
|
966
|
+
if (why !== null) {
|
|
967
|
+
step.rejected = why;
|
|
968
|
+
rejected.push(`bind ${checked.word} to ${checked.field} (${why})`);
|
|
969
|
+
continue;
|
|
970
|
+
}
|
|
971
|
+
bound.push(checked);
|
|
972
|
+
context = {
|
|
973
|
+
...context,
|
|
974
|
+
rulings: withBinding(context.rulings, checked)
|
|
975
|
+
};
|
|
976
|
+
break;
|
|
977
|
+
}
|
|
978
|
+
}
|
|
979
|
+
result = parseQuestion(question, pack, context);
|
|
980
|
+
}
|
|
981
|
+
if (result.status === "resolved") return {
|
|
982
|
+
status: "resolved",
|
|
983
|
+
candidate: result.candidates[0],
|
|
984
|
+
trail
|
|
985
|
+
};
|
|
986
|
+
return {
|
|
987
|
+
status: "escalated",
|
|
988
|
+
result,
|
|
989
|
+
trail
|
|
990
|
+
};
|
|
991
|
+
}
|
|
992
|
+
|
|
993
|
+
//#endregion
|
|
994
|
+
//#region src/ask/run.ts
|
|
995
|
+
const RECALL_K = 10;
|
|
996
|
+
/** The frame a candidate needs to run: its params plus every projectable field of its type. */
|
|
997
|
+
function executable(c, pack) {
|
|
998
|
+
const [key, value] = Object.entries(c.frame)[0];
|
|
999
|
+
const fields = { id: true };
|
|
1000
|
+
for (const [name, f] of Object.entries(own(pack.types, c.type)?.fields ?? {})) if (f.kind !== "metadata") fields[name] = true;
|
|
1001
|
+
return { [key]: {
|
|
1002
|
+
params: value.params,
|
|
1003
|
+
fields
|
|
1004
|
+
} };
|
|
1005
|
+
}
|
|
1006
|
+
async function rows(c, source, http, deps, handoff) {
|
|
1007
|
+
const frame = executable(c, deps.pack);
|
|
1008
|
+
const key = Object.keys(frame)[0];
|
|
1009
|
+
const env = await deps.engine.query(frame, http);
|
|
1010
|
+
const present = env.data !== void 0 && key in env.data;
|
|
1011
|
+
return {
|
|
1012
|
+
kind: "rows",
|
|
1013
|
+
frame,
|
|
1014
|
+
answer: c.answer,
|
|
1015
|
+
data: present ? env.data?.[key] : void 0,
|
|
1016
|
+
...env.errors !== void 0 && env.errors.length > 0 ? { errors: env.errors } : {},
|
|
1017
|
+
source,
|
|
1018
|
+
...handoff !== void 0 ? { handoff } : {}
|
|
1019
|
+
};
|
|
1020
|
+
}
|
|
1021
|
+
async function recall(query, ns, http, deps) {
|
|
1022
|
+
const frame = { "recall/Hit": {
|
|
1023
|
+
params: {
|
|
1024
|
+
to: query,
|
|
1025
|
+
namespace: ns,
|
|
1026
|
+
k: RECALL_K
|
|
1027
|
+
},
|
|
1028
|
+
fields: {
|
|
1029
|
+
id: true,
|
|
1030
|
+
namespace: true,
|
|
1031
|
+
type: true,
|
|
1032
|
+
score: true,
|
|
1033
|
+
row: true
|
|
1034
|
+
}
|
|
1035
|
+
} };
|
|
1036
|
+
const env = await deps.engine.query(frame, http);
|
|
1037
|
+
const hits = env.data?.["recall/Hit"];
|
|
1038
|
+
return {
|
|
1039
|
+
kind: "recall",
|
|
1040
|
+
query,
|
|
1041
|
+
hits: Array.isArray(hits) ? hits : void 0,
|
|
1042
|
+
...env.errors !== void 0 && env.errors.length > 0 ? { errors: env.errors } : {}
|
|
1043
|
+
};
|
|
1044
|
+
}
|
|
1045
|
+
function clarify(question, reason, candidates, pack) {
|
|
1046
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1047
|
+
const options = [];
|
|
1048
|
+
for (const c of candidates) {
|
|
1049
|
+
if (seen.has(c.type)) continue;
|
|
1050
|
+
seen.add(c.type);
|
|
1051
|
+
const phrase = own(pack.types, c.type)?.phrases[0] ?? c.type;
|
|
1052
|
+
options.push({
|
|
1053
|
+
label: `${phrase} (${c.type})`,
|
|
1054
|
+
question,
|
|
1055
|
+
type: c.type
|
|
1056
|
+
});
|
|
1057
|
+
}
|
|
1058
|
+
return {
|
|
1059
|
+
kind: "clarify",
|
|
1060
|
+
question,
|
|
1061
|
+
reason,
|
|
1062
|
+
options
|
|
1063
|
+
};
|
|
1064
|
+
}
|
|
1065
|
+
const isAborted = (http) => http.signal?.aborted === true;
|
|
1066
|
+
async function* runAsk(req, http, deps) {
|
|
1067
|
+
const ctx = {
|
|
1068
|
+
asOf: (deps.now ?? (() => /* @__PURE__ */ new Date()))().toISOString(),
|
|
1069
|
+
rulings: deps.pack.rulings,
|
|
1070
|
+
...req.project !== void 0 ? { project: req.project } : {},
|
|
1071
|
+
...req.type !== void 0 ? { forceType: req.type } : {}
|
|
1072
|
+
};
|
|
1073
|
+
const first = parseQuestion(req.question, deps.pack, ctx);
|
|
1074
|
+
const best = first.candidates[0];
|
|
1075
|
+
yield {
|
|
1076
|
+
event: "parsed",
|
|
1077
|
+
status: first.status,
|
|
1078
|
+
...best !== void 0 ? { frame: executable(best, deps.pack) } : {},
|
|
1079
|
+
evidence: best?.evidence ?? first.evidence ?? [],
|
|
1080
|
+
residual: first.residual,
|
|
1081
|
+
...first.reason !== void 0 ? { reason: first.reason } : {}
|
|
1082
|
+
};
|
|
1083
|
+
if (isAborted(http)) return yield { event: "done" };
|
|
1084
|
+
let outcome;
|
|
1085
|
+
if (first.status === "resolved" && best !== void 0) outcome = await rows(best, "parser", http, deps);
|
|
1086
|
+
else if (first.status === "refuse") outcome = {
|
|
1087
|
+
kind: "refuse",
|
|
1088
|
+
reason: first.reason ?? "refused"
|
|
1089
|
+
};
|
|
1090
|
+
else if (first.status === "clarify") outcome = clarify(req.question, first.reason ?? "which did you mean?", first.candidates, deps.pack);
|
|
1091
|
+
else if (deps.judge === void 0) outcome = best !== void 0 ? await rows(best, "handoff", http, deps, "judge not configured") : clarify(req.question, "judge not configured", first.candidates, deps.pack);
|
|
1092
|
+
else {
|
|
1093
|
+
yield {
|
|
1094
|
+
event: "judging",
|
|
1095
|
+
reason: first.reason ?? first.status
|
|
1096
|
+
};
|
|
1097
|
+
const ac = new AbortController();
|
|
1098
|
+
const timer = setTimeout(() => ac.abort(/* @__PURE__ */ new Error("judge timeout")), deps.judgeTimeoutMs ?? 2e4);
|
|
1099
|
+
const onAbort = () => ac.abort(http.signal?.reason);
|
|
1100
|
+
http.signal?.addEventListener("abort", onAbort, { once: true });
|
|
1101
|
+
if (isAborted(http)) onAbort();
|
|
1102
|
+
let judged;
|
|
1103
|
+
try {
|
|
1104
|
+
const judging = parseWithJudge(req.question, deps.pack, ctx, deps.judge, {
|
|
1105
|
+
signal: ac.signal,
|
|
1106
|
+
exists: async (type, where) => {
|
|
1107
|
+
if (ac.signal.aborted) return false;
|
|
1108
|
+
const key = `${deps.pack.namespace}/${type}`;
|
|
1109
|
+
const r = (await deps.engine.query({ [key]: {
|
|
1110
|
+
params: {
|
|
1111
|
+
where,
|
|
1112
|
+
limit: 1
|
|
1113
|
+
},
|
|
1114
|
+
fields: { id: true }
|
|
1115
|
+
} }, http)).data?.[key];
|
|
1116
|
+
return Array.isArray(r) && r.length > 0;
|
|
1117
|
+
}
|
|
1118
|
+
});
|
|
1119
|
+
judging.catch(() => {});
|
|
1120
|
+
const gone = new Promise((_, reject) => {
|
|
1121
|
+
const fire = () => reject(ac.signal.reason);
|
|
1122
|
+
if (ac.signal.aborted) fire();
|
|
1123
|
+
else ac.signal.addEventListener("abort", fire, { once: true });
|
|
1124
|
+
});
|
|
1125
|
+
judged = await Promise.race([judging, gone]);
|
|
1126
|
+
} catch {
|
|
1127
|
+
judged = void 0;
|
|
1128
|
+
} finally {
|
|
1129
|
+
clearTimeout(timer);
|
|
1130
|
+
http.signal?.removeEventListener("abort", onAbort);
|
|
1131
|
+
}
|
|
1132
|
+
if (isAborted(http)) return yield { event: "done" };
|
|
1133
|
+
if (judged === void 0) outcome = best !== void 0 ? await rows(best, "handoff", http, deps, "judge unavailable") : clarify(req.question, "judge unavailable", first.candidates, deps.pack);
|
|
1134
|
+
else {
|
|
1135
|
+
yield {
|
|
1136
|
+
event: "judged",
|
|
1137
|
+
status: judged.status,
|
|
1138
|
+
trail: judged.trail
|
|
1139
|
+
};
|
|
1140
|
+
switch (judged.status) {
|
|
1141
|
+
case "resolved":
|
|
1142
|
+
outcome = await rows(judged.candidate, "judge", http, deps);
|
|
1143
|
+
break;
|
|
1144
|
+
case "recall":
|
|
1145
|
+
outcome = await recall(judged.query, deps.pack.namespace, http, deps);
|
|
1146
|
+
break;
|
|
1147
|
+
case "refuse":
|
|
1148
|
+
outcome = {
|
|
1149
|
+
kind: "refuse",
|
|
1150
|
+
reason: judged.reason
|
|
1151
|
+
};
|
|
1152
|
+
break;
|
|
1153
|
+
case "clarify":
|
|
1154
|
+
outcome = clarify(req.question, judged.reason, first.candidates, deps.pack);
|
|
1155
|
+
break;
|
|
1156
|
+
case "escalated": outcome = best !== void 0 ? await rows(best, "handoff", http, deps, "judge could not decide") : clarify(req.question, "judge could not decide", first.candidates, deps.pack);
|
|
1157
|
+
}
|
|
1158
|
+
}
|
|
1159
|
+
}
|
|
1160
|
+
yield {
|
|
1161
|
+
event: "outcome",
|
|
1162
|
+
outcome
|
|
1163
|
+
};
|
|
1164
|
+
yield { event: "done" };
|
|
1165
|
+
}
|
|
1166
|
+
|
|
1167
|
+
//#endregion
|
|
1168
|
+
//#region src/judge/ollama.ts
|
|
1169
|
+
/**
|
|
1170
|
+
* Ollama's native chat endpoint with `format` set to the question's schema,
|
|
1171
|
+
* so decoding cannot leave the offered options. Thinking off, temperature 0
|
|
1172
|
+
* and a fixed seed. Sends no `num_ctx` / `keep_alive`: Ollama shares one
|
|
1173
|
+
* runner per model, and differing options force a reload for every client
|
|
1174
|
+
* (see substrate-embedder's README).
|
|
1175
|
+
*/
|
|
1176
|
+
var OllamaJudge = class {
|
|
1177
|
+
#config;
|
|
1178
|
+
constructor(config) {
|
|
1179
|
+
this.#config = {
|
|
1180
|
+
...config,
|
|
1181
|
+
baseUrl: config.baseUrl.replace(/\/+$/, "")
|
|
1182
|
+
};
|
|
1183
|
+
}
|
|
1184
|
+
async decide(prompt, signal) {
|
|
1185
|
+
const timeout = AbortSignal.timeout(this.#config.timeoutMs ?? 6e4);
|
|
1186
|
+
const res = await fetch(`${this.#config.baseUrl}/api/chat`, {
|
|
1187
|
+
method: "POST",
|
|
1188
|
+
headers: { "content-type": "application/json" },
|
|
1189
|
+
signal: signal === void 0 ? timeout : AbortSignal.any([timeout, signal]),
|
|
1190
|
+
body: JSON.stringify({
|
|
1191
|
+
model: this.#config.model,
|
|
1192
|
+
stream: false,
|
|
1193
|
+
think: false,
|
|
1194
|
+
format: prompt.schema,
|
|
1195
|
+
options: {
|
|
1196
|
+
temperature: 0,
|
|
1197
|
+
seed: 0
|
|
1198
|
+
},
|
|
1199
|
+
messages: [{
|
|
1200
|
+
role: "system",
|
|
1201
|
+
content: prompt.system
|
|
1202
|
+
}, {
|
|
1203
|
+
role: "user",
|
|
1204
|
+
content: prompt.user
|
|
1205
|
+
}]
|
|
1206
|
+
})
|
|
1207
|
+
});
|
|
1208
|
+
if (!res.ok) throw new Error(`ollama ${res.status} ${res.statusText}`);
|
|
1209
|
+
const body = await res.json();
|
|
1210
|
+
return JSON.parse(body.message?.content ?? "");
|
|
1211
|
+
}
|
|
1212
|
+
};
|
|
1213
|
+
|
|
1214
|
+
//#endregion
|
|
1215
|
+
//#region src/rulings/schema.ts
|
|
1216
|
+
const ValueAliasSchema = z.object({
|
|
1217
|
+
type: z.string().min(1),
|
|
1218
|
+
field: z.string().min(1),
|
|
1219
|
+
in: z.array(z.string()).optional(),
|
|
1220
|
+
eq: z.string().optional(),
|
|
1221
|
+
startsWith: z.string().optional(),
|
|
1222
|
+
contains: z.string().optional()
|
|
1223
|
+
}).strict().refine((v) => [
|
|
1224
|
+
v.in,
|
|
1225
|
+
v.eq,
|
|
1226
|
+
v.startsWith,
|
|
1227
|
+
v.contains
|
|
1228
|
+
].filter((x) => x !== void 0).length === 1, { message: "a value alias names exactly one of in, eq, startsWith, contains" });
|
|
1229
|
+
const record = z.record(z.string(), z.string());
|
|
1230
|
+
const NamespaceRulingsSchema = z.object({
|
|
1231
|
+
unscopedTypes: z.array(z.string()).optional(),
|
|
1232
|
+
typeAliases: record.optional(),
|
|
1233
|
+
fieldAliases: record.optional(),
|
|
1234
|
+
liveStatus: record.optional(),
|
|
1235
|
+
timeField: record.optional(),
|
|
1236
|
+
typeNotes: record.optional(),
|
|
1237
|
+
valueAliases: z.record(z.string(), z.array(ValueAliasSchema)).optional()
|
|
1238
|
+
}).strict();
|
|
1239
|
+
/** Per top-level key: records merge entry-by-entry (override wins), arrays replace. */
|
|
1240
|
+
function mergeRulings(declared, override) {
|
|
1241
|
+
if (override === void 0) return declared;
|
|
1242
|
+
const out = { ...declared };
|
|
1243
|
+
for (const [key, value] of Object.entries(override)) {
|
|
1244
|
+
const prior = out[key];
|
|
1245
|
+
out[key] = Array.isArray(value) || prior === void 0 || Array.isArray(prior) ? value : {
|
|
1246
|
+
...prior,
|
|
1247
|
+
...value
|
|
1248
|
+
};
|
|
1249
|
+
}
|
|
1250
|
+
return out;
|
|
1251
|
+
}
|
|
1252
|
+
/** The effective rulings for one namespace's schema, plus anything that was dropped. */
|
|
1253
|
+
function readRulings(schema, override) {
|
|
1254
|
+
const warnings = [];
|
|
1255
|
+
let declared = {};
|
|
1256
|
+
const raw = schema.annotations?.nl;
|
|
1257
|
+
if (raw !== void 0) {
|
|
1258
|
+
const parsed = NamespaceRulingsSchema.safeParse(raw);
|
|
1259
|
+
if (parsed.success) declared = parsed.data;
|
|
1260
|
+
else {
|
|
1261
|
+
const where = parsed.error.issues.map((i) => i.path.join(".") || "(root)").join(", ");
|
|
1262
|
+
warnings.push(`declared rulings are invalid and were ignored (${where})`);
|
|
1263
|
+
}
|
|
1264
|
+
}
|
|
1265
|
+
return {
|
|
1266
|
+
rulings: mergeRulings(declared, override),
|
|
1267
|
+
warnings
|
|
1268
|
+
};
|
|
1269
|
+
}
|
|
1270
|
+
|
|
1271
|
+
//#endregion
|
|
1272
|
+
//#region src/pack/build.ts
|
|
1273
|
+
/** Longest value worth matching as a whole phrase. */
|
|
1274
|
+
const MAX_VALUE_LENGTH = 64;
|
|
1275
|
+
/**
|
|
1276
|
+
* A field is vocabulary when its values are labels rather than prose. The
|
|
1277
|
+
* test is length, not distinct count: an `area` with hundreds of one-word
|
|
1278
|
+
* values is vocabulary, a `title` with a dozen sentences is not.
|
|
1279
|
+
*/
|
|
1280
|
+
const MAX_MEAN_WORDS = 3;
|
|
1281
|
+
/** Server-managed fields every type can filter on, with their value type. */
|
|
1282
|
+
const METADATA_FIELDS = {
|
|
1283
|
+
createdAt: {
|
|
1284
|
+
valueType: "date",
|
|
1285
|
+
phrases: ["created", "created at"]
|
|
1286
|
+
},
|
|
1287
|
+
updatedAt: {
|
|
1288
|
+
valueType: "date",
|
|
1289
|
+
phrases: ["updated", "updated at"]
|
|
1290
|
+
},
|
|
1291
|
+
createdById: {
|
|
1292
|
+
valueType: "string",
|
|
1293
|
+
phrases: ["created by", "author"]
|
|
1294
|
+
},
|
|
1295
|
+
createdByKind: {
|
|
1296
|
+
valueType: "string",
|
|
1297
|
+
phrases: ["created by kind"]
|
|
1298
|
+
},
|
|
1299
|
+
updatedById: {
|
|
1300
|
+
valueType: "string",
|
|
1301
|
+
phrases: ["updated by"]
|
|
1302
|
+
},
|
|
1303
|
+
updatedByKind: {
|
|
1304
|
+
valueType: "string",
|
|
1305
|
+
phrases: ["updated by kind"]
|
|
1306
|
+
}
|
|
1307
|
+
};
|
|
1308
|
+
/**
|
|
1309
|
+
* Builds the NL vocabulary for one namespace. Pure: the same schema, operator
|
|
1310
|
+
* table and profile always give byte-identical output, with sorted keys.
|
|
1311
|
+
*/
|
|
1312
|
+
function buildNamespacePack(input) {
|
|
1313
|
+
const { namespace, schema, operators, profile } = input;
|
|
1314
|
+
const maxDistinct = input.maxDistinct ?? 1e3;
|
|
1315
|
+
const types = {};
|
|
1316
|
+
const refs = [];
|
|
1317
|
+
for (const typeName of Object.keys(schema.types).sort()) {
|
|
1318
|
+
const declared = schema.types[typeName]?.fields ?? {};
|
|
1319
|
+
const fields = {};
|
|
1320
|
+
for (const name of Object.keys(declared).sort()) {
|
|
1321
|
+
const def = declared[name];
|
|
1322
|
+
const field = packField(name, def, operators);
|
|
1323
|
+
fields[name] = field;
|
|
1324
|
+
for (const value of field.enum ?? []) refs.push({
|
|
1325
|
+
type: typeName,
|
|
1326
|
+
field: name,
|
|
1327
|
+
value
|
|
1328
|
+
});
|
|
1329
|
+
for (const alias of Object.keys(field.aliases ?? {})) refs.push({
|
|
1330
|
+
type: typeName,
|
|
1331
|
+
field: name,
|
|
1332
|
+
value: alias
|
|
1333
|
+
});
|
|
1334
|
+
const observed = profile?.types[typeName]?.[name];
|
|
1335
|
+
if (observed !== void 0 && field.indexed && isStringy(def) && !observed.truncated && observed.values.length <= maxDistinct && areLabels(observed.values)) {
|
|
1336
|
+
for (const value of observed.values) refs.push({
|
|
1337
|
+
type: typeName,
|
|
1338
|
+
field: name,
|
|
1339
|
+
value
|
|
1340
|
+
});
|
|
1341
|
+
field.distinct = observed.values.length;
|
|
1342
|
+
}
|
|
1343
|
+
}
|
|
1344
|
+
for (const [name, meta] of Object.entries(METADATA_FIELDS)) {
|
|
1345
|
+
if (name in fields) continue;
|
|
1346
|
+
fields[name] = {
|
|
1347
|
+
name,
|
|
1348
|
+
kind: "metadata",
|
|
1349
|
+
valueType: meta.valueType,
|
|
1350
|
+
indexed: true,
|
|
1351
|
+
operators: [...operators[`scalar.${meta.valueType}`] ?? []],
|
|
1352
|
+
phrases: meta.phrases
|
|
1353
|
+
};
|
|
1354
|
+
}
|
|
1355
|
+
types[typeName] = {
|
|
1356
|
+
name: typeName,
|
|
1357
|
+
phrases: typePhrases(typeName),
|
|
1358
|
+
fields: sortKeys(fields)
|
|
1359
|
+
};
|
|
1360
|
+
}
|
|
1361
|
+
return {
|
|
1362
|
+
namespace,
|
|
1363
|
+
schemaVersion: schema.schemaVersion,
|
|
1364
|
+
types,
|
|
1365
|
+
values: indexValues(refs)
|
|
1366
|
+
};
|
|
1367
|
+
}
|
|
1368
|
+
function packField(name, def, operators) {
|
|
1369
|
+
const phrases = uniqueSorted([splitWords(name).join(" "), name.toLowerCase()]);
|
|
1370
|
+
switch (def.kind) {
|
|
1371
|
+
case "scalar": return {
|
|
1372
|
+
name,
|
|
1373
|
+
kind: "scalar",
|
|
1374
|
+
valueType: def.type,
|
|
1375
|
+
indexed: def.indexed === true,
|
|
1376
|
+
operators: [...operators[`scalar.${def.type}`] ?? []],
|
|
1377
|
+
phrases,
|
|
1378
|
+
...def.enum !== void 0 ? { enum: [...def.enum] } : {},
|
|
1379
|
+
...def.aliases !== void 0 ? { aliases: { ...def.aliases } } : {}
|
|
1380
|
+
};
|
|
1381
|
+
case "crossRef": return {
|
|
1382
|
+
name,
|
|
1383
|
+
kind: "crossRef",
|
|
1384
|
+
target: def.type,
|
|
1385
|
+
indexed: def.indexed === true,
|
|
1386
|
+
operators: [...operators.crossRef ?? []],
|
|
1387
|
+
phrases
|
|
1388
|
+
};
|
|
1389
|
+
case "list": return {
|
|
1390
|
+
name,
|
|
1391
|
+
kind: "list",
|
|
1392
|
+
indexed: def.indexed === true,
|
|
1393
|
+
operators: [...operators.list ?? []],
|
|
1394
|
+
phrases
|
|
1395
|
+
};
|
|
1396
|
+
default: return {
|
|
1397
|
+
name,
|
|
1398
|
+
kind: "struct",
|
|
1399
|
+
indexed: false,
|
|
1400
|
+
operators: [],
|
|
1401
|
+
phrases
|
|
1402
|
+
};
|
|
1403
|
+
}
|
|
1404
|
+
}
|
|
1405
|
+
function areLabels(values) {
|
|
1406
|
+
if (values.length === 0) return false;
|
|
1407
|
+
return values.reduce((n, v) => n + v.trim().split(/\s+/).length, 0) / values.length <= MAX_MEAN_WORDS;
|
|
1408
|
+
}
|
|
1409
|
+
/** A string scalar, or a list of string scalars: the fields whose values read as words. */
|
|
1410
|
+
function isStringy(def) {
|
|
1411
|
+
if (def.kind === "scalar") return def.type === "string";
|
|
1412
|
+
return def.kind === "list" && def.of.kind === "scalar" && def.of.type === "string";
|
|
1413
|
+
}
|
|
1414
|
+
function typePhrases(typeName) {
|
|
1415
|
+
const words = splitWords(typeName);
|
|
1416
|
+
const last = words[words.length - 1] ?? typeName.toLowerCase();
|
|
1417
|
+
const head = words.slice(0, -1);
|
|
1418
|
+
const out = [[...head, last].join(" "), [...head, pluralize(last)].join(" ")];
|
|
1419
|
+
if (head.length > 0) out.push(last, pluralize(last));
|
|
1420
|
+
return uniqueSorted(out);
|
|
1421
|
+
}
|
|
1422
|
+
function indexValues(refs) {
|
|
1423
|
+
const byKey = /* @__PURE__ */ new Map();
|
|
1424
|
+
for (const ref of refs) {
|
|
1425
|
+
const key = ref.value.trim().toLowerCase();
|
|
1426
|
+
if (key.length === 0 || key.length > MAX_VALUE_LENGTH) continue;
|
|
1427
|
+
const bucket = byKey.get(key) ?? /* @__PURE__ */ new Map();
|
|
1428
|
+
bucket.set(`${ref.type}\u0000${ref.field}`, ref);
|
|
1429
|
+
byKey.set(key, bucket);
|
|
1430
|
+
}
|
|
1431
|
+
const out = {};
|
|
1432
|
+
for (const key of [...byKey.keys()].sort()) {
|
|
1433
|
+
const bucket = byKey.get(key);
|
|
1434
|
+
out[key] = [...bucket.keys()].sort().map((k) => bucket.get(k));
|
|
1435
|
+
}
|
|
1436
|
+
return out;
|
|
1437
|
+
}
|
|
1438
|
+
function sortKeys(obj) {
|
|
1439
|
+
return Object.fromEntries(Object.keys(obj).sort().map((k) => [k, obj[k]]));
|
|
1440
|
+
}
|
|
1441
|
+
|
|
1442
|
+
//#endregion
|
|
1443
|
+
//#region src/pack/profile.ts
|
|
1444
|
+
/**
|
|
1445
|
+
* Distinct values of every indexed string field (and indexed list-of-string
|
|
1446
|
+
* field) in the namespace, reading up to `maxRows` rows per type. A field that
|
|
1447
|
+
* passes `maxDistinct` is marked truncated and stops growing.
|
|
1448
|
+
*/
|
|
1449
|
+
async function profileNamespace(input) {
|
|
1450
|
+
const maxDistinct = input.maxDistinct ?? 1e3;
|
|
1451
|
+
const maxRows = input.maxRows ?? 1e4;
|
|
1452
|
+
const types = {};
|
|
1453
|
+
for (const typeName of Object.keys(input.schema.types).sort()) {
|
|
1454
|
+
const declared = input.schema.types[typeName]?.fields ?? {};
|
|
1455
|
+
const fields = Object.keys(declared).filter((f) => profiled(declared[f]));
|
|
1456
|
+
if (fields.length === 0) continue;
|
|
1457
|
+
const seen = new Map(fields.map((f) => [f, /* @__PURE__ */ new Set()]));
|
|
1458
|
+
const truncated = /* @__PURE__ */ new Set();
|
|
1459
|
+
let cursor;
|
|
1460
|
+
let rowsRead = 0;
|
|
1461
|
+
do {
|
|
1462
|
+
const page = await input.read(typeName, fields, cursor);
|
|
1463
|
+
rowsRead += page.rows.length;
|
|
1464
|
+
for (const row of page.rows) for (const f of fields) {
|
|
1465
|
+
if (truncated.has(f)) continue;
|
|
1466
|
+
const raw = row[f];
|
|
1467
|
+
const values = Array.isArray(raw) ? raw : [raw];
|
|
1468
|
+
const set = seen.get(f);
|
|
1469
|
+
for (const v of values) {
|
|
1470
|
+
if (typeof v !== "string" || v.length === 0) continue;
|
|
1471
|
+
set.add(v);
|
|
1472
|
+
if (set.size > maxDistinct) {
|
|
1473
|
+
truncated.add(f);
|
|
1474
|
+
break;
|
|
1475
|
+
}
|
|
1476
|
+
}
|
|
1477
|
+
}
|
|
1478
|
+
cursor = page.nextCursor;
|
|
1479
|
+
} while (cursor !== void 0 && truncated.size < fields.length && rowsRead < maxRows);
|
|
1480
|
+
const out = {};
|
|
1481
|
+
for (const f of [...fields].sort()) {
|
|
1482
|
+
const values = [...seen.get(f)].sort();
|
|
1483
|
+
const isTruncated = truncated.has(f);
|
|
1484
|
+
if (values.length === 0 && !isTruncated) continue;
|
|
1485
|
+
out[f] = {
|
|
1486
|
+
values: isTruncated ? values.slice(0, maxDistinct) : values,
|
|
1487
|
+
truncated: isTruncated
|
|
1488
|
+
};
|
|
1489
|
+
}
|
|
1490
|
+
types[typeName] = out;
|
|
1491
|
+
}
|
|
1492
|
+
return { types };
|
|
1493
|
+
}
|
|
1494
|
+
function profiled(def) {
|
|
1495
|
+
if (def.kind === "scalar") return def.type === "string" && def.indexed === true;
|
|
1496
|
+
return def.kind === "list" && def.indexed === true && def.of.kind === "scalar" && def.of.type === "string";
|
|
1497
|
+
}
|
|
1498
|
+
|
|
1499
|
+
//#endregion
|
|
1500
|
+
//#region src/pack/service.ts
|
|
1501
|
+
/**
|
|
1502
|
+
* Packs are built under the CALLER's request: the profile reads distinct
|
|
1503
|
+
* values through `engine.query(frame, req)`, so row policy applies and a
|
|
1504
|
+
* pack can never disclose a value its caller could not read. Hence the
|
|
1505
|
+
* principal is part of the cache key.
|
|
1506
|
+
*/
|
|
1507
|
+
var PackService = class {
|
|
1508
|
+
#opts;
|
|
1509
|
+
#cache = /* @__PURE__ */ new Map();
|
|
1510
|
+
#inflight = /* @__PURE__ */ new Map();
|
|
1511
|
+
#warned = /* @__PURE__ */ new Set();
|
|
1512
|
+
constructor(opts) {
|
|
1513
|
+
this.#opts = opts;
|
|
1514
|
+
}
|
|
1515
|
+
async get(ns, req) {
|
|
1516
|
+
const schemas = this.#opts.engine.federatedSchema();
|
|
1517
|
+
const schema = Object.hasOwn(schemas, ns) ? schemas[ns] : void 0;
|
|
1518
|
+
if (schema === void 0) return void 0;
|
|
1519
|
+
const now = (this.#opts.now ?? Date.now)();
|
|
1520
|
+
const ttl = this.#opts.ttlMs ?? 3e5;
|
|
1521
|
+
const key = `${ns}\u0000${schema.schemaVersion}\u0000${principalKey(req)}`;
|
|
1522
|
+
const hit = this.#cache.get(key);
|
|
1523
|
+
if (hit !== void 0 && now - hit.at < ttl) {
|
|
1524
|
+
this.#cache.delete(key);
|
|
1525
|
+
this.#cache.set(key, hit);
|
|
1526
|
+
return {
|
|
1527
|
+
pack: hit.pack,
|
|
1528
|
+
etag: hit.etag
|
|
1529
|
+
};
|
|
1530
|
+
}
|
|
1531
|
+
let pending = this.#inflight.get(key);
|
|
1532
|
+
if (pending === void 0) {
|
|
1533
|
+
pending = this.#build(ns, schema, req, now).finally(() => this.#inflight.delete(key));
|
|
1534
|
+
this.#inflight.set(key, pending);
|
|
1535
|
+
}
|
|
1536
|
+
const entry = await pending;
|
|
1537
|
+
this.#cache.delete(key);
|
|
1538
|
+
this.#cache.set(key, entry);
|
|
1539
|
+
const max = this.#opts.maxEntries ?? 256;
|
|
1540
|
+
while (this.#cache.size > max) {
|
|
1541
|
+
const oldest = this.#cache.keys().next().value;
|
|
1542
|
+
if (oldest === void 0) break;
|
|
1543
|
+
this.#cache.delete(oldest);
|
|
1544
|
+
}
|
|
1545
|
+
return {
|
|
1546
|
+
pack: entry.pack,
|
|
1547
|
+
etag: entry.etag
|
|
1548
|
+
};
|
|
1549
|
+
}
|
|
1550
|
+
async #build(ns, schema, req, now) {
|
|
1551
|
+
const { rulings, warnings } = readRulings(schema, own(this.#opts.rulings, ns));
|
|
1552
|
+
const warnKey = `${ns}\u0000${schema.schemaVersion}`;
|
|
1553
|
+
if (warnings.length > 0 && !this.#warned.has(warnKey)) {
|
|
1554
|
+
this.#warned.add(warnKey);
|
|
1555
|
+
(this.#opts.log ?? console.warn)(`substrate-nl: ${ns}: ${warnings.join("; ")}`);
|
|
1556
|
+
}
|
|
1557
|
+
const read = async (type, fields, cursor) => {
|
|
1558
|
+
const frameKey = `${ns}/${type}`;
|
|
1559
|
+
const env = await this.#opts.engine.query({ [frameKey]: {
|
|
1560
|
+
params: cursor !== void 0 ? { cursor } : {},
|
|
1561
|
+
fields: Object.fromEntries(fields.map((f) => [f, true]))
|
|
1562
|
+
} }, req);
|
|
1563
|
+
const rows = env.data?.[frameKey];
|
|
1564
|
+
const page = env._pagination?.[frameKey];
|
|
1565
|
+
return {
|
|
1566
|
+
rows: Array.isArray(rows) ? rows : [],
|
|
1567
|
+
nextCursor: page?.hasMore === true ? page.nextCursor : void 0
|
|
1568
|
+
};
|
|
1569
|
+
};
|
|
1570
|
+
const profile = await profileNamespace({
|
|
1571
|
+
schema,
|
|
1572
|
+
read
|
|
1573
|
+
});
|
|
1574
|
+
const pack = {
|
|
1575
|
+
...buildNamespacePack({
|
|
1576
|
+
namespace: ns,
|
|
1577
|
+
schema,
|
|
1578
|
+
operators: this.#opts.engine.operators(ns) ?? {},
|
|
1579
|
+
profile
|
|
1580
|
+
}),
|
|
1581
|
+
rulings,
|
|
1582
|
+
warnings
|
|
1583
|
+
};
|
|
1584
|
+
return {
|
|
1585
|
+
pack,
|
|
1586
|
+
etag: `"${createHash("sha256").update(JSON.stringify(pack)).digest("hex").slice(0, 16)}"`,
|
|
1587
|
+
at: now
|
|
1588
|
+
};
|
|
1589
|
+
}
|
|
1590
|
+
};
|
|
1591
|
+
/**
|
|
1592
|
+
* The Identity canonicalised, less `expiresAt`: a custom row policy may key on
|
|
1593
|
+
* any claim (a tenant, say), so no other subset is safe to share a cached pack
|
|
1594
|
+
* across, while a token's expiry changes with every refresh.
|
|
1595
|
+
*/
|
|
1596
|
+
function principalKey(req) {
|
|
1597
|
+
const id = req.identity;
|
|
1598
|
+
if (id === void 0) return "\0anon";
|
|
1599
|
+
const { expiresAt: _expiresAt, ...stable } = id;
|
|
1600
|
+
return JSON.stringify(canonical(stable));
|
|
1601
|
+
}
|
|
1602
|
+
function canonical(v) {
|
|
1603
|
+
if (Array.isArray(v)) return v.map(canonical);
|
|
1604
|
+
if (v !== null && typeof v === "object") return Object.fromEntries(Object.entries(v).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0).map(([k, x]) => [k, canonical(x)]));
|
|
1605
|
+
return v;
|
|
1606
|
+
}
|
|
1607
|
+
|
|
1608
|
+
//#endregion
|
|
1609
|
+
//#region src/ask/encode.ts
|
|
1610
|
+
/** Same headers as substrate-compose's SSE routes (not exported from its entry). */
|
|
1611
|
+
const SSE_HEADERS = {
|
|
1612
|
+
"content-type": "text/event-stream",
|
|
1613
|
+
"cache-control": "no-store",
|
|
1614
|
+
"x-accel-buffering": "no"
|
|
1615
|
+
};
|
|
1616
|
+
/** One named SSE event. JSON.stringify emits no raw newline, so one data line is safe. */
|
|
1617
|
+
function sseEvent(name, data) {
|
|
1618
|
+
return `event: ${name}\ndata: ${JSON.stringify(data)}\n\n`;
|
|
1619
|
+
}
|
|
1620
|
+
function wireName(ev) {
|
|
1621
|
+
if (ev.event === "outcome") return [ev.outcome.kind, ev.outcome];
|
|
1622
|
+
const { event, ...rest } = ev;
|
|
1623
|
+
return [event, rest];
|
|
1624
|
+
}
|
|
1625
|
+
function toSseStream(events, onCancel, onError) {
|
|
1626
|
+
const enc = new TextEncoder();
|
|
1627
|
+
return new ReadableStream({
|
|
1628
|
+
async pull(controller) {
|
|
1629
|
+
try {
|
|
1630
|
+
const next = await events.next();
|
|
1631
|
+
if (next.done === true) return controller.close();
|
|
1632
|
+
const [name, data] = wireName(next.value);
|
|
1633
|
+
controller.enqueue(enc.encode(sseEvent(name, data)));
|
|
1634
|
+
} catch (err) {
|
|
1635
|
+
onError?.(err);
|
|
1636
|
+
controller.enqueue(enc.encode(sseEvent("error", {
|
|
1637
|
+
code: "nl_engine_error",
|
|
1638
|
+
message: "NL query failed"
|
|
1639
|
+
})));
|
|
1640
|
+
controller.close();
|
|
1641
|
+
}
|
|
1642
|
+
},
|
|
1643
|
+
async cancel() {
|
|
1644
|
+
onCancel?.();
|
|
1645
|
+
await events.return(void 0);
|
|
1646
|
+
}
|
|
1647
|
+
});
|
|
1648
|
+
}
|
|
1649
|
+
async function toJson(events) {
|
|
1650
|
+
const trail = [];
|
|
1651
|
+
let outcome = null;
|
|
1652
|
+
for await (const ev of events) {
|
|
1653
|
+
trail.push(ev);
|
|
1654
|
+
if (ev.event === "outcome") outcome = ev.outcome;
|
|
1655
|
+
}
|
|
1656
|
+
return {
|
|
1657
|
+
outcome,
|
|
1658
|
+
trail
|
|
1659
|
+
};
|
|
1660
|
+
}
|
|
1661
|
+
|
|
1662
|
+
//#endregion
|
|
1663
|
+
//#region src/routes.ts
|
|
1664
|
+
const VARY = { vary: "authorization" };
|
|
1665
|
+
const engineFailure = (req, err, log) => {
|
|
1666
|
+
if (req.signal?.aborted === true) return { status: 499 };
|
|
1667
|
+
log(err);
|
|
1668
|
+
return jsonResponse(502, {
|
|
1669
|
+
error: "NL query failed",
|
|
1670
|
+
code: "nl_engine_error"
|
|
1671
|
+
});
|
|
1672
|
+
};
|
|
1673
|
+
const str = (v) => typeof v === "string" && v.length > 0;
|
|
1674
|
+
/** Longest question parsed; the same bound recall puts on its query text. */
|
|
1675
|
+
const MAX_QUESTION_CHARS = 8192;
|
|
1676
|
+
function askHandler(deps) {
|
|
1677
|
+
const log = deps.log ?? ((err) => console.error("substrate-nl:", err));
|
|
1678
|
+
return async (req) => {
|
|
1679
|
+
const parsed = parseJsonBody(req);
|
|
1680
|
+
if (!parsed.ok) return parsed.response;
|
|
1681
|
+
const body = parsed.value;
|
|
1682
|
+
if (body === null || typeof body !== "object" || Array.isArray(body)) return error400("body must be a JSON object: { namespace, question, project?, type? }");
|
|
1683
|
+
if (!str(body.namespace)) return error400("body.namespace must be a non-empty string");
|
|
1684
|
+
if (!str(body.question)) return error400("body.question must be a non-empty string");
|
|
1685
|
+
if (body.question.length > MAX_QUESTION_CHARS) return error400(`body.question is longer than ${MAX_QUESTION_CHARS} characters`);
|
|
1686
|
+
for (const k of ["project", "type"]) if (body[k] !== void 0 && !str(body[k])) return error400(`body.${k} must be a non-empty string when given`);
|
|
1687
|
+
const ask = {
|
|
1688
|
+
namespace: body.namespace,
|
|
1689
|
+
question: body.question,
|
|
1690
|
+
...str(body.project) ? { project: body.project } : {},
|
|
1691
|
+
...str(body.type) ? { type: body.type } : {}
|
|
1692
|
+
};
|
|
1693
|
+
let served;
|
|
1694
|
+
try {
|
|
1695
|
+
served = await deps.packs.get(ask.namespace, req);
|
|
1696
|
+
} catch (err) {
|
|
1697
|
+
return engineFailure(req, err, log);
|
|
1698
|
+
}
|
|
1699
|
+
if (served === void 0) return jsonResponse(404, {
|
|
1700
|
+
error: `unknown namespace '${ask.namespace}'`,
|
|
1701
|
+
code: "nl_unknown_namespace"
|
|
1702
|
+
});
|
|
1703
|
+
if (ask.type !== void 0 && !Object.hasOwn(served.pack.types, ask.type)) return error400(`unknown type '${ask.type}'`);
|
|
1704
|
+
const events = runAsk(ask, req, {
|
|
1705
|
+
pack: served.pack,
|
|
1706
|
+
engine: deps.engine,
|
|
1707
|
+
...deps.judge !== void 0 ? { judge: deps.judge } : {},
|
|
1708
|
+
...deps.judgeTimeoutMs !== void 0 ? { judgeTimeoutMs: deps.judgeTimeoutMs } : {},
|
|
1709
|
+
...deps.now !== void 0 ? { now: deps.now } : {}
|
|
1710
|
+
});
|
|
1711
|
+
if ((req.headers.accept ?? "").toLowerCase().includes("text/event-stream")) return {
|
|
1712
|
+
status: 200,
|
|
1713
|
+
headers: SSE_HEADERS,
|
|
1714
|
+
body: toSseStream(events, void 0, log)
|
|
1715
|
+
};
|
|
1716
|
+
try {
|
|
1717
|
+
return jsonResponse(200, await toJson(events));
|
|
1718
|
+
} catch (err) {
|
|
1719
|
+
return engineFailure(req, err, log);
|
|
1720
|
+
}
|
|
1721
|
+
};
|
|
1722
|
+
}
|
|
1723
|
+
function packHandler(deps) {
|
|
1724
|
+
const log = deps.log ?? ((err) => console.error("substrate-nl:", err));
|
|
1725
|
+
return async (req) => {
|
|
1726
|
+
const ns = req.params.ns ?? "";
|
|
1727
|
+
let served;
|
|
1728
|
+
try {
|
|
1729
|
+
served = await deps.packs.get(ns, req);
|
|
1730
|
+
} catch (err) {
|
|
1731
|
+
return engineFailure(req, err, log);
|
|
1732
|
+
}
|
|
1733
|
+
if (served === void 0) return jsonResponse(404, {
|
|
1734
|
+
error: `unknown namespace '${ns}'`,
|
|
1735
|
+
code: "nl_unknown_namespace"
|
|
1736
|
+
});
|
|
1737
|
+
if (req.headers["if-none-match"] === served.etag) return {
|
|
1738
|
+
status: 304,
|
|
1739
|
+
headers: {
|
|
1740
|
+
etag: served.etag,
|
|
1741
|
+
...VARY
|
|
1742
|
+
}
|
|
1743
|
+
};
|
|
1744
|
+
return jsonResponse(200, served.pack, {
|
|
1745
|
+
etag: served.etag,
|
|
1746
|
+
"cache-control": "private, no-cache",
|
|
1747
|
+
...VARY
|
|
1748
|
+
});
|
|
1749
|
+
};
|
|
1750
|
+
}
|
|
1751
|
+
|
|
1752
|
+
//#endregion
|
|
1753
|
+
//#region src/plugin.ts
|
|
1754
|
+
/** The routes rely on `KernelRouteClaim.capability`, which every 1.x host honours. */
|
|
1755
|
+
const CONTRACT_VERSION = ">=1.0.0-alpha.0 <2.0.0";
|
|
1756
|
+
function createNlPlugin(cfg) {
|
|
1757
|
+
const packs = new PackService({
|
|
1758
|
+
engine: cfg.engine,
|
|
1759
|
+
...cfg.rulings !== void 0 ? { rulings: cfg.rulings } : {},
|
|
1760
|
+
...cfg.packTtlMs !== void 0 ? { ttlMs: cfg.packTtlMs } : {}
|
|
1761
|
+
});
|
|
1762
|
+
const deps = {
|
|
1763
|
+
engine: cfg.engine,
|
|
1764
|
+
packs,
|
|
1765
|
+
...cfg.judge !== void 0 ? { judge: cfg.judge } : {},
|
|
1766
|
+
...cfg.judgeTimeoutMs !== void 0 ? { judgeTimeoutMs: cfg.judgeTimeoutMs } : {}
|
|
1767
|
+
};
|
|
1768
|
+
return {
|
|
1769
|
+
name: "substrate-nl",
|
|
1770
|
+
contractVersion: CONTRACT_VERSION,
|
|
1771
|
+
kernelRoutes: [{
|
|
1772
|
+
path: "nl",
|
|
1773
|
+
method: "POST",
|
|
1774
|
+
handler: askHandler(deps),
|
|
1775
|
+
capability: "read"
|
|
1776
|
+
}, {
|
|
1777
|
+
path: "nl/pack/:ns",
|
|
1778
|
+
method: "GET",
|
|
1779
|
+
handler: packHandler(deps),
|
|
1780
|
+
capability: "read"
|
|
1781
|
+
}]
|
|
1782
|
+
};
|
|
1783
|
+
}
|
|
1784
|
+
|
|
1785
|
+
//#endregion
|
|
1786
|
+
//#region src/bootstrap.ts
|
|
1787
|
+
/**
|
|
1788
|
+
* Structured-config bootstrap for `@mnmal-ai/substrate-nl`.
|
|
1789
|
+
*
|
|
1790
|
+
* Operator-overridable (`<pluginDir>/substrate-nl.config.json`):
|
|
1791
|
+
* - `judge` optional; Ollama for the constrained judge. Absent = parser-only:
|
|
1792
|
+
* every escalation is an honest hand-off.
|
|
1793
|
+
* - `rulings` optional; per-namespace override merged over declared rulings.
|
|
1794
|
+
* - `packTtlMs` how long a served pack is reused. Default 300000.
|
|
1795
|
+
* Host-supplied (`hostDeps.nlEngine`): schema, operators and query access.
|
|
1796
|
+
*/
|
|
1797
|
+
const JudgeSchema = z.object({
|
|
1798
|
+
baseUrl: z.url(),
|
|
1799
|
+
model: z.string().min(1),
|
|
1800
|
+
timeoutMs: z.number().int().positive().default(2e4)
|
|
1801
|
+
}).strict();
|
|
1802
|
+
const NlBootstrapConfigSchema = z.object({
|
|
1803
|
+
judge: JudgeSchema.optional(),
|
|
1804
|
+
rulings: z.record(z.string(), NamespaceRulingsSchema).optional(),
|
|
1805
|
+
packTtlMs: z.number().int().positive().default(3e5)
|
|
1806
|
+
}).strict();
|
|
1807
|
+
const bootstrap = defineBootstrap.kernel({
|
|
1808
|
+
schema: NlBootstrapConfigSchema,
|
|
1809
|
+
build: (config, deps) => createNlPlugin({
|
|
1810
|
+
engine: deps.nlEngine,
|
|
1811
|
+
packTtlMs: config.packTtlMs,
|
|
1812
|
+
...config.judge !== void 0 ? {
|
|
1813
|
+
judge: new OllamaJudge(config.judge),
|
|
1814
|
+
judgeTimeoutMs: config.judge.timeoutMs
|
|
1815
|
+
} : {},
|
|
1816
|
+
...config.rulings !== void 0 ? { rulings: config.rulings } : {}
|
|
1817
|
+
})
|
|
1818
|
+
});
|
|
1819
|
+
|
|
1820
|
+
//#endregion
|
|
1821
|
+
export { askHandler as a, profileNamespace as c, mergeRulings as d, readRulings as f, parseQuestion as g, parseWithJudge as h, createNlPlugin as i, buildNamespacePack as l, runAsk as m, bootstrap as n, packHandler as o, OllamaJudge as p, CONTRACT_VERSION as r, PackService as s, NlBootstrapConfigSchema as t, NamespaceRulingsSchema as u };
|