@mnmal-ai/substrate-nl 1.0.0-alpha.1 → 1.0.0-alpha.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/access.d.ts +5 -1
- package/dist/access.d.ts.map +1 -1
- package/dist/ask/run.d.ts +8 -1
- package/dist/ask/run.d.ts.map +1 -1
- package/dist/{bootstrap-BpKUloR3.js → bootstrap-CLYwjrrl.js} +1225 -108
- package/dist/bootstrap.js +1 -1
- package/dist/index.js +1 -1
- package/dist/judge/judge.d.ts +1 -0
- package/dist/judge/judge.d.ts.map +1 -1
- package/dist/judge/ollama.d.ts.map +1 -1
- package/dist/parse/compare.d.ts +26 -0
- package/dist/parse/compare.d.ts.map +1 -0
- package/dist/parse/parse.d.ts +10 -1
- package/dist/parse/parse.d.ts.map +1 -1
- package/dist/parse/walks.d.ts +120 -0
- package/dist/parse/walks.d.ts.map +1 -0
- package/dist/rulings/schema.d.ts +3 -1
- package/dist/rulings/schema.d.ts.map +1 -1
- package/package.json +6 -6
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { defineBootstrap, error400, jsonResponse, parseJsonBody } from "@mnmal-ai/substrate/sdk";
|
|
1
|
+
import { METADATA_AUTHOR_FIELDS, METADATA_FILTER_FIELDS, ROW_POLICY_BLOCKS_WALK, WALKS_NEED_PARTITIONED, WALKS_UNSUPPORTED_STORE, defineBootstrap, error400, jsonResponse, parseJsonBody, trimTrailingSlashes } from "@mnmal-ai/substrate/sdk";
|
|
2
2
|
import { z } from "zod";
|
|
3
3
|
import { createHash } from "node:crypto";
|
|
4
4
|
|
|
@@ -28,6 +28,168 @@ function uniqueSorted(xs) {
|
|
|
28
28
|
return [...new Set(xs)].sort();
|
|
29
29
|
}
|
|
30
30
|
|
|
31
|
+
//#endregion
|
|
32
|
+
//#region src/rulings/schema.ts
|
|
33
|
+
/**
|
|
34
|
+
* Rulings arrive from two places: declared by the namespace owner in
|
|
35
|
+
* `defineNamespace(..., { annotations: { nl } })`, and overridden by the
|
|
36
|
+
* operator in `substrate-nl.config.json`. Both are validated here. Bad declared
|
|
37
|
+
* rulings are dropped with a warning rather than failing the boot: one
|
|
38
|
+
* namespace's typo must not take NL down for every namespace, and must not
|
|
39
|
+
* pass silently either.
|
|
40
|
+
*/
|
|
41
|
+
const ValueAliasSchema = z.object({
|
|
42
|
+
type: z.string().min(1),
|
|
43
|
+
field: z.string().min(1),
|
|
44
|
+
in: z.array(z.string()).optional(),
|
|
45
|
+
eq: z.string().optional(),
|
|
46
|
+
startsWith: z.string().optional(),
|
|
47
|
+
contains: z.string().optional()
|
|
48
|
+
}).strict().refine((v) => [
|
|
49
|
+
v.in,
|
|
50
|
+
v.eq,
|
|
51
|
+
v.startsWith,
|
|
52
|
+
v.contains
|
|
53
|
+
].filter((x) => x !== void 0).length === 1, { message: "a value alias names exactly one of in, eq, startsWith, contains" });
|
|
54
|
+
const norm = (p) => p.trim().replace(/\s+/g, " ").toLowerCase();
|
|
55
|
+
const FieldPhraseSchema = z.object({
|
|
56
|
+
field: z.string().min(1).optional(),
|
|
57
|
+
value: z.unknown().optional(),
|
|
58
|
+
exclude: z.record(z.string(), z.unknown()).optional(),
|
|
59
|
+
derive: z.union([z.object({ diff: z.tuple([z.string(), z.string()]) }).strict(), z.object({ ratio: z.tuple([z.string(), z.string()]) }).strict()]).optional(),
|
|
60
|
+
pick: z.enum([
|
|
61
|
+
"latest",
|
|
62
|
+
"max",
|
|
63
|
+
"min"
|
|
64
|
+
]).optional()
|
|
65
|
+
}).strict().refine((v) => v.field !== void 0 || v.value === void 0 && v.exclude === void 0, { message: "value and exclude require field" }).refine((v) => v.field !== void 0 || v.derive !== void 0 || v.pick !== void 0, { message: "an entry needs at least one of field, derive, pick" });
|
|
66
|
+
const FieldPhrasesSchema = z.record(z.string(), FieldPhraseSchema).transform((rec, ctx) => {
|
|
67
|
+
const out = {};
|
|
68
|
+
for (const [phrase, entry] of Object.entries(rec)) {
|
|
69
|
+
const key = norm(phrase);
|
|
70
|
+
if (Object.hasOwn(out, key)) {
|
|
71
|
+
ctx.addIssue({
|
|
72
|
+
code: "custom",
|
|
73
|
+
message: `two phrases normalise to '${key}'`
|
|
74
|
+
});
|
|
75
|
+
return z.NEVER;
|
|
76
|
+
}
|
|
77
|
+
out[key] = entry;
|
|
78
|
+
}
|
|
79
|
+
return out;
|
|
80
|
+
});
|
|
81
|
+
const record = z.record(z.string(), z.string());
|
|
82
|
+
const NamespaceRulingsSchema = z.object({
|
|
83
|
+
fieldPhrases: FieldPhrasesSchema.optional(),
|
|
84
|
+
unscopedTypes: z.array(z.string()).optional(),
|
|
85
|
+
typeAliases: record.optional(),
|
|
86
|
+
fieldAliases: record.optional(),
|
|
87
|
+
liveStatus: record.optional(),
|
|
88
|
+
timeField: record.optional(),
|
|
89
|
+
typeNotes: record.optional(),
|
|
90
|
+
valueAliases: z.record(z.string(), z.array(ValueAliasSchema)).optional(),
|
|
91
|
+
typeExcludes: z.record(z.string(), z.record(z.string(), z.unknown())).optional()
|
|
92
|
+
}).strict();
|
|
93
|
+
/** Per top-level key: records merge entry-by-entry (override wins), arrays replace. */
|
|
94
|
+
function mergeRulings(declared, override) {
|
|
95
|
+
if (override === void 0) return declared;
|
|
96
|
+
const out = { ...declared };
|
|
97
|
+
for (const [key, value] of Object.entries(override)) {
|
|
98
|
+
const prior = out[key];
|
|
99
|
+
out[key] = Array.isArray(value) || prior === void 0 || Array.isArray(prior) ? value : {
|
|
100
|
+
...prior,
|
|
101
|
+
...value
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
return out;
|
|
105
|
+
}
|
|
106
|
+
const COMBINATORS = [
|
|
107
|
+
"AND",
|
|
108
|
+
"OR",
|
|
109
|
+
"NOT"
|
|
110
|
+
];
|
|
111
|
+
/** Field names an `exclude` clause names, walked through AND/OR/NOT. Structural only: operator spellings are the engine's to refuse at query time. */
|
|
112
|
+
function whereFields(w, out) {
|
|
113
|
+
if (Array.isArray(w)) for (const x of w) whereFields(x, out);
|
|
114
|
+
else if (typeof w === "object" && w !== null) for (const [k, v] of Object.entries(w)) if (COMBINATORS.includes(k)) whereFields(v, out);
|
|
115
|
+
else out.push(k);
|
|
116
|
+
}
|
|
117
|
+
/** Why a `fieldPhrases` entry does not fit the schema, or undefined. */
|
|
118
|
+
function phraseProblem(schema, e) {
|
|
119
|
+
const types = /* @__PURE__ */ new Set();
|
|
120
|
+
const refs = [...e.field !== void 0 ? [e.field] : [], ...e.derive ? Object.values(e.derive)[0] ?? [] : []];
|
|
121
|
+
for (const r of refs) {
|
|
122
|
+
const dot = r.indexOf(".");
|
|
123
|
+
const type = dot < 0 ? r : r.slice(0, dot);
|
|
124
|
+
const field = dot < 0 ? "" : r.slice(dot + 1);
|
|
125
|
+
const t = Object.hasOwn(schema.types, type) ? schema.types[type] : void 0;
|
|
126
|
+
if (t === void 0) return `unknown type '${type}'`;
|
|
127
|
+
if (!Object.hasOwn(t.fields, field)) return `unknown field '${type}.${field}'`;
|
|
128
|
+
types.add(type);
|
|
129
|
+
}
|
|
130
|
+
if (types.size > 1) return `refers to more than one type (${[...types].join(", ")})`;
|
|
131
|
+
const [type] = types;
|
|
132
|
+
if (e.exclude !== void 0 && type !== void 0) return excludeProblem(schema, type, e.exclude);
|
|
133
|
+
}
|
|
134
|
+
/** Why an exclude clause does not fit `type`, or undefined. */
|
|
135
|
+
function excludeProblem(schema, type, exclude) {
|
|
136
|
+
const names = [];
|
|
137
|
+
whereFields(exclude, names);
|
|
138
|
+
const fields = schema.types[type]?.fields ?? {};
|
|
139
|
+
const bad = names.find((n) => !n.startsWith("_") && !Object.hasOwn(fields, n) && !Object.hasOwn(METADATA_FILTER_FIELDS, n) && !Object.hasOwn(METADATA_AUTHOR_FIELDS, n));
|
|
140
|
+
return bad !== void 0 ? `exclude names unknown field '${type}.${bad}'` : void 0;
|
|
141
|
+
}
|
|
142
|
+
/** The effective rulings for one namespace's schema, plus anything that was dropped. */
|
|
143
|
+
function readRulings(schema, override) {
|
|
144
|
+
const warnings = [];
|
|
145
|
+
let declared = {};
|
|
146
|
+
const raw = schema.annotations?.nl;
|
|
147
|
+
if (raw !== void 0) {
|
|
148
|
+
const parsed = NamespaceRulingsSchema.safeParse(raw);
|
|
149
|
+
if (parsed.success) declared = parsed.data;
|
|
150
|
+
else {
|
|
151
|
+
const where = parsed.error.issues.map((i) => i.path.join(".") || "(root)").join(", ");
|
|
152
|
+
warnings.push(`declared rulings are invalid and were ignored (${where})`);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
let normOverride = override;
|
|
156
|
+
if (override?.fieldPhrases !== void 0) {
|
|
157
|
+
const p = FieldPhrasesSchema.safeParse(override.fieldPhrases);
|
|
158
|
+
const { fieldPhrases: _drop, ...rest } = override;
|
|
159
|
+
if (p.success) normOverride = {
|
|
160
|
+
...rest,
|
|
161
|
+
fieldPhrases: p.data
|
|
162
|
+
};
|
|
163
|
+
else {
|
|
164
|
+
normOverride = rest;
|
|
165
|
+
warnings.push(`override fieldPhrases are invalid and were ignored (${p.error.issues.map((i) => i.message).join("; ")})`);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
const rulings = mergeRulings(declared, normOverride);
|
|
169
|
+
if (rulings.fieldPhrases !== void 0) {
|
|
170
|
+
const kept = {};
|
|
171
|
+
for (const [phrase, entry] of Object.entries(rulings.fieldPhrases)) {
|
|
172
|
+
const problem = phraseProblem(schema, entry);
|
|
173
|
+
if (problem === void 0) kept[phrase] = entry;
|
|
174
|
+
else warnings.push(`fieldPhrases '${phrase}' was ignored: ${problem}`);
|
|
175
|
+
}
|
|
176
|
+
rulings.fieldPhrases = kept;
|
|
177
|
+
}
|
|
178
|
+
if (rulings.typeExcludes !== void 0) {
|
|
179
|
+
const kept = {};
|
|
180
|
+
for (const [type, where] of Object.entries(rulings.typeExcludes)) {
|
|
181
|
+
const problem = Object.hasOwn(schema.types, type) ? excludeProblem(schema, type, where) : `unknown type '${type}'`;
|
|
182
|
+
if (problem === void 0) kept[type] = where;
|
|
183
|
+
else warnings.push(`typeExcludes '${type}' was ignored: ${problem}`);
|
|
184
|
+
}
|
|
185
|
+
rulings.typeExcludes = kept;
|
|
186
|
+
}
|
|
187
|
+
return {
|
|
188
|
+
rulings,
|
|
189
|
+
warnings
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
|
|
31
193
|
//#endregion
|
|
32
194
|
//#region src/parse/time.ts
|
|
33
195
|
const WEEKDAYS = [
|
|
@@ -109,10 +271,732 @@ function rangeFor(phrase, asOf) {
|
|
|
109
271
|
return null;
|
|
110
272
|
}
|
|
111
273
|
|
|
274
|
+
//#endregion
|
|
275
|
+
//#region src/parse/compare.ts
|
|
276
|
+
const NUMBER = /^\$?(\d+(?:\.\d+)?)$/;
|
|
277
|
+
/** The comparison phrase starting at `words[0]`, or null. */
|
|
278
|
+
function parseComparison(words) {
|
|
279
|
+
const [a = "", b = "", c = ""] = words;
|
|
280
|
+
const pct = /^(\d+(?:\.\d+)?)%$/.exec(a);
|
|
281
|
+
if (pct !== null) {
|
|
282
|
+
const n = Number(pct[1]);
|
|
283
|
+
const up = (100 + n) / 100;
|
|
284
|
+
if (b === "over" || b === "above") return {
|
|
285
|
+
op: "gte",
|
|
286
|
+
times: up,
|
|
287
|
+
length: 2
|
|
288
|
+
};
|
|
289
|
+
if (b === "more" && c === "than") return {
|
|
290
|
+
op: "gte",
|
|
291
|
+
times: up,
|
|
292
|
+
length: 3
|
|
293
|
+
};
|
|
294
|
+
const length = c === "than" ? 3 : 2;
|
|
295
|
+
if (b === "slower") return {
|
|
296
|
+
op: "gte",
|
|
297
|
+
times: up,
|
|
298
|
+
length
|
|
299
|
+
};
|
|
300
|
+
if (b === "faster") return n < 100 ? {
|
|
301
|
+
op: "lte",
|
|
302
|
+
times: (100 - n) / 100,
|
|
303
|
+
length
|
|
304
|
+
} : {
|
|
305
|
+
op: "lte",
|
|
306
|
+
length,
|
|
307
|
+
invalid: `${n}% faster leaves no time to compare against`
|
|
308
|
+
};
|
|
309
|
+
return null;
|
|
310
|
+
}
|
|
311
|
+
const lead = a === "at" && b === "least" ? 2 : 0;
|
|
312
|
+
const w = words[lead] ?? "";
|
|
313
|
+
const mult = /^(\d+(?:\.\d+)?)(?:x|×)$/.exec(w);
|
|
314
|
+
if (mult !== null) return {
|
|
315
|
+
op: "gte",
|
|
316
|
+
times: Number(mult[1]),
|
|
317
|
+
length: lead + 1
|
|
318
|
+
};
|
|
319
|
+
const next = words[lead + 1];
|
|
320
|
+
if (/^\d+(?:\.\d+)?$/.test(w) && (next === "times" || next === "x" || next === "×")) return {
|
|
321
|
+
op: "gte",
|
|
322
|
+
times: Number(w),
|
|
323
|
+
length: lead + 2
|
|
324
|
+
};
|
|
325
|
+
if (lead > 0) return null;
|
|
326
|
+
const amount = /^\$(\d+(?:\.\d+)?)$/.exec(a);
|
|
327
|
+
if (amount !== null) {
|
|
328
|
+
const plus = Number(amount[1]);
|
|
329
|
+
if (b === "over") return {
|
|
330
|
+
op: "gte",
|
|
331
|
+
plus,
|
|
332
|
+
length: 2
|
|
333
|
+
};
|
|
334
|
+
if (b === "more" && c === "than") return {
|
|
335
|
+
op: "gte",
|
|
336
|
+
plus,
|
|
337
|
+
length: 3
|
|
338
|
+
};
|
|
339
|
+
return null;
|
|
340
|
+
}
|
|
341
|
+
const head = a === "over" ? ["gt", 1] : a === "under" ? ["lt", 1] : a === "more" && b === "than" ? ["gt", 2] : a === "less" && b === "than" ? ["lt", 2] : null;
|
|
342
|
+
if (head === null) return null;
|
|
343
|
+
const [op, length] = head;
|
|
344
|
+
const lit = NUMBER.exec(words[length] ?? "");
|
|
345
|
+
return lit !== null ? {
|
|
346
|
+
op,
|
|
347
|
+
value: Number(lit[1]),
|
|
348
|
+
length: length + 1
|
|
349
|
+
} : {
|
|
350
|
+
op,
|
|
351
|
+
length
|
|
352
|
+
};
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
//#endregion
|
|
356
|
+
//#region src/parse/walks.ts
|
|
357
|
+
/** The plan note for a type's default exclusion that yielded to the question's own filter on `field`. */
|
|
358
|
+
function defaultYield(type, field, path = ["where"]) {
|
|
359
|
+
return {
|
|
360
|
+
path: [...path, `ruling:${type} default exclude (yields to the question's ${field} filter)`],
|
|
361
|
+
kind: "filter",
|
|
362
|
+
served: false
|
|
363
|
+
};
|
|
364
|
+
}
|
|
365
|
+
/** Every crossRef hop out of `type`, both directions, inside the pack's namespace. */
|
|
366
|
+
function hopsFrom(pack, type) {
|
|
367
|
+
const out = [];
|
|
368
|
+
for (const [owner, t] of Object.entries(pack.types)) for (const [field, f] of Object.entries(t.fields)) {
|
|
369
|
+
const target = f.kind === "crossRef" ? f.target : void 0;
|
|
370
|
+
if (target === void 0 || target.namespace !== pack.namespace) continue;
|
|
371
|
+
if (own(pack.types, target.type) === void 0) continue;
|
|
372
|
+
if (owner === type) out.push({
|
|
373
|
+
dir: "out",
|
|
374
|
+
from: type,
|
|
375
|
+
owner,
|
|
376
|
+
field,
|
|
377
|
+
to: target.type
|
|
378
|
+
});
|
|
379
|
+
if (target.type === type) out.push({
|
|
380
|
+
dir: "in",
|
|
381
|
+
from: type,
|
|
382
|
+
owner,
|
|
383
|
+
field,
|
|
384
|
+
to: owner
|
|
385
|
+
});
|
|
386
|
+
}
|
|
387
|
+
return out;
|
|
388
|
+
}
|
|
389
|
+
/** All shortest crossRef paths from `from` to `to` (BFS, both directions, no type revisited). */
|
|
390
|
+
function findPaths(pack, from, to, maxHops = 3) {
|
|
391
|
+
if (from === to) return [[]];
|
|
392
|
+
let frontier = [[]];
|
|
393
|
+
for (let depth = 1; depth <= maxHops; depth++) {
|
|
394
|
+
const next = [];
|
|
395
|
+
for (const p of frontier) {
|
|
396
|
+
const at = p[p.length - 1]?.to ?? from;
|
|
397
|
+
for (const h of hopsFrom(pack, at)) {
|
|
398
|
+
if (h.to === from || p.some((x) => x.to === h.to)) continue;
|
|
399
|
+
next.push([...p, h]);
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
const done = next.filter((p) => p[p.length - 1]?.to === to);
|
|
403
|
+
if (done.length > 0) return done;
|
|
404
|
+
frontier = next;
|
|
405
|
+
}
|
|
406
|
+
return [];
|
|
407
|
+
}
|
|
408
|
+
function pathLabel(ns, p) {
|
|
409
|
+
return p.map((h) => h.dir === "in" ? `in ${ns}/${h.owner}.${h.field}` : `out ${h.field}`).join(" > ");
|
|
410
|
+
}
|
|
411
|
+
const PICKED = {
|
|
412
|
+
latest: "latest",
|
|
413
|
+
max: "highest",
|
|
414
|
+
min: "lowest"
|
|
415
|
+
};
|
|
416
|
+
const PICK_WORDS = {
|
|
417
|
+
latest: "latest",
|
|
418
|
+
current: "latest",
|
|
419
|
+
"most recent": "latest",
|
|
420
|
+
highest: "max",
|
|
421
|
+
best: "max",
|
|
422
|
+
lowest: "min",
|
|
423
|
+
cheapest: "min"
|
|
424
|
+
};
|
|
425
|
+
const DERIVE_WORDS = /* @__PURE__ */ new Set([
|
|
426
|
+
"margin",
|
|
427
|
+
"spread",
|
|
428
|
+
"gap"
|
|
429
|
+
]);
|
|
430
|
+
/** A comparative the lexicon does not cover. */
|
|
431
|
+
const COMPARATIVE = /%$|^\d+(?:\.\d+)?(?:x|×)$|^(?:than|twice|double|half|triple)$/;
|
|
432
|
+
const text = (tokens, [s, e]) => tokens.slice(s, e + 1).map((t) => t.text).join(" ");
|
|
433
|
+
const range = ([s, e]) => Array.from({ length: e - s + 1 }, (_, k) => s + k);
|
|
434
|
+
const take = (tokens, [s, e]) => {
|
|
435
|
+
for (let i = s; i <= e; i++) tokens[i].used = true;
|
|
436
|
+
};
|
|
437
|
+
/**
|
|
438
|
+
* Claims field phrases, pick words, "by <derive>" and comparisons. Only a
|
|
439
|
+
* namespace that declares `fieldPhrases` composes walks, so nothing changes
|
|
440
|
+
* for one that does not.
|
|
441
|
+
*/
|
|
442
|
+
function claimWalkParts(tokens, rulings, glue) {
|
|
443
|
+
const claims = {
|
|
444
|
+
phrases: [],
|
|
445
|
+
compares: [],
|
|
446
|
+
picks: [],
|
|
447
|
+
derives: [],
|
|
448
|
+
dropped: []
|
|
449
|
+
};
|
|
450
|
+
const table = rulings.fieldPhrases;
|
|
451
|
+
if (table === void 0 || Object.keys(table).length === 0) return claims;
|
|
452
|
+
for (let n = 6; n >= 1; n--) for (let i = 0; i + n <= tokens.length; i++) {
|
|
453
|
+
const at = [i, i + n - 1];
|
|
454
|
+
if (tokens.slice(i, i + n).some((t) => t.used)) continue;
|
|
455
|
+
const entry = own(table, text(tokens, at));
|
|
456
|
+
if (entry === void 0) continue;
|
|
457
|
+
take(tokens, at);
|
|
458
|
+
const dot = entry.field?.indexOf(".") ?? -1;
|
|
459
|
+
claims.phrases.push({
|
|
460
|
+
span: text(tokens, at),
|
|
461
|
+
at,
|
|
462
|
+
entry,
|
|
463
|
+
...entry.field !== void 0 && dot > 0 ? {
|
|
464
|
+
type: entry.field.slice(0, dot),
|
|
465
|
+
field: entry.field.slice(dot + 1)
|
|
466
|
+
} : {}
|
|
467
|
+
});
|
|
468
|
+
}
|
|
469
|
+
claims.phrases.sort((x, y) => x.at[0] - y.at[0]);
|
|
470
|
+
for (const p of claims.phrases) {
|
|
471
|
+
const s = p.at[0];
|
|
472
|
+
for (const [word, kind] of Object.entries(PICK_WORDS)) {
|
|
473
|
+
const at = [s - word.split(" ").length, s - 1];
|
|
474
|
+
if (at[0] < 0 || tokens.slice(at[0], s).some((t) => t.used)) continue;
|
|
475
|
+
if (text(tokens, at) !== word) continue;
|
|
476
|
+
take(tokens, at);
|
|
477
|
+
claims.picks.push({
|
|
478
|
+
span: word,
|
|
479
|
+
at,
|
|
480
|
+
phrase: p,
|
|
481
|
+
kind
|
|
482
|
+
});
|
|
483
|
+
break;
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
for (const p of claims.phrases) {
|
|
487
|
+
if (p.entry.derive === void 0) continue;
|
|
488
|
+
const by = tokens[p.at[0] - 1];
|
|
489
|
+
if (by?.text === "by" && !by.used) {
|
|
490
|
+
by.used = true;
|
|
491
|
+
claims.derives.push(p);
|
|
492
|
+
} else claims.dropped.push({
|
|
493
|
+
span: p.span,
|
|
494
|
+
reason: `"${p.span}" is a derived value; it orders a list only as "by ${p.span}"`,
|
|
495
|
+
kind: "filter"
|
|
496
|
+
});
|
|
497
|
+
}
|
|
498
|
+
tokens.forEach((t, i) => {
|
|
499
|
+
const next = tokens[i + 1];
|
|
500
|
+
if (t.text !== "by" || t.used || next === void 0 || next.used) return;
|
|
501
|
+
if (!DERIVE_WORDS.has(next.text)) return;
|
|
502
|
+
t.used = true;
|
|
503
|
+
next.used = true;
|
|
504
|
+
claims.dropped.push({
|
|
505
|
+
span: `by ${next.text}`,
|
|
506
|
+
reason: `no derived value "${next.text}" is declared for this namespace`,
|
|
507
|
+
kind: "order"
|
|
508
|
+
});
|
|
509
|
+
});
|
|
510
|
+
const phraseAt = (i) => claims.phrases.find((p) => p.at[0] === i);
|
|
511
|
+
for (let i = 0; i < tokens.length; i++) {
|
|
512
|
+
if (tokens[i]?.used === true) continue;
|
|
513
|
+
const cmp = parseComparison(tokens.slice(i).map((t) => t.text));
|
|
514
|
+
if (cmp === null) continue;
|
|
515
|
+
const at = [i, i + cmp.length - 1];
|
|
516
|
+
const usedAt = range(at).filter((k) => tokens[k]?.used === true);
|
|
517
|
+
const wordPhrase = usedAt.length === 1 ? phraseAt(usedAt[0]) : void 0;
|
|
518
|
+
if (usedAt.length > 0 && (wordPhrase === void 0 || wordPhrase.at[1] !== usedAt[0] || !/^(?:slower|faster)$/.test(wordPhrase.span))) continue;
|
|
519
|
+
let object;
|
|
520
|
+
if (cmp.value === void 0) {
|
|
521
|
+
const pickAt = new Set(claims.picks.flatMap((k) => range(k.at)));
|
|
522
|
+
let j = at[1] + 1;
|
|
523
|
+
while (j < tokens.length && phraseAt(j) === void 0) {
|
|
524
|
+
const t = tokens[j];
|
|
525
|
+
if (!(glue.has(t.text) && !t.used) && !pickAt.has(j)) break;
|
|
526
|
+
j++;
|
|
527
|
+
}
|
|
528
|
+
object = phraseAt(j);
|
|
529
|
+
if (object === void 0) continue;
|
|
530
|
+
}
|
|
531
|
+
const before = [...claims.phrases].reverse().find((p) => p.at[1] < i);
|
|
532
|
+
const adjacent = before !== void 0 && tokens.slice(before.at[1] + 1, i).every((t) => !t.used && glue.has(t.text)) && before.field !== void 0 && before.entry.value === void 0;
|
|
533
|
+
const subject = wordPhrase ?? (adjacent ? before : void 0);
|
|
534
|
+
for (let k = i; k <= at[1]; k++) tokens[k].used = true;
|
|
535
|
+
const span = text(tokens, at);
|
|
536
|
+
claims.compares.push({
|
|
537
|
+
span,
|
|
538
|
+
at,
|
|
539
|
+
cmp,
|
|
540
|
+
...subject !== void 0 ? { subject } : {},
|
|
541
|
+
...object !== void 0 ? { object } : {}
|
|
542
|
+
});
|
|
543
|
+
i = at[1];
|
|
544
|
+
}
|
|
545
|
+
return claims;
|
|
546
|
+
}
|
|
547
|
+
/** Comparatives the lexicon did not claim: dropped, never approximated. */
|
|
548
|
+
function dropComparatives(tokens, dropped) {
|
|
549
|
+
tokens.forEach((t, i) => {
|
|
550
|
+
if (t.used || !COMPARATIVE.test(t.text)) return;
|
|
551
|
+
const prev = tokens[i - 1];
|
|
552
|
+
const words = prev !== void 0 && !prev.used && t.text === "than" ? [prev, t] : [t];
|
|
553
|
+
for (const w of words) w.used = true;
|
|
554
|
+
dropped.push({
|
|
555
|
+
span: words.map((w) => w.text).join(" "),
|
|
556
|
+
reason: "this comparison is not in the comparison lexicon",
|
|
557
|
+
kind: "filter"
|
|
558
|
+
});
|
|
559
|
+
});
|
|
560
|
+
}
|
|
561
|
+
const filter = (clause, path, label) => ({
|
|
562
|
+
clause,
|
|
563
|
+
plan: [{
|
|
564
|
+
path: [...path, label],
|
|
565
|
+
kind: "filter",
|
|
566
|
+
served: false
|
|
567
|
+
}]
|
|
568
|
+
});
|
|
569
|
+
/** Clauses as one where: the clause itself, or an AND with plan paths re-rooted. */
|
|
570
|
+
function andOf(parts) {
|
|
571
|
+
if (parts.length === 0) return null;
|
|
572
|
+
if (parts.length === 1) return {
|
|
573
|
+
where: parts[0].clause,
|
|
574
|
+
plan: parts[0].plan
|
|
575
|
+
};
|
|
576
|
+
return {
|
|
577
|
+
where: { AND: parts.map((p) => p.clause) },
|
|
578
|
+
plan: parts.flatMap((p, i) => p.plan.map((c) => ({
|
|
579
|
+
...c,
|
|
580
|
+
path: [
|
|
581
|
+
"AND",
|
|
582
|
+
String(i),
|
|
583
|
+
...c.path
|
|
584
|
+
]
|
|
585
|
+
})))
|
|
586
|
+
};
|
|
587
|
+
}
|
|
588
|
+
/** Binding names: the type name's capitals ("Listing" -> L, "CiRun" -> CR). */
|
|
589
|
+
function aliasFor(type, taken) {
|
|
590
|
+
const base = type.replace(/[^A-Z]/g, "") || type.slice(0, 1).toUpperCase();
|
|
591
|
+
let name = base;
|
|
592
|
+
for (let n = 2; taken.has(name); n++) name = `${base}${n}`;
|
|
593
|
+
taken.add(name);
|
|
594
|
+
return name;
|
|
595
|
+
}
|
|
596
|
+
function compose(input) {
|
|
597
|
+
const { pack, rulings, seed, claims } = input;
|
|
598
|
+
const ns = pack.namespace;
|
|
599
|
+
const dropped = [...claims.dropped];
|
|
600
|
+
const seedParts = [];
|
|
601
|
+
const walks = /* @__PURE__ */ new Map();
|
|
602
|
+
const dead = /* @__PURE__ */ new Set();
|
|
603
|
+
let clarify = null;
|
|
604
|
+
const timeField = (type) => own(rulings.timeField, type) ?? "createdAt";
|
|
605
|
+
const typeExclude = (type) => {
|
|
606
|
+
const where = own(rulings.typeExcludes, type);
|
|
607
|
+
return where === void 0 ? null : filter({ NOT: where }, ["NOT"], `ruling:${type} default exclude`);
|
|
608
|
+
};
|
|
609
|
+
const walkTo = (type, span) => {
|
|
610
|
+
const known = walks.get(type);
|
|
611
|
+
if (known !== void 0) return known;
|
|
612
|
+
if (dead.has(type)) return null;
|
|
613
|
+
const paths = findPaths(pack, seed, type);
|
|
614
|
+
if (paths.length === 0) {
|
|
615
|
+
dead.add(type);
|
|
616
|
+
dropped.push({
|
|
617
|
+
span,
|
|
618
|
+
reason: `no crossRef path from ${seed} to ${type} within 3 hops`,
|
|
619
|
+
kind: "filter"
|
|
620
|
+
});
|
|
621
|
+
return null;
|
|
622
|
+
}
|
|
623
|
+
if (paths.length > 1) {
|
|
624
|
+
clarify ??= `several equally short paths from ${seed} to ${type}: ${paths.map((p) => pathLabel(ns, p)).join("; ")}`;
|
|
625
|
+
dead.add(type);
|
|
626
|
+
return null;
|
|
627
|
+
}
|
|
628
|
+
const w = {
|
|
629
|
+
type,
|
|
630
|
+
path: paths[0],
|
|
631
|
+
filters: [],
|
|
632
|
+
window: [],
|
|
633
|
+
excludes: [typeExclude(type)].filter((x) => x !== null),
|
|
634
|
+
notes: [],
|
|
635
|
+
applied: /* @__PURE__ */ new Set()
|
|
636
|
+
};
|
|
637
|
+
walks.set(type, w);
|
|
638
|
+
return w;
|
|
639
|
+
};
|
|
640
|
+
/** Where a phrase's own clauses go: the seed, a walk, or nowhere (dropped). */
|
|
641
|
+
const home = (p) => {
|
|
642
|
+
if (p.type === void 0) return null;
|
|
643
|
+
if (p.type === seed) return { seed: true };
|
|
644
|
+
return walkTo(p.type, p.span);
|
|
645
|
+
};
|
|
646
|
+
const isWalk = (h) => !("seed" in h);
|
|
647
|
+
for (const p of claims.phrases) {
|
|
648
|
+
if (p.field === void 0 || p.type === void 0) continue;
|
|
649
|
+
const h = home(p);
|
|
650
|
+
if (h === null) continue;
|
|
651
|
+
if (isWalk(h) && h.applied.has(p.span)) continue;
|
|
652
|
+
const parts = [];
|
|
653
|
+
if (p.entry.value !== void 0) parts.push(filter({ [p.field]: { eq: p.entry.value } }, [p.field], `ruling:${p.span}`));
|
|
654
|
+
const excl = p.entry.exclude;
|
|
655
|
+
if (excl !== void 0) {
|
|
656
|
+
const part = filter({ NOT: excl }, ["NOT"], `ruling:${p.span}`);
|
|
657
|
+
if (isWalk(h)) h.excludes.push(part);
|
|
658
|
+
else seedParts.push(part);
|
|
659
|
+
}
|
|
660
|
+
if (isWalk(h)) {
|
|
661
|
+
h.filters.push(...parts);
|
|
662
|
+
h.applied.add(p.span);
|
|
663
|
+
} else seedParts.push(...parts);
|
|
664
|
+
}
|
|
665
|
+
const picked = /* @__PURE__ */ new Map();
|
|
666
|
+
for (const k of claims.picks) picked.set(k.phrase, k.kind);
|
|
667
|
+
for (const p of claims.phrases) {
|
|
668
|
+
if (p.type === void 0 || p.type === seed || p.field === void 0) continue;
|
|
669
|
+
const w = walks.get(p.type);
|
|
670
|
+
if (w === void 0) continue;
|
|
671
|
+
const kind = picked.get(p) ?? p.entry.pick;
|
|
672
|
+
if (kind === void 0) continue;
|
|
673
|
+
const source = picked.has(p) ? `"${claims.picks.find((k) => k.phrase === p)?.span}"` : "ruling";
|
|
674
|
+
const span = claims.picks.find((k) => k.phrase === p)?.span ?? p.span;
|
|
675
|
+
const pick = {
|
|
676
|
+
kind,
|
|
677
|
+
by: kind === "latest" ? timeField(p.type) : p.field,
|
|
678
|
+
source,
|
|
679
|
+
span
|
|
680
|
+
};
|
|
681
|
+
if (w.pick !== void 0 && (w.pick.kind !== pick.kind || w.pick.by !== pick.by)) {
|
|
682
|
+
clarify ??= `${w.pick.source} and ${pick.source} pick different ${p.type} rows`;
|
|
683
|
+
continue;
|
|
684
|
+
}
|
|
685
|
+
w.pick = pick;
|
|
686
|
+
}
|
|
687
|
+
let seedLatest;
|
|
688
|
+
for (const k of claims.picks) if (k.phrase.type === seed && k.kind === "latest") seedLatest = k.span;
|
|
689
|
+
else if (k.phrase.type === seed) dropped.push({
|
|
690
|
+
span: `${k.span} ${k.phrase.span}`,
|
|
691
|
+
reason: `"${k.span}" picks one neighbour, and ${seed} is the type being asked about`,
|
|
692
|
+
kind: "order"
|
|
693
|
+
});
|
|
694
|
+
const mentionOf = /* @__PURE__ */ new Map();
|
|
695
|
+
const mention = (w, p, text) => {
|
|
696
|
+
const known = mentionOf.get(w);
|
|
697
|
+
if (known === void 0) mentionOf.set(w, {
|
|
698
|
+
at: p.at[0],
|
|
699
|
+
text
|
|
700
|
+
});
|
|
701
|
+
else if (known.at !== p.at[0]) clarify ??= `"${known.text}" and "${text}": one ${w.type} meeting both, or two different ${w.type} rows?`;
|
|
702
|
+
};
|
|
703
|
+
for (const c of claims.compares) {
|
|
704
|
+
if (c.cmp.invalid !== void 0) {
|
|
705
|
+
dropped.push({
|
|
706
|
+
span: c.span,
|
|
707
|
+
reason: c.cmp.invalid,
|
|
708
|
+
kind: "filter"
|
|
709
|
+
});
|
|
710
|
+
continue;
|
|
711
|
+
}
|
|
712
|
+
const s = c.subject;
|
|
713
|
+
if (s?.type === void 0 || s.field === void 0) {
|
|
714
|
+
dropped.push({
|
|
715
|
+
span: c.span,
|
|
716
|
+
reason: "no field phrase directly before it to compare",
|
|
717
|
+
kind: "filter"
|
|
718
|
+
});
|
|
719
|
+
continue;
|
|
720
|
+
}
|
|
721
|
+
const sh = home(s);
|
|
722
|
+
if (sh === null) {
|
|
723
|
+
dropped.push({
|
|
724
|
+
span: c.span,
|
|
725
|
+
reason: `its field "${s.span}" was dropped`,
|
|
726
|
+
kind: "filter"
|
|
727
|
+
});
|
|
728
|
+
continue;
|
|
729
|
+
}
|
|
730
|
+
let operand;
|
|
731
|
+
let kind = "filter";
|
|
732
|
+
if (c.cmp.value !== void 0) operand = c.cmp.value;
|
|
733
|
+
else {
|
|
734
|
+
const o = c.object;
|
|
735
|
+
if (o?.type === void 0 || o.field === void 0) {
|
|
736
|
+
dropped.push({
|
|
737
|
+
span: c.span,
|
|
738
|
+
reason: "nothing to compare against",
|
|
739
|
+
kind: "filter"
|
|
740
|
+
});
|
|
741
|
+
continue;
|
|
742
|
+
}
|
|
743
|
+
let ref;
|
|
744
|
+
if (o.type === seed) ref = `$.${o.field}`;
|
|
745
|
+
else {
|
|
746
|
+
if (o.type === s.type) {
|
|
747
|
+
dropped.push({
|
|
748
|
+
span: `${c.span} ${o.span}`,
|
|
749
|
+
reason: `comparing two fields of one ${o.type} is not expressible`,
|
|
750
|
+
kind: "filter"
|
|
751
|
+
});
|
|
752
|
+
continue;
|
|
753
|
+
}
|
|
754
|
+
const ow = walkTo(o.type, o.span);
|
|
755
|
+
if (ow === null) {
|
|
756
|
+
dropped.push({
|
|
757
|
+
span: c.span,
|
|
758
|
+
reason: `its operand "${o.span}" was dropped`,
|
|
759
|
+
kind: "filter"
|
|
760
|
+
});
|
|
761
|
+
continue;
|
|
762
|
+
}
|
|
763
|
+
mention(ow, o, `${c.span} ${o.span}`);
|
|
764
|
+
if (ow.path.length > 1 || ow.pick !== void 0) {
|
|
765
|
+
dropped.push({
|
|
766
|
+
span: `${c.span} ${o.span}`,
|
|
767
|
+
reason: ow.pick !== void 0 ? `a picked ${o.type} cannot be referenced` : `a comparison against a ${ow.path.length}-hop walk is not expressible`,
|
|
768
|
+
kind: "filter"
|
|
769
|
+
});
|
|
770
|
+
continue;
|
|
771
|
+
}
|
|
772
|
+
ref = `${o.type}\u0000${o.field}`;
|
|
773
|
+
}
|
|
774
|
+
operand = {
|
|
775
|
+
$ref: ref,
|
|
776
|
+
...c.cmp.times !== void 0 ? { times: c.cmp.times } : {},
|
|
777
|
+
...c.cmp.plus !== void 0 ? { plus: c.cmp.plus } : {}
|
|
778
|
+
};
|
|
779
|
+
kind = "ref";
|
|
780
|
+
}
|
|
781
|
+
const part = {
|
|
782
|
+
clause: { [s.field]: { [c.cmp.op]: operand } },
|
|
783
|
+
plan: [{
|
|
784
|
+
path: [
|
|
785
|
+
s.field,
|
|
786
|
+
c.cmp.op,
|
|
787
|
+
...kind === "ref" ? [] : [`compare:${c.span}`]
|
|
788
|
+
],
|
|
789
|
+
kind,
|
|
790
|
+
served: kind === "ref"
|
|
791
|
+
}]
|
|
792
|
+
};
|
|
793
|
+
if (isWalk(sh)) {
|
|
794
|
+
mention(sh, s, `${s.span} ${c.span}`);
|
|
795
|
+
sh.filters.push(part);
|
|
796
|
+
} else seedParts.push(part);
|
|
797
|
+
}
|
|
798
|
+
let timeOnSeed = input.time !== null;
|
|
799
|
+
if (input.time !== null) {
|
|
800
|
+
const t = input.time.at;
|
|
801
|
+
const dist = ([s, e]) => s > t[1] ? s - t[1] : e < t[0] ? t[0] - e : 0;
|
|
802
|
+
const near = /* @__PURE__ */ new Map();
|
|
803
|
+
const note = (type, at) => {
|
|
804
|
+
if (type === void 0) return;
|
|
805
|
+
near.set(type, Math.min(near.get(type) ?? Number.POSITIVE_INFINITY, dist(at)));
|
|
806
|
+
};
|
|
807
|
+
for (const p of claims.phrases) note(p.type, p.at);
|
|
808
|
+
for (const m of input.mentions) note(m.type, m.at);
|
|
809
|
+
const best = Math.min(...near.values());
|
|
810
|
+
const nearest = [...near].filter(([, d]) => d === best).map(([type]) => type);
|
|
811
|
+
if (nearest.length > 1) clarify ??= `"${input.time.span}" could window ${nearest.sort().join(" or ")}`;
|
|
812
|
+
else if (nearest.length === 1 && nearest[0] !== seed) {
|
|
813
|
+
const type = nearest[0];
|
|
814
|
+
const w = walkTo(type, input.time.span);
|
|
815
|
+
timeOnSeed = false;
|
|
816
|
+
if (w === null) dropped.push({
|
|
817
|
+
span: input.time.span,
|
|
818
|
+
reason: `it windows ${type}, which no walk from ${seed} reaches`,
|
|
819
|
+
kind: "filter"
|
|
820
|
+
});
|
|
821
|
+
else {
|
|
822
|
+
const field = timeField(type);
|
|
823
|
+
const ruled = own(rulings.timeField, type) !== void 0;
|
|
824
|
+
w.window.push(filter({ [field]: {
|
|
825
|
+
gte: input.time.range.gte,
|
|
826
|
+
lt: input.time.range.lt
|
|
827
|
+
} }, [field], `window:${input.time.span}${ruled ? "" : ` on ${field} (default)`}`));
|
|
828
|
+
}
|
|
829
|
+
}
|
|
830
|
+
}
|
|
831
|
+
const params = {};
|
|
832
|
+
const plan = [];
|
|
833
|
+
for (const d of claims.derives) {
|
|
834
|
+
const spec = d.entry.derive;
|
|
835
|
+
if (spec === void 0) continue;
|
|
836
|
+
const [op, operands] = Object.entries(spec)[0];
|
|
837
|
+
if (!operands.every((o) => o.startsWith(`${seed}.`))) {
|
|
838
|
+
dropped.push({
|
|
839
|
+
span: `by ${d.span}`,
|
|
840
|
+
reason: `"${d.span}" derives from fields of another type than ${seed}`,
|
|
841
|
+
kind: "order"
|
|
842
|
+
});
|
|
843
|
+
continue;
|
|
844
|
+
}
|
|
845
|
+
const name = d.span.replace(/[^a-z0-9]+(.)?/g, (_, ch) => (ch ?? "").toUpperCase());
|
|
846
|
+
params.derive = {
|
|
847
|
+
...params.derive ?? {},
|
|
848
|
+
[name]: { [op]: operands.map((o) => `$.${o.slice(seed.length + 1)}`) }
|
|
849
|
+
};
|
|
850
|
+
params.orderBy = [{ [`$${name}`]: "desc" }];
|
|
851
|
+
plan.push({
|
|
852
|
+
path: [
|
|
853
|
+
"derive",
|
|
854
|
+
name,
|
|
855
|
+
`ruling:${d.span}`
|
|
856
|
+
],
|
|
857
|
+
kind: "derive",
|
|
858
|
+
served: false
|
|
859
|
+
}, {
|
|
860
|
+
path: [
|
|
861
|
+
"orderBy",
|
|
862
|
+
"0",
|
|
863
|
+
`$${name}`,
|
|
864
|
+
"desc (default)"
|
|
865
|
+
],
|
|
866
|
+
kind: "order",
|
|
867
|
+
served: false
|
|
868
|
+
});
|
|
869
|
+
}
|
|
870
|
+
if (clarify !== null) return { clarify };
|
|
871
|
+
for (const w of walks.values()) {
|
|
872
|
+
const def = typeExclude(w.type);
|
|
873
|
+
const tested = [];
|
|
874
|
+
if (def !== null) whereFields(own(rulings.typeExcludes, w.type), tested);
|
|
875
|
+
const named = [];
|
|
876
|
+
for (const f of w.filters) whereFields(f.clause, named);
|
|
877
|
+
const yieldTo = tested.find((f) => named.includes(f));
|
|
878
|
+
if (yieldTo === void 0) continue;
|
|
879
|
+
w.excludes = w.excludes.filter((x) => !isDefault(x, w.type));
|
|
880
|
+
w.notes.push(defaultYield(w.type, yieldTo, []));
|
|
881
|
+
}
|
|
882
|
+
let pickDropped = false;
|
|
883
|
+
for (const w of walks.values()) {
|
|
884
|
+
if (w.pick === void 0 || w.excludes.length === 0) continue;
|
|
885
|
+
const names = w.excludes.flatMap((x) => x.plan.map((c) => `'${String(c.path.at(-1)).replace(/^ruling:/, "")}'`)).join(", ");
|
|
886
|
+
dropped.push({
|
|
887
|
+
span: w.pick.span,
|
|
888
|
+
reason: `cannot pick the ${PICKED[w.pick.kind]} ${w.type} while the ${names} exclusion applies`,
|
|
889
|
+
kind: "filter"
|
|
890
|
+
});
|
|
891
|
+
delete w.pick;
|
|
892
|
+
pickDropped = true;
|
|
893
|
+
}
|
|
894
|
+
const taken = /* @__PURE__ */ new Set();
|
|
895
|
+
const ordered = [...walks.values()].sort((x, y) => hopKey(ns, x.path[0]).localeCompare(hopKey(ns, y.path[0])));
|
|
896
|
+
for (const w of ordered) if (w.path.length === 1 && w.pick === void 0) w.alias = aliasFor(w.type, taken);
|
|
897
|
+
const aliasOf = new Map(ordered.map((w) => [w.type, w.alias]));
|
|
898
|
+
const resolveRefs = (v) => {
|
|
899
|
+
if (Array.isArray(v)) return v.map(resolveRefs);
|
|
900
|
+
if (typeof v !== "object" || v === null) return v;
|
|
901
|
+
const out = {};
|
|
902
|
+
for (const [k, x] of Object.entries(v)) if (k === "$ref" && typeof x === "string" && x.includes("\0")) {
|
|
903
|
+
const [type, field] = x.split("\0");
|
|
904
|
+
out[k] = `${aliasOf.get(type)}.${field}`;
|
|
905
|
+
} else out[k] = resolveRefs(x);
|
|
906
|
+
return out;
|
|
907
|
+
};
|
|
908
|
+
const parts = [...seedParts];
|
|
909
|
+
for (const w of ordered) parts.push(walkPart(ns, w, typeExclude));
|
|
910
|
+
const composed = parts.map((p) => ({
|
|
911
|
+
...p,
|
|
912
|
+
clause: resolveRefs(p.clause)
|
|
913
|
+
}));
|
|
914
|
+
return {
|
|
915
|
+
parts: composed,
|
|
916
|
+
params,
|
|
917
|
+
plan,
|
|
918
|
+
dropped,
|
|
919
|
+
timeOnSeed,
|
|
920
|
+
...seedLatest !== void 0 ? { seedLatest } : {},
|
|
921
|
+
walked: walks.size > 0 || seedParts.some((p) => p.plan.some((c) => c.kind === "ref")) || plan.length > 0,
|
|
922
|
+
strict: pickDropped || ordered.some((w) => w.pick !== void 0) || composed.some((p) => p.plan.some((c) => c.kind === "ref"))
|
|
923
|
+
};
|
|
924
|
+
}
|
|
925
|
+
const isDefault = (x, type) => x.plan.some((c) => c.path.at(-1) === `ruling:${type} default exclude`);
|
|
926
|
+
function hopKey(ns, h) {
|
|
927
|
+
return h.dir === "in" ? `${ns}/${h.owner}.${h.field}` : h.field;
|
|
928
|
+
}
|
|
929
|
+
/** One walk as a (possibly nested) `_walk` clause. */
|
|
930
|
+
function walkPart(ns, w, exclude) {
|
|
931
|
+
const inner = andOf([
|
|
932
|
+
...w.filters,
|
|
933
|
+
...w.window,
|
|
934
|
+
...w.excludes
|
|
935
|
+
]);
|
|
936
|
+
const label = w.pick === void 0 ? "any" : `pick:${w.pick.kind}(${w.pick.by}) from ${w.pick.source}`;
|
|
937
|
+
let node = {
|
|
938
|
+
...w.alias !== void 0 ? { as: w.alias } : {},
|
|
939
|
+
...edge(ns, w.path[w.path.length - 1]),
|
|
940
|
+
...inner !== null ? { where: inner.where } : {},
|
|
941
|
+
...w.pick !== void 0 ? { pick: { [w.pick.kind]: w.pick.by } } : {}
|
|
942
|
+
};
|
|
943
|
+
let plan = [{
|
|
944
|
+
path: [
|
|
945
|
+
"_walk",
|
|
946
|
+
hopKey(ns, w.path[w.path.length - 1]),
|
|
947
|
+
label
|
|
948
|
+
],
|
|
949
|
+
kind: "hop",
|
|
950
|
+
served: true
|
|
951
|
+
}, ...[...inner?.plan ?? [], ...w.notes].map((c) => ({
|
|
952
|
+
...c,
|
|
953
|
+
path: [
|
|
954
|
+
"_walk",
|
|
955
|
+
"where",
|
|
956
|
+
...c.path
|
|
957
|
+
]
|
|
958
|
+
}))];
|
|
959
|
+
for (let i = w.path.length - 2; i >= 0; i--) {
|
|
960
|
+
const h = w.path[i];
|
|
961
|
+
const ex = exclude(h.to);
|
|
962
|
+
const where = andOf([{
|
|
963
|
+
clause: { _walk: node },
|
|
964
|
+
plan
|
|
965
|
+
}, ...ex !== null ? [ex] : []]);
|
|
966
|
+
node = {
|
|
967
|
+
...edge(ns, h),
|
|
968
|
+
where: where?.where
|
|
969
|
+
};
|
|
970
|
+
plan = [{
|
|
971
|
+
path: [
|
|
972
|
+
"_walk",
|
|
973
|
+
hopKey(ns, h),
|
|
974
|
+
"any"
|
|
975
|
+
],
|
|
976
|
+
kind: "hop",
|
|
977
|
+
served: true
|
|
978
|
+
}, ...(where?.plan ?? []).map((c) => ({
|
|
979
|
+
...c,
|
|
980
|
+
path: [
|
|
981
|
+
"_walk",
|
|
982
|
+
"where",
|
|
983
|
+
...c.path
|
|
984
|
+
]
|
|
985
|
+
}))];
|
|
986
|
+
}
|
|
987
|
+
return {
|
|
988
|
+
clause: { _walk: node },
|
|
989
|
+
plan
|
|
990
|
+
};
|
|
991
|
+
}
|
|
992
|
+
function edge(ns, h) {
|
|
993
|
+
return h.dir === "in" ? { in: `${ns}/${h.owner}.${h.field}` } : { out: h.field };
|
|
994
|
+
}
|
|
995
|
+
|
|
112
996
|
//#endregion
|
|
113
997
|
//#region src/parse/parse.ts
|
|
114
998
|
/** Words that carry no query meaning on their own. */
|
|
115
|
-
const GLUE = new Set("a about again all am an and any anything are as at be been by can could current did do does for from get give go got have hey i i'm in is it it's let let's lets list me more my need needs next now of ok on or our please raised see show so some still talk tell that the their them there these this those to us want was we we've we'd what what's whats when where which who whose why will with would you your yo up thought handled judgement judgment make nice there's anymore stuff things few runs run lives wrote write written created made filed logged".split(/\s+/));
|
|
999
|
+
const GLUE = new Set("a about again all am an and any anything are as at be been by can could current did do does for from get give go got have hey i i'm in is it it's let let's lets list me more my need needs next now of ok on or our please raised see show so some still talk tell that the their them there these this those to us want was we we've we'd what what's whats when where which who whose why will with would you your yo up thought handled judgement judgment make nice there's anymore stuff things few runs run ran lives wrote write written created made filed logged".split(/\s+/));
|
|
116
1000
|
/** First word of an imperative edit, which substrate-nl refuses: it only reads. */
|
|
117
1001
|
const MUTATIONS = new Set("close delete remove archive merge approve approved create add update set mark rename reopen assign ack deploy cut land".split(" "));
|
|
118
1002
|
const LEADING_FILLER = new Set("please can could would you go ahead let let's lets hey ok yo so and then".split(" "));
|
|
@@ -137,6 +1021,13 @@ const CLOSED = [
|
|
|
137
1021
|
"resolved"
|
|
138
1022
|
];
|
|
139
1023
|
const HISTORY_WORDS = new Set("archived archive history historical ever previously past trend trends over-time all-time total".split(" "));
|
|
1024
|
+
/** Connectives a composed reading cannot honour; dropped while composing walks. */
|
|
1025
|
+
const CONNECTIVES = /* @__PURE__ */ new Set([
|
|
1026
|
+
"or",
|
|
1027
|
+
"not",
|
|
1028
|
+
"no",
|
|
1029
|
+
"every"
|
|
1030
|
+
]);
|
|
140
1031
|
const LATEST = new Set("latest newest recent last".split(" "));
|
|
141
1032
|
const OLDEST = new Set("oldest earliest first".split(" "));
|
|
142
1033
|
const AUTHOR_VERBS = new Set("raised created written filed authored opened made sent logged".split(" "));
|
|
@@ -201,7 +1092,7 @@ function parseQuestion(question, pack, ctx) {
|
|
|
201
1092
|
const tokens = (question.replace(CLAUSE, (span, _q, field, raw) => {
|
|
202
1093
|
clauses.push(parseClause(span, field, raw));
|
|
203
1094
|
return " ";
|
|
204
|
-
}).toLowerCase().replace(/[‘’]/g, "'").match(
|
|
1095
|
+
}).toLowerCase().replace(/[‘’]/g, "'").match(/\$\d+(?:\.\d+)?|\d+(?:\.\d+)?[%×]|#\d+|[a-z0-9@[][a-z0-9@._'\]-]*/g) ?? []).map((t) => t.replace(/[.'-]+$/, "").replace(/'s$/, "")).filter((t) => t.length > 0).map((text) => ({
|
|
205
1096
|
text,
|
|
206
1097
|
used: false
|
|
207
1098
|
}));
|
|
@@ -226,6 +1117,7 @@ function parseQuestion(question, pack, ctx) {
|
|
|
226
1117
|
});
|
|
227
1118
|
return true;
|
|
228
1119
|
});
|
|
1120
|
+
const walkClaims = claimWalkParts(tokens, rulings, GLUE);
|
|
229
1121
|
const mentioned = /* @__PURE__ */ new Map();
|
|
230
1122
|
for (const entry of buildTypeIndex(pack, rulings)) {
|
|
231
1123
|
const n = entry.phrase.split(" ").length;
|
|
@@ -235,7 +1127,8 @@ function parseQuestion(question, pack, ctx) {
|
|
|
235
1127
|
for (const t of slice) t.used = true;
|
|
236
1128
|
for (const type of entry.types) mentioned.set(type, {
|
|
237
1129
|
span: entry.phrase,
|
|
238
|
-
plural: entry.plural
|
|
1130
|
+
plural: entry.plural,
|
|
1131
|
+
at: mentioned.get(type)?.at ?? [i, i + n - 1]
|
|
239
1132
|
});
|
|
240
1133
|
}
|
|
241
1134
|
}
|
|
@@ -352,6 +1245,19 @@ function parseQuestion(question, pack, ctx) {
|
|
|
352
1245
|
role: "author",
|
|
353
1246
|
detail: `createdById startsWith ${author}`
|
|
354
1247
|
});
|
|
1248
|
+
const composing = walkClaims.phrases.length + walkClaims.compares.length + walkClaims.dropped.length > 0;
|
|
1249
|
+
if (composing) {
|
|
1250
|
+
dropComparatives(tokens, walkClaims.dropped);
|
|
1251
|
+
for (const t of tokens) {
|
|
1252
|
+
if (t.used || !CONNECTIVES.has(t.text)) continue;
|
|
1253
|
+
t.used = true;
|
|
1254
|
+
walkClaims.dropped.push({
|
|
1255
|
+
span: t.text,
|
|
1256
|
+
reason: `"${t.text}" is not composed: walk clauses are read as all applying together`,
|
|
1257
|
+
kind: "filter"
|
|
1258
|
+
});
|
|
1259
|
+
}
|
|
1260
|
+
}
|
|
355
1261
|
const fieldAliases = {
|
|
356
1262
|
...BUILTIN_FIELD_ALIASES,
|
|
357
1263
|
...rulings.fieldAliases ?? {}
|
|
@@ -411,6 +1317,12 @@ function parseQuestion(question, pack, ctx) {
|
|
|
411
1317
|
detail: ctx.forceType
|
|
412
1318
|
});
|
|
413
1319
|
}
|
|
1320
|
+
if (composing && ctx.forceType === void 0 && mentioned.size > 0) {
|
|
1321
|
+
const reached = new Set(walkClaims.phrases.map((p) => p.type));
|
|
1322
|
+
const byPosition = [...mentioned].sort(([, x], [, y]) => x.at[0] - y.at[0]).map(([t]) => t);
|
|
1323
|
+
const seeds = byPosition.filter((t) => !reached.has(t));
|
|
1324
|
+
types = (seeds.length > 0 ? seeds : byPosition.slice(0, 1)).sort();
|
|
1325
|
+
}
|
|
414
1326
|
if (types.length === 0 && ids.length > 0 && ctx.defaultIdType !== void 0) types = [ctx.defaultIdType];
|
|
415
1327
|
if (types.length === 0 && values.length > 0) {
|
|
416
1328
|
types = typesCovering(pack, values);
|
|
@@ -442,7 +1354,30 @@ function parseQuestion(question, pack, ctx) {
|
|
|
442
1354
|
const candidates = [];
|
|
443
1355
|
const ruledOut = [];
|
|
444
1356
|
for (const type of types) {
|
|
1357
|
+
let composed;
|
|
1358
|
+
if (composing) {
|
|
1359
|
+
const r = compose({
|
|
1360
|
+
pack,
|
|
1361
|
+
rulings,
|
|
1362
|
+
seed: type,
|
|
1363
|
+
claims: walkClaims,
|
|
1364
|
+
mentions: [...mentioned].map(([t, m]) => ({
|
|
1365
|
+
type: t,
|
|
1366
|
+
span: m.span,
|
|
1367
|
+
at: m.at
|
|
1368
|
+
})),
|
|
1369
|
+
time
|
|
1370
|
+
});
|
|
1371
|
+
if ("clarify" in r) return {
|
|
1372
|
+
status: "clarify",
|
|
1373
|
+
candidates: [],
|
|
1374
|
+
residual: [],
|
|
1375
|
+
reason: r.clarify
|
|
1376
|
+
};
|
|
1377
|
+
composed = r;
|
|
1378
|
+
}
|
|
445
1379
|
const c = assemble(type, {
|
|
1380
|
+
composed,
|
|
446
1381
|
pack,
|
|
447
1382
|
ctx,
|
|
448
1383
|
tokens: tokens.map((t) => ({ ...t })),
|
|
@@ -450,7 +1385,7 @@ function parseQuestion(question, pack, ctx) {
|
|
|
450
1385
|
ids,
|
|
451
1386
|
values,
|
|
452
1387
|
clauses,
|
|
453
|
-
time,
|
|
1388
|
+
time: composed === void 0 || composed.timeOnSeed ? time : null,
|
|
454
1389
|
order,
|
|
455
1390
|
count,
|
|
456
1391
|
allProjects,
|
|
@@ -481,6 +1416,12 @@ function parseQuestion(question, pack, ctx) {
|
|
|
481
1416
|
reason: `${candidates.length} readings`
|
|
482
1417
|
};
|
|
483
1418
|
const only = candidates[0];
|
|
1419
|
+
if (only.dropped.length > 0) return {
|
|
1420
|
+
status: "partial",
|
|
1421
|
+
candidates,
|
|
1422
|
+
residual: only.residual,
|
|
1423
|
+
reason: `dropped: ${only.dropped.map((d) => `"${d.span}" (${d.reason})`).join("; ")}`
|
|
1424
|
+
};
|
|
484
1425
|
if (only.residual.length > 0 && ids.length === 0) return {
|
|
485
1426
|
status: "partial",
|
|
486
1427
|
candidates,
|
|
@@ -499,8 +1440,25 @@ function assemble(type, a) {
|
|
|
499
1440
|
const rulings = ctx.rulings ?? {};
|
|
500
1441
|
const fields = own(pack.types, type)?.fields ?? {};
|
|
501
1442
|
const words = tokens.map((t) => t.text);
|
|
502
|
-
const
|
|
1443
|
+
const seedLatest = a.composed?.seedLatest;
|
|
1444
|
+
const evidence = a.order === null && seedLatest !== void 0 ? [...a.evidence, {
|
|
1445
|
+
span: seedLatest,
|
|
1446
|
+
role: "order",
|
|
1447
|
+
detail: "desc, limit 1"
|
|
1448
|
+
}] : a.evidence;
|
|
1449
|
+
const recency = a.order ?? (seedLatest !== void 0 ? "desc" : null);
|
|
1450
|
+
const conflict = a.composed?.params.orderBy !== void 0 && recency !== null;
|
|
1451
|
+
const order = conflict ? null : recency;
|
|
1452
|
+
const orderSpan = evidence.find((e) => e.role === "order")?.span ?? "";
|
|
1453
|
+
const ev = conflict ? evidence.filter((e) => e.role !== "order") : [...evidence];
|
|
1454
|
+
const dropped = [...a.composed?.dropped ?? [], ...conflict ? [{
|
|
1455
|
+
span: orderSpan,
|
|
1456
|
+
reason: "it conflicts with the derived ordering, which is kept",
|
|
1457
|
+
kind: "order"
|
|
1458
|
+
}] : []];
|
|
503
1459
|
const where = [];
|
|
1460
|
+
const wherePlan = /* @__PURE__ */ new Map();
|
|
1461
|
+
const yields = [];
|
|
504
1462
|
if (a.ids.length > 0) where.push({ shortId: a.ids.length === 1 ? { eq: a.ids[0] } : { in: a.ids } });
|
|
505
1463
|
const byField = /* @__PURE__ */ new Map();
|
|
506
1464
|
const usedFields = /* @__PURE__ */ new Set();
|
|
@@ -575,19 +1533,46 @@ function assemble(type, a) {
|
|
|
575
1533
|
}
|
|
576
1534
|
const historical = a.historyWord || a.time !== null || a.closeVerb || terminal;
|
|
577
1535
|
if (a.time !== null) {
|
|
578
|
-
const
|
|
579
|
-
|
|
1536
|
+
const named = Object.entries(fields).find(([, f]) => f.valueType === "date" && f.kind !== "metadata" && creditFieldWords(tokens, f));
|
|
1537
|
+
const dateField = named?.[0] ?? (a.closeVerb || a.updateVerb ? "updatedAt" : own(rulings.timeField, type) ?? "createdAt");
|
|
1538
|
+
const clause = { [dateField]: {
|
|
580
1539
|
gte: a.time.range.gte,
|
|
581
1540
|
lt: a.time.range.lt
|
|
582
|
-
} }
|
|
1541
|
+
} };
|
|
1542
|
+
where.push(clause);
|
|
1543
|
+
if (a.composed !== void 0) {
|
|
1544
|
+
const ruled = named !== void 0 || a.closeVerb || a.updateVerb || own(rulings.timeField, type) !== void 0;
|
|
1545
|
+
wherePlan.set(clause, [{
|
|
1546
|
+
path: [dateField, `window:${a.time.span}${ruled ? "" : ` on ${dateField} (default)`}`],
|
|
1547
|
+
kind: "filter",
|
|
1548
|
+
served: false
|
|
1549
|
+
}]);
|
|
1550
|
+
}
|
|
583
1551
|
}
|
|
584
1552
|
if (a.ids.length === 0) {
|
|
1553
|
+
const excluded = own(rulings.typeExcludes, type);
|
|
1554
|
+
if (excluded !== void 0) {
|
|
1555
|
+
const tested = [];
|
|
1556
|
+
whereFields(excluded, tested);
|
|
1557
|
+
const named = [...usedFields];
|
|
1558
|
+
for (const p of a.composed?.parts ?? []) whereFields(p.clause, named);
|
|
1559
|
+
const yieldTo = tested.find((f) => named.includes(f));
|
|
1560
|
+
if (yieldTo === void 0) {
|
|
1561
|
+
const clause = { NOT: excluded };
|
|
1562
|
+
where.push(clause);
|
|
1563
|
+
wherePlan.set(clause, [{
|
|
1564
|
+
path: ["NOT", `ruling:${type} default exclude`],
|
|
1565
|
+
kind: "filter",
|
|
1566
|
+
served: false
|
|
1567
|
+
}]);
|
|
1568
|
+
} else yields.push(defaultYield(type, yieldTo));
|
|
1569
|
+
}
|
|
585
1570
|
const statusField = fields.status;
|
|
586
1571
|
const live = own(rulings.liveStatus, type);
|
|
587
1572
|
if (a.closeVerb && statusField?.enum !== void 0 && !byField.has("status")) {
|
|
588
1573
|
const closed = statusField.enum.filter((s) => CLOSED.includes(s));
|
|
589
1574
|
if (closed.length > 0) where.push({ status: { in: closed } });
|
|
590
|
-
} else if (!historical && live !== void 0 && statusField?.enum?.includes(live) === true && !usedFields.has("status") &&
|
|
1575
|
+
} else if (!historical && live !== void 0 && statusField?.enum?.includes(live) === true && !usedFields.has("status") && order === null) where.push({ status: { eq: live } });
|
|
591
1576
|
if (!historical && fields.archived !== void 0) where.push({ archived: { eq: false } });
|
|
592
1577
|
else if (historical) ev.push({
|
|
593
1578
|
span: "(history)",
|
|
@@ -607,19 +1592,36 @@ function assemble(type, a) {
|
|
|
607
1592
|
}
|
|
608
1593
|
}
|
|
609
1594
|
const params = {};
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
1595
|
+
const parts = [...where.map((clause) => ({
|
|
1596
|
+
clause,
|
|
1597
|
+
plan: wherePlan.get(clause) ?? []
|
|
1598
|
+
})), ...a.composed?.parts ?? []];
|
|
1599
|
+
const all = andOf(parts);
|
|
1600
|
+
if (all !== null) params.where = all.where;
|
|
1601
|
+
const plan = [
|
|
1602
|
+
...(all?.plan ?? []).map((c) => ({
|
|
1603
|
+
...c,
|
|
1604
|
+
path: ["where", ...c.path]
|
|
1605
|
+
})),
|
|
1606
|
+
...yields,
|
|
1607
|
+
...a.composed?.plan ?? []
|
|
1608
|
+
];
|
|
1609
|
+
if (order !== null) {
|
|
1610
|
+
params.orderBy = [{ [own(rulings.timeField, type) ?? "createdAt"]: order }];
|
|
614
1611
|
params.limit = 1;
|
|
615
1612
|
}
|
|
616
|
-
|
|
1613
|
+
Object.assign(params, a.composed?.params ?? {});
|
|
1614
|
+
if (a.count && a.composed?.walked === true) params.count = true;
|
|
1615
|
+
const answer = a.askField !== null ? a.askField.group ? `group:${a.askField.field}` : `field:${a.askField.field}` : a.count ? "count" : order !== null || a.ids.length === 1 ? "single" : "rows";
|
|
617
1616
|
return {
|
|
618
1617
|
type,
|
|
619
1618
|
frame: { [`${pack.namespace}/${type}`]: { params } },
|
|
620
1619
|
answer,
|
|
621
1620
|
evidence: ev,
|
|
622
|
-
residual: contentWords(tokens)
|
|
1621
|
+
residual: contentWords(tokens),
|
|
1622
|
+
plan,
|
|
1623
|
+
dropped,
|
|
1624
|
+
strict: a.count || order !== null || a.ids.length === 1 || a.composed?.strict === true
|
|
623
1625
|
};
|
|
624
1626
|
}
|
|
625
1627
|
/** A clause's field name: the field itself, case-insensitively, then the rulings' and built-in aliases. */
|
|
@@ -666,7 +1668,16 @@ function creditFieldWords(tokens, field) {
|
|
|
666
1668
|
return hit;
|
|
667
1669
|
}
|
|
668
1670
|
function contentWords(tokens) {
|
|
669
|
-
|
|
1671
|
+
const free = (t) => t !== void 0 && !t.used && !GLUE.has(t.text);
|
|
1672
|
+
const num = (t) => free(t) && /^\d+$/.test(t.text);
|
|
1673
|
+
const out = [];
|
|
1674
|
+
tokens.forEach((t, i) => {
|
|
1675
|
+
if (!free(t) || num(t)) return;
|
|
1676
|
+
const before = num(tokens[i - 1]) ? `${tokens[i - 1]?.text} ` : "";
|
|
1677
|
+
const after = num(tokens[i + 1]) && !(free(tokens[i + 2]) && !num(tokens[i + 2])) ? ` ${tokens[i + 1]?.text}` : "";
|
|
1678
|
+
out.push(`${before}${t.text}${after}`);
|
|
1679
|
+
});
|
|
1680
|
+
return out;
|
|
670
1681
|
}
|
|
671
1682
|
/** Claims the longest n-grams (up to `max` words) for which `accept` returns true. */
|
|
672
1683
|
function claimNgrams(tokens, max, accept) {
|
|
@@ -739,13 +1750,14 @@ function pickField(candidates, fields, words) {
|
|
|
739
1750
|
}
|
|
740
1751
|
function claimTime(tokens, asOf) {
|
|
741
1752
|
let found = null;
|
|
742
|
-
claimNgrams(tokens, 3, (key) => {
|
|
1753
|
+
claimNgrams(tokens, 3, (key, at) => {
|
|
743
1754
|
if (found !== null) return false;
|
|
744
1755
|
const range = rangeFor(key, asOf);
|
|
745
1756
|
if (range === null) return false;
|
|
746
1757
|
found = {
|
|
747
1758
|
span: key,
|
|
748
|
-
range
|
|
1759
|
+
range,
|
|
1760
|
+
at: [at[0], at[at.length - 1]]
|
|
749
1761
|
};
|
|
750
1762
|
return true;
|
|
751
1763
|
});
|
|
@@ -800,7 +1812,10 @@ function buildJudgePrompt(question, result, pack, rulings = {}, rejected = []) {
|
|
|
800
1812
|
]
|
|
801
1813
|
}
|
|
802
1814
|
})] : [],
|
|
803
|
-
branch("recall"
|
|
1815
|
+
branch("recall", { type: {
|
|
1816
|
+
type: "string",
|
|
1817
|
+
enum: allTypes
|
|
1818
|
+
} }),
|
|
804
1819
|
branch("clarify"),
|
|
805
1820
|
branch("refuse"),
|
|
806
1821
|
branch("escalate")
|
|
@@ -824,7 +1839,7 @@ function buildJudgePrompt(question, result, pack, rulings = {}, rejected = []) {
|
|
|
824
1839
|
" of a name or id, contains = a word inside a title or subject. Never bind a word that only",
|
|
825
1840
|
" describes the question (e.g. \"status\", \"called\").",
|
|
826
1841
|
"- recall: the question asks what is known ABOUT a topic (free-text subject words, not field values);",
|
|
827
|
-
" semantic search over the text answers it.",
|
|
1842
|
+
" semantic search over the text answers it. Name the type whose rows hold that knowledge.",
|
|
828
1843
|
"- clarify: RARE. Only when the question has two specific readings and nothing in it chooses one.",
|
|
829
1844
|
" Missing detail is not ambiguity: if a type or a topic fits, choose type or recall.",
|
|
830
1845
|
"- refuse: the question is not about this store at all, or asks to change data.",
|
|
@@ -927,21 +1942,54 @@ async function parseWithJudge(question, pack, ctx, judge, opts = {}) {
|
|
|
927
1942
|
} };
|
|
928
1943
|
trail.push(step);
|
|
929
1944
|
const prompt = buildJudgePrompt(question, result, pack, context.rulings, rejected);
|
|
930
|
-
const
|
|
1945
|
+
const reply = await judge.decide(prompt, opts.signal);
|
|
1946
|
+
const checked = checkChoice(reply, prompt);
|
|
931
1947
|
if (typeof checked === "string") {
|
|
932
1948
|
step.rejected = checked;
|
|
933
1949
|
break;
|
|
934
1950
|
}
|
|
935
1951
|
step.choice = checked;
|
|
1952
|
+
const extra = Object.keys(reply).filter((k) => !Object.hasOwn(checked, k));
|
|
1953
|
+
if (extra.length > 0) step.ignored = extra.sort();
|
|
936
1954
|
switch (checked.action) {
|
|
937
|
-
case "pick":
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
1955
|
+
case "pick": {
|
|
1956
|
+
const chosen = result.candidates[checked.candidate];
|
|
1957
|
+
if (chosen.dropped.length > 0) {
|
|
1958
|
+
const reason = `dropped: ${chosen.dropped.map((d) => `"${d.span}" (${d.reason})`).join("; ")}`;
|
|
1959
|
+
return {
|
|
1960
|
+
status: "escalated",
|
|
1961
|
+
result: {
|
|
1962
|
+
status: "partial",
|
|
1963
|
+
candidates: [chosen],
|
|
1964
|
+
residual: chosen.residual,
|
|
1965
|
+
reason
|
|
1966
|
+
},
|
|
1967
|
+
trail
|
|
1968
|
+
};
|
|
1969
|
+
}
|
|
1970
|
+
if (chosen.strict && chosen.residual.length > 0) {
|
|
1971
|
+
const reason = `unexplained under a strict reading: ${chosen.residual.join(", ")}`;
|
|
1972
|
+
return {
|
|
1973
|
+
status: "escalated",
|
|
1974
|
+
result: {
|
|
1975
|
+
status: "partial",
|
|
1976
|
+
candidates: [chosen],
|
|
1977
|
+
residual: chosen.residual,
|
|
1978
|
+
reason
|
|
1979
|
+
},
|
|
1980
|
+
trail
|
|
1981
|
+
};
|
|
1982
|
+
}
|
|
1983
|
+
return {
|
|
1984
|
+
status: "resolved",
|
|
1985
|
+
candidate: chosen,
|
|
1986
|
+
trail
|
|
1987
|
+
};
|
|
1988
|
+
}
|
|
942
1989
|
case "recall": return {
|
|
943
1990
|
status: "recall",
|
|
944
1991
|
query: question,
|
|
1992
|
+
type: checked.type,
|
|
945
1993
|
trail
|
|
946
1994
|
};
|
|
947
1995
|
case "clarify":
|
|
@@ -993,36 +2041,139 @@ async function parseWithJudge(question, pack, ctx, judge, opts = {}) {
|
|
|
993
2041
|
//#endregion
|
|
994
2042
|
//#region src/ask/run.ts
|
|
995
2043
|
const RECALL_K = 10;
|
|
996
|
-
/**
|
|
997
|
-
|
|
2044
|
+
/**
|
|
2045
|
+
* The frame a candidate needs to run. A count is the server's `count: true`
|
|
2046
|
+
* over the reading's where; `paged` is the rows frame with every projectable
|
|
2047
|
+
* field, which a count falls back to when the server cannot count.
|
|
2048
|
+
*/
|
|
2049
|
+
function executable(c, pack, paged = c.answer !== "count") {
|
|
998
2050
|
const [key, value] = Object.entries(c.frame)[0];
|
|
2051
|
+
const { count: _count, ...params } = value.params;
|
|
2052
|
+
if (!paged) {
|
|
2053
|
+
const { orderBy: _o, limit: _l, derive: _d, ...counted } = params;
|
|
2054
|
+
return { [key]: {
|
|
2055
|
+
params: {
|
|
2056
|
+
...counted,
|
|
2057
|
+
count: true
|
|
2058
|
+
},
|
|
2059
|
+
fields: { id: true }
|
|
2060
|
+
} };
|
|
2061
|
+
}
|
|
999
2062
|
const fields = { id: true };
|
|
1000
2063
|
for (const [name, f] of Object.entries(own(pack.types, c.type)?.fields ?? {})) if (f.kind !== "metadata") fields[name] = true;
|
|
1001
2064
|
return { [key]: {
|
|
1002
|
-
params
|
|
2065
|
+
params,
|
|
1003
2066
|
fields
|
|
1004
2067
|
} };
|
|
1005
2068
|
}
|
|
1006
|
-
|
|
1007
|
-
|
|
2069
|
+
/**
|
|
2070
|
+
* The engine refused `count: true`: the seed is under a row policy, or the
|
|
2071
|
+
* store (memory, legacy-layout pg) answers no walk grammar. A plain paged read
|
|
2072
|
+
* works in every case.
|
|
2073
|
+
*/
|
|
2074
|
+
const COUNT_REFUSED = /* @__PURE__ */ new Set([
|
|
2075
|
+
ROW_POLICY_BLOCKS_WALK,
|
|
2076
|
+
WALKS_UNSUPPORTED_STORE,
|
|
2077
|
+
WALKS_NEED_PARTITIONED
|
|
2078
|
+
]);
|
|
2079
|
+
/**
|
|
2080
|
+
* The parser's plan with the engine's `served`/`recommend` where the engine
|
|
2081
|
+
* planned the same clause (matched by path: the engine's path is a prefix of
|
|
2082
|
+
* the parser's labelled one, or, for a NOT, extends it). Parser-only clauses
|
|
2083
|
+
* keep their entries; engine-only clauses are appended.
|
|
2084
|
+
*/
|
|
2085
|
+
function mergePlan(parsed, engine) {
|
|
2086
|
+
if (engine === void 0) return parsed;
|
|
2087
|
+
const kindOf = (c) => c.kind === "derive" && c.path[0] === "orderBy" ? "order" : c.kind;
|
|
2088
|
+
const prefix = (a, b) => a.length <= b.length && a.every((x, i) => x === b[i]);
|
|
2089
|
+
const used = /* @__PURE__ */ new Set();
|
|
2090
|
+
return [...parsed.map((p) => {
|
|
2091
|
+
const same = engine.filter((e) => kindOf(e) === kindOf(p));
|
|
2092
|
+
const under = same.filter((e) => prefix(e.path, p.path)).sort((x, y) => y.path.length - x.path.length);
|
|
2093
|
+
const matches = under.length > 0 ? [under[0]] : same.filter((e) => prefix(p.path.slice(0, -1), e.path));
|
|
2094
|
+
if (matches.length === 0) return p;
|
|
2095
|
+
for (const m of matches) used.add(m);
|
|
2096
|
+
const { recommend: _r, ...rest } = p;
|
|
2097
|
+
const recommend = matches.find((m) => !m.served)?.recommend;
|
|
2098
|
+
return {
|
|
2099
|
+
...rest,
|
|
2100
|
+
served: matches.every((m) => m.served),
|
|
2101
|
+
...recommend !== void 0 ? { recommend } : {}
|
|
2102
|
+
};
|
|
2103
|
+
}), ...engine.filter((e) => !used.has(e))];
|
|
2104
|
+
}
|
|
2105
|
+
async function rows(c, source, question, http, deps, handoff, leftovers = []) {
|
|
2106
|
+
const dropped = [...c.dropped, ...leftovers];
|
|
2107
|
+
if (c.strict && dropped.some((d) => d.kind === "filter")) return {
|
|
2108
|
+
kind: "unknown",
|
|
2109
|
+
question,
|
|
2110
|
+
dropped,
|
|
2111
|
+
plan: c.plan
|
|
2112
|
+
};
|
|
2113
|
+
let frame = executable(c, deps.pack);
|
|
1008
2114
|
const key = Object.keys(frame)[0];
|
|
1009
|
-
|
|
1010
|
-
const
|
|
2115
|
+
let env = await deps.engine.query(frame, http);
|
|
2116
|
+
const refused = (env.errors ?? []).some((e) => COUNT_REFUSED.has(e.code ?? ""));
|
|
2117
|
+
const paging = c.answer === "count" && !(key in (env.data ?? {})) && refused;
|
|
2118
|
+
if (paging) {
|
|
2119
|
+
frame = executable(c, deps.pack, true);
|
|
2120
|
+
env = await deps.engine.query(frame, http);
|
|
2121
|
+
}
|
|
2122
|
+
const data = env.data !== void 0 && key in env.data ? env.data?.[key] : void 0;
|
|
2123
|
+
const errors = [...env.errors ?? []];
|
|
2124
|
+
let count;
|
|
2125
|
+
if (paging && Array.isArray(data)) {
|
|
2126
|
+
const counted = await countRest(frame, key, data.length, env, http, deps);
|
|
2127
|
+
if (typeof counted === "number") count = counted;
|
|
2128
|
+
else errors.push(counted);
|
|
2129
|
+
} else if (c.answer === "count" && typeof data?.count === "number") count = data.count;
|
|
1011
2130
|
return {
|
|
1012
2131
|
kind: "rows",
|
|
1013
2132
|
frame,
|
|
1014
2133
|
answer: c.answer,
|
|
1015
|
-
data
|
|
1016
|
-
...
|
|
2134
|
+
data,
|
|
2135
|
+
...count !== void 0 ? { count } : {},
|
|
2136
|
+
...errors.length > 0 ? { errors } : {},
|
|
1017
2137
|
source,
|
|
1018
|
-
...handoff !== void 0 ? { handoff } : {}
|
|
2138
|
+
...handoff !== void 0 ? { handoff } : {},
|
|
2139
|
+
dropped,
|
|
2140
|
+
plan: mergePlan(c.plan, env._diagnostics?.[key]?.plan)
|
|
1019
2141
|
};
|
|
1020
2142
|
}
|
|
1021
|
-
|
|
2143
|
+
const MAX_COUNT_PAGES = 1e3;
|
|
2144
|
+
/** The population a count candidate matches: `seen` rows from page one, then id-only pages after its cursor. */
|
|
2145
|
+
async function countRest(frame, key, seen, first, http, deps) {
|
|
2146
|
+
const params = frame[key]?.params ?? {};
|
|
2147
|
+
let count = seen;
|
|
2148
|
+
let page = first._pagination?.[key];
|
|
2149
|
+
for (let pages = 1; page?.hasMore === true; pages++) {
|
|
2150
|
+
if (pages >= MAX_COUNT_PAGES || page.nextCursor === void 0 || isAborted(http)) return {
|
|
2151
|
+
code: "count_incomplete",
|
|
2152
|
+
message: `counted ${count} rows and stopped before the end; the true count is higher`
|
|
2153
|
+
};
|
|
2154
|
+
const env = await deps.engine.query({ [key]: {
|
|
2155
|
+
params: {
|
|
2156
|
+
...params,
|
|
2157
|
+
cursor: page.nextCursor
|
|
2158
|
+
},
|
|
2159
|
+
fields: { id: true }
|
|
2160
|
+
} }, http);
|
|
2161
|
+
const rows = env.data?.[key];
|
|
2162
|
+
if (!Array.isArray(rows)) return {
|
|
2163
|
+
code: "count_incomplete",
|
|
2164
|
+
message: `counted ${count} rows, then a page failed: ${JSON.stringify(env.errors ?? [])}`
|
|
2165
|
+
};
|
|
2166
|
+
count += rows.length;
|
|
2167
|
+
page = env._pagination?.[key];
|
|
2168
|
+
}
|
|
2169
|
+
return count;
|
|
2170
|
+
}
|
|
2171
|
+
async function recall(query, ns, type, http, deps) {
|
|
1022
2172
|
const frame = { "recall/Hit": {
|
|
1023
2173
|
params: {
|
|
1024
2174
|
to: query,
|
|
1025
2175
|
namespace: ns,
|
|
2176
|
+
type,
|
|
1026
2177
|
k: RECALL_K
|
|
1027
2178
|
},
|
|
1028
2179
|
fields: {
|
|
@@ -1081,14 +2232,28 @@ async function* runAsk(req, http, deps) {
|
|
|
1081
2232
|
...first.reason !== void 0 ? { reason: first.reason } : {}
|
|
1082
2233
|
};
|
|
1083
2234
|
if (isAborted(http)) return yield { event: "done" };
|
|
2235
|
+
/**
|
|
2236
|
+
* No judge decided: the parser's reading runs as it stands. Its unexplained
|
|
2237
|
+
* words are dropped, so a strict reading with any is `unknown`, never a number.
|
|
2238
|
+
*/
|
|
2239
|
+
const handoff = async (reason, parsed = first) => {
|
|
2240
|
+
const best = parsed.candidates[0];
|
|
2241
|
+
if (best === void 0) return clarify(req.question, reason, parsed.candidates, deps.pack);
|
|
2242
|
+
const leftovers = best.residual.map((w) => ({
|
|
2243
|
+
span: w,
|
|
2244
|
+
reason: `not understood (${reason})`,
|
|
2245
|
+
kind: "filter"
|
|
2246
|
+
}));
|
|
2247
|
+
return rows(best, "handoff", req.question, http, deps, reason, leftovers);
|
|
2248
|
+
};
|
|
1084
2249
|
let outcome;
|
|
1085
|
-
if (first.status === "resolved" && best !== void 0) outcome = await rows(best, "parser", http, deps);
|
|
2250
|
+
if (first.status === "resolved" && best !== void 0) outcome = await rows(best, "parser", req.question, http, deps);
|
|
1086
2251
|
else if (first.status === "refuse") outcome = {
|
|
1087
2252
|
kind: "refuse",
|
|
1088
2253
|
reason: first.reason ?? "refused"
|
|
1089
2254
|
};
|
|
1090
2255
|
else if (first.status === "clarify") outcome = clarify(req.question, first.reason ?? "which did you mean?", first.candidates, deps.pack);
|
|
1091
|
-
else if (deps.judge === void 0) outcome =
|
|
2256
|
+
else if (deps.judge === void 0) outcome = await handoff("judge not configured");
|
|
1092
2257
|
else {
|
|
1093
2258
|
yield {
|
|
1094
2259
|
event: "judging",
|
|
@@ -1130,7 +2295,7 @@ async function* runAsk(req, http, deps) {
|
|
|
1130
2295
|
http.signal?.removeEventListener("abort", onAbort);
|
|
1131
2296
|
}
|
|
1132
2297
|
if (isAborted(http)) return yield { event: "done" };
|
|
1133
|
-
if (judged === void 0) outcome =
|
|
2298
|
+
if (judged === void 0) outcome = await handoff("judge unavailable");
|
|
1134
2299
|
else {
|
|
1135
2300
|
yield {
|
|
1136
2301
|
event: "judged",
|
|
@@ -1138,11 +2303,17 @@ async function* runAsk(req, http, deps) {
|
|
|
1138
2303
|
trail: judged.trail
|
|
1139
2304
|
};
|
|
1140
2305
|
switch (judged.status) {
|
|
1141
|
-
case "resolved":
|
|
1142
|
-
|
|
2306
|
+
case "resolved": {
|
|
2307
|
+
const noise = judged.candidate.residual.map((w) => ({
|
|
2308
|
+
span: w,
|
|
2309
|
+
reason: "judge: noise",
|
|
2310
|
+
kind: "filter"
|
|
2311
|
+
}));
|
|
2312
|
+
outcome = await rows(judged.candidate, "judge", req.question, http, deps, void 0, noise);
|
|
1143
2313
|
break;
|
|
2314
|
+
}
|
|
1144
2315
|
case "recall":
|
|
1145
|
-
outcome = await recall(judged.query, deps.pack.namespace, http, deps);
|
|
2316
|
+
outcome = await recall(judged.query, deps.pack.namespace, ctx.forceType ?? judged.type, http, deps);
|
|
1146
2317
|
break;
|
|
1147
2318
|
case "refuse":
|
|
1148
2319
|
outcome = {
|
|
@@ -1153,7 +2324,7 @@ async function* runAsk(req, http, deps) {
|
|
|
1153
2324
|
case "clarify":
|
|
1154
2325
|
outcome = clarify(req.question, judged.reason, first.candidates, deps.pack);
|
|
1155
2326
|
break;
|
|
1156
|
-
case "escalated": outcome =
|
|
2327
|
+
case "escalated": outcome = await handoff("judge could not decide", judged.result.candidates.length > 0 ? judged.result : first);
|
|
1157
2328
|
}
|
|
1158
2329
|
}
|
|
1159
2330
|
}
|
|
@@ -1178,7 +2349,7 @@ var OllamaJudge = class {
|
|
|
1178
2349
|
constructor(config) {
|
|
1179
2350
|
this.#config = {
|
|
1180
2351
|
...config,
|
|
1181
|
-
baseUrl: config.baseUrl
|
|
2352
|
+
baseUrl: trimTrailingSlashes(config.baseUrl)
|
|
1182
2353
|
};
|
|
1183
2354
|
}
|
|
1184
2355
|
async decide(prompt, signal) {
|
|
@@ -1211,63 +2382,6 @@ var OllamaJudge = class {
|
|
|
1211
2382
|
}
|
|
1212
2383
|
};
|
|
1213
2384
|
|
|
1214
|
-
//#endregion
|
|
1215
|
-
//#region src/rulings/schema.ts
|
|
1216
|
-
const ValueAliasSchema = z.object({
|
|
1217
|
-
type: z.string().min(1),
|
|
1218
|
-
field: z.string().min(1),
|
|
1219
|
-
in: z.array(z.string()).optional(),
|
|
1220
|
-
eq: z.string().optional(),
|
|
1221
|
-
startsWith: z.string().optional(),
|
|
1222
|
-
contains: z.string().optional()
|
|
1223
|
-
}).strict().refine((v) => [
|
|
1224
|
-
v.in,
|
|
1225
|
-
v.eq,
|
|
1226
|
-
v.startsWith,
|
|
1227
|
-
v.contains
|
|
1228
|
-
].filter((x) => x !== void 0).length === 1, { message: "a value alias names exactly one of in, eq, startsWith, contains" });
|
|
1229
|
-
const record = z.record(z.string(), z.string());
|
|
1230
|
-
const NamespaceRulingsSchema = z.object({
|
|
1231
|
-
unscopedTypes: z.array(z.string()).optional(),
|
|
1232
|
-
typeAliases: record.optional(),
|
|
1233
|
-
fieldAliases: record.optional(),
|
|
1234
|
-
liveStatus: record.optional(),
|
|
1235
|
-
timeField: record.optional(),
|
|
1236
|
-
typeNotes: record.optional(),
|
|
1237
|
-
valueAliases: z.record(z.string(), z.array(ValueAliasSchema)).optional()
|
|
1238
|
-
}).strict();
|
|
1239
|
-
/** Per top-level key: records merge entry-by-entry (override wins), arrays replace. */
|
|
1240
|
-
function mergeRulings(declared, override) {
|
|
1241
|
-
if (override === void 0) return declared;
|
|
1242
|
-
const out = { ...declared };
|
|
1243
|
-
for (const [key, value] of Object.entries(override)) {
|
|
1244
|
-
const prior = out[key];
|
|
1245
|
-
out[key] = Array.isArray(value) || prior === void 0 || Array.isArray(prior) ? value : {
|
|
1246
|
-
...prior,
|
|
1247
|
-
...value
|
|
1248
|
-
};
|
|
1249
|
-
}
|
|
1250
|
-
return out;
|
|
1251
|
-
}
|
|
1252
|
-
/** The effective rulings for one namespace's schema, plus anything that was dropped. */
|
|
1253
|
-
function readRulings(schema, override) {
|
|
1254
|
-
const warnings = [];
|
|
1255
|
-
let declared = {};
|
|
1256
|
-
const raw = schema.annotations?.nl;
|
|
1257
|
-
if (raw !== void 0) {
|
|
1258
|
-
const parsed = NamespaceRulingsSchema.safeParse(raw);
|
|
1259
|
-
if (parsed.success) declared = parsed.data;
|
|
1260
|
-
else {
|
|
1261
|
-
const where = parsed.error.issues.map((i) => i.path.join(".") || "(root)").join(", ");
|
|
1262
|
-
warnings.push(`declared rulings are invalid and were ignored (${where})`);
|
|
1263
|
-
}
|
|
1264
|
-
}
|
|
1265
|
-
return {
|
|
1266
|
-
rulings: mergeRulings(declared, override),
|
|
1267
|
-
warnings
|
|
1268
|
-
};
|
|
1269
|
-
}
|
|
1270
|
-
|
|
1271
2385
|
//#endregion
|
|
1272
2386
|
//#region src/pack/build.ts
|
|
1273
2387
|
/** Longest value worth matching as a whole phrase. */
|
|
@@ -1372,7 +2486,7 @@ function packField(name, def, operators) {
|
|
|
1372
2486
|
name,
|
|
1373
2487
|
kind: "scalar",
|
|
1374
2488
|
valueType: def.type,
|
|
1375
|
-
indexed: def.indexed
|
|
2489
|
+
indexed: def.indexed !== false,
|
|
1376
2490
|
operators: [...operators[`scalar.${def.type}`] ?? []],
|
|
1377
2491
|
phrases,
|
|
1378
2492
|
...def.enum !== void 0 ? { enum: [...def.enum] } : {},
|
|
@@ -1381,8 +2495,11 @@ function packField(name, def, operators) {
|
|
|
1381
2495
|
case "crossRef": return {
|
|
1382
2496
|
name,
|
|
1383
2497
|
kind: "crossRef",
|
|
1384
|
-
target:
|
|
1385
|
-
|
|
2498
|
+
target: {
|
|
2499
|
+
namespace: def.ns,
|
|
2500
|
+
type: def.type
|
|
2501
|
+
},
|
|
2502
|
+
indexed: def.indexed !== false,
|
|
1386
2503
|
operators: [...operators.crossRef ?? []],
|
|
1387
2504
|
phrases
|
|
1388
2505
|
};
|
|
@@ -1492,7 +2609,7 @@ async function profileNamespace(input) {
|
|
|
1492
2609
|
return { types };
|
|
1493
2610
|
}
|
|
1494
2611
|
function profiled(def) {
|
|
1495
|
-
if (def.kind === "scalar") return def.type === "string" && def.indexed
|
|
2612
|
+
if (def.kind === "scalar") return def.type === "string" && def.indexed !== false;
|
|
1496
2613
|
return def.kind === "list" && def.indexed === true && def.of.kind === "scalar" && def.of.type === "string";
|
|
1497
2614
|
}
|
|
1498
2615
|
|
|
@@ -1818,4 +2935,4 @@ const bootstrap = defineBootstrap.kernel({
|
|
|
1818
2935
|
});
|
|
1819
2936
|
|
|
1820
2937
|
//#endregion
|
|
1821
|
-
export { askHandler as a, profileNamespace as c,
|
|
2938
|
+
export { askHandler as a, profileNamespace as c, runAsk as d, parseWithJudge as f, readRulings as g, mergeRulings as h, createNlPlugin as i, buildNamespacePack as l, NamespaceRulingsSchema as m, bootstrap as n, packHandler as o, parseQuestion as p, CONTRACT_VERSION as r, PackService as s, NlBootstrapConfigSchema as t, OllamaJudge as u };
|