@littlebigbrain/create 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,368 @@
1
+ // The candidates of a mention (general linking 4.3, 4.4) and the decision
2
+ // (5.1 to 5.3): the bounds and stop keys, the rules without model reads,
3
+ // the link rule of link version 7, and the packing of the requests.
4
+ import { test } from "node:test";
5
+ import assert from "node:assert/strict";
6
+ import { ENTITY_IRI_NAMESPACE } from "@littlebigbrain/client";
7
+ import {
8
+ BOUNDS,
9
+ candidatesFor,
10
+ neighbourPaths,
11
+ stopKeys,
12
+ type Candidate,
13
+ type KeyHit,
14
+ } from "./candidates.js";
15
+ import {
16
+ decideRequest,
17
+ effectIdOf,
18
+ linkRule,
19
+ packRequests,
20
+ rulesDecisions,
21
+ sameRecord,
22
+ type Asked,
23
+ } from "./decide.js";
24
+ import { blockKeys } from "./names.js";
25
+ import {
26
+ identityItem,
27
+ itemKind,
28
+ rowsOfItem,
29
+ thingItems,
30
+ type Row,
31
+ type TargetView,
32
+ } from "./items.js";
33
+
34
+ const e = (n: number) => `${ENTITY_IRI_NAMESPACE}${String(n).padStart(32, "0")}`;
35
+ const hit = (key: string, n: number, over: Partial<KeyHit> = {}): KeyHit => ({
36
+ key,
37
+ mention: e(n),
38
+ role: "self",
39
+ span: "Mira Tannenholt-Ruiz",
40
+ source: e(100 + n),
41
+ kind: "name",
42
+ window: null,
43
+ thing: e(500 + n),
44
+ label: "Mira Tannenholt-Ruiz",
45
+ ...over,
46
+ });
47
+ const mention = (span: string, keys = blockKeys(span, "text")) => ({
48
+ iri: e(1),
49
+ source: e(101),
50
+ role: "refers" as const,
51
+ span,
52
+ keys,
53
+ cannot: [],
54
+ });
55
+
56
+ test("a strong key comes first; a stop key, the mention's own record and a cannot-link find nothing", () => {
57
+ const m = {
58
+ ...mention("M. Tannenholt"),
59
+ keys: ["email:mira@northbay.example", ...blockKeys("M. Tannenholt", "name")],
60
+ };
61
+ const hits = [
62
+ hit("name:initial:tannenholt m", 2),
63
+ hit("email:mira@northbay.example", 3, { span: "mira@northbay.example", kind: "email" }),
64
+ hit("name:part:tannenholt", 4, { span: "Hanna Tannenholt", label: "Hanna Tannenholt" }),
65
+ hit("name:initial:tannenholt m", 5, { thing: e(505) }),
66
+ ];
67
+ const found = candidatesFor(m, hits, new Set(), []);
68
+ assert.equal(found[0]!.target, e(3));
69
+ assert.ok(found[0]!.strong);
70
+ assert.ok(
71
+ !found.some((c) => c.target === e(4)),
72
+ "a weak key alone and an unlike name stay below the score",
73
+ );
74
+ assert.deepEqual(
75
+ candidatesFor(
76
+ m,
77
+ hits,
78
+ new Set(["email:mira@northbay.example", "name:initial:tannenholt m"]),
79
+ [],
80
+ ).map((c) => c.target),
81
+ [],
82
+ );
83
+ assert.ok(
84
+ !candidatesFor({ ...m, cannot: [e(502)] }, hits, new Set(), []).some((c) => c.thing === e(502)),
85
+ );
86
+ const self = { ...m, role: "self" as const, source: e(103) };
87
+ assert.ok(
88
+ !candidatesFor(self, hits, new Set(), []).some((c) => c.target === e(3)),
89
+ "its own record's mentions",
90
+ );
91
+ });
92
+
93
+ test("a neighbour named alike is a candidate, with how it is reached", () => {
94
+ const m = mention("Mimi T.");
95
+ const neighbour = {
96
+ mention: {
97
+ record: e(200),
98
+ mention: e(7),
99
+ role: "refers",
100
+ span: "M. Tannenholt",
101
+ thing: e(507),
102
+ label: "Mira Tannenholt-Ruiz",
103
+ },
104
+ via: 'ABOUT → Deal "Northbay retrofit"',
105
+ };
106
+ const unlike = {
107
+ ...neighbour,
108
+ mention: { ...neighbour.mention, mention: e(8), span: "Northbay Water", thing: e(508) },
109
+ };
110
+ const found = candidatesFor(m, [], new Set(), [neighbour, unlike]);
111
+ assert.deepEqual(
112
+ found.map((c) => [c.target, c.found, c.via]),
113
+ [[e(7), "neighbour", 'named in ABOUT → Deal "Northbay retrofit"']],
114
+ );
115
+ const paths = neighbourPaths(
116
+ [e(1)],
117
+ [
118
+ [
119
+ {
120
+ from: e(1),
121
+ relation: "ABOUT",
122
+ to: e(2),
123
+ dir: "out",
124
+ name: "Northbay retrofit",
125
+ class: "deal",
126
+ },
127
+ ],
128
+ [{ from: e(2), relation: "INVOLVES", to: e(3), dir: "out", name: "Mira", class: "person" }],
129
+ ],
130
+ );
131
+ assert.equal(
132
+ paths.get(e(1))!.get(e(3)),
133
+ 'ABOUT → deal "Northbay retrofit" → INVOLVES → person "Mira"',
134
+ );
135
+ });
136
+
137
+ test("the stop keys are the keys more than 200 mentions hold", () => {
138
+ const counts = new Map([
139
+ ["email:team@northbay.example", 260],
140
+ ["name:part:alex", BOUNDS.stopKey],
141
+ ["name:part:mira", 3],
142
+ ]);
143
+ assert.deepEqual(stopKeys(counts), ["email:team@northbay.example"]);
144
+ });
145
+
146
+ const candidate = (n: number, over: Partial<Candidate> = {}): Candidate => ({
147
+ target: e(n),
148
+ thing: e(500 + n),
149
+ label: "Mira Tannenholt-Ruiz",
150
+ spans: ["Mira Tannenholt-Ruiz"],
151
+ records: [e(100 + n)],
152
+ score: 0.9,
153
+ found: "name:initial",
154
+ strong: false,
155
+ ...over,
156
+ });
157
+
158
+ test("without model reads a strong key links at 1 and every other candidate goes to a person", () => {
159
+ const refers = { iri: e(1), role: "refers" as const, record: null };
160
+ assert.deepEqual(
161
+ rulesDecisions(refers, [
162
+ candidate(2, { strong: true, found: "email", score: 1 }),
163
+ ]).decisions.map((d) => [d.outcome, d.confidence, d.by]),
164
+ [["link", 1, "strong"]],
165
+ );
166
+ assert.deepEqual(
167
+ rulesDecisions(refers, [
168
+ candidate(2, { strong: true }),
169
+ candidate(3, { strong: true }),
170
+ ]).decisions.map((d) => d.outcome),
171
+ ["review"],
172
+ "two things share the strong key: a person decides",
173
+ );
174
+ assert.deepEqual(
175
+ rulesDecisions(refers, [candidate(2)]).decisions.map((d) => [d.outcome, d.confidence]),
176
+ [["review", 0.9]],
177
+ );
178
+ const alone = rulesDecisions(refers, []).decisions[0]!;
179
+ assert.equal(alone.outcome, "none");
180
+ // A self mention: another record is an identity, never a link.
181
+ const self = { iri: e(1), role: "self" as const, record: e(101) };
182
+ const near = new Map([
183
+ [e(101), { near: new Set(["brightmoor foods"]) }],
184
+ [e(102), { near: new Set(["brightmoor foods"]) }],
185
+ [e(103), { near: new Set(["ridgeway transit"]) }],
186
+ ]);
187
+ const strong = rulesDecisions(self, [candidate(2, { strong: true, found: "email" })], near);
188
+ assert.deepEqual(
189
+ strong.identities.map((i) => [i.b, i.confidence]),
190
+ [[e(102), 1]],
191
+ );
192
+ assert.deepEqual(strong.decisions, []);
193
+ const byName = rulesDecisions(
194
+ self,
195
+ [candidate(2, { found: "name" }), candidate(3, { found: "name" })],
196
+ near,
197
+ );
198
+ assert.deepEqual(
199
+ byName.identities.map((i) => [i.b, i.confidence]),
200
+ [[e(102), 0.9]],
201
+ "a name and a shared related record",
202
+ );
203
+ });
204
+
205
+ test("the link rule: the smaller of the linkable kinds and the names answer, against the gate", () => {
206
+ const asked: Asked = {
207
+ position: 0,
208
+ span: "M. Tannenholt",
209
+ context: "Deal",
210
+ kind: true,
211
+ candidates: [candidate(2), candidate(3)],
212
+ };
213
+ const kind = { choice: "person", confidence: 0.9, probabilities: { person: 0.9, other: 0.1 } };
214
+ const names = {
215
+ choice: "c0",
216
+ confidence: 0.95,
217
+ probabilities: { c0: 0.95, c1: 0.03, new_thing: 0.02 },
218
+ };
219
+ const link = linkRule(e(1), asked, kind, names, 0.8);
220
+ assert.deepEqual([link.outcome, link.target, link.confidence], ["link", e(2), 0.9]);
221
+ assert.equal(linkRule(e(1), asked, kind, names, 0.95).outcome, "review");
222
+ const value = { choice: "value", confidence: 0.8, probabilities: { value: 0.8, person: 0.2 } };
223
+ const noLink = linkRule(e(1), asked, value, names, 0.9);
224
+ assert.deepEqual(
225
+ [noLink.outcome, noLink.target, noLink.confidence],
226
+ ["nolink", e(2), 0.8],
227
+ "no link below the gate offers the best candidate",
228
+ );
229
+ const newThing = {
230
+ choice: "new_thing",
231
+ confidence: 0.97,
232
+ probabilities: { new_thing: 0.97, c0: 0.02 },
233
+ };
234
+ assert.equal(
235
+ linkRule(e(1), asked, kind, newThing, 0.8).outcome,
236
+ "none",
237
+ "a confident new thing writes nothing",
238
+ );
239
+ const strong = { ...asked, kind: false, candidates: [candidate(2, { strong: true })] };
240
+ assert.equal(
241
+ linkRule(e(1), strong, undefined, { choice: "c0", confidence: 0.6 }, 0.8).confidence,
242
+ 1,
243
+ );
244
+ assert.equal(
245
+ sameRecord(
246
+ e(1),
247
+ e(2),
248
+ { choice: "same", confidence: 0.7, probabilities: { same: 0.7 } },
249
+ "name",
250
+ )?.confidence,
251
+ 0.7,
252
+ );
253
+ assert.equal(
254
+ sameRecord(
255
+ e(1),
256
+ e(2),
257
+ { choice: "different", confidence: 0.8, probabilities: { same: 0.1 } },
258
+ "name",
259
+ ),
260
+ null,
261
+ );
262
+ });
263
+
264
+ test("requests hold at most 32 mentions and carry an effect id made of their content", () => {
265
+ const asked: Asked[] = Array.from({ length: 70 }, (_, n) => ({
266
+ position: n,
267
+ span: `Name ${n}`,
268
+ context: "Note",
269
+ kind: true,
270
+ candidates: [candidate(n + 2)],
271
+ }));
272
+ const parts = packRequests({ source: "Note" }, asked);
273
+ assert.deepEqual(
274
+ parts.map((p) => p.length),
275
+ [32, 32, 6],
276
+ );
277
+ const request = decideRequest({ source: "Note" }, parts[2]!);
278
+ assert.equal(Object.keys(request.questions).length, 12);
279
+ assert.equal(effectIdOf(request), effectIdOf(decideRequest({ source: "Note" }, parts[2]!)));
280
+ assert.notEqual(effectIdOf(request), effectIdOf(decideRequest({ source: "Note 2" }, parts[2]!)));
281
+ assert.match(effectIdOf(request), /^link\/d1\/[0-9a-f]{48}$/);
282
+ });
283
+
284
+ test("an item per thing holds its rows; an identity item names its records; the rows read back", () => {
285
+ const view = (n: number) => ({
286
+ iri: e(n),
287
+ span: `Name ${n}`,
288
+ line: `the line of ${n}`,
289
+ source: e(100 + n),
290
+ sourceName: `Note ${n}`,
291
+ sourceClass: "Note",
292
+ field: "body",
293
+ });
294
+ const target: TargetView = {
295
+ iri: e(900),
296
+ anchor: view(9),
297
+ label: "Mira Tannenholt-Ruiz",
298
+ kind: "Person",
299
+ };
300
+ const rows: Row[] = Array.from({ length: 55 }, (_, n) => ({
301
+ decision: {
302
+ mention: e(n + 1),
303
+ outcome: n === 0 ? "nolink" : "review",
304
+ target: e(9),
305
+ thing: e(900),
306
+ confidence: 0.6,
307
+ by: "rule",
308
+ found: "name",
309
+ kind: null,
310
+ },
311
+ mention: view(n + 1),
312
+ }));
313
+ const items = thingItems("link", 1, target, rows);
314
+ assert.deepEqual(
315
+ items.map((i) => [itemKind(i.key!), i.facts!.triplets!.length]),
316
+ [
317
+ ["link", 50],
318
+ ["link", 4],
319
+ ["nolink", 1],
320
+ ],
321
+ );
322
+ assert.ok(items.every((i) => i.origin?.id === "starter:link" && i.origin.label === "link"));
323
+ assert.match(items[0]!.key!, /^starter:link:link:0{29}900:[0-9a-f]{12}$/);
324
+ assert.match(items[1]!.key!, /:1$/);
325
+ // The parts of the pattern of the producer `link`: the sources' classes, the thing's kind, the key kind, link or no link.
326
+ const mapping = (items[0]!.mapping as { link: Record<string, unknown> }).link;
327
+ assert.deepEqual(
328
+ [mapping.sources, mapping.kind, mapping.found, mapping.answer],
329
+ [["Note"], "Person", ["name"], "link"],
330
+ );
331
+ const back = rowsOfItem({ ...items[0]!, status: "open", suggestion_id: "x" } as never);
332
+ assert.equal(back.length, 50);
333
+ assert.deepEqual(
334
+ [back[0]!.mention.iri, back[0]!.decision.target, back[0]!.mention.line],
335
+ [e(2), e(9), "the line of 2"],
336
+ );
337
+ const identity = identityItem(
338
+ "link",
339
+ 0.99,
340
+ {
341
+ records: [e(101), e(102)],
342
+ confidence: 1,
343
+ pairs: [{ a: e(101), b: e(102), confidence: 1, reason: "the same email" }],
344
+ },
345
+ new Map([
346
+ [
347
+ e(101),
348
+ { iri: e(101), name: "Mira Ruiz", class: "Lead", self: { iri: e(1), span: "Mira Ruiz" } },
349
+ ],
350
+ [
351
+ e(102),
352
+ {
353
+ iri: e(102),
354
+ name: "Mira Tannenholt-Ruiz",
355
+ class: "Person",
356
+ self: { iri: e(2), span: "Mira Tannenholt-Ruiz" },
357
+ },
358
+ ],
359
+ ]),
360
+ )!;
361
+ assert.equal(itemKind(identity.key!), "identity");
362
+ assert.equal(identity.identities![0]!.members.length, 2);
363
+ assert.deepEqual(
364
+ (identity.mapping as { identity: { above_gate: boolean } }).identity.above_gate,
365
+ true,
366
+ );
367
+ assert.equal(identity.facts!.triplets![0]!.relation, "SAME_THING_AS");
368
+ });
@@ -0,0 +1,340 @@
1
+ // The decision (general linking 5.1 to 5.3): which thing a mention names.
2
+ // Pure functions: the questions, the state the model reads, the packing of
3
+ // the requests, the link rule, and the decision by keys and rules alone.
4
+ //
5
+ // Without model reads of record values (`modelsReadRecords: false`, the
6
+ // default, decision 1) a strong key links at confidence 1, and every other
7
+ // candidate goes to a person: the rules use the gate 1. With them, the
8
+ // decision model answers a kind question and a names question per mention
9
+ // and a same-record question per record pair; the confidence of a link is
10
+ // the smaller of the linkable kinds' probability and the names answer (the
11
+ // rule of link version 7).
12
+ import type { DecideAnswer, DecideQuestion, DecideRequest } from "@littlebigbrain/client";
13
+ import type { Candidate } from "./candidates.js";
14
+ import { hash, idOf, type IdentityPair } from "./things.js";
15
+
16
+ /** What became of a mention's decision. */
17
+ export type Outcome = "link" | "review" | "nolink" | "none";
18
+
19
+ /** A decision about one mention and one candidate. */
20
+ export interface Decision {
21
+ mention: string;
22
+ outcome: Outcome;
23
+ /** The mention a link pairs with (for `nolink`, the best candidate's). */
24
+ target: string | null;
25
+ thing: string | null;
26
+ /** The value the gate compared: the link's confidence, or the no-link answer's. */
27
+ confidence: number;
28
+ by: "strong" | "model" | "rule";
29
+ found: string | null;
30
+ /** The kind answer of a mention found in text. */
31
+ kind: string | null;
32
+ /** The names answer's probability of each candidate, by target id, the four highest. */
33
+ options?: Record<string, number>;
34
+ /** For a mention left alone: the ids of the candidates it was compared with (for the second look). */
35
+ seen?: string[];
36
+ }
37
+
38
+ /** The kinds of the kind question (5.1). */
39
+ export const KINDS: Record<string, string> = {
40
+ person: "A person, by name, initials or a nickname.",
41
+ organization: "A company, a team or another organization.",
42
+ product: "A product, a part or a service that is offered.",
43
+ place: "A place, a site or an address.",
44
+ document: "A document, a file or a contract.",
45
+ code: "A code or an identifier of something.",
46
+ value: "A value: an amount, a date, a quantity or a property of something.",
47
+ external: "Something outside the records: a standard, a law or a public name.",
48
+ other: "No name: a common word or a phrase.",
49
+ };
50
+
51
+ /** The kinds that link: every kind but `value`, `external` and `other`. */
52
+ export const LINKABLE = ["person", "organization", "product", "place", "document", "code"];
53
+
54
+ const round = (x: number) => Math.round(x * 1000) / 1000;
55
+ /** A short id in decisions and messages: the first 12 hex characters of an entity id. */
56
+ export const shortId = (iri: string) => idOf(iri).slice(0, 12);
57
+
58
+ /** Two candidates are one thing when they share a thing, else when they are one mention. */
59
+ const sameCandidate = (a: Candidate, b: Candidate) =>
60
+ (a.thing ?? a.target) === (b.thing ?? b.target);
61
+
62
+ // ---- by keys and rules ----------------------------------------------------------
63
+
64
+ /** What the rules know of a self mention's record, for an identity: the names near it and in its context fields. */
65
+ export interface RecordContext {
66
+ /** Normal forms of the names of the records it is related to and of its context field values. */
67
+ near: Set<string>;
68
+ }
69
+
70
+ /**
71
+ * The decisions of one mention by keys and rules. A `refers` mention links
72
+ * at 1 to the one thing with its strong key, and goes to a person when two
73
+ * things share it or when only a medium or a weak key, or a neighbour, found
74
+ * the best candidate. A `self` mention links at 1 to a thing without a
75
+ * record that holds its strong key; a candidate with another record is an
76
+ * identity: at 1 for a strong key, at 0.9 for a name when the records share
77
+ * a related record or a context value, else none.
78
+ */
79
+ export function rulesDecisions(
80
+ m: { iri: string; role: "self" | "refers"; record: string | null },
81
+ candidates: Candidate[],
82
+ records: Map<string, RecordContext> = new Map(),
83
+ ): { decisions: Decision[]; identities: IdentityPair[] } {
84
+ const decisions: Decision[] = [];
85
+ const identities: IdentityPair[] = [];
86
+ const decision = (c: Candidate, outcome: Outcome, confidence: number): Decision => ({
87
+ mention: m.iri,
88
+ outcome,
89
+ target: c.target,
90
+ thing: c.thing,
91
+ confidence: round(confidence),
92
+ by: c.strong && outcome === "link" ? "strong" : "rule",
93
+ found: c.found,
94
+ kind: null,
95
+ });
96
+ if (m.role === "refers") {
97
+ const strong = candidates.filter((c) => c.strong);
98
+ const distinct = strong.filter((c, i) => strong.findIndex((d) => sameCandidate(c, d)) === i);
99
+ if (distinct.length === 1) decisions.push(decision(distinct[0]!, "link", 1));
100
+ else if (distinct.length > 1) decisions.push(decision(distinct[0]!, "review", 0.5));
101
+ else if (candidates[0])
102
+ decisions.push(decision(candidates[0], "review", Math.min(candidates[0].score, 0.99)));
103
+ else decisions.push(none(m.iri, candidates));
104
+ return { decisions, identities };
105
+ }
106
+ for (const c of candidates) {
107
+ const others = c.records.filter((r) => r !== m.record);
108
+ if (others.length && m.record) {
109
+ for (const other of others) {
110
+ const identity = ruleIdentity(m.record, other, c, records);
111
+ if (identity) identities.push(identity);
112
+ }
113
+ continue;
114
+ }
115
+ if (c.records.length) continue;
116
+ if (c.strong) decisions.push(decision(c, "link", 1));
117
+ else decisions.push(decision(c, "review", Math.min(c.score, 0.99)));
118
+ }
119
+ if (!decisions.length && !identities.length) decisions.push(none(m.iri, candidates));
120
+ return { decisions, identities };
121
+ }
122
+
123
+ function ruleIdentity(
124
+ record: string,
125
+ other: string,
126
+ c: Candidate,
127
+ records: Map<string, RecordContext>,
128
+ ): IdentityPair | null {
129
+ if (c.strong) return { a: record, b: other, confidence: 1, reason: `the same ${c.found}` };
130
+ if (!(c.found === "name" || c.found === "alias")) return null;
131
+ const [x, y] = [records.get(record)?.near, records.get(other)?.near];
132
+ const shared = x && y ? [...x].find((name) => y.has(name)) : undefined;
133
+ return shared
134
+ ? { a: record, b: other, confidence: 0.9, reason: `the same name, and both name ${shared}` }
135
+ : null;
136
+ }
137
+
138
+ /** A mention left alone: no candidate, or none it names. */
139
+ export function none(mention: string, candidates: Candidate[], confidence = 1): Decision {
140
+ return {
141
+ mention,
142
+ outcome: "none",
143
+ target: null,
144
+ thing: null,
145
+ confidence: round(confidence),
146
+ by: "rule",
147
+ found: null,
148
+ kind: null,
149
+ seen: candidates.map((c) => shortId(c.target)),
150
+ };
151
+ }
152
+
153
+ // ---- by the decision model -------------------------------------------------------
154
+
155
+ /** One mention a decide request asks about. */
156
+ export interface Asked {
157
+ /** The position in the source's list: the question ids are `m<position>-kind` and `m<position>-names`. */
158
+ position: number;
159
+ span: string;
160
+ /** Where the span is: the sentence of a text field, or the field and its value. */
161
+ context: string;
162
+ /** Ask the kind question (a span found in text, not written as a key). */
163
+ kind: boolean;
164
+ candidates: Candidate[];
165
+ }
166
+
167
+ /** One record pair the same-record question asks about. */
168
+ export interface AskedPair {
169
+ id: string;
170
+ a: string;
171
+ b: string;
172
+ match: string;
173
+ }
174
+
175
+ /** The state of a source's decide requests: the source and its relations one hop out. */
176
+ export function sourceState(source: string, relations: string[]): Record<string, unknown> {
177
+ return {
178
+ source,
179
+ relations: relations.slice(0, 12),
180
+ rule: "A candidate is a thing of the graph: a record, or the mentions already linked to it.",
181
+ };
182
+ }
183
+
184
+ const describe = (c: Candidate) =>
185
+ [
186
+ c.label.slice(0, 120),
187
+ c.spans.length ? `names: ${c.spans.map((s) => s.slice(0, 60)).join(", ")}` : "",
188
+ `found by ${c.found}`,
189
+ c.via ? c.via.slice(0, 200) : "",
190
+ ]
191
+ .filter(Boolean)
192
+ .join("; ");
193
+
194
+ /** One decide request: the questions of `asked` and `pairs` with the source's state. */
195
+ export function decideRequest(
196
+ state: Record<string, unknown>,
197
+ asked: Asked[],
198
+ pairs: AskedPair[] = [],
199
+ ): DecideRequest {
200
+ const questions: Record<string, DecideQuestion> = {};
201
+ for (const { position, span, context, kind, candidates } of asked) {
202
+ if (kind)
203
+ questions[`m${position}-kind`] = {
204
+ instructions: `${context} What does the span "${span}" name here?`,
205
+ criteria: KINDS,
206
+ };
207
+ if (!candidates.length) continue;
208
+ const options: Record<string, string> = {};
209
+ candidates.forEach((c, k) => (options[`c${k}`] = describe(c)));
210
+ options.new_thing = "It names a thing that is not in this list.";
211
+ options.not_a_name = "It names no thing.";
212
+ questions[`m${position}-names`] = {
213
+ instructions: `${context} Which thing does the span "${span}" name? Choose a thing only when the span names it: its name in another spelling or order, its initials, a nickname, or one of its keys. A thing whose name is only alike, or that the text only mentions near the span, is no match.`,
214
+ criteria: options,
215
+ };
216
+ }
217
+ for (const pair of pairs)
218
+ questions[`${pair.id}-same`] = {
219
+ instructions: `Record A: ${pair.a}. Record B: ${pair.b}. They match by ${pair.match}. Are they the same thing?`,
220
+ criteria: {
221
+ same: "The two records describe one real thing.",
222
+ different: "They describe two different things.",
223
+ cannot_tell: "The records do not say.",
224
+ },
225
+ };
226
+ return { state, questions };
227
+ }
228
+
229
+ /** Estimated tokens of a request: its JSON length over four. */
230
+ export const estimatedTokens = (request: DecideRequest) =>
231
+ Math.ceil(JSON.stringify(request).length / 4);
232
+
233
+ /** Tokens of one request, at most (5.2). */
234
+ export const REQUEST_TOKENS = 28_000;
235
+ /** Mentions of one request: two questions each, 64 questions at most. */
236
+ export const REQUEST_MENTIONS = 32;
237
+
238
+ /** The mentions of one source in requests of at most 32 mentions and 28,000 estimated tokens. */
239
+ export function packRequests(state: Record<string, unknown>, asked: Asked[]): Asked[][] {
240
+ const out: Asked[][] = [];
241
+ let current: Asked[] = [];
242
+ for (const one of asked) {
243
+ const trial = [...current, one];
244
+ const size = estimatedTokens(decideRequest(state, trial));
245
+ if (current.length && (trial.length > REQUEST_MENTIONS || size > REQUEST_TOKENS)) {
246
+ out.push(current);
247
+ current = [one];
248
+ } else current = trial;
249
+ }
250
+ if (current.length) out.push(current);
251
+ return out;
252
+ }
253
+
254
+ /** The effect id of a decide request, made from its content: the same request gets its first answer. */
255
+ export const effectIdOf = (request: DecideRequest) =>
256
+ `link/d1/${hash(JSON.stringify(request)).slice(0, 48)}`;
257
+
258
+ /**
259
+ * The link rule (5.3) for one asked mention. A chosen candidate with a
260
+ * strong key links at 1. Otherwise a chosen candidate links when the
261
+ * linkable kinds have a probability of at least 0.5, at the smaller of that
262
+ * and the names answer; at or above `gate` it is written, below it a person
263
+ * decides. An answer that chose no thing has the confidence of no link; with
264
+ * a candidate below the gate it goes to a person with the best candidate.
265
+ */
266
+ export function linkRule(
267
+ mention: string,
268
+ asked: Asked,
269
+ kindAnswer: DecideAnswer | undefined,
270
+ namesAnswer: DecideAnswer | undefined,
271
+ gate: number,
272
+ ): Decision {
273
+ const probabilities = kindAnswer?.probabilities ?? {};
274
+ const pLink = !asked.kind
275
+ ? 1
276
+ : Object.keys(probabilities).length
277
+ ? LINKABLE.reduce((sum, id) => sum + (probabilities[id] ?? 0), 0)
278
+ : LINKABLE.includes(kindAnswer?.choice ?? "")
279
+ ? (kindAnswer?.confidence ?? 0)
280
+ : 1 - (kindAnswer?.confidence ?? 1);
281
+ const kind = kindAnswer?.choice ?? null;
282
+ if (!asked.candidates.length || !namesAnswer) {
283
+ const left = none(mention, asked.candidates, 1 - pLink);
284
+ return { ...left, kind, by: namesAnswer || kindAnswer ? "model" : "rule" };
285
+ }
286
+ const options: Record<string, number> = {};
287
+ const ranked = Object.entries(namesAnswer.probabilities ?? {})
288
+ .filter(([id]) => /^c\d+$/.test(id))
289
+ .sort((a, b) => b[1] - a[1]);
290
+ for (const [id, p] of ranked.slice(0, 4)) {
291
+ const c = asked.candidates[Number(id.slice(1))];
292
+ if (c) options[shortId(c.target)] = round(p);
293
+ }
294
+ const choice = namesAnswer.choice ?? "";
295
+ const namesConfidence = namesAnswer.confidence ?? 0;
296
+ const chosen = /^c\d+$/.test(choice) ? asked.candidates[Number(choice.slice(1))] : undefined;
297
+ if (chosen && (chosen.strong || pLink >= 0.5)) {
298
+ const confidence = chosen.strong ? 1 : round(Math.min(pLink, namesConfidence));
299
+ return {
300
+ mention,
301
+ outcome: confidence >= gate ? "link" : "review",
302
+ target: chosen.target,
303
+ thing: chosen.thing,
304
+ confidence,
305
+ by: chosen.strong ? "strong" : "model",
306
+ found: chosen.found,
307
+ kind,
308
+ options,
309
+ };
310
+ }
311
+ const best = ranked[0] ? asked.candidates[Number(ranked[0][0].slice(1))] : asked.candidates[0];
312
+ const confidence = round(Math.max(1 - pLink, chosen ? 0 : namesConfidence));
313
+ if (best && confidence < gate)
314
+ return {
315
+ mention,
316
+ outcome: "nolink",
317
+ target: best.target,
318
+ thing: best.thing,
319
+ confidence,
320
+ by: "model",
321
+ found: best.found,
322
+ kind,
323
+ options,
324
+ };
325
+ return { ...none(mention, asked.candidates, confidence), kind, by: "model", options };
326
+ }
327
+
328
+ /** The identity a same-record answer proposes: `same` with a probability of at least 0.5. */
329
+ export function sameRecord(
330
+ a: string,
331
+ b: string,
332
+ answer: DecideAnswer | undefined,
333
+ match: string,
334
+ ): IdentityPair | null {
335
+ const p =
336
+ answer?.probabilities?.same ?? (answer?.choice === "same" ? (answer.confidence ?? 0) : 0);
337
+ return p >= 0.5
338
+ ? { a, b, confidence: round(p), reason: `the decision model: the same, by ${match}` }
339
+ : null;
340
+ }