@littlebigbrain/create 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +55 -0
- package/README.md +71 -2
- package/bin/create-littlebigbrain.mjs +9 -0
- package/lib/create.mjs +238 -0
- package/package.json +46 -4
- package/project/setup.ts +18 -0
- package/project/try.ts +30 -0
- package/project/worker.ts +23 -0
- package/templates/_shared/testing.ts +490 -0
- package/templates/documents-link/README.md +144 -0
- package/templates/documents-link/link.ts +580 -0
- package/templates/documents-link/samples/ausstattungsliste.pdf +76 -0
- package/templates/documents-link/samples/baubeschreibung.pdf +54 -0
- package/templates/documents-link/samples/expected.json +404 -0
- package/templates/documents-link/samples/pages/ausstattungsliste.json +1150 -0
- package/templates/documents-link/samples/pages/baubeschreibung.json +802 -0
- package/templates/documents-link/samples/pages/raumblaetter.json +1580 -0
- package/templates/documents-link/samples/raumblaetter.pdf +109 -0
- package/templates/documents-link/samples.ts +31 -0
- package/templates/documents-link/workflow.test.ts +470 -0
- package/templates/documents-link/workflow.ts +457 -0
- package/templates/ontology-fit/README.md +81 -0
- package/templates/ontology-fit/samples/expected.md +88 -0
- package/templates/ontology-fit/samples/texts.json +22 -0
- package/templates/ontology-fit/samples.ts +40 -0
- package/templates/ontology-fit/workflow.test.ts +264 -0
- package/templates/ontology-fit/workflow.ts +413 -0
- package/templates/values-extract/README.md +81 -0
- package/templates/values-extract/samples/expected.json +148 -0
- package/templates/values-extract/samples/records.json +26 -0
- package/templates/values-extract/samples.ts +27 -0
- package/templates/values-extract/workflow.test.ts +256 -0
- package/templates/values-extract/workflow.ts +411 -0
- package/templates/versions.json +3 -0
|
@@ -0,0 +1,580 @@
|
|
|
1
|
+
// The parts of documents-link that need no I/O: the page listing the models
|
|
2
|
+
// read, the prompts and their checks, the candidates of a mention, the
|
|
3
|
+
// decision model's questions, the link rule, and the facts the workflow
|
|
4
|
+
// writes. Adapted from the LBB-run `documents.link@v1`
|
|
5
|
+
// (database/crates/lbb-graph-query/src/documents/link.rs and link_store.rs).
|
|
6
|
+
import {
|
|
7
|
+
entityRef,
|
|
8
|
+
type DecideAnswer,
|
|
9
|
+
type DecideQuestion,
|
|
10
|
+
type DecideRequest,
|
|
11
|
+
type FilePages,
|
|
12
|
+
type OwnedWrite,
|
|
13
|
+
type ReasonRequest,
|
|
14
|
+
type Schemas,
|
|
15
|
+
} from "@littlebigbrain/client";
|
|
16
|
+
|
|
17
|
+
export type FilePage = FilePages["pages"][number];
|
|
18
|
+
export type FileLine = FilePage["lines"][number];
|
|
19
|
+
|
|
20
|
+
/** An identifier a page defines, on the line that holds it. */
|
|
21
|
+
export interface Definition {
|
|
22
|
+
line: string;
|
|
23
|
+
identifier: string;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** A span of a line that is, or may be, a code. */
|
|
27
|
+
export interface Mention {
|
|
28
|
+
line: string;
|
|
29
|
+
span: string;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** One entry of the catalogue: an identifier a document defines. */
|
|
33
|
+
export interface Entry {
|
|
34
|
+
identifier: string;
|
|
35
|
+
context: string;
|
|
36
|
+
line: string;
|
|
37
|
+
document: string;
|
|
38
|
+
documentName: string;
|
|
39
|
+
sha256: string;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** What the decision model answered about one mention, kept small. */
|
|
43
|
+
export interface Resolved {
|
|
44
|
+
kind: string | null;
|
|
45
|
+
/** The probability that the span can link at all. */
|
|
46
|
+
pLink: number;
|
|
47
|
+
/** The catalogue index of the entry the names answer chose. */
|
|
48
|
+
target: number | null;
|
|
49
|
+
confidence: number;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** A code or a span longer than this is no code. */
|
|
53
|
+
const CODE_MAX_CHARS = 200;
|
|
54
|
+
/** Characters of an entry's context. */
|
|
55
|
+
export const CONTEXT_CHARS = 700;
|
|
56
|
+
const CANDIDATES_BY_CHARACTERS = 30;
|
|
57
|
+
const CANDIDATES_BY_CONTEXT = 10;
|
|
58
|
+
/** Mentions of one decide request: two questions each, 64 at most. */
|
|
59
|
+
const REQUEST_MENTIONS = 32;
|
|
60
|
+
/** Characters of one decide request (state and questions), well under the route's 200,000. */
|
|
61
|
+
const REQUEST_CHARS = 60_000;
|
|
62
|
+
|
|
63
|
+
// ---- the page as the models read it ---------------------------------------
|
|
64
|
+
|
|
65
|
+
/** The lines of a page grouped by baseline, top to bottom, each row left to right. */
|
|
66
|
+
export function layoutRows(lines: FileLine[]): FileLine[][] {
|
|
67
|
+
const rows: { top: number; bottom: number; lines: FileLine[] }[] = [];
|
|
68
|
+
const sorted = [...lines].sort((a, b) => a.box[1]! - b.box[1]! || a.box[0]! - b.box[0]!);
|
|
69
|
+
for (const line of sorted) {
|
|
70
|
+
const middle = (line.box[1]! + line.box[3]!) / 2;
|
|
71
|
+
const row = rows.find((r) => r.top <= middle && middle <= r.bottom);
|
|
72
|
+
if (row) row.lines.push(line);
|
|
73
|
+
else rows.push({ top: line.box[1]!, bottom: line.box[3]!, lines: [line] });
|
|
74
|
+
}
|
|
75
|
+
return rows.map((row) => row.lines.sort((a, b) => a.box[0]! - b.box[0]!));
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export const rowText = (row: FileLine[]): string => row.map((line) => line.text).join(" | ");
|
|
79
|
+
|
|
80
|
+
/** The text of the row that holds a line. */
|
|
81
|
+
export const rowOf = (lines: FileLine[], id: string): string =>
|
|
82
|
+
rowText(layoutRows(lines).find((row) => row.some((line) => line.id === id)) ?? []);
|
|
83
|
+
|
|
84
|
+
/** A page as the models read it: one row per text line, `[line id] text | [line id] text`. */
|
|
85
|
+
export function listing(lines: FileLine[]): string {
|
|
86
|
+
return layoutRows(lines)
|
|
87
|
+
.map((row) => row.map((line) => `[${line.id}] ${line.text}`).join(" | "))
|
|
88
|
+
.join("\n");
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// ---- the reasoning model's calls ------------------------------------------
|
|
92
|
+
|
|
93
|
+
const DEFINITIONS_PROMPT =
|
|
94
|
+
"Below are the text lines of one page, each with a line id. Lines that share a baseline on the page are on one row, separated by ' | '. Some entries define an identifier: a short code that other documents use to refer to this entry, followed by its description, which can wrap onto the next lines. List every identifier that is defined. row_id is the id of the line that holds the identifier. Copy the identifier exactly as it is written. The description is the entry's text, joined across its lines, at most 200 characters. An identifier can be defined in a table, a list, a numbered heading, or inside a sentence (for example 'Item-Type 07 - a cabinet for …'). Do not list the entries of a table of contents, local list markers such as 'a)' or '1.', page numbers, or references to definitions that are made in other documents.";
|
|
95
|
+
const MENTIONS_PROMPT =
|
|
96
|
+
"Below are the text lines of one page, each with a line id. Lines that share a baseline on the page are on one row, separated by ' | '. List every span that is, or might be, a reference code: a short identifier that points to a definition elsewhere (for example an item, product, position, class or annex code). Copy each span exactly as it is written. Give one code per entry: split lists such as 'A1, A2' into separate entries. When you are not sure whether a span is a code, list it. Do not list plain numbers, prices, dates or page numbers.";
|
|
97
|
+
|
|
98
|
+
const listOf = (key: string, fields: Record<string, { type: string }>) => ({
|
|
99
|
+
type: "object",
|
|
100
|
+
additionalProperties: false,
|
|
101
|
+
required: [key],
|
|
102
|
+
properties: {
|
|
103
|
+
[key]: {
|
|
104
|
+
type: "array",
|
|
105
|
+
items: {
|
|
106
|
+
type: "object",
|
|
107
|
+
additionalProperties: false,
|
|
108
|
+
required: Object.keys(fields),
|
|
109
|
+
properties: fields,
|
|
110
|
+
},
|
|
111
|
+
},
|
|
112
|
+
},
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
/** The definitions call of one page. */
|
|
116
|
+
export function definitionsRequest(page: FilePage): ReasonRequest {
|
|
117
|
+
return {
|
|
118
|
+
system: "You read documents and find the entries that define identifiers.",
|
|
119
|
+
prompt: `${DEFINITIONS_PROMPT}\n\n${listing(page.lines)}`,
|
|
120
|
+
schema: listOf("definitions", {
|
|
121
|
+
row_id: { type: "string" },
|
|
122
|
+
identifier: { type: "string" },
|
|
123
|
+
description: { type: "string" },
|
|
124
|
+
}),
|
|
125
|
+
max_tokens: 16_000,
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/** The mentions call of one page. */
|
|
130
|
+
export function mentionsRequest(page: FilePage): ReasonRequest {
|
|
131
|
+
return {
|
|
132
|
+
system: "You read document pages and find reference codes.",
|
|
133
|
+
prompt: `${MENTIONS_PROMPT}\n\n${listing(page.lines)}`,
|
|
134
|
+
schema: listOf("mentions", { line_id: { type: "string" }, span: { type: "string" } }),
|
|
135
|
+
max_tokens: 16_000,
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
const items = (answer: unknown, key: string): Record<string, unknown>[] => {
|
|
140
|
+
const list = (answer as Record<string, unknown> | null)?.[key];
|
|
141
|
+
return Array.isArray(list) ? list.filter((item) => item && typeof item === "object") : [];
|
|
142
|
+
};
|
|
143
|
+
const text = (value: unknown): string => (typeof value === "string" ? value.normalize("NFC") : "");
|
|
144
|
+
|
|
145
|
+
/** A code is kept only when the line the model cited holds it. */
|
|
146
|
+
function inLine(page: FilePage, line: string, code: string): boolean {
|
|
147
|
+
if (!code.trim() || code.length > CODE_MAX_CHARS) return false;
|
|
148
|
+
return page.lines.some((l) => l.id === line && l.text.includes(code));
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/** The definitions of an answer whose identifier is in the line they cite. */
|
|
152
|
+
export function checkDefinitions(answer: unknown, page: FilePage): Definition[] {
|
|
153
|
+
return items(answer, "definitions")
|
|
154
|
+
.map((item) => ({ line: text(item.row_id), identifier: text(item.identifier) }))
|
|
155
|
+
.filter((d) => inLine(page, d.line, d.identifier));
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/** The mentions of an answer whose span is in the line they cite. */
|
|
159
|
+
export function checkMentions(answer: unknown, page: FilePage): Mention[] {
|
|
160
|
+
const seen = new Set<string>();
|
|
161
|
+
return items(answer, "mentions")
|
|
162
|
+
.map((item) => ({ line: text(item.line_id), span: text(item.span) }))
|
|
163
|
+
.filter(
|
|
164
|
+
(m) =>
|
|
165
|
+
inLine(page, m.line, m.span) &&
|
|
166
|
+
!seen.has(`${m.line}\u0000${m.span}`) &&
|
|
167
|
+
seen.add(`${m.line}\u0000${m.span}`),
|
|
168
|
+
);
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
// ---- entries ---------------------------------------------------------------
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* The key an identifier matches by: no whitespace, lower case, and no dot
|
|
175
|
+
* between leading letters and a digit (`S.4.1`, `s 4.1` and `S4.1` match).
|
|
176
|
+
*/
|
|
177
|
+
export function identifierKey(identifier: string): string {
|
|
178
|
+
const squashed = identifier.replace(/\s+/g, "").toLowerCase();
|
|
179
|
+
return squashed.replace(/^(\p{L}+)\.(?=\d)/u, "$1");
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* The context of each definition: the text of its row up to the row of the
|
|
184
|
+
* next definition, at most 700 characters.
|
|
185
|
+
*/
|
|
186
|
+
export function contextsFor(pages: FilePage[], definitions: Definition[]): string[] {
|
|
187
|
+
const rows: string[] = [];
|
|
188
|
+
const rowOf = new Map<string, number>();
|
|
189
|
+
for (const page of pages)
|
|
190
|
+
for (const row of layoutRows(page.lines)) {
|
|
191
|
+
for (const line of row) rowOf.set(line.id, rows.length);
|
|
192
|
+
rows.push(rowText(row));
|
|
193
|
+
}
|
|
194
|
+
const starts = [...new Set(definitions.map((d) => rowOf.get(d.line) ?? -1))].sort(
|
|
195
|
+
(a, b) => a - b,
|
|
196
|
+
);
|
|
197
|
+
return definitions.map((d) => {
|
|
198
|
+
const start = rowOf.get(d.line);
|
|
199
|
+
if (start === undefined) return d.identifier;
|
|
200
|
+
const end = starts.find((at) => at > start) ?? rows.length;
|
|
201
|
+
return rows.slice(start, end).join(" ").slice(0, CONTEXT_CHARS);
|
|
202
|
+
});
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/** A stable order of the catalogue: by document, line (`p2-l10` after `p2-l9`) and identifier. */
|
|
206
|
+
export const order = (e: Entry): string =>
|
|
207
|
+
[e.documentName, e.sha256, e.line.replace(/\d+/g, (n) => n.padStart(6, "0")), e.identifier].join(
|
|
208
|
+
"\u0000",
|
|
209
|
+
);
|
|
210
|
+
|
|
211
|
+
/** A part of a review item's key: lowercase letters, digits and dots, other runs as one dash. */
|
|
212
|
+
export const keyPart = (text: string): string =>
|
|
213
|
+
identifierKey(text)
|
|
214
|
+
.replace(/[^a-z0-9.]+/g, "-")
|
|
215
|
+
.replace(/^-+|-+$/g, "")
|
|
216
|
+
.slice(0, 48) || "x";
|
|
217
|
+
|
|
218
|
+
// ---- candidates ------------------------------------------------------------
|
|
219
|
+
|
|
220
|
+
function levenshtein(a: string[], b: string[]): number {
|
|
221
|
+
let previous = Array.from({ length: b.length + 1 }, (_, i) => i);
|
|
222
|
+
for (let i = 1; i <= a.length; i++) {
|
|
223
|
+
const current = [i];
|
|
224
|
+
for (let j = 1; j <= b.length; j++)
|
|
225
|
+
current[j] = Math.min(
|
|
226
|
+
previous[j]! + 1,
|
|
227
|
+
current[j - 1]! + 1,
|
|
228
|
+
previous[j - 1]! + (a[i - 1] === b[j - 1] ? 0 : 1),
|
|
229
|
+
);
|
|
230
|
+
previous = current;
|
|
231
|
+
}
|
|
232
|
+
return previous[b.length]!;
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
/** 1 for equal strings, 0 for strings with nothing in common (case folded). */
|
|
236
|
+
function similarity(a: string, b: string): number {
|
|
237
|
+
const [x, y] = [[...a.toLowerCase()], [...b.toLowerCase()]];
|
|
238
|
+
const longest = Math.max(x.length, y.length);
|
|
239
|
+
return longest === 0 ? 1 : 1 - levenshtein(x, y) / longest;
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
/** Whether `text` holds `span` as a word (case folded). */
|
|
243
|
+
export function holdsAsWord(text: string, span: string): boolean {
|
|
244
|
+
const escaped = span.toLowerCase().replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
245
|
+
if (!escaped) return false;
|
|
246
|
+
return new RegExp(`(?<![\\p{L}\\p{N}_.])${escaped}(?![\\p{L}\\p{N}_]|\\.\\d)`, "u").test(
|
|
247
|
+
text.toLowerCase(),
|
|
248
|
+
);
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
/**
|
|
252
|
+
* The candidates of a mention, as catalogue indexes: the 30 entries closest
|
|
253
|
+
* in characters (an entry with the span's identifier key first, then by
|
|
254
|
+
* normalized Levenshtein), up to 10 whose context holds the span as a word,
|
|
255
|
+
* then the ones closest in meaning. A mention never links to the entry its
|
|
256
|
+
* own line defines.
|
|
257
|
+
*/
|
|
258
|
+
export function candidatesFor(
|
|
259
|
+
mention: Mention,
|
|
260
|
+
document: string,
|
|
261
|
+
catalogue: Entry[],
|
|
262
|
+
byMeaning: number[],
|
|
263
|
+
): number[] {
|
|
264
|
+
const key = identifierKey(mention.span);
|
|
265
|
+
const scored = catalogue.map((entry, index) => ({
|
|
266
|
+
index,
|
|
267
|
+
score: identifierKey(entry.identifier) === key ? 2 : similarity(mention.span, entry.identifier),
|
|
268
|
+
}));
|
|
269
|
+
scored.sort((a, b) => b.score - a.score || b.index - a.index);
|
|
270
|
+
const out = scored.slice(0, CANDIDATES_BY_CHARACTERS).map((s) => s.index);
|
|
271
|
+
const limit = CANDIDATES_BY_CHARACTERS + CANDIDATES_BY_CONTEXT;
|
|
272
|
+
catalogue.forEach((entry, index) => {
|
|
273
|
+
if (out.length < limit && !out.includes(index) && holdsAsWord(entry.context, mention.span))
|
|
274
|
+
out.push(index);
|
|
275
|
+
});
|
|
276
|
+
for (const index of byMeaning) if (!out.includes(index)) out.push(index);
|
|
277
|
+
return out.filter(
|
|
278
|
+
(i) => !(catalogue[i]!.document === document && catalogue[i]!.line === mention.line),
|
|
279
|
+
);
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
/** The `k` entries whose unit vectors are closest to `query`. */
|
|
283
|
+
export function nearest(query: number[], vectors: number[][], k = 10): number[] {
|
|
284
|
+
const unit = (v: number[]) => {
|
|
285
|
+
const norm = Math.hypot(...v) || 1;
|
|
286
|
+
return v.map((x) => x / norm);
|
|
287
|
+
};
|
|
288
|
+
const q = unit(query);
|
|
289
|
+
return vectors
|
|
290
|
+
.map((v, index) => ({ index, score: unit(v).reduce((sum, x, i) => sum + x * (q[i] ?? 0), 0) }))
|
|
291
|
+
.sort((a, b) => b.score - a.score || a.index - b.index)
|
|
292
|
+
.slice(0, k)
|
|
293
|
+
.map((s) => s.index);
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
// ---- the decision model's questions -----------------------------------------
|
|
297
|
+
|
|
298
|
+
/** The kind question's options. */
|
|
299
|
+
export const KINDS: Record<string, string> = {
|
|
300
|
+
identifier:
|
|
301
|
+
"An identifier or code that a project document defines, in any spelling. The line points to that entry.",
|
|
302
|
+
abbreviation: "An abbreviation of a term, a system or a group name. It is not a code.",
|
|
303
|
+
value: "A value or property of something: a class, a rating, a dimension, a quantity or a grade.",
|
|
304
|
+
own_number:
|
|
305
|
+
"The number of the entry that this page or row is about, for example its own position number.",
|
|
306
|
+
external:
|
|
307
|
+
"A reference to an annex, a standard, a law or another document that is outside the project's definitions.",
|
|
308
|
+
};
|
|
309
|
+
/** The kinds that link. The LBB-run version also links own numbers, through a same-item question. */
|
|
310
|
+
const LINKABLE = ["identifier", "abbreviation"];
|
|
311
|
+
|
|
312
|
+
/** One mention a decide request asks about. */
|
|
313
|
+
export interface Asked {
|
|
314
|
+
position: number;
|
|
315
|
+
mention: Mention;
|
|
316
|
+
lineText: string;
|
|
317
|
+
candidates: number[];
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
/** The tag of each document in the catalogue: `D1`, `D2`, … */
|
|
321
|
+
export const tags = (catalogue: Entry[]): Map<string, string> =>
|
|
322
|
+
new Map([...new Set(catalogue.map((e) => e.document))].map((doc, i) => [doc, `D${i + 1}`]));
|
|
323
|
+
|
|
324
|
+
/** The state of a page's decide requests: the document, the page's rows and the catalogue's documents. */
|
|
325
|
+
export function pageState(
|
|
326
|
+
name: string,
|
|
327
|
+
page: FilePage,
|
|
328
|
+
catalogue: Entry[],
|
|
329
|
+
): Record<string, unknown> {
|
|
330
|
+
const names = [...tags(catalogue)].map(
|
|
331
|
+
([doc, tag]) => `[${tag}] ${catalogue.find((e) => e.document === doc)!.documentName}`,
|
|
332
|
+
);
|
|
333
|
+
return {
|
|
334
|
+
document: `${name}, one page. Each row holds the lines that share a baseline, left to right, as '[line id] text'.`,
|
|
335
|
+
page: page.page + 1,
|
|
336
|
+
rows: listing(page.lines).split("\n"),
|
|
337
|
+
catalogue: `Each candidate entry is defined in a document of the project: ${names.join(", ")}.`,
|
|
338
|
+
};
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
/** One decide request: the kind and names questions of `asked`, with the candidates' contexts in the state. */
|
|
342
|
+
export function decideRequest(
|
|
343
|
+
state: Record<string, unknown>,
|
|
344
|
+
asked: Asked[],
|
|
345
|
+
catalogue: Entry[],
|
|
346
|
+
): DecideRequest {
|
|
347
|
+
const tag = tags(catalogue);
|
|
348
|
+
const entries: Record<string, string> = {};
|
|
349
|
+
const questions: Record<string, DecideQuestion> = {};
|
|
350
|
+
for (const { position, mention, lineText, candidates } of asked) {
|
|
351
|
+
const line = `Line ${mention.line} reads: "${lineText}".`;
|
|
352
|
+
questions[`m${position}-kind`] = {
|
|
353
|
+
instructions: `${line} What is the span "${mention.span}" in this line?`,
|
|
354
|
+
criteria: KINDS,
|
|
355
|
+
};
|
|
356
|
+
if (!candidates.length) continue;
|
|
357
|
+
const options: Record<string, string> = {};
|
|
358
|
+
for (const index of candidates) {
|
|
359
|
+
const entry = catalogue[index]!;
|
|
360
|
+
options[`e${index}`] =
|
|
361
|
+
`[${tag.get(entry.document)}] ${entry.identifier}: ${entry.context.slice(0, 120)}`;
|
|
362
|
+
entries[`e${index}`] = entry.context;
|
|
363
|
+
}
|
|
364
|
+
options.none_of_these = "The span is not a name for any of these entries.";
|
|
365
|
+
questions[`m${position}-names`] = {
|
|
366
|
+
instructions: `${line} Which entry is the span "${mention.span}" a name for? The state's entries give each candidate's full text. Choose an entry only when the span is that entry's identifier, in any spelling, an abbreviation that the entry defines or is titled by, or the number that the entry's text says it describes. An entry whose text only contains the span, uses the same value, or cites it (for example 'see annex X') is not a match.`,
|
|
367
|
+
criteria: options,
|
|
368
|
+
};
|
|
369
|
+
}
|
|
370
|
+
return { state: { ...state, entries }, questions };
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
/** The mentions of a page in requests of at most 32 mentions and 60,000 characters. */
|
|
374
|
+
export function packRequests(
|
|
375
|
+
state: Record<string, unknown>,
|
|
376
|
+
asked: Asked[],
|
|
377
|
+
catalogue: Entry[],
|
|
378
|
+
): Asked[][] {
|
|
379
|
+
const out: Asked[][] = [];
|
|
380
|
+
let current: Asked[] = [];
|
|
381
|
+
for (const one of asked) {
|
|
382
|
+
const trial = [...current, one];
|
|
383
|
+
const size = JSON.stringify(decideRequest(state, trial, catalogue)).length;
|
|
384
|
+
if (current.length && (trial.length > REQUEST_MENTIONS || size > REQUEST_CHARS)) {
|
|
385
|
+
out.push(current);
|
|
386
|
+
current = [one];
|
|
387
|
+
} else current = trial;
|
|
388
|
+
}
|
|
389
|
+
if (current.length) out.push(current);
|
|
390
|
+
return out;
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
// ---- the link rule -------------------------------------------------------------
|
|
394
|
+
|
|
395
|
+
const entryOf = (option: string | null | undefined): number | null =>
|
|
396
|
+
option && /^e\d+$/.test(option) ? Number(option.slice(1)) : null;
|
|
397
|
+
|
|
398
|
+
/** The kind and names answers of one mention, kept small. */
|
|
399
|
+
export function resolve(kind: DecideAnswer | undefined, names: DecideAnswer | undefined): Resolved {
|
|
400
|
+
const probabilities = kind?.probabilities ?? {};
|
|
401
|
+
const pLink = Object.keys(probabilities).length
|
|
402
|
+
? LINKABLE.reduce((sum, id) => sum + (probabilities[id] ?? 0), 0)
|
|
403
|
+
: LINKABLE.includes(kind?.choice ?? "")
|
|
404
|
+
? (kind?.confidence ?? 0)
|
|
405
|
+
: 1 - (kind?.confidence ?? 1);
|
|
406
|
+
return {
|
|
407
|
+
kind: kind?.choice ?? null,
|
|
408
|
+
pLink: Math.round(pLink * 1000) / 1000,
|
|
409
|
+
target: entryOf(names?.choice),
|
|
410
|
+
confidence: names?.confidence ?? 0,
|
|
411
|
+
};
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
/**
|
|
415
|
+
* A mention links when its linkable kinds have a probability of at least 0.5
|
|
416
|
+
* and the names answer chose an entry; the confidence is the smaller of the
|
|
417
|
+
* two. Otherwise the confidence is that of no link.
|
|
418
|
+
*/
|
|
419
|
+
export function linkRule(r: Resolved): { target: number | null; confidence: number } {
|
|
420
|
+
if (r.target !== null && r.pLink >= 0.5)
|
|
421
|
+
return { target: r.target, confidence: Math.min(r.pLink, r.confidence) };
|
|
422
|
+
return { target: null, confidence: Math.max(1 - r.pLink, r.target === null ? r.confidence : 0) };
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
/** The box of a span on its line: the fewest words whose joined text holds it, else the line's box. */
|
|
426
|
+
export function spanBox(line: FileLine, span: string): number[] {
|
|
427
|
+
const joined = (from: number, to: number) =>
|
|
428
|
+
line.words
|
|
429
|
+
.slice(from, to + 1)
|
|
430
|
+
.map((w) => w.text)
|
|
431
|
+
.join(" ");
|
|
432
|
+
for (let end = 0; end < line.words.length; end++) {
|
|
433
|
+
if (!joined(0, end).includes(span)) continue;
|
|
434
|
+
let start = end;
|
|
435
|
+
while (start > 0 && !joined(start, end).includes(span)) start--;
|
|
436
|
+
const boxes = line.words.slice(start, end + 1).map((w) => w.box);
|
|
437
|
+
const edge = (i: number, pick: (...v: number[]) => number) => pick(...boxes.map((b) => b[i]!));
|
|
438
|
+
return [edge(0, Math.min), edge(1, Math.min), edge(2, Math.max), edge(3, Math.max)];
|
|
439
|
+
}
|
|
440
|
+
return line.box;
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
// ---- the facts ---------------------------------------------------------------
|
|
444
|
+
|
|
445
|
+
/** A document as the facts name it. */
|
|
446
|
+
export interface Doc {
|
|
447
|
+
iri: string;
|
|
448
|
+
name: string;
|
|
449
|
+
sha256: string;
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
/** The fields of an entry and of a mention, owned by the workflow. */
|
|
453
|
+
export const ENTRY_FIELDS = ["identifier", "context", "line_id", "page", "x0", "y0", "x1", "y1"];
|
|
454
|
+
export const MENTION_FIELDS = [
|
|
455
|
+
"span",
|
|
456
|
+
"kind",
|
|
457
|
+
"line_id",
|
|
458
|
+
"line_text",
|
|
459
|
+
"page",
|
|
460
|
+
"x0",
|
|
461
|
+
"y0",
|
|
462
|
+
"x1",
|
|
463
|
+
"y1",
|
|
464
|
+
];
|
|
465
|
+
|
|
466
|
+
const boxOf = (page: number, box: number[]) => ({
|
|
467
|
+
page,
|
|
468
|
+
x0: box[0]!,
|
|
469
|
+
y0: box[1]!,
|
|
470
|
+
x1: box[2]!,
|
|
471
|
+
y1: box[3]!,
|
|
472
|
+
});
|
|
473
|
+
const entryRef = (sha256: string, identifier: string) => ({
|
|
474
|
+
type: "DocEntry",
|
|
475
|
+
name: identifier,
|
|
476
|
+
key: `entry:${sha256}:${identifier}`,
|
|
477
|
+
});
|
|
478
|
+
const mentionRef = (doc: Doc, m: Mention) => ({
|
|
479
|
+
type: "DocMention",
|
|
480
|
+
name: m.span,
|
|
481
|
+
key: `mention:${doc.sha256}:${m.line}:${m.span}`,
|
|
482
|
+
});
|
|
483
|
+
const lineOf = (pages: FilePage[], id: string) => {
|
|
484
|
+
const page = pages.find((p) => p.lines.some((l) => l.id === id))!;
|
|
485
|
+
return { page, line: page.lines.find((l) => l.id === id)! };
|
|
486
|
+
};
|
|
487
|
+
|
|
488
|
+
/** The window's entries `DEFINED_IN` the document, one per identifier with the longest context. */
|
|
489
|
+
export function entriesWrite(doc: Doc, pages: FilePage[], found: Definition[]): OwnedWrite {
|
|
490
|
+
const contexts = contextsFor(pages, found);
|
|
491
|
+
const entries = new Map<string, Schemas["EntityPropertiesInput"]>();
|
|
492
|
+
found.forEach((d, i) => {
|
|
493
|
+
const before = entries.get(d.identifier)?.properties as { context?: string } | undefined;
|
|
494
|
+
if ((before?.context?.length ?? -1) >= contexts[i]!.length) return;
|
|
495
|
+
const { page, line } = lineOf(pages, d.line);
|
|
496
|
+
const fields = { identifier: d.identifier, context: contexts[i]!, line_id: d.line };
|
|
497
|
+
entries.set(d.identifier, {
|
|
498
|
+
...entryRef(doc.sha256, d.identifier),
|
|
499
|
+
properties: { ...fields, ...boxOf(page.page, line.box) },
|
|
500
|
+
});
|
|
501
|
+
});
|
|
502
|
+
const subjects = [...entries.keys()].map((identifier) => entryRef(doc.sha256, identifier));
|
|
503
|
+
const target = entityRef(doc.iri, "Document", doc.name);
|
|
504
|
+
return {
|
|
505
|
+
subjects,
|
|
506
|
+
relations: ["DEFINED_IN"],
|
|
507
|
+
properties: ENTRY_FIELDS,
|
|
508
|
+
entity_properties: [...entries.values()],
|
|
509
|
+
triplets: subjects.map((source) => ({ source, relation: "DEFINED_IN", target })),
|
|
510
|
+
};
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
/** A mention's link to an entry: the line is the evidence, the span's box its region. */
|
|
514
|
+
export function refersTo(
|
|
515
|
+
doc: Doc,
|
|
516
|
+
pages: FilePage[],
|
|
517
|
+
m: Mention,
|
|
518
|
+
entry: Entry,
|
|
519
|
+
confidence: number,
|
|
520
|
+
): Schemas["TripletInput"] {
|
|
521
|
+
const { page, line } = lineOf(pages, m.line);
|
|
522
|
+
const region = { ...boxOf(page.page, spanBox(line, m.span)), confidence };
|
|
523
|
+
return {
|
|
524
|
+
source: mentionRef(doc, m),
|
|
525
|
+
relation: "REFERS_TO",
|
|
526
|
+
target: entryRef(entry.sha256, entry.identifier),
|
|
527
|
+
confidence,
|
|
528
|
+
evidence: { text: line.text, source_id: doc.name, region },
|
|
529
|
+
};
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
/** The window's mentions `MENTIONED_IN` the document, with their kind, box and written links. */
|
|
533
|
+
export function mentionsWrite(
|
|
534
|
+
doc: Doc,
|
|
535
|
+
pages: FilePage[],
|
|
536
|
+
mentions: { m: Mention; r: Resolved }[],
|
|
537
|
+
links: { m: Mention; entry: Entry; confidence: number }[],
|
|
538
|
+
): OwnedWrite {
|
|
539
|
+
const target = entityRef(doc.iri, "Document", doc.name);
|
|
540
|
+
const rows = new Map<string, Schemas["EntityPropertiesInput"]>();
|
|
541
|
+
for (const { m, r } of mentions) {
|
|
542
|
+
const { page, line } = lineOf(pages, m.line);
|
|
543
|
+
const fields = {
|
|
544
|
+
span: m.span,
|
|
545
|
+
line_id: m.line,
|
|
546
|
+
line_text: line.text,
|
|
547
|
+
...(r.kind ? { kind: r.kind } : {}),
|
|
548
|
+
};
|
|
549
|
+
rows.set(m.line + m.span, {
|
|
550
|
+
...mentionRef(doc, m),
|
|
551
|
+
properties: { ...fields, ...boxOf(page.page, spanBox(line, m.span)) },
|
|
552
|
+
});
|
|
553
|
+
}
|
|
554
|
+
const subjects = [...rows.values()].map(({ type, name, key }) => ({ type, name, key: key! }));
|
|
555
|
+
return {
|
|
556
|
+
subjects,
|
|
557
|
+
relations: ["MENTIONED_IN", "REFERS_TO"],
|
|
558
|
+
properties: MENTION_FIELDS,
|
|
559
|
+
entity_properties: [...rows.values()],
|
|
560
|
+
triplets: [
|
|
561
|
+
...subjects.map((source) => ({ source, relation: "MENTIONED_IN", target })),
|
|
562
|
+
...links.map((l) => refersTo(doc, pages, l.m, l.entry, l.confidence)),
|
|
563
|
+
],
|
|
564
|
+
};
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
/** The entities one commit of the workflow names at most: the route takes 1,000. */
|
|
568
|
+
export const PART = 900;
|
|
569
|
+
|
|
570
|
+
/** Part `n` of a write: its subjects `n * PART` to `(n + 1) * PART`, with their fields and edges. */
|
|
571
|
+
export function part(write: OwnedWrite, n: number): OwnedWrite {
|
|
572
|
+
const subjects = write.subjects.slice(n * PART, (n + 1) * PART);
|
|
573
|
+
const keys = new Set(subjects.map((s) => s.key));
|
|
574
|
+
return {
|
|
575
|
+
...write,
|
|
576
|
+
subjects,
|
|
577
|
+
entity_properties: write.entity_properties?.filter((e) => keys.has(e.key)),
|
|
578
|
+
triplets: write.triplets?.filter((t) => keys.has(t.source.key)),
|
|
579
|
+
};
|
|
580
|
+
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
%PDF-1.4
|
|
2
|
+
1 0 obj
|
|
3
|
+
<< /Type /Catalog /Pages 2 0 R >>
|
|
4
|
+
endobj
|
|
5
|
+
2 0 obj
|
|
6
|
+
<< /Type /Pages /Kids [7 0 R] /Count 1 >>
|
|
7
|
+
endobj
|
|
8
|
+
3 0 obj
|
|
9
|
+
<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>
|
|
10
|
+
endobj
|
|
11
|
+
4 0 obj
|
|
12
|
+
<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding >>
|
|
13
|
+
endobj
|
|
14
|
+
5 0 obj
|
|
15
|
+
<< /Title (Ausstattungsliste) /Producer (make-samples.mjs) >>
|
|
16
|
+
endobj
|
|
17
|
+
6 0 obj
|
|
18
|
+
<< /Length 2242 >>
|
|
19
|
+
stream
|
|
20
|
+
0.35 g BT /F1 8 Tf 1 0 0 1 72 806 Tm (Neubau Grundschule Musterstadt) Tj ET 0 g
|
|
21
|
+
0.6 G 0.5 w 72 798 m 523 798 l S 0 G
|
|
22
|
+
BT /F2 16 Tf 1 0 0 1 72 760 Tm (Ausstattungsliste) Tj ET
|
|
23
|
+
BT /F1 10 Tf 1 0 0 1 72 740 Tm (Ausstattung der R\344ume. Die Raumbl\344tter nennen die Positionen dieser Liste.) Tj ET
|
|
24
|
+
BT /F2 10 Tf 1 0 0 1 74.23 712 Tm (Pos.) Tj ET
|
|
25
|
+
BT /F2 10 Tf 1 0 0 1 101.35 712 Tm (Bezeichnung) Tj ET
|
|
26
|
+
0.6 G 0.5 w 72 707 m 523 707 l S 0 G
|
|
27
|
+
BT /F2 10 Tf 1 0 0 1 101.35 690 Tm (Sanit\344r) Tj ET
|
|
28
|
+
BT /F1 10 Tf 1 0 0 1 72 675 Tm (S 1.1) Tj ET
|
|
29
|
+
BT /F1 10 Tf 1 0 0 1 101.35 675 Tm (Waschtisch, Keramik, ca. 55x45 cm, mit Einhebelmischer) Tj ET
|
|
30
|
+
BT /F1 10 Tf 1 0 0 1 72 660 Tm (S 1.2) Tj ET
|
|
31
|
+
BT /F1 10 Tf 1 0 0 1 101.35 660 Tm (WC, wandh\344ngend, mit Unterputz-Sp\374lkasten) Tj ET
|
|
32
|
+
BT /F1 10 Tf 1 0 0 1 72 645 Tm (S 1.3) Tj ET
|
|
33
|
+
BT /F1 10 Tf 1 0 0 1 101.35 645 Tm (Urinal, Keramik, mit Sp\374lsteuerung) Tj ET
|
|
34
|
+
BT /F1 10 Tf 1 0 0 1 72 630 Tm (S 2.1) Tj ET
|
|
35
|
+
BT /F1 10 Tf 1 0 0 1 101.35 630 Tm (Spiegel, Kristallglas, 60x45 cm) Tj ET
|
|
36
|
+
BT /F1 10 Tf 1 0 0 1 72 615 Tm (S 2.2) Tj ET
|
|
37
|
+
BT /F1 10 Tf 1 0 0 1 101.35 615 Tm (Seifenspender, Edelstahl, 0,5 l) Tj ET
|
|
38
|
+
BT /F2 10 Tf 1 0 0 1 101.35 594 Tm (M\366bel) Tj ET
|
|
39
|
+
BT /F1 10 Tf 1 0 0 1 80.9 579 Tm (T 1) Tj ET
|
|
40
|
+
BT /F1 10 Tf 1 0 0 1 101.35 579 Tm (Einzeltisch, 70x50 cm, h\366henverstellbar) Tj ET
|
|
41
|
+
BT /F1 10 Tf 1 0 0 1 80.9 564 Tm (T 2) Tj ET
|
|
42
|
+
BT /F1 10 Tf 1 0 0 1 101.35 564 Tm (Lehrertisch, 140x70 cm) Tj ET
|
|
43
|
+
BT /F1 10 Tf 1 0 0 1 80.9 549 Tm (T 3) Tj ET
|
|
44
|
+
BT /F1 10 Tf 1 0 0 1 101.35 549 Tm (Sch\374lerstuhl, Kunststoff, Gr\366\337e 5) Tj ET
|
|
45
|
+
BT /F2 10 Tf 1 0 0 1 101.35 528 Tm (Leuchten) Tj ET
|
|
46
|
+
BT /F1 10 Tf 1 0 0 1 81.45 513 Tm (L 1) Tj ET
|
|
47
|
+
BT /F1 10 Tf 1 0 0 1 101.35 513 Tm (Wandleuchte LED, 12 W) Tj ET
|
|
48
|
+
BT /F1 10 Tf 1 0 0 1 81.45 498 Tm (L 2) Tj ET
|
|
49
|
+
BT /F1 10 Tf 1 0 0 1 101.35 498 Tm (Deckenleuchte LED, 36 W, dimmbar) Tj ET
|
|
50
|
+
BT /F1 10 Tf 1 0 0 1 81.45 483 Tm (L 3) Tj ET
|
|
51
|
+
BT /F1 10 Tf 1 0 0 1 101.35 483 Tm (Sicherheitsleuchte mit Einzelbatterie) Tj ET
|
|
52
|
+
BT /F2 10 Tf 1 0 0 1 101.35 462 Tm (Tafeln) Tj ET
|
|
53
|
+
BT /F1 10 Tf 1 0 0 1 77.57 447 Tm (W 1) Tj ET
|
|
54
|
+
BT /F1 10 Tf 1 0 0 1 101.35 447 Tm (Wandtafel, Klapptafel, 200x100 cm) Tj ET
|
|
55
|
+
0.6 G 0.5 w 72 52 m 523 52 l S 0 G
|
|
56
|
+
0.35 g BT /F1 8 Tf 1 0 0 1 476.3 40 Tm (Seite 1 von 1) Tj ET 0 g
|
|
57
|
+
endstream
|
|
58
|
+
endobj
|
|
59
|
+
7 0 obj
|
|
60
|
+
<< /Type /Page /Parent 2 0 R /MediaBox [0 0 595 842] /Resources << /Font << /F1 3 0 R /F2 4 0 R >> >> /Contents 6 0 R >>
|
|
61
|
+
endobj
|
|
62
|
+
xref
|
|
63
|
+
0 8
|
|
64
|
+
0000000000 65535 f
|
|
65
|
+
0000000009 00000 n
|
|
66
|
+
0000000058 00000 n
|
|
67
|
+
0000000115 00000 n
|
|
68
|
+
0000000212 00000 n
|
|
69
|
+
0000000314 00000 n
|
|
70
|
+
0000000391 00000 n
|
|
71
|
+
0000002684 00000 n
|
|
72
|
+
trailer
|
|
73
|
+
<< /Size 8 /Root 1 0 R /Info 5 0 R /ID [<913c85a40204b42b73736e88aa12837b> <913c85a40204b42b73736e88aa12837b>] >>
|
|
74
|
+
startxref
|
|
75
|
+
2820
|
|
76
|
+
%%EOF
|