med-pdf-nmo 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +298 -0
- package/README.ru.md +298 -0
- package/dist/bm25.d.ts +47 -0
- package/dist/bm25.js +86 -0
- package/dist/browser-shims/buffer.d.ts +30 -0
- package/dist/browser-shims/buffer.js +31 -0
- package/dist/browser-shims/crypto.d.ts +33 -0
- package/dist/browser-shims/crypto.js +45 -0
- package/dist/browser-shims/fs-promises.d.ts +13 -0
- package/dist/browser-shims/fs-promises.js +25 -0
- package/dist/browser-shims/fs.d.ts +14 -0
- package/dist/browser-shims/fs.js +24 -0
- package/dist/browser-shims/globals.d.ts +9 -0
- package/dist/browser-shims/globals.js +23 -0
- package/dist/browser-shims/path.d.ts +57 -0
- package/dist/browser-shims/path.js +65 -0
- package/dist/browser-shims/process.d.ts +22 -0
- package/dist/browser-shims/process.js +27 -0
- package/dist/browser.d.ts +9 -0
- package/dist/browser.js +12 -0
- package/dist/chunk.d.ts +15 -0
- package/dist/chunk.js +76 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +87 -0
- package/dist/index.d.ts +82 -0
- package/dist/index.js +51 -0
- package/dist/med-pdf-nmo.browser.js +40413 -0
- package/dist/med-pdf-nmo.browser.mjs +40395 -0
- package/dist/normalize.d.ts +73 -0
- package/dist/normalize.js +477 -0
- package/dist/pdf.d.ts +35 -0
- package/dist/pdf.js +396 -0
- package/dist/predictor/config.d.ts +28 -0
- package/dist/predictor/config.js +26 -0
- package/dist/predictor/constants.d.ts +3 -0
- package/dist/predictor/constants.js +59 -0
- package/dist/predictor/runtime.d.ts +15 -0
- package/dist/predictor/runtime.js +59 -0
- package/dist/predictor/scorers/biomedical-symbols.d.ts +36 -0
- package/dist/predictor/scorers/biomedical-symbols.js +347 -0
- package/dist/predictor/scorers/coordinate-table.d.ts +82 -0
- package/dist/predictor/scorers/coordinate-table.js +1210 -0
- package/dist/predictor/scorers/direction.d.ts +71 -0
- package/dist/predictor/scorers/direction.js +345 -0
- package/dist/predictor/scorers/drug-dose.d.ts +6 -0
- package/dist/predictor/scorers/drug-dose.js +221 -0
- package/dist/predictor/scorers/exact-answer.d.ts +10 -0
- package/dist/predictor/scorers/exact-answer.js +75 -0
- package/dist/predictor/scorers/fibrosis-stage.d.ts +6 -0
- package/dist/predictor/scorers/fibrosis-stage.js +103 -0
- package/dist/predictor/scorers/focused.d.ts +40 -0
- package/dist/predictor/scorers/focused.js +204 -0
- package/dist/predictor/scorers/frequency.d.ts +10 -0
- package/dist/predictor/scorers/frequency.js +203 -0
- package/dist/predictor/scorers/numeric.d.ts +77 -0
- package/dist/predictor/scorers/numeric.js +1161 -0
- package/dist/predictor/scorers/recommendation-item.d.ts +27 -0
- package/dist/predictor/scorers/recommendation-item.js +469 -0
- package/dist/predictor/scorers/search.d.ts +41 -0
- package/dist/predictor/scorers/search.js +515 -0
- package/dist/predictor/selection.d.ts +30 -0
- package/dist/predictor/selection.js +370 -0
- package/dist/predictor/text-utils.d.ts +49 -0
- package/dist/predictor/text-utils.js +497 -0
- package/dist/predictor/types.d.ts +23 -0
- package/dist/predictor/types.js +1 -0
- package/dist/predictor.d.ts +52 -0
- package/dist/predictor.js +3834 -0
- package/package.json +82 -0
|
@@ -0,0 +1,515 @@
|
|
|
1
|
+
import { coverage, normalizeForSearch, normalizeText, tokenize, uniqueTokens } from "../../normalize.js";
|
|
2
|
+
import { SECTION_GENERIC_TOKENS } from "../constants.js";
|
|
3
|
+
import { answerHasQuestionNumbers, answerSearchPhrases, betterEvidence, escapeRegExp, evidenceSnippet, findPhraseOccurrences, numberCoverage, pageWindow, proximityBonus, rawSoftCoverage, rawTokens, softCoverage, strictSoftCoverage, tokenizeNormalized, tokenSequenceIncludes, } from "../text-utils.js";
|
|
4
|
+
const ROW_LABEL_RE = /(локализован[а-яa-z0-9-]*|генерализован[а-яa-z0-9-]*|редк[а-яa-z0-9-]*|перинатальн[а-яa-z0-9-]*|ранн[а-яa-z0-9-]*\s+младенческ[а-яa-z0-9-]*|поздн[а-яa-z0-9-]*\s+младенческ[а-яa-z0-9-]*|ювенильн[а-яa-z0-9-]*|подростков[а-яa-z0-9-]*|взросл[а-яa-z0-9-]*|положительн[а-яa-z0-9-]*|сомнительн[а-яa-z0-9-]*|отрицательн[а-яa-z0-9-]*|планов[а-яa-z0-9-]*|экстренн[а-яa-z0-9-]*|неотложн[а-яa-z0-9-]*|перв[а-яa-z0-9-]*\s+лини[ияею]|втор[а-яa-z0-9-]*\s+лини[ияею]|треть[а-яa-z0-9-]*\s+лини[ияею]|не\s+имеющ[а-яa-z0-9-]*\s+фактор[а-яa-z0-9-]*\s+риска|при\s+наличи[а-яa-z0-9-]*\s+фактор[а-яa-z0-9-]*\s+риска)/giu;
|
|
5
|
+
function questionAnchorPhrases(question) {
|
|
6
|
+
const raw = normalizeText(question);
|
|
7
|
+
const normalized = normalizeForSearch(question);
|
|
8
|
+
const phrases = new Set([raw, normalized]);
|
|
9
|
+
const match = raw.match(/^к\s+(.+?)\s+относятся/u);
|
|
10
|
+
if (match) {
|
|
11
|
+
const middleTokens = rawTokens(match[1]).filter((token) => !["группе", "группа", "компонентам", "компоненты", "препараты", "препаратам", "средства", "методы"].includes(token));
|
|
12
|
+
if (middleTokens.length) {
|
|
13
|
+
phrases.add(`к ${middleTokens[0]} относятся`);
|
|
14
|
+
phrases.add(`к ${middleTokens[0]} причинам относятся`);
|
|
15
|
+
phrases.add(`к ${middleTokens[0]} компонентам относятся`);
|
|
16
|
+
phrases.add(`${middleTokens.slice(0, 3).join(" ")} относятся`);
|
|
17
|
+
phrases.add(middleTokens.slice(0, 4).join(" "));
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
const groupMatch = raw.match(/(?:к\s+группе|группы)\s+(.+?)\s+относятся/u);
|
|
21
|
+
if (groupMatch) {
|
|
22
|
+
const tokens = rawTokens(groupMatch[1]);
|
|
23
|
+
if (tokens.length)
|
|
24
|
+
phrases.add(tokens.slice(0, 4).join(" "));
|
|
25
|
+
}
|
|
26
|
+
return [...phrases].filter((phrase) => phrase.length >= 4);
|
|
27
|
+
}
|
|
28
|
+
export function findAnchorSegments(pages, question) {
|
|
29
|
+
const anchors = questionAnchorPhrases(question);
|
|
30
|
+
const segments = [];
|
|
31
|
+
for (const page of pages) {
|
|
32
|
+
for (const anchor of anchors) {
|
|
33
|
+
const normalizedAnchor = normalizeForSearch(anchor);
|
|
34
|
+
let start = 0;
|
|
35
|
+
while (start < page.normalized.length) {
|
|
36
|
+
const index = page.normalized.indexOf(normalizedAnchor, start);
|
|
37
|
+
if (index < 0)
|
|
38
|
+
break;
|
|
39
|
+
const after = page.normalized.slice(index);
|
|
40
|
+
let end = Math.min(after.length, 750);
|
|
41
|
+
end = Math.min(end, findAnchorBoundary(after));
|
|
42
|
+
const text = after.slice(0, end);
|
|
43
|
+
segments.push({
|
|
44
|
+
page: page.page,
|
|
45
|
+
normalized: text,
|
|
46
|
+
text: evidenceSnippet(page.text, anchor),
|
|
47
|
+
anchor,
|
|
48
|
+
});
|
|
49
|
+
start = index + Math.max(1, normalizedAnchor.length);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
return segments.slice(0, 12);
|
|
54
|
+
}
|
|
55
|
+
function questionSectionAnchor(question) {
|
|
56
|
+
const raw = normalizeText(question);
|
|
57
|
+
const match = raw.match(/по\s+([а-яa-z0-9 -]{4,64}?)(?:\s+выдел|\s+отно|\s+явля|\s+различ|\s+классифиц|$)/u);
|
|
58
|
+
if (!match?.[1])
|
|
59
|
+
return null;
|
|
60
|
+
return `по ${match[1].trim()}`;
|
|
61
|
+
}
|
|
62
|
+
export function findSectionSegments(pages, question) {
|
|
63
|
+
const anchor = questionSectionAnchor(question);
|
|
64
|
+
if (!anchor)
|
|
65
|
+
return [];
|
|
66
|
+
const anchorNorm = normalizeForSearch(anchor);
|
|
67
|
+
const segments = [];
|
|
68
|
+
for (const page of pages) {
|
|
69
|
+
const lines = page.lines ?? [];
|
|
70
|
+
for (let index = 0; index < lines.length; index += 1) {
|
|
71
|
+
const lineNorm = normalizeForSearch(lines[index]);
|
|
72
|
+
if (!lineNorm.includes(anchorNorm))
|
|
73
|
+
continue;
|
|
74
|
+
const out = [lines[index]];
|
|
75
|
+
for (let next = index + 1; next < Math.min(lines.length, index + 12); next += 1) {
|
|
76
|
+
const nextNorm = normalizeForSearch(lines[next]);
|
|
77
|
+
if (next > index + 1 && /^по\s+/iu.test(normalizeText(lines[next])))
|
|
78
|
+
break;
|
|
79
|
+
if (nextNorm.length < 2)
|
|
80
|
+
break;
|
|
81
|
+
out.push(lines[next]);
|
|
82
|
+
}
|
|
83
|
+
const text = out.join(" ");
|
|
84
|
+
segments.push({
|
|
85
|
+
page: page.page,
|
|
86
|
+
text,
|
|
87
|
+
normalized: normalizeForSearch(text),
|
|
88
|
+
anchor,
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
return segments.slice(0, 8);
|
|
93
|
+
}
|
|
94
|
+
function findAnchorBoundary(normalizedAfter) {
|
|
95
|
+
let boundary = Math.min(normalizedAfter.length, 750);
|
|
96
|
+
for (let pos = normalizedAfter.indexOf(" k ", 50); pos >= 0; pos = normalizedAfter.indexOf(" k ", pos + 3)) {
|
|
97
|
+
const local = normalizedAfter.slice(pos, pos + 170);
|
|
98
|
+
if (local.includes("othoc")) {
|
|
99
|
+
boundary = Math.min(boundary, pos);
|
|
100
|
+
break;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
for (const keyword of ["уровень", "примечание", "таблица", "раздел", "комментарии"]) {
|
|
104
|
+
const folded = normalizeForSearch(keyword);
|
|
105
|
+
const pos = normalizedAfter.indexOf(` ${folded}`, 80);
|
|
106
|
+
if (pos > 80)
|
|
107
|
+
boundary = Math.min(boundary, pos);
|
|
108
|
+
}
|
|
109
|
+
return Math.max(80, boundary);
|
|
110
|
+
}
|
|
111
|
+
export function bestPhraseSupport({ pages, question, answer, questionTokens, answerTokens, intent }) {
|
|
112
|
+
const answerPhrases = answerSearchPhrases(answer.text);
|
|
113
|
+
const joinedPhrases = answerPhrases.map((phrase) => `${question} ${phrase}`);
|
|
114
|
+
const normalizedQuestion = normalizeForSearch(question);
|
|
115
|
+
let best = null;
|
|
116
|
+
for (const page of pages) {
|
|
117
|
+
const pageNorm = page.normalized;
|
|
118
|
+
for (const joined of joinedPhrases) {
|
|
119
|
+
const directHits = findPhraseOccurrences(pageNorm, joined, { textIsNormalized: true });
|
|
120
|
+
for (const hit of directHits) {
|
|
121
|
+
const window = pageWindow(page, hit, 700);
|
|
122
|
+
const winTokens = tokenize(window);
|
|
123
|
+
const score = 8.5 +
|
|
124
|
+
coverage(questionTokens, winTokens) * 2.2 +
|
|
125
|
+
coverage(answerTokens, winTokens) * 1.2 +
|
|
126
|
+
numberCoverage(answer.text, window) * 0.9 +
|
|
127
|
+
(intent.numeric ? 0.45 : 0);
|
|
128
|
+
best = betterEvidence(best, {
|
|
129
|
+
answerId: answer.id,
|
|
130
|
+
page: page.page,
|
|
131
|
+
text: evidenceSnippet(page.text, joined),
|
|
132
|
+
score,
|
|
133
|
+
kind: "question_answer_phrase",
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
if (!normalizedQuestion)
|
|
138
|
+
continue;
|
|
139
|
+
let qStart = 0;
|
|
140
|
+
while (qStart < pageNorm.length) {
|
|
141
|
+
const qIndex = pageNorm.indexOf(normalizedQuestion, qStart);
|
|
142
|
+
if (qIndex < 0)
|
|
143
|
+
break;
|
|
144
|
+
const afterStart = qIndex + normalizedQuestion.length;
|
|
145
|
+
const after = pageNorm.slice(afterStart, afterStart + 1200);
|
|
146
|
+
for (const answerPhrase of answerPhrases) {
|
|
147
|
+
const normalizedAnswer = normalizeForSearch(answerPhrase);
|
|
148
|
+
const answerIndex = after.indexOf(normalizedAnswer);
|
|
149
|
+
if (answerIndex >= 0) {
|
|
150
|
+
const distance = answerIndex;
|
|
151
|
+
const local = after.slice(Math.max(0, answerIndex - 220), answerIndex + normalizedAnswer.length + 260);
|
|
152
|
+
const score = 6.4 +
|
|
153
|
+
proximityBonus(distance, 1200) * 2.3 +
|
|
154
|
+
coverage(answerTokens, tokenize(local)) * 0.8 +
|
|
155
|
+
numberCoverage(answer.text, local) * 0.7;
|
|
156
|
+
best = betterEvidence(best, {
|
|
157
|
+
answerId: answer.id,
|
|
158
|
+
page: page.page,
|
|
159
|
+
text: evidenceSnippet(page.text, question, answer.text),
|
|
160
|
+
score,
|
|
161
|
+
kind: "answer_after_question",
|
|
162
|
+
});
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
qStart = qIndex + normalizedQuestion.length;
|
|
166
|
+
if (qStart > pageNorm.length)
|
|
167
|
+
break;
|
|
168
|
+
}
|
|
169
|
+
for (const answerPhrase of answerPhrases) {
|
|
170
|
+
const normalizedAnswer = normalizeForSearch(answerPhrase);
|
|
171
|
+
const answerHits = findPhraseOccurrences(pageNorm, answerPhrase, { textIsNormalized: true });
|
|
172
|
+
for (const hit of answerHits) {
|
|
173
|
+
const before = pageNorm.slice(Math.max(0, hit - 450), hit);
|
|
174
|
+
const after = pageNorm.slice(hit + normalizedAnswer.length, hit + normalizedAnswer.length + 450);
|
|
175
|
+
const window = `${before} ${normalizedAnswer} ${after}`;
|
|
176
|
+
const winTokens = tokenize(window);
|
|
177
|
+
const beforeCoverage = coverage(questionTokens, tokenize(before));
|
|
178
|
+
const afterCoverage = coverage(questionTokens, tokenize(after));
|
|
179
|
+
const localCoverage = Math.max(beforeCoverage, afterCoverage);
|
|
180
|
+
const broadCoverage = coverage(questionTokens, winTokens);
|
|
181
|
+
if (Math.max(localCoverage, broadCoverage) <= 0.1)
|
|
182
|
+
continue;
|
|
183
|
+
const qPrefix = normalizedQuestion.slice(0, Math.min(normalizedQuestion.length, 140));
|
|
184
|
+
const prefixBonus = qPrefix && before.includes(qPrefix.slice(0, Math.min(70, qPrefix.length))) ? 1.0 : 0;
|
|
185
|
+
const directionalBonus = localCoverage > 0.24 ? 2.4 + localCoverage * 4.2 : 0;
|
|
186
|
+
const score = 0.8 +
|
|
187
|
+
broadCoverage * 2.7 +
|
|
188
|
+
coverage(answerTokens, winTokens) * 1.2 +
|
|
189
|
+
numberCoverage(answer.text, window) * 1.2 +
|
|
190
|
+
directionalBonus +
|
|
191
|
+
prefixBonus +
|
|
192
|
+
(intent.numeric && answerHasQuestionNumbers(answer.text, question) ? 0.35 : 0);
|
|
193
|
+
best = betterEvidence(best, {
|
|
194
|
+
answerId: answer.id,
|
|
195
|
+
page: page.page,
|
|
196
|
+
text: evidenceSnippet(page.text, answer.text),
|
|
197
|
+
score,
|
|
198
|
+
kind: localCoverage > broadCoverage ? "answer_directional_window" : "answer_window",
|
|
199
|
+
});
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
return best;
|
|
204
|
+
}
|
|
205
|
+
/**
|
|
206
|
+
* Находит ответы-заголовки перед длинным описанием из вопроса.
|
|
207
|
+
*
|
|
208
|
+
* В рекомендациях часто встречается форма `Название: описание...`, а вопрос
|
|
209
|
+
* дословно цитирует только описание. Обычный поиск после вопроса в таком случае
|
|
210
|
+
* может выбрать следующий заголовок, поэтому этот scorer ищет вариант ответа
|
|
211
|
+
* непосредственно перед найденным фрагментом вопроса.
|
|
212
|
+
*/
|
|
213
|
+
export function bestPrecedingQuestionLabelSupport({ mode, pages, question, answer, answerTokens }) {
|
|
214
|
+
if (mode !== "single")
|
|
215
|
+
return null;
|
|
216
|
+
const normalizedQuestion = normalizeForSearch(question);
|
|
217
|
+
if (normalizedQuestion.length < 90)
|
|
218
|
+
return null;
|
|
219
|
+
const answerPhrases = answerSearchPhrases(answer.text);
|
|
220
|
+
let best = null;
|
|
221
|
+
for (const page of pages) {
|
|
222
|
+
const pageNorm = page.normalized;
|
|
223
|
+
let qStart = 0;
|
|
224
|
+
while (qStart < pageNorm.length) {
|
|
225
|
+
const qIndex = pageNorm.indexOf(normalizedQuestion, qStart);
|
|
226
|
+
if (qIndex < 0)
|
|
227
|
+
break;
|
|
228
|
+
const before = pageNorm.slice(Math.max(0, qIndex - 180), qIndex);
|
|
229
|
+
for (const phrase of answerPhrases) {
|
|
230
|
+
const normalizedAnswer = normalizeForSearch(phrase);
|
|
231
|
+
if (!normalizedAnswer || normalizedAnswer.length < 5)
|
|
232
|
+
continue;
|
|
233
|
+
const answerIndex = before.lastIndexOf(normalizedAnswer);
|
|
234
|
+
let tailTokens = [];
|
|
235
|
+
if (answerIndex >= 0) {
|
|
236
|
+
const tail = before.slice(answerIndex + normalizedAnswer.length).trim();
|
|
237
|
+
tailTokens = tokenize(tail, { keepStopwords: true });
|
|
238
|
+
if (tailTokens.length > 8)
|
|
239
|
+
continue;
|
|
240
|
+
}
|
|
241
|
+
const answerCoverage = strictSoftCoverage(answerTokens, tokenizeNormalized(before));
|
|
242
|
+
if (answerIndex < 0 && answerCoverage < 0.82)
|
|
243
|
+
continue;
|
|
244
|
+
const score = 26 + answerCoverage * 2.4 + Math.max(0, 8 - tailTokens.length) * 0.35;
|
|
245
|
+
best = betterEvidence(best, {
|
|
246
|
+
answerId: answer.id,
|
|
247
|
+
page: page.page,
|
|
248
|
+
text: evidenceSnippet(page.text, question, answer.text),
|
|
249
|
+
score,
|
|
250
|
+
kind: "preceding_question_label",
|
|
251
|
+
});
|
|
252
|
+
}
|
|
253
|
+
qStart = qIndex + normalizedQuestion.length;
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
return best;
|
|
257
|
+
}
|
|
258
|
+
export function bestAnchorSupport({ anchorSegments, answer, answerTokens }) {
|
|
259
|
+
if (!anchorSegments?.length)
|
|
260
|
+
return null;
|
|
261
|
+
const answerPhrases = answerSearchPhrases(answer.text);
|
|
262
|
+
let best = null;
|
|
263
|
+
for (const segment of anchorSegments) {
|
|
264
|
+
let phraseHit = false;
|
|
265
|
+
for (const phrase of answerPhrases) {
|
|
266
|
+
const normalizedPhrase = normalizeForSearch(phrase);
|
|
267
|
+
if (normalizedPhrase && segment.normalized.includes(normalizedPhrase)) {
|
|
268
|
+
phraseHit = true;
|
|
269
|
+
break;
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
const segmentTokens = tokenize(segment.normalized);
|
|
273
|
+
const answerCoverage = coverage(answerTokens, segmentTokens);
|
|
274
|
+
const numericCoverage = numberCoverage(answer.text, segment.normalized);
|
|
275
|
+
const support = Math.max(phraseHit ? 1 : 0, answerCoverage, numericCoverage);
|
|
276
|
+
if (support <= 0.15)
|
|
277
|
+
continue;
|
|
278
|
+
const score = 7.4 + support * 3.2 + (phraseHit ? 1.4 : 0) + numericCoverage * 0.8;
|
|
279
|
+
best = betterEvidence(best, {
|
|
280
|
+
answerId: answer.id,
|
|
281
|
+
page: segment.page,
|
|
282
|
+
text: segment.text,
|
|
283
|
+
score,
|
|
284
|
+
kind: "question_anchor_segment",
|
|
285
|
+
});
|
|
286
|
+
}
|
|
287
|
+
return best;
|
|
288
|
+
}
|
|
289
|
+
export function bestSectionSupport({ sectionSegments, answer, answerTokens }) {
|
|
290
|
+
if (!sectionSegments?.length)
|
|
291
|
+
return null;
|
|
292
|
+
const answerPhrases = answerSearchPhrases(answer.text);
|
|
293
|
+
const distinctiveTokens = answerTokens.filter((token) => !SECTION_GENERIC_TOKENS.has(token));
|
|
294
|
+
let best = null;
|
|
295
|
+
for (const segment of sectionSegments) {
|
|
296
|
+
let phraseHit = false;
|
|
297
|
+
for (const phrase of answerPhrases) {
|
|
298
|
+
const normalizedPhrase = normalizeForSearch(phrase);
|
|
299
|
+
if (normalizedPhrase && segment.normalized.includes(normalizedPhrase)) {
|
|
300
|
+
phraseHit = true;
|
|
301
|
+
break;
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
const segmentTokens = tokenize(segment.text);
|
|
305
|
+
const answerCoverage = softCoverage(distinctiveTokens.length ? distinctiveTokens : answerTokens, segmentTokens);
|
|
306
|
+
const numericCoverage = numberCoverage(answer.text, segment.normalized);
|
|
307
|
+
const support = Math.max(phraseHit ? 1 : 0, answerCoverage, numericCoverage);
|
|
308
|
+
if (!phraseHit && support <= 0.25)
|
|
309
|
+
continue;
|
|
310
|
+
const score = 8.2 + support * 4.0 + (phraseHit ? 2.0 : 0) + numericCoverage * 0.8;
|
|
311
|
+
best = betterEvidence(best, {
|
|
312
|
+
answerId: answer.id,
|
|
313
|
+
page: segment.page,
|
|
314
|
+
text: segment.text,
|
|
315
|
+
score,
|
|
316
|
+
kind: "section_list_segment",
|
|
317
|
+
});
|
|
318
|
+
}
|
|
319
|
+
return best;
|
|
320
|
+
}
|
|
321
|
+
function questionRowCues(question) {
|
|
322
|
+
const raw = normalizeText(question);
|
|
323
|
+
const cues = new Set();
|
|
324
|
+
const add = (value) => {
|
|
325
|
+
const cleaned = normalizeText(value).replace(/\s+/g, " ").trim();
|
|
326
|
+
if (cleaned.length >= 3)
|
|
327
|
+
cues.add(cleaned);
|
|
328
|
+
};
|
|
329
|
+
const patterns = [
|
|
330
|
+
/по\s+мкб\s*-?\s*10\s+(.+?)\s+кодируется/u,
|
|
331
|
+
/возраст\s+манифестации\s+(.+?)\s+форм/u,
|
|
332
|
+
/проба.+?считается\s+"?([а-яa-z-]+)"?/u,
|
|
333
|
+
/согласно.+?(стади[яи]\s+(?:[ivx]+|\d+))/u,
|
|
334
|
+
/(стади[яи]\s+(?:[ivx]+|\d+))/u,
|
|
335
|
+
/препарат[а-яa-z0-9-]*\s+((?:перв|втор|треть)[а-яa-z0-9-]*\s+лини[ияею])/u,
|
|
336
|
+
/показани[а-яa-z0-9-]*\s+для\s+(.+?)\s+госпитализац/u,
|
|
337
|
+
/при\s+лечении\s+(.+?)\s+(?:назнач|примен)/u,
|
|
338
|
+
];
|
|
339
|
+
for (const pattern of patterns) {
|
|
340
|
+
const match = raw.match(pattern);
|
|
341
|
+
if (match?.[1])
|
|
342
|
+
add(match[1]);
|
|
343
|
+
}
|
|
344
|
+
for (const match of raw.matchAll(ROW_LABEL_RE))
|
|
345
|
+
add(match[0]);
|
|
346
|
+
return [...cues].slice(0, 8);
|
|
347
|
+
}
|
|
348
|
+
function rowCueMatch(line, cues) {
|
|
349
|
+
const lineRawTokens = rawTokens(line);
|
|
350
|
+
const lineTokens = tokenize(line);
|
|
351
|
+
let best = 0;
|
|
352
|
+
for (const cue of cues) {
|
|
353
|
+
const cueTokens = rawTokens(cue);
|
|
354
|
+
const cueStemTokens = uniqueTokens(cue);
|
|
355
|
+
if (!cueTokens.length)
|
|
356
|
+
continue;
|
|
357
|
+
if (tokenSequenceIncludes(lineRawTokens, cueTokens) || rawSoftCoverage(cueTokens, lineRawTokens) >= 0.8 || coverage(cueStemTokens, lineTokens) >= 0.8) {
|
|
358
|
+
best = Math.max(best, cueTokens.length >= 2 ? 1.15 : 1.0);
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
return best;
|
|
362
|
+
}
|
|
363
|
+
function rowBoundary(line) {
|
|
364
|
+
const raw = normalizeText(line);
|
|
365
|
+
return /^(?:[ivx]+\.\s+|[0-9]+(?:\.[0-9]+)*\s+|уровень\b|комментарии\b|примечание\b)/iu.test(raw);
|
|
366
|
+
}
|
|
367
|
+
function rowSegmentText(lines, startIndex) {
|
|
368
|
+
const current = normalizeText(lines[startIndex]);
|
|
369
|
+
const extendsForward = /[::]$/u.test(current) || /^стади[яи]\s+[ivx0-9]+/iu.test(current);
|
|
370
|
+
if (!extendsForward)
|
|
371
|
+
return lines[startIndex];
|
|
372
|
+
const out = [lines[startIndex]];
|
|
373
|
+
for (let index = startIndex + 1; index < Math.min(lines.length, startIndex + 6); index += 1) {
|
|
374
|
+
const next = lines[index];
|
|
375
|
+
const nextRaw = normalizeText(next);
|
|
376
|
+
if (index > startIndex + 1 && rowBoundary(next))
|
|
377
|
+
break;
|
|
378
|
+
if (/^стади[яи]\s+[ivx0-9]+/iu.test(nextRaw))
|
|
379
|
+
break;
|
|
380
|
+
if (/^по\s+мкб/iu.test(nextRaw))
|
|
381
|
+
break;
|
|
382
|
+
if (!nextRaw)
|
|
383
|
+
break;
|
|
384
|
+
out.push(next);
|
|
385
|
+
if (/[.;]$/u.test(nextRaw) && out.length >= 2)
|
|
386
|
+
break;
|
|
387
|
+
}
|
|
388
|
+
return out.join(" ");
|
|
389
|
+
}
|
|
390
|
+
export function findRowSegments(pages, question, topQuestionPages) {
|
|
391
|
+
const cues = questionRowCues(question);
|
|
392
|
+
if (!cues.length)
|
|
393
|
+
return [];
|
|
394
|
+
const mcbCodeQuestion = /мкб/u.test(normalizeText(question)) && /кодир/u.test(normalizeText(question));
|
|
395
|
+
const segments = [];
|
|
396
|
+
const seen = new Set();
|
|
397
|
+
for (const page of pages) {
|
|
398
|
+
const lines = page.lines ?? [];
|
|
399
|
+
for (let index = 0; index < lines.length; index += 1) {
|
|
400
|
+
if (mcbCodeQuestion && !/^\s*[a-zа-я]?\s*0?\d{2}(?:[.\s]\d)?/iu.test(normalizeText(lines[index])))
|
|
401
|
+
continue;
|
|
402
|
+
const cueScore = rowCueMatch(lines[index], cues);
|
|
403
|
+
if (cueScore <= 0)
|
|
404
|
+
continue;
|
|
405
|
+
const text = rowSegmentText(lines, index);
|
|
406
|
+
const key = `${page.page}:${normalizeForSearch(text).slice(0, 180)}`;
|
|
407
|
+
if (seen.has(key))
|
|
408
|
+
continue;
|
|
409
|
+
seen.add(key);
|
|
410
|
+
segments.push({
|
|
411
|
+
page: page.page,
|
|
412
|
+
text,
|
|
413
|
+
normalized: normalizeForSearch(text),
|
|
414
|
+
cueScore,
|
|
415
|
+
cues: cues.filter((cue) => rowCueMatch(lines[index], [cue]) > 0),
|
|
416
|
+
});
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
return segments.sort((a, b) => b.cueScore - a.cueScore).slice(0, 12);
|
|
420
|
+
}
|
|
421
|
+
function rowCueSpecificityPenalty(segment) {
|
|
422
|
+
const raw = normalizeText(segment.text);
|
|
423
|
+
let penalty = 0;
|
|
424
|
+
for (const cue of segment.cues ?? []) {
|
|
425
|
+
const cueRaw = normalizeText(cue);
|
|
426
|
+
if (!cueRaw || rawTokens(cueRaw).length < 2)
|
|
427
|
+
continue;
|
|
428
|
+
const cueTokens = rawTokens(cueRaw);
|
|
429
|
+
const firstPrefix = cueTokens[0]?.slice(0, Math.min(8, cueTokens[0].length));
|
|
430
|
+
const restPattern = cueTokens
|
|
431
|
+
.slice(1)
|
|
432
|
+
.map((token) => `${escapeRegExp(token.slice(0, Math.min(8, token.length)))}[а-яa-z0-9-]*`)
|
|
433
|
+
.join("\\s+");
|
|
434
|
+
if (firstPrefix && restPattern) {
|
|
435
|
+
const combinedPattern = new RegExp(`${escapeRegExp(firstPrefix)}[а-яa-z0-9-]*.+\\sс\\s+${escapeRegExp(firstPrefix)}[а-яa-z0-9-]*\\s+${restPattern}`, "iu");
|
|
436
|
+
if (combinedPattern.test(raw))
|
|
437
|
+
penalty += 3.0;
|
|
438
|
+
}
|
|
439
|
+
const index = raw.indexOf(cueRaw);
|
|
440
|
+
if (index < 0)
|
|
441
|
+
continue;
|
|
442
|
+
const before = raw
|
|
443
|
+
.slice(0, index)
|
|
444
|
+
.replace(/^[a-zа-я]?\s*\d{2}(?:[.\s]\d)?\s*/iu, "")
|
|
445
|
+
.trim();
|
|
446
|
+
if (before.length > 3)
|
|
447
|
+
penalty += 1.8;
|
|
448
|
+
const after = raw.slice(index + cueRaw.length, index + cueRaw.length + 16);
|
|
449
|
+
if (!cueRaw.includes(" с ") && /^\s+с\s+/u.test(after))
|
|
450
|
+
penalty += 2.6;
|
|
451
|
+
}
|
|
452
|
+
return penalty;
|
|
453
|
+
}
|
|
454
|
+
function answerCodeVariants(answerText) {
|
|
455
|
+
const normalized = normalizeForSearch(answerText);
|
|
456
|
+
const variants = new Set();
|
|
457
|
+
for (const match of normalized.matchAll(/\b([a-zа-я])?0?(\d{2}[.]\d)\b/giu)) {
|
|
458
|
+
const code = match[2];
|
|
459
|
+
variants.add(code);
|
|
460
|
+
variants.add(`0${code}`);
|
|
461
|
+
variants.add(`d${code}`);
|
|
462
|
+
}
|
|
463
|
+
return [...variants];
|
|
464
|
+
}
|
|
465
|
+
function rowAnswerPhrases(answerText) {
|
|
466
|
+
return [...new Set([...answerSearchPhrases(answerText), ...answerCodeVariants(answerText)])].slice(0, 18);
|
|
467
|
+
}
|
|
468
|
+
export function bestRowLabelSupport({ rowSegments, answer, answerTokens }) {
|
|
469
|
+
if (!rowSegments?.length)
|
|
470
|
+
return null;
|
|
471
|
+
const answerPhrases = rowAnswerPhrases(answer.text);
|
|
472
|
+
let best = null;
|
|
473
|
+
for (const segment of rowSegments) {
|
|
474
|
+
let phraseHit = false;
|
|
475
|
+
let phraseIndex = -1;
|
|
476
|
+
for (const phrase of answerPhrases) {
|
|
477
|
+
const normalizedPhrase = normalizeForSearch(phrase);
|
|
478
|
+
if (!normalizedPhrase)
|
|
479
|
+
continue;
|
|
480
|
+
const index = segment.normalized.indexOf(normalizedPhrase);
|
|
481
|
+
if (index >= 0) {
|
|
482
|
+
phraseHit = true;
|
|
483
|
+
phraseIndex = phraseIndex < 0 ? index : Math.min(phraseIndex, index);
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
const segmentTokens = tokenize(segment.text);
|
|
487
|
+
const answerCoverage = strictSoftCoverage(answerTokens, segmentTokens);
|
|
488
|
+
const numericCoverage = numberCoverage(answer.text, segment.normalized);
|
|
489
|
+
const answerHasNegation = /(?:^|\s)(?:не|без|отсутств)/u.test(normalizeText(answer.text));
|
|
490
|
+
const segmentHasNegation = /(?:^|\s)(?:не|без|отсутств)/u.test(normalizeText(segment.text));
|
|
491
|
+
const support = Math.max(phraseHit ? 1 : 0, answerCoverage, numericCoverage);
|
|
492
|
+
const coverageSupport = ((answerTokens.length >= 3 && answerCoverage >= 0.66) || (answerTokens.length >= 2 && answerCoverage >= 0.95)) &&
|
|
493
|
+
(!answerHasNegation || segmentHasNegation);
|
|
494
|
+
if (answerHasNegation && !segmentHasNegation && numericCoverage < 0.65)
|
|
495
|
+
continue;
|
|
496
|
+
if (!phraseHit && numericCoverage < 0.65 && !coverageSupport)
|
|
497
|
+
continue;
|
|
498
|
+
const earlyBonus = phraseIndex >= 0 && phraseIndex <= 180 ? 0.65 : 0;
|
|
499
|
+
const score = 8.0 +
|
|
500
|
+
support * 3.8 +
|
|
501
|
+
segment.cueScore * 1.5 +
|
|
502
|
+
(phraseHit ? 1.7 : 0) +
|
|
503
|
+
numericCoverage * 0.8 +
|
|
504
|
+
earlyBonus -
|
|
505
|
+
rowCueSpecificityPenalty(segment);
|
|
506
|
+
best = betterEvidence(best, {
|
|
507
|
+
answerId: answer.id,
|
|
508
|
+
page: segment.page,
|
|
509
|
+
text: segment.text,
|
|
510
|
+
score,
|
|
511
|
+
kind: "row_label_segment",
|
|
512
|
+
});
|
|
513
|
+
}
|
|
514
|
+
return best;
|
|
515
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { AnswerMode, AnswerScore } from "./types.js";
|
|
2
|
+
import type { PredictorConfig } from "./config.js";
|
|
3
|
+
/**
|
|
4
|
+
* Преобразует raw score вариантов в относительные confidence-like score.
|
|
5
|
+
*/
|
|
6
|
+
export declare function calibrateScores(answerScores: AnswerScore[]): {
|
|
7
|
+
score: number;
|
|
8
|
+
relative: number;
|
|
9
|
+
answer: import("./types.js").AnswerOption;
|
|
10
|
+
raw: number;
|
|
11
|
+
evidence: import("./types.js").EvidenceItem[];
|
|
12
|
+
}[];
|
|
13
|
+
/**
|
|
14
|
+
* Применяет зафиксированные post-scoring корректировки, оставленные после
|
|
15
|
+
* валидационных прогонов.
|
|
16
|
+
*
|
|
17
|
+
* Это по-прежнему детерминированная non-LLM логика: она использует только
|
|
18
|
+
* признаки score/evidence, полученные эвристическими scorers.
|
|
19
|
+
*/
|
|
20
|
+
export declare function applyFrozenFeatureRanker(answerScores: AnswerScore[], mode: AnswerMode, config: PredictorConfig, context?: {
|
|
21
|
+
question?: string;
|
|
22
|
+
}): AnswerScore[];
|
|
23
|
+
/**
|
|
24
|
+
* Выбирает финальные id ответов из калиброванных score для single/multi режима.
|
|
25
|
+
*/
|
|
26
|
+
export declare function selectAnswers(scored: ReturnType<typeof calibrateScores>, mode: AnswerMode, config: PredictorConfig): string[];
|
|
27
|
+
/**
|
|
28
|
+
* Округляет числовой score до четырех знаков после запятой.
|
|
29
|
+
*/
|
|
30
|
+
export declare function round4(value: number): number;
|