chaffjs 0.36.0 → 0.37.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/detectors/acronym-expansion.d.ts.map +1 -1
- package/dist/detectors/acronym-expansion.js +10 -3
- package/dist/detectors/acronym-expansion.js.map +1 -1
- package/dist/detectors/acronym-link.d.ts +5 -0
- package/dist/detectors/acronym-link.d.ts.map +1 -1
- package/dist/detectors/acronym-link.js +33 -0
- package/dist/detectors/acronym-link.js.map +1 -1
- package/dist/detectors/acronym-mention.d.ts +5 -0
- package/dist/detectors/acronym-mention.d.ts.map +1 -0
- package/dist/detectors/acronym-mention.js +34 -0
- package/dist/detectors/acronym-mention.js.map +1 -0
- package/dist/detectors/acronym-name.js +2 -2
- package/dist/detectors/acronym-name.js.map +1 -1
- package/dist/detectors/addressee-title-edge.d.ts +21 -0
- package/dist/detectors/addressee-title-edge.d.ts.map +1 -0
- package/dist/detectors/addressee-title-edge.js +22 -0
- package/dist/detectors/addressee-title-edge.js.map +1 -0
- package/dist/detectors/capitalised-term.d.ts +4 -0
- package/dist/detectors/capitalised-term.d.ts.map +1 -1
- package/dist/detectors/capitalised-term.js +18 -1
- package/dist/detectors/capitalised-term.js.map +1 -1
- package/dist/detectors/caps-caption.d.ts +14 -4
- package/dist/detectors/caps-caption.d.ts.map +1 -1
- package/dist/detectors/caps-caption.js +36 -9
- package/dist/detectors/caps-caption.js.map +1 -1
- package/dist/detectors/char-shape.d.ts.map +1 -1
- package/dist/detectors/char-shape.js +37 -14
- package/dist/detectors/char-shape.js.map +1 -1
- package/dist/detectors/citation-sentence.d.ts +8 -0
- package/dist/detectors/citation-sentence.d.ts.map +1 -0
- package/dist/detectors/citation-sentence.js +40 -0
- package/dist/detectors/citation-sentence.js.map +1 -0
- package/dist/detectors/contraction-mention.d.ts +3 -0
- package/dist/detectors/contraction-mention.d.ts.map +1 -0
- package/dist/detectors/contraction-mention.js +22 -0
- package/dist/detectors/contraction-mention.js.map +1 -0
- package/dist/detectors/dash-break.d.ts +7 -0
- package/dist/detectors/dash-break.d.ts.map +1 -0
- package/dist/detectors/dash-break.js +30 -0
- package/dist/detectors/dash-break.js.map +1 -0
- package/dist/detectors/date-any-year.d.ts.map +1 -1
- package/dist/detectors/date-any-year.js +5 -3
- package/dist/detectors/date-any-year.js.map +1 -1
- package/dist/detectors/definition-use.d.ts.map +1 -1
- package/dist/detectors/definition-use.js +9 -4
- package/dist/detectors/definition-use.js.map +1 -1
- package/dist/detectors/document-shape.d.ts +5 -0
- package/dist/detectors/document-shape.d.ts.map +1 -1
- package/dist/detectors/document-shape.js +15 -1
- package/dist/detectors/document-shape.js.map +1 -1
- package/dist/detectors/duplicate-heading.d.ts +1 -0
- package/dist/detectors/duplicate-heading.d.ts.map +1 -1
- package/dist/detectors/duplicate-heading.js +25 -3
- package/dist/detectors/duplicate-heading.js.map +1 -1
- package/dist/detectors/email-letter.d.ts +9 -2
- package/dist/detectors/email-letter.d.ts.map +1 -1
- package/dist/detectors/email-letter.js +39 -8
- package/dist/detectors/email-letter.js.map +1 -1
- package/dist/detectors/fullwidth-alnum.d.ts.map +1 -1
- package/dist/detectors/fullwidth-alnum.js +7 -1
- package/dist/detectors/fullwidth-alnum.js.map +1 -1
- package/dist/detectors/gram-openers.d.ts +4 -3
- package/dist/detectors/gram-openers.d.ts.map +1 -1
- package/dist/detectors/gram-openers.js +26 -2
- package/dist/detectors/gram-openers.js.map +1 -1
- package/dist/detectors/heading-record-rhythm.d.ts +9 -0
- package/dist/detectors/heading-record-rhythm.d.ts.map +1 -0
- package/dist/detectors/heading-record-rhythm.js +57 -0
- package/dist/detectors/heading-record-rhythm.js.map +1 -0
- package/dist/detectors/kutoten-consistency.d.ts.map +1 -1
- package/dist/detectors/kutoten-consistency.js +7 -0
- package/dist/detectors/kutoten-consistency.js.map +1 -1
- package/dist/detectors/lexicon.d.ts.map +1 -1
- package/dist/detectors/lexicon.js +5 -2
- package/dist/detectors/lexicon.js.map +1 -1
- package/dist/detectors/mark-shape.d.ts +3 -1
- package/dist/detectors/mark-shape.d.ts.map +1 -1
- package/dist/detectors/mark-shape.js +20 -5
- package/dist/detectors/mark-shape.js.map +1 -1
- package/dist/detectors/name-variant.d.ts.map +1 -1
- package/dist/detectors/name-variant.js +6 -0
- package/dist/detectors/name-variant.js.map +1 -1
- package/dist/detectors/organisation-run.d.ts +0 -6
- package/dist/detectors/organisation-run.d.ts.map +1 -1
- package/dist/detectors/organisation-run.js +4 -1
- package/dist/detectors/organisation-run.js.map +1 -1
- package/dist/detectors/orthographic-variant.d.ts.map +1 -1
- package/dist/detectors/orthographic-variant.js +33 -4
- package/dist/detectors/orthographic-variant.js.map +1 -1
- package/dist/detectors/orthography.d.ts.map +1 -1
- package/dist/detectors/orthography.js +4 -2
- package/dist/detectors/orthography.js.map +1 -1
- package/dist/detectors/oxford-comma.d.ts.map +1 -1
- package/dist/detectors/oxford-comma.js +33 -3
- package/dist/detectors/oxford-comma.js.map +1 -1
- package/dist/detectors/range-notation.d.ts +2 -2
- package/dist/detectors/range-notation.d.ts.map +1 -1
- package/dist/detectors/range-notation.js +18 -7
- package/dist/detectors/range-notation.js.map +1 -1
- package/dist/detectors/requirement-smell.d.ts.map +1 -1
- package/dist/detectors/requirement-smell.js +10 -6
- package/dist/detectors/requirement-smell.js.map +1 -1
- package/dist/detectors/sentence-counts.d.ts +10 -2
- package/dist/detectors/sentence-counts.d.ts.map +1 -1
- package/dist/detectors/sentence-counts.js +34 -4
- package/dist/detectors/sentence-counts.js.map +1 -1
- package/dist/detectors/sentence-length.d.ts.map +1 -1
- package/dist/detectors/sentence-length.js +13 -4
- package/dist/detectors/sentence-length.js.map +1 -1
- package/dist/detectors/signals.d.ts.map +1 -1
- package/dist/detectors/signals.js +15 -3
- package/dist/detectors/signals.js.map +1 -1
- package/dist/detectors/superlative-name.d.ts +5 -0
- package/dist/detectors/superlative-name.d.ts.map +1 -1
- package/dist/detectors/superlative-name.js +9 -0
- package/dist/detectors/superlative-name.js.map +1 -1
- package/dist/detectors/token-shape.d.ts.map +1 -1
- package/dist/detectors/token-shape.js +3 -1
- package/dist/detectors/token-shape.js.map +1 -1
- package/dist/detectors/undefined-term.d.ts.map +1 -1
- package/dist/detectors/undefined-term.js +5 -0
- package/dist/detectors/undefined-term.js.map +1 -1
- package/dist/detectors/unknown-word.d.ts +4 -1
- package/dist/detectors/unknown-word.d.ts.map +1 -1
- package/dist/detectors/unknown-word.js +1 -1
- package/dist/detectors/unknown-word.js.map +1 -1
- package/dist/html/html-site-chrome.d.ts.map +1 -1
- package/dist/html/html-site-chrome.js +35 -12
- package/dist/html/html-site-chrome.js.map +1 -1
- package/dist/latin-word-boundary.d.ts +5 -0
- package/dist/latin-word-boundary.d.ts.map +1 -1
- package/dist/latin-word-boundary.js +17 -0
- package/dist/latin-word-boundary.js.map +1 -1
- package/dist/leading-label.d.ts +4 -0
- package/dist/leading-label.d.ts.map +1 -0
- package/dist/leading-label.js +14 -0
- package/dist/leading-label.js.map +1 -0
- package/dist/link-shape.d.ts +3 -2
- package/dist/link-shape.d.ts.map +1 -1
- package/dist/link-shape.js +12 -3
- package/dist/link-shape.js.map +1 -1
- package/dist/name-misspelt-usual.d.ts +5 -0
- package/dist/name-misspelt-usual.d.ts.map +1 -0
- package/dist/name-misspelt-usual.js +19 -0
- package/dist/name-misspelt-usual.js.map +1 -0
- package/dist/notice-header.d.ts +19 -0
- package/dist/notice-header.d.ts.map +1 -0
- package/dist/notice-header.js +31 -0
- package/dist/notice-header.js.map +1 -0
- package/dist/sentence-load.d.ts +2 -1
- package/dist/sentence-load.d.ts.map +1 -1
- package/dist/sentence-load.js +11 -3
- package/dist/sentence-load.js.map +1 -1
- package/dist/structure/attached-parts.d.ts +2 -1
- package/dist/structure/attached-parts.d.ts.map +1 -1
- package/dist/structure/attached-parts.js +22 -9
- package/dist/structure/attached-parts.js.map +1 -1
- package/dist/structure/definition-use.d.ts +5 -2
- package/dist/structure/definition-use.d.ts.map +1 -1
- package/dist/structure/definition-use.js +24 -7
- package/dist/structure/definition-use.js.map +1 -1
- package/dist/structure/gerund-phrase.d.ts +8 -0
- package/dist/structure/gerund-phrase.d.ts.map +1 -0
- package/dist/structure/gerund-phrase.js +30 -0
- package/dist/structure/gerund-phrase.js.map +1 -0
- package/dist/structure/universal.d.ts +7 -3
- package/dist/structure/universal.d.ts.map +1 -1
- package/dist/structure/universal.js +17 -5
- package/dist/structure/universal.js.map +1 -1
- package/dist/title-line.d.ts +2 -0
- package/dist/title-line.d.ts.map +1 -1
- package/dist/title-line.js +2 -1
- package/dist/title-line.js.map +1 -1
- package/dist/topic-clause.d.ts +7 -0
- package/dist/topic-clause.d.ts.map +1 -0
- package/dist/topic-clause.js +87 -0
- package/dist/topic-clause.js.map +1 -0
- package/dist/uncounted-particles.d.ts +18 -0
- package/dist/uncounted-particles.d.ts.map +1 -0
- package/dist/uncounted-particles.js +47 -0
- package/dist/uncounted-particles.js.map +1 -0
- package/package.json +3 -3
- package/rules/adversative-ga-repeat.yaml +1 -0
- package/rules/all-caps-shouting.yaml +2 -2
- package/rules/bare-url-mix.yaml +2 -2
- package/rules/concrete-evidence-density.yaml +1 -0
- package/rules/contraction-consistency.yaml +2 -2
- package/rules/duplicate-definition.yaml +2 -2
- package/rules/duplicate-heading.yaml +2 -2
- package/rules/fullwidth-alnum-consistency.yaml +2 -2
- package/rules/kutoten-consistency.yaml +2 -2
- package/rules/latin-spacing.yaml +2 -2
- package/rules/max-kanji-continuous.yaml +1 -1
- package/rules/max-sentence-length.yaml +2 -2
- package/rules/ngram-repetition.yaml +2 -2
- package/rules/orthographic-variant.yaml +2 -2
- package/rules/oxford-comma-consistency.yaml +2 -2
- package/rules/preamble-length.yaml +3 -2
- package/rules/range-notation-consistency.yaml +2 -2
- package/rules/requirement-smell.yaml +2 -2
- package/rules/title-length.yaml +2 -2
- package/rules/topic-predicate-distance.yaml +2 -2
- package/rules/undefined-acronym.yaml +3 -2
- package/rules/undefined-term.yaml +2 -2
- package/rules/unqualified-superlative.yaml +1 -1
- package/rules/unused-definition.yaml +2 -2
- package/rules/use-before-definition.yaml +2 -2
- package/src/detectors/acronym-expansion.ts +10 -3
- package/src/detectors/acronym-link.ts +38 -0
- package/src/detectors/acronym-mention.ts +44 -0
- package/src/detectors/acronym-name.ts +2 -2
- package/src/detectors/addressee-title-edge.ts +39 -0
- package/src/detectors/capitalised-term.ts +26 -2
- package/src/detectors/caps-caption.ts +43 -9
- package/src/detectors/char-shape.ts +55 -17
- package/src/detectors/citation-sentence.ts +44 -0
- package/src/detectors/contraction-mention.ts +24 -0
- package/src/detectors/dash-break.ts +45 -0
- package/src/detectors/date-any-year.ts +7 -3
- package/src/detectors/definition-use.ts +12 -8
- package/src/detectors/document-shape.ts +20 -1
- package/src/detectors/duplicate-heading.ts +26 -3
- package/src/detectors/email-letter.ts +50 -7
- package/src/detectors/fullwidth-alnum.ts +9 -1
- package/src/detectors/gram-openers.ts +36 -7
- package/src/detectors/heading-record-rhythm.ts +57 -0
- package/src/detectors/kutoten-consistency.ts +9 -0
- package/src/detectors/lexicon.ts +6 -2
- package/src/detectors/mark-shape.ts +30 -5
- package/src/detectors/name-variant.ts +6 -0
- package/src/detectors/organisation-run.ts +6 -1
- package/src/detectors/orthographic-variant.ts +45 -6
- package/src/detectors/orthography.ts +6 -2
- package/src/detectors/oxford-comma.ts +35 -3
- package/src/detectors/range-notation.ts +26 -6
- package/src/detectors/requirement-smell.ts +12 -8
- package/src/detectors/sentence-counts.ts +40 -4
- package/src/detectors/sentence-length.ts +17 -4
- package/src/detectors/signals.ts +18 -3
- package/src/detectors/superlative-name.ts +10 -0
- package/src/detectors/token-shape.ts +3 -1
- package/src/detectors/undefined-term.ts +8 -0
- package/src/detectors/unknown-word.ts +1 -1
- package/src/html/html-site-chrome.ts +70 -13
- package/src/latin-word-boundary.ts +18 -0
- package/src/leading-label.ts +15 -0
- package/src/link-shape.ts +15 -3
- package/src/name-misspelt-usual.ts +25 -0
- package/src/notice-header.ts +57 -0
- package/src/sentence-load.ts +14 -3
- package/src/structure/attached-parts.ts +24 -9
- package/src/structure/definition-use.ts +33 -7
- package/src/structure/gerund-phrase.ts +31 -0
- package/src/structure/universal.ts +22 -7
- package/src/title-line.ts +2 -1
- package/src/topic-clause.ts +109 -0
- package/src/uncounted-particles.ts +72 -0
|
@@ -24,6 +24,9 @@ import { isOrdinaryToken, withOrdinaryWords } from "../name-ordinary-word.ts";
|
|
|
24
24
|
import { isInPersonField } from "../person-field.ts";
|
|
25
25
|
import { labelledSpans, orderNamesOf, quotedSpans, stemOf, titleCaseSpans, wordOrderVariants, type OrderWord } from "../name-word-order.ts";
|
|
26
26
|
import { isGroupPairOf, notDefinedTerms, outsideCapitals, sentenceIndexOf, withGivenNames } from "./name-variant-apart.ts";
|
|
27
|
+
import { usualIsMisspelt } from "../name-misspelt-usual.ts";
|
|
28
|
+
import { spellerOf } from "./unknown-word.ts";
|
|
29
|
+
import { isKnownWord } from "../spelling.ts";
|
|
27
30
|
|
|
28
31
|
// 人の名前と読ませる敬称(様、さん)は語彙表 person-suffix、人を指す前置き(担当の)は person-lead、名前のすぐ後ろに来る語
|
|
29
32
|
// (です、まで)は name-particle、字体の違う同じ字(斎・斉・齋)は name-variant-char が組(group)ごとに言う。
|
|
@@ -387,7 +390,10 @@ export const nameVariant: Detector = (doc): Finding[] => {
|
|
|
387
390
|
const withProducts = [...withPlaces, ...productFindings(doc, prose, withPlaces)];
|
|
388
391
|
const withOrder = [...withProducts, ...orderFindings(doc, prose, withProducts)];
|
|
389
392
|
const codes = modelCodeFindings(doc, prose).filter((code) => !overlapsAny(withOrder, code.offset, code.name.length));
|
|
393
|
+
const dictionary = doc.dictionary?.();
|
|
394
|
+
const speller = dictionary === undefined ? undefined : spellerOf(doc, dictionary);
|
|
390
395
|
return [...withOrder, ...codes]
|
|
396
|
+
.filter(({ name, usual }) => speller === undefined || !usualIsMisspelt(name, usual, (word) => isKnownWord(word, speller)))
|
|
391
397
|
.toSorted((left, right) => left.offset - right.offset)
|
|
392
398
|
.map(({ offset, name, usual, kind }) => ({
|
|
393
399
|
rule: "",
|
|
@@ -86,10 +86,15 @@ const endsInForm = (text: string, closingForms: readonly string[]): boolean =>
|
|
|
86
86
|
* または組織の語で閉じる区切りが 2 つ以上続くもの(国土交通省鉄道局総務課)。
|
|
87
87
|
* 組織の語で閉じない連なり(情報処理推進機構認定試験、総務課長)や、頭に名前の無い 1 区切り(個人情報保護法施行規則改正案検討会)は名前ではない。
|
|
88
88
|
*/
|
|
89
|
+
/** 尻の法人格(独立行政法人)の前が、組織の語で閉じる区切りだけに割れるか(文部科学省独立行政法人: 所管の組織と法人の種類)。 */
|
|
90
|
+
const segmentsBeforeForm = (run: readonly Token[], words: OrganisationWords): boolean =>
|
|
91
|
+
run.some((_, at) => at > 0 && words.closingForms.includes(surfaceOf(run.slice(at))) && segmentCount(run.slice(0, at), words.units) >= 1);
|
|
92
|
+
|
|
89
93
|
export const isOrganisationRun = (tokens: readonly Token[], covering: readonly number[], words: OrganisationWords): boolean => {
|
|
90
94
|
const run = contiguous(tokens, withClosingUnit(tokens, covering, words.units));
|
|
91
95
|
if (run === undefined || run.length === 0) return false;
|
|
92
|
-
|
|
96
|
+
const named = run.some(isProperPiece) || segmentsBeforeForm(run, words);
|
|
97
|
+
if (endsInForm(surfaceOf(run), words.closingForms) && named && !run.some(isNumber)) return true;
|
|
93
98
|
const form = formLength(run, new Set(words.forms));
|
|
94
99
|
const head = form > 0 ? form : properLength(run);
|
|
95
100
|
const segments = segmentCount(run.slice(head), words.units);
|
|
@@ -17,6 +17,7 @@ import { wholeWordRuns, type WholeWordRun } from "../whole-word-runs.ts";
|
|
|
17
17
|
import { entriesForGenre } from "../lexicon-genres.ts";
|
|
18
18
|
import { asNounWhereNominal, documentNounKeys } from "../nominal-stem.ts";
|
|
19
19
|
import { isCompoundHead } from "../compound-head.ts";
|
|
20
|
+
import { leadingLabelSpans } from "../leading-label.ts";
|
|
20
21
|
|
|
21
22
|
/**
|
|
22
23
|
* A word found in the document: where it is, which sentence it is in, its key and spelling, how it is written when that differs, and
|
|
@@ -49,6 +50,14 @@ const FUKUSHI = "hiragana-fukushi";
|
|
|
49
50
|
const IDIOM = "orthographic-variant-idiom";
|
|
50
51
|
/** Lexicon: verbs whose て-form opening a sentence before a comma is a conjunction (あわせて、). */
|
|
51
52
|
const CONJUNCTION = "orthographic-variant-conjunction";
|
|
53
|
+
/** Lexicon: verbs whose judging use (〜とみられる, 増加がみられる) is written in kana while the verb (資料を見る) keeps its kanji. */
|
|
54
|
+
const JUDGING = "orthographic-variant-judging";
|
|
55
|
+
/**
|
|
56
|
+
* Lexicon: words whose kanji is also a person (方 as かた): compared only before が or で (早いほうが, 資料のほうで), where the
|
|
57
|
+
* person is rare; before に, の or は (担当の方に) the kanji is mostly the person.
|
|
58
|
+
*/
|
|
59
|
+
const COMPARATIVE = "orthographic-variant-comparative";
|
|
60
|
+
const COMPARATIVE_PARTICLES = new Set(["が", "で"]);
|
|
52
61
|
/** Lexicon: words the tokenizer cuts in one spelling and not the other (お客様, お客 and さま). Each is read whole, keyed by its group. */
|
|
53
62
|
const WHOLE_WORD = "orthographic-variant-whole-word";
|
|
54
63
|
/** Lexicon: suffixes written onto a word that do not make a compound of it (割当て等, 引受け後): the word keeps its okurigana there. */
|
|
@@ -68,8 +77,9 @@ const inSentence = (sentence: Sentence, spans: readonly Span[]): Span[] =>
|
|
|
68
77
|
spans.map((span) => ({ start: span.start - sentence.span.start, end: span.end - sentence.span.start }));
|
|
69
78
|
|
|
70
79
|
/**
|
|
71
|
-
* Whether a stretch of the sentence is outside a quotation, the team's names, a company's name (関東珪曹硝子㈱)
|
|
72
|
-
* brackets (HTTP(えいちてぃーてぃーぴー)): the first three keep their own spelling,
|
|
80
|
+
* Whether a stretch of the sentence is outside a quotation, the team's names, a company's name (関東珪曹硝子㈱), a reading in
|
|
81
|
+
* brackets (HTTP(えいちてぃーてぃーぴー)) and a label opening the sentence ((答)): the first three keep their own spelling,
|
|
82
|
+
* a reading is not a spelling at all, and a label is a fixed short form.
|
|
73
83
|
*/
|
|
74
84
|
const isOpenIn = (sentence: Sentence, names: Names): IsOpen => {
|
|
75
85
|
const closed = [
|
|
@@ -77,6 +87,7 @@ const isOpenIn = (sentence: Sentence, names: Names): IsOpen => {
|
|
|
77
87
|
...furiganaSpans(sentence.text),
|
|
78
88
|
...japaneseCompanyNameSpans(sentence.text, names.companyForms),
|
|
79
89
|
...inSentence(sentence, names.linkTitles),
|
|
90
|
+
...leadingLabelSpans(sentence.text),
|
|
80
91
|
];
|
|
81
92
|
const named = nameSpans(sentence.text, names.team);
|
|
82
93
|
return (start, end) => !isWithinAny(closed, { start, end }) && !touchesAny(named, start, end);
|
|
@@ -140,18 +151,37 @@ const isConjunctionUse = ({ token, previous, following }: InContext, conjunction
|
|
|
140
151
|
TE.has(following?.[0]?.surface ?? "") &&
|
|
141
152
|
COMMA.has(following?.[1]?.surface ?? "");
|
|
142
153
|
|
|
143
|
-
/**
|
|
144
|
-
|
|
154
|
+
/** と, or と and は or も (〜ともみられる): the quotative particle before a verb of judging. */
|
|
155
|
+
const QUOTATIVE = "と";
|
|
156
|
+
const QUOTATIVE_TOPIC = new Set(["は", "も"]);
|
|
157
|
+
const PASSIVE = new Set(["られる", "れる"]);
|
|
158
|
+
|
|
159
|
+
const followsQuotative = (previous: Token | undefined, before: Token | undefined): boolean =>
|
|
160
|
+
previous?.pos === "ADP" && (previous.surface === QUOTATIVE || (QUOTATIVE_TOPIC.has(previous.surface) && before?.surface === QUOTATIVE));
|
|
161
|
+
|
|
162
|
+
/** The particles of the subject before an observing られる (増加がみられる, 発生はみられない); after を it is looking (映画を見られる). */
|
|
163
|
+
const OBSERVED_SUBJECT = new Set(["が", "は", "も"]);
|
|
164
|
+
|
|
165
|
+
const isObserved = (previous: Token | undefined, following: readonly Token[] | undefined): boolean =>
|
|
166
|
+
previous?.pos === "ADP" && OBSERVED_SUBJECT.has(previous.surface) && PASSIVE.has(following?.[0]?.lemma ?? "");
|
|
167
|
+
|
|
168
|
+
/** A listed verb after the quotative と (〜とみる) or observing with られる (増加がみられる): judging what is seen, not looking. */
|
|
169
|
+
const isJudgingUse = ({ token, previous, before, following }: InContext, judging: ReadonlySet<string>): boolean =>
|
|
170
|
+
token.pos === "VERB" && judging.has(token.lemma ?? "") && (followsQuotative(previous, before) || isObserved(previous, following));
|
|
171
|
+
|
|
172
|
+
/** The lexicons that set a verb's use apart: idioms (気をつける), conjunctions (あわせて、) and judging (〜とみられる). */
|
|
173
|
+
type Uses = { readonly idioms: ReadonlySet<string>; readonly conjunctions: ReadonlySet<string>; readonly judging: ReadonlySet<string> };
|
|
145
174
|
|
|
146
175
|
/**
|
|
147
176
|
* A helper verb or a dependent noun (見て下さい, その事) is written kana by many guides while the full word keeps its kanji (資料を下さい).
|
|
148
177
|
* The two uses are compared apart, so a document that follows that convention is consistent. A suffix (業務用, 全員ぶん) is a third use:
|
|
149
178
|
* it is not the dependent noun of the same reading (このように). The verb of an idiom (気をつける) is a fourth, and a
|
|
150
|
-
* て-form used as a conjunction (あわせて、) a fifth.
|
|
179
|
+
* て-form used as a conjunction (あわせて、) a fifth, and a verb of judging (〜とみられる) a sixth.
|
|
151
180
|
*/
|
|
152
|
-
const useOf = (word: InContext, { idioms, conjunctions }: Uses): string => {
|
|
181
|
+
const useOf = (word: InContext, { idioms, conjunctions, judging }: Uses): string => {
|
|
153
182
|
if (isIdiomVerb(word, idioms)) return "idiom";
|
|
154
183
|
if (isConjunctionUse(word, conjunctions)) return "conjunction";
|
|
184
|
+
if (isJudgingUse(word, judging)) return "judging";
|
|
155
185
|
if (isKanaSuffix(word)) return "suffix";
|
|
156
186
|
if (!isHelper(word)) return "free";
|
|
157
187
|
return word.token.pos === "NOUN" && word.token.features?.["NounType"] !== "Dependent" ? "suffix" : "bound";
|
|
@@ -176,17 +206,24 @@ type Skips = {
|
|
|
176
206
|
readonly skip: ReadonlySet<string>;
|
|
177
207
|
readonly idioms: ReadonlySet<string>;
|
|
178
208
|
readonly conjunctions: ReadonlySet<string>;
|
|
209
|
+
readonly judging: ReadonlySet<string>;
|
|
210
|
+
readonly comparative: ReadonlySet<string>;
|
|
179
211
|
readonly wholeWords: ReadonlyMap<string, string>;
|
|
180
212
|
readonly phraseSuffixes: ReadonlySet<string>;
|
|
181
213
|
/** The document's nouns, as documentNounKeys keys them. */
|
|
182
214
|
readonly nouns: ReadonlySet<string>;
|
|
183
215
|
};
|
|
184
216
|
|
|
217
|
+
/** A listed word outside the comparative frame (担当の方に): the kanji there may be the person of the same reading. */
|
|
218
|
+
const isOutsideComparative = ({ token, following }: InContext, comparative: ReadonlySet<string>): boolean =>
|
|
219
|
+
comparative.has(token.lemma ?? "") && !COMPARATIVE_PARTICLES.has(following?.[0]?.surface ?? "");
|
|
220
|
+
|
|
185
221
|
const readingKeyOf = (word: InContext, skips: Skips): string | undefined => {
|
|
186
222
|
const { skip } = skips;
|
|
187
223
|
const { token } = word;
|
|
188
224
|
const lemma = token.lemma;
|
|
189
225
|
if (!CONTENT.has(token.pos) || token.reading === undefined || lemma === undefined || !JAPANESE_WORD.test(lemma) || skip.has(lemma)) return undefined;
|
|
226
|
+
if (isOutsideComparative(word, skips.comparative)) return undefined;
|
|
190
227
|
const reading = lemmaReading(token.surface, lemma, token.reading);
|
|
191
228
|
if (reading === undefined || [...reading].length < MIN_READING || skip.has(reading) || isCutFromKatakana(word) || isCopulaReadAsVerb(word)) return undefined;
|
|
192
229
|
return `${token.pos}|${useOf(word, skips)}|${reading}`;
|
|
@@ -376,6 +413,8 @@ export const orthographicVariant: Detector = (doc, options): Finding[] => {
|
|
|
376
413
|
skip: new Set([...patternsOf(doc, DISTINCT, options.genre), ...patternsOf(doc, FUKUSHI, options.genre)]),
|
|
377
414
|
idioms: new Set(patternsOf(doc, IDIOM, options.genre)),
|
|
378
415
|
conjunctions: new Set(patternsOf(doc, CONJUNCTION, options.genre)),
|
|
416
|
+
judging: new Set(patternsOf(doc, JUDGING, options.genre)),
|
|
417
|
+
comparative: new Set(patternsOf(doc, COMPARATIVE, options.genre)),
|
|
379
418
|
wholeWords: new Map((doc.lexicons[WHOLE_WORD] ?? []).map((entry) => [entry.pattern, entry.group ?? entry.pattern])),
|
|
380
419
|
phraseSuffixes,
|
|
381
420
|
nouns: documentNounKeys(
|
|
@@ -8,7 +8,7 @@ import { digitRunAround, endsWithDivisionLabel, isNumberName, sequenceLabelStart
|
|
|
8
8
|
import { isSpacedLocationPart, type FloorWords } from "../location-part.ts";
|
|
9
9
|
import { isFieldSeparator, isNameSeparator, type OrganizationWords } from "../name-separator.ts";
|
|
10
10
|
import { addressSpans, clockTimeSpans, identifierSpans, isInsideSpan } from "../spacing-code.ts";
|
|
11
|
-
import { isBoundSuffix, isJoinerSpace } from "../latin-word-boundary.ts";
|
|
11
|
+
import { followsHandle, isBoundSuffix, isJoinerSpace } from "../latin-word-boundary.ts";
|
|
12
12
|
|
|
13
13
|
/** 書いた形の出現に、動詞の活用した形(行う → 行います)を足したもの。同じ位置は一度だけ。 */
|
|
14
14
|
const avoidedStarts = (sentence: Sentence, entry: LexiconEntry, use: string): number[] => {
|
|
@@ -104,8 +104,12 @@ type SentenceCodes = {
|
|
|
104
104
|
* 「本社 5階」のように名前の後ろに空白で区切った階・部屋の番号の前の空白も、所在の組の区切りなので数えない(location-part.ts)。
|
|
105
105
|
* 「経営企画部 IR担当」のように名前だけを並べた行で、名前と名前を分ける空白も、「所属:経済学科 1年」のように項目の値の組を分ける空白も数えない(name-separator.ts)。
|
|
106
106
|
*/
|
|
107
|
+
/** 項目の番号・識別子・チャンネルや宛先の名前(#pj_oss-team)の端の境目。どれも名前や番号を閉じる空白で、空け方の好みではない。 */
|
|
108
|
+
const closesName = (sentence: Sentence, boundary: Boundary, context: NumberContext, codes: SentenceCodes): boolean =>
|
|
109
|
+
followsItemNumber(sentence.text, boundary, context.itemNumbers) || besideSpan(codes.identifiers, boundary) || followsHandle(sentence.text, boundary);
|
|
110
|
+
|
|
107
111
|
const isCounted = (sentence: Sentence, boundary: Boundary, context: NumberContext, codes: SentenceCodes): boolean => {
|
|
108
|
-
if (
|
|
112
|
+
if (closesName(sentence, boundary, context, codes)) return false;
|
|
109
113
|
if (boundary.kind !== "after-digit" && endsWithDivisionLabel(sentence.text.slice(0, boundary.offset), context.divisions)) return false;
|
|
110
114
|
if (boundary.kind === "letter")
|
|
111
115
|
return (
|
|
@@ -26,7 +26,7 @@ import {
|
|
|
26
26
|
shapeOf,
|
|
27
27
|
} from "./list-item.ts";
|
|
28
28
|
import { listReader, type ItemList } from "./list-items.ts";
|
|
29
|
-
import { APPOSITIVE_ANCHOR, isListConjunction, listSentenceOf, VERBAL, type ItemScope, type ListWords } from "./list-sentence.ts";
|
|
29
|
+
import { APPOSITIVE_ANCHOR, isListConjunction, listSentenceOf, NOMINAL, VERBAL, type ItemScope, type ListWords } from "./list-sentence.ts";
|
|
30
30
|
import {
|
|
31
31
|
after,
|
|
32
32
|
contentOnlyWith,
|
|
@@ -42,10 +42,12 @@ import {
|
|
|
42
42
|
some,
|
|
43
43
|
wholeList,
|
|
44
44
|
withoutFirst,
|
|
45
|
+
type Member,
|
|
45
46
|
type View,
|
|
46
47
|
} from "./list-view.ts";
|
|
47
48
|
import { lowerBound, type TokenRange } from "./token-column.ts";
|
|
48
49
|
import { followsPostcode } from "./postcode-end.ts";
|
|
50
|
+
import { dashBreaks } from "./dash-break.ts";
|
|
49
51
|
|
|
50
52
|
const EMPTY: TokenRange = { start: 0, end: 0 };
|
|
51
53
|
|
|
@@ -109,6 +111,31 @@ const afterClosedList = (view: View): View => {
|
|
|
109
111
|
return hasContent(scope, rest) ? { ...after(view, closing), pre: rest } : withoutLead(after(view, closing));
|
|
110
112
|
};
|
|
111
113
|
|
|
114
|
+
const opensAsNoun = (scope: ItemScope, item: TokenRange): boolean => {
|
|
115
|
+
const opening = openingOf(scope, item);
|
|
116
|
+
return opening !== undefined && (opening.pos === "DET" || NOMINAL.has(opening.pos));
|
|
117
|
+
};
|
|
118
|
+
|
|
119
|
+
const opensAsPredicate = (scope: ItemScope, item: TokenRange): boolean => {
|
|
120
|
+
const opening = openingOf(scope, item);
|
|
121
|
+
return opening !== undefined && VERBAL.has(opening.pos) && !isParticiple(opening);
|
|
122
|
+
};
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* 読点で挟んだ例の句が主語と述語のあいだにある形(Any services not listed above, such as connected services, are excluded and
|
|
126
|
+
* are not authorized)。句は読点の直後から始まり、例は名詞の句で、句の後ろの項目は動詞で始まる。主語も句も並びの項目ではなく、
|
|
127
|
+
* 並びがあるなら句の後ろから(are excluded and are not authorized は 2 つ)。例の中の分詞(connected)は名詞にかかる語で、
|
|
128
|
+
* 述語と同じ形と読まない。句の前は並びの最初の項目で、名詞の句(主語)。動詞で始まれば述語の並びの項目
|
|
129
|
+
* (Review backups, such as daily snapshots, update indexes and archive logs は 3 つ)。
|
|
130
|
+
*/
|
|
131
|
+
const isAsideBeforePredicate = (view: View, phrase: Member, examples: TokenRange): boolean => {
|
|
132
|
+
const { scope } = view.list;
|
|
133
|
+
const [subject, next] = [firstOf(view), firstOf(after(view, phrase))];
|
|
134
|
+
if (subject === undefined || next === undefined || subject.j !== phrase.j - 1) return false;
|
|
135
|
+
const opensPhrase = (scope.sentence.exampleEnds[phrase.item.start]?.length ?? 0) > 0;
|
|
136
|
+
return opensPhrase && opensAsNoun(scope, subject.item) && opensAsNoun(scope, examples) && opensAsPredicate(scope, next.item);
|
|
137
|
+
};
|
|
138
|
+
|
|
112
139
|
/**
|
|
113
140
|
* 例の句(such as / including / e.g.)のあとが並び。句の前の項目は並びではない(the screen lock on your phone, such as a PIN or
|
|
114
141
|
* phone-based fingerprint は 2 つ)。句の中の項目の形が揃わなければ句で切らない(materials such as green steel, utilizing ..., and
|
|
@@ -121,7 +148,9 @@ const fromExample = (view: View): View => {
|
|
|
121
148
|
const { scope } = view.list;
|
|
122
149
|
const examples = { start: exampleEnd(scope, phrase.item), end: phrase.item.end };
|
|
123
150
|
if (hasListConjunction(scope, examples)) return view;
|
|
124
|
-
const
|
|
151
|
+
const rest = after(view, phrase);
|
|
152
|
+
if (isAsideBeforePredicate(view, phrase, examples)) return rest;
|
|
153
|
+
const inPhrase = contentOnlyWith(rest, examples);
|
|
125
154
|
return sameShapes(inPhrase) ? inPhrase : view;
|
|
126
155
|
};
|
|
127
156
|
|
|
@@ -288,10 +317,13 @@ export const listVerdicts = (tokens: readonly Token[], words: ListWords, source:
|
|
|
288
317
|
if (!tokens.some(isJudgedConjunction)) return [];
|
|
289
318
|
const sentence = listSentenceOf(tokens, words, source);
|
|
290
319
|
const readItems = listReader(sentence);
|
|
320
|
+
const breaks = dashBreaks(tokens, source);
|
|
291
321
|
return tokens.flatMap((token, at) => {
|
|
292
322
|
if (!isJudgedConjunction(token, at)) return [];
|
|
293
323
|
// 郵便番号で終わる住所のあとの and / or は、住所の行を区切る読点のあと。並びの読点ではない。
|
|
294
|
-
|
|
324
|
+
// 最後の読点のあとに前後を空けたダッシュがあれば(Washington, DC – The …)、読点はダッシュの前で閉じた日付行や見出しのもの。
|
|
325
|
+
const notList = sentence.inCitedTitle(at) || followsPostcode(source, token.span.start) || breaks[at] === true;
|
|
326
|
+
return [notList ? undefined : listAt(readItems(at), at)];
|
|
295
327
|
});
|
|
296
328
|
};
|
|
297
329
|
|
|
@@ -12,8 +12,20 @@ export type WrittenRange = { readonly offset: number; readonly written: string;
|
|
|
12
12
|
const FULLWIDTH_OFFSET = 0xfee0;
|
|
13
13
|
const NUMBER = "[0-90-9]+(?:[..][0-90-9]+)?";
|
|
14
14
|
|
|
15
|
-
/**
|
|
16
|
-
const UNIT_AFTER =
|
|
15
|
+
/** 範囲のすぐ後ろ(同じ行の空白一つまで)の単位や語。番号にも使う記号(ハイフン)は、これが続くときだけ範囲と読む。 */
|
|
16
|
+
const UNIT_AFTER = /^[^\S\n]?[\p{L}%%]/u;
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* 漢字の語に詰めて書いた番号の後ろに仮名が続けば、文書の番号(直資4-5ほか、課資3-1の)。範囲なら後ろは単位(約3-5日)で、
|
|
20
|
+
* 仮名で始まる単位(約3-5か月、3-5つ)は語彙表 range-kana-unit が言う。
|
|
21
|
+
*/
|
|
22
|
+
const CODE_BEFORE = /\p{Script=Han}$/u;
|
|
23
|
+
const KANA_AFTER = /^[^\S\n]?(\p{Script=Hiragana}.*)/u;
|
|
24
|
+
|
|
25
|
+
const isDocumentNumber = (lineBefore: string, after: string, kanaUnits: readonly string[]): boolean => {
|
|
26
|
+
const kana = KANA_AFTER.exec(after)?.[1];
|
|
27
|
+
return CODE_BEFORE.test(lineBefore) && kana !== undefined && !kanaUnits.some((unit) => kana.startsWith(unit));
|
|
28
|
+
};
|
|
17
29
|
|
|
18
30
|
/** 行の頭(見出しや箇条書きの印の後ろ)の「1-2」は節の番号、閉じ括弧のすぐ後ろの「(703) 292-7827」は電話番号。 */
|
|
19
31
|
const LABEL_BEFORE = /(?:^\s*[#>*+|-][\s#>*+|-]*|\)\s?)$/u;
|
|
@@ -30,9 +42,15 @@ const halfWidth = (text: string): string => text.replace(/[0-9.]/gu, (char
|
|
|
30
42
|
/**
|
|
31
43
|
* 文字列の中の数の範囲。前後にさらに数が続く並び(日付 2026-10-02、電話番号、時刻 10:00、版 1.2-3)は範囲ではない。
|
|
32
44
|
* 終わりが始まりより小さいもの(郵便番号 100-0001、試合の 3-2)も範囲ではない。ambiguous の記号は、後ろに単位や語が続き、
|
|
33
|
-
* 前が番号の札(図 2-1、Section 1-2、NSF 13-542
|
|
45
|
+
* 前が番号の札(図 2-1、Section 1-2、NSF 13-542)でも文書の番号(直資4-5ほか)でもないときだけ。
|
|
34
46
|
*/
|
|
35
|
-
export const rangesIn = (
|
|
47
|
+
export const rangesIn = (
|
|
48
|
+
text: string,
|
|
49
|
+
marks: readonly string[],
|
|
50
|
+
ambiguous: readonly string[] = [],
|
|
51
|
+
labels: readonly string[] = [],
|
|
52
|
+
kanaUnits: readonly string[] = [],
|
|
53
|
+
): WrittenRange[] => {
|
|
36
54
|
if (marks.length === 0) return [];
|
|
37
55
|
const joiner = marks
|
|
38
56
|
.toSorted((left, right) => right.length - left.length)
|
|
@@ -46,9 +64,10 @@ export const rangesIn = (text: string, marks: readonly string[], ambiguous: read
|
|
|
46
64
|
const from = Number(halfWidth(match.groups?.["from"] ?? ""));
|
|
47
65
|
const to = Number(halfWidth(match.groups?.["to"] ?? ""));
|
|
48
66
|
const mark = match.groups?.["mark"] ?? "";
|
|
49
|
-
const
|
|
67
|
+
const after = text.slice(match.index + match[0].length);
|
|
50
68
|
const lineBefore = text.slice(text.lastIndexOf("\n", match.index - 1) + 1, match.index);
|
|
51
|
-
|
|
69
|
+
const readsAsRange = UNIT_AFTER.test(after) && !isNumberLabel(lineBefore, labels) && !isDocumentNumber(lineBefore, after, kanaUnits);
|
|
70
|
+
return to > from && !LABEL_BEFORE.test(lineBefore) && (!ambiguous.includes(mark) || readsAsRange)
|
|
52
71
|
? [{ offset: match.index, written: match[0], form: mark }]
|
|
53
72
|
: [];
|
|
54
73
|
});
|
|
@@ -62,6 +81,7 @@ export const rangeNotation: Detector = (doc, options): Finding[] => {
|
|
|
62
81
|
entries.map((entry) => entry.pattern),
|
|
63
82
|
entries.filter((entry) => entry.group === "ambiguous").map((entry) => entry.pattern),
|
|
64
83
|
["figure-label", "numbered-division", "numbered-label"].flatMap((id) => (doc.lexicons[id] ?? []).map((entry) => entry.pattern)),
|
|
84
|
+
(doc.lexicons["range-kana-unit"] ?? []).map((entry) => entry.pattern),
|
|
65
85
|
);
|
|
66
86
|
const minority = formMinority(ranges, options.limit);
|
|
67
87
|
if (minority === undefined) return [];
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { Detector, Finding, Lexicon, LexiconEntry, ProseDocument, Sentence } from "../plugin.ts";
|
|
2
|
+
import { entriesForGenre } from "../lexicon-genres.ts";
|
|
2
3
|
import { entryCloses, entryIn, entryRanges } from "./lexicon-match.ts";
|
|
3
4
|
import { classWordsOf, isClassCue, namesClassAfter } from "./open-end-class.ts";
|
|
4
5
|
import { glossesCondition } from "./open-end-condition.ts";
|
|
@@ -64,9 +65,12 @@ const boundedInTime = (doc: ProseDocument, sentence: Sentence, entry: LexiconEnt
|
|
|
64
65
|
(doc.lexicons[CONCRETE_DEADLINE] ?? []).map((limit) => limit.pattern),
|
|
65
66
|
);
|
|
66
67
|
|
|
67
|
-
/**
|
|
68
|
-
|
|
69
|
-
|
|
68
|
+
/**
|
|
69
|
+
* A lexicon's words that hold in the document's genre ("but not limited to" is a term of art in a contract); the open-end
|
|
70
|
+
* lexicon also holds the cues of a class named after the list and of what holds it, which are no smell.
|
|
71
|
+
*/
|
|
72
|
+
const smellWords = (doc: ProseDocument, lexicon: string, genre: string | undefined): LexiconEntry[] =>
|
|
73
|
+
entriesForGenre(doc.lexicons[lexicon] ?? [], genre).filter((entry) => !isClassCue(entry) && !isScopeCue(entry));
|
|
70
74
|
|
|
71
75
|
/**
|
|
72
76
|
* Where the word stands as a smell. An open-end word inside a name the document gives (業務執行取締役等), giving examples
|
|
@@ -96,9 +100,9 @@ const glossesConditionAt = (doc: ProseDocument, sentence: Sentence, lexicon: str
|
|
|
96
100
|
);
|
|
97
101
|
|
|
98
102
|
/** The first word of each smell in one requirement sentence: one finding per smell, not per word. */
|
|
99
|
-
const smellsIn = (doc: ProseDocument, sentence: Sentence, names: () => OpenEndNames): Finding[] =>
|
|
103
|
+
const smellsIn = (doc: ProseDocument, sentence: Sentence, genre: string | undefined, names: () => OpenEndNames): Finding[] =>
|
|
100
104
|
SMELLS.flatMap(({ lexicon, variant }) =>
|
|
101
|
-
smellWords(doc, lexicon)
|
|
105
|
+
smellWords(doc, lexicon, genre)
|
|
102
106
|
.flatMap((entry) => {
|
|
103
107
|
const offset = smellAt(doc, sentence, entry, lexicon, names);
|
|
104
108
|
const examples = offset !== undefined && (answersQuestion(doc, sentence, entry, lexicon, offset) || glossesConditionAt(doc, sentence, lexicon, offset));
|
|
@@ -107,12 +111,12 @@ const smellsIn = (doc: ProseDocument, sentence: Sentence, names: () => OpenEndNa
|
|
|
107
111
|
.slice(0, 1),
|
|
108
112
|
);
|
|
109
113
|
|
|
110
|
-
export const requirementSmell: Detector = (doc): Finding[] => {
|
|
114
|
+
export const requirementSmell: Detector = (doc, options): Finding[] => {
|
|
111
115
|
const markers = doc.lexicons[MARKERS] ?? [];
|
|
112
116
|
const notMarkers = doc.lexicons[NOT_MARKERS] ?? [];
|
|
113
117
|
const statements = (doc.lexicons[STATEMENTS] ?? []).map((entry) => entry.pattern);
|
|
114
118
|
// Read once, and only when a requirement has an open-end word to tell apart from a name.
|
|
115
119
|
const cache: { names?: OpenEndNames } = {};
|
|
116
|
-
const names = (): OpenEndNames => (cache.names ??= openEndNames(doc, smellWords(doc, OPEN_END), statements));
|
|
117
|
-
return doc.sentences.filter((sentence) => isRequirement(sentence, markers, notMarkers)).flatMap((sentence) => smellsIn(doc, sentence, names));
|
|
120
|
+
const names = (): OpenEndNames => (cache.names ??= openEndNames(doc, smellWords(doc, OPEN_END, options.genre), statements));
|
|
121
|
+
return doc.sentences.filter((sentence) => isRequirement(sentence, markers, notMarkers)).flatMap((sentence) => smellsIn(doc, sentence, options.genre, names));
|
|
118
122
|
};
|
|
@@ -69,9 +69,39 @@ const itemEndBefore = (at: number, tokens: readonly Token[]): number => {
|
|
|
69
69
|
/** The index of the token the next item starts with, past spaces. */
|
|
70
70
|
const itemStartAfter = (at: number, tokens: readonly Token[]): number => tokens.findIndex((token, index) => index > at && !isSpace(token));
|
|
71
71
|
|
|
72
|
-
/** A
|
|
72
|
+
/** A prefix bound to the noun after it (第I, 各部署) starts a noun. */
|
|
73
|
+
const isBoundPrefix = (token: Token | undefined): boolean => token?.features?.["Bound"] === "Yes" && token.pos === "ADJ";
|
|
74
|
+
|
|
75
|
+
const startsNoun = (at: number, tokens: readonly Token[]): boolean => {
|
|
76
|
+
const start = itemStartAfter(at, tokens);
|
|
77
|
+
return isNominal(tokens[start]) || (isBoundPrefix(tokens[start]) && isNominal(tokens[start + 1]));
|
|
78
|
+
};
|
|
79
|
+
|
|
80
|
+
const QUOTE_CLOSERS: ReadonlySet<string> = new Set(["」", "』"]);
|
|
81
|
+
const QUOTE_OPENERS: ReadonlySet<string> = new Set(["「", "『"]);
|
|
82
|
+
|
|
83
|
+
/** Whether the token at `closer` closes a quotation of a noun phrase (「技術の確立」), not of a clause (「成功した」). */
|
|
84
|
+
const closesNounQuotation = (closer: number, tokens: readonly Token[]): boolean =>
|
|
85
|
+
QUOTE_CLOSERS.has(tokens[closer]?.surface ?? "") && isNominal(tokens[closer - 1]);
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* A comma from one quoted noun phrase straight to the next (「計画」、「予算」) lists quoted items. A noun before a quotation
|
|
89
|
+
* (対応するもの、「A」に関しては) may end a clause, and so may a quoted clause, so only noun quotation to noun quotation counts.
|
|
90
|
+
*/
|
|
91
|
+
const betweenQuotations = (at: number, tokens: readonly Token[]): boolean => {
|
|
92
|
+
const before = tokens.findLastIndex((token, index) => index < at && !isSpace(token));
|
|
93
|
+
const start = itemStartAfter(at, tokens);
|
|
94
|
+
if (!QUOTE_OPENERS.has(tokens[start]?.surface ?? "")) return false;
|
|
95
|
+
const after = tokens.findIndex((token, index) => index > start && QUOTE_CLOSERS.has(token.surface));
|
|
96
|
+
return closesNounQuotation(before, tokens) && closesNounQuotation(after, tokens);
|
|
97
|
+
};
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* A comma between two nouns separates listed items (名前、会社名、連絡先), not clauses. A bracketed note after an item is part of
|
|
101
|
+
* it, a noun after its prefix (第I、第II) is a noun, and so is a quotation listed next to another.
|
|
102
|
+
*/
|
|
73
103
|
export const separatesItems = (at: number, tokens: readonly Token[]): boolean =>
|
|
74
|
-
isNominal(tokens[itemEndBefore(at, tokens)]) &&
|
|
104
|
+
betweenQuotations(at, tokens) || (isNominal(tokens[itemEndBefore(at, tokens)]) && startsNoun(at, tokens));
|
|
75
105
|
|
|
76
106
|
/** A sentence cut by more than limit of the lexicon's commas (読点). Commas between listed nouns are not counted. */
|
|
77
107
|
export const sentenceCommaCount: Detector = (doc, options): Finding[] => {
|
|
@@ -81,10 +111,16 @@ export const sentenceCommaCount: Detector = (doc, options): Finding[] => {
|
|
|
81
111
|
|
|
82
112
|
const CONJUNCTIVE = "SCONJ";
|
|
83
113
|
|
|
84
|
-
/**
|
|
114
|
+
/** The phrases ending in a prefatory が (繰り返しになりますが、〜の件でありますが): no contrast with what follows. */
|
|
115
|
+
export const PREFATORY_LEXICON = "adversative-ga-prefatory";
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* A sentence joining more than limit clauses with the lexicon's conjunctive particle (逆接の「が」), not the same word as a
|
|
119
|
+
* case particle, and not inside a prefatory phrase.
|
|
120
|
+
*/
|
|
85
121
|
export const sentenceConjunctiveCount: Detector = (doc, options): Finding[] => {
|
|
86
122
|
const words = wordsOf(options.lexicon);
|
|
87
|
-
return sentencesOver(doc, options.limit, (token) => token.pos === CONJUNCTIVE && words.has(token.surface));
|
|
123
|
+
return sentencesOver(doc, options.limit, (token) => token.pos === CONJUNCTIVE && words.has(token.surface), doc.lexicons[PREFATORY_LEXICON] ?? []);
|
|
88
124
|
};
|
|
89
125
|
|
|
90
126
|
type Opened = { readonly sentence: Sentence; readonly opener: string | undefined };
|
|
@@ -1,26 +1,39 @@
|
|
|
1
1
|
import { lengthOf, proseText } from "../measure.ts";
|
|
2
2
|
import type { Detector, DetectorOptions, Finding, LengthUnit, Sentence } from "../plugin.ts";
|
|
3
|
+
import { citationLead } from "./citation-sentence.ts";
|
|
3
4
|
import { pageMarkerPattern, pageMarkerSize } from "./page-marker.ts";
|
|
4
5
|
|
|
5
6
|
type Measure = { readonly size: number; readonly limit: number; readonly variant?: string };
|
|
6
7
|
|
|
8
|
+
/** The page markers' pattern and the citation signals ("see"), read once per document. */
|
|
9
|
+
type SentenceWords = { readonly markers: RegExp | undefined; readonly citationSignals: readonly string[] };
|
|
10
|
+
|
|
11
|
+
const sizeOf = (text: string, unit: LengthUnit): number =>
|
|
12
|
+
unit === "word" ? text.split(/\s+/u).filter((word) => word.length > 0).length : text.replace(/\s+/gu, "").length;
|
|
13
|
+
|
|
7
14
|
/**
|
|
8
15
|
* 文書と違う言語の文(和文の中の英文)は、その言語の単位と上限で測る。rule がその言語の段を持たなければ文書のもので。
|
|
9
16
|
* 印刷版のページの印(( printed page 52044))は文の言葉ではないので数えない。
|
|
17
|
+
* 脚注の文献の記載(著者 (2025), "題," …)も書き手の文ではないので、その前に書き手が置いた言葉(For example, see …)だけを測る。
|
|
10
18
|
*/
|
|
11
|
-
const measureOf = (sentence: Sentence, unit: LengthUnit, options: DetectorOptions,
|
|
19
|
+
const measureOf = (sentence: Sentence, unit: LengthUnit, options: DetectorOptions, words: SentenceWords): Measure => {
|
|
20
|
+
const lead = citationLead(proseText(sentence), words.citationSignals);
|
|
21
|
+
if (lead !== undefined) return { size: sizeOf(lead, unit), limit: options.limit };
|
|
12
22
|
const embedded = sentence.embeddedLanguage;
|
|
13
23
|
const limit = embedded === undefined ? undefined : options.embeddedLimits?.[embedded.id];
|
|
14
24
|
if (embedded === undefined || limit === undefined) {
|
|
15
|
-
return { size: lengthOf(sentence, unit) - pageMarkerSize(proseText(sentence), markers, unit), limit: options.limit };
|
|
25
|
+
return { size: lengthOf(sentence, unit) - pageMarkerSize(proseText(sentence), words.markers, unit), limit: options.limit };
|
|
16
26
|
}
|
|
17
27
|
return { size: lengthOf(sentence, embedded.lengthUnit), limit, variant: `embedded-${embedded.id}` };
|
|
18
28
|
};
|
|
19
29
|
|
|
20
30
|
export const sentenceLength: Detector = (doc, options): Finding[] => {
|
|
21
|
-
const
|
|
31
|
+
const words: SentenceWords = {
|
|
32
|
+
markers: pageMarkerPattern((options.lexicon ?? []).map((entry) => entry.pattern)),
|
|
33
|
+
citationSignals: (doc.lexicons["citation-signal"] ?? []).map((entry) => entry.pattern),
|
|
34
|
+
};
|
|
22
35
|
return doc.sentences
|
|
23
|
-
.map((sentence) => ({ sentence, measure: measureOf(sentence, doc.lengthUnit, options,
|
|
36
|
+
.map((sentence) => ({ sentence, measure: measureOf(sentence, doc.lengthUnit, options, words) }))
|
|
24
37
|
.filter(({ measure }) => measure.size > measure.limit)
|
|
25
38
|
.map(({ sentence, measure }) => ({
|
|
26
39
|
rule: "max-sentence-length",
|
package/src/detectors/signals.ts
CHANGED
|
@@ -8,7 +8,8 @@ import { abbreviates, expansionAt, termEntryAcronyms, type DefinitionWords, type
|
|
|
8
8
|
import { isExplained } from "./acronym-compound.ts";
|
|
9
9
|
import { statedAcronyms } from "./acronym-definition.ts";
|
|
10
10
|
import { isBracketed, isInHeading, isNamePart } from "./acronym-lone.ts";
|
|
11
|
-
import { linkTitleSpans } from "./acronym-link.ts";
|
|
11
|
+
import { linkedListTerms, linkTitleSpans } from "./acronym-link.ts";
|
|
12
|
+
import { caseVariantsIn, spelledOutAcronyms } from "./acronym-mention.ts";
|
|
12
13
|
import { conjugatedForms } from "./conjugated-form.ts";
|
|
13
14
|
import { evidenceSpans, hasNumeral, isClaim, isLabelOnly, startsWithin, type AdviceWords } from "./concrete-evidence.ts";
|
|
14
15
|
import { letteredIndexEntries } from "./lettered-index.ts";
|
|
@@ -325,10 +326,24 @@ export const undefinedAcronym: Detector = (doc, options): Finding[] => {
|
|
|
325
326
|
]);
|
|
326
327
|
const expandedAt = expansionAt(definitionWordsOf(doc));
|
|
327
328
|
const stated = statedAcronyms(doc.source, { statements: patternsOf(doc, "definition-statement"), markers: patternsOf(doc, "definition-marker") });
|
|
328
|
-
|
|
329
|
+
// 一覧の語のすぐ下でその語を名指すリンク(- GPL / [GNU General Public License …](…))と、略語を話題にして書き出した文
|
|
330
|
+
// (the abbreviation DOB … date of birth)も、略語の説明と読む。
|
|
331
|
+
const spelled = spelledOutAcronyms(
|
|
332
|
+
doc.sentences.map((sentence) => sentence.text),
|
|
333
|
+
patternsOf(doc, "abbreviation-mention"),
|
|
334
|
+
);
|
|
335
|
+
const entries = new Set([...termEntryAcronyms(doc.source), ...stated, ...linkedListTerms(doc.source), ...spelled]);
|
|
329
336
|
const headings = doc.sections.map((section) => section.heading);
|
|
330
|
-
const
|
|
337
|
+
const explainedAsWritten = (word: string): boolean =>
|
|
331
338
|
common.has(word) || entries.has(word) || isExpanded(body, word, expandedAt) || headings.some((heading) => expandsInHeading(heading, word, expandedAt));
|
|
339
|
+
// 大文字の違うだけの書き方(TCoP と TCOP)は同じ略語と読む。どちらかで説明してあれば説明済み。
|
|
340
|
+
const written = [body, ...headings].join("\n");
|
|
341
|
+
const explained = new Map<string, boolean>();
|
|
342
|
+
const explainedAlone = (word: string): boolean => {
|
|
343
|
+
const known = explained.get(word) ?? (explainedAsWritten(word) || caseVariantsIn(written, word).some(explainedAsWritten));
|
|
344
|
+
explained.set(word, known);
|
|
345
|
+
return known;
|
|
346
|
+
};
|
|
332
347
|
const counted = outsideSpans(acronymsOf(doc, notationOf(doc)), [
|
|
333
348
|
...referenceListSpans(doc.source, patternsOf(doc, "reference-list-heading")),
|
|
334
349
|
...outlineLabelSpans(doc.source, patternsOf(doc, "numbered-division")),
|
|
@@ -31,3 +31,13 @@ export const namesQuantity = (tokens: readonly Token[], range: TokenRange, quant
|
|
|
31
31
|
const last = single ? lastJoined(tokens, range.end) : undefined;
|
|
32
32
|
return last !== undefined && measures(last, quantities);
|
|
33
33
|
};
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* 最上級の名詞に空白を挟まず接尾辞が付いて、別の語になっているか(最大化・最大化する)。「最大にする」という動詞で、
|
|
37
|
+
* 何かが一番だという主張ではない。接尾辞は言語パッケージの語彙表が持つ。
|
|
38
|
+
*/
|
|
39
|
+
export const makesWord = (tokens: readonly Token[], range: TokenRange, suffixes: Lexicon): boolean => {
|
|
40
|
+
const last = tokens[range.end - 1];
|
|
41
|
+
const next = tokens[range.end];
|
|
42
|
+
return last !== undefined && next !== undefined && last.span.end === next.span.start && suffixes.some((entry) => entry.pattern === next.surface);
|
|
43
|
+
};
|
|
@@ -3,6 +3,7 @@ import { isWithinAny, quotedIn } from "../quoted-span.ts";
|
|
|
3
3
|
import { chronologyRows } from "../chronology-rows.ts";
|
|
4
4
|
import { externalLinkTitleSpans } from "../link-title.ts";
|
|
5
5
|
import type { Detector, Finding, Span, Token } from "../plugin.ts";
|
|
6
|
+
import { countedTokens, uncountedWordsOf } from "../uncounted-particles.ts";
|
|
6
7
|
|
|
7
8
|
/**
|
|
8
9
|
* 文末が名詞で終わる(体言止め)。箇条書きでは普通だが、本文では文が途中で切れて読める。
|
|
@@ -82,8 +83,9 @@ export const doubledParticle: Detector = (doc, options): Finding[] => {
|
|
|
82
83
|
const nesting = (options.lexicon ?? []).map((entry) => entry.pattern);
|
|
83
84
|
const titles = externalLinkTitleSpans(doc.source, doc.links);
|
|
84
85
|
const breaks = (doc.lexicons[RUN_BREAK_LEXICON] ?? []).map((entry) => entry.pattern);
|
|
86
|
+
const numbering = uncountedWordsOf(doc.lexicons, nesting);
|
|
85
87
|
return doc.sentences.flatMap((sentence) =>
|
|
86
|
-
runsOf(sentence.tokens ?? [], nesting, breaks, [...quotedIn(sentence), ...titles])
|
|
88
|
+
runsOf(countedTokens(sentence.tokens ?? [], numbering), nesting, breaks, [...quotedIn(sentence), ...titles])
|
|
87
89
|
.filter((run) => run.tokens.length > options.limit)
|
|
88
90
|
.map((run) => ({
|
|
89
91
|
rule: "no-doubled-joshi",
|
|
@@ -81,6 +81,12 @@ export const undefinedTerms = (source: string, sentences: readonly (readonly Tok
|
|
|
81
81
|
|
|
82
82
|
const patternsOf = (doc: ProseDocument, lexicon: string): string[] => (doc.lexicons[lexicon] ?? []).map((entry) => entry.pattern);
|
|
83
83
|
|
|
84
|
+
/** The words of one group of a lexicon; undefined: the entries with no group. */
|
|
85
|
+
const groupOf = (doc: ProseDocument, lexicon: string, group: string | undefined): string[] =>
|
|
86
|
+
(doc.lexicons[lexicon] ?? []).filter((entry) => entry.group === group).map((entry) => entry.pattern);
|
|
87
|
+
|
|
88
|
+
const ATTACHED = "attached-document-cue";
|
|
89
|
+
|
|
84
90
|
const findingOf = (doc: ProseDocument, use: UndefinedTerm): Finding => ({
|
|
85
91
|
rule: "undefined-term",
|
|
86
92
|
severity: "info",
|
|
@@ -107,6 +113,8 @@ const capitalisedUses = (doc: ProseDocument, sentences: readonly (readonly Token
|
|
|
107
113
|
definedElsewhere: patternsOf(doc, "defined-elsewhere"),
|
|
108
114
|
capacityMarkers: patternsOf(doc, "capacity-marker"),
|
|
109
115
|
articles: patternsOf(doc, "article"),
|
|
116
|
+
attachedCues: groupOf(doc, ATTACHED, undefined),
|
|
117
|
+
attachedTitleEnds: groupOf(doc, ATTACHED, "title-end"),
|
|
110
118
|
};
|
|
111
119
|
const marked = [...(doc.markup?.headings ?? []), ...doc.links, ...setApartSpans(doc.source)];
|
|
112
120
|
return capitalisedTerms({ source: doc.source, readable: readableText(doc.source), sentences, defined, named: namedWords(doc.source), marked, words });
|
|
@@ -13,7 +13,7 @@ const MAX_LENGTH = 30;
|
|
|
13
13
|
const patternsOf = (doc: ProseDocument, name: string): string[] => (doc.lexicons[name] ?? []).map((entry) => entry.pattern.toLowerCase());
|
|
14
14
|
|
|
15
15
|
/** 辞書と、語彙表 extra-word の語と、チームの names と、別の rule が言う誤り(known-misspelling)を足した語。 */
|
|
16
|
-
const spellerOf = (doc: ProseDocument, dictionary: ReadonlySet<string>): Speller => ({
|
|
16
|
+
export const spellerOf = (doc: ProseDocument, dictionary: ReadonlySet<string>): Speller => ({
|
|
17
17
|
words: [dictionary, new Set([...patternsOf(doc, "extra-word"), ...(doc.names ?? []).map((name) => name.toLowerCase())])],
|
|
18
18
|
suffixes: (doc.lexicons["word-suffix"] ?? []).map((entry) => ({ suffix: entry.pattern, replacement: entry.instead_of ?? "" })),
|
|
19
19
|
prefixes: patternsOf(doc, "word-prefix"),
|