chaffjs 0.36.0 → 0.37.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/detectors/acronym-expansion.d.ts.map +1 -1
- package/dist/detectors/acronym-expansion.js +10 -3
- package/dist/detectors/acronym-expansion.js.map +1 -1
- package/dist/detectors/acronym-link.d.ts +5 -0
- package/dist/detectors/acronym-link.d.ts.map +1 -1
- package/dist/detectors/acronym-link.js +33 -0
- package/dist/detectors/acronym-link.js.map +1 -1
- package/dist/detectors/acronym-mention.d.ts +5 -0
- package/dist/detectors/acronym-mention.d.ts.map +1 -0
- package/dist/detectors/acronym-mention.js +34 -0
- package/dist/detectors/acronym-mention.js.map +1 -0
- package/dist/detectors/acronym-name.js +2 -2
- package/dist/detectors/acronym-name.js.map +1 -1
- package/dist/detectors/addressee-title-edge.d.ts +21 -0
- package/dist/detectors/addressee-title-edge.d.ts.map +1 -0
- package/dist/detectors/addressee-title-edge.js +22 -0
- package/dist/detectors/addressee-title-edge.js.map +1 -0
- package/dist/detectors/capitalised-term.d.ts +4 -0
- package/dist/detectors/capitalised-term.d.ts.map +1 -1
- package/dist/detectors/capitalised-term.js +18 -1
- package/dist/detectors/capitalised-term.js.map +1 -1
- package/dist/detectors/caps-caption.d.ts +14 -4
- package/dist/detectors/caps-caption.d.ts.map +1 -1
- package/dist/detectors/caps-caption.js +36 -9
- package/dist/detectors/caps-caption.js.map +1 -1
- package/dist/detectors/char-shape.d.ts.map +1 -1
- package/dist/detectors/char-shape.js +37 -14
- package/dist/detectors/char-shape.js.map +1 -1
- package/dist/detectors/citation-sentence.d.ts +8 -0
- package/dist/detectors/citation-sentence.d.ts.map +1 -0
- package/dist/detectors/citation-sentence.js +40 -0
- package/dist/detectors/citation-sentence.js.map +1 -0
- package/dist/detectors/contraction-mention.d.ts +3 -0
- package/dist/detectors/contraction-mention.d.ts.map +1 -0
- package/dist/detectors/contraction-mention.js +22 -0
- package/dist/detectors/contraction-mention.js.map +1 -0
- package/dist/detectors/dash-break.d.ts +7 -0
- package/dist/detectors/dash-break.d.ts.map +1 -0
- package/dist/detectors/dash-break.js +30 -0
- package/dist/detectors/dash-break.js.map +1 -0
- package/dist/detectors/date-any-year.d.ts.map +1 -1
- package/dist/detectors/date-any-year.js +5 -3
- package/dist/detectors/date-any-year.js.map +1 -1
- package/dist/detectors/definition-use.d.ts.map +1 -1
- package/dist/detectors/definition-use.js +9 -4
- package/dist/detectors/definition-use.js.map +1 -1
- package/dist/detectors/document-shape.d.ts +5 -0
- package/dist/detectors/document-shape.d.ts.map +1 -1
- package/dist/detectors/document-shape.js +15 -1
- package/dist/detectors/document-shape.js.map +1 -1
- package/dist/detectors/duplicate-heading.d.ts +1 -0
- package/dist/detectors/duplicate-heading.d.ts.map +1 -1
- package/dist/detectors/duplicate-heading.js +25 -3
- package/dist/detectors/duplicate-heading.js.map +1 -1
- package/dist/detectors/email-letter.d.ts +9 -2
- package/dist/detectors/email-letter.d.ts.map +1 -1
- package/dist/detectors/email-letter.js +39 -8
- package/dist/detectors/email-letter.js.map +1 -1
- package/dist/detectors/fullwidth-alnum.d.ts.map +1 -1
- package/dist/detectors/fullwidth-alnum.js +7 -1
- package/dist/detectors/fullwidth-alnum.js.map +1 -1
- package/dist/detectors/gram-openers.d.ts +4 -3
- package/dist/detectors/gram-openers.d.ts.map +1 -1
- package/dist/detectors/gram-openers.js +26 -2
- package/dist/detectors/gram-openers.js.map +1 -1
- package/dist/detectors/heading-record-rhythm.d.ts +9 -0
- package/dist/detectors/heading-record-rhythm.d.ts.map +1 -0
- package/dist/detectors/heading-record-rhythm.js +57 -0
- package/dist/detectors/heading-record-rhythm.js.map +1 -0
- package/dist/detectors/kutoten-consistency.d.ts.map +1 -1
- package/dist/detectors/kutoten-consistency.js +7 -0
- package/dist/detectors/kutoten-consistency.js.map +1 -1
- package/dist/detectors/lexicon.d.ts.map +1 -1
- package/dist/detectors/lexicon.js +5 -2
- package/dist/detectors/lexicon.js.map +1 -1
- package/dist/detectors/mark-shape.d.ts +3 -1
- package/dist/detectors/mark-shape.d.ts.map +1 -1
- package/dist/detectors/mark-shape.js +20 -5
- package/dist/detectors/mark-shape.js.map +1 -1
- package/dist/detectors/name-variant.d.ts.map +1 -1
- package/dist/detectors/name-variant.js +6 -0
- package/dist/detectors/name-variant.js.map +1 -1
- package/dist/detectors/organisation-run.d.ts +0 -6
- package/dist/detectors/organisation-run.d.ts.map +1 -1
- package/dist/detectors/organisation-run.js +4 -1
- package/dist/detectors/organisation-run.js.map +1 -1
- package/dist/detectors/orthographic-variant.d.ts.map +1 -1
- package/dist/detectors/orthographic-variant.js +33 -4
- package/dist/detectors/orthographic-variant.js.map +1 -1
- package/dist/detectors/orthography.d.ts.map +1 -1
- package/dist/detectors/orthography.js +4 -2
- package/dist/detectors/orthography.js.map +1 -1
- package/dist/detectors/oxford-comma.d.ts.map +1 -1
- package/dist/detectors/oxford-comma.js +33 -3
- package/dist/detectors/oxford-comma.js.map +1 -1
- package/dist/detectors/range-notation.d.ts +2 -2
- package/dist/detectors/range-notation.d.ts.map +1 -1
- package/dist/detectors/range-notation.js +18 -7
- package/dist/detectors/range-notation.js.map +1 -1
- package/dist/detectors/requirement-smell.d.ts.map +1 -1
- package/dist/detectors/requirement-smell.js +10 -6
- package/dist/detectors/requirement-smell.js.map +1 -1
- package/dist/detectors/sentence-counts.d.ts +10 -2
- package/dist/detectors/sentence-counts.d.ts.map +1 -1
- package/dist/detectors/sentence-counts.js +34 -4
- package/dist/detectors/sentence-counts.js.map +1 -1
- package/dist/detectors/sentence-length.d.ts.map +1 -1
- package/dist/detectors/sentence-length.js +13 -4
- package/dist/detectors/sentence-length.js.map +1 -1
- package/dist/detectors/signals.d.ts.map +1 -1
- package/dist/detectors/signals.js +15 -3
- package/dist/detectors/signals.js.map +1 -1
- package/dist/detectors/superlative-name.d.ts +5 -0
- package/dist/detectors/superlative-name.d.ts.map +1 -1
- package/dist/detectors/superlative-name.js +9 -0
- package/dist/detectors/superlative-name.js.map +1 -1
- package/dist/detectors/token-shape.d.ts.map +1 -1
- package/dist/detectors/token-shape.js +3 -1
- package/dist/detectors/token-shape.js.map +1 -1
- package/dist/detectors/undefined-term.d.ts.map +1 -1
- package/dist/detectors/undefined-term.js +5 -0
- package/dist/detectors/undefined-term.js.map +1 -1
- package/dist/detectors/unknown-word.d.ts +4 -1
- package/dist/detectors/unknown-word.d.ts.map +1 -1
- package/dist/detectors/unknown-word.js +1 -1
- package/dist/detectors/unknown-word.js.map +1 -1
- package/dist/html/html-site-chrome.d.ts.map +1 -1
- package/dist/html/html-site-chrome.js +35 -12
- package/dist/html/html-site-chrome.js.map +1 -1
- package/dist/latin-word-boundary.d.ts +5 -0
- package/dist/latin-word-boundary.d.ts.map +1 -1
- package/dist/latin-word-boundary.js +17 -0
- package/dist/latin-word-boundary.js.map +1 -1
- package/dist/leading-label.d.ts +4 -0
- package/dist/leading-label.d.ts.map +1 -0
- package/dist/leading-label.js +14 -0
- package/dist/leading-label.js.map +1 -0
- package/dist/link-shape.d.ts +3 -2
- package/dist/link-shape.d.ts.map +1 -1
- package/dist/link-shape.js +12 -3
- package/dist/link-shape.js.map +1 -1
- package/dist/name-misspelt-usual.d.ts +5 -0
- package/dist/name-misspelt-usual.d.ts.map +1 -0
- package/dist/name-misspelt-usual.js +19 -0
- package/dist/name-misspelt-usual.js.map +1 -0
- package/dist/notice-header.d.ts +19 -0
- package/dist/notice-header.d.ts.map +1 -0
- package/dist/notice-header.js +31 -0
- package/dist/notice-header.js.map +1 -0
- package/dist/sentence-load.d.ts +2 -1
- package/dist/sentence-load.d.ts.map +1 -1
- package/dist/sentence-load.js +11 -3
- package/dist/sentence-load.js.map +1 -1
- package/dist/structure/attached-parts.d.ts +2 -1
- package/dist/structure/attached-parts.d.ts.map +1 -1
- package/dist/structure/attached-parts.js +22 -9
- package/dist/structure/attached-parts.js.map +1 -1
- package/dist/structure/definition-use.d.ts +5 -2
- package/dist/structure/definition-use.d.ts.map +1 -1
- package/dist/structure/definition-use.js +24 -7
- package/dist/structure/definition-use.js.map +1 -1
- package/dist/structure/gerund-phrase.d.ts +8 -0
- package/dist/structure/gerund-phrase.d.ts.map +1 -0
- package/dist/structure/gerund-phrase.js +30 -0
- package/dist/structure/gerund-phrase.js.map +1 -0
- package/dist/structure/universal.d.ts +7 -3
- package/dist/structure/universal.d.ts.map +1 -1
- package/dist/structure/universal.js +17 -5
- package/dist/structure/universal.js.map +1 -1
- package/dist/title-line.d.ts +2 -0
- package/dist/title-line.d.ts.map +1 -1
- package/dist/title-line.js +2 -1
- package/dist/title-line.js.map +1 -1
- package/dist/topic-clause.d.ts +7 -0
- package/dist/topic-clause.d.ts.map +1 -0
- package/dist/topic-clause.js +87 -0
- package/dist/topic-clause.js.map +1 -0
- package/dist/uncounted-particles.d.ts +18 -0
- package/dist/uncounted-particles.d.ts.map +1 -0
- package/dist/uncounted-particles.js +47 -0
- package/dist/uncounted-particles.js.map +1 -0
- package/package.json +3 -3
- package/rules/adversative-ga-repeat.yaml +1 -0
- package/rules/all-caps-shouting.yaml +2 -2
- package/rules/bare-url-mix.yaml +2 -2
- package/rules/concrete-evidence-density.yaml +1 -0
- package/rules/contraction-consistency.yaml +2 -2
- package/rules/duplicate-definition.yaml +2 -2
- package/rules/duplicate-heading.yaml +2 -2
- package/rules/fullwidth-alnum-consistency.yaml +2 -2
- package/rules/kutoten-consistency.yaml +2 -2
- package/rules/latin-spacing.yaml +2 -2
- package/rules/max-kanji-continuous.yaml +1 -1
- package/rules/max-sentence-length.yaml +2 -2
- package/rules/ngram-repetition.yaml +2 -2
- package/rules/orthographic-variant.yaml +2 -2
- package/rules/oxford-comma-consistency.yaml +2 -2
- package/rules/preamble-length.yaml +3 -2
- package/rules/range-notation-consistency.yaml +2 -2
- package/rules/requirement-smell.yaml +2 -2
- package/rules/title-length.yaml +2 -2
- package/rules/topic-predicate-distance.yaml +2 -2
- package/rules/undefined-acronym.yaml +3 -2
- package/rules/undefined-term.yaml +2 -2
- package/rules/unqualified-superlative.yaml +1 -1
- package/rules/unused-definition.yaml +2 -2
- package/rules/use-before-definition.yaml +2 -2
- package/src/detectors/acronym-expansion.ts +10 -3
- package/src/detectors/acronym-link.ts +38 -0
- package/src/detectors/acronym-mention.ts +44 -0
- package/src/detectors/acronym-name.ts +2 -2
- package/src/detectors/addressee-title-edge.ts +39 -0
- package/src/detectors/capitalised-term.ts +26 -2
- package/src/detectors/caps-caption.ts +43 -9
- package/src/detectors/char-shape.ts +55 -17
- package/src/detectors/citation-sentence.ts +44 -0
- package/src/detectors/contraction-mention.ts +24 -0
- package/src/detectors/dash-break.ts +45 -0
- package/src/detectors/date-any-year.ts +7 -3
- package/src/detectors/definition-use.ts +12 -8
- package/src/detectors/document-shape.ts +20 -1
- package/src/detectors/duplicate-heading.ts +26 -3
- package/src/detectors/email-letter.ts +50 -7
- package/src/detectors/fullwidth-alnum.ts +9 -1
- package/src/detectors/gram-openers.ts +36 -7
- package/src/detectors/heading-record-rhythm.ts +57 -0
- package/src/detectors/kutoten-consistency.ts +9 -0
- package/src/detectors/lexicon.ts +6 -2
- package/src/detectors/mark-shape.ts +30 -5
- package/src/detectors/name-variant.ts +6 -0
- package/src/detectors/organisation-run.ts +6 -1
- package/src/detectors/orthographic-variant.ts +45 -6
- package/src/detectors/orthography.ts +6 -2
- package/src/detectors/oxford-comma.ts +35 -3
- package/src/detectors/range-notation.ts +26 -6
- package/src/detectors/requirement-smell.ts +12 -8
- package/src/detectors/sentence-counts.ts +40 -4
- package/src/detectors/sentence-length.ts +17 -4
- package/src/detectors/signals.ts +18 -3
- package/src/detectors/superlative-name.ts +10 -0
- package/src/detectors/token-shape.ts +3 -1
- package/src/detectors/undefined-term.ts +8 -0
- package/src/detectors/unknown-word.ts +1 -1
- package/src/html/html-site-chrome.ts +70 -13
- package/src/latin-word-boundary.ts +18 -0
- package/src/leading-label.ts +15 -0
- package/src/link-shape.ts +15 -3
- package/src/name-misspelt-usual.ts +25 -0
- package/src/notice-header.ts +57 -0
- package/src/sentence-load.ts +14 -3
- package/src/structure/attached-parts.ts +24 -9
- package/src/structure/definition-use.ts +33 -7
- package/src/structure/gerund-phrase.ts +31 -0
- package/src/structure/universal.ts +22 -7
- package/src/title-line.ts +2 -1
- package/src/topic-clause.ts +109 -0
- package/src/uncounted-particles.ts +72 -0
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
// A bibliographic citation written out as a sentence, as in a footnote: authors and the year in parentheses, then a title
|
|
2
|
+
// or a publisher. Its length is the length of the reference, not of the writer's prose. Pure.
|
|
3
|
+
import { escapeRegExp } from "../orthography.ts";
|
|
4
|
+
|
|
5
|
+
/** "Keys and Reina (2025)," or "Federal Housing Finance Agency (n.d.),": the year in parentheses, a comma after it. */
|
|
6
|
+
const AUTHOR_YEAR = /\((?:\d{4}[a-z]?|n\.d\.)\),/gu;
|
|
7
|
+
|
|
8
|
+
/** What only a reference carries: a quoted title right after the year, a place and its publisher, or a page range. */
|
|
9
|
+
const PUBLICATION_MARKS: readonly RegExp[] = [
|
|
10
|
+
/\((?:\d{4}[a-z]?|n\.d\.)\),\s*["“]/u,
|
|
11
|
+
/\(\p{Lu}[\p{L}.'’-]*(?:,?\s\p{Lu}[\p{L}.'’-]*)*:\s\p{Lu}[^()]*\)/u,
|
|
12
|
+
/\bpp?\.\s*\d+\s*[-–−]\s*\d+/u,
|
|
13
|
+
];
|
|
14
|
+
|
|
15
|
+
/** Where the citation's year stands: the last year before the first mark of a publication, or the first year when none is before it. */
|
|
16
|
+
const citationYearAt = (text: string): number | undefined => {
|
|
17
|
+
const years = [...text.matchAll(AUTHOR_YEAR)].map((match) => match.index);
|
|
18
|
+
const marks = PUBLICATION_MARKS.flatMap((mark) => {
|
|
19
|
+
const found = mark.exec(text);
|
|
20
|
+
return found === null ? [] : [found.index + found[0].length];
|
|
21
|
+
});
|
|
22
|
+
if (years.length === 0 || marks.length === 0) return undefined;
|
|
23
|
+
const firstMark = Math.min(...marks);
|
|
24
|
+
return years.filter((at) => at < firstMark).at(-1) ?? years[0];
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
export const isCitationSentence = (text: string): boolean => citationYearAt(text) !== undefined;
|
|
28
|
+
|
|
29
|
+
const signalPattern = (signals: readonly string[]): RegExp | undefined =>
|
|
30
|
+
signals.length === 0 ? undefined : new RegExp(String.raw`(?<![\p{L}\p{N}])(?:${signals.map(escapeRegExp).join("|")})(?![\p{L}\p{N}])`, "giu");
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* The writer's own words in a citation sentence: what comes before the last citation signal ("see", from the language's
|
|
34
|
+
* word list) ahead of the citation's year, or everything before the year when no signal stands there (the authors' names
|
|
35
|
+
* are short). "For example, see Lisa Cook (2024), …" leads with "For example,". Undefined when the sentence is not a citation.
|
|
36
|
+
*/
|
|
37
|
+
export const citationLead = (text: string, signals: readonly string[]): string | undefined => {
|
|
38
|
+
const yearAt = citationYearAt(text);
|
|
39
|
+
if (yearAt === undefined) return undefined;
|
|
40
|
+
const beforeYear = text.slice(0, yearAt);
|
|
41
|
+
const pattern = signalPattern(signals);
|
|
42
|
+
const lastSignal = pattern === undefined ? undefined : [...beforeYear.matchAll(pattern)].at(-1);
|
|
43
|
+
return (lastSignal === undefined ? beforeYear : beforeYear.slice(0, lastSignal.index)).trim();
|
|
44
|
+
};
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { quotedSpans } from "../quoted-span.ts";
|
|
2
|
+
|
|
3
|
+
// A short phrase in quotation marks ("Do Not Disturb", ‘we’re going to x’) is a name or an example: the form is mentioned,
|
|
4
|
+
// not used, so it says nothing about the document's register. Spoken words carry punctuation ("We don't know," she
|
|
5
|
+
// said) and stay. The text must already have straight apostrophes, so a ’ left in it closes a quotation.
|
|
6
|
+
|
|
7
|
+
const MENTION_MARKS: Readonly<Record<string, string>> = { "“": "”", '"': '"', "‘": "’" };
|
|
8
|
+
/** More words than this in quotation marks is a quoted sentence, not a phrase named as an example. */
|
|
9
|
+
const MAX_MENTION_WORDS = 4;
|
|
10
|
+
const PHRASE = /^[\p{L}' ]+$/u;
|
|
11
|
+
|
|
12
|
+
const isMention = (quoted: string): boolean => {
|
|
13
|
+
const phrase = quoted.trim();
|
|
14
|
+
return PHRASE.test(phrase) && phrase.split(/\s+/u).length <= MAX_MENTION_WORDS;
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
/** The text with each quoted short phrase blanked out, every other character at its place. */
|
|
18
|
+
export const withoutMentions = (text: string): string => {
|
|
19
|
+
const mentions = quotedSpans(text, MENTION_MARKS).filter((span) => isMention(text.slice(span.start, span.end)));
|
|
20
|
+
if (mentions.length === 0) return text;
|
|
21
|
+
const units = text.split("");
|
|
22
|
+
mentions.forEach((span) => units.fill(" ", span.start, span.end));
|
|
23
|
+
return units.join("");
|
|
24
|
+
};
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
// A dash with space on both sides after the last comma before an and / or ("Washington, DC – The Cybersecurity and
|
|
2
|
+
// Infrastructure Security Agency"). The dash closes off what came before it, a dateline here, so the commas before it
|
|
3
|
+
// do not separate the items of a list the and / or ends. Commas and dashes inside brackets or quotation marks are not
|
|
4
|
+
// the and / or's own. Pure, and one pass over the sentence.
|
|
5
|
+
import type { Token } from "../plugin.ts";
|
|
6
|
+
|
|
7
|
+
const DASHES = new Set(["–", "—", "―", "-", "--"]);
|
|
8
|
+
const OPENERS = new Set(["(", "[", "{", "“"]);
|
|
9
|
+
const CLOSERS = new Set([")", "]", "}", "”"]);
|
|
10
|
+
const SPACE = /\s/u;
|
|
11
|
+
|
|
12
|
+
const isSpacedDash = (token: Token, source: string): boolean =>
|
|
13
|
+
DASHES.has(token.surface) && SPACE.test(source.charAt(token.span.start - 1)) && SPACE.test(source.charAt(token.span.end));
|
|
14
|
+
|
|
15
|
+
/** The last comma and the last spaced dash seen at one depth of brackets. */
|
|
16
|
+
type Seen = { comma: number; dash: number };
|
|
17
|
+
|
|
18
|
+
type Scan = { depth: number; readonly seen: Seen[]; readonly breaks: boolean[] };
|
|
19
|
+
|
|
20
|
+
const seenAt = (scan: Scan): Seen => {
|
|
21
|
+
const seen = scan.seen[scan.depth] ?? { comma: -1, dash: -1 };
|
|
22
|
+
scan.seen[scan.depth] = seen;
|
|
23
|
+
return seen;
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* For each token, whether a spaced dash comes between the last comma before it and it, at its depth, and not right
|
|
28
|
+
* before it: a dash right before the and / or sets off its last item (the plan, the budget – and the schedule).
|
|
29
|
+
*/
|
|
30
|
+
export const dashBreaks = (tokens: readonly Token[], source: string): boolean[] =>
|
|
31
|
+
tokens.reduce<Scan>(
|
|
32
|
+
(scan, token, at) => {
|
|
33
|
+
if (CLOSERS.has(token.surface)) {
|
|
34
|
+
scan.seen.length = scan.depth;
|
|
35
|
+
scan.depth = Math.max(0, scan.depth - 1);
|
|
36
|
+
}
|
|
37
|
+
const seen = seenAt(scan);
|
|
38
|
+
scan.breaks.push(seen.comma !== -1 && seen.dash > seen.comma && seen.dash < at - 1);
|
|
39
|
+
if (token.surface === ",") seen.comma = at;
|
|
40
|
+
if (isSpacedDash(token, source)) seen.dash = at;
|
|
41
|
+
if (OPENERS.has(token.surface)) scan.depth += 1;
|
|
42
|
+
return scan;
|
|
43
|
+
},
|
|
44
|
+
{ depth: 0, seen: [], breaks: [] },
|
|
45
|
+
).breaks;
|
|
@@ -24,7 +24,7 @@ export type AnyYearMarker = { readonly word: string; readonly reach: MarkerReach
|
|
|
24
24
|
/** The span of a yearless date in the source. */
|
|
25
25
|
export type DateSpan = { readonly offset: number; readonly end: number };
|
|
26
26
|
|
|
27
|
-
/** How far apart the two ends of a period may stand
|
|
27
|
+
/** How far apart the two ends of a period may stand besides the joiner itself: a year word (「から翌年」, " to the next "). */
|
|
28
28
|
const SPAN_GAP_REACH = 12;
|
|
29
29
|
|
|
30
30
|
/** How many lines above a list item its introducing line is looked for: a longer list is read as no example. */
|
|
@@ -191,7 +191,11 @@ export const anyYearDate = (date: DateSpan, source: string, markers: readonly An
|
|
|
191
191
|
|
|
192
192
|
/** The date opens a period whose other end, just after it, follows a joiner (「4月1日から翌年3月31日」). */
|
|
193
193
|
export const opensSpan = (date: DateSpan, next: DateSpan | undefined, source: string, joiners: readonly string[]): boolean => {
|
|
194
|
-
|
|
194
|
+
const longest = Math.max(0, ...joiners.map((joiner) => joiner.length));
|
|
195
|
+
if (next === undefined || next.offset - date.end > SPAN_GAP_REACH + longest) return false;
|
|
195
196
|
const gap = source.slice(date.end, next.offset).trimStart();
|
|
196
|
-
return
|
|
197
|
+
return (
|
|
198
|
+
!SENTENCE_END.test(gap) &&
|
|
199
|
+
joiners.some((joiner) => joiner !== "" && gap.length - joiner.length <= SPAN_GAP_REACH && gap.startsWith(joiner) && !runsOn(gap, joiner))
|
|
200
|
+
);
|
|
197
201
|
};
|
|
@@ -27,16 +27,20 @@ const findingAt = (doc: ProseDocument, offset: number, values: Readonly<Record<s
|
|
|
27
27
|
const headingsOf = (doc: ProseDocument): BodyText[] =>
|
|
28
28
|
(doc.markup?.headings ?? []).map((heading) => ({ start: heading.start, text: doc.source.slice(heading.start, heading.end) }));
|
|
29
29
|
|
|
30
|
-
/** 定義したのに、定義の外(文と見出し)で一度も使っていない語。 */
|
|
31
|
-
export const unusedDefinition: Detector = (doc): Finding[] =>
|
|
32
|
-
doc.structure === undefined
|
|
33
|
-
? []
|
|
34
|
-
: unusedDefinitions(definedTerms(doc.structure), [...bodyOf(doc), ...headingsOf(doc)], mentionedIn(doc)).map((defined) =>
|
|
35
|
-
findingAt(doc, defined.span.start, { term: defined.term }),
|
|
36
|
-
);
|
|
37
|
-
|
|
38
30
|
const patternsOf = (doc: ProseDocument, lexicon: string): string[] => (doc.lexicons[lexicon] ?? []).map((entry) => entry.pattern);
|
|
39
31
|
|
|
32
|
+
/** 動名詞の後ろに来て、それが目的語を取る動詞だと分かる語(冠詞と所有の語)。 */
|
|
33
|
+
const DETERMINER_LISTS = ["article", "dependent-possessive"] as const;
|
|
34
|
+
|
|
35
|
+
/** 定義したのに、定義の外(文と見出し)で一度も使っていない語。 */
|
|
36
|
+
export const unusedDefinition: Detector = (doc): Finding[] => {
|
|
37
|
+
if (doc.structure === undefined) return [];
|
|
38
|
+
const determiners = new Set(DETERMINER_LISTS.flatMap((list) => patternsOf(doc, list)));
|
|
39
|
+
return unusedDefinitions(definedTerms(doc.structure), [...bodyOf(doc), ...headingsOf(doc)], mentionedIn(doc), determiners).map((defined) =>
|
|
40
|
+
findingAt(doc, defined.span.start, { term: defined.term }),
|
|
41
|
+
);
|
|
42
|
+
};
|
|
43
|
+
|
|
40
44
|
/** 題・見出し(Markdown の見出し、章・節の行、条の前の見出しの行)と、目次・索引の行の上の位置か。 */
|
|
41
45
|
const titleIn = (doc: ProseDocument): IsTitle => {
|
|
42
46
|
const lines = linesOf(doc.source);
|
|
@@ -99,6 +99,24 @@ const QUESTION_END = /[??]["'”’))」』]*$/u;
|
|
|
99
99
|
|
|
100
100
|
export const isQuestionHeading = (heading: string): boolean => QUESTION_END.test(heading.trim());
|
|
101
101
|
|
|
102
|
+
/** A series label in brackets at a heading's end, a number and a colon opening it ("(490: Employee travel)"): the manual's reference. */
|
|
103
|
+
const SERIES_LABEL_END = /[((]\p{N}+[::][^()()]*[))]$/u;
|
|
104
|
+
const QUOTED = /「[^「」]*」/gu;
|
|
105
|
+
const QUOTED_NAME = "「」";
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* The words the heading's writer chose: without a series label at its end, and with another document's title, quoted
|
|
109
|
+
* right before what is done to it (「…の取扱いについて」等の一部改正, the cited-title-action word list), as one name.
|
|
110
|
+
*/
|
|
111
|
+
export const ownWordsOf = (heading: string, citedTitleActions: readonly string[]): string =>
|
|
112
|
+
heading
|
|
113
|
+
.trimEnd()
|
|
114
|
+
.replace(SERIES_LABEL_END, "")
|
|
115
|
+
.trimEnd()
|
|
116
|
+
.replaceAll(QUOTED, (quote, offset: number) =>
|
|
117
|
+
citedTitleActions.some((action) => action !== "" && heading.startsWith(action, offset + quote.length)) ? QUOTED_NAME : quote,
|
|
118
|
+
);
|
|
119
|
+
|
|
102
120
|
/** The heading's words without a number label at its head (Chapter 2, 1.), from the section the heading opens. */
|
|
103
121
|
const unlabeledOf = (doc: ProseDocument, heading: MarkupHeading): string =>
|
|
104
122
|
doc.sections.find((section) => section.span.start === heading.end)?.unlabeledHeading ?? heading.text;
|
|
@@ -107,10 +125,11 @@ const unlabeledOf = (doc: ProseDocument, heading: MarkupHeading): string =>
|
|
|
107
125
|
export const titleLength: Detector = (doc, options): Finding[] => {
|
|
108
126
|
const headings = writtenHeadings(doc);
|
|
109
127
|
const title = headings.find((heading) => heading.depth === 1);
|
|
128
|
+
const citedTitleActions = (doc.lexicons["cited-title-action"] ?? []).map((entry) => entry.pattern);
|
|
110
129
|
return headings
|
|
111
130
|
.map((heading) => ({ heading, words: unlabeledOf(doc, heading) }))
|
|
112
131
|
.filter(({ words }) => !isParagraphHeading(words) && !isQuestionHeading(words))
|
|
113
|
-
.map(({ heading, words }) => ({ heading, count: headingLength(words, doc.lengthUnit) }))
|
|
132
|
+
.map(({ heading, words }) => ({ heading, count: headingLength(ownWordsOf(words, citedTitleActions), doc.lengthUnit) }))
|
|
114
133
|
.filter(({ count }) => count > options.limit)
|
|
115
134
|
.map(({ heading, count }) => ({
|
|
116
135
|
...findingAt(doc, heading, { heading: heading.text.trim(), count, limit: options.limit }, heading === title ? "title" : undefined),
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { Detector, Finding, MarkupHeading } from "../plugin.ts";
|
|
2
2
|
import { lineNumberAt, linesOf } from "../structure/lines.ts";
|
|
3
|
+
import { recordLabelOf, recordRhythmMask } from "./heading-record-rhythm.ts";
|
|
3
4
|
import { findingAt, writtenHeadings } from "./markup-finding.ts";
|
|
4
5
|
|
|
5
6
|
/** 見出しの言葉を比べる形。全角と半角、大文字と小文字、空白の数は同じ見出しの違いにしない。 */
|
|
@@ -21,22 +22,44 @@ export type RepeatedHeading = { readonly heading: MarkupHeading; readonly first:
|
|
|
21
22
|
|
|
22
23
|
type Seen = Map<number, Map<string, MarkupHeading>>;
|
|
23
24
|
|
|
25
|
+
/** 見出しごとに、同じ親の下の兄弟の型の並びで、決まった見出しの組をくり返す所にあるか(recordRhythmMask)。 */
|
|
26
|
+
const inRecordRhythm = (parents: readonly number[], keys: readonly string[]): boolean[] => {
|
|
27
|
+
const byParent = new Map<number, number[]>();
|
|
28
|
+
parents.forEach((parent, index) => {
|
|
29
|
+
const siblings = byParent.get(parent) ?? [];
|
|
30
|
+
siblings.push(index);
|
|
31
|
+
byParent.set(parent, siblings);
|
|
32
|
+
});
|
|
33
|
+
const inRhythm = keys.map(() => false);
|
|
34
|
+
byParent.forEach((indexes) => {
|
|
35
|
+
const mask = recordRhythmMask(indexes.map((index) => recordLabelOf(keys[index] ?? "")));
|
|
36
|
+
indexes.forEach((index, at) => {
|
|
37
|
+
inRhythm[index] = mask[at] === true;
|
|
38
|
+
});
|
|
39
|
+
});
|
|
40
|
+
return inRhythm;
|
|
41
|
+
};
|
|
42
|
+
|
|
24
43
|
/**
|
|
25
44
|
* 同じ親の下で、前の見出しと同じ言葉の見出し(markdownlint の MD024 siblings_only)。変更履歴の版ごとの「Added」「Fixed」のように、
|
|
26
45
|
* 親が違えば同じ言葉でもよい。深さが違っても親が同じなら兄弟として比べる(飛んだ深さの見出しも同じ親の下にある)。
|
|
46
|
+
* 事例ごとに決まった見出しの組をくり返す所(recordRhythmMask)は、文書の作りとしての重複なので言わない。
|
|
27
47
|
*/
|
|
28
48
|
export const repeatedSiblingHeadings = (headings: readonly MarkupHeading[]): RepeatedHeading[] => {
|
|
29
49
|
const parents = parentIndexes(headings);
|
|
50
|
+
const keys = headings.map((heading) => headingKey(heading.text));
|
|
51
|
+
const inRhythm = inRecordRhythm(parents, keys);
|
|
30
52
|
const seen: Seen = new Map();
|
|
31
|
-
|
|
32
|
-
const key =
|
|
53
|
+
const repeats = headings.flatMap((heading, index) => {
|
|
54
|
+
const key = keys[index] ?? "";
|
|
33
55
|
const parent = parents[index] ?? -1;
|
|
34
56
|
const siblings = seen.get(parent) ?? new Map<string, MarkupHeading>();
|
|
35
57
|
seen.set(parent, siblings);
|
|
36
58
|
const first = siblings.get(key);
|
|
37
59
|
if (first === undefined) siblings.set(key, heading);
|
|
38
|
-
return first === undefined || key === "" ? [] : [{ heading, first }];
|
|
60
|
+
return first === undefined || key === "" || inRhythm[index] === true ? [] : [{ heading, first }];
|
|
39
61
|
});
|
|
62
|
+
return repeats;
|
|
40
63
|
};
|
|
41
64
|
|
|
42
65
|
export const duplicateHeading: Detector = (doc): Finding[] => {
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
// A work email or letter's own parts: a greeting and a closing, a subject line, and an attachment the text mentions.
|
|
2
2
|
// Pure. The words (お世話になっております, Dear, Best regards, 件名, attached) come from the language's email-greeting,
|
|
3
|
-
// email-closing, email-sentence-end, memo-header-label, email-subject-field, attachment-mention, attachment-not-mention and attachment-line lexicons. The
|
|
3
|
+
// email-closing, email-sentence-end, memo-header-label, notice-header, email-subject-field, attachment-mention, attachment-not-mention and attachment-line lexicons. The
|
|
4
4
|
// header and the signature are masked in doc.prose; a quoted reply is someone else's words and is left out.
|
|
5
5
|
import { escapeRegExp } from "../orthography.ts";
|
|
6
6
|
import { quoteAt } from "./structure-tree.ts";
|
|
7
|
+
import { isNoticeHead, noticeWordsOf } from "../notice-header.ts";
|
|
7
8
|
import type { Detector, Finding, ProseDocument, Span } from "../plugin.ts";
|
|
8
9
|
|
|
9
10
|
/** A greeting comes before the body: past the subject, the addressee (株式会社… / 総務部 山田様) and the writer's name. */
|
|
@@ -123,18 +124,51 @@ export const greetingLineCount = (lines: readonly BodyLine[], sentenceEnds: read
|
|
|
123
124
|
return Math.max(GREETING_LINES, firstSentence + 1);
|
|
124
125
|
};
|
|
125
126
|
|
|
126
|
-
/** A
|
|
127
|
+
/** A signing line: a rule of underscores the writer signs on, alone on its line (not "Date: ____", a field to fill in). */
|
|
128
|
+
const SIGNING_LINE = /^[ \t]*_{5,}[ \t]*$/u;
|
|
129
|
+
|
|
130
|
+
/** The signer's name or title under a signing line is short; a paragraph there means the underscores were a section break. */
|
|
131
|
+
const MAX_SIGNER_LENGTH = 60;
|
|
132
|
+
|
|
133
|
+
/** Whether the next line with text is a signer's line: short, holding a letter, not a heading. */
|
|
134
|
+
const signerBelow = (lines: readonly SourceLine[], index: number): boolean => {
|
|
135
|
+
const below = nextText(lines, index);
|
|
136
|
+
const signer = withoutMarks(below);
|
|
137
|
+
return !below.startsWith("#") && [...signer].length <= MAX_SIGNER_LENGTH && LETTER.test(signer);
|
|
138
|
+
};
|
|
139
|
+
|
|
140
|
+
/** Where each signing line starts in the source: underscores alone on a line, a signer's line below, outside a quoted reply. */
|
|
141
|
+
export const signingLines = (source: string, replyQuotes: readonly Span[] = []): number[] => {
|
|
142
|
+
const quoted = insideCursor(replyQuotes);
|
|
143
|
+
const lines = linesWithOffsets(source);
|
|
144
|
+
return lines.flatMap(({ start, text }, index) => (!quoted(start) && SIGNING_LINE.test(text) && signerBelow(lines, index) ? [start] : []));
|
|
145
|
+
};
|
|
146
|
+
|
|
147
|
+
/** Whether a signing line stands among the part's last lines or between them and the next part: the part is signed off. */
|
|
148
|
+
const signedOff = (part: readonly BodyLine[], next: BodyLine | undefined, signings: readonly number[]): boolean => {
|
|
149
|
+
const from = part.slice(-CLOSING_LINES)[0]?.start ?? Infinity;
|
|
150
|
+
return signings.some((offset) => offset >= from && offset < (next?.start ?? Infinity));
|
|
151
|
+
};
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* A missing greeting (none in the first lines, or past the letter's header) and a missing closing: none in the last lines
|
|
155
|
+
* of any part, and no part signed off on a signing line (a letter signed before an acceptance block the recipient signs).
|
|
156
|
+
*/
|
|
127
157
|
export const frameSlips = (
|
|
128
158
|
lines: readonly BodyLine[],
|
|
129
159
|
greetings: readonly string[],
|
|
130
160
|
closings: readonly string[],
|
|
131
161
|
sentenceEnds: readonly string[],
|
|
162
|
+
signings: readonly number[] = [],
|
|
132
163
|
): FrameSlip[] => {
|
|
133
164
|
const first = lines[0];
|
|
134
165
|
const last = lines.at(-1);
|
|
135
166
|
if (first === undefined || last === undefined) return [];
|
|
136
167
|
const greeted = lines.slice(0, greetingLineCount(lines, sentenceEnds)).some((line) => lineHas(line.text, greetings));
|
|
137
|
-
const
|
|
168
|
+
const parts = partsOf(lines);
|
|
169
|
+
const closed = parts.some(
|
|
170
|
+
(part, index) => part.slice(-CLOSING_LINES).some((line) => lineHas(line.text, closings)) || signedOff(part, parts[index + 1]?.[0], signings),
|
|
171
|
+
);
|
|
138
172
|
return [
|
|
139
173
|
...(greeted ? [] : [{ variant: "greeting" as const, offset: first.start }]),
|
|
140
174
|
...(closed ? [] : [{ variant: "closing" as const, offset: last.start }]),
|
|
@@ -169,17 +203,26 @@ export const isMemoLabel = (line: string, labels: readonly string[]): boolean =>
|
|
|
169
203
|
* one). The source, not the body, since a block of TO: / FROM: lines is masked as an email's header. A memo takes no
|
|
170
204
|
* greeting and no closing.
|
|
171
205
|
*/
|
|
172
|
-
export const isMemo = (source: string, labels: readonly string[], sentenceEnds: readonly string[]): boolean =>
|
|
206
|
+
export const isMemo = (source: string, labels: readonly string[], sentenceEnds: readonly string[]): boolean =>
|
|
207
|
+
headLines(source, sentenceEnds).some((line) => isMemoLabel(line, labels));
|
|
208
|
+
|
|
209
|
+
/** The source's lines before the first sentence (the first lines, when no line ends one), past blank lines and a quoted reply. */
|
|
210
|
+
export const headLines = (source: string, sentenceEnds: readonly string[]): string[] => {
|
|
173
211
|
const lines = source.split("\n").filter((line) => line.trim() !== "" && !line.trimStart().startsWith(">"));
|
|
174
212
|
const firstSentence = lines.findIndex((line) => endsSentence(line, sentenceEnds));
|
|
175
|
-
return lines.slice(0, firstSentence === -1 ? HEADER_LINES : firstSentence)
|
|
213
|
+
return lines.slice(0, firstSentence === -1 ? HEADER_LINES : firstSentence);
|
|
176
214
|
};
|
|
177
215
|
|
|
216
|
+
/** A memo (TO: / 発信者:) or an official notice (課資2-7 / 各国税局長 殿) takes no greeting and no closing. */
|
|
217
|
+
const takesNoFrame = (doc: ProseDocument, sentenceEnds: readonly string[]): boolean =>
|
|
218
|
+
isMemo(doc.source, patternsOf(doc, "memo-header-label"), sentenceEnds) ||
|
|
219
|
+
isNoticeHead(headLines(doc.source, sentenceEnds), noticeWordsOf(doc.lexicons["notice-header"] ?? []));
|
|
220
|
+
|
|
178
221
|
const frameSlipsOf = (doc: ProseDocument): FrameSlip[] => {
|
|
179
222
|
const sentenceEnds = patternsOf(doc, "email-sentence-end");
|
|
180
|
-
if (
|
|
223
|
+
if (takesNoFrame(doc, sentenceEnds)) return [];
|
|
181
224
|
const lines = bodyLines(doc.source, doc.prose ?? doc.source, doc.replyQuotes ?? []);
|
|
182
|
-
return frameSlips(lines, patternsOf(doc, "email-greeting"), patternsOf(doc, "email-closing"), sentenceEnds);
|
|
225
|
+
return frameSlips(lines, patternsOf(doc, "email-greeting"), patternsOf(doc, "email-closing"), sentenceEnds, signingLines(doc.source, doc.replyQuotes ?? []));
|
|
183
226
|
};
|
|
184
227
|
|
|
185
228
|
export const emailFrame: Detector = (doc): Finding[] =>
|
|
@@ -91,6 +91,14 @@ const otherWidth = (run: AlnumRun): string => (run.fullwidth ? run.written.norma
|
|
|
91
91
|
|
|
92
92
|
type Located = { readonly sentence: Sentence; readonly run: AlnumRun };
|
|
93
93
|
|
|
94
|
+
const HALFWIDTH_LOWER = /[a-z]/u;
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* 小文字を含む半角の英字の語(AI for Science の for と Science)。英語の語は、略語を全角で書く文書(AI)でも半角で書く。
|
|
98
|
+
* 文書の英字の書き方の票には入れるが、少ないほうとしては指摘しない。
|
|
99
|
+
*/
|
|
100
|
+
const isHalfwidthEnglishWord = (run: AlnumRun): boolean => run.kind === "letters" && !run.fullwidth && HALFWIDTH_LOWER.test(run.written);
|
|
101
|
+
|
|
94
102
|
const findingOf = ({ sentence, run }: Located, count: number, of: number, limit: number): Finding => ({
|
|
95
103
|
rule: "",
|
|
96
104
|
severity: "warning",
|
|
@@ -113,6 +121,6 @@ export const fullwidthAlnum: Detector = (doc, options): Finding[] => {
|
|
|
113
121
|
return KINDS.flatMap((kind) => {
|
|
114
122
|
const ofKind = located.filter(({ run }) => run.kind === kind);
|
|
115
123
|
const odd = minorityWithin(ofKind, ({ run }) => run.fullwidth, options.limit);
|
|
116
|
-
return odd.map((entry) => findingOf(entry, odd.length, ofKind.length, options.limit));
|
|
124
|
+
return odd.filter(({ run }) => !isHalfwidthEnglishWord(run)).map((entry) => findingOf(entry, odd.length, ofKind.length, options.limit));
|
|
117
125
|
});
|
|
118
126
|
};
|
|
@@ -6,16 +6,45 @@ const LIST_OR_TABLE_NEXT = /^[ \t]*\n\s*(?:[-*+]\s|\d{1,3}[.)]\s|\|)/u;
|
|
|
6
6
|
/** How far past the sentence to look for that line: blank lines between, never the rest of a long document. */
|
|
7
7
|
const LOOKAHEAD = 200;
|
|
8
8
|
|
|
9
|
+
/** 番号の印((1)、(2)、3.、第4、⑤)。数の点の後ろに数字が続けば小数(3.14)。 */
|
|
10
|
+
const ITEM_LABEL = String.raw`(?:[((][\d0-9一二三四五六七八九十]{1,3}[))]|[\d0-9]{1,3}[..))](?![\d0-9])|第[\d0-9一二三四五六七八九十]{1,3}|[①-⑳])`;
|
|
11
|
+
/** 箇条書きの印(- 、1. )。 */
|
|
12
|
+
const LIST_MARKER = String.raw`(?:[-*+]|\d{1,3}[.)])[ \t]+`;
|
|
13
|
+
const OPENS_WITH_LABEL = new RegExp(String.raw`^\s*${ITEM_LABEL}`, "u");
|
|
14
|
+
const SENTENCE_END = /[。..!?!?]\s*$/u;
|
|
15
|
+
/** 行の頭か、箇条書きの印の後ろ。 */
|
|
16
|
+
const LINE_START = new RegExp(String.raw`(?:^|\n)[ \t]*(?:${LIST_MARKER})?$`, "u");
|
|
17
|
+
const LINE_START_REACH = 8;
|
|
18
|
+
/**
|
|
19
|
+
* 行の終わりの後ろの、次の空でない行が本文であること。次の行も番号の印で始まれば、番号を振った項目の並び((1)申請書を出す、
|
|
20
|
+
* (2)…)で、見出しではない。
|
|
21
|
+
*/
|
|
22
|
+
const FOLLOWED_BY_BODY = new RegExp(String.raw`^[ \t]*\n\s*(?!(?:${LIST_MARKER})?${ITEM_LABEL})\S`, "u");
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* 行の頭の番号の印で始まり、句点で終わらず、一行を占め、後ろに本文が続く文(「(1)芸術の振興に係る規定に関する改正(第8条関係)」)。項目ごとの見出しで、
|
|
26
|
+
* 並べた見出しは同じ枠(…に係る規定に関する改正)を繰り返す。言い回しがどれもこの文にだけ出るなら見出しのひな形。
|
|
27
|
+
*/
|
|
28
|
+
export const itemTitles = (sentences: readonly Sentence[], source: string): Sentence[] =>
|
|
29
|
+
sentences.filter(
|
|
30
|
+
(sentence) =>
|
|
31
|
+
OPENS_WITH_LABEL.test(sentence.text) &&
|
|
32
|
+
!SENTENCE_END.test(sentence.text) &&
|
|
33
|
+
LINE_START.test(source.slice(Math.max(0, sentence.span.start - LINE_START_REACH), sentence.span.start)) &&
|
|
34
|
+
FOLLOWED_BY_BODY.test(source.slice(sentence.span.end, sentence.span.end + LOOKAHEAD)),
|
|
35
|
+
);
|
|
36
|
+
|
|
9
37
|
/**
|
|
10
38
|
* 見出しのある節の最初の文のうち、次の行から箇条書きか表が続くもの。参照用の文書は、節ごとに同じ書き出しで一覧を導く
|
|
11
39
|
* (「…によると以下の制約である。」の後ろに制約の箇条書き)。言い回しがどれもこの文にだけ出るなら節のひな形で、書き手が言い直したのではない。
|
|
12
40
|
* 後ろが本文の文なら、節ごとの口癖(正直なところ、…)と見分けられないので入れない。導入部(見出しの前)は節ではない。
|
|
13
41
|
*/
|
|
42
|
+
const leadsIntoList = (section: Pick<Section, "depth" | "sentences">, source: string): Sentence[] => {
|
|
43
|
+
const first = section.sentences[0];
|
|
44
|
+
if (section.depth === 0 || first === undefined) return [];
|
|
45
|
+
return LIST_OR_TABLE_NEXT.test(source.slice(first.span.end, first.span.end + LOOKAHEAD)) ? [first] : [];
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
/** 節のひな形の文: 一覧を導く節の最初の文と、項目ごとの見出し(itemTitles)。 */
|
|
14
49
|
export const sectionOpeners = (sections: readonly Pick<Section, "depth" | "sentences">[], source: string): Set<Sentence> =>
|
|
15
|
-
new Set(
|
|
16
|
-
sections.flatMap((section) => {
|
|
17
|
-
const first = section.sentences[0];
|
|
18
|
-
if (section.depth === 0 || first === undefined) return [];
|
|
19
|
-
return LIST_OR_TABLE_NEXT.test(source.slice(first.span.end, first.span.end + LOOKAHEAD)) ? [first] : [];
|
|
20
|
-
}),
|
|
21
|
-
);
|
|
50
|
+
new Set(sections.flatMap((section) => [...leadsIntoList(section, source), ...itemTitles(section.sentences, source)]));
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/** 見出しの言葉から括弧の添え書き(「事例(約款)」の「(約款)」)を除いた、記録の型としての言葉。 */
|
|
2
|
+
export const recordLabelOf = (key: string): string => {
|
|
3
|
+
const stem = key.replace(/[((][^()()]*[))]/gu, "").trim();
|
|
4
|
+
return stem === "" ? key : stem;
|
|
5
|
+
};
|
|
6
|
+
|
|
7
|
+
/** start から period 個の型がみな違うか。窓をずらしながら数えるので、一つの period で列全体を一度なめるだけ。 */
|
|
8
|
+
const distinctWindows = (labels: readonly string[], period: number): boolean[] => {
|
|
9
|
+
const counts = new Map<string, number>();
|
|
10
|
+
const add = (label: string | undefined, by: number): void => {
|
|
11
|
+
if (label !== undefined) counts.set(label, (counts.get(label) ?? 0) + by);
|
|
12
|
+
if (label !== undefined && counts.get(label) === 0) counts.delete(label);
|
|
13
|
+
};
|
|
14
|
+
return labels.map((label, start) => {
|
|
15
|
+
if (start === 0) labels.slice(0, period).forEach((first) => add(first, 1));
|
|
16
|
+
else {
|
|
17
|
+
add(labels[start - 1], -1);
|
|
18
|
+
add(labels[start + period - 1], 1);
|
|
19
|
+
}
|
|
20
|
+
return start + period <= labels.length && counts.size === period;
|
|
21
|
+
});
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
/** 各位置までに、period 前と同じ型だった位置の数(累積)。 */
|
|
25
|
+
const repeatCounts = (labels: readonly string[], period: number): number[] =>
|
|
26
|
+
labels.reduce<number[]>((sums, label, at) => {
|
|
27
|
+
sums.push((sums.at(-1) ?? 0) + (at >= period && labels[at - period] === label ? 1 : 0));
|
|
28
|
+
return sums;
|
|
29
|
+
}, []);
|
|
30
|
+
|
|
31
|
+
/** period の組が二度続く窓 [start, start + 2 * period) を、覆う範囲の印として足す。 */
|
|
32
|
+
const markRepeats = (labels: readonly string[], period: number, cover: number[]): void => {
|
|
33
|
+
const distinct = distinctWindows(labels, period);
|
|
34
|
+
const repeats = repeatCounts(labels, period);
|
|
35
|
+
const repeatsIn = (from: number, to: number): number => (repeats[to - 1] ?? 0) - (repeats[from - 1] ?? 0);
|
|
36
|
+
distinct.forEach((isDistinct, start) => {
|
|
37
|
+
const end = start + 2 * period;
|
|
38
|
+
if (!isDistinct || end > labels.length || repeatsIn(start + period, end) !== period) return;
|
|
39
|
+
cover[start] = (cover[start] ?? 0) + 1;
|
|
40
|
+
cover[end] = (cover[end] ?? 0) - 1;
|
|
41
|
+
});
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* 兄弟の見出しの型(recordLabelOf)の並びで、「二つ以上の違う型の組が、同じ順で二度続く」所にある見出しの印。
|
|
46
|
+
* 事例ごとに【相談事例】【アドバイス】、Question / Answer をくり返す FAQ の決まった見出しで、写し忘れの重複ではない。
|
|
47
|
+
* 組の長さごとに列を一度なめるので、見出しの数の二乗で済む。
|
|
48
|
+
*/
|
|
49
|
+
export const recordRhythmMask = (labels: readonly string[]): boolean[] => {
|
|
50
|
+
const cover = Array.from({ length: labels.length + 1 }, () => 0);
|
|
51
|
+
Array.from({ length: Math.max(0, Math.floor(labels.length / 2) - 1) }, (_, offset) => offset + 2).forEach((period) => markRepeats(labels, period, cover));
|
|
52
|
+
const depths = cover.reduce<number[]>((sums, change) => {
|
|
53
|
+
sums.push((sums.at(-1) ?? 0) + change);
|
|
54
|
+
return sums;
|
|
55
|
+
}, []);
|
|
56
|
+
return labels.map((_, at) => (depths[at] ?? 0) > 0);
|
|
57
|
+
};
|
|
@@ -76,6 +76,14 @@ const isNextInRun = ({ source, offset }: Place): boolean => {
|
|
|
76
76
|
return previous > 0 && countsUpTo(earlier, previous);
|
|
77
77
|
};
|
|
78
78
|
|
|
79
|
+
/**
|
|
80
|
+
* 番号を指す点(「3.から5.までの規定」「(1)4.の改正」)。数の点の直後に、番号を受ける助詞が続く。文の終わりの点の後ろは
|
|
81
|
+
* 次の文の頭。一字の助詞は後ろが平仮名でないときだけ(「のちほど」は文の頭)。文の頭になる語の字(で、も、は、と)は入れない。
|
|
82
|
+
*/
|
|
83
|
+
const ITEM_REFERENCE_AFTER = /^(?:から|まで|及び|又は|並びに|[のをにへ](?![ぁ-ゖ]))/u;
|
|
84
|
+
|
|
85
|
+
const isItemReference = ({ text, at }: Place): boolean => DIGIT.test(text.charAt(at - 1)) && ITEM_REFERENCE_AFTER.test(text.slice(at + 1));
|
|
86
|
+
|
|
79
87
|
const isInsideNumber = (text: string, at: number): boolean => DIGIT.test(text.charAt(at - 1)) && DIGIT.test(text.charAt(at + 1));
|
|
80
88
|
|
|
81
89
|
const isLabelPeriod = (place: Place, kana: ReadonlySet<string>): boolean => {
|
|
@@ -86,6 +94,7 @@ const isLabelPeriod = (place: Place, kana: ReadonlySet<string>): boolean => {
|
|
|
86
94
|
((NOTE_NUMBER_LABEL.test(text.slice(Math.max(0, at - NOTE_REACH), at)) || CLOSED_FIRST_NUMBER.test(beforeNumber)) &&
|
|
87
95
|
isFollowed(place.source, place.offset)) ||
|
|
88
96
|
isNextInRun(place) ||
|
|
97
|
+
isItemReference(place) ||
|
|
89
98
|
LIST_NUMBER.test(beforeNumber) ||
|
|
90
99
|
LATIN_LABEL.test(text.slice(Math.max(0, at - LATIN_LABEL_REACH), at))
|
|
91
100
|
);
|
package/src/detectors/lexicon.ts
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import { joinWords } from "./word-list.ts";
|
|
2
2
|
import { pairMinority } from "./contraction-minority.ts";
|
|
3
|
+
import { withoutMentions } from "./contraction-mention.ts";
|
|
3
4
|
import { straightApostrophes } from "../orthography.ts";
|
|
4
5
|
import { wordsOf } from "./structure.ts";
|
|
5
6
|
import type { Detector, Finding, Lexicon, LexiconEntry, ProseDocument, Sentence, Span, Token } from "../plugin.ts";
|
|
6
7
|
import { entryIn, entryOpens, entryRanges, type TokenRange } from "./lexicon-match.ts";
|
|
7
8
|
import { scoped, scopeMarkersOf, type ScopeMarkers } from "./superlative-scope.ts";
|
|
8
|
-
import { namesQuantity } from "./superlative-name.ts";
|
|
9
|
+
import { makesWord, namesQuantity } from "./superlative-name.ts";
|
|
9
10
|
import { namesAmount } from "./superlative-amount.ts";
|
|
10
11
|
import { restricted, type Restrictors } from "./superlative-clause.ts";
|
|
11
12
|
import { stackedHedges, type StackedHedge } from "./stacked-hedge.ts";
|
|
@@ -121,6 +122,7 @@ type Qualifiers = {
|
|
|
121
122
|
readonly comparison: ComparisonMarkers;
|
|
122
123
|
readonly scope: ScopeMarkers;
|
|
123
124
|
readonly quantityNouns: Lexicon;
|
|
125
|
+
readonly suffixes: Lexicon;
|
|
124
126
|
readonly amounts: Lexicon;
|
|
125
127
|
readonly restrictors: Restrictors;
|
|
126
128
|
};
|
|
@@ -129,6 +131,7 @@ const qualifiersOf = (doc: ProseDocument): Qualifiers => ({
|
|
|
129
131
|
comparison: comparisonMarkersOf(doc.lexicons["comparison-marker"] ?? []),
|
|
130
132
|
scope: scopeMarkersOf(doc.lexicons["superlative-scope"] ?? []),
|
|
131
133
|
quantityNouns: doc.lexicons["quantity-noun"] ?? [],
|
|
134
|
+
suffixes: doc.lexicons["superlative-suffix"] ?? [],
|
|
132
135
|
amounts: doc.lexicons["superlative-amount"] ?? [],
|
|
133
136
|
restrictors: {
|
|
134
137
|
relatives: doc.lexicons["relative-word"] ?? [],
|
|
@@ -143,6 +146,7 @@ const qualifiedAt = (tokens: readonly Token[], range: TokenRange, qualifiers: Qu
|
|
|
143
146
|
scoped(tokens, range, qualifiers.scope) ||
|
|
144
147
|
restricted(tokens, range, qualifiers.restrictors) ||
|
|
145
148
|
namesQuantity(tokens, range, qualifiers.quantityNouns) ||
|
|
149
|
+
makesWord(tokens, range, qualifiers.suffixes) ||
|
|
146
150
|
namesAmount(tokens, range, qualifiers.amounts);
|
|
147
151
|
|
|
148
152
|
/** どの出現も引いた言葉か、限られているか、量を言うときだけ。1 つでもそうでない出現があれば、その文には限定の無い最上級がある。 */
|
|
@@ -283,7 +287,7 @@ const containsWord = (text: string, word: string): boolean => wordUses(text, wor
|
|
|
283
287
|
* アポストロフィの形(don’t と don't)は短縮形かどうかを変えない。違うのは印刷の形で、文体ではない。
|
|
284
288
|
* 折り返した行(「do」で改行して「not」)も、空白 1 つと同じに読む。
|
|
285
289
|
*/
|
|
286
|
-
const comparableText = (sentence: Sentence): string => straightApostrophes(sentence.text).replace(/\s+/gu, " ").toLowerCase();
|
|
290
|
+
const comparableText = (sentence: Sentence): string => withoutMentions(straightApostrophes(sentence.text)).replace(/\s+/gu, " ").toLowerCase();
|
|
287
291
|
|
|
288
292
|
type MixedPair = { readonly minority: string; readonly majority: string; readonly count: number };
|
|
289
293
|
|
|
@@ -5,7 +5,7 @@ import { minorityReport } from "../spacing-minority.ts";
|
|
|
5
5
|
import { externalLinkTitleSpans } from "../link-title.ts";
|
|
6
6
|
import { formulaBracketSpans } from "../formula-brackets.ts";
|
|
7
7
|
import { isWithinAny, quotedSpans } from "../quoted-span.ts";
|
|
8
|
-
import { captionsOf, isCaptionAt,
|
|
8
|
+
import { capitalLineSpans, captionsOf, isCaptionAt, isWithinSorted } from "./caps-caption.ts";
|
|
9
9
|
import { lineStarts } from "../position.ts";
|
|
10
10
|
|
|
11
11
|
/** The letters a mark must touch to count as part of the document's text, by language. */
|
|
@@ -125,6 +125,28 @@ const ENUMERATOR_KIND = "\u0000enumerator";
|
|
|
125
125
|
const withEnumeratorKind = (mark: Mark, enumerators: ReadonlySet<number>): Mark =>
|
|
126
126
|
enumerators.has(mark.offset) ? { ...mark, form: { ...mark.form, kind: `${mark.form.kind}${ENUMERATOR_KIND}` } } : mark;
|
|
127
127
|
|
|
128
|
+
/**
|
|
129
|
+
* A doubled parenthesis around text with no parenthesis inside, such as ((居住の用に供したとき等)) or ((注)): one mark written
|
|
130
|
+
* with two characters, as a directive sets an article's caption. Its width is that notation's, not the prose's.
|
|
131
|
+
*/
|
|
132
|
+
const DOUBLED_PAREN = /\(\([^()()\n]*\)\)|(([^()()\n]*))/gu;
|
|
133
|
+
const PAIR_LENGTH = 2;
|
|
134
|
+
|
|
135
|
+
/** The offsets in `source` of every character of a doubled parenthesis. */
|
|
136
|
+
export const doubledParenOffsets = (source: string): ReadonlySet<number> =>
|
|
137
|
+
new Set(
|
|
138
|
+
[...source.matchAll(DOUBLED_PAREN)].flatMap((match) => {
|
|
139
|
+
const end = match.index + match[0].length;
|
|
140
|
+
return [match.index, match.index + 1, end - PAIR_LENGTH, end - 1];
|
|
141
|
+
}),
|
|
142
|
+
);
|
|
143
|
+
|
|
144
|
+
const DOUBLED_KIND = "\u0000doubled";
|
|
145
|
+
|
|
146
|
+
/** A doubled parenthesis is a kind of its own, compared only with the other doubled parentheses. */
|
|
147
|
+
const withDoubledKind = (mark: Mark, doubled: ReadonlySet<number>): Mark =>
|
|
148
|
+
doubled.has(mark.offset) ? { ...mark, form: { ...mark.form, kind: `${mark.form.kind}${DOUBLED_KIND}` } } : mark;
|
|
149
|
+
|
|
128
150
|
/**
|
|
129
151
|
* A mark written two ways in one document (( and (, “ and "), the less common way reported. The two ways come from the
|
|
130
152
|
* lexicon; neither is called right. Only marks next to a letter of the document's script count, so a bracket in code-like
|
|
@@ -137,7 +159,10 @@ export const markConsistency: Detector = (doc, options): Finding[] => {
|
|
|
137
159
|
const titles = externalLinkTitleSpans(doc.source, doc.links);
|
|
138
160
|
const forms = formsOf(options.lexicon ?? []);
|
|
139
161
|
const enumerators = enumeratorBracketOffsets(doc.source);
|
|
140
|
-
const
|
|
162
|
+
const doubled = doubledParenOffsets(doc.source);
|
|
163
|
+
const marks = ownSentences(doc).flatMap((sentence) =>
|
|
164
|
+
marksIn(doc.source, sentence, forms, script).map((mark) => withDoubledKind(withEnumeratorKind(mark, enumerators), doubled)),
|
|
165
|
+
);
|
|
141
166
|
const kinds = [...new Set(marks.map((mark) => mark.form.kind))];
|
|
142
167
|
return kinds.flatMap((kind) =>
|
|
143
168
|
minorityFindings(
|
|
@@ -263,17 +288,17 @@ export const isShouting = (run: string, known: ReadonlySet<string>, conventional
|
|
|
263
288
|
|
|
264
289
|
/**
|
|
265
290
|
* Runs of three or more all-capital words that are ordinary words, not acronyms ("DO NOT DELETE THIS"), and not a
|
|
266
|
-
* caption, a mention of one, or a
|
|
291
|
+
* caption, a mention of one, a paragraph of capital lines that ends no sentence (a letterhead, a heading), or a roster name.
|
|
267
292
|
*/
|
|
268
293
|
export const capsShouting: Detector = (doc, options): Finding[] => {
|
|
269
294
|
const known = lowerCaseWords(doc.prose ?? doc.source);
|
|
270
295
|
const conventional = new Set((options.lexicon ?? []).map((entry) => entry.pattern));
|
|
271
296
|
const captions = captionsOf(doc.source);
|
|
272
|
-
const
|
|
297
|
+
const capitalLines = capitalLineSpans(doc.source);
|
|
273
298
|
const isCaptionOrLetterhead = (hit: Hit): boolean =>
|
|
274
299
|
captions.has(hit.matched) ||
|
|
275
300
|
isCaptionAt(doc.source, hit.offset, hit.offset + hit.matched.length) ||
|
|
276
|
-
|
|
301
|
+
isWithinSorted(capitalLines, { start: hit.offset, end: hit.offset + hit.matched.length });
|
|
277
302
|
const hits = hitsOf(ownSentences(doc), CAPS_RUN).filter((hit) => isShouting(hit.matched, known, conventional) && !isCaptionOrLetterhead(hit));
|
|
278
303
|
return reportFrom(hits, options.limit);
|
|
279
304
|
};
|