chaffjs 0.11.0 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. package/dist/date-stamp.d.ts +14 -0
  2. package/dist/date-stamp.d.ts.map +1 -0
  3. package/dist/date-stamp.js +40 -0
  4. package/dist/date-stamp.js.map +1 -0
  5. package/dist/declared-lists.d.ts +4 -0
  6. package/dist/declared-lists.d.ts.map +1 -0
  7. package/dist/declared-lists.js +3 -0
  8. package/dist/declared-lists.js.map +1 -0
  9. package/dist/detectors/acronym-context.d.ts +25 -0
  10. package/dist/detectors/acronym-context.d.ts.map +1 -0
  11. package/dist/detectors/acronym-context.js +28 -0
  12. package/dist/detectors/acronym-context.js.map +1 -0
  13. package/dist/detectors/acronym-expansion.d.ts +9 -0
  14. package/dist/detectors/acronym-expansion.d.ts.map +1 -0
  15. package/dist/detectors/acronym-expansion.js +94 -0
  16. package/dist/detectors/acronym-expansion.js.map +1 -0
  17. package/dist/detectors/char-shape.d.ts.map +1 -1
  18. package/dist/detectors/char-shape.js +20 -34
  19. package/dist/detectors/char-shape.js.map +1 -1
  20. package/dist/detectors/doubled-word.d.ts +27 -0
  21. package/dist/detectors/doubled-word.d.ts.map +1 -0
  22. package/dist/detectors/doubled-word.js +94 -0
  23. package/dist/detectors/doubled-word.js.map +1 -0
  24. package/dist/detectors/heading-echo.d.ts.map +1 -1
  25. package/dist/detectors/heading-echo.js +22 -12
  26. package/dist/detectors/heading-echo.js.map +1 -1
  27. package/dist/detectors/index.d.ts.map +1 -1
  28. package/dist/detectors/index.js +2 -0
  29. package/dist/detectors/index.js.map +1 -1
  30. package/dist/detectors/lead-in.d.ts +8 -0
  31. package/dist/detectors/lead-in.d.ts.map +1 -0
  32. package/dist/detectors/lead-in.js +13 -0
  33. package/dist/detectors/lead-in.js.map +1 -0
  34. package/dist/detectors/lexicon-match.d.ts +6 -0
  35. package/dist/detectors/lexicon-match.d.ts.map +1 -1
  36. package/dist/detectors/lexicon-match.js +8 -0
  37. package/dist/detectors/lexicon-match.js.map +1 -1
  38. package/dist/detectors/lexicon.d.ts +5 -0
  39. package/dist/detectors/lexicon.d.ts.map +1 -1
  40. package/dist/detectors/lexicon.js +21 -4
  41. package/dist/detectors/lexicon.js.map +1 -1
  42. package/dist/detectors/orthography.d.ts.map +1 -1
  43. package/dist/detectors/orthography.js +17 -1
  44. package/dist/detectors/orthography.js.map +1 -1
  45. package/dist/detectors/place-run.d.ts +8 -0
  46. package/dist/detectors/place-run.d.ts.map +1 -0
  47. package/dist/detectors/place-run.js +54 -0
  48. package/dist/detectors/place-run.js.map +1 -0
  49. package/dist/detectors/register.d.ts +32 -0
  50. package/dist/detectors/register.d.ts.map +1 -0
  51. package/dist/detectors/register.js +62 -0
  52. package/dist/detectors/register.js.map +1 -0
  53. package/dist/detectors/sentence-ending.d.ts.map +1 -1
  54. package/dist/detectors/sentence-ending.js +17 -28
  55. package/dist/detectors/sentence-ending.js.map +1 -1
  56. package/dist/detectors/signals.d.ts.map +1 -1
  57. package/dist/detectors/signals.js +20 -115
  58. package/dist/detectors/signals.js.map +1 -1
  59. package/dist/detectors/structure.d.ts.map +1 -1
  60. package/dist/detectors/structure.js +14 -1
  61. package/dist/detectors/structure.js.map +1 -1
  62. package/dist/detectors/superlative-scope.d.ts +16 -0
  63. package/dist/detectors/superlative-scope.d.ts.map +1 -0
  64. package/dist/detectors/superlative-scope.js +40 -0
  65. package/dist/detectors/superlative-scope.js.map +1 -0
  66. package/dist/document.d.ts.map +1 -1
  67. package/dist/document.js +17 -11
  68. package/dist/document.js.map +1 -1
  69. package/dist/eval.d.ts.map +1 -1
  70. package/dist/eval.js +3 -2
  71. package/dist/eval.js.map +1 -1
  72. package/dist/heading-title.d.ts +6 -0
  73. package/dist/heading-title.d.ts.map +1 -0
  74. package/dist/heading-title.js +8 -0
  75. package/dist/heading-title.js.map +1 -0
  76. package/dist/in-page-nav.d.ts +19 -0
  77. package/dist/in-page-nav.d.ts.map +1 -0
  78. package/dist/in-page-nav.js +44 -0
  79. package/dist/in-page-nav.js.map +1 -0
  80. package/dist/number-name.d.ts +9 -0
  81. package/dist/number-name.d.ts.map +1 -0
  82. package/dist/number-name.js +57 -0
  83. package/dist/number-name.js.map +1 -0
  84. package/dist/orthography.d.ts +7 -2
  85. package/dist/orthography.d.ts.map +1 -1
  86. package/dist/orthography.js +79 -17
  87. package/dist/orthography.js.map +1 -1
  88. package/dist/plugin.d.ts +4 -0
  89. package/dist/plugin.d.ts.map +1 -1
  90. package/dist/rule-load.d.ts.map +1 -1
  91. package/dist/rule-load.js +1 -0
  92. package/dist/rule-load.js.map +1 -1
  93. package/dist/run.d.ts.map +1 -1
  94. package/dist/run.js +5 -4
  95. package/dist/run.js.map +1 -1
  96. package/package.json +3 -3
  97. package/rules/doubled-word.yaml +27 -0
  98. package/rules/latin-spacing.yaml +4 -2
  99. package/rules/max-kanji-continuous.yaml +2 -0
  100. package/rules/undefined-acronym.yaml +2 -0
  101. package/rules/unqualified-superlative.yaml +1 -0
  102. package/src/date-stamp.ts +65 -0
  103. package/src/declared-lists.ts +5 -0
  104. package/src/detectors/acronym-context.ts +50 -0
  105. package/src/detectors/acronym-expansion.ts +109 -0
  106. package/src/detectors/char-shape.ts +12 -30
  107. package/src/detectors/doubled-word.ts +128 -0
  108. package/src/detectors/heading-echo.ts +13 -3
  109. package/src/detectors/index.ts +2 -0
  110. package/src/detectors/lead-in.ts +14 -0
  111. package/src/detectors/lexicon-match.ts +10 -0
  112. package/src/detectors/lexicon.ts +32 -6
  113. package/src/detectors/orthography.ts +20 -2
  114. package/src/detectors/place-run.ts +63 -0
  115. package/src/detectors/register.ts +83 -0
  116. package/src/detectors/sentence-ending.ts +24 -28
  117. package/src/detectors/signals.ts +24 -120
  118. package/src/detectors/structure.ts +21 -2
  119. package/src/detectors/superlative-scope.ts +66 -0
  120. package/src/document.ts +19 -11
  121. package/src/eval.ts +2 -1
  122. package/src/heading-title.ts +8 -0
  123. package/src/in-page-nav.ts +67 -0
  124. package/src/number-name.ts +68 -0
  125. package/src/orthography.ts +77 -15
  126. package/src/plugin.ts +4 -0
  127. package/src/rule-load.ts +1 -0
  128. package/src/run.ts +5 -4
@@ -1,7 +1,8 @@
1
1
  import { joinWords } from "./word-list.ts";
2
2
  import { wordsOf } from "./structure.ts";
3
- import type { Detector, Finding, Lexicon, ProseDocument, Sentence } from "../plugin.ts";
4
- import { entryIn, entryOpens } from "./lexicon-match.ts";
3
+ import type { Detector, Finding, Lexicon, LexiconEntry, ProseDocument, Sentence } from "../plugin.ts";
4
+ import { entryIn, entryOpens, entryRanges } from "./lexicon-match.ts";
5
+ import { scoped, scopeMarkersOf, type ScopeMarkers } from "./superlative-scope.ts";
5
6
 
6
7
  const PER = 1000;
7
8
 
@@ -38,14 +39,39 @@ const densityRule =
38
39
  export const hedgingDensity = densityRule("excessive-hedging");
39
40
  export const cushionDensity = densityRule("cushion-phrase-density");
40
41
 
42
+ /** 数字は語ではないので言語を問わない。数があれば測った結果を言っている。品詞の数(NUM)は "the best one" の one まで含むので使わない。 */
43
+ const DIGIT = /\d/u;
44
+
45
+ type Qualifiers = { readonly comparison: Lexicon; readonly scope: ScopeMarkers };
46
+
47
+ const qualifiersOf = (doc: ProseDocument): Qualifiers => ({
48
+ comparison: doc.lexicons["comparison-marker"] ?? [],
49
+ scope: scopeMarkersOf(doc.lexicons["superlative-scope"] ?? []),
50
+ });
51
+
52
+ /** どの出現も範囲を持つときだけ。1 つでも範囲の無い出現があれば、その文には限定の無い最上級がある。 */
53
+ const everyScoped = (sentence: Sentence, entry: LexiconEntry, scope: ScopeMarkers): boolean => {
54
+ const ranges = entryRanges(sentence, entry);
55
+ return ranges.length > 0 && ranges.every((range) => scoped(sentence.tokens ?? [], range, scope));
56
+ };
57
+
58
+ const qualified = (sentence: Sentence, entry: LexiconEntry, qualifiers: Qualifiers): boolean =>
59
+ DIGIT.test(sentence.text) || qualifiers.comparison.some((marker) => entryIn(sentence, marker)) || everyScoped(sentence, entry, qualifiers.scope);
60
+
61
+ const bareHits = (doc: ProseDocument, lexicon: Lexicon): Hit[] => {
62
+ const qualifiers = qualifiersOf(doc);
63
+ return doc.sentences.flatMap((sentence) =>
64
+ lexicon.filter((entry) => entryIn(sentence, entry) && !qualified(sentence, entry, qualifiers)).map((entry) => ({ sentence, matched: entry.pattern })),
65
+ );
66
+ };
67
+
41
68
  /**
42
69
  * 限定のない最上級。「最も速い」だけでは、何と比べて最もなのかが無い。
43
- * 同じ文に比較対象や条件があれば、それは主張であって誇張ではない。
70
+ * 同じ文に比較対象・条件・範囲(日本で最も、the best in the world)があれば、それは主張であって誇張ではない。
71
+ * 比較と範囲を言う語は言語パッケージの語彙表が持つ。
44
72
  */
45
- const QUALIFIER = /\d|より|に比べ|のうち|among|than|compared|based on|according/iu;
46
-
47
73
  export const unqualifiedSuperlative: Detector = (doc, options): Finding[] => {
48
- const bare = hitsFor(doc, options.lexicon ?? []).filter((hit) => !QUALIFIER.test(hit.sentence.text));
74
+ const bare = bareHits(doc, options.lexicon ?? []);
49
75
  if (bare.length < options.limit) return [];
50
76
  return bare.map((hit) => ({
51
77
  rule: "unqualified-superlative",
@@ -1,5 +1,6 @@
1
1
  import type { Detector, Finding, Sentence } from "../plugin.ts";
2
- import { latinBoundaries, minorityStyle, occurrencesOutside, type SpacingKind } from "../orthography.ts";
2
+ import { latinBoundaries, minorityStyle, occurrencesOutside, type Boundary, type SpacingKind } from "../orthography.ts";
3
+ import { digitRunAround, isNumberName } from "../number-name.ts";
3
4
 
4
5
  /** チームが chaff.yaml の prefer に書いた「使わない書き方」。書いていなければ何も言わない。 */
5
6
  export const preferredTerm: Detector = (doc, options): Finding[] => {
@@ -23,12 +24,29 @@ const KINDS: readonly SpacingKind[] = ["letter", "before-digit", "after-digit"];
23
24
 
24
25
  type Located = { readonly sentence: Sentence; readonly offset: number; readonly kind: SpacingKind; readonly spaced: boolean };
25
26
 
27
+ /** 境目の隣の数字の、並びの中の 1 字の位置。数字の後ろなら左の字、数字の前なら(空白を越えた)右の字。 */
28
+ const digitBeside = (boundary: Boundary): number => {
29
+ if (boundary.kind === "after-digit") return boundary.offset - 1;
30
+ return boundary.spaced ? boundary.offset + 1 : boundary.offset;
31
+ };
32
+
33
+ /** 番号・識別子として書かれた数の境目は、空け方の好みではないので数えない(number-name.ts)。 */
34
+ const isCounted = (sentence: Sentence, boundary: Boundary): boolean => {
35
+ if (boundary.kind === "letter") return true;
36
+ const run = digitRunAround(sentence.text, digitBeside(boundary));
37
+ return run === undefined || !isNumberName(sentence.text, run, sentence.tokens, sentence.span.start);
38
+ };
39
+
26
40
  /**
27
41
  * 日本語と英字・数字のあいだを、空けるか詰めるか。文書の中で混ざっていたら、少ないほうを指摘する。
28
42
  * どちらが正しいかは決めない。決めるのはチームで、chaff はそろっているかだけを見る。
29
43
  */
30
44
  export const latinSpacing: Detector = (doc, options): Finding[] => {
31
- const located: Located[] = doc.sentences.flatMap((sentence) => latinBoundaries(sentence.text).map((boundary) => ({ sentence, ...boundary })));
45
+ const located: Located[] = doc.sentences.flatMap((sentence) =>
46
+ latinBoundaries(sentence.text, doc.source.slice(sentence.span.start, sentence.span.end))
47
+ .filter((boundary) => isCounted(sentence, boundary))
48
+ .map((boundary) => ({ sentence, ...boundary })),
49
+ );
32
50
  return KINDS.flatMap((kind) => {
33
51
  const ofKind = located.filter((entry) => entry.kind === kind);
34
52
  const minority = minorityStyle(ofKind);
@@ -0,0 +1,63 @@
1
+ import type { Token } from "../plugin.ts";
2
+
3
+ /** 地名(Geo)と、地名に付く単位(GeoUnit: 都・県・郡・市・区・町・村)。 */
4
+ const isGeoName = (token: Token | undefined): boolean => token?.features?.["NameType"] === "Geo";
5
+ const isGeoUnit = (token: Token | undefined): boolean => token?.features?.["NameType"] === "GeoUnit";
6
+ const isGeo = (token: Token | undefined): boolean => isGeoName(token) || isGeoUnit(token);
7
+ const isNumber = (token: Token | undefined): boolean => token?.features?.["NumType"] === "Card";
8
+
9
+ /**
10
+ * 辞書に無い町名は、形態素解析が固有名詞か 1 字ずつに割る(紀美野町は 紀+美野、南伊勢町は 南+伊勢、久米島町は固有名詞,一般)。
11
+ * 2 字以上の普通の語(政策・総務)は地名のかけらにしない。
12
+ */
13
+ const isNamePiece = (token: Token | undefined): boolean => token !== undefined && (token.pos === "PROPN" || [...token.surface].length === 1);
14
+
15
+ /** 連なりの語と、閉じても 1 つの地名にならない単位(都道府県。住所の先頭にしか来ない。語彙表 prefecture-unit)。 */
16
+ type Run = { readonly tokens: readonly Token[]; readonly last: number; readonly topUnits: ReadonlySet<string> };
17
+
18
+ /**
19
+ * index から地名のかけらだけを読み進めて、連なりの終わりまでに都道府県より下の単位(郡・市・町)で閉じるか。
20
+ * 閉じれば、そこまでが 1 つの地名。連なりの外(「に」「の」も 1 字)までは読まない。
21
+ */
22
+ const closedByUnit = (run: Run, index: number): boolean => {
23
+ const token = run.tokens[index];
24
+ if (index > run.last || token === undefined) return false;
25
+ if (isGeoUnit(token)) return !run.topUnits.has(token.surface);
26
+ return isNamePiece(token) && closedByUnit(run, index + 1);
27
+ };
28
+
29
+ /**
30
+ * 住所の語: 地名と地名の単位(NameType=Geo / GeoUnit)、数(NumType=Card)、数のすぐ後ろの助数詞(丁目、NounType=Class)。
31
+ * 辞書に無い町名(新橋)は、地名の後ろで数の前に来るときだけ住所の一部とする。
32
+ * 割られた町名(紀美野町)は、単位の後ろで、地名のかけらだけを挟んで次の単位で閉じるときだけ住所の一部とする。
33
+ */
34
+ const isPlacePart = (run: Run, index: number): boolean => {
35
+ const tokens = run.tokens;
36
+ return (
37
+ isGeo(tokens[index]) ||
38
+ isNumber(tokens[index]) ||
39
+ (isNumber(tokens[index - 1]) && tokens[index]?.features?.["NounType"] === "Class") ||
40
+ (isGeo(tokens[index - 1]) && isNumber(tokens[index + 1])) ||
41
+ (isGeo(tokens[index - 1]) && isNamePiece(tokens[index]) && closedByUnit(run, index + 1))
42
+ );
43
+ };
44
+
45
+ /**
46
+ * 地名が単位を挟まずに続けば地名の並び(東京大阪名古屋福岡)。ただし 2 つだけで郡・市・町が閉じれば、割られた 1 つの地名
47
+ * (北海道虻田郡、富士河口湖町)。3 つ続けば、単位が付いても並び(京都奈良大阪神戸市)。
48
+ * 名古屋大阪市は北海道札幌市と品詞が同じ形なので、ここでは分けられない。
49
+ */
50
+ const isListed = (run: Run, index: number): boolean =>
51
+ isGeoName(run.tokens[index]) && isGeoName(run.tokens[index + 1]) && (isGeoName(run.tokens[index + 2]) || !closedByUnit(run, index + 1));
52
+
53
+ /**
54
+ * 漢字の連なりの語(covering: tokens の添字)が住所か(東京都港区新橋二丁目)。地名で始まり、住所の語だけでできている。
55
+ * 住所は決まった形で、ひらがなを挟んで書けない。組織名(日本経済団体連合会)は地名の後ろに普通の語が続くので住所ではない。
56
+ * topUnits は、住所の先頭にしか来ない単位(都道府県)。言語の語彙表から渡す。
57
+ */
58
+ export const isAddressRun = (tokens: readonly Token[], covering: readonly number[], topUnits: ReadonlySet<string>): boolean => {
59
+ const [first, last] = [covering[0], covering.at(-1)];
60
+ if (first === undefined || last === undefined) return false;
61
+ const run: Run = { tokens, last, topUnits };
62
+ return isGeo(tokens[first]) && !covering.some((index) => isListed(run, index)) && covering.every((index) => isPlacePart(run, index));
63
+ };
@@ -0,0 +1,83 @@
1
+ import type { Sentence, Span, Token } from "../plugin.ts";
2
+
3
+ export type Register = "polite" | "plain";
4
+
5
+ const PREDICATE = new Set(["VERB", "ADJ", "AUX"]);
6
+
7
+ const isPredicate = (token: Token): boolean => PREDICATE.has(token.pos);
8
+
9
+ const isDependent = (token: Token | undefined): boolean => token?.features?.["NounType"] === "Dependent";
10
+
11
+ /** preceding の後ろの述語の連なり: 助動詞の連なりと、その前の語一つ。「おかけしましたこと」なら「し・まし・た」。 */
12
+ const chainAtEnd = (preceding: readonly Token[]): readonly Token[] => {
13
+ const head = preceding.findLastIndex((token) => token.pos !== "AUX");
14
+ return preceding.slice(Math.max(head, 0));
15
+ };
16
+
17
+ /**
18
+ * 非自立名詞で終わる文末は、手前の述語の連なりまで見る。「予約できること。」「おかけしましたこと。」は手前の述語が調子を持つ。
19
+ * 要件や規程のである調は「〜こと。」で書く。「以下のとおり。」は手前も述語でないので、調子を持たないまま。
20
+ */
21
+ const withPreceding = (ending: readonly Token[], preceding: readonly Token[]): readonly Token[] =>
22
+ isDependent(ending[0]) ? [...chainAtEnd(preceding), ...ending] : ending;
23
+
24
+ /**
25
+ * 文末の語の調子。preceding は文末の語より前の語。
26
+ * 文末に述語(動詞・形容詞・助動詞)が無ければ、ですます調でもである調でもないので undefined。
27
+ * 「以下の通り。」「円錐形の麦わら帽子。」のような名詞で終わる文を数えると、そのままである調の少数派になっていた。
28
+ * 書いた形でも原形でも当てる。「ください」の原形は「くださる」で、原形だけを見ると丁寧な文末を見落とす。
29
+ */
30
+ export const registerOf = (ending: readonly Token[], preceding: readonly Token[], polite: readonly string[]): Register | undefined => {
31
+ const judged = withPreceding(ending, preceding);
32
+ if (!judged.some(isPredicate)) return undefined;
33
+ const isPolite = judged.some((token) => polite.includes(token.surface) || (token.lemma !== undefined && polite.includes(token.lemma)));
34
+ return isPolite ? "polite" : "plain";
35
+ };
36
+
37
+ const EXCLAIMED = /[!?!?]$/u;
38
+
39
+ /**
40
+ * 「!」「?」で切れた文が、間を置かずに助詞で始まる次の文へ続くか。「作っていた!!という人」「導入しませんか?が断られた」は、
41
+ * 分割器が切っても一つの文で、切れ目の手前は書き手の文末ではない。「。」で切れた文は続きとは見ない。
42
+ */
43
+ export const continuesInto = (sentence: Sentence, next: Sentence | undefined): boolean =>
44
+ EXCLAIMED.test(sentence.text) && next !== undefined && next.span.start === sentence.span.end && next.tokens?.[0]?.pos === "ADP";
45
+
46
+ /** 位置 offset を含む一番外側の箇条書きの始まり。本文なら undefined。入れ子の項目は外側の箇条書きと一緒に見る。 */
47
+ export const outermostList = (offset: number, lists: readonly Span[]): number | undefined =>
48
+ lists
49
+ .filter((list) => offset >= list.start && offset < list.end)
50
+ .reduce<number | undefined>((outer, list) => (outer === undefined || list.start < outer ? list.start : outer), undefined);
51
+
52
+ /** group: 文が入っている箇条書き(outermostList の値)。本文なら undefined。 */
53
+ export type Judged = { readonly register: Register; readonly group: number | undefined };
54
+
55
+ /** 混ざった文と、その文が属する本文または箇条書きの中の少数派の数。 */
56
+ export type Slip<T extends Judged> = { readonly entry: T; readonly count: number };
57
+
58
+ const countIn = (judged: readonly Judged[], register: Register): number => judged.filter((entry) => entry.register === register).length;
59
+
60
+ /** 少ないほう。同数なら文書全体で少ないほう、それも同数なら丁寧体。 */
61
+ const minorityOf = (group: readonly Judged[], whole: readonly Judged[]): Register => {
62
+ const difference = countIn(group, "polite") - countIn(group, "plain");
63
+ if (difference !== 0) return difference < 0 ? "polite" : "plain";
64
+ return countIn(whole, "polite") <= countIn(whole, "plain") ? "polite" : "plain";
65
+ };
66
+
67
+ const slipsInGroup = <T extends Judged>(judged: readonly T[], group: number | undefined, limit: number): Slip<T>[] => {
68
+ const members = judged.filter((entry) => entry.group === group);
69
+ const minority = minorityOf(members, judged);
70
+ const slips = members.filter((entry) => entry.register === minority);
71
+ // 片方しか無ければ揃っている。少数派が閾値を超えて多ければ、混在ではなく別の文体。
72
+ if (slips.length === 0 || slips.length > limit) return [];
73
+ return slips.map((entry) => ({ entry, count: slips.length }));
74
+ };
75
+
76
+ /**
77
+ * 調子が混ざった文を、judged の順で。本文は本文どうし、箇条書きは 1 つずつ、その中で揃っているかを見る。
78
+ * ですます調の本文に常体の箇条書きを置くのはよくある書き方で、箇条書きが丸ごと揃っていれば混在ではない。
79
+ */
80
+ export const slipsOf = <T extends Judged>(judged: readonly T[], limit: number): Slip<T>[] => {
81
+ const slips = [...new Set(judged.map((entry) => entry.group))].flatMap((group) => slipsInGroup(judged, group, limit));
82
+ return judged.flatMap((entry) => slips.filter((slip) => slip.entry === entry));
83
+ };
@@ -1,4 +1,5 @@
1
- import { endingTokens, hasPredicate, isClosed } from "../sentence-shape.ts";
1
+ import { endingTokens, isClosed } from "../sentence-shape.ts";
2
+ import { continuesInto, outermostList, registerOf, slipsOf, type Register } from "./register.ts";
2
3
  import type { Detector, Finding, Sentence } from "../plugin.ts";
3
4
 
4
5
  /**
@@ -6,36 +7,31 @@ import type { Detector, Finding, Sentence } from "../plugin.ts";
6
7
  *
7
8
  * どちらが正しいかは決めない。決めると、方針の違う書き手にそのまま無視される。
8
9
  * 見るのは文書の中での一貫性だけで、少数派のほうを指摘する。spec §12.3 の方針を日本語にも。
10
+ * 終止符で終わらないもの(見出しの下の `MaaSサービス` のような名前だけの行)は文として数えない。
9
11
  */
10
- /**
11
- * 述語を持たないものは文として数えない。見出しの下の `MaaSサービス` のような
12
- * 名前だけの行がそのまま「である調」の少数派になり、実文書の誤検知はすべてこれだった。
13
- */
14
- const isSentence = (sentence: Sentence): boolean => isClosed(sentence) && hasPredicate(sentence) && endingTokens(sentence).length > 0;
12
+ type Entry = { readonly sentence: Sentence; readonly register: Register; readonly group: number | undefined };
15
13
 
16
- /**
17
- * 文末の語だけを見る。文の途中の引用(「ご覧ください」という表現を使う。)は文末の調子ではない。
18
- * 書いた形でも原形でも当てる。「ください」の原形は「くださる」で、原形だけを見ると丁寧な文末を見落とす。
19
- */
20
- const isPolite = (sentence: Sentence, polite: readonly string[]): boolean =>
21
- endingTokens(sentence).some((token) => polite.includes(token.surface) || (token.lemma !== undefined && polite.includes(token.lemma)));
14
+ const registerOfSentence = (sentence: Sentence, polite: readonly string[]): Register | undefined => {
15
+ const ending = endingTokens(sentence);
16
+ const tokens = sentence.tokens ?? [];
17
+ const head = ending[0] === undefined ? -1 : tokens.indexOf(ending[0]);
18
+ return registerOf(ending, head > 0 ? tokens.slice(0, head) : [], polite);
19
+ };
22
20
 
23
21
  export const sentenceEnding: Detector = (doc, options): Finding[] => {
24
22
  const polite = (options.lexicon ?? []).map((entry) => entry.pattern);
25
- const judged = doc.sentences.filter(isSentence).map((sentence) => ({ sentence, polite: isPolite(sentence, polite) }));
26
- const counts = { polite: judged.filter((entry) => entry.polite).length, plain: judged.length - judged.filter((entry) => entry.polite).length };
27
- const minorityIsPolite = counts.polite <= counts.plain;
28
- const few = minorityIsPolite ? counts.polite : counts.plain;
29
- // 片方しか無ければ一貫している。少数派が閾値を超えて多ければ、混在ではなく別の文体。
30
- if (few === 0 || few > options.limit) return [];
31
- return judged
32
- .filter((entry) => entry.polite === minorityIsPolite)
33
- .map(({ sentence }) => ({
34
- rule: "no-mixed-desumasu",
35
- severity: "warning",
36
- line: 0,
37
- column: 0,
38
- quote: sentence.text.trim(),
39
- values: { count: few, limit: options.limit, offset: sentence.span.start },
40
- }));
23
+ const lists = doc.lists.map((list) => list.span);
24
+ const judged = doc.sentences.flatMap((sentence, index): Entry[] => {
25
+ if (!isClosed(sentence) || continuesInto(sentence, doc.sentences[index + 1])) return [];
26
+ const register = registerOfSentence(sentence, polite);
27
+ return register === undefined ? [] : [{ sentence, register, group: outermostList(sentence.span.start, lists) }];
28
+ });
29
+ return slipsOf(judged, options.limit).map(({ entry: { sentence }, count }) => ({
30
+ rule: "no-mixed-desumasu",
31
+ severity: "warning",
32
+ line: 0,
33
+ column: 0,
34
+ quote: sentence.text.trim(),
35
+ values: { count, limit: options.limit, offset: sentence.span.start },
36
+ }));
41
37
  };
@@ -1,6 +1,8 @@
1
1
  import { proseText } from "../measure.ts";
2
2
  import { wordsOf } from "./structure.ts";
3
3
  import { compacted, placeOf } from "./gram-place.ts";
4
+ import { notAcronymSpansOf, type NotAcronymSpans } from "./acronym-context.ts";
5
+ import { expansionAt, type ExpandedAt } from "./acronym-expansion.ts";
4
6
  import type { Detector, Finding, ProseDocument, Section, Sentence, Token } from "../plugin.ts";
5
7
 
6
8
  const PER = 1000;
@@ -181,148 +183,50 @@ type Span = { readonly start: number; readonly end: number };
181
183
 
182
184
  const spanOf = (match: RegExpExecArray): Span => ({ start: match.index, end: match.index + match[0].length });
183
185
 
184
- const spansOf = (text: string): Span[] => [
186
+ const spansOf = (text: string, notation: NotAcronymSpans): Span[] => [
185
187
  ...[...text.matchAll(UNCASED_STRETCH)].filter((match) => SHOUTED_RUN.test(match[0])).map(spanOf),
186
188
  ...[...text.matchAll(IDENTIFIER)].filter((match) => /\d/u.test(match[0])).map(spanOf),
187
189
  ...[QUOTED_CAPS, REQUIREMENT_WORD, LICENCE].flatMap((pattern) => [...text.matchAll(pattern)].map(spanOf)),
190
+ ...notation(text),
188
191
  ];
189
192
 
190
193
  type AcronymHit = { readonly word: string; readonly hit: Hit };
191
194
 
192
- const acronymsOf = (doc: ProseDocument): AcronymHit[] =>
195
+ const acronymsOf = (doc: ProseDocument, notation: NotAcronymSpans): AcronymHit[] =>
193
196
  doc.sentences.flatMap((sentence) => {
194
- const excluded = spansOf(sentence.text);
197
+ const excluded = spansOf(sentence.text, notation);
195
198
  return [...sentence.text.matchAll(ACRONYM)]
196
199
  .filter((match) => !excluded.some((span) => span.start <= match.index && match.index + match[0].length <= span.end))
197
200
  .map((match) => ({ word: match[0], hit: { sentence, offset: sentence.span.start + match.index } }));
198
201
  });
199
202
 
200
- /** 読み手が説明なしで通じると見なしてよい語。展開すると逆に読みにくい。 */
201
- const COMMON = new Set([
202
- "OK",
203
- "NG",
204
- "URL",
205
- "API",
206
- "CSS",
207
- "HTML",
208
- "JSON",
209
- "YAML",
210
- "HTTP",
211
- "HTTPS",
212
- "PDF",
213
- "CPU",
214
- "GPU",
215
- "RAM",
216
- "USB",
217
- "AI",
218
- "ID",
219
- "FAQ",
220
- "PR",
221
- "OS",
222
- "CLI",
223
- "UI",
224
- "UX",
225
- "SQL",
226
- "CSV",
227
- "XML",
228
- "TODO",
229
- "NOTE",
230
- // 通信と符号化。技術文書でなくても説明なしで通じる。
231
- "DNS",
232
- "IP",
233
- "TCP",
234
- "UDP",
235
- "SSH",
236
- "SSL",
237
- "TLS",
238
- "VPN",
239
- "LAN",
240
- "UTF",
241
- "ASCII",
242
- // 機器・形式・単位。
243
- "PC",
244
- "IT",
245
- "SDK",
246
- "IDE",
247
- "PNG",
248
- "JPEG",
249
- "GIF",
250
- "SVG",
251
- "QR",
252
- "GPS",
253
- "SNS",
254
- "TV",
255
- "KB",
256
- "MB",
257
- "GB",
258
- "TB",
259
- // 役職と国・地域。
260
- "CEO",
261
- "CTO",
262
- "CFO",
263
- "US",
264
- "UK",
265
- "EU",
266
- "UN",
267
- "DNA",
268
- ]);
269
-
270
- /**
271
- * 展開は略語の**すぐ隣**にあるときだけ認める。
272
- * 60 文字も見ると、同じ文のどこかに括弧があるだけで「説明済み」になり、1 件も出なくなる。
273
- *
274
- * 認めるのは 3 つの形。どれも実際によく書かれる。
275
- * CI(継続的インテグレーション) 略語のあとに括弧
276
- * Continuous Integration (CI) 括弧の中が略語
277
- * Tax Cuts and Jobs Act [TCJA] 角括弧の中が略語で、直前の語の頭文字と揃う
278
- * 括弧と略語の間には、引用符((“MNDA”)、(「MNDA」))と空白だけを許す。
279
- */
280
- const WRAP = String.raw`[\s"“”'‘’「」『』]*`;
281
- const OPENS = new RegExp(String.raw`^${WRAP}[((]`, "u");
282
- const CLOSES = new RegExp(String.raw`^${WRAP}[))]`, "u");
283
- const OPENED = new RegExp(String.raw`[((]${WRAP}$`, "u");
284
- const SQUARE_CLOSES = new RegExp(String.raw`^${WRAP}\]`, "u");
285
- const SQUARE_OPENED = new RegExp(String.raw`\[${WRAP}$`, "u");
286
-
287
- /** 括弧と略語の間の幅。空白は 1 つに畳んであるので、(“ MNDA ”) まで収まる。 */
288
- const NEAR = 3;
289
-
290
- /**
291
- * 角括弧は引用の印にも使う([IANA]、[1])。直前の語のうち大文字で始まる語の頭文字が
292
- * 略語と揃うときだけ展開と見なす。見る語は略語の文字数の 2 倍まで(and や of を挟むため)。
293
- */
294
- const spellsOut = (before: string, acronym: string): boolean => {
295
- const letters = acronym.replaceAll("&", "");
296
- const words = before
297
- .replace(SQUARE_OPENED, "")
298
- .trim()
299
- .split(/\s+/u)
300
- .slice(-letters.length * 2);
301
- const initials = words.filter((word) => /^[A-Z]/u.test(word)).map((word) => word.charAt(0));
302
- return initials.join("").endsWith(letters);
303
- };
304
-
305
- const isExpandedAt = (body: string, acronym: string, at: number): boolean => {
306
- const after = body.slice(at + acronym.length, at + acronym.length + NEAR);
307
- const before = body.slice(Math.max(0, at - NEAR), at);
308
- if (OPENS.test(after) || (OPENED.test(before) && CLOSES.test(after))) return true;
309
- return SQUARE_OPENED.test(before) && SQUARE_CLOSES.test(after) && spellsOut(body.slice(0, at), acronym);
310
- };
311
-
312
203
  /**
313
204
  * どこか 1 か所で展開してあればよい。初出が節の見出し代わりの語(「5.3. DPA.」)で、
314
205
  * 展開がその直後の文にあることが契約書では普通にある。見るのは語として現れた所だけ(CISA の中の CI は見ない)。
315
206
  */
316
- const isExpanded = (body: string, acronym: string): boolean =>
317
- [...body.matchAll(new RegExp(`${EDGE_BEFORE}${acronym}${EDGE_AFTER}`, "gu"))].some((match) => isExpandedAt(body, acronym, match.index));
207
+ const isExpanded = (body: string, acronym: string, expandedAt: ExpandedAt): boolean =>
208
+ [...body.matchAll(new RegExp(`${EDGE_BEFORE}${acronym}${EDGE_AFTER}`, "gu"))].some((match) => expandedAt(body, acronym, match.index));
209
+
210
+ const patternsOf = (doc: ProseDocument, id: string): string[] => (doc.lexicons[id] ?? []).map((entry) => entry.pattern);
211
+
212
+ const notationOf = (doc: ProseDocument): NotAcronymSpans =>
213
+ notAcronymSpansOf({
214
+ meridiem: patternsOf(doc, "meridiem"),
215
+ timeZones: patternsOf(doc, "time-zone"),
216
+ currencies: patternsOf(doc, "currency-code"),
217
+ usStates: patternsOf(doc, "us-state-code"),
218
+ emphasis: patternsOf(doc, "emphasis-word"),
219
+ });
318
220
 
319
221
  export const undefinedAcronym: Detector = (doc, options): Finding[] => {
320
222
  const body = bodyOf(doc);
223
+ const common = new Set((options.lexicon ?? []).map((entry) => entry.pattern));
321
224
  const seen = new Map<string, Hit>();
322
- acronymsOf(doc).forEach(({ word, hit }) => {
323
- if (!COMMON.has(word) && !seen.has(word)) seen.set(word, hit);
225
+ acronymsOf(doc, notationOf(doc)).forEach(({ word, hit }) => {
226
+ if (!common.has(word) && !seen.has(word)) seen.set(word, hit);
324
227
  });
325
- const bare = [...seen.entries()].filter(([acronym]) => !isExpanded(body, acronym));
228
+ const expandedAt = expansionAt({ markers: patternsOf(doc, "definition-marker"), verbs: patternsOf(doc, "definition-verb") });
229
+ const bare = [...seen.entries()].filter(([acronym]) => !isExpanded(body, acronym, expandedAt));
326
230
  if (bare.length < options.limit) return [];
327
231
  return bare.map(([acronym, hit]) => ({
328
232
  rule: "undefined-acronym",
@@ -1,5 +1,7 @@
1
1
  import { charLength, lengthOf } from "../measure.ts";
2
- import type { BulletList, Detector, Finding, Paragraph, ProseDocument, Section } from "../plugin.ts";
2
+ import type { BulletList, Detector, Finding, Paragraph, ProseDocument, Section, Span } from "../plugin.ts";
3
+ import { dateStampIndexes, stampCandidates } from "../date-stamp.ts";
4
+ import { inDocumentOrder } from "../structure/issues.ts";
3
5
 
4
6
  /**
5
7
  * ばらつきは変動係数(標準偏差 ÷ 平均)で測る。
@@ -90,6 +92,21 @@ export const ruleOfThree: Detector = (doc, options): Finding[] => {
90
92
  ];
91
93
  };
92
94
 
95
+ const dateSpans = (doc: ProseDocument): Span[] =>
96
+ doc.structure === undefined ? [] : inDocumentOrder(doc.structure).flatMap((node) => (node.kind === "date" ? [node.span] : []));
97
+
98
+ /** 更新日の刻印(「最終更新日:」「2025年6月20日」)は前置きに数えない。木を作るのは、数えて上限を超えたときだけ。 */
99
+ const withoutDateStamps = (doc: ProseDocument, paragraphs: readonly Paragraph[]): Paragraph[] => {
100
+ const labels = (doc.lexicons["date-stamp-label"] ?? []).map((entry) => entry.pattern);
101
+ const candidates = stampCandidates(
102
+ doc.source,
103
+ paragraphs.map((paragraph) => paragraph.span),
104
+ dateSpans(doc),
105
+ );
106
+ const stamps = dateStampIndexes(candidates, labels);
107
+ return paragraphs.filter((_, index) => !stamps.has(index));
108
+ };
109
+
93
110
  /**
94
111
  * 本題に入るまでが長い。業務文書では、読み手は結論を探しに来ている。
95
112
  *
@@ -99,7 +116,9 @@ export const ruleOfThree: Detector = (doc, options): Finding[] => {
99
116
  export const preambleLength: Detector = (doc, options): Finding[] => {
100
117
  const body = doc.sections.find((section) => section.depth >= 2);
101
118
  if (body === undefined) return [];
102
- const before = doc.paragraphs.filter((paragraph) => paragraph.span.start < body.span.start);
119
+ const all = doc.paragraphs.filter((paragraph) => paragraph.span.start < body.span.start);
120
+ if (all.length <= options.limit) return [];
121
+ const before = withoutDateStamps(doc, all);
103
122
  if (before.length <= options.limit) return [];
104
123
  const first = before[0];
105
124
  return first === undefined
@@ -0,0 +1,66 @@
1
+ import type { Lexicon, Token } from "../plugin.ts";
2
+ import type { TokenRange } from "./lexicon-match.ts";
3
+
4
+ /**
5
+ * 最上級が範囲を持っているか。「日本で最も」「国内最大」"the best in the world" は、何の中で最もなのかを言っている。
6
+ * 範囲を示す語は言語で違い、置く側も違う(日本語は最上級の前、英語は後ろ)。どちらの語も言語パッケージの語彙表が持つ。
7
+ */
8
+ export type ScopeMarkers = {
9
+ /** 最上級の直前に置く語(で)。その前に固有名詞か地名が要る。 */
10
+ readonly before: Lexicon;
11
+ /** 最上級の名詞句の直後に置く語(in・of)。その後ろに名詞句が要る。 */
12
+ readonly after: Lexicon;
13
+ };
14
+
15
+ /** 語彙表の position で分ける。position の無い語は、どちら側の語か分からないので使わない。 */
16
+ export const scopeMarkersOf = (lexicon: Lexicon): ScopeMarkers => ({
17
+ before: lexicon.filter((entry) => entry.position === "before"),
18
+ after: lexicon.filter((entry) => entry.position === "after"),
19
+ });
20
+
21
+ const NOMINAL = new Set(["NOUN", "PROPN"]);
22
+
23
+ /** 名詞句の始まり。the world・Chicago・our products・the three・all。 */
24
+ const PHRASE_START = new Set(["DET", "NOUN", "PROPN", "PRON", "NUM", "ADJ"]);
25
+
26
+ /** 名詞の前に来る修飾(most skilled・widely used・stuffed)。名詞が出た後は名詞の連なりだけが続く。 */
27
+ const PREMODIFIER = new Set(["ADJ", "ADV", "VERB", "NOUN", "PROPN", "NUM"]);
28
+ const NOUN_RUN = new Set(["NOUN", "PROPN", "NUM"]);
29
+
30
+ /** 範囲の前に置ける語。修飾だけで終わった句(the best is in …)は範囲ではなく述語。 */
31
+ const PHRASE_END = new Set(["NOUN", "PROPN", "NUM", "ADJ"]);
32
+
33
+ const isMarker = (token: Token | undefined, markers: Lexicon): boolean =>
34
+ token !== undefined && markers.some((entry) => entry.pattern.toLowerCase() === token.surface.toLowerCase());
35
+
36
+ const isNamed = (token: Token | undefined): boolean => token !== undefined && (token.pos === "PROPN" || token.features?.["NameType"] !== undefined);
37
+
38
+ /** 空白も助詞も挟まずに名詞が付いた最上級(国内最大・業界最速・世界唯一)。前の名詞が範囲。 */
39
+ const compound = (tokens: readonly Token[], range: TokenRange): boolean => {
40
+ const before = tokens[range.start - 1];
41
+ const first = tokens[range.start];
42
+ return before !== undefined && first !== undefined && NOMINAL.has(before.pos) && before.span.end === first.span.start;
43
+ };
44
+
45
+ /** 名前 + 範囲の語 + 最上級(日本で最も・トヨタで最も・東京都で最大)。「費用で最大の効果」の「費用」は名前ではないので範囲にしない。 */
46
+ const namedBefore = (tokens: readonly Token[], range: TokenRange, markers: Lexicon): boolean =>
47
+ isMarker(tokens[range.start - 1], markers) && isNamed(tokens[range.start - 2]);
48
+
49
+ /** 最上級に続く名詞句の終わり。名詞が出るまでは修飾を、出た後は名詞の連なりを読む。 */
50
+ const phraseEnd = (tokens: readonly Token[], at: number, nounSeen = false): number => {
51
+ const token = tokens[at];
52
+ if (token === undefined || !(nounSeen ? NOUN_RUN : PREMODIFIER).has(token.pos)) return at;
53
+ return phraseEnd(tokens, at + 1, nounSeen || NOMINAL.has(token.pos));
54
+ };
55
+
56
+ /** 最上級 + 名詞句 + 範囲の語 + 名詞句(the best pizza in Chicago・the most famous of the sculptures)。 */
57
+ const phraseAfter = (tokens: readonly Token[], range: TokenRange, markers: Lexicon): boolean => {
58
+ const at = phraseEnd(tokens, range.end);
59
+ const last = tokens[at - 1];
60
+ const endsPhrase = at === range.end || (last !== undefined && PHRASE_END.has(last.pos));
61
+ const opens = tokens[at + 1];
62
+ return endsPhrase && isMarker(tokens[at], markers) && opens !== undefined && PHRASE_START.has(opens.pos);
63
+ };
64
+
65
+ export const scoped = (tokens: readonly Token[], range: TokenRange, markers: ScopeMarkers): boolean =>
66
+ compound(tokens, range) || namedBefore(tokens, range, markers.before) || phraseAfter(tokens, range, markers.after);