@chaffjs/lang-ja 0.16.0 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (151) hide show
  1. package/README.md +1 -1
  2. package/dist/citation.d.ts +6 -3
  3. package/dist/citation.d.ts.map +1 -1
  4. package/dist/citation.js +31 -9
  5. package/dist/citation.js.map +1 -1
  6. package/dist/closing-quote.d.ts +6 -0
  7. package/dist/closing-quote.d.ts.map +1 -0
  8. package/dist/closing-quote.js +50 -0
  9. package/dist/closing-quote.js.map +1 -0
  10. package/dist/english-run.d.ts +8 -0
  11. package/dist/english-run.d.ts.map +1 -0
  12. package/dist/english-run.js +24 -0
  13. package/dist/english-run.js.map +1 -0
  14. package/dist/index.d.ts.map +1 -1
  15. package/dist/index.js +56 -21
  16. package/dist/index.js.map +1 -1
  17. package/dist/inside-brackets.d.ts +5 -0
  18. package/dist/inside-brackets.d.ts.map +1 -0
  19. package/dist/inside-brackets.js +44 -0
  20. package/dist/inside-brackets.js.map +1 -0
  21. package/dist/label-stop.d.ts +13 -0
  22. package/dist/label-stop.d.ts.map +1 -0
  23. package/dist/label-stop.js +14 -0
  24. package/dist/label-stop.js.map +1 -0
  25. package/dist/lexicons.d.ts.map +1 -1
  26. package/dist/lexicons.js +4 -0
  27. package/dist/lexicons.js.map +1 -1
  28. package/dist/long-vowel-form.d.ts +12 -0
  29. package/dist/long-vowel-form.d.ts.map +1 -0
  30. package/dist/long-vowel-form.js +28 -0
  31. package/dist/long-vowel-form.js.map +1 -0
  32. package/dist/number-stop.d.ts +3 -0
  33. package/dist/number-stop.d.ts.map +1 -0
  34. package/dist/number-stop.js +25 -0
  35. package/dist/number-stop.js.map +1 -0
  36. package/dist/pos.d.ts +17 -1
  37. package/dist/pos.d.ts.map +1 -1
  38. package/dist/pos.js +69 -36
  39. package/dist/pos.js.map +1 -1
  40. package/dist/predicate-frame.d.ts +14 -0
  41. package/dist/predicate-frame.d.ts.map +1 -0
  42. package/dist/predicate-frame.js +22 -0
  43. package/dist/predicate-frame.js.map +1 -0
  44. package/dist/quantities.d.ts.map +1 -1
  45. package/dist/quantities.js +11 -1
  46. package/dist/quantities.js.map +1 -1
  47. package/dist/quoted-stop.d.ts +13 -0
  48. package/dist/quoted-stop.d.ts.map +1 -0
  49. package/dist/quoted-stop.js +45 -0
  50. package/dist/quoted-stop.js.map +1 -0
  51. package/dist/ra-dropped.d.ts +12 -0
  52. package/dist/ra-dropped.d.ts.map +1 -0
  53. package/dist/ra-dropped.js +18 -0
  54. package/dist/ra-dropped.js.map +1 -0
  55. package/dist/reduplication.d.ts +17 -5
  56. package/dist/reduplication.d.ts.map +1 -1
  57. package/dist/reduplication.js +85 -6
  58. package/dist/reduplication.js.map +1 -1
  59. package/dist/regexp.d.ts +3 -0
  60. package/dist/regexp.d.ts.map +1 -0
  61. package/dist/regexp.js +4 -0
  62. package/dist/regexp.js.map +1 -0
  63. package/dist/sentence-close.d.ts +2 -0
  64. package/dist/sentence-close.d.ts.map +1 -0
  65. package/dist/sentence-close.js +10 -0
  66. package/dist/sentence-close.js.map +1 -0
  67. package/dist/sentence-split.d.ts +2 -0
  68. package/dist/sentence-split.d.ts.map +1 -1
  69. package/dist/sentence-split.js +3 -1
  70. package/dist/sentence-split.js.map +1 -1
  71. package/dist/structure.d.ts.map +1 -1
  72. package/dist/structure.js +60 -21
  73. package/dist/structure.js.map +1 -1
  74. package/dist/well-formed.d.ts +4 -1
  75. package/dist/well-formed.d.ts.map +1 -1
  76. package/dist/well-formed.js +5 -2
  77. package/dist/well-formed.js.map +1 -1
  78. package/lexicons/abbreviated-label.yaml +16 -0
  79. package/lexicons/ai-tell.yaml +105 -0
  80. package/lexicons/announcing-opener.yaml +11 -0
  81. package/lexicons/assistant-residue.yaml +37 -0
  82. package/lexicons/calendar-era.yaml +10 -0
  83. package/lexicons/closing-cliche.yaml +2 -0
  84. package/lexicons/comparison-marker.yaml +7 -0
  85. package/lexicons/contrast-frame.yaml +9 -0
  86. package/lexicons/contrast-lead.yaml +10 -0
  87. package/lexicons/contrast-turn.yaml +9 -0
  88. package/lexicons/count-anchor.yaml +10 -0
  89. package/lexicons/count-counter.yaml +17 -0
  90. package/lexicons/count-hedge.yaml +61 -0
  91. package/lexicons/count-number.yaml +16 -0
  92. package/lexicons/distributive-noun.yaml +14 -2
  93. package/lexicons/double-negative.yaml +32 -0
  94. package/lexicons/doubled-word.yaml +12 -0
  95. package/lexicons/email-attachment-note.yaml +8 -0
  96. package/lexicons/email-attribution.yaml +10 -0
  97. package/lexicons/email-header-field.yaml +37 -0
  98. package/lexicons/email-written-field.yaml +11 -0
  99. package/lexicons/enumeration-frame.yaml +32 -0
  100. package/lexicons/enumeration-joiner.yaml +15 -0
  101. package/lexicons/enumeration-negation.yaml +10 -0
  102. package/lexicons/figure-label.yaml +11 -0
  103. package/lexicons/item-number.yaml +38 -0
  104. package/lexicons/long-vowel-distinct.yaml +25 -0
  105. package/lexicons/long-vowel-drop-after.yaml +25 -0
  106. package/lexicons/measure-unit.yaml +78 -0
  107. package/lexicons/name-note.yaml +5 -1
  108. package/lexicons/neutral-ending.yaml +10 -0
  109. package/lexicons/nominalization-event-noun.yaml +13 -0
  110. package/lexicons/nominalization-light-verb.yaml +10 -0
  111. package/lexicons/numbered-label.yaml +50 -0
  112. package/lexicons/passive-predicate-frame.yaml +13 -0
  113. package/lexicons/percent-unit.yaml +7 -0
  114. package/lexicons/placeholder-word.yaml +21 -0
  115. package/lexicons/ra-dropped-verb.yaml +7 -0
  116. package/lexicons/range-closer.yaml +5 -0
  117. package/lexicons/range-connector.yaml +11 -0
  118. package/lexicons/range-opener.yaml +7 -0
  119. package/lexicons/requirement-either.yaml +13 -0
  120. package/lexicons/requirement-loophole.yaml +13 -0
  121. package/lexicons/requirement-marker.yaml +15 -0
  122. package/lexicons/requirement-modal-jis-z8301-2019.yaml +14 -0
  123. package/lexicons/requirement-not-marker.yaml +8 -0
  124. package/lexicons/requirement-open-end.yaml +6 -0
  125. package/lexicons/sasete-itadaku.yaml +11 -0
  126. package/lexicons/share-exception.yaml +7 -0
  127. package/lexicons/share-label.yaml +11 -0
  128. package/lexicons/stock-transition.yaml +11 -0
  129. package/lexicons/vague-clause-pointer.yaml +14 -0
  130. package/lexicons/vague-figure-pointer.yaml +23 -0
  131. package/package.json +1 -1
  132. package/src/citation.ts +35 -10
  133. package/src/closing-quote.ts +53 -0
  134. package/src/english-run.ts +25 -0
  135. package/src/index.ts +54 -13
  136. package/src/inside-brackets.ts +45 -0
  137. package/src/label-stop.ts +26 -0
  138. package/src/lexicons.ts +4 -0
  139. package/src/long-vowel-form.ts +38 -0
  140. package/src/number-stop.ts +30 -0
  141. package/src/pos.ts +78 -29
  142. package/src/predicate-frame.ts +28 -0
  143. package/src/quantities.ts +12 -1
  144. package/src/quoted-stop.ts +53 -0
  145. package/src/ra-dropped.ts +29 -0
  146. package/src/reduplication.ts +119 -8
  147. package/src/regexp.ts +3 -0
  148. package/src/sentence-close.ts +10 -0
  149. package/src/sentence-split.ts +3 -1
  150. package/src/structure.ts +62 -19
  151. package/src/well-formed.ts +5 -2
package/README.md CHANGED
@@ -4,7 +4,7 @@
4
4
 
5
5
  持っているもの:
6
6
 
7
- - 文分割(日本語の文末は `。!?` に限る。`Dr.` のようなラテン略語で切らない)
7
+ - 文分割(日本語の文末は `。!?` と、語の後の `.` に限る。`Dr.` のようなラテン略語で切らない)
8
8
  - L2 の語彙表(空虚な強調 / 水増しの導入 / 定型の結び)
9
9
 
10
10
  新しい言語を足すときは、このパッケージが雛形になります。公式は `@chaffjs/lang-<言語>`、第三者は `chaff-lang-<言語>` と名乗ってください。
@@ -1,4 +1,4 @@
1
- import type { Lexicon } from "chaffjs/plugin";
1
+ import type { Lexicon, NamedDocument } from "chaffjs/plugin";
2
2
  export type CitationVocabulary = {
3
3
  /** 文書の種類の語。長いものから当てる。 */
4
4
  readonly kinds: readonly string[];
@@ -13,8 +13,11 @@ export type CitationVocabulary = {
13
13
  };
14
14
  export declare const citationVocabulary: (lexicons: Readonly<Record<string, Lexicon>>) => CitationVocabulary;
15
15
  /**
16
- * reference の直前に書かれた文書名。「民法第709条」「民法の第709条」なら「民法」。この文書の条を指すなら undefined。
17
- * 名前が種類の語だけ(「契約第3条」)のときも、どの文書か決まらないので undefined にする。
16
+ * reference の直前に書かれた文書名。「民法第709条」「民法の第709条」なら「民法」、「この規則の別表」「本規約第3条」ならこの文書(self)。
17
+ * 名前が種類の語だけ(「契約第3条」)のときは、どの文書か決まらないので undefined にする。
18
+ * 公布の番号を添えた名前(「…に関する基準(昭和五十八年厚生省告示第十四号)第二条」)は、種類の語が無くても文書の名前。
18
19
  */
20
+ export declare const namedDocument: (text: string, reference: number, vocabulary: CitationVocabulary) => NamedDocument | undefined;
21
+ /** reference の直前に書かれた他の文書の名前。この文書を指すか、名前が無ければ undefined。 */
19
22
  export declare const citedDocument: (text: string, reference: number, vocabulary: CitationVocabulary) => string | undefined;
20
23
  //# sourceMappingURL=citation.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"citation.d.ts","sourceRoot":"","sources":["../src/citation.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAI9C,MAAM,MAAM,kBAAkB,GAAG;IAC/B,yBAAyB;IACzB,QAAQ,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,mBAAmB;IACnB,QAAQ,CAAC,cAAc,EAAE,SAAS,MAAM,EAAE,CAAC;IAC3C,yBAAyB;IACzB,QAAQ,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,iCAAiC;IACjC,QAAQ,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,CAAC;IACpC,qBAAqB;IACrB,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;CAC1C,CAAC;AAIF,eAAO,MAAM,kBAAkB,GAAI,UAAU,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,KAAG,kBAM/E,CAAC;AA4BH;;;GAGG;AACH,eAAO,MAAM,aAAa,GAAI,MAAM,MAAM,EAAE,WAAW,MAAM,EAAE,YAAY,kBAAkB,KAAG,MAAM,GAAG,SAOxG,CAAC"}
1
+ {"version":3,"file":"citation.d.ts","sourceRoot":"","sources":["../src/citation.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAI7D,MAAM,MAAM,kBAAkB,GAAG;IAC/B,yBAAyB;IACzB,QAAQ,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,mBAAmB;IACnB,QAAQ,CAAC,cAAc,EAAE,SAAS,MAAM,EAAE,CAAC;IAC3C,yBAAyB;IACzB,QAAQ,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,iCAAiC;IACjC,QAAQ,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,CAAC;IACpC,qBAAqB;IACrB,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;CAC1C,CAAC;AAIF,eAAO,MAAM,kBAAkB,GAAI,UAAU,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,KAAG,kBAM/E,CAAC;AAkCH;;;;GAIG;AACH,eAAO,MAAM,aAAa,GAAI,MAAM,MAAM,EAAE,WAAW,MAAM,EAAE,YAAY,kBAAkB,KAAG,aAAa,GAAG,SAU/G,CAAC;AAEF,4DAA4D;AAC5D,eAAO,MAAM,aAAa,GAAI,MAAM,MAAM,EAAE,WAAW,MAAM,EAAE,YAAY,kBAAkB,KAAG,MAAM,GAAG,SAGxG,CAAC"}
package/dist/citation.js CHANGED
@@ -10,8 +10,8 @@ const NAME_CHAR = /[\p{Script=Han}\p{Script=Katakana}ー・A-Za-z0-9]/u;
10
10
  const TITLE_CHAR = /[\p{Script=Han}\p{Script=Katakana}\p{Script=Hiragana}ー・A-Za-z0-9]/u;
11
11
  /** 名前は長くても数十字。後ろ向きに読む長さを抑え、長い行で遅くならないようにする。 */
12
12
  const MAX_NAME_LENGTH = 30;
13
- /** 名前の後ろの括弧書きが収まる長さ。後ろ向きに読む長さを抑える。 */
14
- const MAX_NOTE_LENGTH = 100;
13
+ /** 名前の後ろの括弧書きが収まる長さ。name-note の最も長い形(略称・公布の番号・「以下…という。」)が収まり、後ろ向きに読む長さを抑える。 */
14
+ const MAX_NOTE_LENGTH = 130;
15
15
  const nameBefore = (text, at, char) => {
16
16
  let start = at;
17
17
  while (start > 0 && at - start < MAX_NAME_LENGTH && char.test(text[start - 1] ?? ""))
@@ -27,17 +27,39 @@ const beforeJoiner = (text, at, joiners) => {
27
27
  const joiner = joiners.find((candidate) => text.endsWith(candidate, at));
28
28
  return joiner === undefined ? at : at - joiner.length;
29
29
  };
30
+ /** 名前が種類の語で終わり、種類の語より長い(「民法」。「契約」だけではどの文書か決まらない)。 */
31
+ const endsWithKind = (name, kinds) => {
32
+ const kind = kinds.find((candidate) => name.endsWith(candidate));
33
+ return kind !== undefined && name.length > kind.length;
34
+ };
30
35
  /**
31
- * reference の直前に書かれた文書名。「民法第709条」「民法の第709条」なら「民法」。この文書の条を指すなら undefined。
32
- * 名前が種類の語だけ(「契約第3条」)のときも、どの文書か決まらないので undefined にする。
36
+ * reference の直前に書かれた文書名。「民法第709条」「民法の第709条」なら「民法」、「この規則の別表」「本規約第3条」ならこの文書(self)。
37
+ * 名前が種類の語だけ(「契約第3条」)のときは、どの文書か決まらないので undefined にする。
38
+ * 公布の番号を添えた名前(「…に関する基準(昭和五十八年厚生省告示第十四号)第二条」)は、種類の語が無くても文書の名前。
33
39
  */
34
- export const citedDocument = (text, reference, vocabulary) => {
35
- const at = beforeNote(text, beforeJoiner(text, reference, vocabulary.joiners), vocabulary.notes);
40
+ export const namedDocument = (text, reference, vocabulary) => {
41
+ const afterName = beforeJoiner(text, reference, vocabulary.joiners);
42
+ const at = beforeNote(text, afterName, vocabulary.notes);
36
43
  const plain = nameBefore(text, at, NAME_CHAR);
37
44
  const name = vocabulary.kanaTitleKinds.includes(plain) ? nameBefore(text, at, TITLE_CHAR) : plain;
38
- const kind = vocabulary.kinds.find((candidate) => name.endsWith(candidate));
39
- if (kind === undefined || name.length === kind.length)
45
+ const self = namesThisDocument(text, at - name.length, name, vocabulary.selfPrefixes);
46
+ if (self && vocabulary.kinds.some((kind) => name.endsWith(kind)))
47
+ return { name, self };
48
+ const numbered = at !== afterName && name !== "";
49
+ if (!numbered && !endsWithKind(name, vocabulary.kinds))
40
50
  return undefined;
41
- return vocabulary.selfPrefixes.some((prefix) => name.startsWith(prefix)) ? undefined : name;
51
+ return { name, self };
52
+ };
53
+ /** reference の直前に書かれた他の文書の名前。この文書を指すか、名前が無ければ undefined。 */
54
+ export const citedDocument = (text, reference, vocabulary) => {
55
+ const named = namedDocument(text, reference, vocabulary);
56
+ return named === undefined || named.self ? undefined : named.name;
57
+ };
58
+ /** 自分を指す頭の語の長さの上限(この・本・当)。後ろ向きに読む長さを抑える。 */
59
+ const MAX_PREFIX_LENGTH = 4;
60
+ /** 名前が自分を指す頭の語で始まるか、そのすぐ前に仮名の頭の語がある(「この基準(…号)第9条」の「この」)。 */
61
+ const namesThisDocument = (text, nameStart, name, selfPrefixes) => {
62
+ const lead = text.slice(Math.max(0, nameStart - MAX_PREFIX_LENGTH), nameStart);
63
+ return selfPrefixes.some((prefix) => name.startsWith(prefix) || lead.endsWith(prefix));
42
64
  };
43
65
  //# sourceMappingURL=citation.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"citation.js","sourceRoot":"","sources":["../src/citation.ts"],"names":[],"mappings":"AAiBA,MAAM,UAAU,GAAG,CAAC,OAA4B,EAAY,EAAE,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;AAE7G,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,QAA2C,EAAsB,EAAE,CAAC,CAAC;IACtG,KAAK,EAAE,UAAU,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IAClG,cAAc,EAAE,UAAU,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;IACvD,KAAK,EAAE,UAAU,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,MAAM,OAAO,IAAI,EAAE,GAAG,CAAC,CAAC;IAC7F,OAAO,EAAE,UAAU,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;IAC5C,YAAY,EAAE,UAAU,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;CAClD,CAAC,CAAC;AAEH,MAAM,SAAS,GAAG,iDAAiD,CAAC;AACpE,MAAM,UAAU,GAAG,oEAAoE,CAAC;AAExF,+CAA+C;AAC/C,MAAM,eAAe,GAAG,EAAE,CAAC;AAE3B,sCAAsC;AACtC,MAAM,eAAe,GAAG,GAAG,CAAC;AAE5B,MAAM,UAAU,GAAG,CAAC,IAAY,EAAE,EAAU,EAAE,IAAY,EAAU,EAAE;IACpE,IAAI,KAAK,GAAG,EAAE,CAAC;IACf,OAAO,KAAK,GAAG,CAAC,IAAI,EAAE,GAAG,KAAK,GAAG,eAAe,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QAAE,KAAK,IAAI,CAAC,CAAC;IACjG,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;AAC/B,CAAC,CAAC;AAEF,MAAM,UAAU,GAAG,CAAC,IAAY,EAAE,EAAU,EAAE,KAAwB,EAAU,EAAE;IAChF,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,GAAG,eAAe,CAAC,EAAE,EAAE,CAAC,CAAC;IACjE,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACvE,OAAO,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;AACpD,CAAC,CAAC;AAEF,MAAM,YAAY,GAAG,CAAC,IAAY,EAAE,EAAU,EAAE,OAA0B,EAAU,EAAE;IACpF,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC;IACzE,OAAO,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC;AACxD,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,IAAY,EAAE,SAAiB,EAAE,UAA8B,EAAsB,EAAE;IACnH,MAAM,EAAE,GAAG,UAAU,CAAC,IAAI,EAAE,YAAY,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,CAAC,OAAO,CAAC,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC;IACjG,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,EAAE,EAAE,EAAE,SAAS,CAAC,CAAC;IAC9C,MAAM,IAAI,GAAG,UAAU,CAAC,cAAc,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,EAAE,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IAClG,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;IAC5E,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM;QAAE,OAAO,SAAS,CAAC;IACxE,OAAO,UAAU,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC;AAC9F,CAAC,CAAC"}
1
+ {"version":3,"file":"citation.js","sourceRoot":"","sources":["../src/citation.ts"],"names":[],"mappings":"AAiBA,MAAM,UAAU,GAAG,CAAC,OAA4B,EAAY,EAAE,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;AAE7G,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,QAA2C,EAAsB,EAAE,CAAC,CAAC;IACtG,KAAK,EAAE,UAAU,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IAClG,cAAc,EAAE,UAAU,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;IACvD,KAAK,EAAE,UAAU,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,MAAM,OAAO,IAAI,EAAE,GAAG,CAAC,CAAC;IAC7F,OAAO,EAAE,UAAU,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;IAC5C,YAAY,EAAE,UAAU,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;CAClD,CAAC,CAAC;AAEH,MAAM,SAAS,GAAG,iDAAiD,CAAC;AACpE,MAAM,UAAU,GAAG,oEAAoE,CAAC;AAExF,+CAA+C;AAC/C,MAAM,eAAe,GAAG,EAAE,CAAC;AAE3B,+EAA+E;AAC/E,MAAM,eAAe,GAAG,GAAG,CAAC;AAE5B,MAAM,UAAU,GAAG,CAAC,IAAY,EAAE,EAAU,EAAE,IAAY,EAAU,EAAE;IACpE,IAAI,KAAK,GAAG,EAAE,CAAC;IACf,OAAO,KAAK,GAAG,CAAC,IAAI,EAAE,GAAG,KAAK,GAAG,eAAe,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QAAE,KAAK,IAAI,CAAC,CAAC;IACjG,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;AAC/B,CAAC,CAAC;AAEF,MAAM,UAAU,GAAG,CAAC,IAAY,EAAE,EAAU,EAAE,KAAwB,EAAU,EAAE;IAChF,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,GAAG,eAAe,CAAC,EAAE,EAAE,CAAC,CAAC;IACjE,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACvE,OAAO,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;AACpD,CAAC,CAAC;AAEF,MAAM,YAAY,GAAG,CAAC,IAAY,EAAE,EAAU,EAAE,OAA0B,EAAU,EAAE;IACpF,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC;IACzE,OAAO,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC;AACxD,CAAC,CAAC;AAEF,qDAAqD;AACrD,MAAM,YAAY,GAAG,CAAC,IAAY,EAAE,KAAwB,EAAW,EAAE;IACvE,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;IACjE,OAAO,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;AACzD,CAAC,CAAC;AAEF;;;;GAIG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,IAAY,EAAE,SAAiB,EAAE,UAA8B,EAA6B,EAAE;IAC1H,MAAM,SAAS,GAAG,YAAY,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,CAAC,OAAO,CAAC,CAAC;IACpE,MAAM,EAAE,GAAG,UAAU,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC;IACzD,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,EAAE,EAAE,EAAE,SAAS,CAAC,CAAC;IAC9C,MAAM,IAAI,GAAG,UAAU,CAAC,cAAc,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,EAAE,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IAClG,MAAM,IAAI,GAAG,iBAAiB,CAAC,IAAI,EAAE,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC,YAAY,CAAC,CAAC;IACtF,IAAI,IAAI,IAAI,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IACxF,MAAM,QAAQ,GAAG,EAAE,KAAK,SAAS,IAAI,IAAI,KAAK,EAAE,CAAC;IACjD,IAAI,CAAC,QAAQ,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,UAAU,CAAC,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IACzE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;AACxB,CAAC,CAAC;AAEF,4DAA4D;AAC5D,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,IAAY,EAAE,SAAiB,EAAE,UAA8B,EAAsB,EAAE;IACnH,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;IACzD,OAAO,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC;AACpE,CAAC,CAAC;AAEF,4CAA4C;AAC5C,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAE5B,4DAA4D;AAC5D,MAAM,iBAAiB,GAAG,CAAC,IAAY,EAAE,SAAiB,EAAE,IAAY,EAAE,YAA+B,EAAW,EAAE;IACpH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,GAAG,iBAAiB,CAAC,EAAE,SAAS,CAAC,CAAC;IAC/E,OAAO,YAAY,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;AACzF,CAAC,CAAC"}
@@ -0,0 +1,6 @@
1
+ import type { Span } from "chaffjs/plugin";
2
+ /** 文頭の閉じ引用符の並びの長さ。後ろに字が続けば開き引用符かアポストロフィなので 0。 */
3
+ export declare const closingRunLength: (sentence: string) => number;
4
+ /** 文の span の並び(text 上の位置)で、文頭に取り残された閉じ引用符を前の文へ戻したもの。 */
5
+ export declare const reattachClosingQuotes: (text: string, spans: readonly Span[]) => Span[];
6
+ //# sourceMappingURL=closing-quote.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"closing-quote.d.ts","sourceRoot":"","sources":["../src/closing-quote.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AAmB3C,iDAAiD;AACjD,eAAO,MAAM,gBAAgB,GAAI,UAAU,MAAM,KAAG,MAGnD,CAAC;AAoBF,wDAAwD;AACxD,eAAO,MAAM,qBAAqB,GAAI,MAAM,MAAM,EAAE,OAAO,SAAS,IAAI,EAAE,KAAG,IAAI,EAOzE,CAAC"}
@@ -0,0 +1,50 @@
1
+ import { NEXT_SENTENCE_HEAD } from "./quoted-stop.js";
2
+ /**
3
+ * sentence-splitter は疑問符・感嘆符の後の曲がった閉じ引用符(“Is it done?” Nobody …)と直線の一重引用符の手前で文を切る。
4
+ * 次の文が「”」で始まり、前の文は引用符が開いたまま終わる。
5
+ *
6
+ * 前の文が句点で終わり、次の文がそこから間を置かずに閉じ引用符(閉じ括弧も)で始まるとき、それを前の文の末尾へ戻す。
7
+ * 戻した後に空白と大文字が続けば文は二つのまま、何も続かなければ前の文だけ、
8
+ * 小文字・数字・句読点が続けば引用は文の途中にあるので一つの文につなぐ(直線の二重引用符で分割器がそうするのと同じ)。
9
+ * 開き引用符(“ ‘)は閉じの分類に入らない。語頭のアポストロフィ(’Tis ’90s)は後ろに字が続くので閉じと読まない。
10
+ */
11
+ const CLOSING_RUN = /^[\p{Pf}\p{Pe}"']+/u;
12
+ const WORD_CHARACTER = /^[\p{L}\p{N}]/u;
13
+ const STOP_AT_END = /[.?!]$/u;
14
+ // 語頭のアポストロフィの後の大文字(’Tis)も次の文の頭。
15
+ const NEXT_SENTENCE = new RegExp(String.raw `^(?:${NEXT_SENTENCE_HEAD}|\s+’\p{Lu})`, "u");
16
+ const LEADING_SPACE = /^\s*/u;
17
+ /** 文頭の閉じ引用符の並びの長さ。後ろに字が続けば開き引用符かアポストロフィなので 0。 */
18
+ export const closingRunLength = (sentence) => {
19
+ const run = CLOSING_RUN.exec(sentence)?.[0].length ?? 0;
20
+ return run > 0 && !WORD_CHARACTER.test(sentence.slice(run)) ? run : 0;
21
+ };
22
+ const follows = (text, previous, span) => previous.end === span.start && STOP_AT_END.test(text.slice(previous.start, previous.end));
23
+ /** previous と span を置き換える文。閉じ引用符が戻らなければ undefined。 */
24
+ const reattached = (text, previous, span) => {
25
+ const run = follows(text, previous, span) ? closingRunLength(text.slice(span.start, span.end)) : 0;
26
+ if (run === 0)
27
+ return undefined;
28
+ const quoteEnd = span.start + run;
29
+ const rest = text.slice(quoteEnd, span.end);
30
+ if (rest.trim() === "")
31
+ return [{ start: previous.start, end: quoteEnd }];
32
+ if (!NEXT_SENTENCE.test(rest))
33
+ return [{ start: previous.start, end: span.end }];
34
+ const next = quoteEnd + (LEADING_SPACE.exec(rest)?.[0].length ?? 0);
35
+ return [
36
+ { start: previous.start, end: quoteEnd },
37
+ { start: next, end: span.end },
38
+ ];
39
+ };
40
+ /** 文の span の並び(text 上の位置)で、文頭に取り残された閉じ引用符を前の文へ戻したもの。 */
41
+ export const reattachClosingQuotes = (text, spans) => spans.reduce((sentences, span) => {
42
+ const previous = sentences.at(-1);
43
+ const replaced = previous === undefined ? undefined : reattached(text, previous, span);
44
+ if (replaced === undefined)
45
+ sentences.push(span);
46
+ else
47
+ sentences.splice(-1, 1, ...replaced);
48
+ return sentences;
49
+ }, []);
50
+ //# sourceMappingURL=closing-quote.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"closing-quote.js","sourceRoot":"","sources":["../src/closing-quote.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAEtD;;;;;;;;GAQG;AACH,MAAM,WAAW,GAAG,qBAAqB,CAAC;AAC1C,MAAM,cAAc,GAAG,gBAAgB,CAAC;AACxC,MAAM,WAAW,GAAG,SAAS,CAAC;AAC9B,gCAAgC;AAChC,MAAM,aAAa,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAA,OAAO,kBAAkB,cAAc,EAAE,GAAG,CAAC,CAAC;AACzF,MAAM,aAAa,GAAG,OAAO,CAAC;AAE9B,iDAAiD;AACjD,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,QAAgB,EAAU,EAAE;IAC3D,MAAM,GAAG,GAAG,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC;IACxD,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;AACxE,CAAC,CAAC;AAEF,MAAM,OAAO,GAAG,CAAC,IAAY,EAAE,QAAc,EAAE,IAAU,EAAW,EAAE,CACpE,QAAQ,CAAC,GAAG,KAAK,IAAI,CAAC,KAAK,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;AAE5F,sDAAsD;AACtD,MAAM,UAAU,GAAG,CAAC,IAAY,EAAE,QAAc,EAAE,IAAU,EAAsB,EAAE;IAClF,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACnG,IAAI,GAAG,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC;IAClC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IAC5C,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE;QAAE,OAAO,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC1E,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IACjF,MAAM,IAAI,GAAG,QAAQ,GAAG,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC;IACpE,OAAO;QACL,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE;QACxC,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE;KAC/B,CAAC;AACJ,CAAC,CAAC;AAEF,wDAAwD;AACxD,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,IAAY,EAAE,KAAsB,EAAU,EAAE,CACpF,KAAK,CAAC,MAAM,CAAS,CAAC,SAAS,EAAE,IAAI,EAAE,EAAE;IACvC,MAAM,QAAQ,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAClC,MAAM,QAAQ,GAAG,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;IACvF,IAAI,QAAQ,KAAK,SAAS;QAAE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;;QAC5C,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,QAAQ,CAAC,CAAC;IAC1C,OAAO,SAAS,CAAC;AACnB,CAAC,EAAE,EAAE,CAAC,CAAC"}
@@ -0,0 +1,8 @@
1
+ export declare const isEnglishRun: (text: string) => boolean;
2
+ /**
3
+ * 分割器が切った二つの断片の間が、英文と英文の切れ目か。
4
+ * 後ろが和文なら切れ目と読まない。「Version 2.0. を使う」「See Fig. 1. これは図」の英字の後のピリオドと見分けられないため。
5
+ * before は前の断片をつないだもので長くなりうるので、短い after を先に見る。
6
+ */
7
+ export declare const isEnglishBoundary: (before: string, after: string) => boolean;
8
+ //# sourceMappingURL=english-run.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"english-run.d.ts","sourceRoot":"","sources":["../src/english-run.ts"],"names":[],"mappings":"AAYA,eAAO,MAAM,YAAY,GAAI,MAAM,MAAM,KAAG,OAI3C,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,iBAAiB,GAAI,QAAQ,MAAM,EAAE,OAAO,MAAM,KAAG,OAC6C,CAAC"}
@@ -0,0 +1,24 @@
1
+ /**
2
+ * 和文の中の英文(英語の要旨、英語の文献)。仮名を含まず、文字(数字と記号は数えない)の半分以上が英字。
3
+ * 数字を数えると「In 1997.」のような年の入った短い英文が英文でなくなる。
4
+ * 漢字だけなら含んでよい(英文の中の「東京」)。仮名があれば和文の一部と読む。
5
+ */
6
+ const KANA = /[\p{Script=Hiragana}\p{Script=Katakana}]/u;
7
+ const LATIN = /\p{Script=Latin}/gu;
8
+ const LETTER = /\p{L}/gu;
9
+ // 英文の文末。閉じ括弧・引用符が続いてよい。断片の末尾だけを見る。
10
+ const ENGLISH_STOP_AT_END = /[.?!][)\]"'”’]{0,3}$/u;
11
+ const TAIL_LENGTH = 4;
12
+ export const isEnglishRun = (text) => {
13
+ if (KANA.test(text))
14
+ return false;
15
+ const latin = [...text.matchAll(LATIN)].length;
16
+ return latin > 0 && latin * 2 >= [...text.matchAll(LETTER)].length;
17
+ };
18
+ /**
19
+ * 分割器が切った二つの断片の間が、英文と英文の切れ目か。
20
+ * 後ろが和文なら切れ目と読まない。「Version 2.0. を使う」「See Fig. 1. これは図」の英字の後のピリオドと見分けられないため。
21
+ * before は前の断片をつないだもので長くなりうるので、短い after を先に見る。
22
+ */
23
+ export const isEnglishBoundary = (before, after) => ENGLISH_STOP_AT_END.test(before.trimEnd().slice(-TAIL_LENGTH)) && isEnglishRun(after) && isEnglishRun(before);
24
+ //# sourceMappingURL=english-run.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"english-run.js","sourceRoot":"","sources":["../src/english-run.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,MAAM,IAAI,GAAG,2CAA2C,CAAC;AACzD,MAAM,KAAK,GAAG,oBAAoB,CAAC;AACnC,MAAM,MAAM,GAAG,SAAS,CAAC;AACzB,mCAAmC;AACnC,MAAM,mBAAmB,GAAG,uBAAuB,CAAC;AACpD,MAAM,WAAW,GAAG,CAAC,CAAC;AAEtB,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,IAAY,EAAW,EAAE;IACpD,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAClC,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC;IAC/C,OAAO,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;AACrE,CAAC,CAAC;AAEF;;;;GAIG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,MAAc,EAAE,KAAa,EAAW,EAAE,CAC1E,mBAAmB,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC,CAAC,IAAI,YAAY,CAAC,KAAK,CAAC,IAAI,YAAY,CAAC,MAAM,CAAC,CAAC"}
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAgB,eAAe,EAAgC,MAAM,gBAAgB,CAAC;AA0DlG,eAAO,MAAM,OAAO,EAAE,eA0BrB,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAeA,OAAO,KAAK,EAAkC,eAAe,EAAgC,MAAM,gBAAgB,CAAC;AAyFpH,eAAO,MAAM,OAAO,EAAE,eA4BrB,CAAC"}
package/dist/index.js CHANGED
@@ -1,13 +1,23 @@
1
1
  import { loadLexicons } from "./lexicons.js";
2
+ import { closesSentence } from "./sentence-close.js";
2
3
  import { sentenceSpans } from "./sentence-split.js";
4
+ import { isEnglishBoundary, isEnglishRun } from "./english-run.js";
5
+ import { unmarkNumberStops } from "./number-stop.js";
6
+ import { labelStops, unmarkLabelStops } from "./label-stop.js";
7
+ import { splitAtQuotedStops } from "./quoted-stop.js";
8
+ import { reattachClosingQuotes } from "./closing-quote.js";
9
+ import { insideBrackets, startsWithCloser } from "./inside-brackets.js";
3
10
  import { structure } from "./structure.js";
4
11
  import { isReady, predicateOnly, prepare, readsAsCounter, readsAsOneAdverb, readsAsOneWord, tokenize } from "./pos.js";
5
12
  import { markSpacedCounters } from "./spaced-counter.js";
6
13
  import { tokensWithin } from "./tokens-within.js";
7
14
  import { distributiveVocabulary, iterationMarkReading, markReduplication } from "./reduplication.js";
15
+ import { knownWordReading, markDroppedLongVowels, remembered } from "./long-vowel-form.js";
8
16
  // chaff からは型だけを取る。実行時の値依存を作らない。アダプタは単体で動く。
17
+ const JAPANESE = /[぀-ゟ゠-ヿ一-鿿]/gu;
18
+ const COUNTABLE = /\S/gu;
9
19
  /**
10
- * 日本語の文末は「。!?」に限られる。
20
+ * 日本語の文末は「。!?」と、語の後の「.」に限られる(closesSentence)。
11
21
  *
12
22
  * sentence-splitter は "." も文末と見なすため、「、Dr. 田中は」で誤分割する。
13
23
  * AbbrMarker の language を差し替えても直らない。原因は略語の保護ではなく、
@@ -16,28 +26,37 @@ import { distributiveVocabulary, iterationMarkReading, markReduplication } from
16
26
  * 文長は sentence-rhythm と max-sentence-length の入力なので、
17
27
  * 誤分割はそのまま指標を動かす。ここで閉じる。
18
28
  */
19
- const CLOSED = /[。!?!?]["))」』]*\s*$/u;
20
- const JAPANESE = /[぀-ゟ゠-ヿ一-鿿]/gu;
21
- const COUNTABLE = /\S/gu;
22
- const isOpen = (text) => text.trim().length > 0 && !CLOSED.test(text);
29
+ const isOpen = (text) => text.trim().length > 0 && !closesSentence(text);
30
+ /**
31
+ * 和文の句点で閉じていない断片、括弧・二重引用符の中で切れた断片、閉じ括弧で始まる断片の前は次へ続く。
32
+ * 閉じ括弧は閉じる文に付き、後ろは分割器が組にした括弧(「…。」と言った)と同じく同じ文に続く。
33
+ * ただし英文どうしは、英語のアダプタと同じく英語の文末で切ったところで切る。
34
+ */
35
+ const continues = (before, after, bracketOpen) => (isOpen(before) || bracketOpen || startsWithCloser(after)) && !isEnglishBoundary(before, after);
36
+ const LINE_BREAK = /[\n\r]/u;
37
+ const ENGLISH = { id: "en", lengthUnit: "word" };
38
+ const withLanguage = (text, span) => (isEnglishRun(text) ? { span, text, embeddedLanguage: ENGLISH } : { span, text });
23
39
  /**
24
40
  * 断片を繋ぐときは raw の連結ではなくオフセットを使う。
25
41
  * sentence-splitter は空白を別ノードに分けるため、raw を繋ぐと空白が落ちて文長が縮む。
26
42
  */
27
- const merge = (source, spans) => spans
28
- .reduce((acc, span) => {
29
- const last = acc.at(-1);
30
- // 断片の「間」に改行があれば繋がない。結合は「Dr. 田中」のような行の途中の
31
- // 誤分割を閉じるためのもので、行またぎは要らない。またぐと、引用ブロックの
32
- // 英文と訳文のように別の行のものまで 1 文に繋がる。
33
- const acrossLines = last !== undefined && source.slice(last.end, span.start).includes("\n");
34
- if (last !== undefined && !acrossLines && isOpen(source.slice(last.start, last.end)))
35
- acc[acc.length - 1] = { start: last.start, end: span.end };
36
- else
37
- acc.push(span);
38
- return acc;
39
- }, [])
40
- .map((span) => ({ span, text: source.slice(span.start, span.end) }));
43
+ const merge = (source, spans) => {
44
+ const open = insideBrackets(source);
45
+ return spans
46
+ .reduce((acc, span) => {
47
+ const last = acc.at(-1);
48
+ // 断片の「間」に改行があれば繋がない。結合は「Dr. 田中」のような行の途中の
49
+ // 誤分割を閉じるためのもので、行またぎは要らない。またぐと、引用ブロックの
50
+ // 英文と訳文のように別の行のものまで 1 文に繋がる。
51
+ const acrossLines = last !== undefined && LINE_BREAK.test(source.slice(last.end, span.start));
52
+ if (last !== undefined && !acrossLines && continues(source.slice(last.start, last.end), source.slice(span.start, span.end), open[last.end] === true))
53
+ acc[acc.length - 1] = { start: last.start, end: span.end };
54
+ else
55
+ acc.push(span);
56
+ return acc;
57
+ }, [])
58
+ .map((span) => withLanguage(source.slice(span.start, span.end), span));
59
+ };
41
60
  /**
42
61
  * token の span は文ではなく、segment に渡した文字列を基準にする。文の span と同じ座標系。
43
62
  * 文ごとに解析して足し戻すのではなく、一度解析して文へ配る。同じ文字列を二度読まない。
@@ -45,17 +64,31 @@ const merge = (source, spans) => spans
45
64
  const LEXICONS = loadLexicons();
46
65
  const DISTRIBUTIVE = distributiveVocabulary(LEXICONS);
47
66
  const TAKES_ITERATION_MARK = iterationMarkReading(LEXICONS, readsAsOneWord);
67
+ const KNOWS_WITH_LONG_VOWEL = remembered(knownWordReading(tokenize));
68
+ /** Whether a running rule reads LongVowelEnding. Asking the dictionary about every katakana word costs, so only then. */
69
+ const askForLongVowels = { value: false };
48
70
  const withTokens = (source, sentences) => {
49
71
  const read = tokenize(source);
50
72
  if (read === undefined)
51
73
  return [...sentences];
52
- const tokens = markSpacedCounters(read, readsAsCounter);
74
+ const counted = markSpacedCounters(read, readsAsCounter);
75
+ const tokens = askForLongVowels.value ? markDroppedLongVowels(counted, KNOWS_WITH_LONG_VOWEL) : counted;
53
76
  return sentences.map((sentence) => ({
54
77
  ...sentence,
55
78
  // 述語かどうかは文の中でしか決まらないので、文へ配ってから印を落とす。
56
79
  tokens: markReduplication(predicateOnly(tokensWithin(tokens, sentence.span)), DISTRIBUTIVE, readsAsOneAdverb, TAKES_ITERATION_MARK),
57
80
  }));
58
81
  };
82
+ const LABEL_STOPS = labelStops((LEXICONS["abbreviated-label"] ?? []).map((entry) => entry.pattern));
83
+ /**
84
+ * 英語のアダプタと同じ手当てを英文にも当てる。番号の前の略した名前(FIG. 1)の点で切らず、
85
+ * 英文の閉じ引用符の内側の句点で切り、文頭に取り残された閉じ引用符を前の文へ戻す。
86
+ */
87
+ const spansOf = (text) => {
88
+ const spans = sentenceSpans(unmarkNumberStops(unmarkLabelStops(text, LABEL_STOPS)));
89
+ const quotedStops = spans.flatMap((span) => (isEnglishRun(text.slice(span.start, span.end)) ? splitAtQuotedStops(text, span) : [span]));
90
+ return reattachClosingQuotes(text, quotedStops);
91
+ };
59
92
  export const adapter = {
60
93
  kind: "language",
61
94
  id: "ja",
@@ -69,6 +102,8 @@ export const adapter = {
69
102
  lengthUnit: "char",
70
103
  },
71
104
  prepare: async (need) => {
105
+ // Once asked, keep marking: files are prepared in parallel, and a later prepare must not unmark an earlier file's run.
106
+ askForLongVowels.value ||= need.features?.includes("LongVowelEnding") ?? false;
72
107
  if (need.pos)
73
108
  await prepare();
74
109
  },
@@ -81,7 +116,7 @@ export const adapter = {
81
116
  lexicons: loadLexicons(),
82
117
  structure,
83
118
  segment: (text) => {
84
- const sentences = merge(text, sentenceSpans(text));
119
+ const sentences = merge(text, spansOf(text));
85
120
  return { sentences: isReady() ? withTokens(text, sentences) : sentences };
86
121
  },
87
122
  };
package/dist/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AACvH,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,sBAAsB,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAGrG,2CAA2C;AAE3C;;;;;;;;;GASG;AACH,MAAM,MAAM,GAAG,sBAAsB,CAAC;AAEtC,MAAM,QAAQ,GAAG,eAAe,CAAC;AACjC,MAAM,SAAS,GAAG,MAAM,CAAC;AAEzB,MAAM,MAAM,GAAG,CAAC,IAAY,EAAW,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEvF;;;GAGG;AACH,MAAM,KAAK,GAAG,CAAC,MAAc,EAAE,KAAsB,EAAc,EAAE,CACnE,KAAK;KACF,MAAM,CAAS,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;IAC5B,MAAM,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACxB,yCAAyC;IACzC,uCAAuC;IACvC,6BAA6B;IAC7B,MAAM,WAAW,GAAG,IAAI,KAAK,SAAS,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC5F,IAAI,IAAI,KAAK,SAAS,IAAI,CAAC,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;QAAE,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC;;QAC5I,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpB,OAAO,GAAG,CAAC;AACb,CAAC,EAAE,EAAE,CAAC;KACL,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;AAEzE;;;GAGG;AACH,MAAM,QAAQ,GAAG,YAAY,EAAE,CAAC;AAChC,MAAM,YAAY,GAAG,sBAAsB,CAAC,QAAQ,CAAC,CAAC;AACtD,MAAM,oBAAoB,GAAG,oBAAoB,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;AAE5E,MAAM,UAAU,GAAG,CAAC,MAAc,EAAE,SAA8B,EAAc,EAAE;IAChF,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC9B,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,CAAC,GAAG,SAAS,CAAC,CAAC;IAC9C,MAAM,MAAM,GAAG,kBAAkB,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IACxD,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAClC,GAAG,QAAQ;QACX,qCAAqC;QACrC,MAAM,EAAE,iBAAiB,CAAC,aAAa,CAAC,YAAY,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,YAAY,EAAE,gBAAgB,EAAE,oBAAoB,CAAC;KACpI,CAAC,CAAC,CAAC;AACN,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,OAAO,GAAoB;IACtC,IAAI,EAAE,UAAU;IAChB,EAAE,EAAE,IAAI;IACR,UAAU,EAAE,CAAC;IACb,YAAY,EAAE;QACZ,aAAa,EAAE,IAAI;QACnB,gDAAgD;QAChD,SAAS,EAAE,IAAI;QACf,GAAG,EAAE,IAAI;QACT,KAAK,EAAE,IAAI;QACX,UAAU,EAAE,MAAM;KACnB;IACD,OAAO,EAAE,KAAK,EAAE,IAAkB,EAAiB,EAAE;QACnD,IAAI,IAAI,CAAC,GAAG;YAAE,MAAM,OAAO,EAAE,CAAC;IAChC,CAAC;IACD,MAAM,EAAE,CAAC,MAAc,EAAU,EAAE;QACjC,MAAM,KAAK,GAAG,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC;QACrD,IAAI,KAAK,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAC1B,OAAO,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC;IACvD,CAAC;IACD,QAAQ,EAAE,YAAY,EAAE;IACxB,SAAS;IACT,OAAO,EAAE,CAAC,IAAY,EAAgB,EAAE;QACtC,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC;QACnD,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;IAC5E,CAAC;CACF,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AACnE,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAC/D,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACxE,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AACvH,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,sBAAsB,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACrG,OAAO,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAG3F,2CAA2C;AAE3C,MAAM,QAAQ,GAAG,eAAe,CAAC;AACjC,MAAM,SAAS,GAAG,MAAM,CAAC;AAEzB;;;;;;;;;GASG;AACH,MAAM,MAAM,GAAG,CAAC,IAAY,EAAW,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;AAE1F;;;;GAIG;AACH,MAAM,SAAS,GAAG,CAAC,MAAc,EAAE,KAAa,EAAE,WAAoB,EAAW,EAAE,CACjF,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,WAAW,IAAI,gBAAgB,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;AAElG,MAAM,UAAU,GAAG,SAAS,CAAC;AAE7B,MAAM,OAAO,GAAqB,EAAE,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC;AAEnE,MAAM,YAAY,GAAG,CAAC,IAAY,EAAE,IAAU,EAAY,EAAE,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;AAE/I;;;GAGG;AACH,MAAM,KAAK,GAAG,CAAC,MAAc,EAAE,KAAsB,EAAc,EAAE;IACnE,MAAM,IAAI,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;IACpC,OAAO,KAAK;SACT,MAAM,CAAS,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;QAC5B,MAAM,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACxB,yCAAyC;QACzC,uCAAuC;QACvC,6BAA6B;QAC7B,MAAM,WAAW,GAAG,IAAI,KAAK,SAAS,IAAI,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QAC9F,IAAI,IAAI,KAAK,SAAS,IAAI,CAAC,WAAW,IAAI,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC;YAClJ,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC;;YACxD,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpB,OAAO,GAAG,CAAC;IACb,CAAC,EAAE,EAAE,CAAC;SACL,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;AAC3E,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,QAAQ,GAAG,YAAY,EAAE,CAAC;AAChC,MAAM,YAAY,GAAG,sBAAsB,CAAC,QAAQ,CAAC,CAAC;AACtD,MAAM,oBAAoB,GAAG,oBAAoB,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;AAC5E,MAAM,qBAAqB,GAAG,UAAU,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC,CAAC;AACrE,yHAAyH;AACzH,MAAM,gBAAgB,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;AAE1C,MAAM,UAAU,GAAG,CAAC,MAAc,EAAE,SAA8B,EAAc,EAAE;IAChF,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC9B,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,CAAC,GAAG,SAAS,CAAC,CAAC;IAC9C,MAAM,OAAO,GAAG,kBAAkB,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IACzD,MAAM,MAAM,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,qBAAqB,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IACxG,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAClC,GAAG,QAAQ;QACX,qCAAqC;QACrC,MAAM,EAAE,iBAAiB,CAAC,aAAa,CAAC,YAAY,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,YAAY,EAAE,gBAAgB,EAAE,oBAAoB,CAAC;KACpI,CAAC,CAAC,CAAC;AACN,CAAC,CAAC;AAEF,MAAM,WAAW,GAAG,UAAU,CAAC,CAAC,QAAQ,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;AAEpG;;;GAGG;AACH,MAAM,OAAO,GAAG,CAAC,IAAY,EAAU,EAAE;IACvC,MAAM,KAAK,GAAG,aAAa,CAAC,iBAAiB,CAAC,gBAAgB,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC;IACpF,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACxI,OAAO,qBAAqB,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;AAClD,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,OAAO,GAAoB;IACtC,IAAI,EAAE,UAAU;IAChB,EAAE,EAAE,IAAI;IACR,UAAU,EAAE,CAAC;IACb,YAAY,EAAE;QACZ,aAAa,EAAE,IAAI;QACnB,gDAAgD;QAChD,SAAS,EAAE,IAAI;QACf,GAAG,EAAE,IAAI;QACT,KAAK,EAAE,IAAI;QACX,UAAU,EAAE,MAAM;KACnB;IACD,OAAO,EAAE,KAAK,EAAE,IAAkB,EAAiB,EAAE;QACnD,uHAAuH;QACvH,gBAAgB,CAAC,KAAK,KAAK,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC;QAC/E,IAAI,IAAI,CAAC,GAAG;YAAE,MAAM,OAAO,EAAE,CAAC;IAChC,CAAC;IACD,MAAM,EAAE,CAAC,MAAc,EAAU,EAAE;QACjC,MAAM,KAAK,GAAG,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC;QACrD,IAAI,KAAK,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAC1B,OAAO,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC;IACvD,CAAC;IACD,QAAQ,EAAE,YAAY,EAAE;IACxB,SAAS;IACT,OAAO,EAAE,CAAC,IAAY,EAAgB,EAAE;QACtC,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;QAC7C,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;IAC5E,CAAC;CACF,CAAC"}
@@ -0,0 +1,5 @@
1
+ /** 閉じ括弧・閉じ引用符で始まるか。前の行で開いた括弧や数え損ねた括弧の閉じが、句点の後で文頭に残ったもの。 */
2
+ export declare const startsWithCloser: (text: string) => boolean;
3
+ /** 各位置(0 から text.length まで)の手前に、同じ行で後から閉じる括弧・二重引用符が開いているか。 */
4
+ export declare const insideBrackets: (text: string) => boolean[];
5
+ //# sourceMappingURL=inside-brackets.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"inside-brackets.d.ts","sourceRoot":"","sources":["../src/inside-brackets.ts"],"names":[],"mappings":"AA6BA,2DAA2D;AAC3D,eAAO,MAAM,gBAAgB,GAAI,MAAM,MAAM,KAAG,OAAkD,CAAC;AAEnG,8DAA8D;AAC9D,eAAO,MAAM,cAAc,GAAI,MAAM,MAAM,KAAG,OAAO,EAWpD,CAAC"}
@@ -0,0 +1,44 @@
1
+ /**
2
+ * 括弧・二重引用符の中の句点は文を閉じない。sentence-splitter は括弧を種類ごとに一つしか数えないので、
3
+ * 同じ種類が入れ子になると内側の閉じ括弧で外側まで閉じたと読み(「(…(…)…。以下同じ。)に」)、曲がった二重引用符(“…”)は数えない。
4
+ * ここでは開きと閉じを入れ子のまま組にする。一重引用符(‘ ’)はアポストロフィと見分けられず、直線の引用符は分割器が数えるので、どちらも数えない。
5
+ */
6
+ const OPENER = /[\p{Ps}“«]/u;
7
+ const CLOSER = /[\p{Pe}”»]/u;
8
+ const LINE_BREAK = /[\n\r]/u;
9
+ /**
10
+ * 同じ行の中で組になった開きと閉じの位置。組にならないもの(「1)」の番号、同上の印の「“」、閉じない括弧)は含めない。
11
+ * 閉じない括弧を数えると、行の終わりまでの文が全部つながる。
12
+ */
13
+ const pairsOf = (text) => {
14
+ const pairs = [];
15
+ const opened = [];
16
+ text.split("").forEach((unit, index) => {
17
+ if (LINE_BREAK.test(unit))
18
+ opened.length = 0;
19
+ else if (OPENER.test(unit))
20
+ opened.push(index);
21
+ else if (CLOSER.test(unit)) {
22
+ const open = opened.pop();
23
+ if (open !== undefined)
24
+ pairs.push({ open, close: index });
25
+ }
26
+ });
27
+ return pairs;
28
+ };
29
+ /** 閉じ括弧・閉じ引用符で始まるか。前の行で開いた括弧や数え損ねた括弧の閉じが、句点の後で文頭に残ったもの。 */
30
+ export const startsWithCloser = (text) => CLOSER.test(text.trimStart().charAt(0));
31
+ /** 各位置(0 から text.length まで)の手前に、同じ行で後から閉じる括弧・二重引用符が開いているか。 */
32
+ export const insideBrackets = (text) => {
33
+ const change = Array.from({ length: text.length + 2 }, () => 0);
34
+ pairsOf(text).forEach((pair) => {
35
+ change[pair.open + 1] = (change[pair.open + 1] ?? 0) + 1;
36
+ change[pair.close + 1] = (change[pair.close + 1] ?? 0) - 1;
37
+ });
38
+ const depth = { value: 0 };
39
+ return change.slice(0, text.length + 1).map((step) => {
40
+ depth.value += step;
41
+ return depth.value > 0;
42
+ });
43
+ };
44
+ //# sourceMappingURL=inside-brackets.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"inside-brackets.js","sourceRoot":"","sources":["../src/inside-brackets.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,MAAM,MAAM,GAAG,aAAa,CAAC;AAC7B,MAAM,MAAM,GAAG,aAAa,CAAC;AAC7B,MAAM,UAAU,GAAG,SAAS,CAAC;AAI7B;;;GAGG;AACH,MAAM,OAAO,GAAG,CAAC,IAAY,EAAU,EAAE;IACvC,MAAM,KAAK,GAAW,EAAE,CAAC;IACzB,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QACrC,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;aACxC,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;aAC1C,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,EAAE,CAAC;YAC1B,IAAI,IAAI,KAAK,SAAS;gBAAE,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;QAC7D,CAAC;IACH,CAAC,CAAC,CAAC;IACH,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF,2DAA2D;AAC3D,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,IAAY,EAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AAEnG,8DAA8D;AAC9D,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,IAAY,EAAa,EAAE;IACxD,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;IAChE,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;QAC7B,MAAM,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QACzD,MAAM,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IAC7D,CAAC,CAAC,CAAC;IACH,MAAM,KAAK,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;IAC3B,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACnD,KAAK,CAAC,KAAK,IAAI,IAAI,CAAC;QACpB,OAAO,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC;IACzB,CAAC,CAAC,CAAC;AACL,CAAC,CAAC"}
@@ -0,0 +1,13 @@
1
+ /**
2
+ * sentence-splitter ends a sentence at the full stop of a label before its number: "FIG." and "1 illustrates …" were two
3
+ * sentences, and so were "Vol." and "XLIII (1979)". Where a number follows, that full stop is replaced with a letter
4
+ * before splitting. The length does not change, so the spans fit the original text.
5
+ * A lone "I" after a label is the pronoun ("He said No. I left."), and a word ("the last Fig. The tree") is not a number.
6
+ */
7
+ /** Built once from the lexicon; undefined when no label ends in a full stop. */
8
+ export type LabelStops = RegExp | undefined;
9
+ /** The labels as written and in capitals, without their full stop ("Fig." → Fig, FIG). */
10
+ export declare const labelStops: (labels: readonly string[]) => LabelStops;
11
+ /** The text with the full stop of every label before a number replaced by a letter. */
12
+ export declare const unmarkLabelStops: (text: string, stops: LabelStops) => string;
13
+ //# sourceMappingURL=label-stop.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"label-stop.d.ts","sourceRoot":"","sources":["../src/label-stop.ts"],"names":[],"mappings":"AAGA;;;;;GAKG;AAEH,gFAAgF;AAChF,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,SAAS,CAAC;AAK5C,0FAA0F;AAC1F,eAAO,MAAM,UAAU,GAAI,QAAQ,SAAS,MAAM,EAAE,KAAG,UAItD,CAAC;AAEF,uFAAuF;AACvF,eAAO,MAAM,gBAAgB,GAAI,MAAM,MAAM,EAAE,OAAO,UAAU,KAAG,MAC0C,CAAC"}
@@ -0,0 +1,14 @@
1
+ // lang-en/src/label-stop.ts の写し。アダプタは互いに依存しないので、和文の中の英文に同じ手当てをするために置く。元とのずれは test_mixed_language_stops.ts で比べる。
2
+ import { escapeRegExp } from "./regexp.js";
3
+ const NUMBER_AFTER = String.raw `(?=\s+(?:\d|(?:[IVXLCDM]{2,}|[VXLCDM])(?![\p{L}\p{N}_])))`;
4
+ const PLAIN_LETTER = "n";
5
+ /** The labels as written and in capitals, without their full stop ("Fig." → Fig, FIG). */
6
+ export const labelStops = (labels) => {
7
+ const stems = labels.filter((label) => label.endsWith(".")).flatMap((label) => [label.slice(0, -1), label.slice(0, -1).toUpperCase()]);
8
+ if (stems.length === 0)
9
+ return undefined;
10
+ return new RegExp(String.raw `(?<![\p{L}\p{N}_.])(?:${[...new Set(stems)].map(escapeRegExp).join("|")})\.${NUMBER_AFTER}`, "gu");
11
+ };
12
+ /** The text with the full stop of every label before a number replaced by a letter. */
13
+ export const unmarkLabelStops = (text, stops) => stops === undefined ? text : text.replace(stops, (label) => `${label.slice(0, -1)}${PLAIN_LETTER}`);
14
+ //# sourceMappingURL=label-stop.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"label-stop.js","sourceRoot":"","sources":["../src/label-stop.ts"],"names":[],"mappings":"AAAA,gHAAgH;AAChH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAY3C,MAAM,YAAY,GAAG,MAAM,CAAC,GAAG,CAAA,2DAA2D,CAAC;AAC3F,MAAM,YAAY,GAAG,GAAG,CAAC;AAEzB,0FAA0F;AAC1F,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,MAAyB,EAAc,EAAE;IAClE,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;IACvI,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IACzC,OAAO,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAA,yBAAyB,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,YAAY,EAAE,EAAE,IAAI,CAAC,CAAC;AAClI,CAAC,CAAC;AAEF,uFAAuF;AACvF,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,IAAY,EAAE,KAAiB,EAAU,EAAE,CAC1E,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,KAAa,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,YAAY,EAAE,CAAC,CAAC"}
@@ -1 +1 @@
1
- {"version":3,"file":"lexicons.d.ts","sourceRoot":"","sources":["../src/lexicons.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,OAAO,EAAgB,MAAM,gBAAgB,CAAC;AAoB5D;;;GAGG;AACH,eAAO,MAAM,YAAY,GAAI,MAAK,MAAY,KAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CASpE,CAAC"}
1
+ {"version":3,"file":"lexicons.d.ts","sourceRoot":"","sources":["../src/lexicons.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,OAAO,EAAgB,MAAM,gBAAgB,CAAC;AAwB5D;;;GAGG;AACH,eAAO,MAAM,YAAY,GAAI,MAAK,MAAY,KAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CASpE,CAAC"}
package/dist/lexicons.js CHANGED
@@ -10,11 +10,15 @@ const toEntry = (raw) => {
10
10
  return undefined;
11
11
  const weight = raw["weight"];
12
12
  const instead = raw["instead_of"];
13
+ const rewrite = raw["rewrite"];
14
+ const group = raw["group"];
13
15
  return {
14
16
  pattern: raw["pattern"],
15
17
  weight: typeof weight === "number" ? weight : undefined,
16
18
  instead_of: typeof instead === "string" ? instead : undefined,
17
19
  position: POSITIONS.find((position) => position === raw["position"]),
20
+ rewrite: typeof rewrite === "string" ? rewrite : undefined,
21
+ ...(typeof group === "string" ? { group } : {}),
18
22
  };
19
23
  };
20
24
  /**
@@ -1 +1 @@
1
- {"version":3,"file":"lexicons.js","sourceRoot":"","sources":["../src/lexicons.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,KAAK,EAAE,MAAM,MAAM,CAAC;AAG7B,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;AAE5E,MAAM,QAAQ,GAAG,CAAC,KAAc,EAAoC,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAE5I,MAAM,SAAS,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAU,CAAC;AAE/C,MAAM,OAAO,GAAG,CAAC,GAAY,EAA4B,EAAE;IACzD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,OAAO,GAAG,CAAC,SAAS,CAAC,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC;IAC3E,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC7B,MAAM,OAAO,GAAG,GAAG,CAAC,YAAY,CAAC,CAAC;IAClC,OAAO;QACL,OAAO,EAAE,GAAG,CAAC,SAAS,CAAC;QACvB,MAAM,EAAE,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;QACvD,UAAU,EAAE,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;QAC7D,QAAQ,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,KAAK,GAAG,CAAC,UAAU,CAAC,CAAC;KACrE,CAAC;AACJ,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,MAAc,GAAG,EAA2B,EAAE,CACzE,MAAM,CAAC,WAAW,CAChB,WAAW,CAAC,GAAG,CAAC;KACb,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;KACxC,OAAO,CAAC,CAAC,IAAI,EAAuB,EAAE;IACrC,MAAM,GAAG,GAAY,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;IAClE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,OAAO,GAAG,CAAC,IAAI,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAAE,OAAO,EAAE,CAAC;IACjG,OAAO,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC;AAC3F,CAAC,CAAC,CACL,CAAC"}
1
+ {"version":3,"file":"lexicons.js","sourceRoot":"","sources":["../src/lexicons.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,KAAK,EAAE,MAAM,MAAM,CAAC;AAG7B,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;AAE5E,MAAM,QAAQ,GAAG,CAAC,KAAc,EAAoC,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAE5I,MAAM,SAAS,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAU,CAAC;AAE/C,MAAM,OAAO,GAAG,CAAC,GAAY,EAA4B,EAAE;IACzD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,OAAO,GAAG,CAAC,SAAS,CAAC,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC;IAC3E,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC7B,MAAM,OAAO,GAAG,GAAG,CAAC,YAAY,CAAC,CAAC;IAClC,MAAM,OAAO,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC;IAC/B,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC;IAC3B,OAAO;QACL,OAAO,EAAE,GAAG,CAAC,SAAS,CAAC;QACvB,MAAM,EAAE,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;QACvD,UAAU,EAAE,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;QAC7D,QAAQ,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,KAAK,GAAG,CAAC,UAAU,CAAC,CAAC;QACpE,OAAO,EAAE,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;QAC1D,GAAG,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAChD,CAAC;AACJ,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,MAAc,GAAG,EAA2B,EAAE,CACzE,MAAM,CAAC,WAAW,CAChB,WAAW,CAAC,GAAG,CAAC;KACb,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;KACxC,OAAO,CAAC,CAAC,IAAI,EAAuB,EAAE;IACrC,MAAM,GAAG,GAAY,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;IAClE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,OAAO,GAAG,CAAC,IAAI,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAAE,OAAO,EAAE,CAAC;IACjG,OAAO,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC;AAC3F,CAAC,CAAC,CACL,CAAC"}
@@ -0,0 +1,12 @@
1
+ import type { Token } from "chaffjs/plugin";
2
+ /**
3
+ * Marks a katakana noun whose form with a final ー is a word the dictionary knows (コンピュータ → コンピューター): LongVowelEnding=Dropped.
4
+ * Only the dictionary can tell a dropped long vowel (メモリ) from a word that never had one (データ, ハードウェア).
5
+ * knowsWithLongVowel is given by index.ts, which holds the analyser; each surface is asked once.
6
+ */
7
+ export declare const markDroppedLongVowels: (tokens: readonly Token[], knowsWithLongVowel: (surface: string) => boolean) => Token[];
8
+ /** Whether text reads as one word the dictionary knows (a lemma, not an unknown run of katakana). */
9
+ export declare const knownWordReading: (tokenize: (text: string) => readonly Token[] | undefined) => (surface: string) => boolean;
10
+ /** Remembers each answer: a document repeats its words, and the analyser is the cost. */
11
+ export declare const remembered: (ask: (surface: string) => boolean) => ((surface: string) => boolean);
12
+ //# sourceMappingURL=long-vowel-form.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"long-vowel-form.d.ts","sourceRoot":"","sources":["../src/long-vowel-form.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAS5C;;;;GAIG;AACH,eAAO,MAAM,qBAAqB,GAAI,QAAQ,SAAS,KAAK,EAAE,EAAE,oBAAoB,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,KAAG,KAAK,EAGrH,CAAC;AAEJ,qGAAqG;AACrG,eAAO,MAAM,gBAAgB,GAC1B,UAAU,CAAC,IAAI,EAAE,MAAM,KAAK,SAAS,KAAK,EAAE,GAAG,SAAS,MACxD,SAAS,MAAM,KAAG,OAGlB,CAAC;AAEJ,yFAAyF;AACzF,eAAO,MAAM,UAAU,GAAI,KAAK,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,KAAG,CAAC,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAS3F,CAAC"}
@@ -0,0 +1,28 @@
1
+ const KATAKANA_WORD = /^[ァ-ヺー]+$/u;
2
+ const LONG_VOWEL = "ー";
3
+ /** A katakana noun written without a final ー, which could be a loanword whose final long vowel was dropped (コンピュータ). */
4
+ const mayHaveDropped = (token) => token.pos === "NOUN" && KATAKANA_WORD.test(token.surface) && !token.surface.endsWith(LONG_VOWEL) && Array.from(token.surface).length > 1;
5
+ /**
6
+ * Marks a katakana noun whose form with a final ー is a word the dictionary knows (コンピュータ → コンピューター): LongVowelEnding=Dropped.
7
+ * Only the dictionary can tell a dropped long vowel (メモリ) from a word that never had one (データ, ハードウェア).
8
+ * knowsWithLongVowel is given by index.ts, which holds the analyser; each surface is asked once.
9
+ */
10
+ export const markDroppedLongVowels = (tokens, knowsWithLongVowel) => tokens.map((token) => mayHaveDropped(token) && knowsWithLongVowel(token.surface) ? { ...token, features: { ...token.features, LongVowelEnding: "Dropped" } } : token);
11
+ /** Whether text reads as one word the dictionary knows (a lemma, not an unknown run of katakana). */
12
+ export const knownWordReading = (tokenize) => (surface) => {
13
+ const read = tokenize(`${surface}${LONG_VOWEL}`);
14
+ return read?.length === 1 && read[0]?.lemma !== undefined && read[0].pos === "NOUN";
15
+ };
16
+ /** Remembers each answer: a document repeats its words, and the analyser is the cost. */
17
+ export const remembered = (ask) => {
18
+ const answers = new Map();
19
+ return (surface) => {
20
+ const known = answers.get(surface);
21
+ if (known !== undefined)
22
+ return known;
23
+ const answer = ask(surface);
24
+ answers.set(surface, answer);
25
+ return answer;
26
+ };
27
+ };
28
+ //# sourceMappingURL=long-vowel-form.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"long-vowel-form.js","sourceRoot":"","sources":["../src/long-vowel-form.ts"],"names":[],"mappings":"AAEA,MAAM,aAAa,GAAG,YAAY,CAAC;AACnC,MAAM,UAAU,GAAG,GAAG,CAAC;AAEvB,wHAAwH;AACxH,MAAM,cAAc,GAAG,CAAC,KAAY,EAAW,EAAE,CAC/C,KAAK,CAAC,GAAG,KAAK,MAAM,IAAI,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;AAE3I;;;;GAIG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,MAAwB,EAAE,kBAAgD,EAAW,EAAE,CAC3H,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CACnB,cAAc,CAAC,KAAK,CAAC,IAAI,kBAAkB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,QAAQ,EAAE,EAAE,GAAG,KAAK,CAAC,QAAQ,EAAE,eAAe,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAC/I,CAAC;AAEJ,qGAAqG;AACrG,MAAM,CAAC,MAAM,gBAAgB,GAC3B,CAAC,QAAwD,EAAE,EAAE,CAC7D,CAAC,OAAe,EAAW,EAAE;IAC3B,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,OAAO,GAAG,UAAU,EAAE,CAAC,CAAC;IACjD,OAAO,IAAI,EAAE,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,KAAK,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,MAAM,CAAC;AACtF,CAAC,CAAC;AAEJ,yFAAyF;AACzF,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,GAAiC,EAAkC,EAAE;IAC9F,MAAM,OAAO,GAAG,IAAI,GAAG,EAAmB,CAAC;IAC3C,OAAO,CAAC,OAAO,EAAE,EAAE;QACjB,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACnC,IAAI,KAAK,KAAK,SAAS;YAAE,OAAO,KAAK,CAAC;QACtC,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC;QAC5B,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC7B,OAAO,MAAM,CAAC;IAChB,CAAC,CAAC;AACJ,CAAC,CAAC"}
@@ -0,0 +1,3 @@
1
+ /** 文末の番号の数字を英字に替えた文字列。長さは変わらない。 */
2
+ export declare const unmarkNumberStops: (text: string) => string;
3
+ //# sourceMappingURL=number-stop.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"number-stop.d.ts","sourceRoot":"","sources":["../src/number-stop.ts"],"names":[],"mappings":"AA2BA,mCAAmC;AACnC,eAAO,MAAM,iBAAiB,GAAI,MAAM,MAAM,KAAG,MACiG,CAAC"}