@chaffjs/lang-ja 0.46.0 → 0.48.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -1
- package/dist/index.js.map +1 -1
- package/dist/initial-stop.d.ts +7 -4
- package/dist/initial-stop.d.ts.map +1 -1
- package/dist/initial-stop.js +16 -8
- package/dist/initial-stop.js.map +1 -1
- package/dist/mixed-width-bracket.d.ts.map +1 -1
- package/dist/mixed-width-bracket.js +5 -1
- package/dist/mixed-width-bracket.js.map +1 -1
- package/dist/run-in-heading.d.ts +7 -0
- package/dist/run-in-heading.d.ts.map +1 -0
- package/dist/run-in-heading.js +101 -0
- package/dist/run-in-heading.js.map +1 -0
- package/lexicons/address-particle.yaml +10 -0
- package/lexicons/closing-abbreviation.yaml +6 -0
- package/lexicons/name-wrong-mark.yaml +10 -0
- package/lexicons/ngram-fixed-ending.yaml +2 -0
- package/lexicons/orthographic-variant-two-readings.yaml +6 -0
- package/lexicons/orthographic-variant-whole-word.yaml +6 -0
- package/lexicons/particle-run-determiner.yaml +9 -0
- package/lexicons/particle-run-set-phrase.yaml +4 -0
- package/lexicons/product-name.yaml +9 -2
- package/lexicons/question-particle.yaml +1 -0
- package/lexicons/requirement-open-end-fixed-term.yaml +8 -0
- package/lexicons/run-in-heading-opener.yaml +15 -0
- package/lexicons/topic-adverb.yaml +8 -0
- package/package.json +1 -1
- package/src/index.ts +4 -0
- package/src/initial-stop.ts +25 -10
- package/src/mixed-width-bracket.ts +6 -1
- package/src/run-in-heading.ts +111 -0
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAqBA,OAAO,KAAK,EAAkC,eAAe,EAAgC,MAAM,gBAAgB,CAAC;AA2GpH,eAAO,MAAM,OAAO,EAAE,eA4BrB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -12,6 +12,7 @@ import { splitAtQuotedLineEnds } from "./quoted-line-stop.js";
|
|
|
12
12
|
import { insideBrackets, startsWithCloser } from "./inside-brackets.js";
|
|
13
13
|
import { structure } from "./structure.js";
|
|
14
14
|
import { cutAtBreaks, lineVocabulary, listItemHeads, listLineBreaks, startsListItem } from "./list-line.js";
|
|
15
|
+
import { cutRunInHeadings, headingOpeners } from "./run-in-heading.js";
|
|
15
16
|
import { listItemHeads as englishListItemHeads, listLineBreaks as englishListLineBreaks } from "./english-list-line.js";
|
|
16
17
|
import { isReady, predicateOnly, prepare, readsAsCounter, readsAsOneAdverb, readsAsOneWord, tokenize } from "./pos.js";
|
|
17
18
|
import { markSpacedCounters } from "./spaced-counter.js";
|
|
@@ -66,6 +67,7 @@ const merge = (source, spans) => {
|
|
|
66
67
|
return acc;
|
|
67
68
|
}, [])
|
|
68
69
|
.flatMap((span) => cutAtBreaks(span, listBreaks))
|
|
70
|
+
.flatMap((span) => (english ? [span] : cutRunInHeadings(source, span, HEADING_OPENERS)))
|
|
69
71
|
.map((span) => withLanguage(source.slice(span.start, span.end), span))
|
|
70
72
|
.map((sentence) => (startsListItem(itemHeads, sentence.span.start) ? asListItem(sentence) : sentence));
|
|
71
73
|
};
|
|
@@ -93,8 +95,9 @@ const withTokens = (source, sentences) => {
|
|
|
93
95
|
};
|
|
94
96
|
const patternsOf = (id) => (LEXICONS[id] ?? []).map((entry) => entry.pattern);
|
|
95
97
|
const LINE_VOCABULARY = lineVocabulary(patternsOf("numbering-counter"), patternsOf("line-head-tag"));
|
|
98
|
+
const HEADING_OPENERS = headingOpeners(patternsOf("run-in-heading-opener"));
|
|
96
99
|
const LABEL_STOPS = labelStops((LEXICONS["abbreviated-label"] ?? []).map((entry) => entry.pattern));
|
|
97
|
-
const INITIAL_STOPS = initialStops((LEXICONS["sentence-start-word"] ?? []).map((entry) => entry.pattern), (LEXICONS["prepositive-abbreviation"] ?? []).map((entry) => entry.pattern));
|
|
100
|
+
const INITIAL_STOPS = initialStops((LEXICONS["sentence-start-word"] ?? []).map((entry) => entry.pattern), (LEXICONS["prepositive-abbreviation"] ?? []).map((entry) => entry.pattern), (LEXICONS["closing-abbreviation"] ?? []).map((entry) => entry.pattern));
|
|
98
101
|
/**
|
|
99
102
|
* 英語のアダプタと同じ手当てを英文にも当てる。番号の前の略した名前(FIG. 1)の点で切らず、
|
|
100
103
|
* 英文の閉じ引用符の内側の句点で切り、文頭に取り残された閉じ引用符を前の文へ戻す。
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AACnE,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAC/D,OAAO,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AACrE,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACxE,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,aAAa,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAC5G,OAAO,EAAE,aAAa,IAAI,oBAAoB,EAAE,cAAc,IAAI,qBAAqB,EAAE,MAAM,wBAAwB,CAAC;AACxH,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AACvH,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,sBAAsB,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACrG,OAAO,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAG3F,2CAA2C;AAE3C,MAAM,QAAQ,GAAG,eAAe,CAAC;AACjC,MAAM,SAAS,GAAG,MAAM,CAAC;AAEzB;;;;;;;;;GASG;AACH,MAAM,MAAM,GAAG,CAAC,IAAY,EAAW,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;AAE1F;;;;GAIG;AACH,MAAM,SAAS,GAAG,CAAC,MAAc,EAAE,KAAa,EAAE,WAAoB,EAAW,EAAE,CACjF,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,WAAW,IAAI,gBAAgB,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;AAElG,MAAM,UAAU,GAAG,SAAS,CAAC;AAE7B,MAAM,OAAO,GAAqB,EAAE,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC;AAEnE,MAAM,UAAU,GAAG,CAAC,QAAkB,EAAY,EAAE,CAAC,CAAC,EAAE,GAAG,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;AAEvF,MAAM,YAAY,GAAG,CAAC,IAAY,EAAE,IAAU,EAAY,EAAE,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;AAE/I;;;GAGG;AACH,MAAM,KAAK,GAAG,CAAC,MAAc,EAAE,KAAsB,EAAc,EAAE;IACnE,MAAM,IAAI,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;IACpC,gEAAgE;IAChE,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;IACrC,MAAM,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC,qBAAqB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IACrG,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;IACjF,OAAO,KAAK;SACT,MAAM,CAAS,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;QAC5B,MAAM,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACxB,yCAAyC;QACzC,uCAAuC;QACvC,6BAA6B;QAC7B,MAAM,WAAW,GAAG,IAAI,KAAK,SAAS,IAAI,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QAC9F,IAAI,IAAI,KAAK,SAAS,IAAI,CAAC,WAAW,IAAI,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC;YAClJ,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC;;YACxD,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpB,OAAO,GAAG,CAAC;IACb,CAAC,EAAE,EAAE,CAAC;SACL,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;SAChD,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;SACrE,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,SAAS,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;AAC3G,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,QAAQ,GAAG,YAAY,EAAE,CAAC;AAChC,MAAM,YAAY,GAAG,sBAAsB,CAAC,QAAQ,CAAC,CAAC;AACtD,MAAM,oBAAoB,GAAG,oBAAoB,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;AAC5E,MAAM,qBAAqB,GAAG,UAAU,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC,CAAC;AACrE,yHAAyH;AACzH,MAAM,gBAAgB,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;AAE1C,MAAM,UAAU,GAAG,CAAC,MAAc,EAAE,SAA8B,EAAc,EAAE;IAChF,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC9B,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,CAAC,GAAG,SAAS,CAAC,CAAC;IAC9C,MAAM,OAAO,GAAG,kBAAkB,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IACzD,MAAM,MAAM,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,qBAAqB,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IACxG,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAClC,GAAG,QAAQ;QACX,qCAAqC;QACrC,MAAM,EAAE,iBAAiB,CAAC,aAAa,CAAC,YAAY,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,YAAY,EAAE,gBAAgB,EAAE,oBAAoB,CAAC;KACpI,CAAC,CAAC,CAAC;AACN,CAAC,CAAC;AAEF,MAAM,UAAU,GAAG,CAAC,EAAU,EAAY,EAAE,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;AAChG,MAAM,eAAe,GAAG,cAAc,CAAC,UAAU,CAAC,mBAAmB,CAAC,EAAE,UAAU,CAAC,eAAe,CAAC,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AACnE,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAC/D,OAAO,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AACrE,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACxE,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,aAAa,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAC5G,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACvE,OAAO,EAAE,aAAa,IAAI,oBAAoB,EAAE,cAAc,IAAI,qBAAqB,EAAE,MAAM,wBAAwB,CAAC;AACxH,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AACvH,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,sBAAsB,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACrG,OAAO,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAG3F,2CAA2C;AAE3C,MAAM,QAAQ,GAAG,eAAe,CAAC;AACjC,MAAM,SAAS,GAAG,MAAM,CAAC;AAEzB;;;;;;;;;GASG;AACH,MAAM,MAAM,GAAG,CAAC,IAAY,EAAW,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;AAE1F;;;;GAIG;AACH,MAAM,SAAS,GAAG,CAAC,MAAc,EAAE,KAAa,EAAE,WAAoB,EAAW,EAAE,CACjF,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,WAAW,IAAI,gBAAgB,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;AAElG,MAAM,UAAU,GAAG,SAAS,CAAC;AAE7B,MAAM,OAAO,GAAqB,EAAE,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC;AAEnE,MAAM,UAAU,GAAG,CAAC,QAAkB,EAAY,EAAE,CAAC,CAAC,EAAE,GAAG,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;AAEvF,MAAM,YAAY,GAAG,CAAC,IAAY,EAAE,IAAU,EAAY,EAAE,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;AAE/I;;;GAGG;AACH,MAAM,KAAK,GAAG,CAAC,MAAc,EAAE,KAAsB,EAAc,EAAE;IACnE,MAAM,IAAI,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;IACpC,gEAAgE;IAChE,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;IACrC,MAAM,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC,qBAAqB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IACrG,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;IACjF,OAAO,KAAK;SACT,MAAM,CAAS,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;QAC5B,MAAM,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACxB,yCAAyC;QACzC,uCAAuC;QACvC,6BAA6B;QAC7B,MAAM,WAAW,GAAG,IAAI,KAAK,SAAS,IAAI,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QAC9F,IAAI,IAAI,KAAK,SAAS,IAAI,CAAC,WAAW,IAAI,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC;YAClJ,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC;;YACxD,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpB,OAAO,GAAG,CAAC;IACb,CAAC,EAAE,EAAE,CAAC;SACL,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;SAChD,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,MAAM,EAAE,IAAI,EAAE,eAAe,CAAC,CAAC,CAAC;SACvF,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;SACrE,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,SAAS,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;AAC3G,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,QAAQ,GAAG,YAAY,EAAE,CAAC;AAChC,MAAM,YAAY,GAAG,sBAAsB,CAAC,QAAQ,CAAC,CAAC;AACtD,MAAM,oBAAoB,GAAG,oBAAoB,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;AAC5E,MAAM,qBAAqB,GAAG,UAAU,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC,CAAC;AACrE,yHAAyH;AACzH,MAAM,gBAAgB,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;AAE1C,MAAM,UAAU,GAAG,CAAC,MAAc,EAAE,SAA8B,EAAc,EAAE;IAChF,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC9B,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,CAAC,GAAG,SAAS,CAAC,CAAC;IAC9C,MAAM,OAAO,GAAG,kBAAkB,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IACzD,MAAM,MAAM,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,qBAAqB,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IACxG,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAClC,GAAG,QAAQ;QACX,qCAAqC;QACrC,MAAM,EAAE,iBAAiB,CAAC,aAAa,CAAC,YAAY,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,YAAY,EAAE,gBAAgB,EAAE,oBAAoB,CAAC;KACpI,CAAC,CAAC,CAAC;AACN,CAAC,CAAC;AAEF,MAAM,UAAU,GAAG,CAAC,EAAU,EAAY,EAAE,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;AAChG,MAAM,eAAe,GAAG,cAAc,CAAC,UAAU,CAAC,mBAAmB,CAAC,EAAE,UAAU,CAAC,eAAe,CAAC,CAAC,CAAC;AACrG,MAAM,eAAe,GAAG,cAAc,CAAC,UAAU,CAAC,uBAAuB,CAAC,CAAC,CAAC;AAE5E,MAAM,WAAW,GAAG,UAAU,CAAC,CAAC,QAAQ,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;AACpG,MAAM,aAAa,GAAG,YAAY,CAChC,CAAC,QAAQ,CAAC,qBAAqB,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,EACrE,CAAC,QAAQ,CAAC,0BAA0B,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,EAC1E,CAAC,QAAQ,CAAC,sBAAsB,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CACvE,CAAC;AAEF;;;GAGG;AACH,MAAM,OAAO,GAAG,CAAC,IAAY,EAAU,EAAE;IACvC,MAAM,KAAK,GAAG,aAAa,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,iBAAiB,CAAC,gBAAgB,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC;IAC/I,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACxI,OAAO,qBAAqB,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,qBAAqB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;AACvG,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,OAAO,GAAoB;IACtC,IAAI,EAAE,UAAU;IAChB,EAAE,EAAE,IAAI;IACR,UAAU,EAAE,CAAC;IACb,YAAY,EAAE;QACZ,aAAa,EAAE,IAAI;QACnB,gDAAgD;QAChD,SAAS,EAAE,IAAI;QACf,GAAG,EAAE,IAAI;QACT,KAAK,EAAE,IAAI;QACX,UAAU,EAAE,MAAM;KACnB;IACD,OAAO,EAAE,KAAK,EAAE,IAAkB,EAAiB,EAAE;QACnD,uHAAuH;QACvH,gBAAgB,CAAC,KAAK,KAAK,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC;QAC/E,IAAI,IAAI,CAAC,GAAG;YAAE,MAAM,OAAO,EAAE,CAAC;IAChC,CAAC;IACD,MAAM,EAAE,CAAC,MAAc,EAAU,EAAE;QACjC,MAAM,KAAK,GAAG,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC;QACrD,IAAI,KAAK,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAC1B,OAAO,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC;IACvD,CAAC;IACD,QAAQ,EAAE,YAAY,EAAE;IACxB,SAAS;IACT,OAAO,EAAE,CAAC,IAAY,EAAgB,EAAE;QACtC,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;QAC7C,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;IAC5E,CAAC;CACF,CAAC"}
|
package/dist/initial-stop.d.ts
CHANGED
|
@@ -6,8 +6,10 @@
|
|
|
6
6
|
* replaced with a plain letter, which the splitter does not read as an initial;
|
|
7
7
|
* - a prepositive abbreviation (U.S.) before any other capitalised word does not end it, unless a comma follows that
|
|
8
8
|
* word ("in the U.S. Yesterday, …"). Its last full stop is replaced with a letter;
|
|
9
|
-
* -
|
|
10
|
-
*
|
|
9
|
+
* - an abbreviation that closes a list (etc.) before a sentence-start-word ends the sentence. The splitter joins it to
|
|
10
|
+
* the next sentence when more than one space or a line break comes between. Its letters are replaced with a plain letter;
|
|
11
|
+
* - capitals with full stops (J., U.K.), or two or three small letters with them (a.m., p.m.), before a word in lower case
|
|
12
|
+
* ("J. meacham", "Hart, K. and J. Reed", "9 a.m. and 4 p.m.") do not end it, since no sentence opens in lower case. A word that is lower case only at its head (iPhone) may be a name that
|
|
11
13
|
* opens one, so it is left out. Their last full stop is replaced with a letter.
|
|
12
14
|
* The length does not change, so the spans fit the original text. Initials first on their line are a list's labels
|
|
13
15
|
* (A. The first item) and are left as they are.
|
|
@@ -15,9 +17,10 @@
|
|
|
15
17
|
export type InitialStops = {
|
|
16
18
|
readonly ends: RegExp | undefined;
|
|
17
19
|
readonly joins: RegExp | undefined;
|
|
20
|
+
readonly closings: RegExp | undefined;
|
|
18
21
|
};
|
|
19
22
|
/** Built once from the lexicons. */
|
|
20
|
-
export declare const initialStops: (startWords: readonly string[], prepositive: readonly string[]) => InitialStops;
|
|
21
|
-
/** The text with every initial that ends a sentence, and every prepositive abbreviation or initial that does not, rewritten. */
|
|
23
|
+
export declare const initialStops: (startWords: readonly string[], prepositive: readonly string[], closings?: readonly string[]) => InitialStops;
|
|
24
|
+
/** The text with every initial or closing abbreviation that ends a sentence, and every prepositive abbreviation or initial that does not, rewritten. */
|
|
22
25
|
export declare const unmarkInitialStops: (text: string, stops: InitialStops) => string;
|
|
23
26
|
//# sourceMappingURL=initial-stop.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"initial-stop.d.ts","sourceRoot":"","sources":["../src/initial-stop.ts"],"names":[],"mappings":"AAGA
|
|
1
|
+
{"version":3,"file":"initial-stop.d.ts","sourceRoot":"","sources":["../src/initial-stop.ts"],"names":[],"mappings":"AAGA;;;;;;;;;;;;;;;GAeG;AAEH,MAAM,MAAM,YAAY,GAAG;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;IAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAA;CAAE,CAAC;AAqB5I,oCAAoC;AACpC,eAAO,MAAM,YAAY,GAAI,YAAY,SAAS,MAAM,EAAE,EAAE,aAAa,SAAS,MAAM,EAAE,EAAE,WAAU,SAAS,MAAM,EAAO,KAAG,YAI9H,CAAC;AAkBF,wJAAwJ;AACxJ,eAAO,MAAM,kBAAkB,GAAI,MAAM,MAAM,EAAE,OAAO,YAAY,KAAG,MAOtE,CAAC"}
|
package/dist/initial-stop.js
CHANGED
|
@@ -4,30 +4,38 @@ const EDGE_BEFORE = String.raw `(?<![\p{L}\p{N}_.'’])`;
|
|
|
4
4
|
const WORD_END = String.raw `(?![\p{L}\p{N}_.'’-])`;
|
|
5
5
|
const PLAIN_LETTER = "n";
|
|
6
6
|
const CAPITAL = /\p{Lu}/gu;
|
|
7
|
-
const
|
|
7
|
+
const LETTER = /\p{L}/gu;
|
|
8
|
+
const BEFORE_LOWER_CASE = new RegExp(String.raw `${EDGE_BEFORE}(?:(?:\p{Lu}\.){1,3}|(?:\p{Ll}\.){2,3})(?=[ \t\r\n]+\p{Ll}(?!\p{Lu}))`, "gu");
|
|
8
9
|
const oneOf = (words) => `(?:${words.map(escapeRegExp).join("|")})`;
|
|
9
|
-
|
|
10
|
-
export const initialStops = (startWords, prepositive) => {
|
|
11
|
-
if (startWords.length === 0)
|
|
12
|
-
return { ends: undefined, joins: undefined };
|
|
13
|
-
const start = String.raw `\s+${oneOf(startWords)}${WORD_END}`;
|
|
10
|
+
const sentenceStops = (start, prepositive) => {
|
|
14
11
|
const ends = new RegExp(String.raw `${EDGE_BEFORE}(?:\p{Lu}\.){1,3}(?=${start})`, "gu");
|
|
15
12
|
if (prepositive.length === 0)
|
|
16
13
|
return { ends, joins: undefined };
|
|
17
14
|
return { ends, joins: new RegExp(String.raw `${EDGE_BEFORE}${oneOf(prepositive)}(?=\s+\p{Lu})(?!${start})(?!\s+\p{Lu}\p{L}*,)`, "gu") };
|
|
18
15
|
};
|
|
16
|
+
/** A closing abbreviation (etc.) before a sentence-start word. */
|
|
17
|
+
const closingEnds = (start, closings) => closings.length === 0 ? undefined : new RegExp(String.raw `${EDGE_BEFORE}${oneOf(closings)}(?=${start})`, "gu");
|
|
18
|
+
/** Built once from the lexicons. */
|
|
19
|
+
export const initialStops = (startWords, prepositive, closings = []) => {
|
|
20
|
+
if (startWords.length === 0)
|
|
21
|
+
return { ends: undefined, joins: undefined, closings: undefined };
|
|
22
|
+
const start = String.raw `\s+${oneOf(startWords)}${WORD_END}`;
|
|
23
|
+
return { ...sentenceStops(start, prepositive), closings: closingEnds(start, closings) };
|
|
24
|
+
};
|
|
19
25
|
/** Only a list's marker, or nothing, before the initials on their line (A. The first item; - B. The second item). */
|
|
20
26
|
const LABEL_HEAD = /^[ \t]*(?:[-*+•][ \t]+)?$/u;
|
|
21
27
|
const lineStartOf = (text, index) => Math.max(text.lastIndexOf("\n", index - 1), text.lastIndexOf("\r", index - 1)) + 1;
|
|
22
28
|
const isFirstOnLine = (text, index) => LABEL_HEAD.test(text.slice(lineStartOf(text, index), index));
|
|
23
29
|
const withoutLastStop = (abbreviation) => `${abbreviation.slice(0, -1)}${PLAIN_LETTER}`;
|
|
30
|
+
const closed = (text, pattern) => pattern === undefined ? text : text.replace(pattern, (abbreviation) => abbreviation.replace(LETTER, PLAIN_LETTER));
|
|
24
31
|
/** Initials before a word in lower case, not first on their line (a list's label stays as it is), rewritten. */
|
|
25
32
|
const joinedBeforeLowerCase = (text) => text.replace(BEFORE_LOWER_CASE, (initials, index) => (isFirstOnLine(text, index) ? initials : withoutLastStop(initials)));
|
|
26
|
-
/** The text with every initial that ends a sentence, and every prepositive abbreviation or initial that does not, rewritten. */
|
|
33
|
+
/** The text with every initial or closing abbreviation that ends a sentence, and every prepositive abbreviation or initial that does not, rewritten. */
|
|
27
34
|
export const unmarkInitialStops = (text, stops) => {
|
|
28
35
|
const ended = stops.ends === undefined
|
|
29
36
|
? text
|
|
30
37
|
: text.replace(stops.ends, (initials, index) => (isFirstOnLine(text, index) ? initials : initials.replace(CAPITAL, PLAIN_LETTER)));
|
|
31
|
-
|
|
38
|
+
const closedText = closed(ended, stops.closings);
|
|
39
|
+
return joinedBeforeLowerCase(stops.joins === undefined ? closedText : closedText.replace(stops.joins, withoutLastStop));
|
|
32
40
|
};
|
|
33
41
|
//# sourceMappingURL=initial-stop.js.map
|
package/dist/initial-stop.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"initial-stop.js","sourceRoot":"","sources":["../src/initial-stop.ts"],"names":[],"mappings":"AAAA,kHAAkH;AAClH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"initial-stop.js","sourceRoot":"","sources":["../src/initial-stop.ts"],"names":[],"mappings":"AAAA,kHAAkH;AAClH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAqB3C,MAAM,WAAW,GAAG,MAAM,CAAC,GAAG,CAAA,uBAAuB,CAAC;AACtD,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAA,uBAAuB,CAAC;AACnD,MAAM,YAAY,GAAG,GAAG,CAAC;AACzB,MAAM,OAAO,GAAG,UAAU,CAAC;AAC3B,MAAM,MAAM,GAAG,SAAS,CAAC;AACzB,MAAM,iBAAiB,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAA,GAAG,WAAW,uEAAuE,EAAE,IAAI,CAAC,CAAC;AAE5I,MAAM,KAAK,GAAG,CAAC,KAAwB,EAAU,EAAE,CAAC,MAAM,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;AAE/F,MAAM,aAAa,GAAG,CAAC,KAAa,EAAE,WAA8B,EAAwC,EAAE;IAC5G,MAAM,IAAI,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAA,GAAG,WAAW,uBAAuB,KAAK,GAAG,EAAE,IAAI,CAAC,CAAC;IACvF,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;IAChE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAA,GAAG,WAAW,GAAG,KAAK,CAAC,WAAW,CAAC,mBAAmB,KAAK,uBAAuB,EAAE,IAAI,CAAC,EAAE,CAAC;AACzI,CAAC,CAAC;AAEF,kEAAkE;AAClE,MAAM,WAAW,GAAG,CAAC,KAAa,EAAE,QAA2B,EAAsB,EAAE,CACrF,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAA,GAAG,WAAW,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,IAAI,CAAC,CAAC;AAEjH,oCAAoC;AACpC,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,UAA6B,EAAE,WAA8B,EAAE,WAA8B,EAAE,EAAgB,EAAE;IAC5I,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;IAC/F,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAA,MAAM,KAAK,CAAC,UAAU,CAAC,GAAG,QAAQ,EAAE,CAAC;IAC7D,OAAO,EAAE,GAAG,aAAa,CAAC,KAAK,EAAE,WAAW,CAAC,EAAE,QAAQ,EAAE,WAAW,CAAC,KAAK,EAAE,QAAQ,CAAC,EAAE,CAAC;AAC1F,CAAC,CAAC;AAEF,qHAAqH;AACrH,MAAM,UAAU,GAAG,4BAA4B,CAAC;AAEhD,MAAM,WAAW,GAAG,CAAC,IAAY,EAAE,KAAa,EAAU,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;AAEhJ,MAAM,aAAa,GAAG,CAAC,IAAY,EAAE,KAAa,EAAW,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;AAE7H,MAAM,eAAe,GAAG,CAAC,YAAoB,EAAU,EAAE,CAAC,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,YAAY,EAAE,CAAC;AAExG,MAAM,MAAM,GAAG,CAAC,IAAY,EAAE,OAA2B,EAAU,EAAE,CACnE,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,YAAoB,EAAE,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC;AAE7H,gHAAgH;AAChH,MAAM,qBAAqB,GAAG,CAAC,IAAY,EAAU,EAAE,CACrD,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAAC,QAAgB,EAAE,KAAa,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;AAE5I,wJAAwJ;AACxJ,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,IAAY,EAAE,KAAmB,EAAU,EAAE;IAC9E,MAAM,KAAK,GACT,KAAK,CAAC,IAAI,KAAK,SAAS;QACtB,CAAC,CAAC,IAAI;QACN,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,QAAgB,EAAE,KAAa,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC;IACvJ,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IACjD,OAAO,qBAAqB,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,eAAe,CAAC,CAAC,CAAC;AAC1H,CAAC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mixed-width-bracket.d.ts","sourceRoot":"","sources":["../src/mixed-width-bracket.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"mixed-width-bracket.d.ts","sourceRoot":"","sources":["../src/mixed-width-bracket.ts"],"names":[],"mappings":"AAsBA,iDAAiD;AACjD,eAAO,MAAM,sBAAsB,GAAI,MAAM,MAAM,KAAG,MAAsG,CAAC"}
|
|
@@ -2,14 +2,18 @@
|
|
|
2
2
|
* sentence-splitter は括弧を幅ごとに別の種類として組にする。有価証券報告書の番号「(1)」のように開きが半角・閉じが全角だと、
|
|
3
3
|
* 開いた「(」が節の終わりまで閉じず、その後ろの句点で一度も切れない(節全体が一文になる)。
|
|
4
4
|
*
|
|
5
|
-
* 短い番号の形(開き + 空白も同じ種類の括弧も含まない数文字 +
|
|
5
|
+
* 短い番号の形(開き + 空白も同じ種類の括弧も含まない数文字 + 幅の違う閉じ)と、句点で終わる括書き((以下「資金」という。))に限り、開きを閉じと同じ幅の字に替えて分割器に渡す。
|
|
6
6
|
* 長さが変わらないので、返る span は元の文字列にそのまま当てはまる。幅の混在そのものは bracket-width-consistency が元の文字列で報告する。
|
|
7
7
|
*/
|
|
8
8
|
// 中身の上限。番号・略号(1、株、注、△、同95.7%)に足りる長さ。長い中身の後の閉じは、別の番号の閉じ(1))かもしれない。
|
|
9
9
|
const MARKER_INNER_MAX = 8;
|
|
10
|
+
// 句点で終わる中身の上限。句点の後の閉じは番号の閉じではなく、括弧の中の文(以下「資金」という。)の閉じ。定義の括書きに足りる長さ。
|
|
11
|
+
const SENTENCE_INNER_MAX = 40;
|
|
10
12
|
const SHORT_MIXED = [
|
|
11
13
|
{ pattern: new RegExp(String.raw `\((?=[^\s()()]{1,${MARKER_INNER_MAX}}))`, "gu"), opener: "(" },
|
|
12
14
|
{ pattern: new RegExp(String.raw `((?=[^\s()()]{1,${MARKER_INNER_MAX}}\))`, "gu"), opener: "(" },
|
|
15
|
+
{ pattern: new RegExp(String.raw `\((?=[^\s()()]{1,${SENTENCE_INNER_MAX}}。))`, "gu"), opener: "(" },
|
|
16
|
+
{ pattern: new RegExp(String.raw `((?=[^\s()()]{1,${SENTENCE_INNER_MAX}}。\))`, "gu"), opener: "(" },
|
|
13
17
|
{ pattern: new RegExp(String.raw `\[(?=[^\s[\][]]{1,${MARKER_INNER_MAX}}])`, "gu"), opener: "[" },
|
|
14
18
|
{ pattern: new RegExp(String.raw `[(?=[^\s[\][]]{1,${MARKER_INNER_MAX}}\])`, "gu"), opener: "[" },
|
|
15
19
|
];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mixed-width-bracket.js","sourceRoot":"","sources":["../src/mixed-width-bracket.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,kEAAkE;AAClE,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAE3B,MAAM,WAAW,GAAG;IAClB,EAAE,OAAO,EAAE,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAA,oBAAoB,gBAAgB,KAAK,EAAE,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE;IAC/F,EAAE,OAAO,EAAE,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAA,mBAAmB,gBAAgB,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE;IAC/F,EAAE,OAAO,EAAE,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAA,qBAAqB,gBAAgB,KAAK,EAAE,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE;IAChG,EAAE,OAAO,EAAE,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAA,oBAAoB,gBAAgB,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE;CACjG,CAAC;AAEF,iDAAiD;AACjD,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,IAAY,EAAU,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,EAAE,IAAI,CAAC,CAAC"}
|
|
1
|
+
{"version":3,"file":"mixed-width-bracket.js","sourceRoot":"","sources":["../src/mixed-width-bracket.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,kEAAkE;AAClE,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAE3B,oEAAoE;AACpE,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAE9B,MAAM,WAAW,GAAG;IAClB,EAAE,OAAO,EAAE,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAA,oBAAoB,gBAAgB,KAAK,EAAE,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE;IAC/F,EAAE,OAAO,EAAE,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAA,mBAAmB,gBAAgB,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE;IAC/F,EAAE,OAAO,EAAE,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAA,oBAAoB,kBAAkB,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE;IAClG,EAAE,OAAO,EAAE,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAA,mBAAmB,kBAAkB,OAAO,EAAE,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE;IAClG,EAAE,OAAO,EAAE,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAA,qBAAqB,gBAAgB,KAAK,EAAE,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE;IAChG,EAAE,OAAO,EAAE,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAA,oBAAoB,gBAAgB,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE;CACjG,CAAC;AAEF,iDAAiD;AACjD,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,IAAY,EAAU,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,EAAE,IAAI,CAAC,CAAC"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { Span } from "chaffjs/plugin";
|
|
2
|
+
export type HeadingOpeners = RegExp | undefined;
|
|
3
|
+
/** 本文を始める語。後ろが助詞か「末」のときだけ(当期純利益の「当期」は語の頭)。 */
|
|
4
|
+
export declare const headingOpeners: (words: readonly string[]) => HeadingOpeners;
|
|
5
|
+
/** span を、頭の見出しと本文に割る。見出しが無ければ span 1 つ。 */
|
|
6
|
+
export declare const cutRunInHeadings: (source: string, span: Span, openers: HeadingOpeners) => Span[];
|
|
7
|
+
//# sourceMappingURL=run-in-heading.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run-in-heading.d.ts","sourceRoot":"","sources":["../src/run-in-heading.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AA+B3C,MAAM,MAAM,cAAc,GAAG,MAAM,GAAG,SAAS,CAAC;AAEhD,8CAA8C;AAC9C,eAAO,MAAM,cAAc,GAAI,OAAO,SAAS,MAAM,EAAE,KAAG,cAIzD,CAAC;AAmEF,4CAA4C;AAC5C,eAAO,MAAM,gBAAgB,GAAI,QAAQ,MAAM,EAAE,MAAM,IAAI,EAAE,SAAS,cAAc,KAAG,IAAI,EAI1F,CAAC"}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import { escapeRegExp } from "./regexp.js";
|
|
2
|
+
/**
|
|
3
|
+
* 改行を落として一行になった報告書では、項目の番号と見出しが本文の文へ句点なしに続く
|
|
4
|
+
* (「…であります。(3)販売の実績当期の販売実績は…」「5【経営上の重要な契約等】(…)当社は…」)。
|
|
5
|
+
* 分割器は見出しを次の文の頭に読むので、見出しの字が文の長さ・漢字の連続・「の」の数に入り、文末の並びは節をまたぐ。
|
|
6
|
+
*
|
|
7
|
+
* 見出しと本文の境が字で分かる所だけ切る:
|
|
8
|
+
* - 節の番号と墨付き括弧の題(5【…】)と、直後の括弧の副題の後ろ。
|
|
9
|
+
* - 行の途中の項目の番号(①、(3)、幅の違う (3))の後ろで、句点・読点の前に本文を始める語(当社、当期、辞書 run-in-heading-opener)が
|
|
10
|
+
* 出る所か、見出しの頭の語をもう一度書き出す所(④固定資産の減損固定資産の減損は)。
|
|
11
|
+
* 境の分からない見出し(③…の最適化レンタル用資産の…)は切らない。
|
|
12
|
+
*/
|
|
13
|
+
const LEADING_SPACE = /^[ \t\u3000]*/u;
|
|
14
|
+
/** 囲み数字と、括弧の番号、後ろの空白まで。開きと閉じの幅が違っても((3))一つの印で、括弧はその中で閉じる。 */
|
|
15
|
+
const ITEM_MARKER = /^(?:[①-⑳㉑-㉟㊱-㊿⑴-⒇]|[((][0-90-9]{1,2}[))])[ \t\u3000]*/u;
|
|
16
|
+
/** 節の番号と墨付き括弧の題。後ろの括弧の副題も題のうち。ただし括弧の番号((1))は次の項目の印。 */
|
|
17
|
+
const NUMBERED_TITLE = /^[0-90-9]{1,2}【[^【】。\n]{1,60}】(?:[((](?![0-90-9a-zA-Za-zA-Z]{1,3}[))])[^(())。\n]{2,60}[))])*/u;
|
|
18
|
+
/** 印の後ろが助詞なら、印は文の中の語(①の規定、5【…】に記載)。 */
|
|
19
|
+
const PARTICLE_AFTER = /^[ \t\u3000]*(?:の|に|を|が|は|と|で|へ|も|から|まで|より|及び|又は|並びに|若しくは)/u;
|
|
20
|
+
/** 見出しは句点・読点・引用符の前で終わる。それより遠くには探さない。 */
|
|
21
|
+
const HEADING_STOP = /[。、,,「」『』\n\r]/u;
|
|
22
|
+
const HEADING_MAX = 40;
|
|
23
|
+
/** 書き出しを繰り返すと読む長さ。短いと見出しの中の同じ字(の状況の)を拾う。 */
|
|
24
|
+
const REPEAT_MIN = 4;
|
|
25
|
+
const REPEATABLE = /^[\p{L}\p{N}・]+$/u;
|
|
26
|
+
/** 見出しは助詞・活用語尾(ひらがな)で終わらない。「売上高は当期…」の「は」の後ろは本文の続き。 */
|
|
27
|
+
const ENDS_IN_HIRAGANA = /\p{Script=Hiragana}$/u;
|
|
28
|
+
/** 本文を始める語。後ろが助詞か「末」のときだけ(当期純利益の「当期」は語の頭)。 */
|
|
29
|
+
export const headingOpeners = (words) => {
|
|
30
|
+
if (words.length === 0)
|
|
31
|
+
return undefined;
|
|
32
|
+
const alternatives = [...words].toSorted((a, b) => b.length - a.length).map(escapeRegExp);
|
|
33
|
+
return new RegExp(`(?:${alternatives.join("|")})(?=[のにはでもがと及、((]|末)`, "gu");
|
|
34
|
+
};
|
|
35
|
+
const headingLength = (rest) => {
|
|
36
|
+
const stop = rest.slice(0, HEADING_MAX).search(HEADING_STOP);
|
|
37
|
+
return stop === -1 ? Math.min(rest.length, HEADING_MAX) : stop;
|
|
38
|
+
};
|
|
39
|
+
const OPENERS = /[(([[【〔]/gu;
|
|
40
|
+
const CLOSERS = /[))]\]】〕]/gu;
|
|
41
|
+
const closesItsBrackets = (heading) => (heading.match(OPENERS)?.length ?? 0) === (heading.match(CLOSERS)?.length ?? 0);
|
|
42
|
+
/** 見出しがそこで終わってよいか: 空(印のすぐ後ろが本文)か、ひらがなで終わらず、開いた括弧を閉じている(経営成績(経営成績に関する分析))。 */
|
|
43
|
+
const canEndAt = (rest, at) => {
|
|
44
|
+
const heading = rest.slice(0, at);
|
|
45
|
+
return at === 0 || (!ENDS_IN_HIRAGANA.test(heading) && closesItsBrackets(heading));
|
|
46
|
+
};
|
|
47
|
+
/**
|
|
48
|
+
* 本文を始める語の位置。印のすぐ後ろの語((2)当社は…)より、見出しの後ろの語(⑥当社及び当社サービスの認知度向上当社は…)を取る。
|
|
49
|
+
* 本文の中の語((2)当社は当期に…)は助詞の後ろに立つので、見出しの終わりにならない。
|
|
50
|
+
*/
|
|
51
|
+
const openerAt = (rest, length, openers) => {
|
|
52
|
+
if (openers === undefined)
|
|
53
|
+
return undefined;
|
|
54
|
+
const places = [...rest.matchAll(openers)].map((match) => match.index).filter((at) => at < length && canEndAt(rest, at));
|
|
55
|
+
return places.find((at) => at > 0) ?? places[0];
|
|
56
|
+
};
|
|
57
|
+
/** 見出しの中で次の項目の印が始まる位置((1)経営成績等の概要①経営成績の状況…)。 */
|
|
58
|
+
const INNER_MARKER = /[①-⑳㉑-㉟㊱-㊿⑴-⒇]|[((][0-90-9]{1,2}[))]/gu;
|
|
59
|
+
const innerMarkerAt = (rest, length) => [...rest.slice(0, length).matchAll(INNER_MARKER)].map((match) => match.index).find((at) => at > 0 && canEndAt(rest, at));
|
|
60
|
+
const repeatAt = (rest, length) => {
|
|
61
|
+
const head = rest.slice(0, REPEAT_MIN);
|
|
62
|
+
if (head.length < REPEAT_MIN || !REPEATABLE.test(head))
|
|
63
|
+
return undefined;
|
|
64
|
+
const repeats = [...rest.slice(0, length).matchAll(new RegExp(escapeRegExp(head), "gu"))].map((match) => match.index);
|
|
65
|
+
return repeats.find((at) => at >= REPEAT_MIN && canEndAt(rest, at));
|
|
66
|
+
};
|
|
67
|
+
/** 印の後ろ(rest)で見出しが終わり本文が始まる位置。分からなければ undefined。 */
|
|
68
|
+
const bodyStart = (rest, openers) => {
|
|
69
|
+
if (PARTICLE_AFTER.test(rest))
|
|
70
|
+
return undefined;
|
|
71
|
+
const length = headingLength(rest);
|
|
72
|
+
const found = [openerAt(rest, length, openers), repeatAt(rest, length), innerMarkerAt(rest, length)].filter((at) => at !== undefined);
|
|
73
|
+
return found.length === 0 ? undefined : Math.min(...found);
|
|
74
|
+
};
|
|
75
|
+
const isLineStart = (source, at) => at === 0 || /[\n\r]/u.test(source.charAt(at - 1));
|
|
76
|
+
/** from から始まる文の頭の見出しの後ろ。題は行の頭でも、項目の番号は行の途中(前の文に続けて書いた所)だけ。 */
|
|
77
|
+
const cutAfterHeading = (source, from, end, openers) => {
|
|
78
|
+
const lead = from + (LEADING_SPACE.exec(source.slice(from, end))?.[0].length ?? 0);
|
|
79
|
+
const head = source.slice(lead, end);
|
|
80
|
+
const title = NUMBERED_TITLE.exec(head)?.[0];
|
|
81
|
+
if (title !== undefined)
|
|
82
|
+
return PARTICLE_AFTER.test(head.slice(title.length)) ? undefined : lead + title.length;
|
|
83
|
+
const marker = ITEM_MARKER.exec(head)?.[0];
|
|
84
|
+
if (marker === undefined || isLineStart(source, lead))
|
|
85
|
+
return undefined;
|
|
86
|
+
const at = bodyStart(head.slice(marker.length), openers);
|
|
87
|
+
return at === undefined ? undefined : lead + marker.length + at;
|
|
88
|
+
};
|
|
89
|
+
const cutsFrom = (source, from, end, openers) => {
|
|
90
|
+
const cut = cutAfterHeading(source, from, end, openers);
|
|
91
|
+
if (cut === undefined || cut <= from || source.slice(cut, end).trim() === "")
|
|
92
|
+
return [];
|
|
93
|
+
return [cut, ...cutsFrom(source, cut, end, openers)];
|
|
94
|
+
};
|
|
95
|
+
/** span を、頭の見出しと本文に割る。見出しが無ければ span 1 つ。 */
|
|
96
|
+
export const cutRunInHeadings = (source, span, openers) => {
|
|
97
|
+
const cuts = cutsFrom(source, span.start, span.end, openers);
|
|
98
|
+
const starts = [span.start, ...cuts];
|
|
99
|
+
return starts.map((start, index) => ({ start, end: cuts[index] ?? span.end }));
|
|
100
|
+
};
|
|
101
|
+
//# sourceMappingURL=run-in-heading.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run-in-heading.js","sourceRoot":"","sources":["../src/run-in-heading.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C;;;;;;;;;;GAUG;AAEH,MAAM,aAAa,GAAG,gBAAgB,CAAC;AACvC,6DAA6D;AAC7D,MAAM,WAAW,GAAG,wDAAwD,CAAC;AAC7E,uDAAuD;AACvD,MAAM,cAAc,GAAG,+FAA+F,CAAC;AACvH,uCAAuC;AACvC,MAAM,cAAc,GAAG,6DAA6D,CAAC;AACrF,wCAAwC;AACxC,MAAM,YAAY,GAAG,iBAAiB,CAAC;AACvC,MAAM,WAAW,GAAG,EAAE,CAAC;AACvB,4CAA4C;AAC5C,MAAM,UAAU,GAAG,CAAC,CAAC;AACrB,MAAM,UAAU,GAAG,mBAAmB,CAAC;AACvC,sDAAsD;AACtD,MAAM,gBAAgB,GAAG,uBAAuB,CAAC;AAIjD,8CAA8C;AAC9C,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,KAAwB,EAAkB,EAAE;IACzE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IACzC,MAAM,YAAY,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;IAC1F,OAAO,IAAI,MAAM,CAAC,MAAM,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,sBAAsB,EAAE,IAAI,CAAC,CAAC;AAC9E,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,CAAC,IAAY,EAAU,EAAE;IAC7C,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;IAC7D,OAAO,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACjE,CAAC,CAAC;AAEF,MAAM,OAAO,GAAG,YAAY,CAAC;AAC7B,MAAM,OAAO,GAAG,aAAa,CAAC;AAC9B,MAAM,iBAAiB,GAAG,CAAC,OAAe,EAAW,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC;AAExI,6EAA6E;AAC7E,MAAM,QAAQ,GAAG,CAAC,IAAY,EAAE,EAAU,EAAW,EAAE;IACrD,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAClC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC;AACrF,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,QAAQ,GAAG,CAAC,IAAY,EAAE,MAAc,EAAE,OAAuB,EAAsB,EAAE;IAC7F,IAAI,OAAO,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC5C,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,GAAG,MAAM,IAAI,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;IACzH,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;AAClD,CAAC,CAAC;AAEF,gDAAgD;AAChD,MAAM,YAAY,GAAG,wCAAwC,CAAC;AAC9D,MAAM,aAAa,GAAG,CAAC,IAAY,EAAE,MAAc,EAAsB,EAAE,CACzE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,GAAG,CAAC,IAAI,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;AAE3H,MAAM,QAAQ,GAAG,CAAC,IAAY,EAAE,MAAc,EAAsB,EAAE;IACpE,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;IACvC,IAAI,IAAI,CAAC,MAAM,GAAG,UAAU,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IACzE,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACtH,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,IAAI,UAAU,IAAI,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;AACtE,CAAC,CAAC;AAEF,oDAAoD;AACpD,MAAM,SAAS,GAAG,CAAC,IAAY,EAAE,OAAuB,EAAsB,EAAE;IAC9E,IAAI,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IAChD,MAAM,MAAM,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,KAAK,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,aAAa,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,KAAK,SAAS,CAAC,CAAC;IACtI,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;AAC7D,CAAC,CAAC;AAEF,MAAM,WAAW,GAAG,CAAC,MAAc,EAAE,EAAU,EAAW,EAAE,CAAC,EAAE,KAAK,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;AAE/G,8DAA8D;AAC9D,MAAM,eAAe,GAAG,CAAC,MAAc,EAAE,IAAY,EAAE,GAAW,EAAE,OAAuB,EAAsB,EAAE;IACjH,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC;IACnF,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACrC,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC7C,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC;IAChH,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC3C,IAAI,MAAM,KAAK,SAAS,IAAI,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IACxE,MAAM,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,CAAC;IACzD,OAAO,EAAE,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,GAAG,MAAM,CAAC,MAAM,GAAG,EAAE,CAAC;AAClE,CAAC,CAAC;AAEF,MAAM,QAAQ,GAAG,CAAC,MAAc,EAAE,IAAY,EAAE,GAAW,EAAE,OAAuB,EAAY,EAAE;IAChG,MAAM,GAAG,GAAG,eAAe,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;IACxD,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,IAAI,IAAI,IAAI,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE;QAAE,OAAO,EAAE,CAAC;IACxF,OAAO,CAAC,GAAG,EAAE,GAAG,QAAQ,CAAC,MAAM,EAAE,GAAG,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC;AACvD,CAAC,CAAC;AAEF,4CAA4C;AAC5C,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,MAAc,EAAE,IAAU,EAAE,OAAuB,EAAU,EAAE;IAC9F,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,IAAI,CAAC,CAAC;IACrC,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;AACjF,CAAC,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# 読み手に話しかける文末の終助詞(…含まれているよ。…違うんだよ。…納めているのかな?)。no-mixed-desumasu が、文の終わりの終助詞を
|
|
2
|
+
# つないだものがこの語で終わる常体の文を指さない(数には入れる)。ですます調の書き手が「である」に滑ったのではなく、読み手に話しかける
|
|
3
|
+
# 声を選んだ形で、子ども向けのページや語りかけの記事が地の文のですます調の中にわざと置く。
|
|
4
|
+
# 「な」だけ(わからないな。)は独り言でも書き損じでも使うので入れない。
|
|
5
|
+
id: address-particle
|
|
6
|
+
language: ja
|
|
7
|
+
entries:
|
|
8
|
+
- pattern: よ
|
|
9
|
+
- pattern: ね
|
|
10
|
+
- pattern: かな
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# 誤りの例を示す印。表記の手引きは「× Rspec / ○ RSpec」のように、わざと誤った形を正しい形と並べて見せる。name-variant は、
|
|
2
|
+
# 同じ行でこの印のすぐ後ろ(空白か「:」を挟んでよい)にある形を、名前の書き分けとして指さない。書いたとおりに当てる。
|
|
3
|
+
id: name-wrong-mark
|
|
4
|
+
language: ja
|
|
5
|
+
entries:
|
|
6
|
+
- pattern: "×"
|
|
7
|
+
- pattern: "✕"
|
|
8
|
+
- pattern: "✖"
|
|
9
|
+
- pattern: "✗"
|
|
10
|
+
- pattern: "✘"
|
|
@@ -54,3 +54,9 @@ entries:
|
|
|
54
54
|
- { pattern: 只今, group: 只今 }
|
|
55
55
|
# 「昨日来」(さくじつらい、昨日から)は一つの語。辞書は「昨日」と「来」に切り、その「昨日」が「きのう」と比べられてしまう。
|
|
56
56
|
- { pattern: 昨日来, group: 昨日来 }
|
|
57
|
+
# 「〜ばよかった」(悔やむ言い方)と「よかったら」(勧める言い方)は決まった言い方で、形容詞を「良い」と書く文書でもかなで書くことが多い。
|
|
58
|
+
# 形容詞の「良い」とは別に、自分の書き方どうし(ばよかった、ば良かった)とだけ比べる。
|
|
59
|
+
- { pattern: ばよかった, group: ば良かった }
|
|
60
|
+
- { pattern: ば良かった, group: ば良かった }
|
|
61
|
+
- { pattern: よかったら, group: 良かったら }
|
|
62
|
+
- { pattern: 良かったら, group: 良かったら }
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# 「それ以外の」とだけ言う語(他の)。no-doubled-joshi は、この語の直後の「の」を入れ子の「の」と数えない。
|
|
2
|
+
# 「他のすべてのディレクティブの代替」の「他の」は名詞を名詞の中に入れず、「ディレクティブ」を絞るだけ。
|
|
3
|
+
# 「すべて」「全て」は入れない。「オブジェクトのすべてのプロパティのアクセス」「全ての世帯の1世帯当たりの」のように、入れ子の輪の一つとして
|
|
4
|
+
# 読むことが多い。
|
|
5
|
+
id: particle-run-determiner
|
|
6
|
+
language: ja
|
|
7
|
+
entries:
|
|
8
|
+
- pattern: 他
|
|
9
|
+
- pattern: ほか
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
# 大文字で終わる製品の名前(Adobe Acrobat Reader DC
|
|
2
|
-
#
|
|
1
|
+
# 大文字で終わる製品の名前(Adobe Acrobat Reader DC)と、作り手の大文字の短い名前を頭に置いた製品の名前(MS Excel)。
|
|
2
|
+
# 大文字は製品の名前の一部で、書き手が展開する略語ではない。名前のまま字面どおりに照らすので、DC や MS だけなら略語のまま。
|
|
3
3
|
id: product-name
|
|
4
4
|
language: ja
|
|
5
5
|
entries:
|
|
@@ -7,3 +7,10 @@ entries:
|
|
|
7
7
|
- pattern: Acrobat Pro DC
|
|
8
8
|
- pattern: Acrobat Standard DC
|
|
9
9
|
- pattern: Acrobat DC
|
|
10
|
+
- pattern: MS Excel
|
|
11
|
+
- pattern: MS Word
|
|
12
|
+
- pattern: MS Office
|
|
13
|
+
- pattern: MS PowerPoint
|
|
14
|
+
- pattern: MS Outlook
|
|
15
|
+
- pattern: MS Access
|
|
16
|
+
- pattern: MS Teams
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# 項目の見出しの後ろで本文を始める語(当連結会計年度における…、当社は…)。改行を落として一行になった報告書では、
|
|
2
|
+
# 「①販売の実績当期の販売実績は…」のように番号の見出しが本文の文へ句点なしに続く。番号で始まる文の頭の句点も読点も
|
|
3
|
+
# 無い所でこの語が出れば、そこで見出しが終わり本文が始まる(run-in-heading.ts)。語の後ろが助詞か「末」のときだけ読む。
|
|
4
|
+
id: run-in-heading-opener
|
|
5
|
+
language: ja
|
|
6
|
+
entries:
|
|
7
|
+
- pattern: 当連結会計年度
|
|
8
|
+
- pattern: 当連結会計期間
|
|
9
|
+
- pattern: 当中間連結会計期間
|
|
10
|
+
- pattern: 当事業年度
|
|
11
|
+
- pattern: 当会計年度
|
|
12
|
+
- pattern: 当期
|
|
13
|
+
- pattern: 当社グループ
|
|
14
|
+
- pattern: 当社
|
|
15
|
+
- pattern: 当グループ
|
package/package.json
CHANGED
package/src/index.ts
CHANGED
|
@@ -12,6 +12,7 @@ import { splitAtQuotedLineEnds } from "./quoted-line-stop.ts";
|
|
|
12
12
|
import { insideBrackets, startsWithCloser } from "./inside-brackets.ts";
|
|
13
13
|
import { structure } from "./structure.ts";
|
|
14
14
|
import { cutAtBreaks, lineVocabulary, listItemHeads, listLineBreaks, startsListItem } from "./list-line.ts";
|
|
15
|
+
import { cutRunInHeadings, headingOpeners } from "./run-in-heading.ts";
|
|
15
16
|
import { listItemHeads as englishListItemHeads, listLineBreaks as englishListLineBreaks } from "./english-list-line.ts";
|
|
16
17
|
import { isReady, predicateOnly, prepare, readsAsCounter, readsAsOneAdverb, readsAsOneWord, tokenize } from "./pos.ts";
|
|
17
18
|
import { markSpacedCounters } from "./spaced-counter.ts";
|
|
@@ -76,6 +77,7 @@ const merge = (source: string, spans: readonly Span[]): Sentence[] => {
|
|
|
76
77
|
return acc;
|
|
77
78
|
}, [])
|
|
78
79
|
.flatMap((span) => cutAtBreaks(span, listBreaks))
|
|
80
|
+
.flatMap((span) => (english ? [span] : cutRunInHeadings(source, span, HEADING_OPENERS)))
|
|
79
81
|
.map((span) => withLanguage(source.slice(span.start, span.end), span))
|
|
80
82
|
.map((sentence) => (startsListItem(itemHeads, sentence.span.start) ? asListItem(sentence) : sentence));
|
|
81
83
|
};
|
|
@@ -105,11 +107,13 @@ const withTokens = (source: string, sentences: readonly Sentence[]): Sentence[]
|
|
|
105
107
|
|
|
106
108
|
const patternsOf = (id: string): string[] => (LEXICONS[id] ?? []).map((entry) => entry.pattern);
|
|
107
109
|
const LINE_VOCABULARY = lineVocabulary(patternsOf("numbering-counter"), patternsOf("line-head-tag"));
|
|
110
|
+
const HEADING_OPENERS = headingOpeners(patternsOf("run-in-heading-opener"));
|
|
108
111
|
|
|
109
112
|
const LABEL_STOPS = labelStops((LEXICONS["abbreviated-label"] ?? []).map((entry) => entry.pattern));
|
|
110
113
|
const INITIAL_STOPS = initialStops(
|
|
111
114
|
(LEXICONS["sentence-start-word"] ?? []).map((entry) => entry.pattern),
|
|
112
115
|
(LEXICONS["prepositive-abbreviation"] ?? []).map((entry) => entry.pattern),
|
|
116
|
+
(LEXICONS["closing-abbreviation"] ?? []).map((entry) => entry.pattern),
|
|
113
117
|
);
|
|
114
118
|
|
|
115
119
|
/**
|
package/src/initial-stop.ts
CHANGED
|
@@ -9,32 +9,43 @@ import { escapeRegExp } from "./regexp.ts";
|
|
|
9
9
|
* replaced with a plain letter, which the splitter does not read as an initial;
|
|
10
10
|
* - a prepositive abbreviation (U.S.) before any other capitalised word does not end it, unless a comma follows that
|
|
11
11
|
* word ("in the U.S. Yesterday, …"). Its last full stop is replaced with a letter;
|
|
12
|
-
* -
|
|
13
|
-
*
|
|
12
|
+
* - an abbreviation that closes a list (etc.) before a sentence-start-word ends the sentence. The splitter joins it to
|
|
13
|
+
* the next sentence when more than one space or a line break comes between. Its letters are replaced with a plain letter;
|
|
14
|
+
* - capitals with full stops (J., U.K.), or two or three small letters with them (a.m., p.m.), before a word in lower case
|
|
15
|
+
* ("J. meacham", "Hart, K. and J. Reed", "9 a.m. and 4 p.m.") do not end it, since no sentence opens in lower case. A word that is lower case only at its head (iPhone) may be a name that
|
|
14
16
|
* opens one, so it is left out. Their last full stop is replaced with a letter.
|
|
15
17
|
* The length does not change, so the spans fit the original text. Initials first on their line are a list's labels
|
|
16
18
|
* (A. The first item) and are left as they are.
|
|
17
19
|
*/
|
|
18
20
|
|
|
19
|
-
export type InitialStops = { readonly ends: RegExp | undefined; readonly joins: RegExp | undefined };
|
|
21
|
+
export type InitialStops = { readonly ends: RegExp | undefined; readonly joins: RegExp | undefined; readonly closings: RegExp | undefined };
|
|
20
22
|
|
|
21
23
|
const EDGE_BEFORE = String.raw`(?<![\p{L}\p{N}_.'’])`;
|
|
22
24
|
const WORD_END = String.raw`(?![\p{L}\p{N}_.'’-])`;
|
|
23
25
|
const PLAIN_LETTER = "n";
|
|
24
26
|
const CAPITAL = /\p{Lu}/gu;
|
|
25
|
-
const
|
|
27
|
+
const LETTER = /\p{L}/gu;
|
|
28
|
+
const BEFORE_LOWER_CASE = new RegExp(String.raw`${EDGE_BEFORE}(?:(?:\p{Lu}\.){1,3}|(?:\p{Ll}\.){2,3})(?=[ \t\r\n]+\p{Ll}(?!\p{Lu}))`, "gu");
|
|
26
29
|
|
|
27
30
|
const oneOf = (words: readonly string[]): string => `(?:${words.map(escapeRegExp).join("|")})`;
|
|
28
31
|
|
|
29
|
-
|
|
30
|
-
export const initialStops = (startWords: readonly string[], prepositive: readonly string[]): InitialStops => {
|
|
31
|
-
if (startWords.length === 0) return { ends: undefined, joins: undefined };
|
|
32
|
-
const start = String.raw`\s+${oneOf(startWords)}${WORD_END}`;
|
|
32
|
+
const sentenceStops = (start: string, prepositive: readonly string[]): Pick<InitialStops, "ends" | "joins"> => {
|
|
33
33
|
const ends = new RegExp(String.raw`${EDGE_BEFORE}(?:\p{Lu}\.){1,3}(?=${start})`, "gu");
|
|
34
34
|
if (prepositive.length === 0) return { ends, joins: undefined };
|
|
35
35
|
return { ends, joins: new RegExp(String.raw`${EDGE_BEFORE}${oneOf(prepositive)}(?=\s+\p{Lu})(?!${start})(?!\s+\p{Lu}\p{L}*,)`, "gu") };
|
|
36
36
|
};
|
|
37
37
|
|
|
38
|
+
/** A closing abbreviation (etc.) before a sentence-start word. */
|
|
39
|
+
const closingEnds = (start: string, closings: readonly string[]): RegExp | undefined =>
|
|
40
|
+
closings.length === 0 ? undefined : new RegExp(String.raw`${EDGE_BEFORE}${oneOf(closings)}(?=${start})`, "gu");
|
|
41
|
+
|
|
42
|
+
/** Built once from the lexicons. */
|
|
43
|
+
export const initialStops = (startWords: readonly string[], prepositive: readonly string[], closings: readonly string[] = []): InitialStops => {
|
|
44
|
+
if (startWords.length === 0) return { ends: undefined, joins: undefined, closings: undefined };
|
|
45
|
+
const start = String.raw`\s+${oneOf(startWords)}${WORD_END}`;
|
|
46
|
+
return { ...sentenceStops(start, prepositive), closings: closingEnds(start, closings) };
|
|
47
|
+
};
|
|
48
|
+
|
|
38
49
|
/** Only a list's marker, or nothing, before the initials on their line (A. The first item; - B. The second item). */
|
|
39
50
|
const LABEL_HEAD = /^[ \t]*(?:[-*+•][ \t]+)?$/u;
|
|
40
51
|
|
|
@@ -44,15 +55,19 @@ const isFirstOnLine = (text: string, index: number): boolean => LABEL_HEAD.test(
|
|
|
44
55
|
|
|
45
56
|
const withoutLastStop = (abbreviation: string): string => `${abbreviation.slice(0, -1)}${PLAIN_LETTER}`;
|
|
46
57
|
|
|
58
|
+
const closed = (text: string, pattern: RegExp | undefined): string =>
|
|
59
|
+
pattern === undefined ? text : text.replace(pattern, (abbreviation: string) => abbreviation.replace(LETTER, PLAIN_LETTER));
|
|
60
|
+
|
|
47
61
|
/** Initials before a word in lower case, not first on their line (a list's label stays as it is), rewritten. */
|
|
48
62
|
const joinedBeforeLowerCase = (text: string): string =>
|
|
49
63
|
text.replace(BEFORE_LOWER_CASE, (initials: string, index: number) => (isFirstOnLine(text, index) ? initials : withoutLastStop(initials)));
|
|
50
64
|
|
|
51
|
-
/** The text with every initial that ends a sentence, and every prepositive abbreviation or initial that does not, rewritten. */
|
|
65
|
+
/** The text with every initial or closing abbreviation that ends a sentence, and every prepositive abbreviation or initial that does not, rewritten. */
|
|
52
66
|
export const unmarkInitialStops = (text: string, stops: InitialStops): string => {
|
|
53
67
|
const ended =
|
|
54
68
|
stops.ends === undefined
|
|
55
69
|
? text
|
|
56
70
|
: text.replace(stops.ends, (initials: string, index: number) => (isFirstOnLine(text, index) ? initials : initials.replace(CAPITAL, PLAIN_LETTER)));
|
|
57
|
-
|
|
71
|
+
const closedText = closed(ended, stops.closings);
|
|
72
|
+
return joinedBeforeLowerCase(stops.joins === undefined ? closedText : closedText.replace(stops.joins, withoutLastStop));
|
|
58
73
|
};
|
|
@@ -2,15 +2,20 @@
|
|
|
2
2
|
* sentence-splitter は括弧を幅ごとに別の種類として組にする。有価証券報告書の番号「(1)」のように開きが半角・閉じが全角だと、
|
|
3
3
|
* 開いた「(」が節の終わりまで閉じず、その後ろの句点で一度も切れない(節全体が一文になる)。
|
|
4
4
|
*
|
|
5
|
-
* 短い番号の形(開き + 空白も同じ種類の括弧も含まない数文字 +
|
|
5
|
+
* 短い番号の形(開き + 空白も同じ種類の括弧も含まない数文字 + 幅の違う閉じ)と、句点で終わる括書き((以下「資金」という。))に限り、開きを閉じと同じ幅の字に替えて分割器に渡す。
|
|
6
6
|
* 長さが変わらないので、返る span は元の文字列にそのまま当てはまる。幅の混在そのものは bracket-width-consistency が元の文字列で報告する。
|
|
7
7
|
*/
|
|
8
8
|
// 中身の上限。番号・略号(1、株、注、△、同95.7%)に足りる長さ。長い中身の後の閉じは、別の番号の閉じ(1))かもしれない。
|
|
9
9
|
const MARKER_INNER_MAX = 8;
|
|
10
10
|
|
|
11
|
+
// 句点で終わる中身の上限。句点の後の閉じは番号の閉じではなく、括弧の中の文(以下「資金」という。)の閉じ。定義の括書きに足りる長さ。
|
|
12
|
+
const SENTENCE_INNER_MAX = 40;
|
|
13
|
+
|
|
11
14
|
const SHORT_MIXED = [
|
|
12
15
|
{ pattern: new RegExp(String.raw`\((?=[^\s()()]{1,${MARKER_INNER_MAX}}))`, "gu"), opener: "(" },
|
|
13
16
|
{ pattern: new RegExp(String.raw`((?=[^\s()()]{1,${MARKER_INNER_MAX}}\))`, "gu"), opener: "(" },
|
|
17
|
+
{ pattern: new RegExp(String.raw`\((?=[^\s()()]{1,${SENTENCE_INNER_MAX}}。))`, "gu"), opener: "(" },
|
|
18
|
+
{ pattern: new RegExp(String.raw`((?=[^\s()()]{1,${SENTENCE_INNER_MAX}}。\))`, "gu"), opener: "(" },
|
|
14
19
|
{ pattern: new RegExp(String.raw`\[(?=[^\s[\][]]{1,${MARKER_INNER_MAX}}])`, "gu"), opener: "[" },
|
|
15
20
|
{ pattern: new RegExp(String.raw`[(?=[^\s[\][]]{1,${MARKER_INNER_MAX}}\])`, "gu"), opener: "[" },
|
|
16
21
|
];
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import type { Span } from "chaffjs/plugin";
|
|
2
|
+
import { escapeRegExp } from "./regexp.ts";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* 改行を落として一行になった報告書では、項目の番号と見出しが本文の文へ句点なしに続く
|
|
6
|
+
* (「…であります。(3)販売の実績当期の販売実績は…」「5【経営上の重要な契約等】(…)当社は…」)。
|
|
7
|
+
* 分割器は見出しを次の文の頭に読むので、見出しの字が文の長さ・漢字の連続・「の」の数に入り、文末の並びは節をまたぐ。
|
|
8
|
+
*
|
|
9
|
+
* 見出しと本文の境が字で分かる所だけ切る:
|
|
10
|
+
* - 節の番号と墨付き括弧の題(5【…】)と、直後の括弧の副題の後ろ。
|
|
11
|
+
* - 行の途中の項目の番号(①、(3)、幅の違う (3))の後ろで、句点・読点の前に本文を始める語(当社、当期、辞書 run-in-heading-opener)が
|
|
12
|
+
* 出る所か、見出しの頭の語をもう一度書き出す所(④固定資産の減損固定資産の減損は)。
|
|
13
|
+
* 境の分からない見出し(③…の最適化レンタル用資産の…)は切らない。
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
const LEADING_SPACE = /^[ \t\u3000]*/u;
|
|
17
|
+
/** 囲み数字と、括弧の番号、後ろの空白まで。開きと閉じの幅が違っても((3))一つの印で、括弧はその中で閉じる。 */
|
|
18
|
+
const ITEM_MARKER = /^(?:[①-⑳㉑-㉟㊱-㊿⑴-⒇]|[((][0-90-9]{1,2}[))])[ \t\u3000]*/u;
|
|
19
|
+
/** 節の番号と墨付き括弧の題。後ろの括弧の副題も題のうち。ただし括弧の番号((1))は次の項目の印。 */
|
|
20
|
+
const NUMBERED_TITLE = /^[0-90-9]{1,2}【[^【】。\n]{1,60}】(?:[((](?![0-90-9a-zA-Za-zA-Z]{1,3}[))])[^(())。\n]{2,60}[))])*/u;
|
|
21
|
+
/** 印の後ろが助詞なら、印は文の中の語(①の規定、5【…】に記載)。 */
|
|
22
|
+
const PARTICLE_AFTER = /^[ \t\u3000]*(?:の|に|を|が|は|と|で|へ|も|から|まで|より|及び|又は|並びに|若しくは)/u;
|
|
23
|
+
/** 見出しは句点・読点・引用符の前で終わる。それより遠くには探さない。 */
|
|
24
|
+
const HEADING_STOP = /[。、,,「」『』\n\r]/u;
|
|
25
|
+
const HEADING_MAX = 40;
|
|
26
|
+
/** 書き出しを繰り返すと読む長さ。短いと見出しの中の同じ字(の状況の)を拾う。 */
|
|
27
|
+
const REPEAT_MIN = 4;
|
|
28
|
+
const REPEATABLE = /^[\p{L}\p{N}・]+$/u;
|
|
29
|
+
/** 見出しは助詞・活用語尾(ひらがな)で終わらない。「売上高は当期…」の「は」の後ろは本文の続き。 */
|
|
30
|
+
const ENDS_IN_HIRAGANA = /\p{Script=Hiragana}$/u;
|
|
31
|
+
|
|
32
|
+
export type HeadingOpeners = RegExp | undefined;
|
|
33
|
+
|
|
34
|
+
/** 本文を始める語。後ろが助詞か「末」のときだけ(当期純利益の「当期」は語の頭)。 */
|
|
35
|
+
export const headingOpeners = (words: readonly string[]): HeadingOpeners => {
|
|
36
|
+
if (words.length === 0) return undefined;
|
|
37
|
+
const alternatives = [...words].toSorted((a, b) => b.length - a.length).map(escapeRegExp);
|
|
38
|
+
return new RegExp(`(?:${alternatives.join("|")})(?=[のにはでもがと及、((]|末)`, "gu");
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
const headingLength = (rest: string): number => {
|
|
42
|
+
const stop = rest.slice(0, HEADING_MAX).search(HEADING_STOP);
|
|
43
|
+
return stop === -1 ? Math.min(rest.length, HEADING_MAX) : stop;
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
const OPENERS = /[(([[【〔]/gu;
|
|
47
|
+
const CLOSERS = /[))]\]】〕]/gu;
|
|
48
|
+
const closesItsBrackets = (heading: string): boolean => (heading.match(OPENERS)?.length ?? 0) === (heading.match(CLOSERS)?.length ?? 0);
|
|
49
|
+
|
|
50
|
+
/** 見出しがそこで終わってよいか: 空(印のすぐ後ろが本文)か、ひらがなで終わらず、開いた括弧を閉じている(経営成績(経営成績に関する分析))。 */
|
|
51
|
+
const canEndAt = (rest: string, at: number): boolean => {
|
|
52
|
+
const heading = rest.slice(0, at);
|
|
53
|
+
return at === 0 || (!ENDS_IN_HIRAGANA.test(heading) && closesItsBrackets(heading));
|
|
54
|
+
};
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* 本文を始める語の位置。印のすぐ後ろの語((2)当社は…)より、見出しの後ろの語(⑥当社及び当社サービスの認知度向上当社は…)を取る。
|
|
58
|
+
* 本文の中の語((2)当社は当期に…)は助詞の後ろに立つので、見出しの終わりにならない。
|
|
59
|
+
*/
|
|
60
|
+
const openerAt = (rest: string, length: number, openers: HeadingOpeners): number | undefined => {
|
|
61
|
+
if (openers === undefined) return undefined;
|
|
62
|
+
const places = [...rest.matchAll(openers)].map((match) => match.index).filter((at) => at < length && canEndAt(rest, at));
|
|
63
|
+
return places.find((at) => at > 0) ?? places[0];
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
/** 見出しの中で次の項目の印が始まる位置((1)経営成績等の概要①経営成績の状況…)。 */
|
|
67
|
+
const INNER_MARKER = /[①-⑳㉑-㉟㊱-㊿⑴-⒇]|[((][0-90-9]{1,2}[))]/gu;
|
|
68
|
+
const innerMarkerAt = (rest: string, length: number): number | undefined =>
|
|
69
|
+
[...rest.slice(0, length).matchAll(INNER_MARKER)].map((match) => match.index).find((at) => at > 0 && canEndAt(rest, at));
|
|
70
|
+
|
|
71
|
+
const repeatAt = (rest: string, length: number): number | undefined => {
|
|
72
|
+
const head = rest.slice(0, REPEAT_MIN);
|
|
73
|
+
if (head.length < REPEAT_MIN || !REPEATABLE.test(head)) return undefined;
|
|
74
|
+
const repeats = [...rest.slice(0, length).matchAll(new RegExp(escapeRegExp(head), "gu"))].map((match) => match.index);
|
|
75
|
+
return repeats.find((at) => at >= REPEAT_MIN && canEndAt(rest, at));
|
|
76
|
+
};
|
|
77
|
+
|
|
78
|
+
/** 印の後ろ(rest)で見出しが終わり本文が始まる位置。分からなければ undefined。 */
|
|
79
|
+
const bodyStart = (rest: string, openers: HeadingOpeners): number | undefined => {
|
|
80
|
+
if (PARTICLE_AFTER.test(rest)) return undefined;
|
|
81
|
+
const length = headingLength(rest);
|
|
82
|
+
const found = [openerAt(rest, length, openers), repeatAt(rest, length), innerMarkerAt(rest, length)].filter((at) => at !== undefined);
|
|
83
|
+
return found.length === 0 ? undefined : Math.min(...found);
|
|
84
|
+
};
|
|
85
|
+
|
|
86
|
+
const isLineStart = (source: string, at: number): boolean => at === 0 || /[\n\r]/u.test(source.charAt(at - 1));
|
|
87
|
+
|
|
88
|
+
/** from から始まる文の頭の見出しの後ろ。題は行の頭でも、項目の番号は行の途中(前の文に続けて書いた所)だけ。 */
|
|
89
|
+
const cutAfterHeading = (source: string, from: number, end: number, openers: HeadingOpeners): number | undefined => {
|
|
90
|
+
const lead = from + (LEADING_SPACE.exec(source.slice(from, end))?.[0].length ?? 0);
|
|
91
|
+
const head = source.slice(lead, end);
|
|
92
|
+
const title = NUMBERED_TITLE.exec(head)?.[0];
|
|
93
|
+
if (title !== undefined) return PARTICLE_AFTER.test(head.slice(title.length)) ? undefined : lead + title.length;
|
|
94
|
+
const marker = ITEM_MARKER.exec(head)?.[0];
|
|
95
|
+
if (marker === undefined || isLineStart(source, lead)) return undefined;
|
|
96
|
+
const at = bodyStart(head.slice(marker.length), openers);
|
|
97
|
+
return at === undefined ? undefined : lead + marker.length + at;
|
|
98
|
+
};
|
|
99
|
+
|
|
100
|
+
const cutsFrom = (source: string, from: number, end: number, openers: HeadingOpeners): number[] => {
|
|
101
|
+
const cut = cutAfterHeading(source, from, end, openers);
|
|
102
|
+
if (cut === undefined || cut <= from || source.slice(cut, end).trim() === "") return [];
|
|
103
|
+
return [cut, ...cutsFrom(source, cut, end, openers)];
|
|
104
|
+
};
|
|
105
|
+
|
|
106
|
+
/** span を、頭の見出しと本文に割る。見出しが無ければ span 1 つ。 */
|
|
107
|
+
export const cutRunInHeadings = (source: string, span: Span, openers: HeadingOpeners): Span[] => {
|
|
108
|
+
const cuts = cutsFrom(source, span.start, span.end, openers);
|
|
109
|
+
const starts = [span.start, ...cuts];
|
|
110
|
+
return starts.map((start, index) => ({ start, end: cuts[index] ?? span.end }));
|
|
111
|
+
};
|