@chaffjs/lang-ja 0.10.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAgB,eAAe,EAAgC,MAAM,gBAAgB,CAAC;AAsDlG,eAAO,MAAM,OAAO,EAAE,eA0BrB,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAgB,eAAe,EAAgC,MAAM,gBAAgB,CAAC;AA2DlG,eAAO,MAAM,OAAO,EAAE,eA0BrB,CAAC"}
package/dist/index.js CHANGED
@@ -2,6 +2,7 @@ import { loadLexicons } from "./lexicons.js";
2
2
  import { sentenceSpans } from "./sentence-split.js";
3
3
  import { structure } from "./structure.js";
4
4
  import { isReady, predicateOnly, prepare, tokenize } from "./pos.js";
5
+ import { distributiveVocabulary, markReduplication } from "./reduplication.js";
5
6
  // chaff からは型だけを取る。実行時の値依存を作らない。アダプタは単体で動く。
6
7
  /**
7
8
  * 日本語の文末は「。!?」に限られる。
@@ -38,6 +39,7 @@ const merge = (source, spans) => spans
38
39
  * token の span は文ではなく、segment に渡した文字列を基準にする。文の span と同じ座標系。
39
40
  * 文ごとに解析して足し戻すのではなく、一度解析して文へ配る。同じ文字列を二度読まない。
40
41
  */
42
+ const DISTRIBUTIVE = distributiveVocabulary(loadLexicons());
41
43
  const withTokens = (source, sentences) => {
42
44
  const tokens = tokenize(source);
43
45
  if (tokens === undefined)
@@ -45,7 +47,7 @@ const withTokens = (source, sentences) => {
45
47
  return sentences.map((sentence) => ({
46
48
  ...sentence,
47
49
  // 述語かどうかは文の中でしか決まらないので、文へ配ってから印を落とす。
48
- tokens: predicateOnly(tokens.filter((token) => token.span.start >= sentence.span.start && token.span.end <= sentence.span.end)),
50
+ tokens: markReduplication(predicateOnly(tokens.filter((token) => token.span.start >= sentence.span.start && token.span.end <= sentence.span.end)), DISTRIBUTIVE),
49
51
  }));
50
52
  };
51
53
  export const adapter = {
package/dist/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAGrE,2CAA2C;AAE3C;;;;;;;;;GASG;AACH,MAAM,MAAM,GAAG,sBAAsB,CAAC;AAEtC,MAAM,QAAQ,GAAG,eAAe,CAAC;AACjC,MAAM,SAAS,GAAG,MAAM,CAAC;AAEzB,MAAM,MAAM,GAAG,CAAC,IAAY,EAAW,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEvF;;;GAGG;AACH,MAAM,KAAK,GAAG,CAAC,MAAc,EAAE,KAAsB,EAAc,EAAE,CACnE,KAAK;KACF,MAAM,CAAS,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;IAC5B,MAAM,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACxB,yCAAyC;IACzC,uCAAuC;IACvC,6BAA6B;IAC7B,MAAM,WAAW,GAAG,IAAI,KAAK,SAAS,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC5F,IAAI,IAAI,KAAK,SAAS,IAAI,CAAC,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QACrF,OAAO,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IACrE,CAAC;IACD,OAAO,CAAC,GAAG,GAAG,EAAE,IAAI,CAAC,CAAC;AACxB,CAAC,EAAE,EAAE,CAAC;KACL,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;AAEzE;;;GAGG;AACH,MAAM,UAAU,GAAG,CAAC,MAAc,EAAE,SAA8B,EAAc,EAAE;IAChF,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;IAChC,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,CAAC,GAAG,SAAS,CAAC,CAAC;IAChD,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAClC,GAAG,QAAQ;QACX,qCAAqC;QACrC,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,QAAQ,CAAC,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;KAChI,CAAC,CAAC,CAAC;AACN,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,OAAO,GAAoB;IACtC,IAAI,EAAE,UAAU;IAChB,EAAE,EAAE,IAAI;IACR,UAAU,EAAE,CAAC;IACb,YAAY,EAAE;QACZ,aAAa,EAAE,IAAI;QACnB,gDAAgD;QAChD,SAAS,EAAE,IAAI;QACf,GAAG,EAAE,IAAI;QACT,KAAK,EAAE,IAAI;QACX,UAAU,EAAE,MAAM;KACnB;IACD,OAAO,EAAE,KAAK,EAAE,IAAkB,EAAiB,EAAE;QACnD,IAAI,IAAI,CAAC,GAAG;YAAE,MAAM,OAAO,EAAE,CAAC;IAChC,CAAC;IACD,MAAM,EAAE,CAAC,MAAc,EAAU,EAAE;QACjC,MAAM,KAAK,GAAG,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC;QACrD,IAAI,KAAK,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAC1B,OAAO,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC;IACvD,CAAC;IACD,QAAQ,EAAE,YAAY,EAAE;IACxB,SAAS;IACT,OAAO,EAAE,CAAC,IAAY,EAAgB,EAAE;QACtC,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC;QACnD,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;IAC5E,CAAC;CACF,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AACrE,OAAO,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAG/E,2CAA2C;AAE3C;;;;;;;;;GASG;AACH,MAAM,MAAM,GAAG,sBAAsB,CAAC;AAEtC,MAAM,QAAQ,GAAG,eAAe,CAAC;AACjC,MAAM,SAAS,GAAG,MAAM,CAAC;AAEzB,MAAM,MAAM,GAAG,CAAC,IAAY,EAAW,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEvF;;;GAGG;AACH,MAAM,KAAK,GAAG,CAAC,MAAc,EAAE,KAAsB,EAAc,EAAE,CACnE,KAAK;KACF,MAAM,CAAS,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;IAC5B,MAAM,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACxB,yCAAyC;IACzC,uCAAuC;IACvC,6BAA6B;IAC7B,MAAM,WAAW,GAAG,IAAI,KAAK,SAAS,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC5F,IAAI,IAAI,KAAK,SAAS,IAAI,CAAC,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QACrF,OAAO,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IACrE,CAAC;IACD,OAAO,CAAC,GAAG,GAAG,EAAE,IAAI,CAAC,CAAC;AACxB,CAAC,EAAE,EAAE,CAAC;KACL,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;AAEzE;;;GAGG;AACH,MAAM,YAAY,GAAG,sBAAsB,CAAC,YAAY,EAAE,CAAC,CAAC;AAE5D,MAAM,UAAU,GAAG,CAAC,MAAc,EAAE,SAA8B,EAAc,EAAE;IAChF,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;IAChC,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,CAAC,GAAG,SAAS,CAAC,CAAC;IAChD,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAClC,GAAG,QAAQ;QACX,qCAAqC;QACrC,MAAM,EAAE,iBAAiB,CACvB,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,QAAQ,CAAC,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EACvH,YAAY,CACb;KACF,CAAC,CAAC,CAAC;AACN,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,OAAO,GAAoB;IACtC,IAAI,EAAE,UAAU;IAChB,EAAE,EAAE,IAAI;IACR,UAAU,EAAE,CAAC;IACb,YAAY,EAAE;QACZ,aAAa,EAAE,IAAI;QACnB,gDAAgD;QAChD,SAAS,EAAE,IAAI;QACf,GAAG,EAAE,IAAI;QACT,KAAK,EAAE,IAAI;QACX,UAAU,EAAE,MAAM;KACnB;IACD,OAAO,EAAE,KAAK,EAAE,IAAkB,EAAiB,EAAE;QACnD,IAAI,IAAI,CAAC,GAAG;YAAE,MAAM,OAAO,EAAE,CAAC;IAChC,CAAC;IACD,MAAM,EAAE,CAAC,MAAc,EAAU,EAAE;QACjC,MAAM,KAAK,GAAG,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC;QACrD,IAAI,KAAK,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAC1B,OAAO,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC;IACvD,CAAC;IACD,QAAQ,EAAE,YAAY,EAAE;IACxB,SAAS;IACT,OAAO,EAAE,CAAC,IAAY,EAAgB,EAAE;QACtC,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC;QACnD,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;IAC5E,CAAC;CACF,CAAC"}
@@ -1 +1 @@
1
- {"version":3,"file":"lexicons.d.ts","sourceRoot":"","sources":["../src/lexicons.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,OAAO,EAAgB,MAAM,gBAAgB,CAAC;AAiB5D;;;GAGG;AACH,eAAO,MAAM,YAAY,GAAI,MAAK,MAAY,KAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAQ7D,CAAC"}
1
+ {"version":3,"file":"lexicons.d.ts","sourceRoot":"","sources":["../src/lexicons.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,OAAO,EAAgB,MAAM,gBAAgB,CAAC;AAoB5D;;;GAGG;AACH,eAAO,MAAM,YAAY,GAAI,MAAK,MAAY,KAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAQ7D,CAAC"}
package/dist/lexicons.js CHANGED
@@ -4,6 +4,7 @@ import { dirname, join } from "node:path";
4
4
  import { parse } from "yaml";
5
5
  const DIR = join(dirname(fileURLToPath(import.meta.url)), "..", "lexicons");
6
6
  const isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
7
+ const POSITIONS = ["before", "after"];
7
8
  const toEntry = (raw) => {
8
9
  if (!isRecord(raw) || typeof raw["pattern"] !== "string")
9
10
  return undefined;
@@ -13,6 +14,7 @@ const toEntry = (raw) => {
13
14
  pattern: raw["pattern"],
14
15
  weight: typeof weight === "number" ? weight : undefined,
15
16
  instead_of: typeof instead === "string" ? instead : undefined,
17
+ position: POSITIONS.find((position) => position === raw["position"]),
16
18
  };
17
19
  };
18
20
  /**
@@ -1 +1 @@
1
- {"version":3,"file":"lexicons.js","sourceRoot":"","sources":["../src/lexicons.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,KAAK,EAAE,MAAM,MAAM,CAAC;AAG7B,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;AAE5E,MAAM,QAAQ,GAAG,CAAC,KAAc,EAAoC,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAE5I,MAAM,OAAO,GAAG,CAAC,GAAY,EAA4B,EAAE;IACzD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,OAAO,GAAG,CAAC,SAAS,CAAC,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC;IAC3E,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC7B,MAAM,OAAO,GAAG,GAAG,CAAC,YAAY,CAAC,CAAC;IAClC,OAAO;QACL,OAAO,EAAE,GAAG,CAAC,SAAS,CAAC;QACvB,MAAM,EAAE,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;QACvD,UAAU,EAAE,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;KAC9D,CAAC;AACJ,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,MAAc,GAAG,EAA2B,EAAE,CACzE,WAAW,CAAC,GAAG,CAAC;KACb,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;KACxC,MAAM,CAA0B,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;IAC7C,MAAM,GAAG,GAAY,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;IAClE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,OAAO,GAAG,CAAC,IAAI,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAAE,OAAO,GAAG,CAAC;IAClG,MAAM,OAAO,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;IACnF,OAAO,EAAE,GAAG,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC;AAC1C,CAAC,EAAE,EAAE,CAAC,CAAC"}
1
+ {"version":3,"file":"lexicons.js","sourceRoot":"","sources":["../src/lexicons.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,KAAK,EAAE,MAAM,MAAM,CAAC;AAG7B,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;AAE5E,MAAM,QAAQ,GAAG,CAAC,KAAc,EAAoC,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAE5I,MAAM,SAAS,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAU,CAAC;AAE/C,MAAM,OAAO,GAAG,CAAC,GAAY,EAA4B,EAAE;IACzD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,OAAO,GAAG,CAAC,SAAS,CAAC,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC;IAC3E,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC7B,MAAM,OAAO,GAAG,GAAG,CAAC,YAAY,CAAC,CAAC;IAClC,OAAO;QACL,OAAO,EAAE,GAAG,CAAC,SAAS,CAAC;QACvB,MAAM,EAAE,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;QACvD,UAAU,EAAE,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;QAC7D,QAAQ,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,KAAK,GAAG,CAAC,UAAU,CAAC,CAAC;KACrE,CAAC;AACJ,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,MAAc,GAAG,EAA2B,EAAE,CACzE,WAAW,CAAC,GAAG,CAAC;KACb,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;KACxC,MAAM,CAA0B,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;IAC7C,MAAM,GAAG,GAAY,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;IAClE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,OAAO,GAAG,CAAC,IAAI,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAAE,OAAO,GAAG,CAAC;IAClG,MAAM,OAAO,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;IACnF,OAAO,EAAE,GAAG,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC;AAC1C,CAAC,EAAE,EAAE,CAAC,CAAC"}
@@ -0,0 +1,12 @@
1
+ import type { Lexicon, Token } from "chaffjs/plugin";
2
+ /**
3
+ * 名詞を重ねて「それぞれの」を言う形(会社会社で、部署部署の)。重ねられるのは、まとまりや場合を指す名詞だけで、
4
+ * 後ろに助詞が続く。二つ目の名詞に UD の Echo=Rdp を付ける。「資料資料の」「確認確認する」は書き損じのまま。
5
+ */
6
+ export type Distributive = {
7
+ readonly nouns: ReadonlySet<string>;
8
+ readonly particles: ReadonlySet<string>;
9
+ };
10
+ export declare const distributiveVocabulary: (lexicons: Readonly<Record<string, Lexicon>>) => Distributive;
11
+ export declare const markReduplication: (tokens: readonly Token[], vocabulary: Distributive) => Token[];
12
+ //# sourceMappingURL=reduplication.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"reduplication.d.ts","sourceRoot":"","sources":["../src/reduplication.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAErD;;;GAGG;AACH,MAAM,MAAM,YAAY,GAAG;IAAE,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IAAC,QAAQ,CAAC,SAAS,EAAE,WAAW,CAAC,MAAM,CAAC,CAAA;CAAE,CAAC;AAI5G,eAAO,MAAM,sBAAsB,GAAI,UAAU,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,KAAG,YAGnF,CAAC;AAeH,eAAO,MAAM,iBAAiB,GAAI,QAAQ,SAAS,KAAK,EAAE,EAAE,YAAY,YAAY,KAAG,KAAK,EAGzF,CAAC"}
@@ -0,0 +1,15 @@
1
+ const patternsOf = (lexicon) => new Set((lexicon ?? []).map((entry) => entry.pattern));
2
+ export const distributiveVocabulary = (lexicons) => ({
3
+ nouns: patternsOf(lexicons["distributive-noun"]),
4
+ particles: patternsOf(lexicons["distributive-particle"]),
5
+ });
6
+ const isSameNoun = (first, second, nouns) => first !== undefined &&
7
+ first.pos === "NOUN" &&
8
+ second.pos === "NOUN" &&
9
+ first.surface === second.surface &&
10
+ first.span.end === second.span.start &&
11
+ nouns.has(second.surface);
12
+ const isParticleAfter = (next, second, particles) => next !== undefined && next.pos === "ADP" && next.span.start === second.span.end && particles.has(next.surface);
13
+ const echoed = (token) => ({ ...token, features: { ...token.features, Echo: "Rdp" } });
14
+ export const markReduplication = (tokens, vocabulary) => tokens.map((token, index) => isSameNoun(tokens[index - 1], token, vocabulary.nouns) && isParticleAfter(tokens[index + 1], token, vocabulary.particles) ? echoed(token) : token);
15
+ //# sourceMappingURL=reduplication.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"reduplication.js","sourceRoot":"","sources":["../src/reduplication.ts"],"names":[],"mappings":"AAQA,MAAM,UAAU,GAAG,CAAC,OAA4B,EAAuB,EAAE,CAAC,IAAI,GAAG,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;AAEjI,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,QAA2C,EAAgB,EAAE,CAAC,CAAC;IACpG,KAAK,EAAE,UAAU,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC;IAChD,SAAS,EAAE,UAAU,CAAC,QAAQ,CAAC,uBAAuB,CAAC,CAAC;CACzD,CAAC,CAAC;AAEH,MAAM,UAAU,GAAG,CAAC,KAAwB,EAAE,MAAa,EAAE,KAA0B,EAAW,EAAE,CAClG,KAAK,KAAK,SAAS;IACnB,KAAK,CAAC,GAAG,KAAK,MAAM;IACpB,MAAM,CAAC,GAAG,KAAK,MAAM;IACrB,KAAK,CAAC,OAAO,KAAK,MAAM,CAAC,OAAO;IAChC,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,MAAM,CAAC,IAAI,CAAC,KAAK;IACpC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;AAE5B,MAAM,eAAe,GAAG,CAAC,IAAuB,EAAE,MAAa,EAAE,SAA8B,EAAW,EAAE,CAC1G,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,GAAG,KAAK,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,KAAK,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AAEjH,MAAM,MAAM,GAAG,CAAC,KAAY,EAAS,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,QAAQ,EAAE,EAAE,GAAG,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;AAErG,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,MAAwB,EAAE,UAAwB,EAAW,EAAE,CAC/F,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAC1B,UAAU,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,UAAU,CAAC,KAAK,CAAC,IAAI,eAAe,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAClJ,CAAC"}
@@ -0,0 +1,72 @@
1
+ # 読み手が説明なしで通じると見なしてよい略語。展開すると逆に読みにくい。
2
+ id: common-acronym
3
+ language: ja
4
+ entries:
5
+ - pattern: OK
6
+ - pattern: NG
7
+ - pattern: URL
8
+ - pattern: API
9
+ - pattern: CSS
10
+ - pattern: HTML
11
+ - pattern: JSON
12
+ - pattern: YAML
13
+ - pattern: HTTP
14
+ - pattern: HTTPS
15
+ - pattern: PDF
16
+ - pattern: CPU
17
+ - pattern: GPU
18
+ - pattern: RAM
19
+ - pattern: USB
20
+ - pattern: AI
21
+ - pattern: ID
22
+ - pattern: FAQ
23
+ - pattern: PR
24
+ - pattern: OS
25
+ - pattern: CLI
26
+ - pattern: UI
27
+ - pattern: UX
28
+ - pattern: SQL
29
+ - pattern: CSV
30
+ - pattern: XML
31
+ - pattern: TODO
32
+ - pattern: NOTE
33
+ - pattern: TIP
34
+ # 通信と符号化。技術文書でなくても説明なしで通じる。
35
+ - pattern: DNS
36
+ - pattern: IP
37
+ - pattern: TCP
38
+ - pattern: UDP
39
+ - pattern: SSH
40
+ - pattern: SSL
41
+ - pattern: TLS
42
+ - pattern: VPN
43
+ - pattern: LAN
44
+ - pattern: UTF
45
+ - pattern: ASCII
46
+ # 機器・形式・単位。
47
+ - pattern: PC
48
+ - pattern: IT
49
+ - pattern: SDK
50
+ - pattern: IDE
51
+ - pattern: PNG
52
+ - pattern: JPEG
53
+ - pattern: GIF
54
+ - pattern: SVG
55
+ - pattern: QR
56
+ - pattern: GPS
57
+ - pattern: SNS
58
+ - pattern: TV
59
+ - pattern: KB
60
+ - pattern: MB
61
+ - pattern: GB
62
+ - pattern: TB
63
+ # 役職と国・地域。
64
+ - pattern: CEO
65
+ - pattern: CTO
66
+ - pattern: CFO
67
+ - pattern: US
68
+ - pattern: USA
69
+ - pattern: UK
70
+ - pattern: EU
71
+ - pattern: UN
72
+ - pattern: DNA
@@ -0,0 +1,8 @@
1
+ # 最上級が何と比べているかを言う語。文にあれば「最も速い」は誇張ではなく主張になる。
2
+ # 動詞は原形で書く。「に比べる」は「に比べて」「に比べると」「に比べ、」に当たる。
3
+ id: comparison-marker
4
+ language: ja
5
+ entries:
6
+ - pattern: より
7
+ - pattern: に比べる
8
+ - pattern: のうち
@@ -0,0 +1,14 @@
1
+ # 更新日・公開日の刻印のラベル。これと日付だけの段落は、ページの付帯情報として本題までの段落に数えない。
2
+ id: date-stamp-label
3
+ language: ja
4
+ entries:
5
+ - pattern: 最終更新日
6
+ - pattern: 最終更新
7
+ - pattern: 更新日
8
+ - pattern: 公開日
9
+ - pattern: 掲載日
10
+ - pattern: 作成日
11
+ - pattern: 発行日
12
+ - pattern: 更新
13
+ - pattern: 公開
14
+ - pattern: 掲載
@@ -0,0 +1,6 @@
1
+ # 括弧の中で、略語の前に書いて定義を始める語(人事部(以下「HR」という。)、Human Resource(以下、HR))。
2
+ # 契約書・白書・社内の手引きのどれでも書く、ふつうの書き方。括弧の中が語と略語だけのときに限って、略語の説明と見なす。
3
+ id: definition-marker
4
+ language: ja
5
+ entries:
6
+ - pattern: 以下
@@ -0,0 +1,8 @@
1
+ # 括弧の中で、略語の後ろに書いて定義を閉じる語((以下「HR」という。))。
2
+ id: definition-verb
3
+ language: ja
4
+ entries:
5
+ - pattern: という
6
+ - pattern: と呼ぶ
7
+ - pattern: と称する
8
+ - pattern: と略す
@@ -0,0 +1,25 @@
1
+ # 重ねて「それぞれの」を言える名詞。まとまり(会社・部署・家庭)と、場合や時(場面・時代)を指すもの。
2
+ # 「会社会社で考え方が違う」「場面場面で使い分ける」。ここに無い名詞の重なり(資料資料の)は、doubled-word が書き損じとして数える。
3
+ id: distributive-noun
4
+ language: ja
5
+ entries:
6
+ - pattern: 会社
7
+ - pattern: 企業
8
+ - pattern: 組織
9
+ - pattern: 部署
10
+ - pattern: 部門
11
+ - pattern: チーム
12
+ - pattern: 店舗
13
+ - pattern: 現場
14
+ - pattern: 職場
15
+ - pattern: 学校
16
+ - pattern: 家庭
17
+ - pattern: 地域
18
+ - pattern: 国
19
+ - pattern: 個人
20
+ - pattern: 場面
21
+ - pattern: 場所
22
+ - pattern: 時代
23
+ - pattern: 時期
24
+ - pattern: 季節
25
+ - pattern: 案件
@@ -0,0 +1,11 @@
1
+ # 重ねた名詞(distributive-noun)の後ろに続いて「それぞれの」を言う助詞。「会社会社で」「部署部署の」「場面場面に」。
2
+ # ここに無い助詞が続く重なり(会社会社を)は、doubled-word が書き損じとして数える。
3
+ id: distributive-particle
4
+ language: ja
5
+ entries:
6
+ - pattern: で
7
+ - pattern: に
8
+ - pattern: の
9
+ - pattern: によって
10
+ - pattern: により
11
+ - pattern: による
@@ -0,0 +1,10 @@
1
+ # doubled-word が数えない重なり。形態素解析が一語にしない重ね言葉だけを置く。
2
+ # 「まだまだ」「ますます」「いろいろ」「時々」は一語になり、「そうそう」「もっともっと」は副詞なので、ここには要らない。
3
+ # 名詞を重ねて「それぞれの」を言う形(会社会社で)は、後ろの助詞でアダプタが見分ける(distributive-particle)。
4
+ id: doubled-word
5
+ language: ja
6
+ entries:
7
+ - pattern: 一つ一つ
8
+ - pattern: 見る見る
9
+ - pattern: 毎日毎日
10
+ - pattern: 毎回毎回
@@ -0,0 +1,10 @@
1
+ # 後ろの箇条書きや表へ読者を渡す言い回し。これを含む文は、見出しの語を繰り返していても heading-echo に数えない。
2
+ id: lead-in
3
+ language: ja
4
+ entries:
5
+ - pattern: 次のとおり
6
+ - pattern: 次の通り
7
+ - pattern: 以下のとおり
8
+ - pattern: 以下の通り
9
+ - pattern: 下記のとおり
10
+ - pattern: 下記の通り
@@ -0,0 +1,7 @@
1
+ # 最上級の範囲(何の中で最もなのか)を言う語。position は最上級のどちら側に置くか。
2
+ # 日本語は前に置く(日本で最も)。前の語が固有名詞か地名のときだけ範囲と読む。「最少の費用で最大の効果」の「費用で」は手段。
3
+ id: superlative-scope
4
+ language: ja
5
+ entries:
6
+ - pattern: で
7
+ position: before
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@chaffjs/lang-ja",
3
- "version": "0.10.0",
3
+ "version": "0.11.0",
4
4
  "description": "Japanese language adapter for chaff",
5
5
  "license": "MIT",
6
6
  "author": "isamu",
package/src/index.ts CHANGED
@@ -2,6 +2,7 @@ import { loadLexicons } from "./lexicons.ts";
2
2
  import { sentenceSpans } from "./sentence-split.ts";
3
3
  import { structure } from "./structure.ts";
4
4
  import { isReady, predicateOnly, prepare, tokenize } from "./pos.ts";
5
+ import { distributiveVocabulary, markReduplication } from "./reduplication.ts";
5
6
  import type { AdapterNeeds, LanguageAdapter, Segmentation, Sentence, Span } from "chaffjs/plugin";
6
7
 
7
8
  // chaff からは型だけを取る。実行時の値依存を作らない。アダプタは単体で動く。
@@ -46,13 +47,18 @@ const merge = (source: string, spans: readonly Span[]): Sentence[] =>
46
47
  * token の span は文ではなく、segment に渡した文字列を基準にする。文の span と同じ座標系。
47
48
  * 文ごとに解析して足し戻すのではなく、一度解析して文へ配る。同じ文字列を二度読まない。
48
49
  */
50
+ const DISTRIBUTIVE = distributiveVocabulary(loadLexicons());
51
+
49
52
  const withTokens = (source: string, sentences: readonly Sentence[]): Sentence[] => {
50
53
  const tokens = tokenize(source);
51
54
  if (tokens === undefined) return [...sentences];
52
55
  return sentences.map((sentence) => ({
53
56
  ...sentence,
54
57
  // 述語かどうかは文の中でしか決まらないので、文へ配ってから印を落とす。
55
- tokens: predicateOnly(tokens.filter((token) => token.span.start >= sentence.span.start && token.span.end <= sentence.span.end)),
58
+ tokens: markReduplication(
59
+ predicateOnly(tokens.filter((token) => token.span.start >= sentence.span.start && token.span.end <= sentence.span.end)),
60
+ DISTRIBUTIVE,
61
+ ),
56
62
  }));
57
63
  };
58
64
 
package/src/lexicons.ts CHANGED
@@ -8,6 +8,8 @@ const DIR = join(dirname(fileURLToPath(import.meta.url)), "..", "lexicons");
8
8
 
9
9
  const isRecord = (value: unknown): value is Record<string, unknown> => typeof value === "object" && value !== null && !Array.isArray(value);
10
10
 
11
+ const POSITIONS = ["before", "after"] as const;
12
+
11
13
  const toEntry = (raw: unknown): LexiconEntry | undefined => {
12
14
  if (!isRecord(raw) || typeof raw["pattern"] !== "string") return undefined;
13
15
  const weight = raw["weight"];
@@ -16,6 +18,7 @@ const toEntry = (raw: unknown): LexiconEntry | undefined => {
16
18
  pattern: raw["pattern"],
17
19
  weight: typeof weight === "number" ? weight : undefined,
18
20
  instead_of: typeof instead === "string" ? instead : undefined,
21
+ position: POSITIONS.find((position) => position === raw["position"]),
19
22
  };
20
23
  };
21
24
 
@@ -0,0 +1,32 @@
1
+ import type { Lexicon, Token } from "chaffjs/plugin";
2
+
3
+ /**
4
+ * 名詞を重ねて「それぞれの」を言う形(会社会社で、部署部署の)。重ねられるのは、まとまりや場合を指す名詞だけで、
5
+ * 後ろに助詞が続く。二つ目の名詞に UD の Echo=Rdp を付ける。「資料資料の」「確認確認する」は書き損じのまま。
6
+ */
7
+ export type Distributive = { readonly nouns: ReadonlySet<string>; readonly particles: ReadonlySet<string> };
8
+
9
+ const patternsOf = (lexicon: Lexicon | undefined): ReadonlySet<string> => new Set((lexicon ?? []).map((entry) => entry.pattern));
10
+
11
+ export const distributiveVocabulary = (lexicons: Readonly<Record<string, Lexicon>>): Distributive => ({
12
+ nouns: patternsOf(lexicons["distributive-noun"]),
13
+ particles: patternsOf(lexicons["distributive-particle"]),
14
+ });
15
+
16
+ const isSameNoun = (first: Token | undefined, second: Token, nouns: ReadonlySet<string>): boolean =>
17
+ first !== undefined &&
18
+ first.pos === "NOUN" &&
19
+ second.pos === "NOUN" &&
20
+ first.surface === second.surface &&
21
+ first.span.end === second.span.start &&
22
+ nouns.has(second.surface);
23
+
24
+ const isParticleAfter = (next: Token | undefined, second: Token, particles: ReadonlySet<string>): boolean =>
25
+ next !== undefined && next.pos === "ADP" && next.span.start === second.span.end && particles.has(next.surface);
26
+
27
+ const echoed = (token: Token): Token => ({ ...token, features: { ...token.features, Echo: "Rdp" } });
28
+
29
+ export const markReduplication = (tokens: readonly Token[], vocabulary: Distributive): Token[] =>
30
+ tokens.map((token, index) =>
31
+ isSameNoun(tokens[index - 1], token, vocabulary.nouns) && isParticleAfter(tokens[index + 1], token, vocabulary.particles) ? echoed(token) : token,
32
+ );