@chaffjs/lang-ja 0.0.1

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 isamu
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,12 @@
+# @chaffjs/lang-ja
+
+[chaff](https://www.npmjs.com/package/chaffjs) の公式日本語アダプタ。`chaff` に同梱されているので、単体で入れる必要はありません。
+
+持っているもの:
+
+- 文分割（日本語の文末は `。！？` に限る。`Dr.` のようなラテン略語で切らない）
+- L2 の語彙表（空虚な強調 / 水増しの導入 / 定型の結び）
+
+新しい言語を足すときは、このパッケージが雛形になります。公式は `@chaffjs/<言語>`、第三者は `chaff-lang-<言語>` と名乗ってください。
+
+MIT

package/dist/index.d.ts

@@ -0,0 +1,3 @@
+import type { LanguageAdapter } from "chaffjs/plugin";
+export declare const adapter: LanguageAdapter;
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAgC,MAAM,gBAAgB,CAAC;AAyCpF,eAAO,MAAM,OAAO,EAAE,eAmBrB,CAAC"}

package/dist/index.js

@@ -0,0 +1,55 @@
+import { split, SentenceSplitterSyntax } from "sentence-splitter";
+import { loadLexicons } from "./lexicons.js";
+// chaff からは型だけを取る。実行時の値依存を作らない。アダプタは単体で動く。
+/**
+ * 日本語の文末は「。！？」に限られる。
+ *
+ * sentence-splitter は "." も文末と見なすため、「、Dr. 田中は」で誤分割する。
+ * AbbrMarker の language を差し替えても直らない。原因は略語の保護ではなく、
+ * "." を終端と見なすこと自体にあるため。spec §7.2。
+ *
+ * 文長は sentence-rhythm と max-sentence-length の入力なので、
+ * 誤分割はそのまま指標を動かす。ここで閉じる。
+ */
+const CLOSED = /[。！？!?][")）」』]*\s*$/u;
+const JAPANESE = /[぀-ゟ゠-ヿ一-鿿]/gu;
+const COUNTABLE = /\S/gu;
+const isOpen = (text) => text.trim().length > 0 && !CLOSED.test(text);
+const rawSpans = (text) => split(text)
+    .filter((node) => node.type === SentenceSplitterSyntax.Sentence)
+    .map((node) => ({ start: node.range[0], end: node.range[1] }));
+/**
+ * 断片を繋ぐときは raw の連結ではなくオフセットを使う。
+ * sentence-splitter は空白を別ノードに分けるため、raw を繋ぐと空白が落ちて文長が縮む。
+ */
+const merge = (source, spans) => spans
+    .reduce((acc, span) => {
+    const last = acc.at(-1);
+    if (last !== undefined && isOpen(source.slice(last.start, last.end))) {
+        return [...acc.slice(0, -1), { start: last.start, end: span.end }];
+    }
+    return [...acc, span];
+}, [])
+    .map((span) => ({ span, text: source.slice(span.start, span.end) }));
+export const adapter = {
+    kind: "language",
+    id: "ja",
+    apiVersion: 1,
+    capabilities: {
+        sentenceSplit: true,
+        // budoux も形態素解析も、まだ繋いでいない。spec §16 の Tier 0。
+        wordSplit: false,
+        pos: false,
+        lemma: false,
+        lengthUnit: "char",
+    },
+    detect: (source) => {
+        const total = [...source.matchAll(COUNTABLE)].length;
+        if (total === 0)
+            return 0;
+        return [...source.matchAll(JAPANESE)].length / total;
+    },
+    lexicons: loadLexicons(),
+    segment: (text) => ({ sentences: merge(text, rawSpans(text)) }),
+};
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AAClE,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAG7C,2CAA2C;AAE3C;;;;;;;;;GASG;AACH,MAAM,MAAM,GAAG,sBAAsB,CAAC;AAEtC,MAAM,QAAQ,GAAG,eAAe,CAAC;AACjC,MAAM,SAAS,GAAG,MAAM,CAAC;AAEzB,MAAM,MAAM,GAAG,CAAC,IAAY,EAAW,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEvF,MAAM,QAAQ,GAAG,CAAC,IAAY,EAAU,EAAE,CACxC,KAAK,CAAC,IAAI,CAAC;KACR,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,sBAAsB,CAAC,QAAQ,CAAC;KAC/D,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAEnE;;;GAGG;AACH,MAAM,KAAK,GAAG,CAAC,MAAc,EAAE,KAAsB,EAAc,EAAE,CACnE,KAAK;KACF,MAAM,CAAS,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;IAC5B,MAAM,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACxB,IAAI,IAAI,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QACrE,OAAO,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IACrE,CAAC;IACD,OAAO,CAAC,GAAG,GAAG,EAAE,IAAI,CAAC,CAAC;AACxB,CAAC,EAAE,EAAE,CAAC;KACL,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;AAEzE,MAAM,CAAC,MAAM,OAAO,GAAoB;IACtC,IAAI,EAAE,UAAU;IAChB,EAAE,EAAE,IAAI;IACR,UAAU,EAAE,CAAC;IACb,YAAY,EAAE;QACZ,aAAa,EAAE,IAAI;QACnB,6CAA6C;QAC7C,SAAS,EAAE,KAAK;QAChB,GAAG,EAAE,KAAK;QACV,KAAK,EAAE,KAAK;QACZ,UAAU,EAAE,MAAM;KACnB;IACD,MAAM,EAAE,CAAC,MAAc,EAAU,EAAE;QACjC,MAAM,KAAK,GAAG,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC;QACrD,IAAI,KAAK,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAC1B,OAAO,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC;IACvD,CAAC;IACD,QAAQ,EAAE,YAAY,EAAE;IACxB,OAAO,EAAE,CAAC,IAAY,EAAgB,EAAE,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;CACtF,CAAC"}

package/dist/lexicons.d.ts

@@ -0,0 +1,7 @@
+import type { Lexicon } from "chaffjs/plugin";
+/**
+ * 語彙表はアダプタが持つ。detector は共通で、これだけが言語別。spec §11。
+ * 新しい言語のサポートは、ここを書くところから始まる。
+ */
+export declare const loadLexicons: (dir?: string) => Record<string, Lexicon>;
+//# sourceMappingURL=lexicons.d.ts.map

package/dist/lexicons.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"lexicons.d.ts","sourceRoot":"","sources":["../src/lexicons.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,OAAO,EAAgB,MAAM,gBAAgB,CAAC;AAY5D;;;GAGG;AACH,eAAO,MAAM,YAAY,GAAI,MAAK,MAAY,KAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAQ7D,CAAC"}

package/dist/lexicons.js

@@ -0,0 +1,26 @@
+import { readFileSync, readdirSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import { dirname, join } from "node:path";
+import { parse } from "yaml";
+const DIR = join(dirname(fileURLToPath(import.meta.url)), "..", "lexicons");
+const isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
+const toEntry = (raw) => {
+    if (!isRecord(raw) || typeof raw["pattern"] !== "string")
+        return undefined;
+    const weight = raw["weight"];
+    return { pattern: raw["pattern"], weight: typeof weight === "number" ? weight : undefined };
+};
+/**
+ * 語彙表はアダプタが持つ。detector は共通で、これだけが言語別。spec §11。
+ * 新しい言語のサポートは、ここを書くところから始まる。
+ */
+export const loadLexicons = (dir = DIR) => readdirSync(dir)
+    .filter((file) => file.endsWith(".yaml"))
+    .reduce((acc, file) => {
+    const raw = parse(readFileSync(join(dir, file), "utf8"));
+    if (!isRecord(raw) || typeof raw["id"] !== "string" || !Array.isArray(raw["entries"]))
+        return acc;
+    const entries = raw["entries"].map(toEntry).filter((entry) => entry !== undefined);
+    return { ...acc, [raw["id"]]: entries };
+}, {});
+//# sourceMappingURL=lexicons.js.map

package/dist/lexicons.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"lexicons.js","sourceRoot":"","sources":["../src/lexicons.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,KAAK,EAAE,MAAM,MAAM,CAAC;AAG7B,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;AAE5E,MAAM,QAAQ,GAAG,CAAC,KAAc,EAAoC,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAE5I,MAAM,OAAO,GAAG,CAAC,GAAY,EAA4B,EAAE;IACzD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,OAAO,GAAG,CAAC,SAAS,CAAC,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC;IAC3E,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC7B,OAAO,EAAE,OAAO,EAAE,GAAG,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;AAC9F,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,MAAc,GAAG,EAA2B,EAAE,CACzE,WAAW,CAAC,GAAG,CAAC;KACb,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;KACxC,MAAM,CAA0B,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;IAC7C,MAAM,GAAG,GAAY,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;IAClE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,OAAO,GAAG,CAAC,IAAI,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAAE,OAAO,GAAG,CAAC;IAClG,MAAM,OAAO,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;IACnF,OAAO,EAAE,GAAG,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC;AAC1C,CAAC,EAAE,EAAE,CAAC,CAAC"}

package/lexicons/closing-cliche.yaml

@@ -0,0 +1,10 @@
+# 定型の結び。読者に何も残さない締め。
+id: closing-cliche
+language: ja
+entries:
+  - pattern: いかがでしたか
+  - pattern: いかがでしょうか
+  - pattern: まとめると
+  - pattern: 最後までお読みいただき
+  - pattern: 参考になれば幸いです
+  - pattern: ぜひ試してみてください

package/lexicons/empty-intensifier.yaml

@@ -0,0 +1,13 @@
+# 空虚な強調。程度を言っているようで、何も言っていない表現。
+id: empty-intensifier
+language: ja
+entries:
+  - pattern: 非常に重要
+  - pattern: 極めて重要
+  - pattern: きわめて重要
+  - pattern: 大変重要
+  - pattern: とても重要
+  - pattern: 最も重要なのは
+  - pattern: 言うまでもなく
+  - pattern: 不可欠です
+  - pattern: 欠かせません

package/lexicons/padded-intro.yaml

@@ -0,0 +1,12 @@
+# 水増しの導入。どの記事にも当てはまる書き出し。
+id: padded-intro
+language: ja
+entries:
+  - pattern: 近年
+  - pattern: 昨今
+  - pattern: 注目されています
+  - pattern: 注目を集めています
+  - pattern: 重要性が高まっています
+  - pattern: 方も多いのではないでしょうか
+  - pattern: ではないでしょうか
+  - pattern: 皆さんは

package/package.json

@@ -0,0 +1,41 @@
+{
+  "name": "@chaffjs/lang-ja",
+  "version": "0.0.1",
+  "description": "Japanese language adapter for chaff",
+  "license": "MIT",
+  "author": "isamu",
+  "type": "module",
+  "exports": {
+    ".": {
+      "types": "./src/index.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "files": [
+    "LICENSE",
+    "README.md",
+    "dist",
+    "lexicons",
+    "src"
+  ],
+  "engines": {
+    "node": ">=24"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.build.json",
+    "prepack": "yarn build"
+  },
+  "dependencies": {
+    "sentence-splitter": "^5.0.1",
+    "yaml": "^2.9.0"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/isamu/lab.git",
+    "directory": "text/packages/lang-ja"
+  },
+  "homepage": "https://github.com/isamu/lab/tree/main/text",
+  "publishConfig": {
+    "access": "public"
+  }
+}

package/src/index.ts

@@ -0,0 +1,63 @@
+import { split, SentenceSplitterSyntax } from "sentence-splitter";
+import { loadLexicons } from "./lexicons.ts";
+import type { LanguageAdapter, Segmentation, Sentence, Span } from "chaffjs/plugin";
+
+// chaff からは型だけを取る。実行時の値依存を作らない。アダプタは単体で動く。
+
+/**
+ * 日本語の文末は「。！？」に限られる。
+ *
+ * sentence-splitter は "." も文末と見なすため、「、Dr. 田中は」で誤分割する。
+ * AbbrMarker の language を差し替えても直らない。原因は略語の保護ではなく、
+ * "." を終端と見なすこと自体にあるため。spec §7.2。
+ *
+ * 文長は sentence-rhythm と max-sentence-length の入力なので、
+ * 誤分割はそのまま指標を動かす。ここで閉じる。
+ */
+const CLOSED = /[。！？!?][")）」』]*\s*$/u;
+
+const JAPANESE = /[぀-ゟ゠-ヿ一-鿿]/gu;
+const COUNTABLE = /\S/gu;
+
+const isOpen = (text: string): boolean => text.trim().length > 0 && !CLOSED.test(text);
+
+const rawSpans = (text: string): Span[] =>
+  split(text)
+    .filter((node) => node.type === SentenceSplitterSyntax.Sentence)
+    .map((node) => ({ start: node.range[0], end: node.range[1] }));
+
+/**
+ * 断片を繋ぐときは raw の連結ではなくオフセットを使う。
+ * sentence-splitter は空白を別ノードに分けるため、raw を繋ぐと空白が落ちて文長が縮む。
+ */
+const merge = (source: string, spans: readonly Span[]): Sentence[] =>
+  spans
+    .reduce<Span[]>((acc, span) => {
+      const last = acc.at(-1);
+      if (last !== undefined && isOpen(source.slice(last.start, last.end))) {
+        return [...acc.slice(0, -1), { start: last.start, end: span.end }];
+      }
+      return [...acc, span];
+    }, [])
+    .map((span) => ({ span, text: source.slice(span.start, span.end) }));
+
+export const adapter: LanguageAdapter = {
+  kind: "language",
+  id: "ja",
+  apiVersion: 1,
+  capabilities: {
+    sentenceSplit: true,
+    // budoux も形態素解析も、まだ繋いでいない。spec §16 の Tier 0。
+    wordSplit: false,
+    pos: false,
+    lemma: false,
+    lengthUnit: "char",
+  },
+  detect: (source: string): number => {
+    const total = [...source.matchAll(COUNTABLE)].length;
+    if (total === 0) return 0;
+    return [...source.matchAll(JAPANESE)].length / total;
+  },
+  lexicons: loadLexicons(),
+  segment: (text: string): Segmentation => ({ sentences: merge(text, rawSpans(text)) }),
+};

package/src/lexicons.ts

@@ -0,0 +1,29 @@
+import { readFileSync, readdirSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import { dirname, join } from "node:path";
+import { parse } from "yaml";
+import type { Lexicon, LexiconEntry } from "chaffjs/plugin";
+
+const DIR = join(dirname(fileURLToPath(import.meta.url)), "..", "lexicons");
+
+const isRecord = (value: unknown): value is Record<string, unknown> => typeof value === "object" && value !== null && !Array.isArray(value);
+
+const toEntry = (raw: unknown): LexiconEntry | undefined => {
+  if (!isRecord(raw) || typeof raw["pattern"] !== "string") return undefined;
+  const weight = raw["weight"];
+  return { pattern: raw["pattern"], weight: typeof weight === "number" ? weight : undefined };
+};
+
+/**
+ * 語彙表はアダプタが持つ。detector は共通で、これだけが言語別。spec §11。
+ * 新しい言語のサポートは、ここを書くところから始まる。
+ */
+export const loadLexicons = (dir: string = DIR): Record<string, Lexicon> =>
+  readdirSync(dir)
+    .filter((file) => file.endsWith(".yaml"))
+    .reduce<Record<string, Lexicon>>((acc, file) => {
+      const raw: unknown = parse(readFileSync(join(dir, file), "utf8"));
+      if (!isRecord(raw) || typeof raw["id"] !== "string" || !Array.isArray(raw["entries"])) return acc;
+      const entries = raw["entries"].map(toEntry).filter((entry) => entry !== undefined);
+      return { ...acc, [raw["id"]]: entries };
+    }, {});