@chaffjs/lang-en 0.1.0 → 0.3.0

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAgB,MAAM,gBAAgB,CAAC;AAOpE;;;GAGG;AACH,eAAO,MAAM,OAAO,EAAE,eAsBrB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAgB,eAAe,EAA0B,MAAM,gBAAgB,CAAC;AAoB5F;;;GAGG;AACH,eAAO,MAAM,OAAO,EAAE,eA6BrB,CAAC"}

package/dist/index.js

@@ -1,8 +1,22 @@
 import { split, SentenceSplitterSyntax } from "sentence-splitter";
 import { loadLexicons } from "./lexicons.js";
+import { isReady, prepare, tokenize } from "./pos.js";
 // chaff からは型だけを取る。実行時の値依存を作らない。アダプタは単体で動く。
 const LATIN_LETTER = /[a-z]/giu;
 const COUNTABLE = /\S/gu;
+/**
+ * token の span は文ではなく、segment に渡した文字列を基準にする。文の span と同じ座標系。
+ * wink は位置を返さないため、文ごとに解析してから文の開始位置を足す。
+ */
+const withTokens = (sentence) => {
+    const tokens = tokenize(sentence.text);
+    if (tokens === undefined)
+        return sentence;
+    return {
+        ...sentence,
+        tokens: tokens.map((token) => ({ ...token, span: { start: sentence.span.start + token.span.start, end: sentence.span.start + token.span.end } })),
+    };
+};
 /**
  * 英語は sentence-splitter の既定で足りる。"Dr." "e.g." "U.S." "$3.50" を
  * いずれも文末と誤認しない。日本語のような後処理は要らない。spec §7.2。
@@ -14,10 +28,17 @@ export const adapter = {
     capabilities: {
         sentenceSplit: true,
         wordSplit: true,
-        pos: false,
-        lemma: false,
+        // 「払えばできる」の宣言。実際に払うのは prepare。英語は 130 ms ほど。
+        pos: true,
+        lemma: true,
         lengthUnit: "word",
     },
+    prepare: (need) => {
+        // 英語の解析器は同期に読める。契約は Promise なので、そこだけ合わせる。
+        if (need.pos)
+            prepare();
+        return Promise.resolve();
+    },
     detect: (source) => {
         const total = [...source.matchAll(COUNTABLE)].length;
         if (total === 0)
@@ -25,10 +46,11 @@ export const adapter = {
         return [...source.matchAll(LATIN_LETTER)].length / total;
     },
     lexicons: loadLexicons(),
-    segment: (text) => ({
-        sentences: split(text)
+    segment: (text) => {
+        const sentences = split(text)
             .filter((node) => node.type === SentenceSplitterSyntax.Sentence)
-            .map((node) => ({ span: { start: node.range[0], end: node.range[1] }, text: text.slice(node.range[0], node.range[1]) })),
-    }),
+            .map((node) => ({ span: { start: node.range[0], end: node.range[1] }, text: text.slice(node.range[0], node.range[1]) }));
+        return { sentences: isReady() ? sentences.map(withTokens) : sentences };
+    },
 };
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AAClE,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAG7C,2CAA2C;AAE3C,MAAM,YAAY,GAAG,UAAU,CAAC;AAChC,MAAM,SAAS,GAAG,MAAM,CAAC;AAEzB;;;GAGG;AACH,MAAM,CAAC,MAAM,OAAO,GAAoB;IACtC,IAAI,EAAE,UAAU;IAChB,EAAE,EAAE,IAAI;IACR,UAAU,EAAE,CAAC;IACb,YAAY,EAAE;QACZ,aAAa,EAAE,IAAI;QACnB,SAAS,EAAE,IAAI;QACf,GAAG,EAAE,KAAK;QACV,KAAK,EAAE,KAAK;QACZ,UAAU,EAAE,MAAM;KACnB;IACD,MAAM,EAAE,CAAC,MAAc,EAAU,EAAE;QACjC,MAAM,KAAK,GAAG,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC;QACrD,IAAI,KAAK,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAC1B,OAAO,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC;IAC3D,CAAC;IACD,QAAQ,EAAE,YAAY,EAAE;IACxB,OAAO,EAAE,CAAC,IAAY,EAAgB,EAAE,CAAC,CAAC;QACxC,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC;aACnB,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,sBAAsB,CAAC,QAAQ,CAAC;aAC/D,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;KAC3H,CAAC;CACH,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AAClE,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAGtD,2CAA2C;AAE3C,MAAM,YAAY,GAAG,UAAU,CAAC;AAChC,MAAM,SAAS,GAAG,MAAM,CAAC;AAEzB;;;GAGG;AACH,MAAM,UAAU,GAAG,CAAC,QAAkB,EAAY,EAAE;IAClD,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IACvC,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC;IAC1C,OAAO;QACL,GAAG,QAAQ;QACX,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;KAClJ,CAAC;AACJ,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,OAAO,GAAoB;IACtC,IAAI,EAAE,UAAU;IAChB,EAAE,EAAE,IAAI;IACR,UAAU,EAAE,CAAC;IACb,YAAY,EAAE;QACZ,aAAa,EAAE,IAAI;QACnB,SAAS,EAAE,IAAI;QACf,6CAA6C;QAC7C,GAAG,EAAE,IAAI;QACT,KAAK,EAAE,IAAI;QACX,UAAU,EAAE,MAAM;KACnB;IACD,OAAO,EAAE,CAAC,IAAkB,EAAiB,EAAE;QAC7C,0CAA0C;QAC1C,IAAI,IAAI,CAAC,GAAG;YAAE,OAAO,EAAE,CAAC;QACxB,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;IACD,MAAM,EAAE,CAAC,MAAc,EAAU,EAAE;QACjC,MAAM,KAAK,GAAG,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC;QACrD,IAAI,KAAK,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAC1B,OAAO,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC;IAC3D,CAAC;IACD,QAAQ,EAAE,YAAY,EAAE;IACxB,OAAO,EAAE,CAAC,IAAY,EAAgB,EAAE;QACtC,MAAM,SAAS,GAAe,KAAK,CAAC,IAAI,CAAC;aACtC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,sBAAsB,CAAC,QAAQ,CAAC;aAC/D,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC3H,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;IAC1E,CAAC;CACF,CAAC"}

package/dist/pos.d.ts

@@ -0,0 +1,6 @@
+import type { Token } from "chaffjs/plugin";
+export declare const upos: (tag: string) => string;
+export declare const prepare: () => void;
+export declare const isReady: () => boolean;
+export declare const tokenize: (text: string) => Token[] | undefined;
+//# sourceMappingURL=pos.d.ts.map

package/dist/pos.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pos.d.ts","sourceRoot":"","sources":["../src/pos.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AA4E5C,eAAO,MAAM,IAAI,GAAI,KAAK,MAAM,KAAG,MAAiE,CAAC;AAqBrG,eAAO,MAAM,OAAO,QAAO,IAE1B,CAAC;AAEF,eAAO,MAAM,OAAO,QAAO,OAAoC,CAAC;AAwBhE,eAAO,MAAM,QAAQ,GAAI,MAAM,MAAM,KAAG,KAAK,EAAE,GAAG,SAIjD,CAAC"}

package/dist/pos.js

@@ -0,0 +1,109 @@
+import { createRequire } from "node:module";
+const require = createRequire(import.meta.url);
+const isRecord = (value) => typeof value === "object" && value !== null;
+const isCallable = (value) => typeof value === "function";
+const isTagged = (value) => isRecord(value) && typeof value["value"] === "string" && typeof value["pos"] === "string";
+const toArray = (value) => (Array.isArray(value) ? value : []);
+/** 取り出した関数をそのまま呼ぶと receiver が外れる。wink の tagSentence は自分の状態を読む。 */
+const callMethod = (owner, name, args) => {
+    const method = owner[name];
+    if (!isCallable(method))
+        throw new Error(`wink-pos-tagger が ${name} を持っていません`);
+    return Reflect.apply(method, owner, args);
+};
+const build = () => {
+    const factory = require("wink-pos-tagger");
+    if (!isCallable(factory))
+        throw new Error("wink-pos-tagger が関数を export していません");
+    const tagger = factory();
+    if (!isRecord(tagger))
+        throw new Error("wink-pos-tagger が object を返しませんでした");
+    return tagger;
+};
+/**
+ * Penn Treebank を UPOS に寄せる。detector にアダプタ固有の体系を見せない。spec §6。
+ * 引けなかったものは X。誤った品詞を当てるより、分からないと言うほうがまし。
+ */
+const BY_TAG = {
+    CC: "CCONJ",
+    CD: "NUM",
+    DT: "DET",
+    EX: "PRON",
+    FW: "X",
+    IN: "ADP",
+    JJ: "ADJ",
+    JJR: "ADJ",
+    JJS: "ADJ",
+    MD: "AUX",
+    NN: "NOUN",
+    NNS: "NOUN",
+    NNP: "PROPN",
+    NNPS: "PROPN",
+    PDT: "DET",
+    POS: "PART",
+    PRP: "PRON",
+    PRP$: "PRON",
+    RB: "ADV",
+    RBR: "ADV",
+    RBS: "ADV",
+    RP: "ADP",
+    SYM: "SYM",
+    TO: "PART",
+    UH: "INTJ",
+    VB: "VERB",
+    VBD: "VERB",
+    VBG: "VERB",
+    VBN: "VERB",
+    VBP: "VERB",
+    VBZ: "VERB",
+    WDT: "DET",
+    WP: "PRON",
+    WP$: "PRON",
+    WRB: "ADV",
+};
+const PUNCTUATION = new Set([".", ",", ":", "(", ")", "``", "''", "#", "$"]);
+export const upos = (tag) => (PUNCTUATION.has(tag) ? "PUNCT" : (BY_TAG[tag] ?? "X"));
+/**
+ * 受動は be + 過去分詞。過去分詞だけでは完了形（has reviewed）と見分けられないので、
+ * 直前の be を見る。間に副詞が挟まる（was quickly approved）ぶんだけ遡る。
+ */
+const ADVERB_TAGS = new Set(["RB", "RBR", "RBS"]);
+const BE = new Set(["be", "am", "is", "are", "was", "were", "been", "being"]);
+const isBe = (entry) => BE.has(entry.lemma ?? entry.value.toLowerCase()) || BE.has(entry.value.toLowerCase());
+const isPassive = (tagged, at) => {
+    if (tagged[at]?.pos !== "VBN")
+        return false;
+    const before = tagged.slice(0, at).reverse();
+    const head = before.find((entry) => !ADVERB_TAGS.has(entry.pos));
+    return head !== undefined && isBe(head);
+};
+const state = { ready: undefined };
+export const prepare = () => {
+    state.ready ??= build();
+};
+export const isReady = () => state.ready !== undefined;
+/**
+ * wink は位置を返さないので、表層を順に照合して復元する。
+ * 見つからないものは飛ばし、カーソルは進めない。位置の当てずっぽうを下流に流さない。
+ */
+const locate = (text, tagged) => tagged.reduce((acc, entry, at) => {
+    const start = text.indexOf(entry.value, acc.cursor);
+    if (start === -1)
+        return acc;
+    const end = start + entry.value.length;
+    const token = {
+        span: { start, end },
+        surface: entry.value,
+        pos: upos(entry.pos),
+        ...(entry.lemma === undefined ? {} : { lemma: entry.lemma }),
+        ...(isPassive(tagged, at) ? { features: { Voice: "Pass" } } : {}),
+    };
+    return { tokens: [...acc.tokens, token], cursor: end };
+}, { tokens: [], cursor: 0 }).tokens;
+export const tokenize = (text) => {
+    const tagger = state.ready;
+    if (tagger === undefined)
+        return undefined;
+    return locate(text, toArray(callMethod(tagger, "tagSentence", [text])).filter(isTagged));
+};
+//# sourceMappingURL=pos.js.map

package/dist/pos.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pos.js","sourceRoot":"","sources":["../src/pos.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAG5C,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAO/C,MAAM,QAAQ,GAAG,CAAC,KAAc,EAAoC,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,CAAC;AAEnH,MAAM,UAAU,GAAG,CAAC,KAAc,EAAqD,EAAE,CAAC,OAAO,KAAK,KAAK,UAAU,CAAC;AAEtH,MAAM,QAAQ,GAAG,CAAC,KAAc,EAAmB,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,KAAK,CAAC,KAAK,CAAC,KAAK,QAAQ,CAAC;AAEhJ,MAAM,OAAO,GAAG,CAAC,KAAc,EAAsB,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AAE5F,kEAAkE;AAClE,MAAM,UAAU,GAAG,CAAC,KAA8B,EAAE,IAAY,EAAE,IAAwB,EAAW,EAAE;IACrG,MAAM,MAAM,GAAY,KAAK,CAAC,IAAI,CAAC,CAAC;IACpC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,qBAAqB,IAAI,WAAW,CAAC,CAAC;IAC/E,OAAO,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;AAC5C,CAAC,CAAC;AAEF,MAAM,KAAK,GAAG,GAAW,EAAE;IACzB,MAAM,OAAO,GAAY,OAAO,CAAC,iBAAiB,CAAC,CAAC;IACpD,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;IAChF,MAAM,MAAM,GAAY,OAAO,EAAE,CAAC;IAClC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;IAC7E,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,MAAM,GAAqC;IAC/C,EAAE,EAAE,OAAO;IACX,EAAE,EAAE,KAAK;IACT,EAAE,EAAE,KAAK;IACT,EAAE,EAAE,MAAM;IACV,EAAE,EAAE,GAAG;IACP,EAAE,EAAE,KAAK;IACT,EAAE,EAAE,KAAK;IACT,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,EAAE,EAAE,KAAK;IACT,EAAE,EAAE,MAAM;IACV,GAAG,EAAE,MAAM;IACX,GAAG,EAAE,OAAO;IACZ,IAAI,EAAE,OAAO;IACb,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,MAAM;IACX,GAAG,EAAE,MAAM;IACX,IAAI,EAAE,MAAM;IACZ,EAAE,EAAE,KAAK;IACT,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,EAAE,EAAE,KAAK;IACT,GAAG,EAAE,KAAK;IACV,EAAE,EAAE,MAAM;IACV,EAAE,EAAE,MAAM;IACV,EAAE,EAAE,MAAM;IACV,GAAG,EAAE,MAAM;IACX,GAAG,EAAE,MAAM;IACX,GAAG,EAAE,MAAM;IACX,GAAG,EAAE,MAAM;IACX,GAAG,EAAE,MAAM;IACX,GAAG,EAAE,KAAK;IACV,EAAE,EAAE,MAAM;IACV,GAAG,EAAE,MAAM;IACX,GAAG,EAAE,KAAK;CACX,CAAC;AAEF,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAE7E,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,GAAW,EAAU,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;AAErG;;;GAGG;AACH,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;AAElD,MAAM,EAAE,GAAG,IAAI,GAAG,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;AAE9E,MAAM,IAAI,GAAG,CAAC,KAAa,EAAW,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;AAE/H,MAAM,SAAS,GAAG,CAAC,MAAyB,EAAE,EAAU,EAAW,EAAE;IACnE,IAAI,MAAM,CAAC,EAAE,CAAC,EAAE,GAAG,KAAK,KAAK;QAAE,OAAO,KAAK,CAAC;IAC5C,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;IAC7C,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;IACjE,OAAO,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1C,CAAC,CAAC;AAEF,MAAM,KAAK,GAAkC,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;AAElE,MAAM,CAAC,MAAM,OAAO,GAAG,GAAS,EAAE;IAChC,KAAK,CAAC,KAAK,KAAK,KAAK,EAAE,CAAC;AAC1B,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,OAAO,GAAG,GAAY,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC;AAEhE;;;GAGG;AACH,MAAM,MAAM,GAAG,CAAC,IAAY,EAAE,MAAyB,EAAW,EAAE,CAClE,MAAM,CAAC,MAAM,CACX,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE;IACjB,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACpD,IAAI,KAAK,KAAK,CAAC,CAAC;QAAE,OAAO,GAAG,CAAC;IAC7B,MAAM,GAAG,GAAG,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC;IACvC,MAAM,KAAK,GAAG;QACZ,IAAI,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE;QACpB,OAAO,EAAE,KAAK,CAAC,KAAK;QACpB,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC;QACpB,GAAG,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC;QAC5D,GAAG,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAClE,CAAC;IACF,OAAO,EAAE,MAAM,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;AACzD,CAAC,EACD,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,CAC1B,CAAC,MAAM,CAAC;AAEX,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,IAAY,EAAuB,EAAE;IAC5D,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC;IAC3B,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC3C,OAAO,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,MAAM,EAAE,aAAa,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;AAC3F,CAAC,CAAC"}

package/lexicons/agent-marker.yaml

@@ -0,0 +1,5 @@
+# Words that name the actor. A passive carrying one of these says who did it.
+id: agent-marker
+language: en
+entries:
+  - pattern: " by "

package/lexicons/ai-tell.yaml

@@ -0,0 +1,21 @@
+# Signals of generated text. One says nothing, so the weights are summed.
+# Never a verdict on its own. spec §20.2.
+id: ai-tell
+language: en
+entries:
+  - pattern: delve into
+    weight: 0.9
+  - pattern: in today's fast-paced world
+    weight: 0.9
+  - pattern: tapestry
+    weight: 0.6
+  - pattern: it's not just
+    weight: 0.7
+  - pattern: plays a crucial role
+    weight: 0.8
+  - pattern: navigate the complexities
+    weight: 0.8
+  - pattern: a testament to
+    weight: 0.7
+  - pattern: unlock the potential
+    weight: 0.8

package/lexicons/cushion-phrase.yaml

@@ -0,0 +1,11 @@
+# Softeners. One is courtesy. Several push the point further away.
+id: cushion-phrase
+language: en
+entries:
+  - pattern: if you don't mind
+  - pattern: sorry to bother
+  - pattern: just wanted to
+  - pattern: i was wondering if
+  - pattern: when you get a chance
+  - pattern: no rush but
+  - pattern: apologies for

package/lexicons/excessive-hedging.yaml

@@ -0,0 +1,12 @@
+# Hedges. Avoiding a claim leaves the reader unsure what was decided.
+id: excessive-hedging
+language: en
+entries:
+  - pattern: it may be
+  - pattern: it could be
+  - pattern: it seems
+  - pattern: arguably
+  - pattern: to some extent
+  - pattern: in some cases
+  - pattern: it is possible that
+  - pattern: one might argue

package/lexicons/paragraph-opener.yaml

@@ -0,0 +1,12 @@
+# Conjunctions that open a paragraph. A run of them makes every paragraph an afterthought.
+id: paragraph-opener
+language: en
+entries:
+  - pattern: also
+  - pattern: moreover
+  - pattern: furthermore
+  - pattern: however
+  - pattern: additionally
+  - pattern: in addition
+  - pattern: that said
+  - pattern: on the other hand

package/lexicons/sentence-opener.yaml

@@ -0,0 +1,10 @@
+# Conjunctions that open a sentence. One is fine; a run of them makes every
+# sentence read as an afterthought to the last.
+id: sentence-opener
+language: en
+entries:
+  - pattern: And
+  - pattern: But
+  - pattern: So
+  - pattern: Yet
+  - pattern: Also

package/lexicons/superlative.yaml

@@ -0,0 +1,11 @@
+# Superlatives. They need something to be superlative against.
+id: superlative
+language: en
+entries:
+  - pattern: the best
+  - pattern: the fastest
+  - pattern: the most
+  - pattern: world-first
+  - pattern: industry-leading
+  - pattern: unmatched
+  - pattern: unparalleled

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chaffjs/lang-en",
-  "version": "0.1.0",
+  "version": "0.3.0",
   "description": "English language adapter for chaff",
   "license": "MIT",
   "author": "isamu",
@@ -27,6 +27,7 @@
   },
   "dependencies": {
     "sentence-splitter": "^5.0.1",
+    "wink-pos-tagger": "^2.2.2",
     "yaml": "^2.9.0"
   },
   "repository": {

package/src/index.ts

@@ -1,12 +1,26 @@
 import { split, SentenceSplitterSyntax } from "sentence-splitter";
 import { loadLexicons } from "./lexicons.ts";
-import type { LanguageAdapter, Segmentation } from "chaffjs/plugin";
+import { isReady, prepare, tokenize } from "./pos.ts";
+import type { AdapterNeeds, LanguageAdapter, Segmentation, Sentence } from "chaffjs/plugin";
 
 // chaff からは型だけを取る。実行時の値依存を作らない。アダプタは単体で動く。
 
 const LATIN_LETTER = /[a-z]/giu;
 const COUNTABLE = /\S/gu;
 
+/**
+ * token の span は文ではなく、segment に渡した文字列を基準にする。文の span と同じ座標系。
+ * wink は位置を返さないため、文ごとに解析してから文の開始位置を足す。
+ */
+const withTokens = (sentence: Sentence): Sentence => {
+  const tokens = tokenize(sentence.text);
+  if (tokens === undefined) return sentence;
+  return {
+    ...sentence,
+    tokens: tokens.map((token) => ({ ...token, span: { start: sentence.span.start + token.span.start, end: sentence.span.start + token.span.end } })),
+  };
+};
+
 /**
  * 英語は sentence-splitter の既定で足りる。"Dr." "e.g." "U.S." "$3.50" を
  * いずれも文末と誤認しない。日本語のような後処理は要らない。spec §7.2。
@@ -18,19 +32,26 @@ export const adapter: LanguageAdapter = {
   capabilities: {
     sentenceSplit: true,
     wordSplit: true,
-    pos: false,
-    lemma: false,
+    // 「払えばできる」の宣言。実際に払うのは prepare。英語は 130 ms ほど。
+    pos: true,
+    lemma: true,
     lengthUnit: "word",
   },
+  prepare: (need: AdapterNeeds): Promise<void> => {
+    // 英語の解析器は同期に読める。契約は Promise なので、そこだけ合わせる。
+    if (need.pos) prepare();
+    return Promise.resolve();
+  },
   detect: (source: string): number => {
     const total = [...source.matchAll(COUNTABLE)].length;
     if (total === 0) return 0;
     return [...source.matchAll(LATIN_LETTER)].length / total;
   },
   lexicons: loadLexicons(),
-  segment: (text: string): Segmentation => ({
-    sentences: split(text)
+  segment: (text: string): Segmentation => {
+    const sentences: Sentence[] = split(text)
       .filter((node) => node.type === SentenceSplitterSyntax.Sentence)
-      .map((node) => ({ span: { start: node.range[0], end: node.range[1] }, text: text.slice(node.range[0], node.range[1]) })),
-  }),
+      .map((node) => ({ span: { start: node.range[0], end: node.range[1] }, text: text.slice(node.range[0], node.range[1]) }));
+    return { sentences: isReady() ? sentences.map(withTokens) : sentences };
+  },
 };

package/src/pos.ts

@@ -0,0 +1,131 @@
+import { createRequire } from "node:module";
+import type { Token } from "chaffjs/plugin";
+
+const require = createRequire(import.meta.url);
+
+/** wink は CommonJS で、辞書を同期に持つ。初期化は 130 ms ほど。 */
+type Tagged = { readonly value: string; readonly pos: string; readonly lemma?: string };
+
+type Tagger = Record<string, unknown>;
+
+const isRecord = (value: unknown): value is Record<string, unknown> => typeof value === "object" && value !== null;
+
+const isCallable = (value: unknown): value is (...args: readonly unknown[]) => unknown => typeof value === "function";
+
+const isTagged = (value: unknown): value is Tagged => isRecord(value) && typeof value["value"] === "string" && typeof value["pos"] === "string";
+
+const toArray = (value: unknown): readonly unknown[] => (Array.isArray(value) ? value : []);
+
+/** 取り出した関数をそのまま呼ぶと receiver が外れる。wink の tagSentence は自分の状態を読む。 */
+const callMethod = (owner: Record<string, unknown>, name: string, args: readonly unknown[]): unknown => {
+  const method: unknown = owner[name];
+  if (!isCallable(method)) throw new Error(`wink-pos-tagger が ${name} を持っていません`);
+  return Reflect.apply(method, owner, args);
+};
+
+const build = (): Tagger => {
+  const factory: unknown = require("wink-pos-tagger");
+  if (!isCallable(factory)) throw new Error("wink-pos-tagger が関数を export していません");
+  const tagger: unknown = factory();
+  if (!isRecord(tagger)) throw new Error("wink-pos-tagger が object を返しませんでした");
+  return tagger;
+};
+
+/**
+ * Penn Treebank を UPOS に寄せる。detector にアダプタ固有の体系を見せない。spec §6。
+ * 引けなかったものは X。誤った品詞を当てるより、分からないと言うほうがまし。
+ */
+const BY_TAG: Readonly<Record<string, string>> = {
+  CC: "CCONJ",
+  CD: "NUM",
+  DT: "DET",
+  EX: "PRON",
+  FW: "X",
+  IN: "ADP",
+  JJ: "ADJ",
+  JJR: "ADJ",
+  JJS: "ADJ",
+  MD: "AUX",
+  NN: "NOUN",
+  NNS: "NOUN",
+  NNP: "PROPN",
+  NNPS: "PROPN",
+  PDT: "DET",
+  POS: "PART",
+  PRP: "PRON",
+  PRP$: "PRON",
+  RB: "ADV",
+  RBR: "ADV",
+  RBS: "ADV",
+  RP: "ADP",
+  SYM: "SYM",
+  TO: "PART",
+  UH: "INTJ",
+  VB: "VERB",
+  VBD: "VERB",
+  VBG: "VERB",
+  VBN: "VERB",
+  VBP: "VERB",
+  VBZ: "VERB",
+  WDT: "DET",
+  WP: "PRON",
+  WP$: "PRON",
+  WRB: "ADV",
+};
+
+const PUNCTUATION = new Set([".", ",", ":", "(", ")", "``", "''", "#", "$"]);
+
+export const upos = (tag: string): string => (PUNCTUATION.has(tag) ? "PUNCT" : (BY_TAG[tag] ?? "X"));
+
+/**
+ * 受動は be + 過去分詞。過去分詞だけでは完了形（has reviewed）と見分けられないので、
+ * 直前の be を見る。間に副詞が挟まる（was quickly approved）ぶんだけ遡る。
+ */
+const ADVERB_TAGS = new Set(["RB", "RBR", "RBS"]);
+
+const BE = new Set(["be", "am", "is", "are", "was", "were", "been", "being"]);
+
+const isBe = (entry: Tagged): boolean => BE.has(entry.lemma ?? entry.value.toLowerCase()) || BE.has(entry.value.toLowerCase());
+
+const isPassive = (tagged: readonly Tagged[], at: number): boolean => {
+  if (tagged[at]?.pos !== "VBN") return false;
+  const before = tagged.slice(0, at).reverse();
+  const head = before.find((entry) => !ADVERB_TAGS.has(entry.pos));
+  return head !== undefined && isBe(head);
+};
+
+const state: { ready: Tagger | undefined } = { ready: undefined };
+
+export const prepare = (): void => {
+  state.ready ??= build();
+};
+
+export const isReady = (): boolean => state.ready !== undefined;
+
+/**
+ * wink は位置を返さないので、表層を順に照合して復元する。
+ * 見つからないものは飛ばし、カーソルは進めない。位置の当てずっぽうを下流に流さない。
+ */
+const locate = (text: string, tagged: readonly Tagged[]): Token[] =>
+  tagged.reduce<{ tokens: Token[]; cursor: number }>(
+    (acc, entry, at) => {
+      const start = text.indexOf(entry.value, acc.cursor);
+      if (start === -1) return acc;
+      const end = start + entry.value.length;
+      const token = {
+        span: { start, end },
+        surface: entry.value,
+        pos: upos(entry.pos),
+        ...(entry.lemma === undefined ? {} : { lemma: entry.lemma }),
+        ...(isPassive(tagged, at) ? { features: { Voice: "Pass" } } : {}),
+      };
+      return { tokens: [...acc.tokens, token], cursor: end };
+    },
+    { tokens: [], cursor: 0 },
+  ).tokens;
+
+export const tokenize = (text: string): Token[] | undefined => {
+  const tagger = state.ready;
+  if (tagger === undefined) return undefined;
+  return locate(text, toArray(callMethod(tagger, "tagSentence", [text])).filter(isTagged));
+};