ja-compromise 0.0.1 → 0.1.0

package/src/01-one/romanji/compute/kanji-reading/words.js

@@ -0,0 +1,31 @@
+// per-character readings can't know that 日本 is にほん and not にちほん.
+// this is a small override-table for the words that get it wrong most often.
+// (a full reading-dictionary is the real fix - see the changelog)
+export default {
+  '私': 'わたし', '僕': 'ぼく', '俺': 'おれ', '君': 'きみ', '彼': 'かれ', '彼女': 'かのじょ',
+  '日本': 'にほん', '日本語': 'にほんご', '日本人': 'にほんじん', '東京': 'とうきょう',
+  '英語': 'えいご', '中国': 'ちゅうごく', '韓国': 'かんこく', '大阪': 'おおさか', '京都': 'きょうと',
+  '今日': 'きょう', '明日': 'あした', '昨日': 'きのう', '毎日': 'まいにち', '今朝': 'けさ',
+  '一昨日': 'おととい', '明後日': 'あさって', '今年': 'ことし', '去年': 'きょねん', '来年': 'らいねん',
+  '時間': 'じかん', '時計': 'とけい', '手紙': 'てがみ', '写真': 'しゃしん', '電話': 'でんわ',
+  '学生': 'がくせい', '先生': 'せんせい', '学校': 'がっこう', '大学': 'だいがく', '会社': 'かいしゃ',
+  '友達': 'ともだち', '家族': 'かぞく', '子供': 'こども', '大人': 'おとな', '両親': 'りょうしん',
+  '本': 'ほん', '人': 'ひと', '男': 'おとこ', '女': 'おんな', '水': 'みず', '火': 'ひ', '木': 'き',
+  '空': 'そら', '海': 'うみ', '山': 'やま', '川': 'かわ', '花': 'はな', '犬': 'いぬ', '猫': 'ねこ',
+  '車': 'くるま', '家': 'いえ', '店': 'みせ', '駅': 'えき', '道': 'みち', '町': 'まち', '国': 'くに',
+  '目': 'め', '耳': 'みみ', '口': 'くち', '手': 'て', '足': 'あし', '頭': 'あたま', '顔': 'かお',
+  '朝': 'あさ', '昼': 'ひる', '夜': 'よる', '雨': 'あめ', '雪': 'ゆき', '風': 'かぜ', '天気': 'てんき',
+  '食べ物': 'たべもの', '飲み物': 'のみもの', '御飯': 'ごはん', '料理': 'りょうり', '野菜': 'やさい',
+  '映画': 'えいが', '音楽': 'おんがく', '仕事': 'しごと', '勉強': 'べんきょう', '質問': 'しつもん',
+  '一': 'いち', '二': 'に', '三': 'さん', '四': 'よん', '五': 'ご', '六': 'ろく', '七': 'なな',
+  '八': 'はち', '九': 'きゅう', '十': 'じゅう', '百': 'ひゃく', '千': 'せん', '万': 'まん', '円': 'えん',
+  // 熟字訓 - readings that belong to the whole word, not its characters
+  '美味しい': 'おいしい', '大人': 'おとな', '今日': 'きょう', '一人': 'ひとり', '二人': 'ふたり',
+  '大丈夫': 'だいじょうぶ', '上手': 'じょうず', '下手': 'へた', '眼鏡': 'めがね', '果物': 'くだもの',
+  '八百屋': 'やおや', '土産': 'みやげ', '相撲': 'すもう', '風邪': 'かぜ', '田舎': 'いなか',
+  '素敵': 'すてき', '素晴らしい': 'すばらしい', '綺麗': 'きれい', '沢山': 'たくさん',
+  // 来る is irregular in speech as well as in grammar - こ / き / く
+  '来る': 'くる', '来ます': 'きます', '来ました': 'きました', '来た': 'きた', '来て': 'きて',
+  '来ない': 'こない', '来なかった': 'こなかった', '来い': 'こい', '来られる': 'こられる',
+  '食べる': 'たべる', '食べます': 'たべます', '食べた': 'たべた', '食べて': 'たべて',
+}

package/src/01-one/romanji/compute/toRomanji/hiragana-map.js

@@ -0,0 +1,175 @@
+// https://github.com/zkayser/jay_verb/blob/master/lib/japanese/to_romaji.rb
+export default [
+  {},
+  // single-char
+  {
+    "あ": "a",
+    "い": "i",
+    "う": "u",
+    "え": "e",
+    "お": "o",
+    "か": "ka",
+    "き": "ki",
+    "く": "ku",
+    "け": "ke",
+    "こ": "ko",
+    "さ": "sa",
+    "し": "shi",
+    "す": "su",
+    "せ": "se",
+    "そ": "so",
+    "た": "ta",
+    "ち": "chi",
+    "つ": "tsu",
+    "て": "te",
+    "と": "to",
+    "な": "na",
+    "に": "ni",
+    "ぬ": "nu",
+    "ね": "ne",
+    "の": "no",
+    "は": "ha",
+    "ひ": "hi",
+    "ふ": "fu",
+    "へ": "he",
+    "ほ": "ho",
+    "ま": "ma",
+    "み": "mi",
+    "む": "mu",
+    "め": "me",
+    "も": "mo",
+    "や": "ya",
+    "ゆ": "yu",
+    "よ": "yo",
+    "ら": "ra",
+    "り": "ri",
+    "る": "ru",
+    "れ": "re",
+    "ろ": "ro",
+    "わ": "wa",
+    "を": "wo",
+    "ん": "n",
+    "が": "ga",
+    "ぎ": "gi",
+    "ぐ": "gu",
+    "げ": "ge",
+    "ご": "go",
+    "ざ": "za",
+    "じ": "ji",
+    "ず": "zu",
+    "ぜ": "ze",
+    "ぞ": "zo",
+    "だ": "da",
+    "ぢ": "dchi",
+    "づ": "dzu",
+    "で": "de",
+    "ど": "do",
+    "ば": "ba",
+    "び": "bi",
+    "ぶ": "bu",
+    "べ": "be",
+    "ぼ": "bo",
+    "ぱ": "pa",
+    "ぴ": "pi",
+    "ぷ": "pu",
+    "ぺ": "pe",
+    "ぽ": "po",
+    //specials
+    "ゃ": "ya",
+    "ゅ": "yu",
+    "ょ": "yo",
+    "っ": "",
+    "ぁ": "a",
+    "ぃ": "i",
+    "ぅ": "u",
+    "ぇ": "e",
+    "ぉ": "o",
+    "。": "."
+  },
+  // two-char
+  {
+    "きゃ": "kya",
+    "きゅ": "kyu",
+    "きょ": "kyo",
+    "しゃ": "sha",
+    "しゅ": "shu",
+    "しょ": "sho",
+    "ちゃ": "cha",
+    "ちゅ": "chu",
+    "ちょ": "cho",
+    "にゃ": "nya",
+    "にゅ": "nyu",
+    "にょ": "nyo",
+    "ひゃ": "hya",
+    "ひゅ": "hyu",
+    "ひょ": "hyo",
+    "みゃ": "mya",
+    "みゅ": "myu",
+    "みょ": "myo",
+    "りゃ": "rya",
+    "りゅ": "ryu",
+    "りょ": "ryo",
+    "ぎゃ": "gya",
+    "ぎゅ": "gyu",
+    "ぎょ": "gyo",
+    "じゃ": "ja",
+    "じゅ": "ju",
+    "じょ": "jo",
+    "ぢゃ": "dja",
+    "ぢゅ": "dju",
+    "ぢょ": "djo",
+    "びゃ": "bya",
+    "びゅ": "byu",
+    "びょ": "byo",
+    "ぴゃ": "pya",
+    "ぴゅ": "pyu",
+    "ぴょ": "pyo",
+    "てぃ": "ti",
+    "っか": "kka",
+    "っき": "kki",
+    "っく": "kku",
+    "っけ": "kke",
+    "っこ": "kko",
+    "っさ": "ssa",
+    "っし": "sshi",
+    "っす": "ssu",
+    "っせ": "sse",
+    "っそ": "sso",
+    "った": "tta",
+    "っち": "cchi",
+    "っつ": "ttsu",
+    "って": "tte",
+    "っと": "tto",
+    "っば": "bba",
+    "っび": "bbi",
+    "っぶ": "bbu",
+    "っべ": "bbe",
+    "っぼ": "bbo",
+    "っぱ": "ppa",
+    "っぴ": "ppi",
+    "っぷ": "ppu",
+    "っぺ": "ppe",
+    "っぽ": "ppo"
+  },
+  // three-char
+  {
+    "っきゃ": "kkya",
+    "っきゅ": "kkyu",
+    "っきょ": "kkyo",
+    "っしゃ": "ssha",
+    "っしゅ": "sshu",
+    "っしょ": "ssho",
+    "っちゃ": "ccha",
+    "っちゅ": "cchu",
+    "っちょ": "ccho",
+    "っじゃ": "jja",
+    "っじゅ": "jju",
+    "っじょ": "jjo",
+    "っびゃ": "bbya",
+    "っびゅ": "bbyu",
+    "っびょ": "bbyo",
+    "っぴゃ": "ppya",
+    "っぴゅ": "ppyu",
+    "っぴょ": "ppyo"
+  }
+]

package/src/01-one/romanji/compute/toRomanji/index.js

@@ -0,0 +1,84 @@
+import hMap from './hiragana-map.js'
+
+let hasMulti = new Set(['き', 'ぎ', 'し', 'じ', 'ち', 'ぢ', 'っ', 'て', 'に', 'ひ', 'び', 'ぴ', 'み', 'り'])
+
+// katakana and hiragana are the same 46 sounds, 0x60 apart in unicode.
+// fold katakana down so one map handles both scripts.
+const toHiragana = function (str) {
+  let out = ''
+  for (let i = 0; i < str.length; i += 1) {
+    let c = str[i]
+    if (c >= 'ァ' && c <= 'ヶ') {
+      out += String.fromCharCode(c.charCodeAt(0) - 0x60)
+    } else {
+      out += c
+    }
+  }
+  return out
+}
+
+// there are 46 of these
+const isHiragana = function (ch) {
+  return ch >= "\u3040" && ch <= "\u309f";
+}
+
+const vowels = 'aiueo'
+
+// sound-out japanese script in latin alphabet
+const toRomanji = function (str) {
+  let chars = toHiragana(str).split('')
+  let out = ''
+
+  for (let i = 0; i < chars.length; i += 1) {
+    let c = chars[i]
+    // ー is the 長音符 - it lengthens the vowel of the syllable before it
+    if (c === 'ー' || c === 'ｰ') {
+      let last = out[out.length - 1]
+      if (last && vowels.includes(last)) {
+        out += last
+      }
+      continue
+    }
+    // pass non-hiragana right through
+    if (!isHiragana(c)) {
+      out += c
+      continue
+    }
+    // a lone っ doubles the consonant that follows it, and is silent at the
+    // end of a word - it never spells anything by itself
+    if (c === 'っ') {
+      let next = chars[i + 1] ? hMap[1][chars[i + 1]] : ''
+      if (next && !vowels.includes(next[0])) {
+        out += next[0]
+      }
+      continue
+    }
+    // look ahead at greedy multi-char sequences
+    if (hasMulti.has(c)) {
+      if (chars[i + 1]) {
+        let two = c + chars[i + 1]
+        if (hMap[2].hasOwnProperty(two)) {
+          out += hMap[2][two]
+          i += 1
+          continue
+        }
+        if (chars[i + 2]) {
+          let three = c + chars[i + 1] + chars[i + 2]
+          if (hMap[3].hasOwnProperty(three)) {
+            out += hMap[3][three]
+            i += 1
+            continue
+          }
+        }
+      }
+    }
+    // single-char map
+    out += hMap[1][c] || c
+  }
+  return out
+}
+
+export default toRomanji
+
+// console.log(toRomanji('ひらがな　カタカナ'))
+// console.log(toRomanji('あっきょっつああ'))

package/src/01-one/romanji/compute/toRomanji/katakana-map.js

@@ -0,0 +1,128 @@
+let out = {
+  "a": "ア",
+  "i": "イ",
+  "u": "ウ",
+  "e": "エ",
+  "o": "オ",
+  "n": "ン",
+  "m": "ン",
+  "ka": "カ",
+  "ki": "キ",
+  "ku": "ク",
+  "ke": "ケ",
+  "ko": "コ",
+  "sa": "サ",
+  "si": "シ",
+  "su": "ス",
+  "se": "セ",
+  "so": "ソ",
+  "ta": "タ",
+  "ti": "チ",
+  "tu": "ツ",
+  "te": "テ",
+  "to": "ト",
+  "na": "ナ",
+  "ni": "ニ",
+  "nu": "ヌ",
+  "ne": "ネ",
+  "no": "ノ",
+  "ha": "ハ",
+  "hi": "ヒ",
+  "fu": "フ",
+  "he": "ヘ",
+  "ho": "ホ",
+  "ma": "マ",
+  "mi": "ミ",
+  "mu": "ム",
+  "me": "メ",
+  "mo": "モ",
+  "ya": "ヤ",
+  "yu": "ユ",
+  "yo": "ヨ",
+  "ra": "ラ",
+  "ri": "リ",
+  "ru": "ル",
+  "re": "レ",
+  "ro": "ロ",
+  "wa": "ワ",
+  "wo": "ヲ",
+  "ga": "ガ",
+  "gi": "ギ",
+  "gu": "グ",
+  "ge": "ゲ",
+  "go": "ゴ",
+  "za": "ザ",
+  "zi": "ジ",
+  "zu": "ズ",
+  "ze": "ゼ",
+  "zo": "ゾ",
+  "da": "ダ",
+  "di": "ヂ",
+  "du": "ヅ",
+  "de": "デ",
+  "do": "ド",
+  "ba": "バ",
+  "bi": "ビ",
+  "bu": "ブ",
+  "be": "ベ",
+  "bo": "ボ",
+  "pa": "パ",
+  "pi": "ピ",
+  "pu": "プ",
+  "pe": "ペ",
+  "po": "ポ",
+  "ja": "ジャ",
+  "ju": "ジュ",
+  "jo": "ジョ",
+  "ji": "ジ",
+  "vi": "ヴィ",
+  "kya": "キャ",
+  "kyu": "キュ",
+  "kyo": "キョ",
+  "sha": "シャ",
+  "shu": "シュ",
+  "sho": "ショ",
+  "shi": "シ",
+  "tsu": "ツ",
+  "cha": "チャ",
+  "chu": "チュ",
+  "cho": "チョ",
+  "chi": "チ",
+  "nya": "ニャ",
+  "nyu": "ニュ",
+  "nyo": "ニョ",
+  "hya": "ヒャ",
+  "hyu": "ヒュ",
+  "hyo": "ヒョ",
+  "mya": "ミャ",
+  "myu": "ミュ",
+  "myo": "ミョ",
+  "rya": "リャ",
+  "ryu": "リュ",
+  "ryo": "リョ",
+  "gya": "ギャ",
+  "gyu": "ギュ",
+  "gyo": "ギョ",
+  "bya": "ビャ",
+  "byu": "ビュ",
+  "byo": "ビョ",
+  "pya": "ピャ",
+  "pyu": "ピュ",
+  "pyo": "ピョ",
+  "tsu": "ッ"
+}
+
+const toUtf8 = function (s) {
+  return decodeURIComponent(s)
+}
+
+Object.entries(out).forEach(a => {
+  let num = a[1].replace(/&#/g, '').replace(/;/, '')
+  let hex = Number(num).toString(16);
+  // console.log('\\u' + hex)
+  console.log(toUtf8('\\u' + hex)) //eslint-disable-line
+})
+
+
+// console.log(toUtf8('\u30a2\u30e1\u30ea\u30ab\u5408\u8846\u56fd')) //アメリカ合衆国 (USA)
+// console.log(toUtf8('\u12496'))

package/src/01-one/romanji/plugin.js

@@ -0,0 +1,7 @@
+import api from './api.js'
+import compute from './compute/index.js'
+
+export default {
+  compute,
+  api
+}

package/src/01-one/tokenizer/methods/join-numbers.js

@@ -0,0 +1,32 @@
+import { isNumeral } from '../../numbers/kanji-number.js'
+
+const allNumeral = function (str) {
+  for (let i = 0; i < str.length; i += 1) {
+    if (!isNumeral(str[i])) {
+      return false
+    }
+  }
+  return str.length > 0
+}
+
+/**
+ * 二十三 arrives as 二|十|三, because each of them is a lexicon entry on its own.
+ * a run of numerals is one number.
+ */
+const joinNumbers = function (arr) {
+  let out = []
+  for (let i = 0; i < arr.length; i += 1) {
+    if (!allNumeral(arr[i])) {
+      out.push(arr[i])
+      continue
+    }
+    let run = arr[i]
+    while (arr[i + 1] !== undefined && allNumeral(arr[i + 1])) {
+      run += arr[i + 1]
+      i += 1
+    }
+    out.push(run)
+  }
+  return out
+}
+export default joinNumbers

package/src/01-one/tokenizer/methods/join-up.js

@@ -0,0 +1,66 @@
+import lexicon from '../../lexicon/lexicon.js'
+import { getType, isKanji } from './lib.js'
+
+// suffixes that belong to the word in front of them
+const suffixes = new Set(['たち', '達'])
+// honorific prefixes that belong to the word behind them
+const prefixes = new Set(['お', 'ご', '御'])
+
+// two unknown characters may only merge if they're the same script.
+// 'kanji then hiragana' used to merge too, which glued particles onto nouns.
+const mergeTypes = function (a, b) {
+  return a === b && a !== 'punctuation' && a !== 'other'
+}
+
+// an unknown kanji run longer than this is almost certainly two words.
+// katakana has no such limit - ニュージーランド is one word.
+const MAX_RUN = 4
+const runLimit = (type) => (type === 'kanji' || type === 'hiragana' ? MAX_RUN : Infinity)
+
+/** glue neighbouring unknown characters into plausible words */
+const joinUp = function (arr) {
+  let out = []
+  for (let i = 0; i < arr.length; i += 1) {
+    let c = arr[i]
+    if (c === null || c === '') {
+      continue
+    }
+    let last = out[out.length - 1]
+    // 私 + たち,  田中 + さん
+    if (suffixes.has(c) && last && !lexicon[last + c]) {
+      out[out.length - 1] = last + c
+      continue
+    }
+    // お + 金,  ご + 飯
+    if (prefixes.has(c) && arr[i + 1] && isKanji(arr[i + 1][0]) && !lexicon[c]) {
+      out.push(c + arr[i + 1])
+      i += 1
+      continue
+    }
+    if (c.length === 1 && !lexicon[c]) {
+      let type = getType(c)
+      let run = c
+      // race ahead, joining same-script characters.
+      // a *known* single kanji may join an already-started run (日+本+人),
+      // but never starts one - so 「読んでいる人」 keeps 人 on its own.
+      let limit = runLimit(type)
+      while (run.length < limit && arr[i + 1] !== undefined) {
+        let next = arr[i + 1]
+        if (next.length !== 1 || !mergeTypes(type, getType(next))) {
+          break
+        }
+        if (lexicon[next] && type !== 'kanji') {
+          break
+        }
+        run += next
+        i += 1
+      }
+      out.push(run)
+      continue
+    }
+    out.push(c)
+  }
+  return out
+}
+
+export default joinUp

package/src/01-one/tokenizer/methods/lib.js

@@ -0,0 +1,66 @@
+// https://github.com/darren-lester/nihongo/blob/master/src/analysers.js
+
+const isHiragana = function (ch) {
+  return ch >= '぀' && ch <= 'ゟ'
+}
+
+const isKatakana = function (ch) {
+  // ・ (U+30FB) sits inside the katakana block but is punctuation - it
+  // separates the parts of a foreign name, ジョン・スミス
+  if (ch === '・' || ch === '゠') {
+    return false
+  }
+  return (ch >= '゠' && ch <= 'ヿ') || (ch >= 'ㇰ' && ch <= 'ㇿ')
+}
+
+const isKanji = function (ch) {
+  return (
+    (ch >= '一' && ch <= '龯') ||
+    (ch >= '㐀' && ch <= '䶿') ||
+    ch === '々' || // 々 - the repeat-mark, as in 人々
+    ch === '〆' ||
+    ch === 'ヶ' ||
+    ch === '𠮟'
+  )
+}
+
+const isNumber = function (c) {
+  return (c >= '0' && c <= '9') || (c >= '０' && c <= '９') // half & full-width
+}
+
+const isAscii = function (c) {
+  return /[a-zA-Z]/.test(c) || (c >= 'Ａ' && c <= 'ｚ')
+}
+
+// 、。！？ and friends
+const isPunctuation = function (c) {
+  return /[、。，．！？!?,.:：;；・…〜~「」『』（）()【】〔〕《》〈〉\s]/.test(c)
+}
+
+const getType = function (c) {
+  // ー is the 長音符 - it continues whatever script it follows
+  if (c === 'ー' || c === 'ｰ') {
+    return 'katakana'
+  }
+  if (isHiragana(c)) {
+    return 'hiragana'
+  }
+  if (isKatakana(c)) {
+    return 'katakana'
+  }
+  if (isKanji(c)) {
+    return 'kanji'
+  }
+  if (isNumber(c)) {
+    return 'number'
+  }
+  if (isAscii(c)) {
+    return 'ascii'
+  }
+  if (isPunctuation(c)) {
+    return 'punctuation'
+  }
+  return 'other'
+}
+
+export { isHiragana, isKatakana, isKanji, isNumber, isAscii, isPunctuation, getType }

package/src/01-one/tokenizer/methods/naiive-split.js

@@ -0,0 +1,88 @@
+import { isHiragana, isKatakana, isKanji, isNumber, isAscii } from './lib.js'
+
+let markers = [
+  // 'は',// - topic marker
+  // 'が',// - subject
+  // 'を',// - direct object
+  // 'の',// - possessive (reverse "of"), question mark (plain)
+  // 'な',// - marks an adjective
+  //'も',// - "also" (substitutes for wa, ga, or w
+  //'で',// - "by means of", "in"/"at" for actio
+  //'に',// - indirect object, "in"/"at" for existen
+  //'と',// - "and", object of "say" or "thin
+  //'や',// - "and" for a li
+  //'へ',// - destination "t
+  //'か',// - question mark (polit
+
+  '・', //word-splitter
+  '、',//comma
+  '：', //colon
+  ' ',//space
+  '\t',//tab
+  '\n',//newline
+]
+
+let always = new Set(markers)
+
+const getType = function (c) {
+  if (isHiragana(c)) {
+    return 'hiragana'
+  }
+  if (isKatakana(c)) {
+    return 'katakana'
+  }
+  if (isKanji(c)) {
+    return 'kanji'
+  }
+  if (isNumber(c)) {
+    return 'number'
+  }
+  if (isAscii(c)) {
+    return 'ascii'
+  }
+  return null
+}
+
+// naiive split by Hiragana/Katakana/Kanji segments
+const tokenize = function (txt) {
+  let chars = txt.split('')
+  let arr = []
+  let past = null
+  let run = []
+  // loop through every character
+  for (let i = 0; i < chars.length; i += 1) {
+    let c = chars[i]
+    // clear split punctuation
+    if (always.has(c)) {
+      run.push(c)
+      arr.push(run.join(''))
+      run = []
+      past = null
+      continue
+    }
+    // which script is the character?
+    let type = getType(c)
+
+    if (past == null) {
+      past = type
+    }
+    // keep it going
+    if (type === past) {
+      run.push(c)
+      continue
+    }
+    // start of a new type
+    if (run.length > 0) {
+      arr.push(run.join(''))
+    }
+    run = [c]
+    past = type
+  }
+  if (run.length > 0) {
+    arr.push(run.join(''))
+  }
+  return arr//.filter(s => s.length > 1)
+}
+export default tokenize
+// console.log(tokenize('小さな子供は食料品店に歩いた'))
+