ja-compromise 0.0.1 → 0.1.0

package/src/01-one/tokenizer/methods/okurigana.js

@@ -0,0 +1,58 @@
+import lexicon from '../../lexicon/lexicon.js'
+import deconjugate from '../../conjugate/deconjugate.js'
+import { isKanji, isHiragana } from './lib.js'
+
+// a particle can't be the first kana of a verb's okurigana - 「木で作った」
+// is 木 + で + 作った, not 木で + 作った
+const particleStart = new Set(['は', 'が', 'を', 'に', 'へ', 'と', 'も', 'の', 'か', 'ね', 'よ', 'や', 'で', 'ば'])
+const MAX_TAIL = 6
+
+const isHiraganaWord = (str) => str.split('').every(isHiragana)
+
+/**
+ * 含まれている is a verb even though 含む isn't in the lexicon.
+ * glue an unknown kanji stem onto the inflectional tail that follows it.
+ */
+const attachOkurigana = function (arr) {
+  let out = []
+  for (let i = 0; i < arr.length; i += 1) {
+    let head = arr[i]
+    // only an unknown kanji chunk can pick up okurigana
+    if (lexicon[head] || !head.split('').every(isKanji)) {
+      out.push(head)
+      continue
+    }
+    // collect the hiragana tokens that follow it
+    let tail = ''
+    let end = i
+    for (let o = i + 1; o < arr.length; o += 1) {
+      if (!isHiraganaWord(arr[o]) || tail.length + arr[o].length > MAX_TAIL) {
+        break
+      }
+      tail += arr[o]
+      end = o
+    }
+    if (tail === '' || particleStart.has(tail[0])) {
+      out.push(head)
+      continue
+    }
+    // try the longest tail first, shrinking a token at a time
+    let joined = head
+    let taken = i
+    for (let o = end; o > i; o -= 1) {
+      let candidate = head + arr.slice(i + 1, o + 1).join('')
+      if (candidate.length - head.length < 2) {
+        continue
+      }
+      if (lexicon[candidate] || deconjugate(candidate)) {
+        joined = candidate
+        taken = o
+        break
+      }
+    }
+    out.push(joined)
+    i = taken
+  }
+  return out
+}
+export default attachOkurigana

package/src/01-one/tokenizer/methods/terms.js

@@ -0,0 +1,143 @@
+import buildIndex from './trie/build.js'
+import lexicon from '../../lexicon/lexicon.js'
+import splitUp from './trie/split-up.js'
+import joinUp from './join-up.js'
+import attachOkurigana from './okurigana.js'
+import joinNumbers from './join-numbers.js'
+import { isPunctuation } from './lib.js'
+
+const { words, maxLen } = buildIndex(Object.keys(lexicon))
+// a very long 'word' is nearly always two words - cap the lookahead
+const LOOKAHEAD = Math.min(maxLen, 12)
+
+const allHiragana = /^[぀-ゟ]+$/
+
+// a case-particle at either end of a kana run isn't part of the word
+const edgeParticle = new Set(['は', 'が', 'を', 'に', 'へ', 'と', 'も', 'の', 'で', 'や', 'か'])
+// beyond this, a string of loose kana really is several words
+const MAX_KANA_RUN = 7
+
+const isSingleKana = (str) => str.length === 1 && allHiragana.test(str)
+
+// 「『（(【〔《〈 at the end of a punctuation run
+const trailingOpen = /[「『（(【〔《〈]+$/
+
+/**
+ * ひらがな comes back from the matcher as ひ|ら|が|な, because が and な happen
+ * to be particles.  a stretch of single kana that matched nothing is one
+ * unknown word - a name, or a word the lexicon is missing.
+ */
+const rejoinKana = function (arr) {
+  let out = []
+  for (let i = 0; i < arr.length; i += 1) {
+    if (!isSingleKana(arr[i])) {
+      out.push(arr[i])
+      continue
+    }
+    // how far does this stretch of loose kana go?
+    let end = i
+    while (end < arr.length && isSingleKana(arr[end])) {
+      end += 1
+    }
+    let run = arr.slice(i, end)
+    i = end - 1
+    // を is only ever a particle, so it's a hard word-boundary
+    let segments = [[]]
+    run.forEach(c => {
+      if (c === 'を') {
+        segments.push(['を'], [])
+      } else {
+        segments[segments.length - 1].push(c)
+      }
+    })
+    segments.forEach(seg => {
+      if (seg.length === 0) {
+        return
+      }
+      if (seg.length === 1 || seg[0] === 'を' || seg.length > MAX_KANA_RUN) {
+        out = out.concat(seg)
+        return
+      }
+      // 「での」 is two particles in a row, not a word
+      if (seg.every(c => edgeParticle.has(c))) {
+        out = out.concat(seg)
+        return
+      }
+      // 「ひらがなと」 is ひらがな + と, 「経験につながる」 is 経験 | に | つながる.
+      // a particle can't begin a clause, so only trim the head when something
+      // real came before it
+      let hasLeft = out.length > 0 && !edgeParticle.has(out[out.length - 1])
+      let head = ''
+      let tail = ''
+      if (seg.length > 2 && hasLeft && edgeParticle.has(seg[0])) {
+        head = seg.shift()
+      }
+      if (seg.length > 2 && edgeParticle.has(seg[seg.length - 1])) {
+        tail = seg.pop()
+      }
+      if (head) {
+        out.push(head)
+      }
+      out.push(seg.join(''))
+      if (tail) {
+        out.push(tail)
+      }
+    })
+  }
+  return out
+}
+
+const tokenize = function (txt, isChunk) {
+  // when the writer used a space, that chunk is one word - beginner japanese
+  // is often written 「たろう は のりこ を…」, and 「のりこ」 must not be
+  // greedily read as the particle の plus りこ
+  if (isChunk === true && txt.length <= 6 && allHiragana.test(txt) && !lexicon[txt]) {
+    return [txt]
+  }
+  // split by known-word segments
+  let arr = splitUp(txt, words, LOOKAHEAD)
+  // a kana run that fell apart into loose characters is one unknown word
+  arr = rejoinKana(arr)
+  // join-up neighbouring unknown characters
+  arr = joinUp(arr)
+  // 二|十|三 is one number
+  arr = joinNumbers(arr)
+  // give an unknown kanji stem its inflectional tail - 含 + まれている
+  arr = attachOkurigana(arr)
+  // punctuation is never its own term - it hangs off the word before it,
+  // except an opening bracket, which hangs off the word after it - 彼は「はい」と
+  let out = []
+  let carry = ''
+  arr.forEach(str => {
+    if (str === '') {
+      return
+    }
+    if (str.split('').every(isPunctuation)) {
+      let open = ''
+      str = str.replace(trailingOpen, m => {
+        open = m
+        return ''
+      })
+      if (str !== '' && out.length > 0) {
+        out[out.length - 1] += str
+      } else {
+        carry += str
+      }
+      carry += open
+      return
+    }
+    out.push(carry + str)
+    carry = ''
+  })
+  // punctuation with no word after it
+  if (carry !== '') {
+    if (out.length > 0) {
+      out[out.length - 1] += carry
+    } else {
+      out.push(carry)
+    }
+  }
+  return out
+}
+export default tokenize
+export { words }

package/src/01-one/tokenizer/methods/trie/build.js

@@ -0,0 +1,13 @@
+// index the lexicon for longest-match lookup
+const buildIndex = function (arr) {
+  let words = new Set()
+  let maxLen = 1
+  for (let i = 0; i < arr.length; i += 1) {
+    words.add(arr[i])
+    if (arr[i].length > maxLen) {
+      maxLen = arr[i].length
+    }
+  }
+  return { words, maxLen }
+}
+export default buildIndex

package/src/01-one/tokenizer/methods/trie/split-up.js

@@ -0,0 +1,29 @@
+// longest-match segmentation.
+//
+// this used to build a character-trie over the whole lexicon, which cost
+// ~40mb of nodes.  a Set of the words plus a bounded backwards scan is the
+// same answer for a fraction of the memory.
+
+const splitUp = function (txt, words, maxLen) {
+  let out = []
+  let i = 0
+  while (i < txt.length) {
+    let max = Math.min(maxLen, txt.length - i)
+    let found = ''
+    for (let len = max; len > 1; len -= 1) {
+      let str = txt.substr(i, len)
+      if (words.has(str)) {
+        found = str
+        break
+      }
+    }
+    if (found === '') {
+      // a single character is only a 'word' if the lexicon says so
+      found = txt[i]
+    }
+    out.push(found)
+    i += found.length
+  }
+  return out
+}
+export default splitUp

package/src/01-one/tokenizer/methods/whitespace.js

@@ -0,0 +1,44 @@
+import toTerms from './terms.js'
+const before = /^[\s「『（(〽【〔《〈]+/
+const after = /[\s、。，．：；・」』）)…〜~】〕》〉\.?!？！]+$/
+
+const getPunct = function (str) {
+  let pre = ''
+  let post = ''
+  let inside = str
+  // strip-off leading punctuation
+  inside = inside.replace(before, m => {
+    pre = m || ''
+    return ''
+  })
+  // strip-off ending punctuation
+  inside = inside.replace(after, m => {
+    post = m || ''
+    return ''
+  })
+  return { pre, inside, post }
+}
+
+const tokenize = function (txt, isChunk) {
+  let terms = []
+  toTerms(txt, isChunk).forEach(str => {
+    let { pre, post, inside } = getPunct(str)
+    if (inside === '') {
+      // punctuation-only - fold it into the term before it
+      if (terms.length > 0) {
+        terms[terms.length - 1].post += pre + post
+        return
+      }
+      return
+    }
+    terms.push({
+      text: inside,
+      normal: inside.toLowerCase(),
+      pre,
+      post,
+      tags: new Set(),
+    })
+  })
+  return terms
+}
+export default tokenize

package/src/01-one/tokenizer/plugin.js

@@ -0,0 +1,13 @@
+// import unicode from './unicode.js'
+// import contractions from './contractions.js'
+import toTerms from './methods/terms.js'
+
+export default {
+  // mutate: (world) => {
+  //   world.model.one.unicode = unicode
+  //   world.model.one.contractions = contractions
+  // },
+  methods: {
+    toTerms: toTerms
+  }
+}

package/src/02-two/preTagger/compute/01-script.js

@@ -0,0 +1,44 @@
+import { isHiragana, isKatakana, isKanji, isAscii, isNumber } from '../../../01-one/tokenizer/methods/lib.js'
+
+const every = function (str, fn) {
+  for (let i = 0; i < str.length; i += 1) {
+    if (!fn(str[i])) {
+      return false
+    }
+  }
+  return str.length > 0
+}
+
+const tagScript = function (terms, setTag, world) {
+  const reason = 'script'
+  terms.forEach(term => {
+    let str = term.text
+    if (every(str, isHiragana)) {
+      setTag([term], 'Hiragana', world, null, reason)
+      return
+    }
+    if (every(str, c => isKatakana(c) || c === 'ー')) {
+      setTag([term], 'Katakana', world, null, reason)
+      // a katakana word is nearly always a loanword noun
+      if (term.tags.size <= 1) {
+        setTag([term], 'Noun', world, null, reason)
+      }
+      return
+    }
+    if (every(str, isKanji)) {
+      setTag([term], 'Kanji', world, null, reason)
+      return
+    }
+    if (every(str, isNumber)) {
+      setTag([term], 'Value', world, null, reason)
+      setTag([term], 'Cardinal', world, null, reason)
+      return
+    }
+    if (every(str, isAscii)) {
+      setTag([term], 'Ascii', world, null, reason)
+      return
+    }
+    // a mixed kanji+hiragana word (書いた, 食べる) - no single script tag
+  })
+}
+export default tagScript

package/src/02-two/preTagger/compute/02-particles.js

@@ -0,0 +1,82 @@
+// what a particle tells us about the word in front of it.
+// this is the workhorse of japanese pos-tagging: the particle *is* the syntax.
+const reason = 'particle'
+
+// particles that only ever follow a noun-phrase
+const nounBefore = {
+  'は': 'Topic',
+  'が': 'Noun',
+  'を': 'Object',
+  'へ': 'Noun',
+  'から': 'Noun',
+  'まで': 'Noun',
+  'より': 'Noun',
+  'の': 'Possessive',
+}
+// に and で follow nouns, but also verb-stems (見に行く) and な-adjectives (静かで)
+const softNounBefore = new Set(['に', 'で', 'と', 'も'])
+
+// particles that only ever follow a verb or adjective
+const verbBefore = new Set(['て', 'ば', 'たら', 'ながら', 'ので', 'のに', 'けれど', 'けれども', 'けど', 'たり'])
+
+const tagParticles = function (terms, setTag, world) {
+  for (let i = 0; i < terms.length; i += 1) {
+    let t = terms[i]
+    let prev = terms[i - 1]
+    let next = terms[i + 1]
+    let str = t.text
+
+    // な before a noun is the adnominal copula (静かな人), not a final particle
+    if (str === 'な' && next && !next.tags.has('Verb')) {
+      setTag([t], 'AdnominalParticle', world, null, reason)
+      if (prev) {
+        setTag([prev], 'NaAdjective', world, null, reason)
+      }
+      continue
+    }
+    // か at the end of a clause is a question-marker, not 'or'
+    if (str === 'か' && (!next || next.tags.has('SentenceParticle'))) {
+      setTag([t], 'SentenceParticle', world, null, reason)
+      setTag([t], 'QuestionWord', world, null, reason)
+      continue
+    }
+    // と after a verb introduces a quote;  between two nouns it means 'and'
+    if (str === 'と' && prev) {
+      if (prev.tags.has('Verb') || prev.tags.has('Copula')) {
+        setTag([t], 'QuotativeParticle', world, null, reason)
+        continue
+      }
+      if (next && !next.tags.has('Verb')) {
+        setTag([t], 'Conjunction', world, null, reason)
+      }
+    }
+    // で after a noun is 'at/by'; after a な-adjective it's the copula
+    if (str === 'で' && prev && prev.tags.has('NaAdjective')) {
+      setTag([t], 'Copula', world, null, reason)
+      setTag([t], 'Gerund', world, null, reason)
+      continue
+    }
+
+    // a particle never marks another particle
+    if (!prev || prev.tags.has('Particle')) {
+      continue
+    }
+    if (nounBefore[str] !== undefined) {
+      if (!prev.tags.has('Verb') && !prev.tags.has('Adjective')) {
+        setTag([prev], nounBefore[str], world, null, reason)
+      } else if (str === 'の' || str === 'が') {
+        // 走るの / 行くが - the verb is nominalized or the clause continues
+        setTag([t], str === 'の' ? 'AdnominalParticle' : 'ConjunctiveParticle', world, null, reason)
+      }
+      continue
+    }
+    if (softNounBefore.has(str) && !prev.tags.has('Verb') && !prev.tags.has('Adjective') && prev.tags.size <= 1) {
+      setTag([prev], 'Noun', world, null, reason)
+      continue
+    }
+    if (verbBefore.has(str) && prev.tags.size <= 1) {
+      setTag([prev], 'Verb', world, null, reason)
+    }
+  }
+}
+export default tagParticles

package/src/02-two/preTagger/compute/03-verbs.js

@@ -0,0 +1,111 @@
+import { deconjugate, conjugateVerb } from '../../../01-one/conjugate/index.js'
+import lexicon from '../../../01-one/lexicon/lexicon.js'
+
+const reason = 'verbSuffix'
+const isKnown = (w) => lexicon.hasOwnProperty(w)
+const scriptOnly = new Set(['Kanji', 'Hiragana', 'Katakana', 'Ascii'])
+
+// a verb we've never seen can still be read off its ending.
+// 「散歩した」 is a verb even if 散歩する isn't in the lexicon.
+const tagUnknownVerbs = function (terms, setTag, world) {
+  terms.forEach(term => {
+    // only guess at words the lexicon hasn't already resolved.
+    // すみません is an expression, not the negative of a verb 済む.
+    let known = [...term.tags].some(t => !scriptOnly.has(t))
+    if (known) {
+      return
+    }
+    if (term.text.length < 2) {
+      return
+    }
+    let found = deconjugate(term.text, isKnown)
+    if (!found || found.guess === true) {
+      return
+    }
+    found.tags.forEach(tag => setTag([term], tag, world, null, reason))
+    term.root = found.root
+  })
+}
+
+// する attaches to the noun before it - 「勉強 します」
+const tagSuruVerbs = function (terms, setTag, world) {
+  for (let i = 1; i < terms.length; i += 1) {
+    let t = terms[i]
+    if (!t.tags.has('Verb')) {
+      continue
+    }
+    if (/^(し|す|さ)/.test(t.text) === false) {
+      continue
+    }
+    let prev = terms[i - 1]
+    if (prev && prev.tags.has('SuruVerb')) {
+      setTag([prev], 'Verb', world, null, 'suruVerb')
+    }
+  }
+}
+
+// 〜て + いる/ある/おく/しまう - the helper verb carries the aspect,
+// but the meaning belongs to the て-form in front of it
+// 〜ている marks aspect;  〜てしまう, 〜てくれる and friends only add nuance
+const aspectVerbs = ['いる', '居る', 'おる', 'ある']
+const otherHelpers = ['おく', 'しまう', '来る', 'くる', '行く', 'いく', 'みる', '見る', 'あげる', 'くれる', 'もらう']
+
+// every surface-form of every helper, so we can recognise いました / しまった
+const formsOf = function (list) {
+  let set = new Set()
+  list.forEach(dict => {
+    let forms = conjugateVerb(dict)
+    if (forms) {
+      Object.keys(forms).forEach(k => typeof forms[k] === 'string' && set.add(forms[k]))
+    }
+  })
+  return set
+}
+let isAspect = formsOf(aspectVerbs)
+let isHelper = formsOf(otherHelpers)
+
+const tagCompoundVerbs = function (terms, setTag, world) {
+  for (let i = 1; i < terms.length; i += 1) {
+    let t = terms[i]
+    let prev = terms[i - 1]
+    if (!prev || !prev.tags.has('Gerund') || !t.tags.has('Verb')) {
+      continue
+    }
+    if (!isAspect.has(t.text) && !isHelper.has(t.text)) {
+      continue
+    }
+    if (isAspect.has(t.text)) {
+      setTag([prev], 'Progressive', world, null, 'compoundVerb')
+    }
+    ;['PastTense', 'Negative', 'Polite'].forEach(tag => {
+      if (t.tags.has(tag)) {
+        setTag([prev], tag, world, null, 'compoundVerb')
+      }
+    })
+  }
+}
+
+// an auxiliary hands its tense back to the verb it attaches to
+const tagAuxiliary = function (terms, setTag, world) {
+  for (let i = 1; i < terms.length; i += 1) {
+    let t = terms[i]
+    if (!t.tags.has('Auxiliary')) {
+      continue
+    }
+    let prev = terms[i - 1]
+    if (!prev) {
+      continue
+    }
+    if (!prev.tags.has('Verb') && !prev.tags.has('Adjective') && !prev.tags.has('Noun')) {
+      setTag([prev], 'Verb', world, null, 'beforeAuxiliary')
+    }
+    // ませんでした after 行き makes the whole phrase past-polite-negative
+    ;['PastTense', 'PresentTense', 'Negative', 'Polite'].forEach(tag => {
+      if (t.tags.has(tag) && prev.tags.has('Verb')) {
+        setTag([prev], tag, world, null, 'fromAuxiliary')
+      }
+    })
+  }
+}
+
+export default { tagUnknownVerbs, tagSuruVerbs, tagAuxiliary, tagCompoundVerbs }

package/src/02-two/preTagger/compute/04-adjectives.js

@@ -0,0 +1,43 @@
+import { isKanji } from '../../../01-one/tokenizer/methods/lib.js'
+
+const reason = 'adjSuffix'
+
+// い-adjective endings.  the old rule was 'ends in い → Adjective', which
+// swallowed every negative verb (行かない), every masu-stem noun (お願い),
+// and 弟/はい/せい besides.
+const looksAdjective = /(し|た|な|か|ら|わ|が|ば|ざ|さ|は|ま|よ|る|す|つ|ぬ|ぶ|む|ぐ|ゆ|ろ|そ|お|こ|の|ほ|も|ど|ぼ|ぽ|ご|ぞ|づ|べ|で|げ|ぜ|え|け|せ|て|ね|へ|め|れ)い$/
+// endings that look like an い-adjective but aren't
+const notAdjective = /(ない|たい|らしい|っぽい)$/
+
+const adjSuffixes = function (terms, setTag, world) {
+  terms.forEach(term => {
+    let str = term.text
+    if (term.tags.size > 1 || term.tags.has('Verb') || term.tags.has('Adjective')) {
+      return
+    }
+    if (str.length < 3 || !str.endsWith('い')) {
+      return
+    }
+    // ない/たい are verb endings, and are already handled as such
+    if (notAdjective.test(str) && !term.tags.has('Adjective')) {
+      return
+    }
+    // 美しい - kanji stem plus a hiragana tail is the classic shape
+    if (isKanji(str[0]) && looksAdjective.test(str)) {
+      setTag([term], 'Adjective', world, null, reason)
+      setTag([term], 'IAdjective', world, null, reason)
+    }
+  })
+}
+
+// 〜さ nominalises an adjective (高さ), 〜たち pluralises a noun (子供たち)
+const nounSuffixes = function (terms, setTag, world) {
+  terms.forEach(term => {
+    if (term.text.length > 2 && /(たち|達)$/.test(term.text)) {
+      setTag([term], 'Plural', world, null, 'nounSuffix')
+      setTag([term], 'Noun', world, null, 'nounSuffix')
+    }
+  })
+}
+
+export default { adjSuffixes, nounSuffixes }

package/src/02-two/preTagger/compute/05-people.js

@@ -0,0 +1,22 @@
+const reason = 'honorific'
+
+// 田中さん - an honorific proves the word before it is a person
+const tagPeople = function (terms, setTag, world) {
+  for (let i = 0; i < terms.length; i += 1) {
+    let t = terms[i]
+    if (!t.tags.has('Honorific')) {
+      continue
+    }
+    let prev = terms[i - 1]
+    if (prev && !prev.tags.has('Verb') && !prev.tags.has('Particle')) {
+      setTag([prev], 'Person', world, null, reason)
+    }
+  }
+  // 田中さん may have tokenized as one word - the suffix is still a signal
+  terms.forEach(term => {
+    if (term.text.length > 2 && /(さん|様|さま|ちゃん|くん|氏)$/.test(term.text) && term.tags.size <= 1) {
+      setTag([term], 'Person', world, null, reason)
+    }
+  })
+}
+export default tagPeople