de-compromise 0.0.10 → 0.1.0

package/src/01-one/lexicon/methods/verbs/irregulars.js

@@ -0,0 +1,202 @@
+import { prefixes } from './custom/rules.js'
+
+// hand-curated conjugations for the highest-frequency irregular verbs.
+// the trained models lose some of these to compression, and they are
+// too important to get wrong.
+// person order: [ich, du, er/sie/es, wir, ihr, sie]
+const irregulars = {
+  sein: {
+    present: ['bin', 'bist', 'ist', 'sind', 'seid', 'sind'],
+    past: ['war', 'warst', 'war', 'waren', 'wart', 'waren'],
+    subj1: ['sei', 'seist', 'sei', 'seien', 'seiet', 'seien'],
+    subj2: ['wäre', 'wärst', 'wäre', 'wären', 'wärt', 'wären'],
+    imperative: ['sei', 'seid'],
+    pastParticiple: 'gewesen',
+    presentParticiple: 'seiend',
+  },
+  haben: {
+    present: ['habe', 'hast', 'hat', 'haben', 'habt', 'haben'],
+    past: ['hatte', 'hattest', 'hatte', 'hatten', 'hattet', 'hatten'],
+    subj2: ['hätte', 'hättest', 'hätte', 'hätten', 'hättet', 'hätten'],
+    imperative: ['hab', 'habt'],
+    pastParticiple: 'gehabt',
+  },
+  werden: {
+    present: ['werde', 'wirst', 'wird', 'werden', 'werdet', 'werden'],
+    past: ['wurde', 'wurdest', 'wurde', 'wurden', 'wurdet', 'wurden'],
+    subj2: ['würde', 'würdest', 'würde', 'würden', 'würdet', 'würden'],
+    imperative: ['werde', 'werdet'],
+    pastParticiple: 'geworden',
+  },
+  tun: {
+    present: ['tue', 'tust', 'tut', 'tun', 'tut', 'tun'],
+    past: ['tat', 'tatest', 'tat', 'taten', 'tatet', 'taten'],
+    subj2: ['täte', 'tätest', 'täte', 'täten', 'tätet', 'täten'],
+    pastParticiple: 'getan',
+    presentParticiple: 'tuend',
+  },
+  wissen: {
+    present: ['weiß', 'weißt', 'weiß', 'wissen', 'wisst', 'wissen'],
+    past: ['wusste', 'wusstest', 'wusste', 'wussten', 'wusstet', 'wussten'],
+    subj2: ['wüsste', 'wüsstest', 'wüsste', 'wüssten', 'wüsstet', 'wüssten'],
+    pastParticiple: 'gewusst',
+  },
+  essen: {
+    present: ['esse', 'isst', 'isst', 'essen', 'esst', 'essen'],
+    past: ['aß', 'aßest', 'aß', 'aßen', 'aßt', 'aßen'],
+    subj2: ['äße', 'äßest', 'äße', 'äßen', 'äßet', 'äßen'],
+    imperative: ['iss', 'esst'],
+    pastParticiple: 'gegessen',
+  },
+  lesen: {
+    present: ['lese', 'liest', 'liest', 'lesen', 'lest', 'lesen'],
+    past: ['las', 'lasest', 'las', 'lasen', 'last', 'lasen'],
+    imperative: ['lies', 'lest'],
+    pastParticiple: 'gelesen',
+  },
+  geben: {
+    present: ['gebe', 'gibst', 'gibt', 'geben', 'gebt', 'geben'],
+    past: ['gab', 'gabst', 'gab', 'gaben', 'gabt', 'gaben'],
+    subj2: ['gäbe', 'gäbest', 'gäbe', 'gäben', 'gäbet', 'gäben'],
+    imperative: ['gib', 'gebt'],
+    pastParticiple: 'gegeben',
+  },
+  kommen: {
+    present: ['komme', 'kommst', 'kommt', 'kommen', 'kommt', 'kommen'],
+    past: ['kam', 'kamst', 'kam', 'kamen', 'kamt', 'kamen'],
+    subj2: ['käme', 'kämest', 'käme', 'kämen', 'kämet', 'kämen'],
+    imperative: ['komm', 'kommt'],
+    pastParticiple: 'gekommen',
+  },
+  heißen: {
+    pastParticiple: 'geheißen',
+  },
+  // modal verbs
+  können: {
+    present: ['kann', 'kannst', 'kann', 'können', 'könnt', 'können'],
+    past: ['konnte', 'konntest', 'konnte', 'konnten', 'konntet', 'konnten'],
+    subj2: ['könnte', 'könntest', 'könnte', 'könnten', 'könntet', 'könnten'],
+    pastParticiple: 'gekonnt',
+  },
+  müssen: {
+    present: ['muss', 'musst', 'muss', 'müssen', 'müsst', 'müssen'],
+    past: ['musste', 'musstest', 'musste', 'mussten', 'musstet', 'mussten'],
+    subj2: ['müsste', 'müsstest', 'müsste', 'müssten', 'müsstet', 'müssten'],
+    pastParticiple: 'gemusst',
+  },
+  dürfen: {
+    present: ['darf', 'darfst', 'darf', 'dürfen', 'dürft', 'dürfen'],
+    past: ['durfte', 'durftest', 'durfte', 'durften', 'durftet', 'durften'],
+    subj2: ['dürfte', 'dürftest', 'dürfte', 'dürften', 'dürftet', 'dürften'],
+    pastParticiple: 'gedurft',
+  },
+  mögen: {
+    present: ['mag', 'magst', 'mag', 'mögen', 'mögt', 'mögen'],
+    past: ['mochte', 'mochtest', 'mochte', 'mochten', 'mochtet', 'mochten'],
+    subj2: ['möchte', 'möchtest', 'möchte', 'möchten', 'möchtet', 'möchten'],
+    pastParticiple: 'gemocht',
+  },
+  wollen: {
+    present: ['will', 'willst', 'will', 'wollen', 'wollt', 'wollen'],
+    past: ['wollte', 'wolltest', 'wollte', 'wollten', 'wolltet', 'wollten'],
+    subj2: ['wollte', 'wolltest', 'wollte', 'wollten', 'wolltet', 'wollten'],
+    pastParticiple: 'gewollt',
+  },
+  sollen: {
+    present: ['soll', 'sollst', 'soll', 'sollen', 'sollt', 'sollen'],
+    past: ['sollte', 'solltest', 'sollte', 'sollten', 'solltet', 'sollten'],
+    subj2: ['sollte', 'solltest', 'sollte', 'sollten', 'solltet', 'sollten'],
+    pastParticiple: 'gesollt',
+  },
+  // mixed verbs — weak endings, strong vowel change
+  denken: {
+    past: ['dachte', 'dachtest', 'dachte', 'dachten', 'dachtet', 'dachten'],
+    subj2: ['dächte', 'dächtest', 'dächte', 'dächten', 'dächtet', 'dächten'],
+    pastParticiple: 'gedacht',
+  },
+  bringen: {
+    past: ['brachte', 'brachtest', 'brachte', 'brachten', 'brachtet', 'brachten'],
+    subj2: ['brächte', 'brächtest', 'brächte', 'brächten', 'brächtet', 'brächten'],
+    pastParticiple: 'gebracht',
+  },
+  kennen: {
+    past: ['kannte', 'kanntest', 'kannte', 'kannten', 'kanntet', 'kannten'],
+    pastParticiple: 'gekannt',
+  },
+  nennen: {
+    past: ['nannte', 'nanntest', 'nannte', 'nannten', 'nanntet', 'nannten'],
+    pastParticiple: 'genannt',
+  },
+  rennen: {
+    past: ['rannte', 'ranntest', 'rannte', 'rannten', 'ranntet', 'rannten'],
+    pastParticiple: 'gerannt',
+  },
+  brennen: {
+    past: ['brannte', 'branntest', 'brannte', 'brannten', 'branntet', 'brannten'],
+    pastParticiple: 'gebrannt',
+  },
+  senden: {
+    past: ['sandte', 'sandtest', 'sandte', 'sandten', 'sandtet', 'sandten'],
+    pastParticiple: 'gesandt',
+  },
+  wenden: {
+    past: ['wandte', 'wandtest', 'wandte', 'wandten', 'wandtet', 'wandten'],
+    pastParticiple: 'gewandt',
+  },
+}
+
+const inseparable = /^(be|emp|ent|er|ge|miss|ver|zer|wiederer)/
+
+// 'verstehen' → ver + stehen,  'aufstehen' → auf + stehen
+const find = function (str) {
+  if (irregulars.hasOwnProperty(str)) {
+    return { base: irregulars[str], prefix: '', separable: false }
+  }
+  let m = str.match(inseparable)
+  if (m && irregulars.hasOwnProperty(str.substring(m[0].length))) {
+    return { base: irregulars[str.substring(m[0].length)], prefix: m[0], separable: false }
+  }
+  for (let i = 0; i < prefixes.length; i += 1) {
+    let p = prefixes[i]
+    if (str.startsWith(p) && irregulars.hasOwnProperty(str.substring(p.length))) {
+      return { base: irregulars[str.substring(p.length)], prefix: p, separable: true }
+    }
+  }
+  return null
+}
+
+// returns conjugated forms for one tense, or null.
+// tense: present | past | subj1 | subj2 | imperative | pastParticiple | presentParticiple
+const getIrregular = function (str, tense) {
+  let found = find(str)
+  if (!found || !found.base[tense]) {
+    return null
+  }
+  let { base, prefix, separable } = found
+  let forms = base[tense]
+  if (typeof forms === 'string') {
+    // participles - 'auf' + 'gestanden', but 'ver' + 'standen'
+    if (tense === 'pastParticiple' && separable === false && prefix) {
+      return prefix + forms.replace(/^ge/, '')
+    }
+    return prefix + forms
+  }
+  return forms.map(f => prefix + f)
+}
+
+// form → infinitive, for root-recovery
+let toInfinitive = {}
+Object.keys(irregulars).forEach(inf => {
+  Object.keys(irregulars[inf]).forEach(tense => {
+    let forms = irregulars[inf][tense]
+    forms = typeof forms === 'string' ? [forms] : forms
+    forms.forEach(w => {
+      if (!toInfinitive.hasOwnProperty(w)) {
+        toInfinitive[w] = inf
+      }
+    })
+  })
+})
+
+export default getIrregular
+export { toInfinitive }

package/src/01-one/lexicon/methods/verbs/toRoot.js

@@ -0,0 +1,55 @@
+import { convert, reverse } from 'suffix-thumb'
+import model from '../models.js'
+import { toInfinitive } from './irregulars.js'
+import ppRules from './custom/fromPastParticiple.js'
+let { presentTense, pastTense, subjunctive1, subjunctive2, imperative, presentParticiple, pastParticiple } = model
+
+// =-=-
+const revAll = function (m) {
+  return Object.keys(m).reduce((h, k) => {
+    h[k] = reverse(m[k])
+    return h
+  }, {})
+}
+
+let presentRev = revAll(presentTense)
+let pastRev = revAll(pastTense)
+let subjRev1 = revAll(subjunctive1)
+let subjRev2 = revAll(subjunctive2)
+let impRev = revAll(imperative)
+let presentPartRev = reverse(presentParticiple.presentParticiple)
+let pastPartRev = pastParticiple ? reverse(pastParticiple.pastParticiple) : null
+
+const allForms = function (str, form, m) {
+  if (toInfinitive.hasOwnProperty(str)) {
+    return toInfinitive[str]
+  }
+  if (m.hasOwnProperty(form)) {
+    return convert(str, m[form])
+  }
+  return str
+}
+
+const fromPresent = (str, form) => allForms(str, form, presentRev)
+const fromPast = (str, form) => allForms(str, form, pastRev)
+const fromSubjunctive1 = (str, form) => allForms(str, form, subjRev1)
+const fromSubjunctive2 = (str, form) => allForms(str, form, subjRev2)
+const fromImperative = (str, form) => allForms(str, form, impRev)
+const fromPresentParticiple = (str) => {
+  return toInfinitive[str] || convert(str, presentPartRev)
+}
+const fromPastParticiple = (str) => {
+  if (toInfinitive.hasOwnProperty(str)) {
+    return toInfinitive[str]
+  }
+  if (pastPartRev) {
+    return convert(str, pastPartRev)
+  }
+  return ppRules(str)
+}
+
+export {
+  fromPresent, fromPast, fromSubjunctive1, fromSubjunctive2, fromImperative, fromPresentParticiple, fromPastParticiple
+}
+
+// console.log(fromPresent('tanzt', 'secondPlural'))

package/src/01-one/lexicon/misc.js

@@ -0,0 +1,78 @@
+// hand-set tags for core function words.
+// these win over the packed lexicon and the conjugation expansion.
+export default {
+  // question words
+  'wo': 'QuestionWord',// where
+  'woher': 'QuestionWord',//where  from
+  'wohin': 'QuestionWord',//where  to
+  'wann': 'QuestionWord',// when
+  'was': 'QuestionWord',// what
+  'wer': 'QuestionWord',// who
+  'wie': 'QuestionWord',// how
+  'warum': 'QuestionWord',// why
+  'achte': 'TextOrdinal',
+
+  // negation
+  'nicht': ['Adverb', 'Negative'],
+  'nein': ['Negative', 'Expression'],
+  'kein': ['Determiner', 'Negative'],
+  'keine': ['Determiner', 'Negative'],
+  'keinem': ['Determiner', 'Negative'],
+  'keinen': ['Determiner', 'Negative'],
+  'keiner': ['Determiner', 'Negative'],
+  'keines': ['Determiner', 'Negative'],
+
+  // demonstrative determiners
+  'dieser': 'Determiner',
+  'diese': 'Determiner',
+  'dieses': 'Determiner',
+  'diesem': 'Determiner',
+  'diesen': 'Determiner',
+  'jener': 'Determiner',
+  'jene': 'Determiner',
+  'jenes': 'Determiner',
+  'jenem': 'Determiner',
+  'jenen': 'Determiner',
+
+  // quantifier determiners
+  'jeder': 'Determiner',
+  'jede': 'Determiner',
+  'jedes': 'Determiner',
+  'jedem': 'Determiner',
+  'jeden': 'Determiner',
+  'mancher': 'Determiner',
+  'manche': 'Determiner',
+  'manches': 'Determiner',
+  'manchem': 'Determiner',
+  'manchen': 'Determiner',
+  'solcher': 'Determiner',
+  'solche': 'Determiner',
+  'solches': 'Determiner',
+  'solchem': 'Determiner',
+  'solchen': 'Determiner',
+  'welcher': 'Determiner',
+  'welche': 'Determiner',
+  'welches': 'Determiner',
+  'welchem': 'Determiner',
+  'welchen': 'Determiner',
+  'alle': 'Determiner',
+  'allen': 'Determiner',
+  'aller': 'Determiner',
+  'beide': 'Determiner',
+  'beiden': 'Determiner',
+  'viele': 'Determiner',
+  'vielen': 'Determiner',
+  'vieler': 'Determiner',
+  'mehrere': 'Determiner',
+  'mehreren': 'Determiner',
+  'wenige': 'Determiner',
+  'wenigen': 'Determiner',
+  'einige': 'Determiner',
+  'einigen': 'Determiner',
+  'einiger': 'Determiner',
+
+  // common adverbs/particles the lexicon mis-tags
+  'morgen': 'Adverb',// tomorrow — 'Morgen' (morning) is caught by the titlecase pass
+  'bitte': 'Adverb',// please — far more common than 'ich bitte'
+  'etwas': 'Pronoun',// something
+}

package/src/01-one/lexicon/plugin.js

@@ -0,0 +1,14 @@
+import methods from './methods/index.js'
+import lexicon from './lexicon.js'
+import root from './compute/root.js'
+
+export default {
+  compute: { root: root },
+  methods: {
+    two: {
+      transform: methods
+    }
+  },
+  words: lexicon,
+  hooks: ['lexicon']
+}

package/src/01-one/splitter/build.js

@@ -0,0 +1,20 @@
+import { addWord } from './trie.js'
+
+const buildIndex = function (world) {
+  let words = Object.entries(world.model.one.lexicon)
+  let { nouns, values } = world.model.one.splitter
+  words.forEach((a) => {
+    let [w, tag] = a
+    if (tag === 'TextOrdinal' || tag === 'TextCardinal') {
+      addWord(w, values)
+    }
+    if (tag === 'Noun' || tag === 'FemaleNoun' || tag === 'MaleNoun' || tag === 'NeuterNoun') {
+      addWord(w, nouns)
+    }
+  })
+  // misc words
+  addWord('und', values) //'and'
+  addWord('ein', values) //'one'
+  addWord('hunderttausend', values)
+}
+export default buildIndex

package/src/01-one/splitter/compute/splitter.js

@@ -0,0 +1,54 @@
+// links between words
+// https://digital.lib.washington.edu/researchworks/bitstream/handle/1773/44691/Callow_washington_0250O_20779.pdf
+// 'er|en|es|s|e|n
+const hasLink = /^(e[rns]|s|n)/
+import { findMatch } from '../trie.js'
+
+const findSplits = function (str, root) {
+  let found = []
+  while (str.length > 0) {
+    let match = findMatch(str, root)
+    if (!match) {
+      //we done
+      // allow a connector - [foo, en, bar]
+      if (found.length > 0 && hasLink.test(str)) {
+        let tmp = str.replace(hasLink, '')
+        match = findMatch(tmp, root)
+        if (match) {
+          let link = str.match(hasLink)[0]
+          found.push(link)
+          str = str.substr(link.length)
+        }
+      }
+    }
+    if (!match) {
+      found.push(str)
+      break
+    }
+    // found a prefix
+    found.push(match)
+    str = str.substr(match.length)
+  }
+  return found
+}
+
+const splitter = function (view) {
+  let { values } = view.model.one.splitter
+  view.docs.forEach((terms) => {
+    terms.forEach((term) => {
+      // split numbers
+      if (term.tags.has('Value')) {
+        term.splits = findSplits(term.normal, values)
+      }
+      // split nouns
+      // if (term.tags.has('Noun')) {
+      // term.splits = findSplits(term.normal, nouns)
+      // }
+      // split adjectives
+      // if (term.tags.has('Adjective')) {
+      //   term.byChar = findbyChar(term, adjectives)
+      // }
+    })
+  })
+}
+export default { splitter }

package/src/01-one/splitter/model/index.js

@@ -0,0 +1,20 @@
+// import values from './splitter/values.js'
+// import nouns from './splitter/nouns.js'
+// import verbs from './splitter/verbs.js'
+// import adjectives from './splitter/adjectives.js'
+
+let values = { more: [] }
+let nouns = { more: [] }
+let verbs = { more: [] }
+let adjectives = { more: [] }
+
+export default {
+  one: {
+    splitter: {
+      values,
+      nouns,
+      verbs,
+      adjectives
+    }
+  }
+}

package/src/01-one/splitter/plugin.js

@@ -0,0 +1,10 @@
+import model from '../splitter/model/index.js'
+import compute from '../splitter/compute/splitter.js'
+import mutate from './build.js'
+
+export default {
+  mutate,
+  model,
+  compute,
+  // hooks: ['splitter'],
+}

package/src/01-one/splitter/trie.js

@@ -0,0 +1,40 @@
+const addWord = function (str, root) {
+  let chars = str.split('')
+  let node = root
+  chars.forEach(c => {
+    node.more[c] = node.more[c] || { more: {} }
+    node = node.more[c]
+  })
+  node.end = true
+}
+
+const makeTrie = function (arr) {
+  let root = {
+    more: {}
+  }
+  arr.forEach(str => addWord(str, root))
+  return root
+}
+
+const findMatch = function (str, root) {
+  let match = null
+  let chars = str.split('')
+  let node = root
+  for (let i = 0; i < chars.length; i += 1) {
+    if (!node.more[chars[i]]) {
+      // dead end
+      return match
+    }
+    node = node.more[chars[i]]
+    if (node.end === true) {
+      match = chars.slice(0, i + 1).join('')
+    }
+  }
+  return match
+}
+
+export { makeTrie, findMatch, addWord }
+
+// let trie = makeTrie(['car', 'cab', 'stab', 'carbon'])
+// console.dir(trie, { depth: 15 })
+// console.log(findMatch('carbon', trie))

package/src/01-one/tagset/plugin.js

@@ -0,0 +1,11 @@
+import nouns from './tags/nouns.js'
+import verbs from './tags/verbs.js'
+import values from './tags/values.js'
+import dates from './tags/dates.js'
+import misc from './tags/misc.js'
+
+let tags = Object.assign({}, nouns, verbs, values, dates, misc)
+
+export default {
+  tags
+}

package/src/01-one/tagset/tags/dates.js

@@ -0,0 +1,45 @@
+export default {
+  Date: {
+    not: ['Verb', 'Adverb', 'Adjective'],
+  },
+  Month: {
+    is: 'Singular',
+    also: ['Date'],
+    not: ['Year', 'WeekDay', 'Time'],
+  },
+  WeekDay: {
+    is: 'Noun',
+    also: ['Date'],
+  },
+  Year: {
+    is: 'Date',
+    not: ['RomanNumeral'],
+  },
+  FinancialQuarter: {
+    is: 'Date',
+    not: 'Fraction',
+  },
+  // 'easter'
+  Holiday: {
+    is: 'Date',
+    also: ['Noun'],
+  },
+  // 'summer'
+  Season: {
+    is: 'Date',
+  },
+  Timezone: {
+    is: 'Noun',
+    also: ['Date'],
+    not: ['ProperNoun'],
+  },
+  Time: {
+    is: 'Date',
+    not: ['AtMention'],
+  },
+  // 'months'
+  Duration: {
+    is: 'Noun',
+    also: ['Date'],
+  },
+}

package/src/01-one/tagset/tags/misc.js

@@ -0,0 +1,70 @@
+const anything = ['Noun', 'Verb', 'Adjective', 'Adverb', 'Value', 'QuestionWord']
+
+export default {
+  Adjective: {
+    not: ['Noun', 'Verb', 'Adverb', 'Value'],
+  },
+  Comparable: {
+    is: 'Adjective',
+  },
+  Comparative: {
+    is: 'Adjective',
+  },
+  Superlative: {
+    is: 'Adjective',
+    not: ['Comparative'],
+  },
+  NumberRange: {},
+  Adverb: {
+    not: ['Noun', 'Verb', 'Adjective', 'Value'],
+  },
+
+  Determiner: {
+    not: ['Noun', 'Verb', 'Adjective', 'Adverb', 'QuestionWord', 'Conjunction'], //allow 'a' to be a Determiner/Value
+  },
+  Conjunction: {
+    not: anything,
+  },
+  Preposition: {
+    not: ['Noun', 'Verb', 'Adjective', 'Adverb', 'QuestionWord'],
+  },
+  QuestionWord: {
+    not: ['Determiner'],
+  },
+  Currency: {
+    is: 'Noun',
+  },
+  Expression: {
+    not: ['Noun', 'Adjective', 'Verb', 'Adverb'],
+  },
+  Abbreviation: {},
+  Url: {
+    not: ['HashTag', 'PhoneNumber', 'Verb', 'Adjective', 'Value', 'AtMention', 'Email'],
+  },
+  PhoneNumber: {
+    not: ['HashTag', 'Verb', 'Adjective', 'Value', 'AtMention', 'Email'],
+  },
+  HashTag: {},
+  AtMention: {
+    is: 'Noun',
+    not: ['HashTag', 'Email'],
+  },
+  Emoji: {
+    not: ['HashTag', 'Verb', 'Adjective', 'Value', 'AtMention'],
+  },
+  Emoticon: {
+    not: ['HashTag', 'Verb', 'Adjective', 'Value', 'AtMention'],
+  },
+  Email: {
+    not: ['HashTag', 'Verb', 'Adjective', 'Value', 'AtMention'],
+  },
+  Acronym: {
+    not: ['Plural', 'RomanNumeral'],
+  },
+  Negative: {
+    not: ['Noun', 'Adjective', 'Value'],
+  },
+  Condition: {
+    not: ['Verb', 'Adjective', 'Noun', 'Value'],
+  },
+}