de-compromise 0.0.9 → 0.1.0

package/src/02-two/preTagger/compute/2nd-pass/prefix-lookup.js

@@ -0,0 +1,33 @@
+
+//sweep-through all prefixes
+const prefixLoop = function (str = '', prefixes = []) {
+  const len = str.length
+  let max = 10
+  if (max > len - 3) {
+    max = len - 3
+  }
+  for (let i = max; i > 2; i -= 1) {
+    let prefix = str.substring(0, i)
+    if (prefixes[prefix.length].hasOwnProperty(prefix) === true) {
+      let tag = prefixes[prefix.length][prefix]
+      return tag
+    }
+  }
+  return null
+}
+
+// give 'overwork' the same tag as 'work'
+const checkPrefix = function (terms, i, world) {
+  let setTag = world.methods.one.setTag
+  let term = terms[i]
+  if (term.tags.size === 0) {
+    let tag = prefixLoop(term.normal, world.model.two.prefixPatterns)
+    if (tag !== null) {
+      setTag([term], tag, world, false, '2-prefix')
+      term.confidence = 0.5
+      return true
+    }
+  }
+  return null
+}
+export default checkPrefix

package/src/02-two/preTagger/compute/2nd-pass/suffix-lookup.js

@@ -0,0 +1,44 @@
+
+//sweep-through all suffixes
+const suffixLoop = function (str = '', suffixes = []) {
+  const len = str.length
+  let max = 10
+  if (len <= max) {
+    max = len - 1
+  }
+  for (let i = max; i > 1; i -= 1) {
+    let suffix = str.substr(len - i, len)
+    if (suffixes[suffix.length].hasOwnProperty(suffix) === true) {
+      // console.log(suffix)
+      let tag = suffixes[suffix.length][suffix]
+      return tag
+    }
+  }
+  return null
+}
+
+// decide tag from the ending of the word
+const suffixCheck = function (terms, i, world) {
+  let setTag = world.methods.one.setTag
+  let suffixes = world.model.two.suffixPatterns
+  let term = terms[i]
+  if (term.tags.size === 0) {
+    let tag = suffixLoop(term.normal, suffixes)
+    if (tag !== null) {
+      setTag([term], tag, world, false, '2-suffix')
+      term.confidence = 0.7
+      return true
+    }
+    // try implicit form of word, too
+    if (term.implicit) {
+      tag = suffixLoop(term.implicit, suffixes)
+      if (tag !== null) {
+        setTag([term], tag, world, false, '2-implicit-suffix')
+        term.confidence = 0.7
+        return true
+      }
+    }
+  }
+  return null
+}
+export default suffixCheck

package/src/02-two/preTagger/compute/index.js

@@ -0,0 +1,42 @@
+// 1st pass
+import checkRegex from './1st-pass/regex.js'
+import titleCase from './1st-pass/titlecase.js'
+import checkYear from './1st-pass/year.js'
+// 2nd pass
+import acronym from './2nd-pass/acronym.js'
+// import neighbours from './2nd-pass/neighbours.js'
+import fallback from './2nd-pass/fallback.js'
+import suffixCheck from './2nd-pass/suffix-lookup.js'
+import prefixCheck from './2nd-pass/prefix-lookup.js'
+
+// these methods don't care about word-neighbours
+const firstPass = function (terms, world) {
+  for (let i = 0; i < terms.length; i += 1) {
+    //  is it titlecased?
+    let found = titleCase(terms, i, world)
+    // try look-like rules
+    found = found || checkRegex(terms, i, world)
+    // turn '1993' into a year
+    checkYear(terms, i, world)
+  }
+}
+const secondPass = function (terms, world) {
+  for (let i = 0; i < terms.length; i += 1) {
+    let found = acronym(terms, i, world)
+    found = found || suffixCheck(terms, i, world)
+    found = found || prefixCheck(terms, i, world)
+    // found = found || neighbours(terms, i, world)
+    found = found || fallback(terms, i, world)
+  }
+}
+
+
+const tagger = function (view) {
+  let world = view.world
+  view.docs.forEach(terms => {
+    firstPass(terms, world)
+    secondPass(terms, world)
+  })
+  return view
+}
+export default tagger

package/src/02-two/preTagger/model/index.js

@@ -0,0 +1,14 @@
+import regexNormal from './regex/regex-normal.js'
+import regexNumbers from './regex/regex-numbers.js'
+import regexText from './regex/regex-text.js'
+import suffixPatterns from './suffixes.js'
+import prefixPatterns from './prefixes.js'
+
+
+export default {
+  regexNormal,
+  regexNumbers,
+  regexText,
+  suffixPatterns,
+  prefixPatterns
+}

package/src/02-two/preTagger/model/prefixes.js

@@ -0,0 +1,46 @@
+const val = 'TextValue'
+
+export default [
+  null,
+  {
+    // one-letter prefixes
+  },
+  {
+    // two-letter prefixes
+  },
+  {
+    // three-letter prefixes
+  },
+  {
+    // four-letter prefixes
+    "eins": val,
+    "zwei": val,
+    "drei": val,
+    "vier": val,
+    "fünf": val,
+    "acht": val,
+    "neun": val,
+  },
+  {
+    // five-letter prefixes
+    "sechs": val,
+    "sieben": val,
+  },
+  {
+    // six-letter prefixes
+  },
+  {
+    // seven-letter prefixes
+  },
+  {
+    // eight-letter prefixes
+  },
+  {
+    // nine-letter prefixes
+    einhunder: val
+  },
+  {
+    // ten-letter prefixes
+    zweihunder: val
+  },
+]

package/src/02-two/preTagger/model/regex/regex-normal.js

@@ -0,0 +1,25 @@
+export default [
+  //web tags
+  [/^[\w.]+@[\w.]+\.[a-z]{2,3}$/, 'Email'],
+  [/^(https?:\/\/|www\.)+\w+\.[a-z]{2,3}/, 'Url', 'http..'],
+  [/^[a-z0-9./].+\.(com|net|gov|org|ly|edu|info|biz|dev|ru|jp|de|in|uk|br|io|ai)/, 'Url', '.com'],
+
+  // timezones
+  [/^[PMCE]ST$/, 'Timezone', 'EST'],
+
+  //names
+  [/^ma?c'.*/, 'LastName', "mc'neil"],
+  [/^o'[drlkn].*/, 'LastName', "o'connor"],
+  [/^ma?cd[aeiou]/, 'LastName', 'mcdonald'],
+
+  //slang things
+  [/^(lol)+[sz]$/, 'Expression', 'lol'],
+  [/^wo{2,}a*h?$/, 'Expression', 'wooah'],
+  [/^(hee?){2,}h?$/, 'Expression', 'hehe'],
+  [/^(un|de|re)\\-[a-z\u00C0-\u00FF]{2}/, 'Verb', 'un-vite'],
+
+  // m/h
+  [/^(m|k|cm|km)\/(s|h|hr)$/, 'Unit', '5 k/m'],
+  // μg/g
+  [/^(ug|ng|mg)\/(l|m3|ft3)$/, 'Unit', 'ug/L'],
+]

package/src/02-two/preTagger/model/regex/regex-numbers.js

@@ -0,0 +1,62 @@
+export default [
+
+  [/^@1?[0-9](am|pm)$/i, 'Time', '3pm'],
+  [/^[0-9]{2}[:.][0-9]{2}(am|pm)?$/i, 'Time', '13.30pm'],
+  [/^'[0-9]{2}$/, 'Year'],
+  // times
+  [/^[012]?[0-9](:[0-5][0-9])(:[0-5][0-9])$/, 'Time', '3:12:31'],
+  [/^[012]?[0-9](:[0-5][0-9])?(:[0-5][0-9])? ?(am|pm)$/i, 'Time', '1:12pm'],
+  [/^[012]?[0-9](:[0-5][0-9])(:[0-5][0-9])? ?(am|pm)?$/i, 'Time', '1:12:31pm'], //can remove?
+
+  // iso-dates
+  [/^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}/i, 'Date', 'iso-date'],
+  [/^[0-9]{1,4}-[0-9]{1,2}-[0-9]{1,4}$/, 'Date', 'iso-dash'],
+  [/^[0-9]{1,4}\/[0-9]{1,2}\/[0-9]{1,4}$/, 'Date', 'iso-slash'],
+  [/^[0-9]{1,4}\.[0-9]{1,2}\.[0-9]{1,4}$/, 'Date', 'iso-dot'],
+  [/^[0-9]{1,4}-[a-z]{2,9}-[0-9]{1,4}$/i, 'Date', '12-dec-2019'],
+
+  // timezones
+  [/^utc ?[+-]?[0-9]+$/, 'Timezone', 'utc-9'],
+  [/^(gmt|utc)[+-][0-9]{1,2}$/i, 'Timezone', 'gmt-3'],
+
+  //phone numbers
+  [/^[0-9]{3}-[0-9]{4}$/, 'PhoneNumber', '421-0029'],
+  [/^(\+?[0-9][ -])?[0-9]{3}[ -]?[0-9]{3}-[0-9]{4}$/, 'PhoneNumber', '1-800-'],
+
+
+  //money
+  //like $5.30
+  [
+    /^[-+]?[$\xA2-\xA5\u058F\u060B\u09F2\u09F3\u09FB\u0AF1\u0BF9\u0E3F\u17DB\u20A0-\u20BD\uA838\uFDFC\uFE69\uFF04\uFFE0\uFFE1\uFFE5\uFFE6][-+]?[0-9]+(,[0-9]{3})*(\.[0-9]+)?([kmb]|bn)?\+?$/,
+    ['Money', 'Value'],
+    '$5.30',
+  ],
+  //like 5.30$
+  [
+    /^[-+]?[0-9]+(,[0-9]{3})*(\.[0-9]+)?[$\xA2-\xA5\u058F\u060B\u09F2\u09F3\u09FB\u0AF1\u0BF9\u0E3F\u17DB\u20A0-\u20BD\uA838\uFDFC\uFE69\uFF04\uFFE0\uFFE1\uFFE5\uFFE6]\+?$/,
+    ['Money', 'Value'],
+    '5.30£',
+  ],
+  //like
+  [/^[-+]?[$£]?[0-9]([0-9,.])+(usd|eur|jpy|gbp|cad|aud|chf|cny|hkd|nzd|kr|rub)$/i, ['Money', 'Value'], '$400usd'],
+
+  //numbers
+  // german-format - '.' groups thousands
+  [/^[-+]?[0-9]{1,3}(\.[0-9]{3})+(,[0-9]+)?$/, ['Cardinal', 'NumericValue'], '1.234.567'],
+  // 50 | -50 | 3.23  | 5,999.0  | 10+
+  [/^[-+]?[0-9]+(,[0-9]{3})*(\.[0-9]+)?\+?$/, ['Cardinal', 'NumericValue'], '5,999'],
+  [/^[-+]?[0-9]+(,[0-9]{3})*(\.[0-9]+)?(st|nd|rd|r?th)$/, ['Ordinal', 'NumericValue'], '53rd'],
+  // .73th
+  [/^\.[0-9]+\+?$/, ['Cardinal', 'NumericValue'], '.73th'],
+  //percent
+  [/^[-+]?[0-9]+(,[0-9]{3})*(\.[0-9]+)?%\+?$/, ['Percent', 'Cardinal', 'NumericValue'], '-4%'],
+  [/^\.[0-9]+%$/, ['Percent', 'Cardinal', 'NumericValue'], '.3%'],
+  //fraction
+  [/^[0-9]{1,4}\/[0-9]{1,4}(st|nd|rd|th)?s?$/, ['Fraction', 'NumericValue'], '2/3rds'],
+  //range
+  [/^[0-9.]{1,3}[a-z]{0,2}[-–—][0-9]{1,3}[a-z]{0,2}$/, ['Value', 'NumberRange'], '3-4'],
+  //time-range
+  [/^[0-9]{1,2}(:[0-9][0-9])?(am|pm)? ?[-–—] ?[0-9]{1,2}(:[0-9][0-9])?(am|pm)$/, ['Time', 'NumberRange'], '3-4pm'],
+  //with unit
+  [/^[0-9.]+([a-z]{1,4})$/, 'Value', '9km'],
+]

package/src/02-two/preTagger/model/regex/regex-text.js

@@ -0,0 +1,17 @@
+export default [
+  // #coolguy
+  [/^#[a-z0-9_\u00C0-\u00FF]{2,}$/i, 'HashTag'],
+
+  // @spencermountain
+  [/^@\w{2,}$/, 'AtMention'],
+
+  // period-ones acronyms - f.b.i.
+  [/^([A-ZÄÖÜ]\.){2}[A-ZÄÖÜ]?/i, ['Acronym', 'Noun'], 'F.B.I'], //ascii-only
+
+  // ending-apostrophes
+  [/.{3}[lkmnp]in['‘’‛‵′`´]$/, 'Gerund', "chillin'"],
+  [/.{4}s['‘’‛‵′`´]$/, 'Possessive', "flanders'"],
+
+  // german ordinals '4.'
+  // [/^[0-9]+\.$/, 'Ordinal'],
+]

package/src/02-two/preTagger/model/suffixes.js

@@ -0,0 +1,171 @@
+const rb = 'Adverb'
+const nn = 'Noun'
+const vb = 'Verb'
+const jj = 'Adjective'
+const ord = 'TextOrdinal'
+const card = 'TextCardinal'
+
+
+export default [
+  null,
+  {
+    // one-letter suffixes
+    s: nn,
+    l: nn,
+    a: nn,
+    k: nn,
+    d: nn,
+    z: nn,
+  },
+  {
+    // two-letter suffixes
+    ig: jj,
+    ls: rb,
+    // er: nn,
+    el: nn,
+    et: vb,
+    gt: vb,
+    lt: vb,
+    // en: vb,
+    ns: nn,
+    ts: nn,
+    // ht: vb,
+    'ßt': vb
+  },
+  {
+    // three-letter suffixes
+    che: jj,
+    ige: jj,
+    // ger: jj,
+    tig: jj,
+    end: jj,
+    // mal: rb,
+    hin: rb,
+    ung: nn,
+    ion: nn,
+    ter: nn,
+    ert: vb,
+    tet: vb,
+    ten: vb,
+    eln: vb,
+    ern: vb,
+    gen: vb,
+    fen: vb,
+    // ben: vb,
+    // ren: vb,
+    // fen: vb,
+    // men: vb,
+    igt: vb,
+    tzt: vb,
+    det: vb,
+    elt: vb,
+    ete: vb,
+    elf: card,//eleven
+  },
+  {
+    // four-letter suffixes
+    chen: jj,
+    lich: jj,
+    igen: jj,
+    cher: jj,
+    ende: jj,
+    isch: jj,
+    enen: jj,
+    tige: jj,
+    tens: rb,
+    mals: rb,
+    rung: nn,
+    iert: vb,
+    eben: vb,
+    ehen: vb,
+    mmen: vb,
+    llen: vb,
+    ngen: vb,
+    tzen: vb,
+    hren: vb,
+    cken: vb,
+    ssen: vb,
+    eßen: vb,
+    hnen: vb,
+    ufen: vb,
+    lten: vb,
+    hten: vb,
+    zehn: card,//10s
+    eins: card,
+    zwei: card,
+    drei: card,
+    vier: card,
+    fünf: card,
+    acht: card,
+    neun: card,
+    // zehn: card,
+    // lten: vb,
+    // ssen: vb
+  },
+  {
+    // five-letter suffixes
+    ische: jj,
+    zwölf: card,
+    sechs: card,
+    liche: jj,
+    // enden: jj,
+    tlich: jj,
+    // tigen: jj,
+    stens: rb,
+    falls: rb,
+    weise: rb,
+    seits: rb,
+    ungen: nn,
+    erung: nn,
+    ieren: vb,
+  },
+  {
+    // six-letter suffixes
+    igsten: jj,
+    ischen: jj,
+    sieben: card,
+    lichen: jj,
+    tische: jj,
+    nische: jj,
+    zehnte: ord,
+    zigste: ord,
+    ßigste: ord,
+  },
+  {
+    // seven-letter suffixes
+    zigsten: ord,
+    tischen: jj,
+    tlichen: jj,
+    nischen: jj,
+    zwanzig: card,//20
+    dreißig: card,//30
+    vierzig: card,//40
+    fünfzig: card,//50
+    sechzig: card, //60
+    siebzig: card, //70
+    achtzig: card, //80
+    neunzig: card, //90
+    hundert: card,//100
+    tausend: card,//1,000
+  },
+  // eight-letter suffixes
+  {
+    dreizehn: card,
+    vierzehn: card,
+    fünfzehn: card,
+    sechzehn: card,
+    siebzehn: card,
+    achtzehn: card,
+    neunzehn: card,
+
+  },
+  // nine-letter suffixes
+  {
+  },
+  // ten-letter suffixes
+  {
+    hundertste: ord,
+    tausendste: ord,
+    millionste: ord
+  },
+]

package/src/02-two/preTagger/plugin.js

@@ -0,0 +1,12 @@
+import tagger from './compute/index.js'
+import model from './model/index.js'
+
+export default {
+  compute: {
+    tagger
+  },
+  model: {
+    two: model
+  },
+  hooks: ['tagger']
+}

package/src/03-three/adjectives/api.js

@@ -0,0 +1,34 @@
+export const getNth = (doc, n) => (typeof n === 'number' ? doc.eq(n) : doc)
+
+// get root form of adjective
+const getRoot = function (m) {
+  m = m.eq(0).compute('root')
+  return m.text('root')
+}
+
+const api = function (View) {
+  class Adjectives extends View {
+    constructor(document, pointer, groups) {
+      super(document, pointer, groups)
+      this.viewType = 'Adjectives'
+    }
+    conjugate(n) {
+      const methods = this.methods.two.transform.adjective
+      const { inflect, toRoot } = methods
+      return getNth(this, n).map(m => {
+        let str = getRoot(m, methods)
+        let root = toRoot(str) || str
+        let res = inflect(root)
+        res.infinitive = root
+        return res
+      }, [])
+    }
+  }
+
+  View.prototype.adjectives = function (n) {
+    let m = this.match('#Adjective')
+    m = getNth(m, n)
+    return new Adjectives(this.document, m.pointer)
+  }
+}
+export default api

package/src/03-three/adjectives/plugin.js

@@ -0,0 +1,5 @@
+import api from './api.js'
+
+export default {
+  api,
+}

package/src/03-three/nouns/api.js

@@ -0,0 +1,74 @@
+export const getNth = (doc, n) => (typeof n === 'number' ? doc.eq(n) : doc)
+
+// get (lower-cased) singular form of noun
+const getRoot = function (m) {
+  m = m.eq(0).compute('root')
+  return m.text('root')
+}
+
+// german nouns are capitalized - keep the casing of the original word
+const matchCase = function (m, str) {
+  if (/^[A-ZÄÖÜ]/.test(m.text().trim())) {
+    return str.charAt(0).toUpperCase() + str.substring(1)
+  }
+  return str
+}
+
+// 'hunde' → 'hund' → 'hunde' round-trips, so it is already a plural,
+// even when the tagger could not tell - 'die Hunde' vs 'der Kunde'
+const isPluralForm = function (methods, str) {
+  let single = methods.toSingular(str)
+  return single !== str && methods.toPlural(single).one === str
+}
+
+const api = function (View) {
+  class Nouns extends View {
+    constructor(document, pointer, groups) {
+      super(document, pointer, groups)
+      this.viewType = 'Nouns'
+    }
+    conjugate(n) {
+      const methods = this.methods.two.transform.noun
+      return getNth(this, n).map(m => {
+        let root = getRoot(m)
+        return {
+          singular: root,
+          plural: methods.toPlural(root).one,
+        }
+      }, [])
+    }
+
+    isPlural(n) {
+      return getNth(this, n).if('#Plural')
+    }
+    toPlural(n) {
+      const methods = this.methods.two.transform.noun
+      return getNth(this, n).not('#Plural').map(m => {
+        if (isPluralForm(methods, m.text('normal'))) {
+          return m
+        }
+        let str = getRoot(m)
+        let plural = methods.toPlural(str).one
+        return m.replaceWith(matchCase(m, plural))
+      })
+    }
+    toSingular(n) {
+      const methods = this.methods.two.transform.noun
+      return getNth(this, n).not('#Singular').map(m => {
+        let str = m.text('normal')
+        if (!m.has('#Plural') && !isPluralForm(methods, str)) {
+          return m
+        }
+        let singular = methods.toSingular(str)
+        return m.replaceWith(matchCase(m, singular))
+      })
+    }
+  }
+
+  View.prototype.nouns = function (n) {
+    let m = this.match('#Noun+')
+    m = getNth(m, n)
+    return new Nouns(this.document, m.pointer)
+  }
+}
+export default api

package/src/03-three/nouns/plugin.js

@@ -0,0 +1,5 @@
+import api from './api.js'
+
+export default {
+  api,
+}