de-compromise 0.0.9 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. package/README.md +1 -1
  2. package/builds/de-compromise.cjs +2357 -1687
  3. package/builds/de-compromise.min.js +1 -1
  4. package/builds/de-compromise.mjs +1 -1
  5. package/package.json +21 -14
  6. package/src/01-one/lexicon/_data.js +32 -0
  7. package/src/01-one/lexicon/compute/root.js +55 -0
  8. package/src/01-one/lexicon/lexicon.js +95 -0
  9. package/src/01-one/lexicon/methods/_data.js +193 -0
  10. package/src/01-one/lexicon/methods/adjectives/_test-back.js +29 -0
  11. package/src/01-one/lexicon/methods/adjectives/_test.js +49 -0
  12. package/src/01-one/lexicon/methods/adjectives/inflect.js +27 -0
  13. package/src/01-one/lexicon/methods/adjectives/model.js +126 -0
  14. package/src/01-one/lexicon/methods/adjectives/new/adjective.js +8 -0
  15. package/src/01-one/lexicon/methods/adjectives/new/comparative.js +7 -0
  16. package/src/01-one/lexicon/methods/adjectives/new/superlative.js +159 -0
  17. package/src/01-one/lexicon/methods/adjectives/toRoot.js +47 -0
  18. package/src/01-one/lexicon/methods/index.js +56 -0
  19. package/src/01-one/lexicon/methods/models.js +10 -0
  20. package/src/01-one/lexicon/methods/nouns/_test-back.js +45 -0
  21. package/src/01-one/lexicon/methods/nouns/new.js +56 -0
  22. package/src/01-one/lexicon/methods/nouns/toPlural.js +49 -0
  23. package/src/01-one/lexicon/methods/nouns/toSingular.js +46 -0
  24. package/src/01-one/lexicon/methods/verbs/conjugate.js +87 -0
  25. package/src/01-one/lexicon/methods/verbs/custom/_test-from.js +53 -0
  26. package/src/01-one/lexicon/methods/verbs/custom/_test-to.js +50 -0
  27. package/src/01-one/lexicon/methods/verbs/custom/fromPastParticiple.js +37 -0
  28. package/src/01-one/lexicon/methods/verbs/custom/rules.js +182 -0
  29. package/src/01-one/lexicon/methods/verbs/custom/toPastParticiple.js +54 -0
  30. package/src/01-one/lexicon/methods/verbs/irregulars.js +202 -0
  31. package/src/01-one/lexicon/methods/verbs/toRoot.js +55 -0
  32. package/src/01-one/lexicon/misc.js +78 -0
  33. package/src/01-one/lexicon/plugin.js +14 -0
  34. package/src/01-one/splitter/build.js +20 -0
  35. package/src/01-one/splitter/compute/splitter.js +54 -0
  36. package/src/01-one/splitter/model/index.js +20 -0
  37. package/src/01-one/splitter/plugin.js +10 -0
  38. package/src/01-one/splitter/trie.js +40 -0
  39. package/src/01-one/tagset/plugin.js +11 -0
  40. package/src/01-one/tagset/tags/dates.js +45 -0
  41. package/src/01-one/tagset/tags/misc.js +70 -0
  42. package/src/01-one/tagset/tags/nouns.js +116 -0
  43. package/src/01-one/tagset/tags/values.js +47 -0
  44. package/src/01-one/tagset/tags/verbs.js +64 -0
  45. package/src/01-one/tokenizer/contractions.js +40 -0
  46. package/src/01-one/tokenizer/is-sentence.js +37 -0
  47. package/src/01-one/tokenizer/plugin.js +11 -0
  48. package/src/01-one/tokenizer/unicode.js +55 -0
  49. package/src/02-two/postTagger/matches.js +47 -0
  50. package/src/02-two/postTagger/plugin.js +8 -0
  51. package/src/02-two/preTagger/compute/1st-pass/regex.js +43 -0
  52. package/src/02-two/preTagger/compute/1st-pass/titlecase.js +21 -0
  53. package/src/02-two/preTagger/compute/1st-pass/year.js +54 -0
  54. package/src/02-two/preTagger/compute/2nd-pass/acronym.js +82 -0
  55. package/src/02-two/preTagger/compute/2nd-pass/fallback.js +27 -0
  56. package/src/02-two/preTagger/compute/2nd-pass/prefix-lookup.js +33 -0
  57. package/src/02-two/preTagger/compute/2nd-pass/suffix-lookup.js +44 -0
  58. package/src/02-two/preTagger/compute/index.js +42 -0
  59. package/src/02-two/preTagger/model/index.js +14 -0
  60. package/src/02-two/preTagger/model/prefixes.js +46 -0
  61. package/src/02-two/preTagger/model/regex/regex-normal.js +25 -0
  62. package/src/02-two/preTagger/model/regex/regex-numbers.js +62 -0
  63. package/src/02-two/preTagger/model/regex/regex-text.js +17 -0
  64. package/src/02-two/preTagger/model/suffixes.js +171 -0
  65. package/src/02-two/preTagger/plugin.js +12 -0
  66. package/src/03-three/adjectives/api.js +34 -0
  67. package/src/03-three/adjectives/plugin.js +5 -0
  68. package/src/03-three/nouns/api.js +74 -0
  69. package/src/03-three/nouns/plugin.js +5 -0
  70. package/src/03-three/numbers/api.js +259 -0
  71. package/src/03-three/numbers/data.js +77 -0
  72. package/src/03-three/numbers/format/index.js +27 -0
  73. package/src/03-three/numbers/format/toText.js +118 -0
  74. package/src/03-three/numbers/parse/fromText.js +52 -0
  75. package/src/03-three/numbers/parse/index.js +63 -0
  76. package/src/03-three/numbers/plugin.js +5 -0
  77. package/src/03-three/verbs/api.js +41 -0
  78. package/src/03-three/verbs/plugin.js +5 -0
  79. package/src/_lib.js +4 -0
  80. package/src/_version.js +1 -0
  81. package/src/index.js +53 -0
  82. package/types/index.d.ts +46 -0
  83. package/types/misc.ts +92 -0
  84. package/types/view/de.ts +61 -0
  85. package/types/view/one.ts +235 -0
@@ -0,0 +1,116 @@
1
+ const entity = ['Person', 'Place', 'Organization']
2
+
3
+ export default {
4
+ Noun: {
5
+ not: ['Verb', 'Adjective', 'Adverb', 'Value', 'Determiner'],
6
+ },
7
+ Singular: {
8
+ is: 'Noun',
9
+ not: ['Plural'],
10
+ },
11
+ ProperNoun: {
12
+ is: 'Noun',
13
+ },
14
+ Person: {
15
+ is: 'Singular',
16
+ also: ['ProperNoun'],
17
+ not: ['Place', 'Organization', 'Date'],
18
+ },
19
+ FirstName: {
20
+ is: 'Person',
21
+ },
22
+ MaleName: {
23
+ is: 'FirstName',
24
+ not: ['FemaleName', 'LastName'],
25
+ },
26
+ FemaleName: {
27
+ is: 'FirstName',
28
+ not: ['MaleName', 'LastName'],
29
+ },
30
+ LastName: {
31
+ is: 'Person',
32
+ not: ['FirstName'],
33
+ },
34
+ Honorific: {
35
+ is: 'Noun',
36
+ not: ['FirstName', 'LastName', 'Value'],
37
+ },
38
+ Place: {
39
+ is: 'Singular',
40
+ not: ['Person', 'Organization'],
41
+ },
42
+ Country: {
43
+ is: 'Place',
44
+ also: ['ProperNoun'],
45
+ not: ['City'],
46
+ },
47
+ City: {
48
+ is: 'Place',
49
+ also: ['ProperNoun'],
50
+ not: ['Country'],
51
+ },
52
+ Region: {
53
+ is: 'Place',
54
+ also: ['ProperNoun'],
55
+ },
56
+ Address: {
57
+ // is: 'Place',
58
+ },
59
+ Organization: {
60
+ is: 'ProperNoun',
61
+ not: ['Person', 'Place'],
62
+ },
63
+ SportsTeam: {
64
+ is: 'Organization',
65
+ },
66
+ School: {
67
+ is: 'Organization',
68
+ },
69
+ Company: {
70
+ is: 'Organization',
71
+ },
72
+ Plural: {
73
+ is: 'Noun',
74
+ not: ['Singular'],
75
+ },
76
+ Uncountable: {
77
+ is: 'Noun',
78
+ },
79
+ Pronoun: {
80
+ is: 'Noun',
81
+ not: entity,
82
+ },
83
+ Actor: {
84
+ is: 'Noun',
85
+ not: entity,
86
+ },
87
+ Activity: {
88
+ is: 'Noun',
89
+ not: ['Person', 'Place'],
90
+ },
91
+ Unit: {
92
+ is: 'Noun',
93
+ not: entity,
94
+ },
95
+ Demonym: {
96
+ is: 'Noun',
97
+ also: ['ProperNoun'],
98
+ not: entity,
99
+ },
100
+ // standalone — 'meinem' is #Possessive as both a determiner ('meinem Freund')
101
+ // and a pronoun ('das ist meins')
102
+ Possessive: {},
103
+ // german genders
104
+ MaleNoun: {
105
+ is: 'Noun',
106
+ not: ['FemaleNoun', 'NeuterNoun'],
107
+ },
108
+ FemaleNoun: {
109
+ is: 'Noun',
110
+ not: ['MaleNoun', 'NeuterNoun'],
111
+ },
112
+ NeuterNoun: {
113
+ is: 'Noun',
114
+ not: ['MaleNoun', 'FemaleNoun'],
115
+ },
116
+ }
@@ -0,0 +1,47 @@
1
+ export default {
2
+ Value: {
3
+ not: ['Verb', 'Adjective', 'Adverb'],
4
+ },
5
+ Ordinal: {
6
+ is: 'Value',
7
+ not: ['Cardinal'],
8
+ },
9
+ Cardinal: {
10
+ is: 'Value',
11
+ not: ['Ordinal'],
12
+ },
13
+ Fraction: {
14
+ is: 'Value',
15
+ not: ['Noun'],
16
+ },
17
+ Multiple: {
18
+ is: 'Value',
19
+ },
20
+ RomanNumeral: {
21
+ is: 'Cardinal',
22
+ not: ['TextValue'],
23
+ },
24
+ TextValue: {
25
+ is: 'Value',
26
+ not: ['NumericValue'],
27
+ },
28
+ TextOrdinal: {
29
+ is: 'TextValue',
30
+ also: ['Ordinal']
31
+ },
32
+ TextCardinal: {
33
+ is: 'TextValue',
34
+ also: ['Cardinal']
35
+ },
36
+
37
+ NumericValue: {
38
+ is: 'Value',
39
+ not: ['TextValue'],
40
+ },
41
+ Money: {
42
+ is: 'Cardinal',
43
+ },
44
+ Percent: {
45
+ is: 'Value',
46
+ },
47
+ }
@@ -0,0 +1,64 @@
1
+ export default {
2
+ Verb: {
3
+ not: ['Noun', 'Adjective', 'Adverb', 'Value', 'Expression'],
4
+ },
5
+ PresentTense: {
6
+ is: 'Verb',
7
+ not: ['PastTense'],
8
+ },
9
+ Infinitive: {
10
+ is: 'PresentTense',
11
+ not: ['Gerund'],
12
+ },
13
+ Imperative: {
14
+ is: 'Infinitive',
15
+ },
16
+ Subjunctive: {
17
+ is: 'Verb',
18
+ },
19
+ Participle: {
20
+ is: 'Verb',
21
+ },
22
+ Gerund: {
23
+ is: 'PresentTense',
24
+ not: ['Copula'],
25
+ },
26
+ PastTense: {
27
+ is: 'Verb',
28
+ not: ['PresentTense', 'Gerund'],
29
+ },
30
+ Copula: {
31
+ is: 'Verb',
32
+ },
33
+ Modal: {
34
+ is: 'Verb',
35
+ not: ['Infinitive'],
36
+ },
37
+ PerfectTense: {
38
+ is: 'Verb',
39
+ not: ['Gerund'],
40
+ },
41
+ Pluperfect: {
42
+ is: 'Verb',
43
+ },
44
+ PhrasalVerb: {
45
+ is: 'Verb',
46
+ },
47
+ Particle: {
48
+ is: 'PhrasalVerb',
49
+ not: ['PastTense', 'PresentTense', 'Copula', 'Gerund'],
50
+ },
51
+ Auxiliary: {
52
+ is: 'Verb',
53
+ not: ['PastTense', 'PresentTense', 'Gerund', 'Conjunction'],
54
+ },
55
+ FirstPerson: {
56
+ is: 'Verb'
57
+ },
58
+ SecondPerson: {
59
+ is: 'Verb'
60
+ },
61
+ ThirdPerson: {
62
+ is: 'Verb'
63
+ },
64
+ }
@@ -0,0 +1,40 @@
1
+ // https://www.fluentu.com/blog/german/german-contractions/
2
+ export default [
3
+ { word: 'am', out: ['an', 'dem'] }, //(on the / at the)
4
+ { word: 'ans', out: ['an', 'das'] }, //(on the / at the)
5
+ { word: 'aufs', out: ['auf', 'das'] }, //(on the / at the)
6
+ { word: 'beim', out: ['bei', 'dem'] }, //(at the / in the / with the)
7
+ { word: 'durchs', out: ['durch', 'das'] }, //(through the)
8
+ { word: 'fürs', out: ['für', 'das'] }, //(for the)
9
+ { word: 'hinterm', out: ['hinter', 'dem'] }, //(behind the)
10
+ { word: 'ins', out: ['in', 'das'] }, //(in the / into the / to the)
11
+ { word: 'im', out: ['in', 'dem'] }, //(at / in the)
12
+ { word: 'übers', out: ['über', 'das'] }, //(over the / about the)
13
+ { word: 'ums', out: ['um', 'das'] }, //(at the / around)
14
+ { word: 'unters', out: ['unter', 'das'] }, //(under the)
15
+ { word: 'unterm', out: ['unter', 'dem'] }, //(under the)
16
+ { word: 'vom', out: ['von', 'dem'] }, //(from the)
17
+ { word: 'vors', out: ['vor', 'das'] }, //(in front of the)
18
+ { word: 'vorm', out: ['vor', 'dem'] }, //(in front of the / from)
19
+ { word: 'zum', out: ['zu', 'dem'] }, //(to the / to)
20
+ { word: 'zur', out: ['zu', 'der'] }, //(to the / to)
21
+
22
+ // { word: "n", out: ['das', 'ans'] },
23
+ // { word: "an", out: ['dem', 'am'] },
24
+ // { word: "auf", out: ['das', 'aufs'] },
25
+ // { word: "bei", out: ['dem', 'beim'] },
26
+ // { word: "durch", out: ['das', 'durchs'] },
27
+ // { word: "für", out: ['das', 'fürs'] },
28
+ // { word: "in", out: ['das', 'ins'] },
29
+ // { word: "in", out: ['dem', 'im'] },
30
+ // { word: "über", out: ['das', 'übers'] },
31
+ // { word: "um", out: ['das', 'ums'] },
32
+ // { word: "unter", out: ['das', 'unters'] },
33
+ // { word: "von", out: ['dem', 'vom'] },
34
+ // { word: "vor", out: ['das', 'vors'] },
35
+ // { word: "vor", out: ['dem', 'vorm'] },
36
+ // { word: "zu", out: ['dem', 'zum'] },
37
+ // { word: "zu", out: ['der', 'zur'] },
38
+ ]
39
+
40
+
@@ -0,0 +1,37 @@
1
+ const isAcronym = /[ .][A-Z]\.? *$/i
2
+ const hasEllipse = /(?:\u2026|\.{2,}) *$/
3
+ const hasLetter = /\p{L}/u
4
+ const isOrdinal = /[0-9]\. *$/
5
+
6
+ /** does this look like a sentence? */
7
+ const isSentence = function (str, abbrevs) {
8
+ // must have a letter
9
+ if (hasLetter.test(str) === false) {
10
+ return false
11
+ }
12
+ // check for 'F.B.I.'
13
+ if (isAcronym.test(str) === true) {
14
+ return false
15
+ }
16
+ // german ordinals like '4.'
17
+ if (isOrdinal.test(str) === true) {
18
+ return false
19
+ }
20
+ //check for '...'
21
+ if (hasEllipse.test(str) === true) {
22
+ return false
23
+ }
24
+ let txt = str.replace(/[.!?\u203D\u2E18\u203C\u2047-\u2049] *$/, '')
25
+ let words = txt.split(' ')
26
+ let lastWord = words[words.length - 1].toLowerCase()
27
+ // check for 'Mr.'
28
+ if (abbrevs.hasOwnProperty(lastWord) === true) {
29
+ return false
30
+ }
31
+ // //check for jeopardy!
32
+ // if (blacklist.hasOwnProperty(lastWord)) {
33
+ // return false
34
+ // }
35
+ return true
36
+ }
37
+ export default isSentence
@@ -0,0 +1,11 @@
1
+ import unicode from './unicode.js'
2
+ import contractions from './contractions.js'
3
+ import isSentence from './is-sentence.js'
4
+
5
+ export default {
6
+ mutate: (world) => {
7
+ world.model.one.unicode = unicode
8
+ world.model.one.contractions = contractions
9
+ world.methods.one.tokenize.isSentence = isSentence
10
+ },
11
+ }
@@ -0,0 +1,55 @@
1
+ //a hugely-ignorant, and widely subjective transliteration of latin, cryllic, greek unicode characters to english ascii.
2
+ //approximate visual (not semantic or phonetic) relationship between unicode and ascii characters
3
+ //http://en.wikipedia.org/wiki/List_of_Unicode_characters
4
+ //https://docs.google.com/spreadsheet/ccc?key=0Ah46z755j7cVdFRDM1A2YVpwa1ZYWlpJM2pQZ003M0E
5
+
6
+
7
+ // geman accented letters
8
+ // ä | Ä
9
+ // ö | Ö
10
+ // ü | Ü
11
+ // ß
12
+
13
+ let compact = {
14
+ '!': '¡',
15
+ '?': '¿Ɂ',
16
+ '"': '“”"❝❞',
17
+ "'": '‘‛❛❜’',
18
+ '-': '—–',
19
+ a: 'ªÁÃÅáãåĀāĂ㥹ǍǎǞǟǠǡǺǻȀȁȂȃȦȧȺΆΑΔΛάαλАаѦѧӐӑӒӓƛæ',
20
+ b: 'þƀƁƂƃƄƅɃϦБВЪЬвъьѢѣҌҍ',
21
+ c: '¢©ĆćĈĉĊċČčƆƇƈȻȼͻͼϲϹϽϾСсєҀҁҪҫ',
22
+ d: 'ÐĎďĐđƉƊȡƋƌ',
23
+ e: 'ĒēĔĕĖėĘęĚěƐȄȅȆȇȨȩɆɇΈΕΞΣέεξϵЀЁЕеѐёҼҽҾҿӖӗ',
24
+ f: 'ƑƒϜϝӺӻҒғſ',
25
+ g: 'ĜĝĞğĠġĢģƓǤǥǦǧǴǵ',
26
+ h: 'ĤĥĦħƕǶȞȟΉΗЂЊЋНнђћҢңҤҥҺһӉӊ',
27
+ I: 'Í',
28
+ i: 'íĨĩĪīĬĭĮįİıƖƗȈȉȊȋΊΐΪίιϊІЇії',
29
+ j: 'ĴĵǰȷɈɉϳЈј',
30
+ k: 'ĶķĸƘƙǨǩΚκЌЖКжкќҚқҜҝҞҟҠҡ',
31
+ l: 'ĹĺĻļĽľĿŀŁłƚƪǀǏǐȴȽΙӀӏ',
32
+ m: 'ΜϺϻМмӍӎ',
33
+ n: 'ÑñŃńŅņŇňʼnŊŋƝƞǸǹȠȵΝΠήηϞЍИЙЛПийлпѝҊҋӅӆӢӣӤӥπ',
34
+ o: 'ÓÕØðóõøŌōŎŏŐőƟƠơǑǒǪǫǬǭǾǿȌȍȎȏȪȫȬȭȮȯȰȱΌΘΟθοσόϕϘϙϬϴОФоѲѳӦӧӨөӪӫ',
35
+ p: 'ƤΡρϷϸϼРрҎҏÞ',
36
+ q: 'Ɋɋ',
37
+ r: 'ŔŕŖŗŘřƦȐȑȒȓɌɍЃГЯгяѓҐґ',
38
+ s: 'ŚśŜŝŞşŠšƧƨȘșȿЅѕ',
39
+ t: 'ŢţŤťŦŧƫƬƭƮȚțȶȾΓΤτϮТт',
40
+ u: 'µÚúŨũŪūŬŭŮůŰűŲųƯưƱƲǓǔǕǖǗǘǙǚǛǜȔȕȖȗɄΰμυϋύ',
41
+ v: 'νѴѵѶѷ',
42
+ w: 'ŴŵƜωώϖϢϣШЩшщѡѿ',
43
+ x: '×ΧχϗϰХхҲҳӼӽӾӿ',
44
+ y: 'ÝýÿŶŷŸƳƴȲȳɎɏΎΥΫγψϒϓϔЎУучўѰѱҮүҰұӮӯӰӱӲӳ',
45
+ z: 'ŹźŻżŽžƵƶȤȥɀΖ',
46
+ ß: 'β' //greek beta to german Eszett surrogate
47
+ }
48
+ //decompress data into two hashes
49
+ let unicode = {}
50
+ Object.keys(compact).forEach(function (k) {
51
+ compact[k].split('').forEach(function (s) {
52
+ unicode[s] = k
53
+ })
54
+ })
55
+ export default unicode
@@ -0,0 +1,47 @@
1
+ const separablePrefixes = '(ab|an|auf|aus|bei|ein|fest|fort|her|heraus|hervor|hin|hinzu|los|mit|nach|nieder|vor|voraus|weg|weiter|zu|zurecht|zurück|zusammen|zuteil)'
2
+
3
+ const postTagger = function (doc) {
4
+ // eine as 1 or the
5
+ doc.match('eine #Value').tag('TextValue', 'eine-value')
6
+ // 6.30 Uhr
7
+ doc.match('#Value uhr').tag('Time', 'time-Uhr')
8
+ // a value before a currency is money - '12,5 Euro'
9
+ doc.match('[#Value] #Currency', 0).tag('Money', 'value-currency')
10
+
11
+ // prenominal possessive is a determiner - 'mein Bruder', 'seine kleine Schwester'
12
+ doc.match('[(mein|dein|sein|ihr|unser|euer)] #Adjective? #Noun', 0).tag('Determiner', 'possessive-det')
13
+ doc.match('[#Possessive] #Adjective? #Noun', 0).tag('Determiner', 'possessive-det')
14
+
15
+ // 'weiß' beside a pronoun is wissen, not white - 'das weiß ich'
16
+ doc.match('[weiß] #Pronoun', 0).tag('PresentTense', 'weiß-pronoun')
17
+ doc.match('#Pronoun [weiß]', 0).tag('PresentTense', 'pronoun-weiß')
18
+
19
+ // a verb between a determiner and a noun is an inflected adjective - 'das schnelle Auto'
20
+ doc.match('#Determiner [(#Verb && !#Infinitive && /e[nrsm]?$/)] (#Noun && !#Pronoun)', 0).tag('Adjective', 'det-adj-noun')
21
+
22
+ // a determiner with no noun after it is a pronoun - 'das ist gut', 'alle außer ihm'
23
+ doc.match('[#Determiner] (#Verb|#Preposition)', 0).tag('Pronoun', 'det-pronoun')
24
+
25
+ // modal verbs - the infinitive goes to the end of the clause:
26
+ // dürfen (may), können (can), mögen (like), müssen (must), sollen (should), wollen (want)
27
+ doc.match('[{dürfen}] * #Infinitive', 0).tag('Modal', 'dürfen-verb')
28
+ doc.match('[{können}] * #Infinitive', 0).tag('Modal', 'können-verb')
29
+ doc.match('[{mögen}] * #Infinitive', 0).tag('Modal', 'mögen-verb')
30
+ doc.match('[{müssen}] * #Infinitive', 0).tag('Modal', 'müssen-verb')
31
+ doc.match('[{sollen}] * #Infinitive', 0).tag('Modal', 'sollen-verb')
32
+ doc.match('[{wollen}] * #Infinitive', 0).tag('Modal', 'wollen-verb')
33
+
34
+ // auxiliary + participle at the end of the clause (Satzklammer):
35
+ // 'sie hat gestern ein Buch gelesen', 'er ist nach Hause gegangen'
36
+ doc.match('[{haben}] * #Participle', 0).tag('Auxiliary', 'haben-participle')
37
+ doc.match('[{haben}] * #Infinitive', 0).tag('Auxiliary', 'haben-infinitive')
38
+ doc.match('[{sein}] * #Participle', 0).ifNo('#Determiner').tag('Auxiliary', 'sein-participle')
39
+ // werden + infinitive is the future tense - 'wir werden nach Berlin fahren'
40
+ doc.match('[{werden}] * #Infinitive', 0).tag('Auxiliary', 'werden-future')
41
+ // werden + participle is the passive - 'das Buch wird gelesen'
42
+ doc.match('[{werden}] * #Participle', 0).tag('Auxiliary', 'werden-passive')
43
+
44
+ // a separable verb-prefix, at the end of the clause - 'ich stehe um sieben Uhr auf'
45
+ doc.match('#Verb * [#Preposition]$', 0).match(separablePrefixes).tag('Particle', 'separable-prefix')
46
+ }
47
+ export default postTagger
@@ -0,0 +1,8 @@
1
+ import postTagger from './matches.js'
2
+
3
+ export default {
4
+ compute: {
5
+ postTagger
6
+ },
7
+ hooks: ['postTagger']
8
+ }
@@ -0,0 +1,43 @@
1
+ const hasApostrophe = /['‘’‛‵′`´]/
2
+ const hasPeriod = /\./
3
+ const isNum = /^[0-9+\-,]+$/
4
+
5
+ // normal regexes
6
+ const doRegs = function (str, regs) {
7
+ for (let i = 0; i < regs.length; i += 1) {
8
+ if (regs[i][0].test(str) === true) {
9
+ return regs[i]
10
+ }
11
+ }
12
+ return null
13
+ }
14
+
15
+ const checkRegex = function (terms, i, world) {
16
+ let setTag = world.methods.one.setTag
17
+ let term = terms[i]
18
+ let { regexText, regexNormal, regexNumbers } = world.model.two
19
+ let normal = term.machine || term.normal
20
+ let text = term.text
21
+ // keep dangling apostrophe?
22
+ if (hasApostrophe.test(term.post) && !hasApostrophe.test(term.pre)) {
23
+ text += term.post.trim()
24
+ }
25
+ // keep period in number ordinals?
26
+ if (hasPeriod.test(term.post) && isNum.test(text)) {
27
+ setTag([term], ['Ordinal', 'NumericValue'], world, false, `1-regex-ordinal`)
28
+ term.confidence = 0.6
29
+ return true
30
+ }
31
+ let arr = doRegs(text, regexText) || doRegs(normal, regexNormal)
32
+ // hide a bunch of number regexes behind this one
33
+ if (!arr && /[0-9]/.test(normal)) {
34
+ arr = doRegs(normal, regexNumbers)
35
+ }
36
+ if (arr) {
37
+ setTag([term], arr[1], world, false, `1-regex- '${arr[2] || arr[0]}'`)
38
+ term.confidence = 0.6
39
+ return true
40
+ }
41
+ return null
42
+ }
43
+ export default checkRegex
@@ -0,0 +1,21 @@
1
+ const isTitleCase = function (str) {
2
+ return /^[A-ZÄÖÜ][a-z'\u00C0-\u00FF]/.test(str) || /^[A-ZÄÖÜ]$/.test(str)
3
+ }
4
+
5
+ // add a noun to any non-0 index titlecased word, with no existing tag
6
+ const titleCaseNoun = function (terms, i, world) {
7
+ let setTag = world.methods.one.setTag
8
+ let term = terms[i]
9
+ // don't over-write any tags
10
+ // if (term.tags.size > 0) {
11
+ // return
12
+ // }
13
+ // skip first-word, for now
14
+ if (i === 0) {
15
+ return
16
+ }
17
+ if (isTitleCase(term.text)) {
18
+ setTag([term], 'Noun', world, false, `1-titlecase`)
19
+ }
20
+ }
21
+ export default titleCaseNoun
@@ -0,0 +1,54 @@
1
+ const min = 1400
2
+ const max = 2100
3
+
4
+ const dateWords = new Set(['pendant', 'dans', 'avant', 'apres', 'pour', 'en'])
5
+
6
+ const seemsGood = function (term) {
7
+ if (!term) {
8
+ return false
9
+ }
10
+ if (dateWords.has(term.normal)) {
11
+ return true
12
+ }
13
+ if (term.tags.has('Date') || term.tags.has('Month') || term.tags.has('WeekDay')) {
14
+ return true
15
+ }
16
+ return false
17
+ }
18
+
19
+ const seemsOkay = function (term) {
20
+ if (!term) {
21
+ return false
22
+ }
23
+ if (term.tags.has('Ordinal')) {
24
+ return true
25
+ }
26
+ return false
27
+ }
28
+
29
+ // recognize '1993' as a year
30
+ const tagYear = function (terms, i, world) {
31
+ let setTag = world.methods.one.setTag
32
+ const term = terms[i]
33
+ if (term.tags.has('NumericValue') && term.tags.has('Cardinal') && term.normal.length === 4) {
34
+ let num = Number(term.normal)
35
+ // number between 1400 and 2100
36
+ if (num && !isNaN(num)) {
37
+ if (num > min && num < max) {
38
+ if (seemsGood(terms[i - 1]) || seemsGood(terms[i + 1])) {
39
+ setTag([term], 'Year', world, false, '1-tagYear')
40
+ return true
41
+ }
42
+ // or is it really-close to a year?
43
+ if (num > 1950 && num < 2025) {
44
+ if (seemsOkay(terms[i - 1]) || seemsOkay(terms[i + 1])) {
45
+ setTag([term], 'Year', world, false, '1-tagYear-close')
46
+ return true
47
+ }
48
+ }
49
+ }
50
+ }
51
+ }
52
+ return null
53
+ }
54
+ export default tagYear
@@ -0,0 +1,82 @@
1
+ const oneLetterAcronym = /^[A-ZÄÖÜ]('s|,)?$/
2
+ const isUpperCase = /^[A-Z-ÄÖÜ]+$/
3
+ const periodAcronym = /([A-ZÄÖÜ]\.)+[A-ZÄÖÜ]?,?$/
4
+ const noPeriodAcronym = /[A-ZÄÖÜ]{2,}('s|,)?$/
5
+ const lowerCaseAcronym = /([a-zäöü]\.)+[a-zäöü]\.?$/
6
+
7
+
8
+
9
+ const oneLetterWord = {
10
+ I: true,
11
+ A: true,
12
+ }
13
+ // just uppercase acronyms, no periods - 'UNOCHA'
14
+ const isNoPeriodAcronym = function (term, model) {
15
+ let str = term.text
16
+ // ensure it's all upper-case
17
+ if (isUpperCase.test(str) === false) {
18
+ return false
19
+ }
20
+ // long capitalized words are not usually either
21
+ if (str.length > 5) {
22
+ return false
23
+ }
24
+ // 'I' is not a acronym
25
+ if (oneLetterWord.hasOwnProperty(str)) {
26
+ return false
27
+ }
28
+ // known-words, like 'PIZZA' is not an acronym.
29
+ if (model.one.lexicon.hasOwnProperty(term.normal)) {
30
+ return false
31
+ }
32
+ //like N.D.A
33
+ if (periodAcronym.test(str) === true) {
34
+ return true
35
+ }
36
+ //like c.e.o
37
+ if (lowerCaseAcronym.test(str) === true) {
38
+ return true
39
+ }
40
+ //like 'F.'
41
+ if (oneLetterAcronym.test(str) === true) {
42
+ return true
43
+ }
44
+ //like NDA
45
+ if (noPeriodAcronym.test(str) === true) {
46
+ return true
47
+ }
48
+ return false
49
+ }
50
+
51
+ const isAcronym = function (terms, i, world) {
52
+ let setTag = world.methods.one.setTag
53
+ let term = terms[i]
54
+ //these are not acronyms
55
+ if (term.tags.has('RomanNumeral') || term.tags.has('Acronym')) {
56
+ return null
57
+ }
58
+ //non-period ones are harder
59
+ if (isNoPeriodAcronym(term, world.model)) {
60
+ term.tags.clear()
61
+ setTag([term], ['Acronym', 'Noun'], world, false, '3-no-period-acronym')
62
+ return true
63
+ }
64
+ // one-letter acronyms
65
+ if (!oneLetterWord.hasOwnProperty(term.text) && oneLetterAcronym.test(term.text)) {
66
+ term.tags.clear()
67
+ setTag([term], ['Acronym', 'Noun'], world, false, '3-one-letter-acronym')
68
+ return true
69
+ }
70
+ //if it's a very-short organization?
71
+ if (term.tags.has('Organization') && term.text.length <= 3) {
72
+ setTag([term], 'Acronym', world, false, '3-org-acronym')
73
+ return true
74
+ }
75
+ // upper-case org, like UNESCO
76
+ if (term.tags.has('Organization') && isUpperCase.test(term.text) && term.text.length <= 6) {
77
+ setTag([term], 'Acronym', world, false, '3-titlecase-acronym')
78
+ return true
79
+ }
80
+ return null
81
+ }
82
+ export default isAcronym
@@ -0,0 +1,27 @@
1
+ const isTitleCase = function (str) {
2
+ return /^[A-ZÄÖÜ][a-z'\u00C0-\u00FF]/.test(str) || /^[A-ZÄÖÜ]$/.test(str)
3
+ }
4
+
5
+ const hasNoVerb = function (terms) {
6
+ return !terms.find(t => t.tags.has('Verb'))
7
+ }
8
+
9
+ const fallback = function (terms, i, world) {
10
+ let setTag = world.methods.one.setTag
11
+ let term = terms[i]
12
+ if (term.tags.size === 0) {
13
+
14
+ // is it first-word titlecase?
15
+ if (i === 0 && isTitleCase(term.text)) {
16
+ setTag([term], 'Noun', world, false, `1-titlecase`)// Noun still the safest bet?
17
+ return
18
+ }
19
+
20
+ let tag = 'Adjective'
21
+ if (terms.length > 10 && hasNoVerb(terms)) {
22
+ tag = 'Verb'
23
+ }
24
+ setTag([term], tag, world, false, '2-fallback')
25
+ }
26
+ }
27
+ export default fallback