de-compromise 0.0.9 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/builds/de-compromise.cjs +2357 -1687
- package/builds/de-compromise.min.js +1 -1
- package/builds/de-compromise.mjs +1 -1
- package/package.json +21 -14
- package/src/01-one/lexicon/_data.js +32 -0
- package/src/01-one/lexicon/compute/root.js +55 -0
- package/src/01-one/lexicon/lexicon.js +95 -0
- package/src/01-one/lexicon/methods/_data.js +193 -0
- package/src/01-one/lexicon/methods/adjectives/_test-back.js +29 -0
- package/src/01-one/lexicon/methods/adjectives/_test.js +49 -0
- package/src/01-one/lexicon/methods/adjectives/inflect.js +27 -0
- package/src/01-one/lexicon/methods/adjectives/model.js +126 -0
- package/src/01-one/lexicon/methods/adjectives/new/adjective.js +8 -0
- package/src/01-one/lexicon/methods/adjectives/new/comparative.js +7 -0
- package/src/01-one/lexicon/methods/adjectives/new/superlative.js +159 -0
- package/src/01-one/lexicon/methods/adjectives/toRoot.js +47 -0
- package/src/01-one/lexicon/methods/index.js +56 -0
- package/src/01-one/lexicon/methods/models.js +10 -0
- package/src/01-one/lexicon/methods/nouns/_test-back.js +45 -0
- package/src/01-one/lexicon/methods/nouns/new.js +56 -0
- package/src/01-one/lexicon/methods/nouns/toPlural.js +49 -0
- package/src/01-one/lexicon/methods/nouns/toSingular.js +46 -0
- package/src/01-one/lexicon/methods/verbs/conjugate.js +87 -0
- package/src/01-one/lexicon/methods/verbs/custom/_test-from.js +53 -0
- package/src/01-one/lexicon/methods/verbs/custom/_test-to.js +50 -0
- package/src/01-one/lexicon/methods/verbs/custom/fromPastParticiple.js +37 -0
- package/src/01-one/lexicon/methods/verbs/custom/rules.js +182 -0
- package/src/01-one/lexicon/methods/verbs/custom/toPastParticiple.js +54 -0
- package/src/01-one/lexicon/methods/verbs/irregulars.js +202 -0
- package/src/01-one/lexicon/methods/verbs/toRoot.js +55 -0
- package/src/01-one/lexicon/misc.js +78 -0
- package/src/01-one/lexicon/plugin.js +14 -0
- package/src/01-one/splitter/build.js +20 -0
- package/src/01-one/splitter/compute/splitter.js +54 -0
- package/src/01-one/splitter/model/index.js +20 -0
- package/src/01-one/splitter/plugin.js +10 -0
- package/src/01-one/splitter/trie.js +40 -0
- package/src/01-one/tagset/plugin.js +11 -0
- package/src/01-one/tagset/tags/dates.js +45 -0
- package/src/01-one/tagset/tags/misc.js +70 -0
- package/src/01-one/tagset/tags/nouns.js +116 -0
- package/src/01-one/tagset/tags/values.js +47 -0
- package/src/01-one/tagset/tags/verbs.js +64 -0
- package/src/01-one/tokenizer/contractions.js +40 -0
- package/src/01-one/tokenizer/is-sentence.js +37 -0
- package/src/01-one/tokenizer/plugin.js +11 -0
- package/src/01-one/tokenizer/unicode.js +55 -0
- package/src/02-two/postTagger/matches.js +47 -0
- package/src/02-two/postTagger/plugin.js +8 -0
- package/src/02-two/preTagger/compute/1st-pass/regex.js +43 -0
- package/src/02-two/preTagger/compute/1st-pass/titlecase.js +21 -0
- package/src/02-two/preTagger/compute/1st-pass/year.js +54 -0
- package/src/02-two/preTagger/compute/2nd-pass/acronym.js +82 -0
- package/src/02-two/preTagger/compute/2nd-pass/fallback.js +27 -0
- package/src/02-two/preTagger/compute/2nd-pass/prefix-lookup.js +33 -0
- package/src/02-two/preTagger/compute/2nd-pass/suffix-lookup.js +44 -0
- package/src/02-two/preTagger/compute/index.js +42 -0
- package/src/02-two/preTagger/model/index.js +14 -0
- package/src/02-two/preTagger/model/prefixes.js +46 -0
- package/src/02-two/preTagger/model/regex/regex-normal.js +25 -0
- package/src/02-two/preTagger/model/regex/regex-numbers.js +62 -0
- package/src/02-two/preTagger/model/regex/regex-text.js +17 -0
- package/src/02-two/preTagger/model/suffixes.js +171 -0
- package/src/02-two/preTagger/plugin.js +12 -0
- package/src/03-three/adjectives/api.js +34 -0
- package/src/03-three/adjectives/plugin.js +5 -0
- package/src/03-three/nouns/api.js +74 -0
- package/src/03-three/nouns/plugin.js +5 -0
- package/src/03-three/numbers/api.js +259 -0
- package/src/03-three/numbers/data.js +77 -0
- package/src/03-three/numbers/format/index.js +27 -0
- package/src/03-three/numbers/format/toText.js +118 -0
- package/src/03-three/numbers/parse/fromText.js +52 -0
- package/src/03-three/numbers/parse/index.js +63 -0
- package/src/03-three/numbers/plugin.js +5 -0
- package/src/03-three/verbs/api.js +41 -0
- package/src/03-three/verbs/plugin.js +5 -0
- package/src/_lib.js +4 -0
- package/src/_version.js +1 -0
- package/src/index.js +53 -0
- package/types/index.d.ts +46 -0
- package/types/misc.ts +92 -0
- package/types/view/de.ts +61 -0
- package/types/view/one.ts +235 -0
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
const entity = ['Person', 'Place', 'Organization']
|
|
2
|
+
|
|
3
|
+
export default {
|
|
4
|
+
Noun: {
|
|
5
|
+
not: ['Verb', 'Adjective', 'Adverb', 'Value', 'Determiner'],
|
|
6
|
+
},
|
|
7
|
+
Singular: {
|
|
8
|
+
is: 'Noun',
|
|
9
|
+
not: ['Plural'],
|
|
10
|
+
},
|
|
11
|
+
ProperNoun: {
|
|
12
|
+
is: 'Noun',
|
|
13
|
+
},
|
|
14
|
+
Person: {
|
|
15
|
+
is: 'Singular',
|
|
16
|
+
also: ['ProperNoun'],
|
|
17
|
+
not: ['Place', 'Organization', 'Date'],
|
|
18
|
+
},
|
|
19
|
+
FirstName: {
|
|
20
|
+
is: 'Person',
|
|
21
|
+
},
|
|
22
|
+
MaleName: {
|
|
23
|
+
is: 'FirstName',
|
|
24
|
+
not: ['FemaleName', 'LastName'],
|
|
25
|
+
},
|
|
26
|
+
FemaleName: {
|
|
27
|
+
is: 'FirstName',
|
|
28
|
+
not: ['MaleName', 'LastName'],
|
|
29
|
+
},
|
|
30
|
+
LastName: {
|
|
31
|
+
is: 'Person',
|
|
32
|
+
not: ['FirstName'],
|
|
33
|
+
},
|
|
34
|
+
Honorific: {
|
|
35
|
+
is: 'Noun',
|
|
36
|
+
not: ['FirstName', 'LastName', 'Value'],
|
|
37
|
+
},
|
|
38
|
+
Place: {
|
|
39
|
+
is: 'Singular',
|
|
40
|
+
not: ['Person', 'Organization'],
|
|
41
|
+
},
|
|
42
|
+
Country: {
|
|
43
|
+
is: 'Place',
|
|
44
|
+
also: ['ProperNoun'],
|
|
45
|
+
not: ['City'],
|
|
46
|
+
},
|
|
47
|
+
City: {
|
|
48
|
+
is: 'Place',
|
|
49
|
+
also: ['ProperNoun'],
|
|
50
|
+
not: ['Country'],
|
|
51
|
+
},
|
|
52
|
+
Region: {
|
|
53
|
+
is: 'Place',
|
|
54
|
+
also: ['ProperNoun'],
|
|
55
|
+
},
|
|
56
|
+
Address: {
|
|
57
|
+
// is: 'Place',
|
|
58
|
+
},
|
|
59
|
+
Organization: {
|
|
60
|
+
is: 'ProperNoun',
|
|
61
|
+
not: ['Person', 'Place'],
|
|
62
|
+
},
|
|
63
|
+
SportsTeam: {
|
|
64
|
+
is: 'Organization',
|
|
65
|
+
},
|
|
66
|
+
School: {
|
|
67
|
+
is: 'Organization',
|
|
68
|
+
},
|
|
69
|
+
Company: {
|
|
70
|
+
is: 'Organization',
|
|
71
|
+
},
|
|
72
|
+
Plural: {
|
|
73
|
+
is: 'Noun',
|
|
74
|
+
not: ['Singular'],
|
|
75
|
+
},
|
|
76
|
+
Uncountable: {
|
|
77
|
+
is: 'Noun',
|
|
78
|
+
},
|
|
79
|
+
Pronoun: {
|
|
80
|
+
is: 'Noun',
|
|
81
|
+
not: entity,
|
|
82
|
+
},
|
|
83
|
+
Actor: {
|
|
84
|
+
is: 'Noun',
|
|
85
|
+
not: entity,
|
|
86
|
+
},
|
|
87
|
+
Activity: {
|
|
88
|
+
is: 'Noun',
|
|
89
|
+
not: ['Person', 'Place'],
|
|
90
|
+
},
|
|
91
|
+
Unit: {
|
|
92
|
+
is: 'Noun',
|
|
93
|
+
not: entity,
|
|
94
|
+
},
|
|
95
|
+
Demonym: {
|
|
96
|
+
is: 'Noun',
|
|
97
|
+
also: ['ProperNoun'],
|
|
98
|
+
not: entity,
|
|
99
|
+
},
|
|
100
|
+
// standalone — 'meinem' is #Possessive as both a determiner ('meinem Freund')
|
|
101
|
+
// and a pronoun ('das ist meins')
|
|
102
|
+
Possessive: {},
|
|
103
|
+
// german genders
|
|
104
|
+
MaleNoun: {
|
|
105
|
+
is: 'Noun',
|
|
106
|
+
not: ['FemaleNoun', 'NeuterNoun'],
|
|
107
|
+
},
|
|
108
|
+
FemaleNoun: {
|
|
109
|
+
is: 'Noun',
|
|
110
|
+
not: ['MaleNoun', 'NeuterNoun'],
|
|
111
|
+
},
|
|
112
|
+
NeuterNoun: {
|
|
113
|
+
is: 'Noun',
|
|
114
|
+
not: ['MaleNoun', 'FemaleNoun'],
|
|
115
|
+
},
|
|
116
|
+
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
export default {
|
|
2
|
+
Value: {
|
|
3
|
+
not: ['Verb', 'Adjective', 'Adverb'],
|
|
4
|
+
},
|
|
5
|
+
Ordinal: {
|
|
6
|
+
is: 'Value',
|
|
7
|
+
not: ['Cardinal'],
|
|
8
|
+
},
|
|
9
|
+
Cardinal: {
|
|
10
|
+
is: 'Value',
|
|
11
|
+
not: ['Ordinal'],
|
|
12
|
+
},
|
|
13
|
+
Fraction: {
|
|
14
|
+
is: 'Value',
|
|
15
|
+
not: ['Noun'],
|
|
16
|
+
},
|
|
17
|
+
Multiple: {
|
|
18
|
+
is: 'Value',
|
|
19
|
+
},
|
|
20
|
+
RomanNumeral: {
|
|
21
|
+
is: 'Cardinal',
|
|
22
|
+
not: ['TextValue'],
|
|
23
|
+
},
|
|
24
|
+
TextValue: {
|
|
25
|
+
is: 'Value',
|
|
26
|
+
not: ['NumericValue'],
|
|
27
|
+
},
|
|
28
|
+
TextOrdinal: {
|
|
29
|
+
is: 'TextValue',
|
|
30
|
+
also: ['Ordinal']
|
|
31
|
+
},
|
|
32
|
+
TextCardinal: {
|
|
33
|
+
is: 'TextValue',
|
|
34
|
+
also: ['Cardinal']
|
|
35
|
+
},
|
|
36
|
+
|
|
37
|
+
NumericValue: {
|
|
38
|
+
is: 'Value',
|
|
39
|
+
not: ['TextValue'],
|
|
40
|
+
},
|
|
41
|
+
Money: {
|
|
42
|
+
is: 'Cardinal',
|
|
43
|
+
},
|
|
44
|
+
Percent: {
|
|
45
|
+
is: 'Value',
|
|
46
|
+
},
|
|
47
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
export default {
|
|
2
|
+
Verb: {
|
|
3
|
+
not: ['Noun', 'Adjective', 'Adverb', 'Value', 'Expression'],
|
|
4
|
+
},
|
|
5
|
+
PresentTense: {
|
|
6
|
+
is: 'Verb',
|
|
7
|
+
not: ['PastTense'],
|
|
8
|
+
},
|
|
9
|
+
Infinitive: {
|
|
10
|
+
is: 'PresentTense',
|
|
11
|
+
not: ['Gerund'],
|
|
12
|
+
},
|
|
13
|
+
Imperative: {
|
|
14
|
+
is: 'Infinitive',
|
|
15
|
+
},
|
|
16
|
+
Subjunctive: {
|
|
17
|
+
is: 'Verb',
|
|
18
|
+
},
|
|
19
|
+
Participle: {
|
|
20
|
+
is: 'Verb',
|
|
21
|
+
},
|
|
22
|
+
Gerund: {
|
|
23
|
+
is: 'PresentTense',
|
|
24
|
+
not: ['Copula'],
|
|
25
|
+
},
|
|
26
|
+
PastTense: {
|
|
27
|
+
is: 'Verb',
|
|
28
|
+
not: ['PresentTense', 'Gerund'],
|
|
29
|
+
},
|
|
30
|
+
Copula: {
|
|
31
|
+
is: 'Verb',
|
|
32
|
+
},
|
|
33
|
+
Modal: {
|
|
34
|
+
is: 'Verb',
|
|
35
|
+
not: ['Infinitive'],
|
|
36
|
+
},
|
|
37
|
+
PerfectTense: {
|
|
38
|
+
is: 'Verb',
|
|
39
|
+
not: ['Gerund'],
|
|
40
|
+
},
|
|
41
|
+
Pluperfect: {
|
|
42
|
+
is: 'Verb',
|
|
43
|
+
},
|
|
44
|
+
PhrasalVerb: {
|
|
45
|
+
is: 'Verb',
|
|
46
|
+
},
|
|
47
|
+
Particle: {
|
|
48
|
+
is: 'PhrasalVerb',
|
|
49
|
+
not: ['PastTense', 'PresentTense', 'Copula', 'Gerund'],
|
|
50
|
+
},
|
|
51
|
+
Auxiliary: {
|
|
52
|
+
is: 'Verb',
|
|
53
|
+
not: ['PastTense', 'PresentTense', 'Gerund', 'Conjunction'],
|
|
54
|
+
},
|
|
55
|
+
FirstPerson: {
|
|
56
|
+
is: 'Verb'
|
|
57
|
+
},
|
|
58
|
+
SecondPerson: {
|
|
59
|
+
is: 'Verb'
|
|
60
|
+
},
|
|
61
|
+
ThirdPerson: {
|
|
62
|
+
is: 'Verb'
|
|
63
|
+
},
|
|
64
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
// https://www.fluentu.com/blog/german/german-contractions/
|
|
2
|
+
export default [
|
|
3
|
+
{ word: 'am', out: ['an', 'dem'] }, //(on the / at the)
|
|
4
|
+
{ word: 'ans', out: ['an', 'das'] }, //(on the / at the)
|
|
5
|
+
{ word: 'aufs', out: ['auf', 'das'] }, //(on the / at the)
|
|
6
|
+
{ word: 'beim', out: ['bei', 'dem'] }, //(at the / in the / with the)
|
|
7
|
+
{ word: 'durchs', out: ['durch', 'das'] }, //(through the)
|
|
8
|
+
{ word: 'fürs', out: ['für', 'das'] }, //(for the)
|
|
9
|
+
{ word: 'hinterm', out: ['hinter', 'dem'] }, //(behind the)
|
|
10
|
+
{ word: 'ins', out: ['in', 'das'] }, //(in the / into the / to the)
|
|
11
|
+
{ word: 'im', out: ['in', 'dem'] }, //(at / in the)
|
|
12
|
+
{ word: 'übers', out: ['über', 'das'] }, //(over the / about the)
|
|
13
|
+
{ word: 'ums', out: ['um', 'das'] }, //(at the / around)
|
|
14
|
+
{ word: 'unters', out: ['unter', 'das'] }, //(under the)
|
|
15
|
+
{ word: 'unterm', out: ['unter', 'dem'] }, //(under the)
|
|
16
|
+
{ word: 'vom', out: ['von', 'dem'] }, //(from the)
|
|
17
|
+
{ word: 'vors', out: ['vor', 'das'] }, //(in front of the)
|
|
18
|
+
{ word: 'vorm', out: ['vor', 'dem'] }, //(in front of the / from)
|
|
19
|
+
{ word: 'zum', out: ['zu', 'dem'] }, //(to the / to)
|
|
20
|
+
{ word: 'zur', out: ['zu', 'der'] }, //(to the / to)
|
|
21
|
+
|
|
22
|
+
// { word: "n", out: ['das', 'ans'] },
|
|
23
|
+
// { word: "an", out: ['dem', 'am'] },
|
|
24
|
+
// { word: "auf", out: ['das', 'aufs'] },
|
|
25
|
+
// { word: "bei", out: ['dem', 'beim'] },
|
|
26
|
+
// { word: "durch", out: ['das', 'durchs'] },
|
|
27
|
+
// { word: "für", out: ['das', 'fürs'] },
|
|
28
|
+
// { word: "in", out: ['das', 'ins'] },
|
|
29
|
+
// { word: "in", out: ['dem', 'im'] },
|
|
30
|
+
// { word: "über", out: ['das', 'übers'] },
|
|
31
|
+
// { word: "um", out: ['das', 'ums'] },
|
|
32
|
+
// { word: "unter", out: ['das', 'unters'] },
|
|
33
|
+
// { word: "von", out: ['dem', 'vom'] },
|
|
34
|
+
// { word: "vor", out: ['das', 'vors'] },
|
|
35
|
+
// { word: "vor", out: ['dem', 'vorm'] },
|
|
36
|
+
// { word: "zu", out: ['dem', 'zum'] },
|
|
37
|
+
// { word: "zu", out: ['der', 'zur'] },
|
|
38
|
+
]
|
|
39
|
+
|
|
40
|
+
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
const isAcronym = /[ .][A-Z]\.? *$/i
|
|
2
|
+
const hasEllipse = /(?:\u2026|\.{2,}) *$/
|
|
3
|
+
const hasLetter = /\p{L}/u
|
|
4
|
+
const isOrdinal = /[0-9]\. *$/
|
|
5
|
+
|
|
6
|
+
/** does this look like a sentence? */
|
|
7
|
+
const isSentence = function (str, abbrevs) {
|
|
8
|
+
// must have a letter
|
|
9
|
+
if (hasLetter.test(str) === false) {
|
|
10
|
+
return false
|
|
11
|
+
}
|
|
12
|
+
// check for 'F.B.I.'
|
|
13
|
+
if (isAcronym.test(str) === true) {
|
|
14
|
+
return false
|
|
15
|
+
}
|
|
16
|
+
// german ordinals like '4.'
|
|
17
|
+
if (isOrdinal.test(str) === true) {
|
|
18
|
+
return false
|
|
19
|
+
}
|
|
20
|
+
//check for '...'
|
|
21
|
+
if (hasEllipse.test(str) === true) {
|
|
22
|
+
return false
|
|
23
|
+
}
|
|
24
|
+
let txt = str.replace(/[.!?\u203D\u2E18\u203C\u2047-\u2049] *$/, '')
|
|
25
|
+
let words = txt.split(' ')
|
|
26
|
+
let lastWord = words[words.length - 1].toLowerCase()
|
|
27
|
+
// check for 'Mr.'
|
|
28
|
+
if (abbrevs.hasOwnProperty(lastWord) === true) {
|
|
29
|
+
return false
|
|
30
|
+
}
|
|
31
|
+
// //check for jeopardy!
|
|
32
|
+
// if (blacklist.hasOwnProperty(lastWord)) {
|
|
33
|
+
// return false
|
|
34
|
+
// }
|
|
35
|
+
return true
|
|
36
|
+
}
|
|
37
|
+
export default isSentence
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import unicode from './unicode.js'
|
|
2
|
+
import contractions from './contractions.js'
|
|
3
|
+
import isSentence from './is-sentence.js'
|
|
4
|
+
|
|
5
|
+
export default {
|
|
6
|
+
mutate: (world) => {
|
|
7
|
+
world.model.one.unicode = unicode
|
|
8
|
+
world.model.one.contractions = contractions
|
|
9
|
+
world.methods.one.tokenize.isSentence = isSentence
|
|
10
|
+
},
|
|
11
|
+
}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
//a hugely-ignorant, and widely subjective transliteration of latin, cryllic, greek unicode characters to english ascii.
|
|
2
|
+
//approximate visual (not semantic or phonetic) relationship between unicode and ascii characters
|
|
3
|
+
//http://en.wikipedia.org/wiki/List_of_Unicode_characters
|
|
4
|
+
//https://docs.google.com/spreadsheet/ccc?key=0Ah46z755j7cVdFRDM1A2YVpwa1ZYWlpJM2pQZ003M0E
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
// geman accented letters
|
|
8
|
+
// ä | Ä
|
|
9
|
+
// ö | Ö
|
|
10
|
+
// ü | Ü
|
|
11
|
+
// ß
|
|
12
|
+
|
|
13
|
+
let compact = {
|
|
14
|
+
'!': '¡',
|
|
15
|
+
'?': '¿Ɂ',
|
|
16
|
+
'"': '“”"❝❞',
|
|
17
|
+
"'": '‘‛❛❜’',
|
|
18
|
+
'-': '—–',
|
|
19
|
+
a: 'ªÁÃÅáãåĀāĂ㥹ǍǎǞǟǠǡǺǻȀȁȂȃȦȧȺΆΑΔΛάαλАаѦѧӐӑӒӓƛæ',
|
|
20
|
+
b: 'þƀƁƂƃƄƅɃϦБВЪЬвъьѢѣҌҍ',
|
|
21
|
+
c: '¢©ĆćĈĉĊċČčƆƇƈȻȼͻͼϲϹϽϾСсєҀҁҪҫ',
|
|
22
|
+
d: 'ÐĎďĐđƉƊȡƋƌ',
|
|
23
|
+
e: 'ĒēĔĕĖėĘęĚěƐȄȅȆȇȨȩɆɇΈΕΞΣέεξϵЀЁЕеѐёҼҽҾҿӖӗ',
|
|
24
|
+
f: 'ƑƒϜϝӺӻҒғſ',
|
|
25
|
+
g: 'ĜĝĞğĠġĢģƓǤǥǦǧǴǵ',
|
|
26
|
+
h: 'ĤĥĦħƕǶȞȟΉΗЂЊЋНнђћҢңҤҥҺһӉӊ',
|
|
27
|
+
I: 'Í',
|
|
28
|
+
i: 'íĨĩĪīĬĭĮįİıƖƗȈȉȊȋΊΐΪίιϊІЇії',
|
|
29
|
+
j: 'ĴĵǰȷɈɉϳЈј',
|
|
30
|
+
k: 'ĶķĸƘƙǨǩΚκЌЖКжкќҚқҜҝҞҟҠҡ',
|
|
31
|
+
l: 'ĹĺĻļĽľĿŀŁłƚƪǀǏǐȴȽΙӀӏ',
|
|
32
|
+
m: 'ΜϺϻМмӍӎ',
|
|
33
|
+
n: 'ÑñŃńŅņŇňʼnŊŋƝƞǸǹȠȵΝΠήηϞЍИЙЛПийлпѝҊҋӅӆӢӣӤӥπ',
|
|
34
|
+
o: 'ÓÕØðóõøŌōŎŏŐőƟƠơǑǒǪǫǬǭǾǿȌȍȎȏȪȫȬȭȮȯȰȱΌΘΟθοσόϕϘϙϬϴОФоѲѳӦӧӨөӪӫ',
|
|
35
|
+
p: 'ƤΡρϷϸϼРрҎҏÞ',
|
|
36
|
+
q: 'Ɋɋ',
|
|
37
|
+
r: 'ŔŕŖŗŘřƦȐȑȒȓɌɍЃГЯгяѓҐґ',
|
|
38
|
+
s: 'ŚśŜŝŞşŠšƧƨȘșȿЅѕ',
|
|
39
|
+
t: 'ŢţŤťŦŧƫƬƭƮȚțȶȾΓΤτϮТт',
|
|
40
|
+
u: 'µÚúŨũŪūŬŭŮůŰűŲųƯưƱƲǓǔǕǖǗǘǙǚǛǜȔȕȖȗɄΰμυϋύ',
|
|
41
|
+
v: 'νѴѵѶѷ',
|
|
42
|
+
w: 'ŴŵƜωώϖϢϣШЩшщѡѿ',
|
|
43
|
+
x: '×ΧχϗϰХхҲҳӼӽӾӿ',
|
|
44
|
+
y: 'ÝýÿŶŷŸƳƴȲȳɎɏΎΥΫγψϒϓϔЎУучўѰѱҮүҰұӮӯӰӱӲӳ',
|
|
45
|
+
z: 'ŹźŻżŽžƵƶȤȥɀΖ',
|
|
46
|
+
ß: 'β' //greek beta to german Eszett surrogate
|
|
47
|
+
}
|
|
48
|
+
//decompress data into two hashes
|
|
49
|
+
let unicode = {}
|
|
50
|
+
Object.keys(compact).forEach(function (k) {
|
|
51
|
+
compact[k].split('').forEach(function (s) {
|
|
52
|
+
unicode[s] = k
|
|
53
|
+
})
|
|
54
|
+
})
|
|
55
|
+
export default unicode
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
const separablePrefixes = '(ab|an|auf|aus|bei|ein|fest|fort|her|heraus|hervor|hin|hinzu|los|mit|nach|nieder|vor|voraus|weg|weiter|zu|zurecht|zurück|zusammen|zuteil)'
|
|
2
|
+
|
|
3
|
+
const postTagger = function (doc) {
|
|
4
|
+
// eine as 1 or the
|
|
5
|
+
doc.match('eine #Value').tag('TextValue', 'eine-value')
|
|
6
|
+
// 6.30 Uhr
|
|
7
|
+
doc.match('#Value uhr').tag('Time', 'time-Uhr')
|
|
8
|
+
// a value before a currency is money - '12,5 Euro'
|
|
9
|
+
doc.match('[#Value] #Currency', 0).tag('Money', 'value-currency')
|
|
10
|
+
|
|
11
|
+
// prenominal possessive is a determiner - 'mein Bruder', 'seine kleine Schwester'
|
|
12
|
+
doc.match('[(mein|dein|sein|ihr|unser|euer)] #Adjective? #Noun', 0).tag('Determiner', 'possessive-det')
|
|
13
|
+
doc.match('[#Possessive] #Adjective? #Noun', 0).tag('Determiner', 'possessive-det')
|
|
14
|
+
|
|
15
|
+
// 'weiß' beside a pronoun is wissen, not white - 'das weiß ich'
|
|
16
|
+
doc.match('[weiß] #Pronoun', 0).tag('PresentTense', 'weiß-pronoun')
|
|
17
|
+
doc.match('#Pronoun [weiß]', 0).tag('PresentTense', 'pronoun-weiß')
|
|
18
|
+
|
|
19
|
+
// a verb between a determiner and a noun is an inflected adjective - 'das schnelle Auto'
|
|
20
|
+
doc.match('#Determiner [(#Verb && !#Infinitive && /e[nrsm]?$/)] (#Noun && !#Pronoun)', 0).tag('Adjective', 'det-adj-noun')
|
|
21
|
+
|
|
22
|
+
// a determiner with no noun after it is a pronoun - 'das ist gut', 'alle außer ihm'
|
|
23
|
+
doc.match('[#Determiner] (#Verb|#Preposition)', 0).tag('Pronoun', 'det-pronoun')
|
|
24
|
+
|
|
25
|
+
// modal verbs - the infinitive goes to the end of the clause:
|
|
26
|
+
// dürfen (may), können (can), mögen (like), müssen (must), sollen (should), wollen (want)
|
|
27
|
+
doc.match('[{dürfen}] * #Infinitive', 0).tag('Modal', 'dürfen-verb')
|
|
28
|
+
doc.match('[{können}] * #Infinitive', 0).tag('Modal', 'können-verb')
|
|
29
|
+
doc.match('[{mögen}] * #Infinitive', 0).tag('Modal', 'mögen-verb')
|
|
30
|
+
doc.match('[{müssen}] * #Infinitive', 0).tag('Modal', 'müssen-verb')
|
|
31
|
+
doc.match('[{sollen}] * #Infinitive', 0).tag('Modal', 'sollen-verb')
|
|
32
|
+
doc.match('[{wollen}] * #Infinitive', 0).tag('Modal', 'wollen-verb')
|
|
33
|
+
|
|
34
|
+
// auxiliary + participle at the end of the clause (Satzklammer):
|
|
35
|
+
// 'sie hat gestern ein Buch gelesen', 'er ist nach Hause gegangen'
|
|
36
|
+
doc.match('[{haben}] * #Participle', 0).tag('Auxiliary', 'haben-participle')
|
|
37
|
+
doc.match('[{haben}] * #Infinitive', 0).tag('Auxiliary', 'haben-infinitive')
|
|
38
|
+
doc.match('[{sein}] * #Participle', 0).ifNo('#Determiner').tag('Auxiliary', 'sein-participle')
|
|
39
|
+
// werden + infinitive is the future tense - 'wir werden nach Berlin fahren'
|
|
40
|
+
doc.match('[{werden}] * #Infinitive', 0).tag('Auxiliary', 'werden-future')
|
|
41
|
+
// werden + participle is the passive - 'das Buch wird gelesen'
|
|
42
|
+
doc.match('[{werden}] * #Participle', 0).tag('Auxiliary', 'werden-passive')
|
|
43
|
+
|
|
44
|
+
// a separable verb-prefix, at the end of the clause - 'ich stehe um sieben Uhr auf'
|
|
45
|
+
doc.match('#Verb * [#Preposition]$', 0).match(separablePrefixes).tag('Particle', 'separable-prefix')
|
|
46
|
+
}
|
|
47
|
+
export default postTagger
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
const hasApostrophe = /['‘’‛‵′`´]/
|
|
2
|
+
const hasPeriod = /\./
|
|
3
|
+
const isNum = /^[0-9+\-,]+$/
|
|
4
|
+
|
|
5
|
+
// normal regexes
|
|
6
|
+
const doRegs = function (str, regs) {
|
|
7
|
+
for (let i = 0; i < regs.length; i += 1) {
|
|
8
|
+
if (regs[i][0].test(str) === true) {
|
|
9
|
+
return regs[i]
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
return null
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
const checkRegex = function (terms, i, world) {
|
|
16
|
+
let setTag = world.methods.one.setTag
|
|
17
|
+
let term = terms[i]
|
|
18
|
+
let { regexText, regexNormal, regexNumbers } = world.model.two
|
|
19
|
+
let normal = term.machine || term.normal
|
|
20
|
+
let text = term.text
|
|
21
|
+
// keep dangling apostrophe?
|
|
22
|
+
if (hasApostrophe.test(term.post) && !hasApostrophe.test(term.pre)) {
|
|
23
|
+
text += term.post.trim()
|
|
24
|
+
}
|
|
25
|
+
// keep period in number ordinals?
|
|
26
|
+
if (hasPeriod.test(term.post) && isNum.test(text)) {
|
|
27
|
+
setTag([term], ['Ordinal', 'NumericValue'], world, false, `1-regex-ordinal`)
|
|
28
|
+
term.confidence = 0.6
|
|
29
|
+
return true
|
|
30
|
+
}
|
|
31
|
+
let arr = doRegs(text, regexText) || doRegs(normal, regexNormal)
|
|
32
|
+
// hide a bunch of number regexes behind this one
|
|
33
|
+
if (!arr && /[0-9]/.test(normal)) {
|
|
34
|
+
arr = doRegs(normal, regexNumbers)
|
|
35
|
+
}
|
|
36
|
+
if (arr) {
|
|
37
|
+
setTag([term], arr[1], world, false, `1-regex- '${arr[2] || arr[0]}'`)
|
|
38
|
+
term.confidence = 0.6
|
|
39
|
+
return true
|
|
40
|
+
}
|
|
41
|
+
return null
|
|
42
|
+
}
|
|
43
|
+
export default checkRegex
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
const isTitleCase = function (str) {
|
|
2
|
+
return /^[A-ZÄÖÜ][a-z'\u00C0-\u00FF]/.test(str) || /^[A-ZÄÖÜ]$/.test(str)
|
|
3
|
+
}
|
|
4
|
+
|
|
5
|
+
// add a noun to any non-0 index titlecased word, with no existing tag
|
|
6
|
+
const titleCaseNoun = function (terms, i, world) {
|
|
7
|
+
let setTag = world.methods.one.setTag
|
|
8
|
+
let term = terms[i]
|
|
9
|
+
// don't over-write any tags
|
|
10
|
+
// if (term.tags.size > 0) {
|
|
11
|
+
// return
|
|
12
|
+
// }
|
|
13
|
+
// skip first-word, for now
|
|
14
|
+
if (i === 0) {
|
|
15
|
+
return
|
|
16
|
+
}
|
|
17
|
+
if (isTitleCase(term.text)) {
|
|
18
|
+
setTag([term], 'Noun', world, false, `1-titlecase`)
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
export default titleCaseNoun
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
const min = 1400
|
|
2
|
+
const max = 2100
|
|
3
|
+
|
|
4
|
+
const dateWords = new Set(['pendant', 'dans', 'avant', 'apres', 'pour', 'en'])
|
|
5
|
+
|
|
6
|
+
const seemsGood = function (term) {
|
|
7
|
+
if (!term) {
|
|
8
|
+
return false
|
|
9
|
+
}
|
|
10
|
+
if (dateWords.has(term.normal)) {
|
|
11
|
+
return true
|
|
12
|
+
}
|
|
13
|
+
if (term.tags.has('Date') || term.tags.has('Month') || term.tags.has('WeekDay')) {
|
|
14
|
+
return true
|
|
15
|
+
}
|
|
16
|
+
return false
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
const seemsOkay = function (term) {
|
|
20
|
+
if (!term) {
|
|
21
|
+
return false
|
|
22
|
+
}
|
|
23
|
+
if (term.tags.has('Ordinal')) {
|
|
24
|
+
return true
|
|
25
|
+
}
|
|
26
|
+
return false
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
// recognize '1993' as a year
|
|
30
|
+
const tagYear = function (terms, i, world) {
|
|
31
|
+
let setTag = world.methods.one.setTag
|
|
32
|
+
const term = terms[i]
|
|
33
|
+
if (term.tags.has('NumericValue') && term.tags.has('Cardinal') && term.normal.length === 4) {
|
|
34
|
+
let num = Number(term.normal)
|
|
35
|
+
// number between 1400 and 2100
|
|
36
|
+
if (num && !isNaN(num)) {
|
|
37
|
+
if (num > min && num < max) {
|
|
38
|
+
if (seemsGood(terms[i - 1]) || seemsGood(terms[i + 1])) {
|
|
39
|
+
setTag([term], 'Year', world, false, '1-tagYear')
|
|
40
|
+
return true
|
|
41
|
+
}
|
|
42
|
+
// or is it really-close to a year?
|
|
43
|
+
if (num > 1950 && num < 2025) {
|
|
44
|
+
if (seemsOkay(terms[i - 1]) || seemsOkay(terms[i + 1])) {
|
|
45
|
+
setTag([term], 'Year', world, false, '1-tagYear-close')
|
|
46
|
+
return true
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
return null
|
|
53
|
+
}
|
|
54
|
+
export default tagYear
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
const oneLetterAcronym = /^[A-ZÄÖÜ]('s|,)?$/
|
|
2
|
+
const isUpperCase = /^[A-Z-ÄÖÜ]+$/
|
|
3
|
+
const periodAcronym = /([A-ZÄÖÜ]\.)+[A-ZÄÖÜ]?,?$/
|
|
4
|
+
const noPeriodAcronym = /[A-ZÄÖÜ]{2,}('s|,)?$/
|
|
5
|
+
const lowerCaseAcronym = /([a-zäöü]\.)+[a-zäöü]\.?$/
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
const oneLetterWord = {
|
|
10
|
+
I: true,
|
|
11
|
+
A: true,
|
|
12
|
+
}
|
|
13
|
+
// just uppercase acronyms, no periods - 'UNOCHA'
|
|
14
|
+
const isNoPeriodAcronym = function (term, model) {
|
|
15
|
+
let str = term.text
|
|
16
|
+
// ensure it's all upper-case
|
|
17
|
+
if (isUpperCase.test(str) === false) {
|
|
18
|
+
return false
|
|
19
|
+
}
|
|
20
|
+
// long capitalized words are not usually either
|
|
21
|
+
if (str.length > 5) {
|
|
22
|
+
return false
|
|
23
|
+
}
|
|
24
|
+
// 'I' is not a acronym
|
|
25
|
+
if (oneLetterWord.hasOwnProperty(str)) {
|
|
26
|
+
return false
|
|
27
|
+
}
|
|
28
|
+
// known-words, like 'PIZZA' is not an acronym.
|
|
29
|
+
if (model.one.lexicon.hasOwnProperty(term.normal)) {
|
|
30
|
+
return false
|
|
31
|
+
}
|
|
32
|
+
//like N.D.A
|
|
33
|
+
if (periodAcronym.test(str) === true) {
|
|
34
|
+
return true
|
|
35
|
+
}
|
|
36
|
+
//like c.e.o
|
|
37
|
+
if (lowerCaseAcronym.test(str) === true) {
|
|
38
|
+
return true
|
|
39
|
+
}
|
|
40
|
+
//like 'F.'
|
|
41
|
+
if (oneLetterAcronym.test(str) === true) {
|
|
42
|
+
return true
|
|
43
|
+
}
|
|
44
|
+
//like NDA
|
|
45
|
+
if (noPeriodAcronym.test(str) === true) {
|
|
46
|
+
return true
|
|
47
|
+
}
|
|
48
|
+
return false
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const isAcronym = function (terms, i, world) {
|
|
52
|
+
let setTag = world.methods.one.setTag
|
|
53
|
+
let term = terms[i]
|
|
54
|
+
//these are not acronyms
|
|
55
|
+
if (term.tags.has('RomanNumeral') || term.tags.has('Acronym')) {
|
|
56
|
+
return null
|
|
57
|
+
}
|
|
58
|
+
//non-period ones are harder
|
|
59
|
+
if (isNoPeriodAcronym(term, world.model)) {
|
|
60
|
+
term.tags.clear()
|
|
61
|
+
setTag([term], ['Acronym', 'Noun'], world, false, '3-no-period-acronym')
|
|
62
|
+
return true
|
|
63
|
+
}
|
|
64
|
+
// one-letter acronyms
|
|
65
|
+
if (!oneLetterWord.hasOwnProperty(term.text) && oneLetterAcronym.test(term.text)) {
|
|
66
|
+
term.tags.clear()
|
|
67
|
+
setTag([term], ['Acronym', 'Noun'], world, false, '3-one-letter-acronym')
|
|
68
|
+
return true
|
|
69
|
+
}
|
|
70
|
+
//if it's a very-short organization?
|
|
71
|
+
if (term.tags.has('Organization') && term.text.length <= 3) {
|
|
72
|
+
setTag([term], 'Acronym', world, false, '3-org-acronym')
|
|
73
|
+
return true
|
|
74
|
+
}
|
|
75
|
+
// upper-case org, like UNESCO
|
|
76
|
+
if (term.tags.has('Organization') && isUpperCase.test(term.text) && term.text.length <= 6) {
|
|
77
|
+
setTag([term], 'Acronym', world, false, '3-titlecase-acronym')
|
|
78
|
+
return true
|
|
79
|
+
}
|
|
80
|
+
return null
|
|
81
|
+
}
|
|
82
|
+
export default isAcronym
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
const isTitleCase = function (str) {
|
|
2
|
+
return /^[A-ZÄÖÜ][a-z'\u00C0-\u00FF]/.test(str) || /^[A-ZÄÖÜ]$/.test(str)
|
|
3
|
+
}
|
|
4
|
+
|
|
5
|
+
const hasNoVerb = function (terms) {
|
|
6
|
+
return !terms.find(t => t.tags.has('Verb'))
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
const fallback = function (terms, i, world) {
|
|
10
|
+
let setTag = world.methods.one.setTag
|
|
11
|
+
let term = terms[i]
|
|
12
|
+
if (term.tags.size === 0) {
|
|
13
|
+
|
|
14
|
+
// is it first-word titlecase?
|
|
15
|
+
if (i === 0 && isTitleCase(term.text)) {
|
|
16
|
+
setTag([term], 'Noun', world, false, `1-titlecase`)// Noun still the safest bet?
|
|
17
|
+
return
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
let tag = 'Adjective'
|
|
21
|
+
if (terms.length > 10 && hasNoVerb(terms)) {
|
|
22
|
+
tag = 'Verb'
|
|
23
|
+
}
|
|
24
|
+
setTag([term], tag, world, false, '2-fallback')
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
export default fallback
|