psychowl 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +21 -0
- data/LICENSE.txt +21 -0
- data/README.md +212 -0
- data/exe/psychowl +106 -0
- data/ext/psychowl_native/core/data/LICENSE-whatlang +26 -0
- data/ext/psychowl_native/core/data/README.md +32 -0
- data/ext/psychowl_native/core/data/alphabets.tsv +16 -0
- data/ext/psychowl_native/core/data/languages.tsv +20 -0
- data/ext/psychowl_native/core/data/scripts.tsv +28 -0
- data/ext/psychowl_native/core/data/trigrams/ara.txt +302 -0
- data/ext/psychowl_native/core/data/trigrams/deu.txt +302 -0
- data/ext/psychowl_native/core/data/trigrams/eng.txt +302 -0
- data/ext/psychowl_native/core/data/trigrams/fra.txt +302 -0
- data/ext/psychowl_native/core/data/trigrams/hin.txt +302 -0
- data/ext/psychowl_native/core/data/trigrams/ind.txt +302 -0
- data/ext/psychowl_native/core/data/trigrams/ita.txt +302 -0
- data/ext/psychowl_native/core/data/trigrams/nld.txt +302 -0
- data/ext/psychowl_native/core/data/trigrams/pol.txt +302 -0
- data/ext/psychowl_native/core/data/trigrams/por.txt +302 -0
- data/ext/psychowl_native/core/data/trigrams/rus.txt +302 -0
- data/ext/psychowl_native/core/data/trigrams/spa.txt +302 -0
- data/ext/psychowl_native/core/data/trigrams/tur.txt +302 -0
- data/ext/psychowl_native/core/data/trigrams/ukr.txt +302 -0
- data/ext/psychowl_native/core/data/trigrams/vie.txt +302 -0
- data/lib/psychowl/active_model.rb +64 -0
- data/lib/psychowl/data_file.rb +14 -0
- data/lib/psychowl/detector.rb +123 -0
- data/lib/psychowl/engine.rb +23 -0
- data/lib/psychowl/info.rb +39 -0
- data/lib/psychowl/lang.rb +55 -0
- data/lib/psychowl/locale/en.yml +6 -0
- data/lib/psychowl/lookup.rb +25 -0
- data/lib/psychowl/native_speedup.rb +46 -0
- data/lib/psychowl/postgres.rb +35 -0
- data/lib/psychowl/railtie.rb +10 -0
- data/lib/psychowl/ruby_engine/segmentation.rb +84 -0
- data/lib/psychowl/ruby_engine/trigrams.rb +74 -0
- data/lib/psychowl/ruby_engine.rb +166 -0
- data/lib/psychowl/script.rb +39 -0
- data/lib/psychowl/segment.rb +24 -0
- data/lib/psychowl/tables.rb +55 -0
- data/lib/psychowl/text.rb +26 -0
- data/lib/psychowl/version.rb +5 -0
- data/lib/psychowl.rb +82 -0
- metadata +91 -0
|
@@ -0,0 +1,302 @@
|
|
|
1
|
+
# tur: 300 most frequent trigrams, most frequent first. "_" marks a word boundary.
|
|
2
|
+
# Ported from whatlang 0.18.0 (MIT), see ../LICENSE-whatlang.
|
|
3
|
+
_ve
|
|
4
|
+
_ha
|
|
5
|
+
ve_
|
|
6
|
+
ler
|
|
7
|
+
lar
|
|
8
|
+
ir_
|
|
9
|
+
in_
|
|
10
|
+
hak
|
|
11
|
+
_he
|
|
12
|
+
her
|
|
13
|
+
bir
|
|
14
|
+
er_
|
|
15
|
+
an_
|
|
16
|
+
arı
|
|
17
|
+
eri
|
|
18
|
+
ya_
|
|
19
|
+
_bi
|
|
20
|
+
ak_
|
|
21
|
+
r_h
|
|
22
|
+
eti
|
|
23
|
+
ın_
|
|
24
|
+
iye
|
|
25
|
+
yet
|
|
26
|
+
_ka
|
|
27
|
+
ası
|
|
28
|
+
ını
|
|
29
|
+
_ol
|
|
30
|
+
tle
|
|
31
|
+
eya
|
|
32
|
+
kkı
|
|
33
|
+
ara
|
|
34
|
+
akk
|
|
35
|
+
etl
|
|
36
|
+
sın
|
|
37
|
+
esi
|
|
38
|
+
na_
|
|
39
|
+
de_
|
|
40
|
+
ek_
|
|
41
|
+
_ta
|
|
42
|
+
nda
|
|
43
|
+
ini
|
|
44
|
+
_bu
|
|
45
|
+
ile
|
|
46
|
+
rın
|
|
47
|
+
rin
|
|
48
|
+
vey
|
|
49
|
+
ne_
|
|
50
|
+
kla
|
|
51
|
+
e_h
|
|
52
|
+
ine
|
|
53
|
+
ır_
|
|
54
|
+
ere
|
|
55
|
+
ama
|
|
56
|
+
dır
|
|
57
|
+
n_h
|
|
58
|
+
_sa
|
|
59
|
+
ına
|
|
60
|
+
sin
|
|
61
|
+
e_k
|
|
62
|
+
le_
|
|
63
|
+
_ge
|
|
64
|
+
mas
|
|
65
|
+
ınd
|
|
66
|
+
nın
|
|
67
|
+
ı_v
|
|
68
|
+
_va
|
|
69
|
+
lan
|
|
70
|
+
lma
|
|
71
|
+
erk
|
|
72
|
+
rke
|
|
73
|
+
nma
|
|
74
|
+
tin
|
|
75
|
+
rle
|
|
76
|
+
_te
|
|
77
|
+
nin
|
|
78
|
+
akl
|
|
79
|
+
a_v
|
|
80
|
+
da_
|
|
81
|
+
_de
|
|
82
|
+
let
|
|
83
|
+
ill
|
|
84
|
+
e_m
|
|
85
|
+
ard
|
|
86
|
+
en_
|
|
87
|
+
riy
|
|
88
|
+
aya
|
|
89
|
+
nı_
|
|
90
|
+
_hü
|
|
91
|
+
_şa
|
|
92
|
+
e_b
|
|
93
|
+
k_v
|
|
94
|
+
kın
|
|
95
|
+
k_h
|
|
96
|
+
_me
|
|
97
|
+
mil
|
|
98
|
+
san
|
|
99
|
+
_il
|
|
100
|
+
si_
|
|
101
|
+
rdı
|
|
102
|
+
e_d
|
|
103
|
+
dan
|
|
104
|
+
hür
|
|
105
|
+
var
|
|
106
|
+
ana
|
|
107
|
+
e_a
|
|
108
|
+
kes
|
|
109
|
+
et_
|
|
110
|
+
mes
|
|
111
|
+
şah
|
|
112
|
+
dir
|
|
113
|
+
_mi
|
|
114
|
+
ret
|
|
115
|
+
rri
|
|
116
|
+
_se
|
|
117
|
+
ola
|
|
118
|
+
ürr
|
|
119
|
+
irl
|
|
120
|
+
bu_
|
|
121
|
+
mak
|
|
122
|
+
_ma
|
|
123
|
+
mek
|
|
124
|
+
n_e
|
|
125
|
+
kı_
|
|
126
|
+
n_v
|
|
127
|
+
n_i
|
|
128
|
+
lik
|
|
129
|
+
lle
|
|
130
|
+
_ed
|
|
131
|
+
_hi
|
|
132
|
+
n_b
|
|
133
|
+
a_h
|
|
134
|
+
_ba
|
|
135
|
+
nsa
|
|
136
|
+
_iş
|
|
137
|
+
eli
|
|
138
|
+
kar
|
|
139
|
+
_iç
|
|
140
|
+
ı_h
|
|
141
|
+
ala
|
|
142
|
+
li_
|
|
143
|
+
ulu
|
|
144
|
+
rak
|
|
145
|
+
evl
|
|
146
|
+
e_i
|
|
147
|
+
ni_
|
|
148
|
+
re_
|
|
149
|
+
r_ş
|
|
150
|
+
eme
|
|
151
|
+
etm
|
|
152
|
+
e_t
|
|
153
|
+
ik_
|
|
154
|
+
e_s
|
|
155
|
+
a_b
|
|
156
|
+
iş_
|
|
157
|
+
n_k
|
|
158
|
+
hai
|
|
159
|
+
nde
|
|
160
|
+
aiz
|
|
161
|
+
_eş
|
|
162
|
+
izd
|
|
163
|
+
un_
|
|
164
|
+
olm
|
|
165
|
+
hiç
|
|
166
|
+
zdi
|
|
167
|
+
ar_
|
|
168
|
+
unm
|
|
169
|
+
ma_
|
|
170
|
+
_gö
|
|
171
|
+
ilm
|
|
172
|
+
lme
|
|
173
|
+
im_
|
|
174
|
+
n_t
|
|
175
|
+
tir
|
|
176
|
+
dil
|
|
177
|
+
mal
|
|
178
|
+
e_g
|
|
179
|
+
i_v
|
|
180
|
+
_ko
|
|
181
|
+
lun
|
|
182
|
+
e_e
|
|
183
|
+
mel
|
|
184
|
+
ket
|
|
185
|
+
ık_
|
|
186
|
+
n_s
|
|
187
|
+
ele
|
|
188
|
+
la_
|
|
189
|
+
el_
|
|
190
|
+
r_v
|
|
191
|
+
ede
|
|
192
|
+
şit
|
|
193
|
+
ili
|
|
194
|
+
eşi
|
|
195
|
+
yla
|
|
196
|
+
a_i
|
|
197
|
+
_an
|
|
198
|
+
anı
|
|
199
|
+
_et
|
|
200
|
+
rı_
|
|
201
|
+
ahs
|
|
202
|
+
_ya
|
|
203
|
+
sı_
|
|
204
|
+
edi
|
|
205
|
+
siy
|
|
206
|
+
t_v
|
|
207
|
+
i_b
|
|
208
|
+
se_
|
|
209
|
+
içi
|
|
210
|
+
çin
|
|
211
|
+
bul
|
|
212
|
+
ame
|
|
213
|
+
_da
|
|
214
|
+
miş
|
|
215
|
+
may
|
|
216
|
+
tim
|
|
217
|
+
a_k
|
|
218
|
+
tme
|
|
219
|
+
r_b
|
|
220
|
+
ins
|
|
221
|
+
yan
|
|
222
|
+
nla
|
|
223
|
+
mle
|
|
224
|
+
_di
|
|
225
|
+
eye
|
|
226
|
+
ger
|
|
227
|
+
ye_
|
|
228
|
+
uğu
|
|
229
|
+
erd
|
|
230
|
+
din
|
|
231
|
+
ser
|
|
232
|
+
_mü
|
|
233
|
+
mem
|
|
234
|
+
vle
|
|
235
|
+
_ke
|
|
236
|
+
nam
|
|
237
|
+
ind
|
|
238
|
+
len
|
|
239
|
+
eke
|
|
240
|
+
es_
|
|
241
|
+
_ki
|
|
242
|
+
n_m
|
|
243
|
+
it_
|
|
244
|
+
_in
|
|
245
|
+
_ku
|
|
246
|
+
rşı
|
|
247
|
+
a_s
|
|
248
|
+
arş
|
|
249
|
+
_ay
|
|
250
|
+
eml
|
|
251
|
+
lek
|
|
252
|
+
oru
|
|
253
|
+
rme
|
|
254
|
+
kor
|
|
255
|
+
rde
|
|
256
|
+
i_m
|
|
257
|
+
_so
|
|
258
|
+
tür
|
|
259
|
+
al_
|
|
260
|
+
lam
|
|
261
|
+
eni
|
|
262
|
+
nun
|
|
263
|
+
_uy
|
|
264
|
+
ken
|
|
265
|
+
hsı
|
|
266
|
+
i_i
|
|
267
|
+
a_d
|
|
268
|
+
ri_
|
|
269
|
+
dev
|
|
270
|
+
ün_
|
|
271
|
+
a_m
|
|
272
|
+
r_a
|
|
273
|
+
mey
|
|
274
|
+
cak
|
|
275
|
+
ıyl
|
|
276
|
+
maz
|
|
277
|
+
e_v
|
|
278
|
+
ece
|
|
279
|
+
ade
|
|
280
|
+
iç_
|
|
281
|
+
şma
|
|
282
|
+
mse
|
|
283
|
+
te_
|
|
284
|
+
tün
|
|
285
|
+
ims
|
|
286
|
+
kim
|
|
287
|
+
e_y
|
|
288
|
+
şı_
|
|
289
|
+
end
|
|
290
|
+
k_g
|
|
291
|
+
ndi
|
|
292
|
+
alı
|
|
293
|
+
_ce
|
|
294
|
+
lem
|
|
295
|
+
öğr
|
|
296
|
+
ütü
|
|
297
|
+
k_i
|
|
298
|
+
r_t
|
|
299
|
+
_öğ
|
|
300
|
+
büt
|
|
301
|
+
anl
|
|
302
|
+
_bü
|
|
@@ -0,0 +1,302 @@
|
|
|
1
|
+
# ukr: 300 most frequent trigrams, most frequent first. "_" marks a word boundary.
|
|
2
|
+
# Ported from whatlang 0.18.0 (MIT), see ../LICENSE-whatlang.
|
|
3
|
+
на_
|
|
4
|
+
_пр
|
|
5
|
+
_і_
|
|
6
|
+
пра
|
|
7
|
+
рав
|
|
8
|
+
_на
|
|
9
|
+
ня_
|
|
10
|
+
ння
|
|
11
|
+
_за
|
|
12
|
+
ого
|
|
13
|
+
_по
|
|
14
|
+
ти_
|
|
15
|
+
го_
|
|
16
|
+
люд
|
|
17
|
+
_лю
|
|
18
|
+
во_
|
|
19
|
+
_ко
|
|
20
|
+
_ма
|
|
21
|
+
льн
|
|
22
|
+
юди
|
|
23
|
+
их_
|
|
24
|
+
о_н
|
|
25
|
+
_не
|
|
26
|
+
аво
|
|
27
|
+
анн
|
|
28
|
+
дин
|
|
29
|
+
_св
|
|
30
|
+
сво
|
|
31
|
+
ожн
|
|
32
|
+
кож
|
|
33
|
+
енн
|
|
34
|
+
пов
|
|
35
|
+
жна
|
|
36
|
+
_до
|
|
37
|
+
ати
|
|
38
|
+
ина
|
|
39
|
+
ає_
|
|
40
|
+
а_л
|
|
41
|
+
_бу
|
|
42
|
+
аці
|
|
43
|
+
не_
|
|
44
|
+
ува
|
|
45
|
+
обо
|
|
46
|
+
_ос
|
|
47
|
+
_як
|
|
48
|
+
має
|
|
49
|
+
_ви
|
|
50
|
+
них
|
|
51
|
+
аль
|
|
52
|
+
або
|
|
53
|
+
є_п
|
|
54
|
+
_та
|
|
55
|
+
ні_
|
|
56
|
+
ть_
|
|
57
|
+
ови
|
|
58
|
+
бо_
|
|
59
|
+
_ві
|
|
60
|
+
_аб
|
|
61
|
+
ере
|
|
62
|
+
і_п
|
|
63
|
+
а_м
|
|
64
|
+
вин
|
|
65
|
+
без
|
|
66
|
+
при
|
|
67
|
+
іль
|
|
68
|
+
ног
|
|
69
|
+
о_п
|
|
70
|
+
ми_
|
|
71
|
+
та_
|
|
72
|
+
ом_
|
|
73
|
+
ою_
|
|
74
|
+
бод
|
|
75
|
+
ста
|
|
76
|
+
воб
|
|
77
|
+
_бе
|
|
78
|
+
до_
|
|
79
|
+
ва_
|
|
80
|
+
ті_
|
|
81
|
+
_об
|
|
82
|
+
о_в
|
|
83
|
+
ост
|
|
84
|
+
_в_
|
|
85
|
+
_що
|
|
86
|
+
ий_
|
|
87
|
+
ся_
|
|
88
|
+
і_с
|
|
89
|
+
_сп
|
|
90
|
+
инн
|
|
91
|
+
від
|
|
92
|
+
ств
|
|
93
|
+
и_п
|
|
94
|
+
ван
|
|
95
|
+
нов
|
|
96
|
+
нан
|
|
97
|
+
кон
|
|
98
|
+
_у_
|
|
99
|
+
ват
|
|
100
|
+
она
|
|
101
|
+
ії_
|
|
102
|
+
но_
|
|
103
|
+
дно
|
|
104
|
+
ій_
|
|
105
|
+
езп
|
|
106
|
+
пер
|
|
107
|
+
_де
|
|
108
|
+
ути
|
|
109
|
+
ьно
|
|
110
|
+
ист
|
|
111
|
+
під
|
|
112
|
+
сті
|
|
113
|
+
бут
|
|
114
|
+
_мо
|
|
115
|
+
и_і
|
|
116
|
+
ідн
|
|
117
|
+
ако
|
|
118
|
+
нні
|
|
119
|
+
ід_
|
|
120
|
+
тис
|
|
121
|
+
що_
|
|
122
|
+
род
|
|
123
|
+
і_в
|
|
124
|
+
а_з
|
|
125
|
+
ава
|
|
126
|
+
_пе
|
|
127
|
+
му_
|
|
128
|
+
і_н
|
|
129
|
+
а_п
|
|
130
|
+
соб
|
|
131
|
+
ої_
|
|
132
|
+
а_в
|
|
133
|
+
спр
|
|
134
|
+
ів_
|
|
135
|
+
ний
|
|
136
|
+
яко
|
|
137
|
+
ду_
|
|
138
|
+
вно
|
|
139
|
+
і_д
|
|
140
|
+
ну_
|
|
141
|
+
аро
|
|
142
|
+
и_с
|
|
143
|
+
_ін
|
|
144
|
+
ля_
|
|
145
|
+
рів
|
|
146
|
+
у_в
|
|
147
|
+
_рі
|
|
148
|
+
и_д
|
|
149
|
+
нар
|
|
150
|
+
нен
|
|
151
|
+
ова
|
|
152
|
+
ому
|
|
153
|
+
лен
|
|
154
|
+
нац
|
|
155
|
+
ним
|
|
156
|
+
ися
|
|
157
|
+
чи_
|
|
158
|
+
ав_
|
|
159
|
+
і_р
|
|
160
|
+
ном
|
|
161
|
+
_ро
|
|
162
|
+
нос
|
|
163
|
+
ві_
|
|
164
|
+
вни
|
|
165
|
+
овн
|
|
166
|
+
_її
|
|
167
|
+
ові
|
|
168
|
+
мож
|
|
169
|
+
віл
|
|
170
|
+
у_п
|
|
171
|
+
_пі
|
|
172
|
+
_су
|
|
173
|
+
її_
|
|
174
|
+
одн
|
|
175
|
+
_вс
|
|
176
|
+
ово
|
|
177
|
+
ють
|
|
178
|
+
іст
|
|
179
|
+
сть
|
|
180
|
+
і_з
|
|
181
|
+
_ст
|
|
182
|
+
буд
|
|
183
|
+
_ра
|
|
184
|
+
чен
|
|
185
|
+
про
|
|
186
|
+
роз
|
|
187
|
+
івн
|
|
188
|
+
оду
|
|
189
|
+
а_о
|
|
190
|
+
ьни
|
|
191
|
+
ни_
|
|
192
|
+
о_с
|
|
193
|
+
сно
|
|
194
|
+
зна
|
|
195
|
+
рац
|
|
196
|
+
им_
|
|
197
|
+
о_д
|
|
198
|
+
ими
|
|
199
|
+
я_і
|
|
200
|
+
ції
|
|
201
|
+
х_п
|
|
202
|
+
дер
|
|
203
|
+
чин
|
|
204
|
+
_со
|
|
205
|
+
а_с
|
|
206
|
+
ерж
|
|
207
|
+
и_з
|
|
208
|
+
и_в
|
|
209
|
+
е_п
|
|
210
|
+
ди_
|
|
211
|
+
заб
|
|
212
|
+
осо
|
|
213
|
+
у_с
|
|
214
|
+
е_б
|
|
215
|
+
сі_
|
|
216
|
+
тер
|
|
217
|
+
ніх
|
|
218
|
+
я_н
|
|
219
|
+
і_б
|
|
220
|
+
кла
|
|
221
|
+
спі
|
|
222
|
+
в_і
|
|
223
|
+
_ні
|
|
224
|
+
о_з
|
|
225
|
+
ржа
|
|
226
|
+
сту
|
|
227
|
+
їх_
|
|
228
|
+
а_н
|
|
229
|
+
нна
|
|
230
|
+
так
|
|
231
|
+
я_п
|
|
232
|
+
зпе
|
|
233
|
+
_од
|
|
234
|
+
абе
|
|
235
|
+
для
|
|
236
|
+
ту_
|
|
237
|
+
і_м
|
|
238
|
+
печ
|
|
239
|
+
_дл
|
|
240
|
+
же_
|
|
241
|
+
ки_
|
|
242
|
+
віт
|
|
243
|
+
ніс
|
|
244
|
+
гал
|
|
245
|
+
ага
|
|
246
|
+
е_м
|
|
247
|
+
ами
|
|
248
|
+
зах
|
|
249
|
+
рим
|
|
250
|
+
ї_о
|
|
251
|
+
тан
|
|
252
|
+
ког
|
|
253
|
+
рес
|
|
254
|
+
удь
|
|
255
|
+
_ре
|
|
256
|
+
то_
|
|
257
|
+
ков
|
|
258
|
+
тор
|
|
259
|
+
ара
|
|
260
|
+
сві
|
|
261
|
+
тва
|
|
262
|
+
а_б
|
|
263
|
+
оже
|
|
264
|
+
соц
|
|
265
|
+
оці
|
|
266
|
+
ціа
|
|
267
|
+
осн
|
|
268
|
+
роб
|
|
269
|
+
дь‐
|
|
270
|
+
ь‐я
|
|
271
|
+
‐як
|
|
272
|
+
і_і
|
|
273
|
+
заг
|
|
274
|
+
ахи
|
|
275
|
+
хис
|
|
276
|
+
піл
|
|
277
|
+
цій
|
|
278
|
+
х_в
|
|
279
|
+
лив
|
|
280
|
+
осв
|
|
281
|
+
іал
|
|
282
|
+
руч
|
|
283
|
+
ь_п
|
|
284
|
+
інш
|
|
285
|
+
в_я
|
|
286
|
+
ги_
|
|
287
|
+
аги
|
|
288
|
+
_ді
|
|
289
|
+
ком
|
|
290
|
+
ини
|
|
291
|
+
а_і
|
|
292
|
+
оди
|
|
293
|
+
нал
|
|
294
|
+
тво
|
|
295
|
+
кої
|
|
296
|
+
всі
|
|
297
|
+
я_в
|
|
298
|
+
ною
|
|
299
|
+
об_
|
|
300
|
+
о_у
|
|
301
|
+
о_о
|
|
302
|
+
і_о
|