tokenator 0.2.0 → 0.2.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 35c1d63916cde346788a0c80a4782eff19afedc3
4
- data.tar.gz: 4fb32d3b8ec35383b8ac3cf076b9ec213e3ccfc3
3
+ metadata.gz: ba4933f843a64a58ceafa12879d2527fad555a84
4
+ data.tar.gz: 2ddf0bc5ab90202d7c946b2a7c0ff9dcbcc85587
5
5
  SHA512:
6
- metadata.gz: c158a5069c3d3edd59cc3ea6b474b83a5f62b05bf3c3d9ed87de1eab2d0e51edb369eb12db854b7873110c2d771d67fbb29f886f4ca1cf094261ec932a3c1854
7
- data.tar.gz: cc005580610d0ef1a3668876c29a0afdda2164510951cd1ab142ad1afbdbf7e53fb3e222efff271cae51fe00849394cc301988377aaa18ef5a7e5f4237eb3f30
6
+ metadata.gz: ddded9a096194ad8c0f7faf7db11bccfd56da2b3e91c12b2b3c92560df4a242cd69df5a5896e5c446bb33d545b5621493c0ab720475e004532e2c8a6ebb8a5b2
7
+ data.tar.gz: 2648ce29678aa3454d3b2849776f6c6fdfdbf43e3ccf03d32e17ac1c4e35eeffcdb2aff9131729456b8c4daf9208a818e9cf5865560ebacf88a54fc2a85e5200
@@ -0,0 +1,1048 @@
1
+ <!DOCTYPE html><html><head><meta charset="utf-8"><style>body {
2
+ width: 45em;
3
+ border: 1px solid #ddd;
4
+ outline: 1300px solid #fff;
5
+ margin: 16px auto;
6
+ }
7
+
8
+ body .markdown-body
9
+ {
10
+ padding: 30px;
11
+ }
12
+
13
+ @font-face {
14
+ font-family: fontawesome-mini;
15
+ src: url(data:font/woff;charset=utf-8;base64,d09GRgABAAAAAAzUABAAAAAAFNgAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAABGRlRNAAABbAAAABwAAAAcZMzaOEdERUYAAAGIAAAAHQAAACAAOQAET1MvMgAAAagAAAA+AAAAYHqhde9jbWFwAAAB6AAAAFIAAAFa4azkLWN2dCAAAAI8AAAAKAAAACgFgwioZnBnbQAAAmQAAAGxAAACZVO0L6dnYXNwAAAEGAAAAAgAAAAIAAAAEGdseWYAAAQgAAAFDgAACMz7eroHaGVhZAAACTAAAAAwAAAANgWEOEloaGVhAAAJYAAAAB0AAAAkDGEGa2htdHgAAAmAAAAAEwAAADBEgAAQbG9jYQAACZQAAAAaAAAAGgsICJBtYXhwAAAJsAAAACAAAAAgASgBD25hbWUAAAnQAAACZwAABOD4no+3cG9zdAAADDgAAABsAAAAmF+yXM9wcmVwAAAMpAAAAC4AAAAusPIrFAAAAAEAAAAAyYlvMQAAAADLVHQgAAAAAM/u9uZ4nGNgZGBg4ANiCQYQYGJgBEJuIGYB8xgABMMAPgAAAHicY2Bm42OcwMDKwMLSw2LMwMDQBqGZihmiwHycoKCyqJjB4YPDh4NsDP+BfNb3DIuAFCOSEgUGRgAKDgt4AAB4nGNgYGBmgGAZBkYGEAgB8hjBfBYGCyDNxcDBwMTA9MHhQ9SHrA8H//9nYACyQyFs/sP86/kX8HtB9UIBIxsDXICRCUgwMaACRoZhDwA3fxKSAAAAAAHyAHABJQB/AIEAdAFGAOsBIwC/ALgAxACGAGYAugBNACcA/wCIeJxdUbtOW0EQ3Q0PA4HE2CA52hSzmZDGe6EFCcTVjWJkO4XlCGk3cpGLcQEfQIFEDdqvGaChpEibBiEXSHxCPiESM2uIojQ7O7NzzpkzS8qRqnfpa89T5ySQwt0GzTb9Tki1swD3pOvrjYy0gwdabGb0ynX7/gsGm9GUO2oA5T1vKQ8ZTTuBWrSn/tH8Cob7/B/zOxi0NNP01DoJ6SEE5ptxS4PvGc26yw/6gtXhYjAwpJim4i4/plL+tzTnasuwtZHRvIMzEfnJNEBTa20Emv7UIdXzcRRLkMumsTaYmLL+JBPBhcl0VVO1zPjawV2ys+hggyrNgQfYw1Z5DB4ODyYU0rckyiwNEfZiq8QIEZMcCjnl3Mn+pED5SBLGvElKO+OGtQbGkdfAoDZPs/88m01tbx3C+FkcwXe/GUs6+MiG2hgRYjtiKYAJREJGVfmGGs+9LAbkUvvPQJSA5fGPf50ItO7YRDyXtXUOMVYIen7b3PLLirtWuc6LQndvqmqo0inN+17OvscDnh4Lw0FjwZvP+/5Kgfo8LK40aA4EQ3o3ev+iteqIq7wXPrIn07+xWgAAAAABAAH//wAPeJyFlctvG1UUh+/12DPN1B7P3JnYjj2Ox4/MuDHxJH5N3UdaEUQLqBIkfQQioJWQ6AMEQkIqsPGCPwA1otuWSmTBhjtps2ADWbJg3EpIXbGouqSbCraJw7kzNo2dRN1cnXN1ZvT7zuuiMEI7ncizyA0URofRBJpCdbQuIFShYY+GZRrxMDVtih5TwQPHtXDFFSIKoWIbuREBjLH27Ny4MsbVx+uOJThavebgVrNRLAiYx06rXsvhxLgWx9xpfHdrs/ekc2Pl2cpPCVEITQpwbj8VQhfXSq2m+Wxqaq2D73Kne5e3NjHqQNj3CRYlJlgUl/jRNP+2Gs2pNYRQiOnmUaQDqm30KqKiTTWPWjboxnTWpvgxjXo0KrtZXAHt7hwIz0YVcj88JnKlJKi3NPAwLyDwZudSmJSMMJFDYaOkaol6XtESx3Gt1VTytdZJ3DCLeaVhVnCBH1fycHTxFXwPX+l2e3d6H/TufGGmMTLTnbSJUdo00zuBswMO/nl3YLeL/wnu9/limCuD3vC54h5NBVz6Li414AI8Vx3iiosKcQXUbrvhFFiYb++HN4DaF4XzFW0fIN4XDWJ3a3XQoq9V8WiyRmdsatV9xUcHims1JloH0YUa090G3Tro3mC6c01f+YwCPquINr1PTaCP6rVTOOmf0GE2dBc7zWIhji3/5MchSuBHgDbU99RMWt3YUNMZMJmx92YP6NsHx/5/M1yvInpnkIOM3Z8fA3JQ2lW1RFC1KaBPDFXNAHYYvGy73aYZZZ3HifbeuiVZCpwA3oQBs0wGPYJbJfg60xrKEbKiNtTe1adwrpBRwlAuQ3q3VRaX0QmQ9a49BTSCuF1MLfQ6+tinOubRBZuWPNoMevGMT+V41KitO1is3D/tpMcq1JHZqDHGs8DoYGDkxJgKjHROeTCmhZvzPm9pod+ltKm4PN7Dyvvldlpsg8D+4AUJZ3F/JBstZz7cbFRxsaAGV6yX/dkcycWf8eS3QlQea+YLjdm3yrOnrhFpUyKVvFE4lpv4bO3Svx/6F/4xmiDu/RT5iI++lko18mY1oX+5UGKR6kmVjM/Zb76yfHtxy+h/SyQ0lLdpdKy/lWB6szatetQJ8nZ80A2Qt6ift6gJeavU3BO4gtxs/KCtNPVibCtYCWY3SIlSBPKXZALXiIR9oZeJ1AuMyxLpHIy/yO7vSiSE+kZvk0ihJ30HgHfzZtEMmvV58x6dtqns0XTAW7Vdm4HJ04OCp/crOO7rd9SGxQAE/mVA9xRN+kVSMRFF6S9JFGUtthkjBA5tFCWc2l4V43Ex9GmUP3SI37Jjmir9KqlaDJ4S4JB3vuM/jzyH1+8MuoZ+QGzfnvPoJb96cZlWjMcKLfgDwB7E634JTY+asjsPzS5CiVnEWY+KsrsIN5rn3mAPjqmQBxGjcGKB9f9ZxY3mYC2L85CJ2FXIxKKyHk+dg0FHbuEc7D5NzWUX32WxFcWNGRAbvwSx0RmIXVDuYySafluQBmzA/ssqJAMLnli+WIC90Gw4lm85wcp0qjArEDPJJV/sSx4P9ungTpgMw5gVC1XO4uULq0s3v1rqLi0vX/z65vlH50f8T/RHmSPTk5xxWBWOluMT6WiOy+tdvWxlV/XQb3o3c6Ssr+r6I708GsX9/nzp1tKFh0s3v7m4vAy/Hnb/KMOvc1wump6Il48K6mGDy02X9Yd65pa+nQIjk76lWxCkG8NBCP0HQS9IpAAAeJxjYGRgYGBhcCrq214Qz2/zlUGenQEEzr/77oug/zewFbB+AHI5GJhAogBwKQ0qeJxjYGRgYH3/P46BgZ0BBNgKGBgZUAEPAE/7At0AAAB4nGNngAB2IGYjhBsYBAAIYADVAAAAAAAAAAAAAFwAyAEeAaACCgKmAx4DggRmAAAAAQAAAAwAagAEAAAAAAACAAEAAgAWAAABAAChAAAAAHiclZI7bxQxFIWPd/JkUYQChEhIyAVKgdBMskm1QkKrRETpQiLRUczueB/K7HhlOxttg8LvoKPgP9DxFxANDR0tHRWi4NjrPIBEgh1p/dm+vufcawNYFWsQmP6e4jSyQB2fI9cwj++RE9wTjyPP4LYoI89iWbyLPIe6+Bh5Hs9rryMv4GbtW+RF3EhuRa7jbrIbeQkPkjdUETOLnL0Kip4FVvAhco1RXyMnSPEz8gzWxE7kWTwUp5HnsCLeR57HW/El8gJWa58iL+JO7UfkOh4l9yMv4UnyEtvQGGECgwF66MNBooF1bGCL1ELB/TYU+ZBRlvsKQ44Se6jQ4a7hef+fh72Crv25kp+8lNWGmeKoOI5jJLb1aGIGvb6TjfWNLdkqdFvJw4l1amjlXtXRZqRN7lSRylZZyhBqpVFWmTEXgWfUrpi/hZOQXdOd4rKuXOtEWT3k5IArPRzTUU5tHKjecZkTpnVbNOnt6jzN8240GD4xtikvZW56043rPMg/dS+dlOceXoR+WPbJ55Dsekq1lJpnypsMUsYOdCW30o103Ytu/lvh+5RWFLfBjm9/N8hJntPhvx92rnoE/kyHdGasGy754kw36vsVf/lFeBi+0COu+cfgQr42G3CRpeLoZ53gmfe3X6rcKt5oVxnptHR9JS8ehVUd5wvvahN2uqxOOpMXapibI5k7Zwbt4xBSaTfoKBufhAnO/uqNcfK8OTs0OQ6l7JIqFjDhYj5WcjevCnI/1DDiI8j4ndWb/5YzDZWh79yomWXeXj7Nnw70/2TIeFPTrlSh89k1ObOSRVZWZfgF0r/zJQB4nG2JUQuCQBCEd07TTg36fb2IyBaLd3vWaUh/vmSJnvpgmG8YcmS8X3Shf3R7QA4OBUocUKHGER5NNbOOEvwc1txnuWkTRb/aPjimJ5vXabI+3VfOiyS15UWvyezM2xiGOPyuMohOH8O8JiO4Af+FsAGNAEuwCFBYsQEBjlmxRgYrWCGwEFlLsBRSWCGwgFkdsAYrXFhZsBQrAAA=) format('woff');
16
+ }
17
+
18
+ @font-face {
19
+ font-family: octicons-anchor;
20
+ src: url(data:font/woff;charset=utf-8;base64,d09GRgABAAAAAAYcAA0AAAAACjQAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAABGRlRNAAABMAAAABwAAAAca8vGTk9TLzIAAAFMAAAARAAAAFZG1VHVY21hcAAAAZAAAAA+AAABQgAP9AdjdnQgAAAB0AAAAAQAAAAEACICiGdhc3AAAAHUAAAACAAAAAj//wADZ2x5ZgAAAdwAAADRAAABEKyikaNoZWFkAAACsAAAAC0AAAA2AtXoA2hoZWEAAALgAAAAHAAAACQHngNFaG10eAAAAvwAAAAQAAAAEAwAACJsb2NhAAADDAAAAAoAAAAKALIAVG1heHAAAAMYAAAAHwAAACABEAB2bmFtZQAAAzgAAALBAAAFu3I9x/Nwb3N0AAAF/AAAAB0AAAAvaoFvbwAAAAEAAAAAzBdyYwAAAADP2IQvAAAAAM/bz7t4nGNgZGFgnMDAysDB1Ml0hoGBoR9CM75mMGLkYGBgYmBlZsAKAtJcUxgcPsR8iGF2+O/AEMPsznAYKMwIkgMA5REMOXicY2BgYGaAYBkGRgYQsAHyGMF8FgYFIM0ChED+h5j//yEk/3KoSgZGNgYYk4GRCUgwMaACRoZhDwCs7QgGAAAAIgKIAAAAAf//AAJ4nHWMMQrCQBBF/0zWrCCIKUQsTDCL2EXMohYGSSmorScInsRGL2DOYJe0Ntp7BK+gJ1BxF1stZvjz/v8DRghQzEc4kIgKwiAppcA9LtzKLSkdNhKFY3HF4lK69ExKslx7Xa+vPRVS43G98vG1DnkDMIBUgFN0MDXflU8tbaZOUkXUH0+U27RoRpOIyCKjbMCVejwypzJJG4jIwb43rfl6wbwanocrJm9XFYfskuVC5K/TPyczNU7b84CXcbxks1Un6H6tLH9vf2LRnn8Ax7A5WQAAAHicY2BkYGAA4teL1+yI57f5ysDNwgAC529f0kOmWRiYVgEpDgYmEA8AUzEKsQAAAHicY2BkYGB2+O/AEMPCAAJAkpEBFbAAADgKAe0EAAAiAAAAAAQAAAAEAAAAAAAAKgAqACoAiAAAeJxjYGRgYGBhsGFgYgABEMkFhAwM/xn0QAIAD6YBhwB4nI1Ty07cMBS9QwKlQapQW3VXySvEqDCZGbGaHULiIQ1FKgjWMxknMfLEke2A+IJu+wntrt/QbVf9gG75jK577Lg8K1qQPCfnnnt8fX1NRC/pmjrk/zprC+8D7tBy9DHgBXoWfQ44Av8t4Bj4Z8CLtBL9CniJluPXASf0Lm4CXqFX8Q84dOLnMB17N4c7tBo1AS/Qi+hTwBH4rwHHwN8DXqQ30XXAS7QaLwSc0Gn8NuAVWou/gFmnjLrEaEh9GmDdDGgL3B4JsrRPDU2hTOiMSuJUIdKQQayiAth69r6akSSFqIJuA19TrzCIaY8sIoxyrNIrL//pw7A2iMygkX5vDj+G+kuoLdX4GlGK/8Lnlz6/h9MpmoO9rafrz7ILXEHHaAx95s9lsI7AHNMBWEZHULnfAXwG9/ZqdzLI08iuwRloXE8kfhXYAvE23+23DU3t626rbs8/8adv+9DWknsHp3E17oCf+Z48rvEQNZ78paYM38qfk3v/u3l3u3GXN2Dmvmvpf1Srwk3pB/VSsp512bA/GG5i2WJ7wu430yQ5K3nFGiOqgtmSB5pJVSizwaacmUZzZhXLlZTq8qGGFY2YcSkqbth6aW1tRmlaCFs2016m5qn36SbJrqosG4uMV4aP2PHBmB3tjtmgN2izkGQyLWprekbIntJFing32a5rKWCN/SdSoga45EJykyQ7asZvHQ8PTm6cslIpwyeyjbVltNikc2HTR7YKh9LBl9DADC0U/jLcBZDKrMhUBfQBvXRzLtFtjU9eNHKin0x5InTqb8lNpfKv1s1xHzTXRqgKzek/mb7nB8RZTCDhGEX3kK/8Q75AmUM/eLkfA+0Hi908Kx4eNsMgudg5GLdRD7a84npi+YxNr5i5KIbW5izXas7cHXIMAau1OueZhfj+cOcP3P8MNIWLyYOBuxL6DRylJ4cAAAB4nGNgYoAALjDJyIAOWMCiTIxMLDmZedkABtIBygAAAA==) format('woff');
21
+ }
22
+
23
+ .markdown-body {
24
+ font-family: sans-serif;
25
+ -ms-text-size-adjust: 100%;
26
+ -webkit-text-size-adjust: 100%;
27
+ color: #333333;
28
+ overflow: hidden;
29
+ font-family: "Helvetica Neue", Helvetica, "Segoe UI", Arial, freesans, sans-serif;
30
+ font-size: 16px;
31
+ line-height: 1.6;
32
+ word-wrap: break-word;
33
+ }
34
+
35
+ .markdown-body a {
36
+ background: transparent;
37
+ }
38
+
39
+ .markdown-body a:active,
40
+ .markdown-body a:hover {
41
+ outline: 0;
42
+ }
43
+
44
+ .markdown-body b,
45
+ .markdown-body strong {
46
+ font-weight: bold;
47
+ }
48
+
49
+ .markdown-body mark {
50
+ background: #ff0;
51
+ color: #000;
52
+ font-style: italic;
53
+ font-weight: bold;
54
+ }
55
+
56
+ .markdown-body sub,
57
+ .markdown-body sup {
58
+ font-size: 75%;
59
+ line-height: 0;
60
+ position: relative;
61
+ vertical-align: baseline;
62
+ }
63
+ .markdown-body sup {
64
+ top: -0.5em;
65
+ }
66
+ .markdown-body sub {
67
+ bottom: -0.25em;
68
+ }
69
+
70
+ .markdown-body h1 {
71
+ font-size: 2em;
72
+ margin: 0.67em 0;
73
+ }
74
+
75
+ .markdown-body img {
76
+ border: 0;
77
+ }
78
+
79
+ .markdown-body hr {
80
+ -moz-box-sizing: content-box;
81
+ box-sizing: content-box;
82
+ height: 0;
83
+ }
84
+
85
+ .markdown-body pre {
86
+ overflow: auto;
87
+ }
88
+
89
+ .markdown-body code,
90
+ .markdown-body kbd,
91
+ .markdown-body pre,
92
+ .markdown-body samp {
93
+ font-family: monospace, monospace;
94
+ font-size: 1em;
95
+ }
96
+
97
+ .markdown-body input {
98
+ color: inherit;
99
+ font: inherit;
100
+ margin: 0;
101
+ }
102
+
103
+ .markdown-body html input[disabled] {
104
+ cursor: default;
105
+ }
106
+
107
+ .markdown-body input {
108
+ line-height: normal;
109
+ }
110
+
111
+ .markdown-body input[type="checkbox"] {
112
+ box-sizing: border-box;
113
+ padding: 0;
114
+ }
115
+
116
+ .markdown-body table {
117
+ border-collapse: collapse;
118
+ border-spacing: 0;
119
+ }
120
+
121
+ .markdown-body td,
122
+ .markdown-body th {
123
+ padding: 0;
124
+ }
125
+
126
+ .markdown-body .codehilitetable {
127
+ border: 0;
128
+ border-spacing: 0;
129
+ }
130
+
131
+ .markdown-body .codehilitetable tr {
132
+ border: 0;
133
+ }
134
+
135
+ .markdown-body .codehilitetable pre,
136
+ .markdown-body .codehilitetable div.codehilite {
137
+ margin: 0;
138
+ }
139
+
140
+ .markdown-body .linenos,
141
+ .markdown-body .code,
142
+ .markdown-body .codehilitetable td {
143
+ border: 0;
144
+ padding: 0;
145
+ }
146
+
147
+ .markdown-body td:not(.linenos) .linenodiv {
148
+ padding: 0 !important;
149
+ }
150
+
151
+ .markdown-body .code {
152
+ width: 100%;
153
+ }
154
+
155
+ .markdown-body .linenos div pre,
156
+ .markdown-body .linenodiv pre,
157
+ .markdown-body .linenodiv {
158
+ border: 0;
159
+ -webkit-border-radius: 0;
160
+ -moz-border-radius: 0;
161
+ border-radius: 0;
162
+ -webkit-border-top-left-radius: 3px;
163
+ -webkit-border-bottom-left-radius: 3px;
164
+ -moz-border-radius-topleft: 3px;
165
+ -moz-border-radius-bottomleft: 3px;
166
+ border-top-left-radius: 3px;
167
+ border-bottom-left-radius: 3px;
168
+ }
169
+
170
+ .markdown-body .code div pre,
171
+ .markdown-body .code div {
172
+ border: 0;
173
+ -webkit-border-radius: 0;
174
+ -moz-border-radius: 0;
175
+ border-radius: 0;
176
+ -webkit-border-top-right-radius: 3px;
177
+ -webkit-border-bottom-right-radius: 3px;
178
+ -moz-border-radius-topright: 3px;
179
+ -moz-border-radius-bottomright: 3px;
180
+ border-top-right-radius: 3px;
181
+ border-bottom-right-radius: 3px;
182
+ }
183
+
184
+ .markdown-body * {
185
+ -moz-box-sizing: border-box;
186
+ box-sizing: border-box;
187
+ }
188
+
189
+ .markdown-body input {
190
+ font: 13px Helvetica, arial, freesans, clean, sans-serif, "Segoe UI Emoji", "Segoe UI Symbol";
191
+ line-height: 1.4;
192
+ }
193
+
194
+ .markdown-body a {
195
+ color: #4183c4;
196
+ text-decoration: none;
197
+ }
198
+
199
+ .markdown-body a:hover,
200
+ .markdown-body a:focus,
201
+ .markdown-body a:active {
202
+ text-decoration: underline;
203
+ }
204
+
205
+ .markdown-body hr {
206
+ height: 0;
207
+ margin: 15px 0;
208
+ overflow: hidden;
209
+ background: transparent;
210
+ border: 0;
211
+ border-bottom: 1px solid #ddd;
212
+ }
213
+
214
+ .markdown-body hr:before,
215
+ .markdown-body hr:after {
216
+ display: table;
217
+ content: " ";
218
+ }
219
+
220
+ .markdown-body hr:after {
221
+ clear: both;
222
+ }
223
+
224
+ .markdown-body h1,
225
+ .markdown-body h2,
226
+ .markdown-body h3,
227
+ .markdown-body h4,
228
+ .markdown-body h5,
229
+ .markdown-body h6 {
230
+ margin-top: 15px;
231
+ margin-bottom: 15px;
232
+ line-height: 1.1;
233
+ }
234
+
235
+ .markdown-body h1 {
236
+ font-size: 30px;
237
+ }
238
+
239
+ .markdown-body h2 {
240
+ font-size: 21px;
241
+ }
242
+
243
+ .markdown-body h3 {
244
+ font-size: 16px;
245
+ }
246
+
247
+ .markdown-body h4 {
248
+ font-size: 14px;
249
+ }
250
+
251
+ .markdown-body h5 {
252
+ font-size: 12px;
253
+ }
254
+
255
+ .markdown-body h6 {
256
+ font-size: 11px;
257
+ }
258
+
259
+ .markdown-body blockquote {
260
+ margin: 0;
261
+ }
262
+
263
+ .markdown-body ul,
264
+ .markdown-body ol {
265
+ padding: 0;
266
+ margin-top: 0;
267
+ margin-bottom: 0;
268
+ }
269
+
270
+ .markdown-body ol ol,
271
+ .markdown-body ul ol {
272
+ list-style-type: lower-roman;
273
+ }
274
+
275
+ .markdown-body ul ul ol,
276
+ .markdown-body ul ol ol,
277
+ .markdown-body ol ul ol,
278
+ .markdown-body ol ol ol {
279
+ list-style-type: lower-alpha;
280
+ }
281
+
282
+ .markdown-body dd {
283
+ margin-left: 0;
284
+ }
285
+
286
+ .markdown-body code,
287
+ .markdown-body pre,
288
+ .markdown-body samp {
289
+ font-family: Consolas, "Liberation Mono", Menlo, Courier, monospace;
290
+ font-size: 12px;
291
+ }
292
+
293
+ .markdown-body pre {
294
+ margin-top: 0;
295
+ margin-bottom: 0;
296
+ }
297
+
298
+ .markdown-body kbd {
299
+ background-color: #e7e7e7;
300
+ background-image: -moz-linear-gradient(#fefefe, #e7e7e7);
301
+ background-image: -webkit-linear-gradient(#fefefe, #e7e7e7);
302
+ background-image: linear-gradient(#fefefe, #e7e7e7);
303
+ background-repeat: repeat-x;
304
+ border-radius: 2px;
305
+ border: 1px solid #cfcfcf;
306
+ color: #000;
307
+ padding: 3px 5px;
308
+ line-height: 10px;
309
+ font: 11px Consolas, "Liberation Mono", Menlo, Courier, monospace;
310
+ display: inline-block;
311
+ }
312
+
313
+ .markdown-body>*:first-child {
314
+ margin-top: 0 !important;
315
+ }
316
+
317
+ .markdown-body>*:last-child {
318
+ margin-bottom: 0 !important;
319
+ }
320
+
321
+ .markdown-body .headeranchor-link {
322
+ position: absolute;
323
+ top: 0;
324
+ bottom: 0;
325
+ left: 0;
326
+ display: block;
327
+ padding-right: 6px;
328
+ padding-left: 30px;
329
+ margin-left: -30px;
330
+ }
331
+
332
+ .markdown-body .headeranchor-link:focus {
333
+ outline: none;
334
+ }
335
+
336
+ .markdown-body h1,
337
+ .markdown-body h2,
338
+ .markdown-body h3,
339
+ .markdown-body h4,
340
+ .markdown-body h5,
341
+ .markdown-body h6 {
342
+ position: relative;
343
+ margin-top: 1em;
344
+ margin-bottom: 16px;
345
+ font-weight: bold;
346
+ line-height: 1.4;
347
+ }
348
+
349
+ .markdown-body h1 .headeranchor,
350
+ .markdown-body h2 .headeranchor,
351
+ .markdown-body h3 .headeranchor,
352
+ .markdown-body h4 .headeranchor,
353
+ .markdown-body h5 .headeranchor,
354
+ .markdown-body h6 .headeranchor {
355
+ display: none;
356
+ color: #000;
357
+ vertical-align: middle;
358
+ }
359
+
360
+ .markdown-body h1:hover .headeranchor-link,
361
+ .markdown-body h2:hover .headeranchor-link,
362
+ .markdown-body h3:hover .headeranchor-link,
363
+ .markdown-body h4:hover .headeranchor-link,
364
+ .markdown-body h5:hover .headeranchor-link,
365
+ .markdown-body h6:hover .headeranchor-link {
366
+ height: 1em;
367
+ padding-left: 8px;
368
+ margin-left: -30px;
369
+ line-height: 1;
370
+ text-decoration: none;
371
+ }
372
+
373
+ .markdown-body h1:hover .headeranchor-link .headeranchor,
374
+ .markdown-body h2:hover .headeranchor-link .headeranchor,
375
+ .markdown-body h3:hover .headeranchor-link .headeranchor,
376
+ .markdown-body h4:hover .headeranchor-link .headeranchor,
377
+ .markdown-body h5:hover .headeranchor-link .headeranchor,
378
+ .markdown-body h6:hover .headeranchor-link .headeranchor {
379
+ display: inline-block;
380
+ }
381
+
382
+ .markdown-body h1 {
383
+ padding-bottom: 0.3em;
384
+ font-size: 2.25em;
385
+ line-height: 1.2;
386
+ border-bottom: 1px solid #eee;
387
+ }
388
+
389
+ .markdown-body h2 {
390
+ padding-bottom: 0.3em;
391
+ font-size: 1.75em;
392
+ line-height: 1.225;
393
+ border-bottom: 1px solid #eee;
394
+ }
395
+
396
+ .markdown-body h3 {
397
+ font-size: 1.5em;
398
+ line-height: 1.43;
399
+ }
400
+
401
+ .markdown-body h4 {
402
+ font-size: 1.25em;
403
+ }
404
+
405
+ .markdown-body h5 {
406
+ font-size: 1em;
407
+ }
408
+
409
+ .markdown-body h6 {
410
+ font-size: 1em;
411
+ color: #777;
412
+ }
413
+
414
+ .markdown-body p,
415
+ .markdown-body blockquote,
416
+ .markdown-body ul,
417
+ .markdown-body ol,
418
+ .markdown-body dl,
419
+ .markdown-body table,
420
+ .markdown-body pre,
421
+ .markdown-body .admonition {
422
+ margin-top: 0;
423
+ margin-bottom: 16px;
424
+ }
425
+
426
+ .markdown-body hr {
427
+ height: 4px;
428
+ padding: 0;
429
+ margin: 16px 0;
430
+ background-color: #e7e7e7;
431
+ border: 0 none;
432
+ }
433
+
434
+ .markdown-body ul,
435
+ .markdown-body ol {
436
+ padding-left: 2em;
437
+ }
438
+
439
+ .markdown-body ul ul,
440
+ .markdown-body ul ol,
441
+ .markdown-body ol ol,
442
+ .markdown-body ol ul {
443
+ margin-top: 0;
444
+ margin-bottom: 0;
445
+ }
446
+
447
+ .markdown-body li>p {
448
+ margin-top: 16px;
449
+ }
450
+
451
+ .markdown-body dl {
452
+ padding: 0;
453
+ }
454
+
455
+ .markdown-body dl dt {
456
+ padding: 0;
457
+ margin-top: 16px;
458
+ font-size: 1em;
459
+ font-style: italic;
460
+ font-weight: bold;
461
+ }
462
+
463
+ .markdown-body dl dd {
464
+ padding: 0 16px;
465
+ margin-bottom: 16px;
466
+ }
467
+
468
+ .markdown-body blockquote {
469
+ padding: 0 15px;
470
+ color: #777;
471
+ border-left: 4px solid #ddd;
472
+ }
473
+
474
+ .markdown-body blockquote>:first-child {
475
+ margin-top: 0;
476
+ }
477
+
478
+ .markdown-body blockquote>:last-child {
479
+ margin-bottom: 0;
480
+ }
481
+
482
+ .markdown-body table {
483
+ display: block;
484
+ width: 100%;
485
+ overflow: auto;
486
+ word-break: normal;
487
+ word-break: keep-all;
488
+ }
489
+
490
+ .markdown-body table th {
491
+ font-weight: bold;
492
+ }
493
+
494
+ .markdown-body table th,
495
+ .markdown-body table td {
496
+ padding: 6px 13px;
497
+ border: 1px solid #ddd;
498
+ }
499
+
500
+ .markdown-body table tr {
501
+ background-color: #fff;
502
+ border-top: 1px solid #ccc;
503
+ }
504
+
505
+ .markdown-body table tr:nth-child(2n) {
506
+ background-color: #f8f8f8;
507
+ }
508
+
509
+ .markdown-body img {
510
+ max-width: 100%;
511
+ -moz-box-sizing: border-box;
512
+ box-sizing: border-box;
513
+ }
514
+
515
+ .markdown-body code,
516
+ .markdown-body samp {
517
+ padding: 0;
518
+ padding-top: 0.2em;
519
+ padding-bottom: 0.2em;
520
+ margin: 0;
521
+ font-size: 85%;
522
+ background-color: rgba(0,0,0,0.04);
523
+ border-radius: 3px;
524
+ }
525
+
526
+ .markdown-body code:before,
527
+ .markdown-body code:after {
528
+ letter-spacing: -0.2em;
529
+ content: "\00a0";
530
+ }
531
+
532
+ .markdown-body pre>code {
533
+ padding: 0;
534
+ margin: 0;
535
+ font-size: 100%;
536
+ word-break: normal;
537
+ white-space: pre;
538
+ background: transparent;
539
+ border: 0;
540
+ }
541
+
542
+ .markdown-body .codehilite {
543
+ margin-bottom: 16px;
544
+ }
545
+
546
+ .markdown-body .codehilite pre,
547
+ .markdown-body pre {
548
+ padding: 16px;
549
+ overflow: auto;
550
+ font-size: 85%;
551
+ line-height: 1.45;
552
+ background-color: #f7f7f7;
553
+ border-radius: 3px;
554
+ }
555
+
556
+ .markdown-body .codehilite pre {
557
+ margin-bottom: 0;
558
+ word-break: normal;
559
+ }
560
+
561
+ .markdown-body pre {
562
+ word-wrap: normal;
563
+ }
564
+
565
+ .markdown-body pre code {
566
+ display: inline;
567
+ max-width: initial;
568
+ padding: 0;
569
+ margin: 0;
570
+ overflow: initial;
571
+ line-height: inherit;
572
+ word-wrap: normal;
573
+ background-color: transparent;
574
+ border: 0;
575
+ }
576
+
577
+ .markdown-body pre code:before,
578
+ .markdown-body pre code:after {
579
+ content: normal;
580
+ }
581
+
582
+ /* Admonition */
583
+ .markdown-body .admonition {
584
+ -webkit-border-radius: 3px;
585
+ -moz-border-radius: 3px;
586
+ position: relative;
587
+ border-radius: 3px;
588
+ border: 1px solid #e0e0e0;
589
+ border-left: 6px solid #333;
590
+ padding: 10px 10px 10px 30px;
591
+ }
592
+
593
+ .markdown-body .admonition table {
594
+ color: #333;
595
+ }
596
+
597
+ .markdown-body .admonition p {
598
+ padding: 0;
599
+ }
600
+
601
+ .markdown-body .admonition-title {
602
+ font-weight: bold;
603
+ margin: 0;
604
+ }
605
+
606
+ .markdown-body .admonition>.admonition-title {
607
+ color: #333;
608
+ }
609
+
610
+ .markdown-body .attention>.admonition-title {
611
+ color: #a6d796;
612
+ }
613
+
614
+ .markdown-body .caution>.admonition-title {
615
+ color: #d7a796;
616
+ }
617
+
618
+ .markdown-body .hint>.admonition-title {
619
+ color: #96c6d7;
620
+ }
621
+
622
+ .markdown-body .danger>.admonition-title {
623
+ color: #c25f77;
624
+ }
625
+
626
+ .markdown-body .question>.admonition-title {
627
+ color: #96a6d7;
628
+ }
629
+
630
+ .markdown-body .note>.admonition-title {
631
+ color: #d7c896;
632
+ }
633
+
634
+ .markdown-body .admonition:before,
635
+ .markdown-body .attention:before,
636
+ .markdown-body .caution:before,
637
+ .markdown-body .hint:before,
638
+ .markdown-body .danger:before,
639
+ .markdown-body .question:before,
640
+ .markdown-body .note:before {
641
+ font: normal normal 16px fontawesome-mini;
642
+ -moz-osx-font-smoothing: grayscale;
643
+ -webkit-user-select: none;
644
+ -moz-user-select: none;
645
+ -ms-user-select: none;
646
+ user-select: none;
647
+ line-height: 1.5;
648
+ color: #333;
649
+ position: absolute;
650
+ left: 0;
651
+ top: 0;
652
+ padding-top: 10px;
653
+ padding-left: 10px;
654
+ }
655
+
656
+ .markdown-body .admonition:before {
657
+ content: "\f056\00a0";
658
+ color: 333;
659
+ }
660
+
661
+ .markdown-body .attention:before {
662
+ content: "\f058\00a0";
663
+ color: #a6d796;
664
+ }
665
+
666
+ .markdown-body .caution:before {
667
+ content: "\f06a\00a0";
668
+ color: #d7a796;
669
+ }
670
+
671
+ .markdown-body .hint:before {
672
+ content: "\f05a\00a0";
673
+ color: #96c6d7;
674
+ }
675
+
676
+ .markdown-body .danger:before {
677
+ content: "\f057\00a0";
678
+ color: #c25f77;
679
+ }
680
+
681
+ .markdown-body .question:before {
682
+ content: "\f059\00a0";
683
+ color: #96a6d7;
684
+ }
685
+
686
+ .markdown-body .note:before {
687
+ content: "\f040\00a0";
688
+ color: #d7c896;
689
+ }
690
+
691
+ .markdown-body .admonition::after {
692
+ content: normal;
693
+ }
694
+
695
+ .markdown-body .attention {
696
+ border-left: 6px solid #a6d796;
697
+ }
698
+
699
+ .markdown-body .caution {
700
+ border-left: 6px solid #d7a796;
701
+ }
702
+
703
+ .markdown-body .hint {
704
+ border-left: 6px solid #96c6d7;
705
+ }
706
+
707
+ .markdown-body .danger {
708
+ border-left: 6px solid #c25f77;
709
+ }
710
+
711
+ .markdown-body .question {
712
+ border-left: 6px solid #96a6d7;
713
+ }
714
+
715
+ .markdown-body .note {
716
+ border-left: 6px solid #d7c896;
717
+ }
718
+
719
+ .markdown-body .admonition>*:first-child {
720
+ margin-top: 0 !important;
721
+ }
722
+
723
+ .markdown-body .admonition>*:last-child {
724
+ margin-bottom: 0 !important;
725
+ }
726
+
727
+ /* progress bar*/
728
+ .markdown-body .progress {
729
+ display: block;
730
+ width: 300px;
731
+ margin: 10px 0;
732
+ height: 24px;
733
+ -webkit-border-radius: 3px;
734
+ -moz-border-radius: 3px;
735
+ border-radius: 3px;
736
+ background-color: #ededed;
737
+ position: relative;
738
+ box-shadow: inset -1px 1px 3px rgba(0, 0, 0, .1);
739
+ }
740
+
741
+ .markdown-body .progress-label {
742
+ position: absolute;
743
+ text-align: center;
744
+ font-weight: bold;
745
+ width: 100%; margin: 0;
746
+ line-height: 24px;
747
+ color: #333;
748
+ text-shadow: 1px 1px 0 #fefefe, -1px -1px 0 #fefefe, -1px 1px 0 #fefefe, 1px -1px 0 #fefefe, 0 1px 0 #fefefe, 0 -1px 0 #fefefe, 1px 0 0 #fefefe, -1px 0 0 #fefefe, 1px 1px 2px #000;
749
+ -webkit-font-smoothing: antialiased !important;
750
+ white-space: nowrap;
751
+ overflow: hidden;
752
+ }
753
+
754
+ .markdown-body .progress-bar {
755
+ height: 24px;
756
+ float: left;
757
+ -webkit-border-radius: 3px;
758
+ -moz-border-radius: 3px;
759
+ border-radius: 3px;
760
+ background-color: #96c6d7;
761
+ box-shadow: inset 0 1px 0 rgba(255, 255, 255, .5), inset 0 -1px 0 rgba(0, 0, 0, .1);
762
+ background-size: 30px 30px;
763
+ background-image: -webkit-linear-gradient(
764
+ 135deg, rgba(255, 255, 255, .4) 27%,
765
+ transparent 27%,
766
+ transparent 52%, rgba(255, 255, 255, .4) 52%,
767
+ rgba(255, 255, 255, .4) 77%,
768
+ transparent 77%, transparent
769
+ );
770
+ background-image: -moz-linear-gradient(
771
+ 135deg,
772
+ rgba(255, 255, 255, .4) 27%, transparent 27%,
773
+ transparent 52%, rgba(255, 255, 255, .4) 52%,
774
+ rgba(255, 255, 255, .4) 77%, transparent 77%,
775
+ transparent
776
+ );
777
+ background-image: -ms-linear-gradient(
778
+ 135deg,
779
+ rgba(255, 255, 255, .4) 27%, transparent 27%,
780
+ transparent 52%, rgba(255, 255, 255, .4) 52%,
781
+ rgba(255, 255, 255, .4) 77%, transparent 77%,
782
+ transparent
783
+ );
784
+ background-image: -o-linear-gradient(
785
+ 135deg,
786
+ rgba(255, 255, 255, .4) 27%, transparent 27%,
787
+ transparent 52%, rgba(255, 255, 255, .4) 52%,
788
+ rgba(255, 255, 255, .4) 77%, transparent 77%,
789
+ transparent
790
+ );
791
+ background-image: linear-gradient(
792
+ 135deg,
793
+ rgba(255, 255, 255, .4) 27%, transparent 27%,
794
+ transparent 52%, rgba(255, 255, 255, .4) 52%,
795
+ rgba(255, 255, 255, .4) 77%, transparent 77%,
796
+ transparent
797
+ );
798
+ }
799
+
800
+ .markdown-body .progress-100plus .progress-bar {
801
+ background-color: #a6d796;
802
+ }
803
+
804
+ .markdown-body .progress-80plus .progress-bar {
805
+ background-color: #c6d796;
806
+ }
807
+
808
+ .markdown-body .progress-60plus .progress-bar {
809
+ background-color: #d7c896;
810
+ }
811
+
812
+ .markdown-body .progress-40plus .progress-bar {
813
+ background-color: #d7a796;
814
+ }
815
+
816
+ .markdown-body .progress-20plus .progress-bar {
817
+ background-color: #d796a6;
818
+ }
819
+
820
+ .markdown-body .progress-0plus .progress-bar {
821
+ background-color: #c25f77;
822
+ }
823
+
824
+ .markdown-body .candystripe-animate .progress-bar{
825
+ -webkit-animation: animate-stripes 3s linear infinite;
826
+ -moz-animation: animate-stripes 3s linear infinite;
827
+ animation: animate-stripes 3s linear infinite;
828
+ }
829
+
830
+ @-webkit-keyframes animate-stripes {
831
+ 0% {
832
+ background-position: 0 0;
833
+ }
834
+
835
+ 100% {
836
+ background-position: 60px 0;
837
+ }
838
+ }
839
+
840
+ @-moz-keyframes animate-stripes {
841
+ 0% {
842
+ background-position: 0 0;
843
+ }
844
+
845
+ 100% {
846
+ background-position: 60px 0;
847
+ }
848
+ }
849
+
850
+ @keyframes animate-stripes {
851
+ 0% {
852
+ background-position: 0 0;
853
+ }
854
+
855
+ 100% {
856
+ background-position: 60px 0;
857
+ }
858
+ }
859
+
860
+ .markdown-body .gloss .progress-bar {
861
+ box-shadow:
862
+ inset 0 4px 12px rgba(255, 255, 255, .7),
863
+ inset 0 -12px 0 rgba(0, 0, 0, .05);
864
+ }
865
+
866
+ /* Multimarkdown Critic Blocks */
867
+ .markdown-body .critic_mark {
868
+ background: #ff0;
869
+ }
870
+
871
+ .markdown-body .critic_delete {
872
+ color: #c82829;
873
+ text-decoration: line-through;
874
+ }
875
+
876
+ .markdown-body .critic_insert {
877
+ color: #718c00 ;
878
+ text-decoration: underline;
879
+ }
880
+
881
+ .markdown-body .critic_comment {
882
+ color: #8e908c;
883
+ font-style: italic;
884
+ }
885
+
886
+ .markdown-body .headeranchor {
887
+ font: normal normal 16px octicons-anchor;
888
+ line-height: 1;
889
+ display: inline-block;
890
+ text-decoration: none;
891
+ -webkit-font-smoothing: antialiased;
892
+ -moz-osx-font-smoothing: grayscale;
893
+ -webkit-user-select: none;
894
+ -moz-user-select: none;
895
+ -ms-user-select: none;
896
+ user-select: none;
897
+ }
898
+
899
+ .headeranchor:before {
900
+ content: '\f05c';
901
+ }
902
+
903
+ .markdown-body .task-list-item {
904
+ list-style-type: none;
905
+ }
906
+
907
+ .markdown-body .task-list-item+.task-list-item {
908
+ margin-top: 3px;
909
+ }
910
+
911
+ .markdown-body .task-list-item input {
912
+ margin: 0 4px 0.25em -20px;
913
+ vertical-align: middle;
914
+ }
915
+
916
+ /* Media */
917
+ @media only screen and (min-width: 480px) {
918
+ .markdown-body {
919
+ font-size:14px;
920
+ }
921
+ }
922
+
923
+ @media only screen and (min-width: 768px) {
924
+ .markdown-body {
925
+ font-size:16px;
926
+ }
927
+ }
928
+
929
+ @media print {
930
+ .markdown-body * {
931
+ background: transparent !important;
932
+ color: black !important;
933
+ filter:none !important;
934
+ -ms-filter: none !important;
935
+ }
936
+
937
+ .markdown-body {
938
+ font-size:12pt;
939
+ max-width:100%;
940
+ outline:none;
941
+ border: 0;
942
+ }
943
+
944
+ .markdown-body a,
945
+ .markdown-body a:visited {
946
+ text-decoration: underline;
947
+ }
948
+
949
+ .markdown-body .headeranchor-link {
950
+ display: none;
951
+ }
952
+
953
+ .markdown-body a[href]:after {
954
+ content: " (" attr(href) ")";
955
+ }
956
+
957
+ .markdown-body abbr[title]:after {
958
+ content: " (" attr(title) ")";
959
+ }
960
+
961
+ .markdown-body .ir a:after,
962
+ .markdown-body a[href^="javascript:"]:after,
963
+ .markdown-body a[href^="#"]:after {
964
+ content: "";
965
+ }
966
+
967
+ .markdown-body pre {
968
+ white-space: pre;
969
+ white-space: pre-wrap;
970
+ word-wrap: break-word;
971
+ }
972
+
973
+ .markdown-body pre,
974
+ .markdown-body blockquote {
975
+ border: 1px solid #999;
976
+ padding-right: 1em;
977
+ page-break-inside: avoid;
978
+ }
979
+
980
+ .markdown-body .progress,
981
+ .markdown-body .progress-bar {
982
+ -moz-box-shadow: none;
983
+ -webkit-box-shadow: none;
984
+ box-shadow: none;
985
+ }
986
+
987
+ .markdown-body .progress {
988
+ border: 1px solid #ddd;
989
+ }
990
+
991
+ .markdown-body .progress-bar {
992
+ height: 22px;
993
+ border-right: 1px solid #ddd;
994
+ }
995
+
996
+ .markdown-body tr,
997
+ .markdown-body img {
998
+ page-break-inside: avoid;
999
+ }
1000
+
1001
+ .markdown-body img {
1002
+ max-width: 100% !important;
1003
+ }
1004
+
1005
+ .markdown-body p,
1006
+ .markdown-body h2,
1007
+ .markdown-body h3 {
1008
+ orphans: 3;
1009
+ widows: 3;
1010
+ }
1011
+
1012
+ .markdown-body h2,
1013
+ .markdown-body h3 {
1014
+ page-break-after: avoid;
1015
+ }
1016
+ }
1017
+ </style><title>README</title></head><body><article class="markdown-body"><h1 id="tokenator"><a name="user-content-tokenator" href="#tokenator" class="headeranchor-link" aria-hidden="true"><span class="headeranchor"></span></a>Tokenator</h1>
1018
+ <p>This is a simple rule-based tokenizer, based on the one developed by Robert Macyntyre in 1995 for the Penn Treebank, and adapted from a script lifted from <a href="https://github.com/louismullie/treat">the Treat gem</a>.</p>
1019
+ <p>It&rsquo;s taylored to my specific use case: I don&rsquo;t need part-of-speech tagging or punctuation preservation.</p>
1020
+ <h2 id="installation"><a name="user-content-installation" href="#installation" class="headeranchor-link" aria-hidden="true"><span class="headeranchor"></span></a>Installation</h2>
1021
+ <p>Add this line to your application&rsquo;s Gemfile:</p>
1022
+ <pre><code class="ruby">gem 'tokenator'
1023
+ </code></pre>
1024
+
1025
+ <p>And then execute:</p>
1026
+ <pre><code>$ bundle
1027
+ </code></pre>
1028
+ <p>Or install it yourself as:</p>
1029
+ <pre><code>$ gem install tokenator
1030
+ </code></pre>
1031
+ <h2 id="usage"><a name="user-content-usage" href="#usage" class="headeranchor-link" aria-hidden="true"><span class="headeranchor"></span></a>Usage</h2>
1032
+ <pre><code class="ruby">tokenizer = Tokenator.new
1033
+
1034
+ tokenizer.tokenize &quot;This? It's just a basic example!&quot;
1035
+ # =&gt; [&quot;This&quot;, &quot;It&quot;, &quot;is&quot;, &quot;just&quot;, &quot;a&quot;, &quot;basic&quot;, &quot;example&quot;]
1036
+
1037
+ tokenizer.tokenize &quot;This? It's just a basic example!&quot;, {:remove_punctuation =&gt; false}
1038
+ # =&gt; [&quot;This&quot;, &quot;?&quot;, &quot;It&quot;, &quot;is&quot;, &quot;just&quot;, &quot;a&quot;, &quot;basic&quot;, &quot;example&quot;, &quot;!&quot;]
1039
+
1040
+ </code></pre>
1041
+
1042
+ <h2 id="development"><a name="user-content-development" href="#development" class="headeranchor-link" aria-hidden="true"><span class="headeranchor"></span></a>Development</h2>
1043
+ <p>After checking out the repo, run <code>bin/setup</code> to install dependencies. Then, run <code>rake test</code> to run the tests. You can also run <code>bin/console</code> for an interactive prompt that will allow you to experiment.</p>
1044
+ <p>To install this gem onto your local machine, run <code>bundle exec rake install</code>. To release a new version, update the version number in <code>version.rb</code>, and then run <code>bundle exec rake release</code>, which will create a git tag for the version, push git commits and tags, and push the <code>.gem</code> file to <a href="https://rubygems.org">rubygems.org</a>.</p>
1045
+ <h2 id="contributing"><a name="user-content-contributing" href="#contributing" class="headeranchor-link" aria-hidden="true"><span class="headeranchor"></span></a>Contributing</h2>
1046
+ <p>Bug reports and pull requests are welcome on GitHub at <a href="https://github.com/Amval/tokenator">https://github.com/Amval/tokenator</a>.</p>
1047
+ <h2 id="license"><a name="user-content-license" href="#license" class="headeranchor-link" aria-hidden="true"><span class="headeranchor"></span></a>License</h2>
1048
+ <p>The gem is available as open source under the terms of the <a href="http://opensource.org/licenses/MIT">MIT License</a>.</p></article></body></html>
data/README.md CHANGED
@@ -22,7 +22,16 @@ Or install it yourself as:
22
22
 
23
23
  ## Usage
24
24
 
25
- TODO: Write usage instructions here
25
+ ```ruby
26
+ tokenizer = Tokenator.new
27
+
28
+ tokenizer.tokenize "This? It's just a basic example!"
29
+ # => ["This", "It", "is", "just", "a", "basic", "example"]
30
+
31
+ tokenizer.tokenize "This? It's just a basic example!", {:remove_punctuation => false}
32
+ # => ["This", "?", "It", "is", "just", "a", "basic", "example", "!"]
33
+
34
+ ```
26
35
 
27
36
  ## Development
28
37
 
@@ -75,8 +75,11 @@ class PTB
75
75
  s.gsub!(/\.\.\./, '')
76
76
  s.gsub!(/ \./, ' ')
77
77
  s.gsub!(/- /, ' ')
78
+ s.gsub!(/["`'][^\w]/, ' ')
79
+ s.gsub!(/"$/, ' ')
78
80
  end
79
81
  s.split(/\s+/)
80
82
  end
81
83
 
82
84
  end
85
+
@@ -1,3 +1,3 @@
1
1
  module Tokenator
2
- VERSION = "0.2.0"
2
+ VERSION = "0.2.1"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: tokenator
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.0
4
+ version: 0.2.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Amval
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2016-01-26 00:00:00.000000000 Z
11
+ date: 2016-02-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -63,6 +63,7 @@ files:
63
63
  - ".travis.yml"
64
64
  - Gemfile
65
65
  - LICENSE.txt
66
+ - README.html
66
67
  - README.md
67
68
  - Rakefile
68
69
  - bin/console