tokenator 0.2.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 35c1d63916cde346788a0c80a4782eff19afedc3
4
- data.tar.gz: 4fb32d3b8ec35383b8ac3cf076b9ec213e3ccfc3
3
+ metadata.gz: ba4933f843a64a58ceafa12879d2527fad555a84
4
+ data.tar.gz: 2ddf0bc5ab90202d7c946b2a7c0ff9dcbcc85587
5
5
  SHA512:
6
- metadata.gz: c158a5069c3d3edd59cc3ea6b474b83a5f62b05bf3c3d9ed87de1eab2d0e51edb369eb12db854b7873110c2d771d67fbb29f886f4ca1cf094261ec932a3c1854
7
- data.tar.gz: cc005580610d0ef1a3668876c29a0afdda2164510951cd1ab142ad1afbdbf7e53fb3e222efff271cae51fe00849394cc301988377aaa18ef5a7e5f4237eb3f30
6
+ metadata.gz: ddded9a096194ad8c0f7faf7db11bccfd56da2b3e91c12b2b3c92560df4a242cd69df5a5896e5c446bb33d545b5621493c0ab720475e004532e2c8a6ebb8a5b2
7
+ data.tar.gz: 2648ce29678aa3454d3b2849776f6c6fdfdbf43e3ccf03d32e17ac1c4e35eeffcdb2aff9131729456b8c4daf9208a818e9cf5865560ebacf88a54fc2a85e5200
@@ -0,0 +1,1048 @@
1
+ <!DOCTYPE html><html><head><meta charset="utf-8"><style>body {
2
+ width: 45em;
3
+ border: 1px solid #ddd;
4
+ outline: 1300px solid #fff;
5
+ margin: 16px auto;
6
+ }
7
+
8
+ body .markdown-body
9
+ {
10
+ padding: 30px;
11
+ }
12
+
13
+ @font-face {
14
+ font-family: fontawesome-mini;
15
+ src: url(data:font/woff;charset=utf-8;base64,d09GRgABAAAAAAzUABAAAAAAFNgAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAABGRlRNAAABbAAAABwAAAAcZMzaOEdERUYAAAGIAAAAHQAAACAAOQAET1MvMgAAAagAAAA+AAAAYHqhde9jbWFwAAAB6AAAAFIAAAFa4azkLWN2dCAAAAI8AAAAKAAAACgFgwioZnBnbQAAAmQAAAGxAAACZVO0L6dnYXNwAAAEGAAAAAgAAAAIAAAAEGdseWYAAAQgAAAFDgAACMz7eroHaGVhZAAACTAAAAAwAAAANgWEOEloaGVhAAAJYAAAAB0AAAAkDGEGa2htdHgAAAmAAAAAEwAAADBEgAAQbG9jYQAACZQAAAAaAAAAGgsICJBtYXhwAAAJsAAAACAAAAAgASgBD25hbWUAAAnQAAACZwAABOD4no+3cG9zdAAADDgAAABsAAAAmF+yXM9wcmVwAAAMpAAAAC4AAAAusPIrFAAAAAEAAAAAyYlvMQAAAADLVHQgAAAAAM/u9uZ4nGNgZGBg4ANiCQYQYGJgBEJuIGYB8xgABMMAPgAAAHicY2Bm42OcwMDKwMLSw2LMwMDQBqGZihmiwHycoKCyqJjB4YPDh4NsDP+BfNb3DIuAFCOSEgUGRgAKDgt4AAB4nGNgYGBmgGAZBkYGEAgB8hjBfBYGCyDNxcDBwMTA9MHhQ9SHrA8H//9nYACyQyFs/sP86/kX8HtB9UIBIxsDXICRCUgwMaACRoZhDwA3fxKSAAAAAAHyAHABJQB/AIEAdAFGAOsBIwC/ALgAxACGAGYAugBNACcA/wCIeJxdUbtOW0EQ3Q0PA4HE2CA52hSzmZDGe6EFCcTVjWJkO4XlCGk3cpGLcQEfQIFEDdqvGaChpEibBiEXSHxCPiESM2uIojQ7O7NzzpkzS8qRqnfpa89T5ySQwt0GzTb9Tki1swD3pOvrjYy0gwdabGb0ynX7/gsGm9GUO2oA5T1vKQ8ZTTuBWrSn/tH8Cob7/B/zOxi0NNP01DoJ6SEE5ptxS4PvGc26yw/6gtXhYjAwpJim4i4/plL+tzTnasuwtZHRvIMzEfnJNEBTa20Emv7UIdXzcRRLkMumsTaYmLL+JBPBhcl0VVO1zPjawV2ys+hggyrNgQfYw1Z5DB4ODyYU0rckyiwNEfZiq8QIEZMcCjnl3Mn+pED5SBLGvElKO+OGtQbGkdfAoDZPs/88m01tbx3C+FkcwXe/GUs6+MiG2hgRYjtiKYAJREJGVfmGGs+9LAbkUvvPQJSA5fGPf50ItO7YRDyXtXUOMVYIen7b3PLLirtWuc6LQndvqmqo0inN+17OvscDnh4Lw0FjwZvP+/5Kgfo8LK40aA4EQ3o3ev+iteqIq7wXPrIn07+xWgAAAAABAAH//wAPeJyFlctvG1UUh+/12DPN1B7P3JnYjj2Ox4/MuDHxJH5N3UdaEUQLqBIkfQQioJWQ6AMEQkIqsPGCPwA1otuWSmTBhjtps2ADWbJg3EpIXbGouqSbCraJw7kzNo2dRN1cnXN1ZvT7zuuiMEI7ncizyA0URofRBJpCdbQuIFShYY+GZRrxMDVtih5TwQPHtXDFFSIKoWIbuREBjLH27Ny4MsbVx+uOJThavebgVrNRLAiYx06rXsvhxLgWx9xpfHdrs/ekc2Pl2cpPCVEITQpwbj8VQhfXSq2m+Wxqaq2D73Kne5e3NjHqQNj3CRYlJlgUl/jRNP+2Gs2pNYRQiOnmUaQDqm30KqKiTTWPWjboxnTWpvgxjXo0KrtZXAHt7hwIz0YVcj88JnKlJKi3NPAwLyDwZudSmJSMMJFDYaOkaol6XtESx3Gt1VTytdZJ3DCLeaVhVnCBH1fycHTxFXwPX+l2e3d6H/TufGGmMTLTnbSJUdo00zuBswMO/nl3YLeL/wnu9/limCuD3vC54h5NBVz6Li414AI8Vx3iiosKcQXUbrvhFFiYb++HN4DaF4XzFW0fIN4XDWJ3a3XQoq9V8WiyRmdsatV9xUcHims1JloH0YUa090G3Tro3mC6c01f+YwCPquINr1PTaCP6rVTOOmf0GE2dBc7zWIhji3/5MchSuBHgDbU99RMWt3YUNMZMJmx92YP6NsHx/5/M1yvInpnkIOM3Z8fA3JQ2lW1RFC1KaBPDFXNAHYYvGy73aYZZZ3HifbeuiVZCpwA3oQBs0wGPYJbJfg60xrKEbKiNtTe1adwrpBRwlAuQ3q3VRaX0QmQ9a49BTSCuF1MLfQ6+tinOubRBZuWPNoMevGMT+V41KitO1is3D/tpMcq1JHZqDHGs8DoYGDkxJgKjHROeTCmhZvzPm9pod+ltKm4PN7Dyvvldlpsg8D+4AUJZ3F/JBstZz7cbFRxsaAGV6yX/dkcycWf8eS3QlQea+YLjdm3yrOnrhFpUyKVvFE4lpv4bO3Svx/6F/4xmiDu/RT5iI++lko18mY1oX+5UGKR6kmVjM/Zb76yfHtxy+h/SyQ0lLdpdKy/lWB6szatetQJ8nZ80A2Qt6ift6gJeavU3BO4gtxs/KCtNPVibCtYCWY3SIlSBPKXZALXiIR9oZeJ1AuMyxLpHIy/yO7vSiSE+kZvk0ihJ30HgHfzZtEMmvV58x6dtqns0XTAW7Vdm4HJ04OCp/crOO7rd9SGxQAE/mVA9xRN+kVSMRFF6S9JFGUtthkjBA5tFCWc2l4V43Ex9GmUP3SI37Jjmir9KqlaDJ4S4JB3vuM/jzyH1+8MuoZ+QGzfnvPoJb96cZlWjMcKLfgDwB7E634JTY+asjsPzS5CiVnEWY+KsrsIN5rn3mAPjqmQBxGjcGKB9f9ZxY3mYC2L85CJ2FXIxKKyHk+dg0FHbuEc7D5NzWUX32WxFcWNGRAbvwSx0RmIXVDuYySafluQBmzA/ssqJAMLnli+WIC90Gw4lm85wcp0qjArEDPJJV/sSx4P9ungTpgMw5gVC1XO4uULq0s3v1rqLi0vX/z65vlH50f8T/RHmSPTk5xxWBWOluMT6WiOy+tdvWxlV/XQb3o3c6Ssr+r6I708GsX9/nzp1tKFh0s3v7m4vAy/Hnb/KMOvc1wump6Il48K6mGDy02X9Yd65pa+nQIjk76lWxCkG8NBCP0HQS9IpAAAeJxjYGRgYGBhcCrq214Qz2/zlUGenQEEzr/77oug/zewFbB+AHI5GJhAogBwKQ0qeJxjYGRgYH3/P46BgZ0BBNgKGBgZUAEPAE/7At0AAAB4nGNngAB2IGYjhBsYBAAIYADVAAAAAAAAAAAAAFwAyAEeAaACCgKmAx4DggRmAAAAAQAAAAwAagAEAAAAAAACAAEAAgAWAAABAAChAAAAAHiclZI7bxQxFIWPd/JkUYQChEhIyAVKgdBMskm1QkKrRETpQiLRUczueB/K7HhlOxttg8LvoKPgP9DxFxANDR0tHRWi4NjrPIBEgh1p/dm+vufcawNYFWsQmP6e4jSyQB2fI9cwj++RE9wTjyPP4LYoI89iWbyLPIe6+Bh5Hs9rryMv4GbtW+RF3EhuRa7jbrIbeQkPkjdUETOLnL0Kip4FVvAhco1RXyMnSPEz8gzWxE7kWTwUp5HnsCLeR57HW/El8gJWa58iL+JO7UfkOh4l9yMv4UnyEtvQGGECgwF66MNBooF1bGCL1ELB/TYU+ZBRlvsKQ44Se6jQ4a7hef+fh72Crv25kp+8lNWGmeKoOI5jJLb1aGIGvb6TjfWNLdkqdFvJw4l1amjlXtXRZqRN7lSRylZZyhBqpVFWmTEXgWfUrpi/hZOQXdOd4rKuXOtEWT3k5IArPRzTUU5tHKjecZkTpnVbNOnt6jzN8240GD4xtikvZW56043rPMg/dS+dlOceXoR+WPbJ55Dsekq1lJpnypsMUsYOdCW30o103Ytu/lvh+5RWFLfBjm9/N8hJntPhvx92rnoE/kyHdGasGy754kw36vsVf/lFeBi+0COu+cfgQr42G3CRpeLoZ53gmfe3X6rcKt5oVxnptHR9JS8ehVUd5wvvahN2uqxOOpMXapibI5k7Zwbt4xBSaTfoKBufhAnO/uqNcfK8OTs0OQ6l7JIqFjDhYj5WcjevCnI/1DDiI8j4ndWb/5YzDZWh79yomWXeXj7Nnw70/2TIeFPTrlSh89k1ObOSRVZWZfgF0r/zJQB4nG2JUQuCQBCEd07TTg36fb2IyBaLd3vWaUh/vmSJnvpgmG8YcmS8X3Shf3R7QA4OBUocUKHGER5NNbOOEvwc1txnuWkTRb/aPjimJ5vXabI+3VfOiyS15UWvyezM2xiGOPyuMohOH8O8JiO4Af+FsAGNAEuwCFBYsQEBjlmxRgYrWCGwEFlLsBRSWCGwgFkdsAYrXFhZsBQrAAA=) format('woff');
16
+ }
17
+
18
+ @font-face {
19
+ font-family: octicons-anchor;
20
+ src: url(data:font/woff;charset=utf-8;base64,d09GRgABAAAAAAYcAA0AAAAACjQAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAABGRlRNAAABMAAAABwAAAAca8vGTk9TLzIAAAFMAAAARAAAAFZG1VHVY21hcAAAAZAAAAA+AAABQgAP9AdjdnQgAAAB0AAAAAQAAAAEACICiGdhc3AAAAHUAAAACAAAAAj//wADZ2x5ZgAAAdwAAADRAAABEKyikaNoZWFkAAACsAAAAC0AAAA2AtXoA2hoZWEAAALgAAAAHAAAACQHngNFaG10eAAAAvwAAAAQAAAAEAwAACJsb2NhAAADDAAAAAoAAAAKALIAVG1heHAAAAMYAAAAHwAAACABEAB2bmFtZQAAAzgAAALBAAAFu3I9x/Nwb3N0AAAF/AAAAB0AAAAvaoFvbwAAAAEAAAAAzBdyYwAAAADP2IQvAAAAAM/bz7t4nGNgZGFgnMDAysDB1Ml0hoGBoR9CM75mMGLkYGBgYmBlZsAKAtJcUxgcPsR8iGF2+O/AEMPsznAYKMwIkgMA5REMOXicY2BgYGaAYBkGRgYQsAHyGMF8FgYFIM0ChED+h5j//yEk/3KoSgZGNgYYk4GRCUgwMaACRoZhDwCs7QgGAAAAIgKIAAAAAf//AAJ4nHWMMQrCQBBF/0zWrCCIKUQsTDCL2EXMohYGSSmorScInsRGL2DOYJe0Ntp7BK+gJ1BxF1stZvjz/v8DRghQzEc4kIgKwiAppcA9LtzKLSkdNhKFY3HF4lK69ExKslx7Xa+vPRVS43G98vG1DnkDMIBUgFN0MDXflU8tbaZOUkXUH0+U27RoRpOIyCKjbMCVejwypzJJG4jIwb43rfl6wbwanocrJm9XFYfskuVC5K/TPyczNU7b84CXcbxks1Un6H6tLH9vf2LRnn8Ax7A5WQAAAHicY2BkYGAA4teL1+yI57f5ysDNwgAC529f0kOmWRiYVgEpDgYmEA8AUzEKsQAAAHicY2BkYGB2+O/AEMPCAAJAkpEBFbAAADgKAe0EAAAiAAAAAAQAAAAEAAAAAAAAKgAqACoAiAAAeJxjYGRgYGBhsGFgYgABEMkFhAwM/xn0QAIAD6YBhwB4nI1Ty07cMBS9QwKlQapQW3VXySvEqDCZGbGaHULiIQ1FKgjWMxknMfLEke2A+IJu+wntrt/QbVf9gG75jK577Lg8K1qQPCfnnnt8fX1NRC/pmjrk/zprC+8D7tBy9DHgBXoWfQ44Av8t4Bj4Z8CLtBL9CniJluPXASf0Lm4CXqFX8Q84dOLnMB17N4c7tBo1AS/Qi+hTwBH4rwHHwN8DXqQ30XXAS7QaLwSc0Gn8NuAVWou/gFmnjLrEaEh9GmDdDGgL3B4JsrRPDU2hTOiMSuJUIdKQQayiAth69r6akSSFqIJuA19TrzCIaY8sIoxyrNIrL//pw7A2iMygkX5vDj+G+kuoLdX4GlGK/8Lnlz6/h9MpmoO9rafrz7ILXEHHaAx95s9lsI7AHNMBWEZHULnfAXwG9/ZqdzLI08iuwRloXE8kfhXYAvE23+23DU3t626rbs8/8adv+9DWknsHp3E17oCf+Z48rvEQNZ78paYM38qfk3v/u3l3u3GXN2Dmvmvpf1Srwk3pB/VSsp512bA/GG5i2WJ7wu430yQ5K3nFGiOqgtmSB5pJVSizwaacmUZzZhXLlZTq8qGGFY2YcSkqbth6aW1tRmlaCFs2016m5qn36SbJrqosG4uMV4aP2PHBmB3tjtmgN2izkGQyLWprekbIntJFing32a5rKWCN/SdSoga45EJykyQ7asZvHQ8PTm6cslIpwyeyjbVltNikc2HTR7YKh9LBl9DADC0U/jLcBZDKrMhUBfQBvXRzLtFtjU9eNHKin0x5InTqb8lNpfKv1s1xHzTXRqgKzek/mb7nB8RZTCDhGEX3kK/8Q75AmUM/eLkfA+0Hi908Kx4eNsMgudg5GLdRD7a84npi+YxNr5i5KIbW5izXas7cHXIMAau1OueZhfj+cOcP3P8MNIWLyYOBuxL6DRylJ4cAAAB4nGNgYoAALjDJyIAOWMCiTIxMLDmZedkABtIBygAAAA==) format('woff');
21
+ }
22
+
23
+ .markdown-body {
24
+ font-family: sans-serif;
25
+ -ms-text-size-adjust: 100%;
26
+ -webkit-text-size-adjust: 100%;
27
+ color: #333333;
28
+ overflow: hidden;
29
+ font-family: "Helvetica Neue", Helvetica, "Segoe UI", Arial, freesans, sans-serif;
30
+ font-size: 16px;
31
+ line-height: 1.6;
32
+ word-wrap: break-word;
33
+ }
34
+
35
+ .markdown-body a {
36
+ background: transparent;
37
+ }
38
+
39
+ .markdown-body a:active,
40
+ .markdown-body a:hover {
41
+ outline: 0;
42
+ }
43
+
44
+ .markdown-body b,
45
+ .markdown-body strong {
46
+ font-weight: bold;
47
+ }
48
+
49
+ .markdown-body mark {
50
+ background: #ff0;
51
+ color: #000;
52
+ font-style: italic;
53
+ font-weight: bold;
54
+ }
55
+
56
+ .markdown-body sub,
57
+ .markdown-body sup {
58
+ font-size: 75%;
59
+ line-height: 0;
60
+ position: relative;
61
+ vertical-align: baseline;
62
+ }
63
+ .markdown-body sup {
64
+ top: -0.5em;
65
+ }
66
+ .markdown-body sub {
67
+ bottom: -0.25em;
68
+ }
69
+
70
+ .markdown-body h1 {
71
+ font-size: 2em;
72
+ margin: 0.67em 0;
73
+ }
74
+
75
+ .markdown-body img {
76
+ border: 0;
77
+ }
78
+
79
+ .markdown-body hr {
80
+ -moz-box-sizing: content-box;
81
+ box-sizing: content-box;
82
+ height: 0;
83
+ }
84
+
85
+ .markdown-body pre {
86
+ overflow: auto;
87
+ }
88
+
89
+ .markdown-body code,
90
+ .markdown-body kbd,
91
+ .markdown-body pre,
92
+ .markdown-body samp {
93
+ font-family: monospace, monospace;
94
+ font-size: 1em;
95
+ }
96
+
97
+ .markdown-body input {
98
+ color: inherit;
99
+ font: inherit;
100
+ margin: 0;
101
+ }
102
+
103
+ .markdown-body html input[disabled] {
104
+ cursor: default;
105
+ }
106
+
107
+ .markdown-body input {
108
+ line-height: normal;
109
+ }
110
+
111
+ .markdown-body input[type="checkbox"] {
112
+ box-sizing: border-box;
113
+ padding: 0;
114
+ }
115
+
116
+ .markdown-body table {
117
+ border-collapse: collapse;
118
+ border-spacing: 0;
119
+ }
120
+
121
+ .markdown-body td,
122
+ .markdown-body th {
123
+ padding: 0;
124
+ }
125
+
126
+ .markdown-body .codehilitetable {
127
+ border: 0;
128
+ border-spacing: 0;
129
+ }
130
+
131
+ .markdown-body .codehilitetable tr {
132
+ border: 0;
133
+ }
134
+
135
+ .markdown-body .codehilitetable pre,
136
+ .markdown-body .codehilitetable div.codehilite {
137
+ margin: 0;
138
+ }
139
+
140
+ .markdown-body .linenos,
141
+ .markdown-body .code,
142
+ .markdown-body .codehilitetable td {
143
+ border: 0;
144
+ padding: 0;
145
+ }
146
+
147
+ .markdown-body td:not(.linenos) .linenodiv {
148
+ padding: 0 !important;
149
+ }
150
+
151
+ .markdown-body .code {
152
+ width: 100%;
153
+ }
154
+
155
+ .markdown-body .linenos div pre,
156
+ .markdown-body .linenodiv pre,
157
+ .markdown-body .linenodiv {
158
+ border: 0;
159
+ -webkit-border-radius: 0;
160
+ -moz-border-radius: 0;
161
+ border-radius: 0;
162
+ -webkit-border-top-left-radius: 3px;
163
+ -webkit-border-bottom-left-radius: 3px;
164
+ -moz-border-radius-topleft: 3px;
165
+ -moz-border-radius-bottomleft: 3px;
166
+ border-top-left-radius: 3px;
167
+ border-bottom-left-radius: 3px;
168
+ }
169
+
170
+ .markdown-body .code div pre,
171
+ .markdown-body .code div {
172
+ border: 0;
173
+ -webkit-border-radius: 0;
174
+ -moz-border-radius: 0;
175
+ border-radius: 0;
176
+ -webkit-border-top-right-radius: 3px;
177
+ -webkit-border-bottom-right-radius: 3px;
178
+ -moz-border-radius-topright: 3px;
179
+ -moz-border-radius-bottomright: 3px;
180
+ border-top-right-radius: 3px;
181
+ border-bottom-right-radius: 3px;
182
+ }
183
+
184
+ .markdown-body * {
185
+ -moz-box-sizing: border-box;
186
+ box-sizing: border-box;
187
+ }
188
+
189
+ .markdown-body input {
190
+ font: 13px Helvetica, arial, freesans, clean, sans-serif, "Segoe UI Emoji", "Segoe UI Symbol";
191
+ line-height: 1.4;
192
+ }
193
+
194
+ .markdown-body a {
195
+ color: #4183c4;
196
+ text-decoration: none;
197
+ }
198
+
199
+ .markdown-body a:hover,
200
+ .markdown-body a:focus,
201
+ .markdown-body a:active {
202
+ text-decoration: underline;
203
+ }
204
+
205
+ .markdown-body hr {
206
+ height: 0;
207
+ margin: 15px 0;
208
+ overflow: hidden;
209
+ background: transparent;
210
+ border: 0;
211
+ border-bottom: 1px solid #ddd;
212
+ }
213
+
214
+ .markdown-body hr:before,
215
+ .markdown-body hr:after {
216
+ display: table;
217
+ content: " ";
218
+ }
219
+
220
+ .markdown-body hr:after {
221
+ clear: both;
222
+ }
223
+
224
+ .markdown-body h1,
225
+ .markdown-body h2,
226
+ .markdown-body h3,
227
+ .markdown-body h4,
228
+ .markdown-body h5,
229
+ .markdown-body h6 {
230
+ margin-top: 15px;
231
+ margin-bottom: 15px;
232
+ line-height: 1.1;
233
+ }
234
+
235
+ .markdown-body h1 {
236
+ font-size: 30px;
237
+ }
238
+
239
+ .markdown-body h2 {
240
+ font-size: 21px;
241
+ }
242
+
243
+ .markdown-body h3 {
244
+ font-size: 16px;
245
+ }
246
+
247
+ .markdown-body h4 {
248
+ font-size: 14px;
249
+ }
250
+
251
+ .markdown-body h5 {
252
+ font-size: 12px;
253
+ }
254
+
255
+ .markdown-body h6 {
256
+ font-size: 11px;
257
+ }
258
+
259
+ .markdown-body blockquote {
260
+ margin: 0;
261
+ }
262
+
263
+ .markdown-body ul,
264
+ .markdown-body ol {
265
+ padding: 0;
266
+ margin-top: 0;
267
+ margin-bottom: 0;
268
+ }
269
+
270
+ .markdown-body ol ol,
271
+ .markdown-body ul ol {
272
+ list-style-type: lower-roman;
273
+ }
274
+
275
+ .markdown-body ul ul ol,
276
+ .markdown-body ul ol ol,
277
+ .markdown-body ol ul ol,
278
+ .markdown-body ol ol ol {
279
+ list-style-type: lower-alpha;
280
+ }
281
+
282
+ .markdown-body dd {
283
+ margin-left: 0;
284
+ }
285
+
286
+ .markdown-body code,
287
+ .markdown-body pre,
288
+ .markdown-body samp {
289
+ font-family: Consolas, "Liberation Mono", Menlo, Courier, monospace;
290
+ font-size: 12px;
291
+ }
292
+
293
+ .markdown-body pre {
294
+ margin-top: 0;
295
+ margin-bottom: 0;
296
+ }
297
+
298
+ .markdown-body kbd {
299
+ background-color: #e7e7e7;
300
+ background-image: -moz-linear-gradient(#fefefe, #e7e7e7);
301
+ background-image: -webkit-linear-gradient(#fefefe, #e7e7e7);
302
+ background-image: linear-gradient(#fefefe, #e7e7e7);
303
+ background-repeat: repeat-x;
304
+ border-radius: 2px;
305
+ border: 1px solid #cfcfcf;
306
+ color: #000;
307
+ padding: 3px 5px;
308
+ line-height: 10px;
309
+ font: 11px Consolas, "Liberation Mono", Menlo, Courier, monospace;
310
+ display: inline-block;
311
+ }
312
+
313
+ .markdown-body>*:first-child {
314
+ margin-top: 0 !important;
315
+ }
316
+
317
+ .markdown-body>*:last-child {
318
+ margin-bottom: 0 !important;
319
+ }
320
+
321
+ .markdown-body .headeranchor-link {
322
+ position: absolute;
323
+ top: 0;
324
+ bottom: 0;
325
+ left: 0;
326
+ display: block;
327
+ padding-right: 6px;
328
+ padding-left: 30px;
329
+ margin-left: -30px;
330
+ }
331
+
332
+ .markdown-body .headeranchor-link:focus {
333
+ outline: none;
334
+ }
335
+
336
+ .markdown-body h1,
337
+ .markdown-body h2,
338
+ .markdown-body h3,
339
+ .markdown-body h4,
340
+ .markdown-body h5,
341
+ .markdown-body h6 {
342
+ position: relative;
343
+ margin-top: 1em;
344
+ margin-bottom: 16px;
345
+ font-weight: bold;
346
+ line-height: 1.4;
347
+ }
348
+
349
+ .markdown-body h1 .headeranchor,
350
+ .markdown-body h2 .headeranchor,
351
+ .markdown-body h3 .headeranchor,
352
+ .markdown-body h4 .headeranchor,
353
+ .markdown-body h5 .headeranchor,
354
+ .markdown-body h6 .headeranchor {
355
+ display: none;
356
+ color: #000;
357
+ vertical-align: middle;
358
+ }
359
+
360
+ .markdown-body h1:hover .headeranchor-link,
361
+ .markdown-body h2:hover .headeranchor-link,
362
+ .markdown-body h3:hover .headeranchor-link,
363
+ .markdown-body h4:hover .headeranchor-link,
364
+ .markdown-body h5:hover .headeranchor-link,
365
+ .markdown-body h6:hover .headeranchor-link {
366
+ height: 1em;
367
+ padding-left: 8px;
368
+ margin-left: -30px;
369
+ line-height: 1;
370
+ text-decoration: none;
371
+ }
372
+
373
+ .markdown-body h1:hover .headeranchor-link .headeranchor,
374
+ .markdown-body h2:hover .headeranchor-link .headeranchor,
375
+ .markdown-body h3:hover .headeranchor-link .headeranchor,
376
+ .markdown-body h4:hover .headeranchor-link .headeranchor,
377
+ .markdown-body h5:hover .headeranchor-link .headeranchor,
378
+ .markdown-body h6:hover .headeranchor-link .headeranchor {
379
+ display: inline-block;
380
+ }
381
+
382
+ .markdown-body h1 {
383
+ padding-bottom: 0.3em;
384
+ font-size: 2.25em;
385
+ line-height: 1.2;
386
+ border-bottom: 1px solid #eee;
387
+ }
388
+
389
+ .markdown-body h2 {
390
+ padding-bottom: 0.3em;
391
+ font-size: 1.75em;
392
+ line-height: 1.225;
393
+ border-bottom: 1px solid #eee;
394
+ }
395
+
396
+ .markdown-body h3 {
397
+ font-size: 1.5em;
398
+ line-height: 1.43;
399
+ }
400
+
401
+ .markdown-body h4 {
402
+ font-size: 1.25em;
403
+ }
404
+
405
+ .markdown-body h5 {
406
+ font-size: 1em;
407
+ }
408
+
409
+ .markdown-body h6 {
410
+ font-size: 1em;
411
+ color: #777;
412
+ }
413
+
414
+ .markdown-body p,
415
+ .markdown-body blockquote,
416
+ .markdown-body ul,
417
+ .markdown-body ol,
418
+ .markdown-body dl,
419
+ .markdown-body table,
420
+ .markdown-body pre,
421
+ .markdown-body .admonition {
422
+ margin-top: 0;
423
+ margin-bottom: 16px;
424
+ }
425
+
426
+ .markdown-body hr {
427
+ height: 4px;
428
+ padding: 0;
429
+ margin: 16px 0;
430
+ background-color: #e7e7e7;
431
+ border: 0 none;
432
+ }
433
+
434
+ .markdown-body ul,
435
+ .markdown-body ol {
436
+ padding-left: 2em;
437
+ }
438
+
439
+ .markdown-body ul ul,
440
+ .markdown-body ul ol,
441
+ .markdown-body ol ol,
442
+ .markdown-body ol ul {
443
+ margin-top: 0;
444
+ margin-bottom: 0;
445
+ }
446
+
447
+ .markdown-body li>p {
448
+ margin-top: 16px;
449
+ }
450
+
451
+ .markdown-body dl {
452
+ padding: 0;
453
+ }
454
+
455
+ .markdown-body dl dt {
456
+ padding: 0;
457
+ margin-top: 16px;
458
+ font-size: 1em;
459
+ font-style: italic;
460
+ font-weight: bold;
461
+ }
462
+
463
+ .markdown-body dl dd {
464
+ padding: 0 16px;
465
+ margin-bottom: 16px;
466
+ }
467
+
468
+ .markdown-body blockquote {
469
+ padding: 0 15px;
470
+ color: #777;
471
+ border-left: 4px solid #ddd;
472
+ }
473
+
474
+ .markdown-body blockquote>:first-child {
475
+ margin-top: 0;
476
+ }
477
+
478
+ .markdown-body blockquote>:last-child {
479
+ margin-bottom: 0;
480
+ }
481
+
482
+ .markdown-body table {
483
+ display: block;
484
+ width: 100%;
485
+ overflow: auto;
486
+ word-break: normal;
487
+ word-break: keep-all;
488
+ }
489
+
490
+ .markdown-body table th {
491
+ font-weight: bold;
492
+ }
493
+
494
+ .markdown-body table th,
495
+ .markdown-body table td {
496
+ padding: 6px 13px;
497
+ border: 1px solid #ddd;
498
+ }
499
+
500
+ .markdown-body table tr {
501
+ background-color: #fff;
502
+ border-top: 1px solid #ccc;
503
+ }
504
+
505
+ .markdown-body table tr:nth-child(2n) {
506
+ background-color: #f8f8f8;
507
+ }
508
+
509
+ .markdown-body img {
510
+ max-width: 100%;
511
+ -moz-box-sizing: border-box;
512
+ box-sizing: border-box;
513
+ }
514
+
515
+ .markdown-body code,
516
+ .markdown-body samp {
517
+ padding: 0;
518
+ padding-top: 0.2em;
519
+ padding-bottom: 0.2em;
520
+ margin: 0;
521
+ font-size: 85%;
522
+ background-color: rgba(0,0,0,0.04);
523
+ border-radius: 3px;
524
+ }
525
+
526
+ .markdown-body code:before,
527
+ .markdown-body code:after {
528
+ letter-spacing: -0.2em;
529
+ content: "\00a0";
530
+ }
531
+
532
+ .markdown-body pre>code {
533
+ padding: 0;
534
+ margin: 0;
535
+ font-size: 100%;
536
+ word-break: normal;
537
+ white-space: pre;
538
+ background: transparent;
539
+ border: 0;
540
+ }
541
+
542
+ .markdown-body .codehilite {
543
+ margin-bottom: 16px;
544
+ }
545
+
546
+ .markdown-body .codehilite pre,
547
+ .markdown-body pre {
548
+ padding: 16px;
549
+ overflow: auto;
550
+ font-size: 85%;
551
+ line-height: 1.45;
552
+ background-color: #f7f7f7;
553
+ border-radius: 3px;
554
+ }
555
+
556
+ .markdown-body .codehilite pre {
557
+ margin-bottom: 0;
558
+ word-break: normal;
559
+ }
560
+
561
+ .markdown-body pre {
562
+ word-wrap: normal;
563
+ }
564
+
565
+ .markdown-body pre code {
566
+ display: inline;
567
+ max-width: initial;
568
+ padding: 0;
569
+ margin: 0;
570
+ overflow: initial;
571
+ line-height: inherit;
572
+ word-wrap: normal;
573
+ background-color: transparent;
574
+ border: 0;
575
+ }
576
+
577
+ .markdown-body pre code:before,
578
+ .markdown-body pre code:after {
579
+ content: normal;
580
+ }
581
+
582
+ /* Admonition */
583
+ .markdown-body .admonition {
584
+ -webkit-border-radius: 3px;
585
+ -moz-border-radius: 3px;
586
+ position: relative;
587
+ border-radius: 3px;
588
+ border: 1px solid #e0e0e0;
589
+ border-left: 6px solid #333;
590
+ padding: 10px 10px 10px 30px;
591
+ }
592
+
593
+ .markdown-body .admonition table {
594
+ color: #333;
595
+ }
596
+
597
+ .markdown-body .admonition p {
598
+ padding: 0;
599
+ }
600
+
601
+ .markdown-body .admonition-title {
602
+ font-weight: bold;
603
+ margin: 0;
604
+ }
605
+
606
+ .markdown-body .admonition>.admonition-title {
607
+ color: #333;
608
+ }
609
+
610
+ .markdown-body .attention>.admonition-title {
611
+ color: #a6d796;
612
+ }
613
+
614
+ .markdown-body .caution>.admonition-title {
615
+ color: #d7a796;
616
+ }
617
+
618
+ .markdown-body .hint>.admonition-title {
619
+ color: #96c6d7;
620
+ }
621
+
622
+ .markdown-body .danger>.admonition-title {
623
+ color: #c25f77;
624
+ }
625
+
626
+ .markdown-body .question>.admonition-title {
627
+ color: #96a6d7;
628
+ }
629
+
630
+ .markdown-body .note>.admonition-title {
631
+ color: #d7c896;
632
+ }
633
+
634
+ .markdown-body .admonition:before,
635
+ .markdown-body .attention:before,
636
+ .markdown-body .caution:before,
637
+ .markdown-body .hint:before,
638
+ .markdown-body .danger:before,
639
+ .markdown-body .question:before,
640
+ .markdown-body .note:before {
641
+ font: normal normal 16px fontawesome-mini;
642
+ -moz-osx-font-smoothing: grayscale;
643
+ -webkit-user-select: none;
644
+ -moz-user-select: none;
645
+ -ms-user-select: none;
646
+ user-select: none;
647
+ line-height: 1.5;
648
+ color: #333;
649
+ position: absolute;
650
+ left: 0;
651
+ top: 0;
652
+ padding-top: 10px;
653
+ padding-left: 10px;
654
+ }
655
+
656
+ .markdown-body .admonition:before {
657
+ content: "\f056\00a0";
658
+ color: 333;
659
+ }
660
+
661
+ .markdown-body .attention:before {
662
+ content: "\f058\00a0";
663
+ color: #a6d796;
664
+ }
665
+
666
+ .markdown-body .caution:before {
667
+ content: "\f06a\00a0";
668
+ color: #d7a796;
669
+ }
670
+
671
+ .markdown-body .hint:before {
672
+ content: "\f05a\00a0";
673
+ color: #96c6d7;
674
+ }
675
+
676
+ .markdown-body .danger:before {
677
+ content: "\f057\00a0";
678
+ color: #c25f77;
679
+ }
680
+
681
+ .markdown-body .question:before {
682
+ content: "\f059\00a0";
683
+ color: #96a6d7;
684
+ }
685
+
686
+ .markdown-body .note:before {
687
+ content: "\f040\00a0";
688
+ color: #d7c896;
689
+ }
690
+
691
+ .markdown-body .admonition::after {
692
+ content: normal;
693
+ }
694
+
695
+ .markdown-body .attention {
696
+ border-left: 6px solid #a6d796;
697
+ }
698
+
699
+ .markdown-body .caution {
700
+ border-left: 6px solid #d7a796;
701
+ }
702
+
703
+ .markdown-body .hint {
704
+ border-left: 6px solid #96c6d7;
705
+ }
706
+
707
+ .markdown-body .danger {
708
+ border-left: 6px solid #c25f77;
709
+ }
710
+
711
+ .markdown-body .question {
712
+ border-left: 6px solid #96a6d7;
713
+ }
714
+
715
+ .markdown-body .note {
716
+ border-left: 6px solid #d7c896;
717
+ }
718
+
719
+ .markdown-body .admonition>*:first-child {
720
+ margin-top: 0 !important;
721
+ }
722
+
723
+ .markdown-body .admonition>*:last-child {
724
+ margin-bottom: 0 !important;
725
+ }
726
+
727
+ /* progress bar*/
728
+ .markdown-body .progress {
729
+ display: block;
730
+ width: 300px;
731
+ margin: 10px 0;
732
+ height: 24px;
733
+ -webkit-border-radius: 3px;
734
+ -moz-border-radius: 3px;
735
+ border-radius: 3px;
736
+ background-color: #ededed;
737
+ position: relative;
738
+ box-shadow: inset -1px 1px 3px rgba(0, 0, 0, .1);
739
+ }
740
+
741
+ .markdown-body .progress-label {
742
+ position: absolute;
743
+ text-align: center;
744
+ font-weight: bold;
745
+ width: 100%; margin: 0;
746
+ line-height: 24px;
747
+ color: #333;
748
+ text-shadow: 1px 1px 0 #fefefe, -1px -1px 0 #fefefe, -1px 1px 0 #fefefe, 1px -1px 0 #fefefe, 0 1px 0 #fefefe, 0 -1px 0 #fefefe, 1px 0 0 #fefefe, -1px 0 0 #fefefe, 1px 1px 2px #000;
749
+ -webkit-font-smoothing: antialiased !important;
750
+ white-space: nowrap;
751
+ overflow: hidden;
752
+ }
753
+
754
+ .markdown-body .progress-bar {
755
+ height: 24px;
756
+ float: left;
757
+ -webkit-border-radius: 3px;
758
+ -moz-border-radius: 3px;
759
+ border-radius: 3px;
760
+ background-color: #96c6d7;
761
+ box-shadow: inset 0 1px 0 rgba(255, 255, 255, .5), inset 0 -1px 0 rgba(0, 0, 0, .1);
762
+ background-size: 30px 30px;
763
+ background-image: -webkit-linear-gradient(
764
+ 135deg, rgba(255, 255, 255, .4) 27%,
765
+ transparent 27%,
766
+ transparent 52%, rgba(255, 255, 255, .4) 52%,
767
+ rgba(255, 255, 255, .4) 77%,
768
+ transparent 77%, transparent
769
+ );
770
+ background-image: -moz-linear-gradient(
771
+ 135deg,
772
+ rgba(255, 255, 255, .4) 27%, transparent 27%,
773
+ transparent 52%, rgba(255, 255, 255, .4) 52%,
774
+ rgba(255, 255, 255, .4) 77%, transparent 77%,
775
+ transparent
776
+ );
777
+ background-image: -ms-linear-gradient(
778
+ 135deg,
779
+ rgba(255, 255, 255, .4) 27%, transparent 27%,
780
+ transparent 52%, rgba(255, 255, 255, .4) 52%,
781
+ rgba(255, 255, 255, .4) 77%, transparent 77%,
782
+ transparent
783
+ );
784
+ background-image: -o-linear-gradient(
785
+ 135deg,
786
+ rgba(255, 255, 255, .4) 27%, transparent 27%,
787
+ transparent 52%, rgba(255, 255, 255, .4) 52%,
788
+ rgba(255, 255, 255, .4) 77%, transparent 77%,
789
+ transparent
790
+ );
791
+ background-image: linear-gradient(
792
+ 135deg,
793
+ rgba(255, 255, 255, .4) 27%, transparent 27%,
794
+ transparent 52%, rgba(255, 255, 255, .4) 52%,
795
+ rgba(255, 255, 255, .4) 77%, transparent 77%,
796
+ transparent
797
+ );
798
+ }
799
+
800
+ .markdown-body .progress-100plus .progress-bar {
801
+ background-color: #a6d796;
802
+ }
803
+
804
+ .markdown-body .progress-80plus .progress-bar {
805
+ background-color: #c6d796;
806
+ }
807
+
808
+ .markdown-body .progress-60plus .progress-bar {
809
+ background-color: #d7c896;
810
+ }
811
+
812
+ .markdown-body .progress-40plus .progress-bar {
813
+ background-color: #d7a796;
814
+ }
815
+
816
+ .markdown-body .progress-20plus .progress-bar {
817
+ background-color: #d796a6;
818
+ }
819
+
820
+ .markdown-body .progress-0plus .progress-bar {
821
+ background-color: #c25f77;
822
+ }
823
+
824
+ .markdown-body .candystripe-animate .progress-bar{
825
+ -webkit-animation: animate-stripes 3s linear infinite;
826
+ -moz-animation: animate-stripes 3s linear infinite;
827
+ animation: animate-stripes 3s linear infinite;
828
+ }
829
+
830
+ @-webkit-keyframes animate-stripes {
831
+ 0% {
832
+ background-position: 0 0;
833
+ }
834
+
835
+ 100% {
836
+ background-position: 60px 0;
837
+ }
838
+ }
839
+
840
+ @-moz-keyframes animate-stripes {
841
+ 0% {
842
+ background-position: 0 0;
843
+ }
844
+
845
+ 100% {
846
+ background-position: 60px 0;
847
+ }
848
+ }
849
+
850
+ @keyframes animate-stripes {
851
+ 0% {
852
+ background-position: 0 0;
853
+ }
854
+
855
+ 100% {
856
+ background-position: 60px 0;
857
+ }
858
+ }
859
+
860
+ .markdown-body .gloss .progress-bar {
861
+ box-shadow:
862
+ inset 0 4px 12px rgba(255, 255, 255, .7),
863
+ inset 0 -12px 0 rgba(0, 0, 0, .05);
864
+ }
865
+
866
+ /* Multimarkdown Critic Blocks */
867
+ .markdown-body .critic_mark {
868
+ background: #ff0;
869
+ }
870
+
871
+ .markdown-body .critic_delete {
872
+ color: #c82829;
873
+ text-decoration: line-through;
874
+ }
875
+
876
+ .markdown-body .critic_insert {
877
+ color: #718c00 ;
878
+ text-decoration: underline;
879
+ }
880
+
881
+ .markdown-body .critic_comment {
882
+ color: #8e908c;
883
+ font-style: italic;
884
+ }
885
+
886
+ .markdown-body .headeranchor {
887
+ font: normal normal 16px octicons-anchor;
888
+ line-height: 1;
889
+ display: inline-block;
890
+ text-decoration: none;
891
+ -webkit-font-smoothing: antialiased;
892
+ -moz-osx-font-smoothing: grayscale;
893
+ -webkit-user-select: none;
894
+ -moz-user-select: none;
895
+ -ms-user-select: none;
896
+ user-select: none;
897
+ }
898
+
899
+ .headeranchor:before {
900
+ content: '\f05c';
901
+ }
902
+
903
+ .markdown-body .task-list-item {
904
+ list-style-type: none;
905
+ }
906
+
907
+ .markdown-body .task-list-item+.task-list-item {
908
+ margin-top: 3px;
909
+ }
910
+
911
+ .markdown-body .task-list-item input {
912
+ margin: 0 4px 0.25em -20px;
913
+ vertical-align: middle;
914
+ }
915
+
916
+ /* Media */
917
+ @media only screen and (min-width: 480px) {
918
+ .markdown-body {
919
+ font-size:14px;
920
+ }
921
+ }
922
+
923
+ @media only screen and (min-width: 768px) {
924
+ .markdown-body {
925
+ font-size:16px;
926
+ }
927
+ }
928
+
929
+ @media print {
930
+ .markdown-body * {
931
+ background: transparent !important;
932
+ color: black !important;
933
+ filter:none !important;
934
+ -ms-filter: none !important;
935
+ }
936
+
937
+ .markdown-body {
938
+ font-size:12pt;
939
+ max-width:100%;
940
+ outline:none;
941
+ border: 0;
942
+ }
943
+
944
+ .markdown-body a,
945
+ .markdown-body a:visited {
946
+ text-decoration: underline;
947
+ }
948
+
949
+ .markdown-body .headeranchor-link {
950
+ display: none;
951
+ }
952
+
953
+ .markdown-body a[href]:after {
954
+ content: " (" attr(href) ")";
955
+ }
956
+
957
+ .markdown-body abbr[title]:after {
958
+ content: " (" attr(title) ")";
959
+ }
960
+
961
+ .markdown-body .ir a:after,
962
+ .markdown-body a[href^="javascript:"]:after,
963
+ .markdown-body a[href^="#"]:after {
964
+ content: "";
965
+ }
966
+
967
+ .markdown-body pre {
968
+ white-space: pre;
969
+ white-space: pre-wrap;
970
+ word-wrap: break-word;
971
+ }
972
+
973
+ .markdown-body pre,
974
+ .markdown-body blockquote {
975
+ border: 1px solid #999;
976
+ padding-right: 1em;
977
+ page-break-inside: avoid;
978
+ }
979
+
980
+ .markdown-body .progress,
981
+ .markdown-body .progress-bar {
982
+ -moz-box-shadow: none;
983
+ -webkit-box-shadow: none;
984
+ box-shadow: none;
985
+ }
986
+
987
+ .markdown-body .progress {
988
+ border: 1px solid #ddd;
989
+ }
990
+
991
+ .markdown-body .progress-bar {
992
+ height: 22px;
993
+ border-right: 1px solid #ddd;
994
+ }
995
+
996
+ .markdown-body tr,
997
+ .markdown-body img {
998
+ page-break-inside: avoid;
999
+ }
1000
+
1001
+ .markdown-body img {
1002
+ max-width: 100% !important;
1003
+ }
1004
+
1005
+ .markdown-body p,
1006
+ .markdown-body h2,
1007
+ .markdown-body h3 {
1008
+ orphans: 3;
1009
+ widows: 3;
1010
+ }
1011
+
1012
+ .markdown-body h2,
1013
+ .markdown-body h3 {
1014
+ page-break-after: avoid;
1015
+ }
1016
+ }
1017
+ </style><title>README</title></head><body><article class="markdown-body"><h1 id="tokenator"><a name="user-content-tokenator" href="#tokenator" class="headeranchor-link" aria-hidden="true"><span class="headeranchor"></span></a>Tokenator</h1>
1018
+ <p>This is a simple rule-based tokenizer, based on the one developed by Robert Macyntyre in 1995 for the Penn Treebank, and adapted from a script lifted from <a href="https://github.com/louismullie/treat">the Treat gem</a>.</p>
1019
+ <p>It&rsquo;s taylored to my specific use case: I don&rsquo;t need part-of-speech tagging or punctuation preservation.</p>
1020
+ <h2 id="installation"><a name="user-content-installation" href="#installation" class="headeranchor-link" aria-hidden="true"><span class="headeranchor"></span></a>Installation</h2>
1021
+ <p>Add this line to your application&rsquo;s Gemfile:</p>
1022
+ <pre><code class="ruby">gem 'tokenator'
1023
+ </code></pre>
1024
+
1025
+ <p>And then execute:</p>
1026
+ <pre><code>$ bundle
1027
+ </code></pre>
1028
+ <p>Or install it yourself as:</p>
1029
+ <pre><code>$ gem install tokenator
1030
+ </code></pre>
1031
+ <h2 id="usage"><a name="user-content-usage" href="#usage" class="headeranchor-link" aria-hidden="true"><span class="headeranchor"></span></a>Usage</h2>
1032
+ <pre><code class="ruby">tokenizer = Tokenator.new
1033
+
1034
+ tokenizer.tokenize &quot;This? It's just a basic example!&quot;
1035
+ # =&gt; [&quot;This&quot;, &quot;It&quot;, &quot;is&quot;, &quot;just&quot;, &quot;a&quot;, &quot;basic&quot;, &quot;example&quot;]
1036
+
1037
+ tokenizer.tokenize &quot;This? It's just a basic example!&quot;, {:remove_punctuation =&gt; false}
1038
+ # =&gt; [&quot;This&quot;, &quot;?&quot;, &quot;It&quot;, &quot;is&quot;, &quot;just&quot;, &quot;a&quot;, &quot;basic&quot;, &quot;example&quot;, &quot;!&quot;]
1039
+
1040
+ </code></pre>
1041
+
1042
+ <h2 id="development"><a name="user-content-development" href="#development" class="headeranchor-link" aria-hidden="true"><span class="headeranchor"></span></a>Development</h2>
1043
+ <p>After checking out the repo, run <code>bin/setup</code> to install dependencies. Then, run <code>rake test</code> to run the tests. You can also run <code>bin/console</code> for an interactive prompt that will allow you to experiment.</p>
1044
+ <p>To install this gem onto your local machine, run <code>bundle exec rake install</code>. To release a new version, update the version number in <code>version.rb</code>, and then run <code>bundle exec rake release</code>, which will create a git tag for the version, push git commits and tags, and push the <code>.gem</code> file to <a href="https://rubygems.org">rubygems.org</a>.</p>
1045
+ <h2 id="contributing"><a name="user-content-contributing" href="#contributing" class="headeranchor-link" aria-hidden="true"><span class="headeranchor"></span></a>Contributing</h2>
1046
+ <p>Bug reports and pull requests are welcome on GitHub at <a href="https://github.com/Amval/tokenator">https://github.com/Amval/tokenator</a>.</p>
1047
+ <h2 id="license"><a name="user-content-license" href="#license" class="headeranchor-link" aria-hidden="true"><span class="headeranchor"></span></a>License</h2>
1048
+ <p>The gem is available as open source under the terms of the <a href="http://opensource.org/licenses/MIT">MIT License</a>.</p></article></body></html>
data/README.md CHANGED
@@ -22,7 +22,16 @@ Or install it yourself as:
22
22
 
23
23
  ## Usage
24
24
 
25
- TODO: Write usage instructions here
25
+ ```ruby
26
+ tokenizer = Tokenator.new
27
+
28
+ tokenizer.tokenize "This? It's just a basic example!"
29
+ # => ["This", "It", "is", "just", "a", "basic", "example"]
30
+
31
+ tokenizer.tokenize "This? It's just a basic example!", {:remove_punctuation => false}
32
+ # => ["This", "?", "It", "is", "just", "a", "basic", "example", "!"]
33
+
34
+ ```
26
35
 
27
36
  ## Development
28
37
 
@@ -75,8 +75,11 @@ class PTB
75
75
  s.gsub!(/\.\.\./, '')
76
76
  s.gsub!(/ \./, ' ')
77
77
  s.gsub!(/- /, ' ')
78
+ s.gsub!(/["`'][^\w]/, ' ')
79
+ s.gsub!(/"$/, ' ')
78
80
  end
79
81
  s.split(/\s+/)
80
82
  end
81
83
 
82
84
  end
85
+
@@ -1,3 +1,3 @@
1
1
  module Tokenator
2
- VERSION = "0.2.0"
2
+ VERSION = "0.2.1"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: tokenator
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.0
4
+ version: 0.2.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Amval
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2016-01-26 00:00:00.000000000 Z
11
+ date: 2016-02-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -63,6 +63,7 @@ files:
63
63
  - ".travis.yml"
64
64
  - Gemfile
65
65
  - LICENSE.txt
66
+ - README.html
66
67
  - README.md
67
68
  - Rakefile
68
69
  - bin/console