uv-rays 1.0.6 → 1.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/uv-rays/abstract_tokenizer.rb +6 -7
- data/lib/uv-rays/buffered_tokenizer.rb +10 -8
- data/lib/uv-rays/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: dbda330c1c29c6e1d6d83dd262df51d94322ec4f
|
4
|
+
data.tar.gz: db15f804d9f4514d39af9926021627169a59b185
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: bf595aaa5a464e8ee0c111f964c528e190b76581a69b8509909028b0a1e06f01962f19b6cc1dc5c81e7974a4c1260e62027e640a17a57e69a071a2e25450fe5c
|
7
|
+
data.tar.gz: 3947075d8337536b045a03d9aad35cfd6551876ce8153a55c11d5f6e648605bc5394c49fac316d18f187bfceb91ef3a6c1280f34c80704ce69cf3b0d24f585f1
|
@@ -6,6 +6,7 @@ module UV
|
|
6
6
|
# callback based system for application level tokenization without
|
7
7
|
# the heavy lifting.
|
8
8
|
class AbstractTokenizer
|
9
|
+
DEFAULT_ENCODING = 'ASCII-8BIT'.freeze
|
9
10
|
|
10
11
|
attr_accessor :callback, :indicator, :size_limit, :verbose
|
11
12
|
|
@@ -15,16 +16,14 @@ module UV
|
|
15
16
|
@indicator = options[:indicator]
|
16
17
|
@size_limit = options[:size_limit]
|
17
18
|
@verbose = options[:verbose] if @size_limit
|
18
|
-
@encoding = options[:encoding]
|
19
|
+
@encoding = options[:encoding] || DEFAULT_ENCODING
|
19
20
|
|
20
21
|
raise ArgumentError, 'no indicator provided' unless @indicator
|
21
22
|
raise ArgumentError, 'no callback provided' unless @callback
|
22
23
|
|
23
24
|
@input = ''
|
24
|
-
|
25
|
-
|
26
|
-
@indicator.force_encoding(@encoding) if @indicator.is_a?(String)
|
27
|
-
end
|
25
|
+
@input.force_encoding(@encoding)
|
26
|
+
@indicator.force_encoding(@encoding) if @indicator.is_a?(String)
|
28
27
|
end
|
29
28
|
|
30
29
|
# Extract takes an arbitrary string of input data and returns an array of
|
@@ -37,7 +36,7 @@ module UV
|
|
37
36
|
#
|
38
37
|
# @param [String] data
|
39
38
|
def extract(data)
|
40
|
-
data.force_encoding(@encoding)
|
39
|
+
data.force_encoding(@encoding)
|
41
40
|
@input << data
|
42
41
|
|
43
42
|
messages = @input.split(@indicator, -1)
|
@@ -120,7 +119,7 @@ module UV
|
|
120
119
|
|
121
120
|
def reset
|
122
121
|
@input = ''
|
123
|
-
@input.force_encoding(@encoding)
|
122
|
+
@input.force_encoding(@encoding)
|
124
123
|
end
|
125
124
|
end
|
126
125
|
end
|
@@ -15,6 +15,7 @@
|
|
15
15
|
# end
|
16
16
|
module UV
|
17
17
|
class BufferedTokenizer
|
18
|
+
DEFAULT_ENCODING = 'ASCII-8BIT'.freeze
|
18
19
|
|
19
20
|
attr_accessor :delimiter, :indicator, :size_limit, :verbose
|
20
21
|
|
@@ -26,11 +27,14 @@ module UV
|
|
26
27
|
@size_limit = options[:size_limit]
|
27
28
|
@min_length = options[:min_length] || 1
|
28
29
|
@verbose = options[:verbose] if @size_limit
|
29
|
-
@encoding = options[:encoding]
|
30
|
+
@encoding = options[:encoding] || DEFAULT_ENCODING
|
30
31
|
|
31
32
|
if @delimiter
|
33
|
+
@delimiter.force_encoding(@encoding) if @delimiter.is_a?(String)
|
34
|
+
@indicator.force_encoding(@encoding) if @indicator.is_a?(String)
|
32
35
|
@extract_method = method(:delimiter_extract)
|
33
36
|
elsif @indicator && @msg_length
|
37
|
+
@indicator.force_encoding(@encoding) if @indicator.is_a?(String)
|
34
38
|
@extract_method = method(:length_extract)
|
35
39
|
else
|
36
40
|
raise ArgumentError, 'no delimiter provided'
|
@@ -49,7 +53,7 @@ module UV
|
|
49
53
|
#
|
50
54
|
# @param [String] data
|
51
55
|
def extract(data)
|
52
|
-
data.force_encoding(@encoding)
|
56
|
+
data.force_encoding(@encoding)
|
53
57
|
@input << data
|
54
58
|
|
55
59
|
@extract_method.call
|
@@ -143,16 +147,14 @@ module UV
|
|
143
147
|
|
144
148
|
def init_buffer
|
145
149
|
@input = ''
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
@indicator.force_encoding(@encoding) if @indicator.is_a?(String)
|
150
|
-
end
|
150
|
+
@input.force_encoding(@encoding)
|
151
|
+
@delimiter.force_encoding(@encoding) if @delimiter.is_a?(String)
|
152
|
+
@indicator.force_encoding(@encoding) if @indicator.is_a?(String)
|
151
153
|
end
|
152
154
|
|
153
155
|
def reset(value = '')
|
154
156
|
@input = value
|
155
|
-
@input.force_encoding(@encoding)
|
157
|
+
@input.force_encoding(@encoding)
|
156
158
|
end
|
157
159
|
end
|
158
160
|
end
|
data/lib/uv-rays/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: uv-rays
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Stephen von Takach
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-03-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: libuv
|