invoca-utils 0.3.0 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +7 -0
- data/Gemfile +5 -1
- data/Gemfile.lock +23 -20
- data/Rakefile +1 -0
- data/invoca-utils.gemspec +2 -3
- data/lib/invoca/utils.rb +6 -5
- data/lib/invoca/utils/exceptions.rb +25 -0
- data/lib/invoca/utils/guaranteed_utf8_string.rb +22 -7
- data/lib/invoca/utils/hash.rb +0 -1
- data/lib/invoca/utils/version.rb +1 -1
- data/test/unit/exceptions_test.rb +84 -0
- data/test/unit/guaranteed_utf8_string_test.rb +23 -0
- metadata +6 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 54dcad50905bf9c0d48905b88e563c6418ea4bc852e4f85d4cd892e2b2031096
|
4
|
+
data.tar.gz: 592dde85e812312a5fb5ff67d34da1d5a0127f0c2c2f5f60538baab3c63d32ff
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6fac4beb8eef9b3a4ebc3615cb7711c9134e401e34ba26a2d0012b89c7962ca19440b2ceb2e0fafd5aaafcd4576717d10fddd276c4583858ffc725904194464a
|
7
|
+
data.tar.gz: c312101bd66d62f9c145610be64a8e75ec931387d4dc33f581885dc5707ff7565e474fd618ddbf569ed9091a591d0f2e0a164b7899ffe311b889c62fbc5a7d37
|
data/CHANGELOG.md
CHANGED
@@ -4,6 +4,12 @@ Inspired by [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
|
|
4
4
|
|
5
5
|
Note: This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
6
6
|
|
7
|
+
## [0.4.0] - 2020-06-09
|
8
|
+
### Added
|
9
|
+
- Added `Invoca::Utils.retry_on_exception`.
|
10
|
+
- Added `Invoca::Utils::GuaranteedUTF8String.normalize_all_strings` to normalize
|
11
|
+
all strings found in a JSON doc of hashes, arrays, and values.
|
12
|
+
|
7
13
|
## [0.3.0] - 2020-04-28
|
8
14
|
### Added
|
9
15
|
- Array::* operator changed to use alias_method instead of prepend to prevent infinite recursion when HoboSupport gem is present
|
@@ -17,5 +23,6 @@ Note: This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0
|
|
17
23
|
- Enumerable::build_hash method ported from HoboSupport
|
18
24
|
- Enumerable::* operator ported from HoboSupport
|
19
25
|
|
26
|
+
[0.4.0]: https://github.com/Invoca/invoca-utils/compare/v0.3.0...v0.4.0
|
20
27
|
[0.3.0]: https://github.com/Invoca/invoca-utils/compare/v0.2.0...v0.3.0
|
21
28
|
[0.2.0]: https://github.com/Invoca/invoca-utils/compare/v0.1.1...v0.2.0
|
data/Gemfile
CHANGED
@@ -4,7 +4,7 @@ source 'https://rubygems.org'
|
|
4
4
|
|
5
5
|
gemspec
|
6
6
|
|
7
|
-
|
7
|
+
source 'https://rubygems.org' do
|
8
8
|
gem 'activesupport', '~> 4.2'
|
9
9
|
gem 'minitest'
|
10
10
|
gem 'minitest-reporters'
|
@@ -16,3 +16,7 @@ group :development do
|
|
16
16
|
gem 'test-unit', '= 1.2.3'
|
17
17
|
gem 'tzinfo'
|
18
18
|
end
|
19
|
+
|
20
|
+
source 'https://gem.fury.io/invoca' do
|
21
|
+
gem 'test_overrides', '~> 0.13'
|
22
|
+
end
|
data/Gemfile.lock
CHANGED
@@ -1,37 +1,38 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
invoca-utils (0.
|
4
|
+
invoca-utils (0.4.0)
|
5
5
|
|
6
6
|
GEM
|
7
7
|
remote: https://rubygems.org/
|
8
|
+
remote: https://gem.fury.io/invoca/
|
8
9
|
specs:
|
9
|
-
activesupport (4.2.11.
|
10
|
+
activesupport (4.2.11.3)
|
10
11
|
i18n (~> 0.7)
|
11
12
|
minitest (~> 5.1)
|
12
13
|
thread_safe (~> 0.3, >= 0.3.4)
|
13
14
|
tzinfo (~> 1.1)
|
14
15
|
ansi (1.5.0)
|
15
16
|
builder (3.2.4)
|
16
|
-
coderay (1.1.
|
17
|
+
coderay (1.1.3)
|
17
18
|
concurrent-ruby (1.1.6)
|
18
19
|
hoe (3.22.1)
|
19
20
|
rake (>= 0.8, < 15.0)
|
20
21
|
i18n (0.9.5)
|
21
22
|
concurrent-ruby (~> 1.0)
|
22
|
-
method_source (0.
|
23
|
-
minitest (5.14.
|
24
|
-
minitest-reporters (1.
|
23
|
+
method_source (1.0.0)
|
24
|
+
minitest (5.14.1)
|
25
|
+
minitest-reporters (1.4.2)
|
25
26
|
ansi
|
26
27
|
builder
|
27
28
|
minitest (>= 5.0)
|
28
29
|
ruby-progressbar
|
29
|
-
pry (0.
|
30
|
-
coderay (~> 1.1
|
31
|
-
method_source (~>
|
30
|
+
pry (0.13.1)
|
31
|
+
coderay (~> 1.1)
|
32
|
+
method_source (~> 1.0)
|
32
33
|
rake (13.0.1)
|
33
34
|
rr (1.1.2)
|
34
|
-
ruby-prof (
|
35
|
+
ruby-prof (1.4.1)
|
35
36
|
ruby-progressbar (1.10.1)
|
36
37
|
shoulda (3.5.0)
|
37
38
|
shoulda-context (~> 1.0, >= 1.0.1)
|
@@ -41,6 +42,7 @@ GEM
|
|
41
42
|
activesupport (>= 3.0.0)
|
42
43
|
test-unit (1.2.3)
|
43
44
|
hoe (>= 1.5.1)
|
45
|
+
test_overrides (0.13.0)
|
44
46
|
thread_safe (0.3.6)
|
45
47
|
tzinfo (1.2.7)
|
46
48
|
thread_safe (~> 0.1)
|
@@ -49,17 +51,18 @@ PLATFORMS
|
|
49
51
|
ruby
|
50
52
|
|
51
53
|
DEPENDENCIES
|
52
|
-
activesupport (~> 4.2)
|
54
|
+
activesupport (~> 4.2)!
|
53
55
|
invoca-utils!
|
54
|
-
minitest
|
55
|
-
minitest-reporters
|
56
|
-
pry
|
57
|
-
rake
|
58
|
-
rr (= 1.1.2)
|
59
|
-
ruby-prof
|
60
|
-
shoulda (= 3.5.0)
|
61
|
-
test-unit (= 1.2.3)
|
62
|
-
|
56
|
+
minitest!
|
57
|
+
minitest-reporters!
|
58
|
+
pry!
|
59
|
+
rake!
|
60
|
+
rr (= 1.1.2)!
|
61
|
+
ruby-prof!
|
62
|
+
shoulda (= 3.5.0)!
|
63
|
+
test-unit (= 1.2.3)!
|
64
|
+
test_overrides (~> 0.13)!
|
65
|
+
tzinfo!
|
63
66
|
|
64
67
|
BUNDLED WITH
|
65
68
|
1.17.3
|
data/Rakefile
CHANGED
data/invoca-utils.gemspec
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
lib = File.expand_path('
|
1
|
+
lib = File.expand_path('lib', __dir__)
|
2
2
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
3
3
|
require 'invoca/utils/version'
|
4
4
|
|
@@ -7,8 +7,7 @@ Gem::Specification.new do |spec|
|
|
7
7
|
spec.version = Invoca::Utils::VERSION
|
8
8
|
spec.authors = ["Invoca development"]
|
9
9
|
spec.email = ["development@invoca.com"]
|
10
|
-
spec.summary =
|
11
|
-
spec.description = %q{A public collection of helpers used in multiple projects}
|
10
|
+
spec.summary = "A public collection of helpers used in multiple projects"
|
12
11
|
spec.homepage = ""
|
13
12
|
spec.license = "MIT"
|
14
13
|
|
data/lib/invoca/utils.rb
CHANGED
@@ -1,5 +1,10 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
module Invoca
|
4
|
+
module Utils
|
5
|
+
end
|
6
|
+
end
|
7
|
+
|
3
8
|
require "invoca/utils/module"
|
4
9
|
require "invoca/utils/array"
|
5
10
|
require "invoca/utils/enumerable"
|
@@ -11,14 +16,10 @@ require "invoca/utils/map_compact"
|
|
11
16
|
require "invoca/utils/min_max"
|
12
17
|
require "invoca/utils/stable_sort"
|
13
18
|
require "invoca/utils/time"
|
19
|
+
require "invoca/utils/exceptions"
|
14
20
|
require "invoca/utils/guaranteed_utf8_string"
|
15
21
|
require "invoca/utils/version"
|
16
22
|
|
17
|
-
module Invoca
|
18
|
-
module Utils
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
23
|
unless defined?(Diff)
|
23
24
|
Diff = Invoca::Utils::Diff
|
24
25
|
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Invoca
|
4
|
+
module Utils
|
5
|
+
class << self
|
6
|
+
# Yields and rescues any exceptions given in `exception_classes`, retrying the given number of times.
|
7
|
+
# The final retry does not rescue any exceptions.
|
8
|
+
# The try number (0..retries) is yielded as a block param.
|
9
|
+
#
|
10
|
+
# @param [Class or Array(Class)] exception_classes - exception()s) to rescue
|
11
|
+
# @param [Integer] retries: - 1+ count of retries (1 retry = up to 2 tries total)
|
12
|
+
# @param [Proc] before_retry - optional proc which is called before each retry, with the exception passed as a block param
|
13
|
+
# @return the value from yield
|
14
|
+
def retry_on_exception(exception_classes, retries: 1, before_retry: nil)
|
15
|
+
retries.times do |attempt_number|
|
16
|
+
return yield(attempt_number)
|
17
|
+
rescue *Array(exception_classes) => ex
|
18
|
+
before_retry&.call(ex)
|
19
|
+
end
|
20
|
+
|
21
|
+
yield(retries) # no rescue for this last try, so any exceptions will raise out
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -21,6 +21,7 @@ module Invoca
|
|
21
21
|
REPLACE_CHARACTER = '~'
|
22
22
|
|
23
23
|
class << self
|
24
|
+
# normalizes a string to UTF-8
|
24
25
|
def normalize_string(orig_string,
|
25
26
|
normalize_utf16: true,
|
26
27
|
normalize_cp1252: true,
|
@@ -43,6 +44,22 @@ module Invoca
|
|
43
44
|
replace_unicode_beyond_ffff: replace_unicode_beyond_ffff)
|
44
45
|
end
|
45
46
|
|
47
|
+
# Walks a JSON doc of hashes, arrays, and values and normalizes all strings found to UTF-8
|
48
|
+
def normalize_all_strings(value, **options)
|
49
|
+
case value
|
50
|
+
when Hash
|
51
|
+
value.each_with_object({}) do |(k, v), result|
|
52
|
+
result[normalize_all_strings(k, **options)] = normalize_all_strings(v, **options)
|
53
|
+
end
|
54
|
+
when Array
|
55
|
+
value.map { |v| normalize_all_strings(v, **options) }
|
56
|
+
when String
|
57
|
+
normalize_string(value, **options)
|
58
|
+
else
|
59
|
+
value
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
46
63
|
private
|
47
64
|
|
48
65
|
def normalize_string_from_utf8(string,
|
@@ -54,13 +71,11 @@ module Invoca
|
|
54
71
|
found_utf_16 = normalize_utf_16(string, normalize_cp1252: normalize_cp1252) if normalize_utf16
|
55
72
|
if found_utf_16
|
56
73
|
string.encode!('UTF-8')
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
raise ArgumentError, 'Could not normalize to utf8 due to invalid characters (probably CP1252)'
|
63
|
-
end
|
74
|
+
elsif !string.valid_encoding?
|
75
|
+
if normalize_cp1252
|
76
|
+
cp1252_to_utf_8(string)
|
77
|
+
else
|
78
|
+
raise ArgumentError, 'Could not normalize to utf8 due to invalid characters (probably CP1252)'
|
64
79
|
end
|
65
80
|
end
|
66
81
|
normalize_newlines(string) if normalize_newlines
|
data/lib/invoca/utils/hash.rb
CHANGED
data/lib/invoca/utils/version.rb
CHANGED
@@ -0,0 +1,84 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative '../test_helper'
|
4
|
+
|
5
|
+
class ExceptionsTest < Minitest::Test
|
6
|
+
context "exceptions" do
|
7
|
+
context ".retry_on_exception" do
|
8
|
+
should "default retries: to 1" do
|
9
|
+
times = 0
|
10
|
+
tries = []
|
11
|
+
result = Invoca::Utils.retry_on_exception(ArgumentError) do |try|
|
12
|
+
tries << try
|
13
|
+
times += 1
|
14
|
+
end
|
15
|
+
assert_equal 1, result
|
16
|
+
assert_equal [0], tries
|
17
|
+
end
|
18
|
+
|
19
|
+
context "when never raising an exception" do
|
20
|
+
should "return result" do
|
21
|
+
times = 0
|
22
|
+
tries = []
|
23
|
+
result = Invoca::Utils.retry_on_exception(ArgumentError, retries: 2) do |try|
|
24
|
+
tries << try
|
25
|
+
times += 1
|
26
|
+
try == 0 and raise ArgumentError, '!!!'
|
27
|
+
times
|
28
|
+
end
|
29
|
+
assert_equal 2, result
|
30
|
+
assert_equal [0, 1], tries
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
context "when always raising an exception" do
|
35
|
+
should "retry and finally raise" do
|
36
|
+
tries = []
|
37
|
+
assert_raises(ArgumentError, /!!! 2/) do
|
38
|
+
Invoca::Utils.retry_on_exception(ArgumentError, retries: 1) do |try|
|
39
|
+
tries << try
|
40
|
+
raise ArgumentError, "!!! #{try + 1}"
|
41
|
+
end
|
42
|
+
end
|
43
|
+
assert_equal [0, 1], tries
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
context "when raising but then succeeding" do
|
48
|
+
should "retry and finally return result" do
|
49
|
+
times = 0
|
50
|
+
result = Invoca::Utils.retry_on_exception(ArgumentError, retries: 1) do
|
51
|
+
times += 1
|
52
|
+
if times == 1
|
53
|
+
raise ArgumentError, "!!! #{times}"
|
54
|
+
else
|
55
|
+
times
|
56
|
+
end
|
57
|
+
end
|
58
|
+
assert_equal 2, result
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
context "when raising different exceptions (array notation) but then succeeding" do
|
63
|
+
should "retry and finally return result" do
|
64
|
+
times = 0
|
65
|
+
tries = []
|
66
|
+
result = Invoca::Utils.retry_on_exception([ArgumentError, RuntimeError], retries: 2) do |try|
|
67
|
+
tries << try
|
68
|
+
times += 1
|
69
|
+
case times
|
70
|
+
when 1
|
71
|
+
raise ArgumentError, "!!! #{times}"
|
72
|
+
when 2
|
73
|
+
raise RuntimeError, "!!! #{times}"
|
74
|
+
else
|
75
|
+
times
|
76
|
+
end
|
77
|
+
end
|
78
|
+
assert_equal 3, result
|
79
|
+
assert_equal [0, 1, 2], tries
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
@@ -221,6 +221,29 @@ class GuaranteedUTF8StringTest < Minitest::Test
|
|
221
221
|
assert_equal Encoding::UTF_8, encoded_string.encoding
|
222
222
|
end
|
223
223
|
end
|
224
|
+
|
225
|
+
context ".normalize_strings" do
|
226
|
+
should "walk json doc, replacing strings in: values, inside array elements, and hash keys and values" do
|
227
|
+
json_doc = {
|
228
|
+
'😹' => "\xE2\x9C\x93 laughing cat",
|
229
|
+
['😹'] => ["\xE2", "\xf0\x9f\x98\xb9", { "newline" => "\r\n" }],
|
230
|
+
'cp1252' => "\x91smart quotes\x92"
|
231
|
+
}
|
232
|
+
|
233
|
+
normalized_json = Invoca::Utils::GuaranteedUTF8String.normalize_all_strings(json_doc,
|
234
|
+
normalize_utf16: true,
|
235
|
+
normalize_cp1252: true,
|
236
|
+
normalize_newlines: true,
|
237
|
+
remove_utf8_bom: true,
|
238
|
+
replace_unicode_beyond_ffff: true)
|
239
|
+
|
240
|
+
assert_equal({
|
241
|
+
'~' => "✓ laughing cat",
|
242
|
+
['~'] => ["â", "~", { "newline" => "\n" }],
|
243
|
+
'cp1252' => "‘smart quotes’"
|
244
|
+
}, normalized_json)
|
245
|
+
end
|
246
|
+
end
|
224
247
|
end
|
225
248
|
|
226
249
|
context 'constructor' do
|
metadata
CHANGED
@@ -1,16 +1,16 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: invoca-utils
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Invoca development
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-06-09 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
|
-
description:
|
13
|
+
description:
|
14
14
|
email:
|
15
15
|
- development@invoca.com
|
16
16
|
executables: []
|
@@ -33,6 +33,7 @@ files:
|
|
33
33
|
- lib/invoca/utils/array.rb
|
34
34
|
- lib/invoca/utils/diff.rb
|
35
35
|
- lib/invoca/utils/enumerable.rb
|
36
|
+
- lib/invoca/utils/exceptions.rb
|
36
37
|
- lib/invoca/utils/guaranteed_utf8_string.rb
|
37
38
|
- lib/invoca/utils/hash.rb
|
38
39
|
- lib/invoca/utils/hash_with_indifferent_access.rb
|
@@ -48,6 +49,7 @@ files:
|
|
48
49
|
- test/test_helper.rb
|
49
50
|
- test/unit/array_test.rb
|
50
51
|
- test/unit/enumerable_test.rb
|
52
|
+
- test/unit/exceptions_test.rb
|
51
53
|
- test/unit/guaranteed_utf8_string_test.rb
|
52
54
|
- test/unit/hash_test.rb
|
53
55
|
- test/unit/hash_with_indifferent_access_test.rb
|
@@ -86,6 +88,7 @@ test_files:
|
|
86
88
|
- test/test_helper.rb
|
87
89
|
- test/unit/array_test.rb
|
88
90
|
- test/unit/enumerable_test.rb
|
91
|
+
- test/unit/exceptions_test.rb
|
89
92
|
- test/unit/guaranteed_utf8_string_test.rb
|
90
93
|
- test/unit/hash_test.rb
|
91
94
|
- test/unit/hash_with_indifferent_access_test.rb
|