format_parser 0.27.0 → 0.28.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/main.yml +104 -0
- data/CHANGELOG.md +3 -0
- data/format_parser.gemspec +1 -0
- data/lib/format_parser/version.rb +1 -1
- data/lib/remote_io.rb +7 -1
- data/spec/remote_fetching_spec.rb +11 -0
- data/spec/remote_io_spec.rb +38 -13
- metadata +20 -6
- data/.travis.yml +0 -12
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: d2212d92c3c0ca7aab0e60c9aa6559d0508ec2b8b39e4b2067990d97a7d78327
|
|
4
|
+
data.tar.gz: 27736f9d52ea1e73a147d96d38608e268040648e61d2b913ca38f164f2d2d0b6
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 5f63f74444df32d016fc25dc5c3731671690185b7ecf80ffc06da350b37d82b566f451508091245fec255c816121a115286dd04fb626aa4c1b694836fd645d97
|
|
7
|
+
data.tar.gz: ee1917287bb7c50bd71d9a7756321e30fbc73aa5e3a4f42e4844eb9494dfe77e89a9a26622732772c9b29c4ea673f3401324d2e6bf67e5bdaa5dac991d12f821
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on: [push,pull_request]
|
|
4
|
+
|
|
5
|
+
env:
|
|
6
|
+
BUNDLE_PATH: vendor/bundle
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
lint:
|
|
10
|
+
name: Code Style
|
|
11
|
+
runs-on: ubuntu-18.04
|
|
12
|
+
if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
|
|
13
|
+
strategy:
|
|
14
|
+
matrix:
|
|
15
|
+
ruby:
|
|
16
|
+
- 2.7
|
|
17
|
+
- 2.6
|
|
18
|
+
- 2.5
|
|
19
|
+
- 2.4
|
|
20
|
+
- 2.3
|
|
21
|
+
- 2.2
|
|
22
|
+
- jruby
|
|
23
|
+
steps:
|
|
24
|
+
- name: Checkout
|
|
25
|
+
uses: actions/checkout@v2
|
|
26
|
+
- name: Setup Ruby
|
|
27
|
+
uses: ruby/setup-ruby@v1
|
|
28
|
+
with:
|
|
29
|
+
ruby-version: ${{ matrix.ruby }}
|
|
30
|
+
- name: Gemfile Cache
|
|
31
|
+
uses: actions/cache@v2
|
|
32
|
+
with:
|
|
33
|
+
path: Gemfile.lock
|
|
34
|
+
key: ${{ runner.os }}-gemlock-${{ matrix.ruby }}-${{ hashFiles('Gemfile', 'format_parser.gemspec') }}
|
|
35
|
+
restore-keys: |
|
|
36
|
+
${{ runner.os }}-gemlock-${{ matrix.ruby }}-
|
|
37
|
+
- name: Bundle Cache
|
|
38
|
+
id: cache-gems
|
|
39
|
+
uses: actions/cache@v2
|
|
40
|
+
with:
|
|
41
|
+
path: vendor/bundle
|
|
42
|
+
key: ${{ runner.os }}-gems-${{ matrix.ruby }}-${{ hashFiles('Gemfile', 'Gemfile.lock', 'format_parser.gemspec') }}
|
|
43
|
+
restore-keys: |
|
|
44
|
+
${{ runner.os }}-gems-${{ matrix.ruby }}-
|
|
45
|
+
${{ runner.os }}-gems-
|
|
46
|
+
- name: Bundle Install
|
|
47
|
+
if: steps.cache-gems.outputs.cache-hit != 'true'
|
|
48
|
+
run: bundle install --jobs 4 --retry 3
|
|
49
|
+
- name: Rubocop Cache
|
|
50
|
+
uses: actions/cache@v2
|
|
51
|
+
with:
|
|
52
|
+
path: ~/.cache/rubocop_cache
|
|
53
|
+
key: ${{ runner.os }}-rubocop-${{ hashFiles('.rubocop.yml') }}
|
|
54
|
+
restore-keys: |
|
|
55
|
+
${{ runner.os }}-rubocop-
|
|
56
|
+
- name: Rubocop
|
|
57
|
+
run: bundle exec rubocop
|
|
58
|
+
test:
|
|
59
|
+
name: Specs
|
|
60
|
+
runs-on: ubuntu-18.04
|
|
61
|
+
if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
|
|
62
|
+
strategy:
|
|
63
|
+
matrix:
|
|
64
|
+
ruby:
|
|
65
|
+
- 2.7
|
|
66
|
+
- 2.6
|
|
67
|
+
- 2.5
|
|
68
|
+
- 2.4
|
|
69
|
+
- 2.3
|
|
70
|
+
- 2.2
|
|
71
|
+
- jruby
|
|
72
|
+
experimental: [false]
|
|
73
|
+
include:
|
|
74
|
+
- ruby: 3.0
|
|
75
|
+
experimental: true
|
|
76
|
+
steps:
|
|
77
|
+
- name: Checkout
|
|
78
|
+
uses: actions/checkout@v2
|
|
79
|
+
- name: Setup Ruby
|
|
80
|
+
uses: ruby/setup-ruby@v1
|
|
81
|
+
with:
|
|
82
|
+
ruby-version: ${{ matrix.ruby }}
|
|
83
|
+
- name: Gemfile Cache
|
|
84
|
+
uses: actions/cache@v2
|
|
85
|
+
with:
|
|
86
|
+
path: Gemfile.lock
|
|
87
|
+
key: ${{ runner.os }}-gemlock-${{ matrix.ruby }}-${{ hashFiles('Gemfile', 'format_parser.gemspec') }}
|
|
88
|
+
restore-keys: |
|
|
89
|
+
${{ runner.os }}-gemlock-${{ matrix.ruby }}-
|
|
90
|
+
- name: Bundle Cache
|
|
91
|
+
id: cache-gems
|
|
92
|
+
uses: actions/cache@v2
|
|
93
|
+
with:
|
|
94
|
+
path: vendor/bundle
|
|
95
|
+
key: ${{ runner.os }}-gems-${{ matrix.ruby }}-${{ hashFiles('Gemfile', 'Gemfile.lock', 'format_parser.gemspec') }}
|
|
96
|
+
restore-keys: |
|
|
97
|
+
${{ runner.os }}-gems-${{ matrix.ruby }}-
|
|
98
|
+
${{ runner.os }}-gems-
|
|
99
|
+
- name: Bundle Install
|
|
100
|
+
if: steps.cache-gems.outputs.cache-hit != 'true'
|
|
101
|
+
run: bundle install --jobs 4 --retry 3
|
|
102
|
+
- name: RSpec
|
|
103
|
+
continue-on-error: ${{ matrix.experimental }}
|
|
104
|
+
run: bundle exec rake parallel:spec
|
data/CHANGELOG.md
CHANGED
data/format_parser.gemspec
CHANGED
|
@@ -34,6 +34,7 @@ Gem::Specification.new do |spec|
|
|
|
34
34
|
spec.add_dependency 'exifr', '~> 1', '>= 1.3.8'
|
|
35
35
|
spec.add_dependency 'id3tag', '~> 0.14'
|
|
36
36
|
spec.add_dependency 'faraday', '~> 0.13'
|
|
37
|
+
spec.add_dependency 'faraday_middleware', '~> 0.14'
|
|
37
38
|
spec.add_dependency 'measurometer', '~> 1'
|
|
38
39
|
|
|
39
40
|
spec.add_development_dependency 'rspec', '~> 3.0'
|
data/lib/remote_io.rb
CHANGED
|
@@ -26,6 +26,7 @@ class FormatParser::RemoteIO
|
|
|
26
26
|
# @param uri[URI, String] the remote URL to obtain
|
|
27
27
|
def initialize(uri)
|
|
28
28
|
require 'faraday'
|
|
29
|
+
require 'faraday_middleware/response/follow_redirects'
|
|
29
30
|
@uri = uri
|
|
30
31
|
@pos = 0
|
|
31
32
|
@remote_size = false
|
|
@@ -78,7 +79,12 @@ class FormatParser::RemoteIO
|
|
|
78
79
|
# We use a GET and not a HEAD request followed by a GET because
|
|
79
80
|
# S3 does not allow HEAD requests if you only presigned your URL for GETs, so we
|
|
80
81
|
# combine the first GET of a segment and retrieving the size of the resource
|
|
81
|
-
|
|
82
|
+
conn = Faraday.new do |faraday|
|
|
83
|
+
faraday.use FaradayMiddleware::FollowRedirects
|
|
84
|
+
# we still need the default adapter, more details: https://blog.thecodewhisperer.com/permalink/losing-time-to-faraday
|
|
85
|
+
faraday.adapter Faraday.default_adapter
|
|
86
|
+
end
|
|
87
|
+
response = conn.get(@uri, nil, range: 'bytes=%d-%d' % [range.begin, range.end])
|
|
82
88
|
|
|
83
89
|
case response.status
|
|
84
90
|
when 200, 206
|
|
@@ -15,6 +15,10 @@ describe 'Fetching data from HTTP remotes' do
|
|
|
15
15
|
}
|
|
16
16
|
@server = WEBrick::HTTPServer.new(options)
|
|
17
17
|
@server.mount '/', WEBrick::HTTPServlet::FileHandler, fixtures_dir
|
|
18
|
+
@server.mount_proc '/redirect' do |req, res|
|
|
19
|
+
res.status = 302
|
|
20
|
+
res.header['Location'] = req.path.sub('/redirect', '')
|
|
21
|
+
end
|
|
18
22
|
trap('INT') { @server.stop }
|
|
19
23
|
@server_thread = Thread.new { @server.start }
|
|
20
24
|
end
|
|
@@ -91,6 +95,13 @@ describe 'Fetching data from HTTP remotes' do
|
|
|
91
95
|
end
|
|
92
96
|
end
|
|
93
97
|
|
|
98
|
+
context 'when the server responds with a redirect' do
|
|
99
|
+
it 'follows the redirect' do
|
|
100
|
+
file_information = FormatParser.parse_http('http://localhost:9399/redirect/TIFF/test.tif')
|
|
101
|
+
expect(file_information.format).to eq(:tif)
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
|
|
94
105
|
after(:all) do
|
|
95
106
|
@server.stop
|
|
96
107
|
@server_thread.join(0.5)
|
data/spec/remote_io_spec.rb
CHANGED
|
@@ -7,7 +7,9 @@ describe FormatParser::RemoteIO do
|
|
|
7
7
|
rio = described_class.new('https://images.invalid/img.jpg')
|
|
8
8
|
|
|
9
9
|
fake_resp = double(headers: {'Content-Range' => '10-109/2577'}, status: 206, body: 'This is the response')
|
|
10
|
-
|
|
10
|
+
faraday_conn = instance_double(Faraday::Connection, get: fake_resp)
|
|
11
|
+
allow(Faraday).to receive(:new).and_return(faraday_conn)
|
|
12
|
+
expect(faraday_conn).to receive(:get).with('https://images.invalid/img.jpg', nil, range: 'bytes=10-109')
|
|
11
13
|
|
|
12
14
|
rio.seek(10)
|
|
13
15
|
read_result = rio.read(100)
|
|
@@ -18,7 +20,9 @@ describe FormatParser::RemoteIO do
|
|
|
18
20
|
rio = described_class.new('https://images.invalid/img.jpg')
|
|
19
21
|
|
|
20
22
|
fake_resp = double(headers: {'Content-Range' => '10-109/2577'}, status: 200, body: 'This is the response')
|
|
21
|
-
|
|
23
|
+
faraday_conn = instance_double(Faraday::Connection, get: fake_resp)
|
|
24
|
+
allow(Faraday).to receive(:new).and_return(faraday_conn)
|
|
25
|
+
expect(faraday_conn).to receive(:get).with('https://images.invalid/img.jpg', nil, range: 'bytes=10-109')
|
|
22
26
|
|
|
23
27
|
rio.seek(10)
|
|
24
28
|
read_result = rio.read(100)
|
|
@@ -29,7 +33,9 @@ describe FormatParser::RemoteIO do
|
|
|
29
33
|
rio = described_class.new('https://images.invalid/img.jpg')
|
|
30
34
|
|
|
31
35
|
fake_resp = double(headers: {}, status: 403, body: 'Please log in')
|
|
32
|
-
|
|
36
|
+
faraday_conn = instance_double(Faraday::Connection, get: fake_resp)
|
|
37
|
+
allow(Faraday).to receive(:new).and_return(faraday_conn)
|
|
38
|
+
expect(faraday_conn).to receive(:get).with('https://images.invalid/img.jpg', nil, range: 'bytes=100-199')
|
|
33
39
|
|
|
34
40
|
rio.seek(100)
|
|
35
41
|
expect { rio.read(100) }.to raise_error(/replied with a 403 and refused/)
|
|
@@ -39,7 +45,9 @@ describe FormatParser::RemoteIO do
|
|
|
39
45
|
rio = described_class.new('https://images.invalid/img.jpg')
|
|
40
46
|
|
|
41
47
|
fake_resp = double(headers: {}, status: 416, body: 'You stepped off the ledge of the range')
|
|
42
|
-
|
|
48
|
+
faraday_conn = instance_double(Faraday::Connection, get: fake_resp)
|
|
49
|
+
allow(Faraday).to receive(:new).and_return(faraday_conn)
|
|
50
|
+
expect(faraday_conn).to receive(:get).with('https://images.invalid/img.jpg', nil, range: 'bytes=100-199')
|
|
43
51
|
|
|
44
52
|
rio.seek(100)
|
|
45
53
|
expect(rio.read(100)).to be_nil
|
|
@@ -49,7 +57,9 @@ describe FormatParser::RemoteIO do
|
|
|
49
57
|
rio = described_class.new('https://images.invalid/img.jpg')
|
|
50
58
|
|
|
51
59
|
fake_resp = double(headers: {}, status: 403, body: 'Please log in')
|
|
52
|
-
|
|
60
|
+
faraday_conn = instance_double(Faraday::Connection, get: fake_resp)
|
|
61
|
+
allow(Faraday).to receive(:new).and_return(faraday_conn)
|
|
62
|
+
expect(faraday_conn).to receive(:get).with('https://images.invalid/img.jpg', nil, range: 'bytes=100-199')
|
|
53
63
|
|
|
54
64
|
rio.seek(100)
|
|
55
65
|
# rubocop: disable Lint/AmbiguousBlockAssociation
|
|
@@ -60,7 +70,9 @@ describe FormatParser::RemoteIO do
|
|
|
60
70
|
rio = described_class.new('https://images.invalid/img.jpg')
|
|
61
71
|
|
|
62
72
|
fake_resp = double(headers: {}, status: 416, body: 'You jumped off the end of the file maam')
|
|
63
|
-
|
|
73
|
+
faraday_conn = instance_double(Faraday::Connection, get: fake_resp)
|
|
74
|
+
allow(Faraday).to receive(:new).and_return(faraday_conn)
|
|
75
|
+
expect(faraday_conn).to receive(:get).with('https://images.invalid/img.jpg', nil, range: 'bytes=100-199')
|
|
64
76
|
|
|
65
77
|
rio.seek(100)
|
|
66
78
|
expect(rio.read(100)).to be_nil
|
|
@@ -69,15 +81,24 @@ describe FormatParser::RemoteIO do
|
|
|
69
81
|
it 'does not overwrite size when the range cannot be satisfied and the response is 416' do
|
|
70
82
|
rio = described_class.new('https://images.invalid/img.jpg')
|
|
71
83
|
|
|
72
|
-
|
|
73
|
-
|
|
84
|
+
fake_resp1 = double(headers: {'Content-Range' => 'bytes 0-0/13'}, status: 206, body: 'a')
|
|
85
|
+
fake_resp2 = double(headers: {}, status: 416, body: 'You jumped off the end of the file maam')
|
|
86
|
+
|
|
87
|
+
faraday_conn = instance_double(Faraday::Connection)
|
|
88
|
+
allow(Faraday).to receive(:new).and_return(faraday_conn)
|
|
89
|
+
expect(faraday_conn).to receive(:get)
|
|
90
|
+
.with('https://images.invalid/img.jpg', nil, range: 'bytes=0-0')
|
|
91
|
+
.ordered
|
|
92
|
+
.and_return(fake_resp1)
|
|
93
|
+
expect(faraday_conn).to receive(:get)
|
|
94
|
+
.with('https://images.invalid/img.jpg', nil, range: 'bytes=100-199')
|
|
95
|
+
.ordered
|
|
96
|
+
.and_return(fake_resp2)
|
|
97
|
+
|
|
74
98
|
rio.read(1)
|
|
75
99
|
|
|
76
100
|
expect(rio.size).to eq(13)
|
|
77
101
|
|
|
78
|
-
fake_resp = double(headers: {}, status: 416, body: 'You jumped off the end of the file maam')
|
|
79
|
-
expect(Faraday).to receive(:get).with('https://images.invalid/img.jpg', nil, range: 'bytes=100-199').and_return(fake_resp)
|
|
80
|
-
|
|
81
102
|
rio.seek(100)
|
|
82
103
|
expect(rio.read(100)).to be_nil
|
|
83
104
|
|
|
@@ -88,7 +109,9 @@ describe FormatParser::RemoteIO do
|
|
|
88
109
|
rio = described_class.new('https://images.invalid/img.jpg')
|
|
89
110
|
|
|
90
111
|
fake_resp = double(headers: {}, status: 502, body: 'Guru meditation')
|
|
91
|
-
|
|
112
|
+
faraday_conn = instance_double(Faraday::Connection, get: fake_resp)
|
|
113
|
+
allow(Faraday).to receive(:new).and_return(faraday_conn)
|
|
114
|
+
expect(faraday_conn).to receive(:get).with('https://images.invalid/img.jpg', nil, range: 'bytes=100-199')
|
|
92
115
|
|
|
93
116
|
rio.seek(100)
|
|
94
117
|
expect { rio.read(100) }.to raise_error(/replied with a 502 and we might want to retry/)
|
|
@@ -100,7 +123,9 @@ describe FormatParser::RemoteIO do
|
|
|
100
123
|
expect(rio.pos).to eq(0)
|
|
101
124
|
|
|
102
125
|
fake_resp = double(headers: {'Content-Range' => 'bytes 0-0/13'}, status: 206, body: 'a')
|
|
103
|
-
|
|
126
|
+
faraday_conn = instance_double(Faraday::Connection, get: fake_resp)
|
|
127
|
+
allow(Faraday).to receive(:new).and_return(faraday_conn)
|
|
128
|
+
expect(faraday_conn).to receive(:get).with('https://images.invalid/img.jpg', nil, range: 'bytes=0-0')
|
|
104
129
|
rio.read(1)
|
|
105
130
|
|
|
106
131
|
expect(rio.pos).to eq(1)
|
metadata
CHANGED
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: format_parser
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.28.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Noah Berman
|
|
8
8
|
- Julik Tarkhanov
|
|
9
|
-
autorequire:
|
|
9
|
+
autorequire:
|
|
10
10
|
bindir: exe
|
|
11
11
|
cert_chain: []
|
|
12
|
-
date: 2021-
|
|
12
|
+
date: 2021-02-17 00:00:00.000000000 Z
|
|
13
13
|
dependencies:
|
|
14
14
|
- !ruby/object:Gem::Dependency
|
|
15
15
|
name: ks
|
|
@@ -73,6 +73,20 @@ dependencies:
|
|
|
73
73
|
- - "~>"
|
|
74
74
|
- !ruby/object:Gem::Version
|
|
75
75
|
version: '0.13'
|
|
76
|
+
- !ruby/object:Gem::Dependency
|
|
77
|
+
name: faraday_middleware
|
|
78
|
+
requirement: !ruby/object:Gem::Requirement
|
|
79
|
+
requirements:
|
|
80
|
+
- - "~>"
|
|
81
|
+
- !ruby/object:Gem::Version
|
|
82
|
+
version: '0.14'
|
|
83
|
+
type: :runtime
|
|
84
|
+
prerelease: false
|
|
85
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
86
|
+
requirements:
|
|
87
|
+
- - "~>"
|
|
88
|
+
- !ruby/object:Gem::Version
|
|
89
|
+
version: '0.14'
|
|
76
90
|
- !ruby/object:Gem::Dependency
|
|
77
91
|
name: measurometer
|
|
78
92
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -183,10 +197,10 @@ executables:
|
|
|
183
197
|
extensions: []
|
|
184
198
|
extra_rdoc_files: []
|
|
185
199
|
files:
|
|
200
|
+
- ".github/workflows/main.yml"
|
|
186
201
|
- ".gitignore"
|
|
187
202
|
- ".rspec"
|
|
188
203
|
- ".rubocop.yml"
|
|
189
|
-
- ".travis.yml"
|
|
190
204
|
- CHANGELOG.md
|
|
191
205
|
- CODE_OF_CONDUCT.md
|
|
192
206
|
- CONTRIBUTING.md
|
|
@@ -280,7 +294,7 @@ licenses:
|
|
|
280
294
|
- MIT (Hippocratic)
|
|
281
295
|
metadata:
|
|
282
296
|
allowed_push_host: https://rubygems.org
|
|
283
|
-
post_install_message:
|
|
297
|
+
post_install_message:
|
|
284
298
|
rdoc_options: []
|
|
285
299
|
require_paths:
|
|
286
300
|
- lib
|
|
@@ -296,7 +310,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
296
310
|
version: '0'
|
|
297
311
|
requirements: []
|
|
298
312
|
rubygems_version: 3.0.3
|
|
299
|
-
signing_key:
|
|
313
|
+
signing_key:
|
|
300
314
|
specification_version: 4
|
|
301
315
|
summary: A library for efficient parsing of file metadata
|
|
302
316
|
test_files: []
|