format_parser 0.27.0 → 0.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/main.yml +104 -0
- data/CHANGELOG.md +3 -0
- data/format_parser.gemspec +1 -0
- data/lib/format_parser/version.rb +1 -1
- data/lib/remote_io.rb +7 -1
- data/spec/remote_fetching_spec.rb +11 -0
- data/spec/remote_io_spec.rb +38 -13
- metadata +20 -6
- data/.travis.yml +0 -12
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: d2212d92c3c0ca7aab0e60c9aa6559d0508ec2b8b39e4b2067990d97a7d78327
|
|
4
|
+
data.tar.gz: 27736f9d52ea1e73a147d96d38608e268040648e61d2b913ca38f164f2d2d0b6
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 5f63f74444df32d016fc25dc5c3731671690185b7ecf80ffc06da350b37d82b566f451508091245fec255c816121a115286dd04fb626aa4c1b694836fd645d97
|
|
7
|
+
data.tar.gz: ee1917287bb7c50bd71d9a7756321e30fbc73aa5e3a4f42e4844eb9494dfe77e89a9a26622732772c9b29c4ea673f3401324d2e6bf67e5bdaa5dac991d12f821
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on: [push,pull_request]
|
|
4
|
+
|
|
5
|
+
env:
|
|
6
|
+
BUNDLE_PATH: vendor/bundle
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
lint:
|
|
10
|
+
name: Code Style
|
|
11
|
+
runs-on: ubuntu-18.04
|
|
12
|
+
if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
|
|
13
|
+
strategy:
|
|
14
|
+
matrix:
|
|
15
|
+
ruby:
|
|
16
|
+
- 2.7
|
|
17
|
+
- 2.6
|
|
18
|
+
- 2.5
|
|
19
|
+
- 2.4
|
|
20
|
+
- 2.3
|
|
21
|
+
- 2.2
|
|
22
|
+
- jruby
|
|
23
|
+
steps:
|
|
24
|
+
- name: Checkout
|
|
25
|
+
uses: actions/checkout@v2
|
|
26
|
+
- name: Setup Ruby
|
|
27
|
+
uses: ruby/setup-ruby@v1
|
|
28
|
+
with:
|
|
29
|
+
ruby-version: ${{ matrix.ruby }}
|
|
30
|
+
- name: Gemfile Cache
|
|
31
|
+
uses: actions/cache@v2
|
|
32
|
+
with:
|
|
33
|
+
path: Gemfile.lock
|
|
34
|
+
key: ${{ runner.os }}-gemlock-${{ matrix.ruby }}-${{ hashFiles('Gemfile', 'format_parser.gemspec') }}
|
|
35
|
+
restore-keys: |
|
|
36
|
+
${{ runner.os }}-gemlock-${{ matrix.ruby }}-
|
|
37
|
+
- name: Bundle Cache
|
|
38
|
+
id: cache-gems
|
|
39
|
+
uses: actions/cache@v2
|
|
40
|
+
with:
|
|
41
|
+
path: vendor/bundle
|
|
42
|
+
key: ${{ runner.os }}-gems-${{ matrix.ruby }}-${{ hashFiles('Gemfile', 'Gemfile.lock', 'format_parser.gemspec') }}
|
|
43
|
+
restore-keys: |
|
|
44
|
+
${{ runner.os }}-gems-${{ matrix.ruby }}-
|
|
45
|
+
${{ runner.os }}-gems-
|
|
46
|
+
- name: Bundle Install
|
|
47
|
+
if: steps.cache-gems.outputs.cache-hit != 'true'
|
|
48
|
+
run: bundle install --jobs 4 --retry 3
|
|
49
|
+
- name: Rubocop Cache
|
|
50
|
+
uses: actions/cache@v2
|
|
51
|
+
with:
|
|
52
|
+
path: ~/.cache/rubocop_cache
|
|
53
|
+
key: ${{ runner.os }}-rubocop-${{ hashFiles('.rubocop.yml') }}
|
|
54
|
+
restore-keys: |
|
|
55
|
+
${{ runner.os }}-rubocop-
|
|
56
|
+
- name: Rubocop
|
|
57
|
+
run: bundle exec rubocop
|
|
58
|
+
test:
|
|
59
|
+
name: Specs
|
|
60
|
+
runs-on: ubuntu-18.04
|
|
61
|
+
if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
|
|
62
|
+
strategy:
|
|
63
|
+
matrix:
|
|
64
|
+
ruby:
|
|
65
|
+
- 2.7
|
|
66
|
+
- 2.6
|
|
67
|
+
- 2.5
|
|
68
|
+
- 2.4
|
|
69
|
+
- 2.3
|
|
70
|
+
- 2.2
|
|
71
|
+
- jruby
|
|
72
|
+
experimental: [false]
|
|
73
|
+
include:
|
|
74
|
+
- ruby: 3.0
|
|
75
|
+
experimental: true
|
|
76
|
+
steps:
|
|
77
|
+
- name: Checkout
|
|
78
|
+
uses: actions/checkout@v2
|
|
79
|
+
- name: Setup Ruby
|
|
80
|
+
uses: ruby/setup-ruby@v1
|
|
81
|
+
with:
|
|
82
|
+
ruby-version: ${{ matrix.ruby }}
|
|
83
|
+
- name: Gemfile Cache
|
|
84
|
+
uses: actions/cache@v2
|
|
85
|
+
with:
|
|
86
|
+
path: Gemfile.lock
|
|
87
|
+
key: ${{ runner.os }}-gemlock-${{ matrix.ruby }}-${{ hashFiles('Gemfile', 'format_parser.gemspec') }}
|
|
88
|
+
restore-keys: |
|
|
89
|
+
${{ runner.os }}-gemlock-${{ matrix.ruby }}-
|
|
90
|
+
- name: Bundle Cache
|
|
91
|
+
id: cache-gems
|
|
92
|
+
uses: actions/cache@v2
|
|
93
|
+
with:
|
|
94
|
+
path: vendor/bundle
|
|
95
|
+
key: ${{ runner.os }}-gems-${{ matrix.ruby }}-${{ hashFiles('Gemfile', 'Gemfile.lock', 'format_parser.gemspec') }}
|
|
96
|
+
restore-keys: |
|
|
97
|
+
${{ runner.os }}-gems-${{ matrix.ruby }}-
|
|
98
|
+
${{ runner.os }}-gems-
|
|
99
|
+
- name: Bundle Install
|
|
100
|
+
if: steps.cache-gems.outputs.cache-hit != 'true'
|
|
101
|
+
run: bundle install --jobs 4 --retry 3
|
|
102
|
+
- name: RSpec
|
|
103
|
+
continue-on-error: ${{ matrix.experimental }}
|
|
104
|
+
run: bundle exec rake parallel:spec
|
data/CHANGELOG.md
CHANGED
data/format_parser.gemspec
CHANGED
|
@@ -34,6 +34,7 @@ Gem::Specification.new do |spec|
|
|
|
34
34
|
spec.add_dependency 'exifr', '~> 1', '>= 1.3.8'
|
|
35
35
|
spec.add_dependency 'id3tag', '~> 0.14'
|
|
36
36
|
spec.add_dependency 'faraday', '~> 0.13'
|
|
37
|
+
spec.add_dependency 'faraday_middleware', '~> 0.14'
|
|
37
38
|
spec.add_dependency 'measurometer', '~> 1'
|
|
38
39
|
|
|
39
40
|
spec.add_development_dependency 'rspec', '~> 3.0'
|
data/lib/remote_io.rb
CHANGED
|
@@ -26,6 +26,7 @@ class FormatParser::RemoteIO
|
|
|
26
26
|
# @param uri[URI, String] the remote URL to obtain
|
|
27
27
|
def initialize(uri)
|
|
28
28
|
require 'faraday'
|
|
29
|
+
require 'faraday_middleware/response/follow_redirects'
|
|
29
30
|
@uri = uri
|
|
30
31
|
@pos = 0
|
|
31
32
|
@remote_size = false
|
|
@@ -78,7 +79,12 @@ class FormatParser::RemoteIO
|
|
|
78
79
|
# We use a GET and not a HEAD request followed by a GET because
|
|
79
80
|
# S3 does not allow HEAD requests if you only presigned your URL for GETs, so we
|
|
80
81
|
# combine the first GET of a segment and retrieving the size of the resource
|
|
81
|
-
|
|
82
|
+
conn = Faraday.new do |faraday|
|
|
83
|
+
faraday.use FaradayMiddleware::FollowRedirects
|
|
84
|
+
# we still need the default adapter, more details: https://blog.thecodewhisperer.com/permalink/losing-time-to-faraday
|
|
85
|
+
faraday.adapter Faraday.default_adapter
|
|
86
|
+
end
|
|
87
|
+
response = conn.get(@uri, nil, range: 'bytes=%d-%d' % [range.begin, range.end])
|
|
82
88
|
|
|
83
89
|
case response.status
|
|
84
90
|
when 200, 206
|
|
@@ -15,6 +15,10 @@ describe 'Fetching data from HTTP remotes' do
|
|
|
15
15
|
}
|
|
16
16
|
@server = WEBrick::HTTPServer.new(options)
|
|
17
17
|
@server.mount '/', WEBrick::HTTPServlet::FileHandler, fixtures_dir
|
|
18
|
+
@server.mount_proc '/redirect' do |req, res|
|
|
19
|
+
res.status = 302
|
|
20
|
+
res.header['Location'] = req.path.sub('/redirect', '')
|
|
21
|
+
end
|
|
18
22
|
trap('INT') { @server.stop }
|
|
19
23
|
@server_thread = Thread.new { @server.start }
|
|
20
24
|
end
|
|
@@ -91,6 +95,13 @@ describe 'Fetching data from HTTP remotes' do
|
|
|
91
95
|
end
|
|
92
96
|
end
|
|
93
97
|
|
|
98
|
+
context 'when the server responds with a redirect' do
|
|
99
|
+
it 'follows the redirect' do
|
|
100
|
+
file_information = FormatParser.parse_http('http://localhost:9399/redirect/TIFF/test.tif')
|
|
101
|
+
expect(file_information.format).to eq(:tif)
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
|
|
94
105
|
after(:all) do
|
|
95
106
|
@server.stop
|
|
96
107
|
@server_thread.join(0.5)
|
data/spec/remote_io_spec.rb
CHANGED
|
@@ -7,7 +7,9 @@ describe FormatParser::RemoteIO do
|
|
|
7
7
|
rio = described_class.new('https://images.invalid/img.jpg')
|
|
8
8
|
|
|
9
9
|
fake_resp = double(headers: {'Content-Range' => '10-109/2577'}, status: 206, body: 'This is the response')
|
|
10
|
-
|
|
10
|
+
faraday_conn = instance_double(Faraday::Connection, get: fake_resp)
|
|
11
|
+
allow(Faraday).to receive(:new).and_return(faraday_conn)
|
|
12
|
+
expect(faraday_conn).to receive(:get).with('https://images.invalid/img.jpg', nil, range: 'bytes=10-109')
|
|
11
13
|
|
|
12
14
|
rio.seek(10)
|
|
13
15
|
read_result = rio.read(100)
|
|
@@ -18,7 +20,9 @@ describe FormatParser::RemoteIO do
|
|
|
18
20
|
rio = described_class.new('https://images.invalid/img.jpg')
|
|
19
21
|
|
|
20
22
|
fake_resp = double(headers: {'Content-Range' => '10-109/2577'}, status: 200, body: 'This is the response')
|
|
21
|
-
|
|
23
|
+
faraday_conn = instance_double(Faraday::Connection, get: fake_resp)
|
|
24
|
+
allow(Faraday).to receive(:new).and_return(faraday_conn)
|
|
25
|
+
expect(faraday_conn).to receive(:get).with('https://images.invalid/img.jpg', nil, range: 'bytes=10-109')
|
|
22
26
|
|
|
23
27
|
rio.seek(10)
|
|
24
28
|
read_result = rio.read(100)
|
|
@@ -29,7 +33,9 @@ describe FormatParser::RemoteIO do
|
|
|
29
33
|
rio = described_class.new('https://images.invalid/img.jpg')
|
|
30
34
|
|
|
31
35
|
fake_resp = double(headers: {}, status: 403, body: 'Please log in')
|
|
32
|
-
|
|
36
|
+
faraday_conn = instance_double(Faraday::Connection, get: fake_resp)
|
|
37
|
+
allow(Faraday).to receive(:new).and_return(faraday_conn)
|
|
38
|
+
expect(faraday_conn).to receive(:get).with('https://images.invalid/img.jpg', nil, range: 'bytes=100-199')
|
|
33
39
|
|
|
34
40
|
rio.seek(100)
|
|
35
41
|
expect { rio.read(100) }.to raise_error(/replied with a 403 and refused/)
|
|
@@ -39,7 +45,9 @@ describe FormatParser::RemoteIO do
|
|
|
39
45
|
rio = described_class.new('https://images.invalid/img.jpg')
|
|
40
46
|
|
|
41
47
|
fake_resp = double(headers: {}, status: 416, body: 'You stepped off the ledge of the range')
|
|
42
|
-
|
|
48
|
+
faraday_conn = instance_double(Faraday::Connection, get: fake_resp)
|
|
49
|
+
allow(Faraday).to receive(:new).and_return(faraday_conn)
|
|
50
|
+
expect(faraday_conn).to receive(:get).with('https://images.invalid/img.jpg', nil, range: 'bytes=100-199')
|
|
43
51
|
|
|
44
52
|
rio.seek(100)
|
|
45
53
|
expect(rio.read(100)).to be_nil
|
|
@@ -49,7 +57,9 @@ describe FormatParser::RemoteIO do
|
|
|
49
57
|
rio = described_class.new('https://images.invalid/img.jpg')
|
|
50
58
|
|
|
51
59
|
fake_resp = double(headers: {}, status: 403, body: 'Please log in')
|
|
52
|
-
|
|
60
|
+
faraday_conn = instance_double(Faraday::Connection, get: fake_resp)
|
|
61
|
+
allow(Faraday).to receive(:new).and_return(faraday_conn)
|
|
62
|
+
expect(faraday_conn).to receive(:get).with('https://images.invalid/img.jpg', nil, range: 'bytes=100-199')
|
|
53
63
|
|
|
54
64
|
rio.seek(100)
|
|
55
65
|
# rubocop: disable Lint/AmbiguousBlockAssociation
|
|
@@ -60,7 +70,9 @@ describe FormatParser::RemoteIO do
|
|
|
60
70
|
rio = described_class.new('https://images.invalid/img.jpg')
|
|
61
71
|
|
|
62
72
|
fake_resp = double(headers: {}, status: 416, body: 'You jumped off the end of the file maam')
|
|
63
|
-
|
|
73
|
+
faraday_conn = instance_double(Faraday::Connection, get: fake_resp)
|
|
74
|
+
allow(Faraday).to receive(:new).and_return(faraday_conn)
|
|
75
|
+
expect(faraday_conn).to receive(:get).with('https://images.invalid/img.jpg', nil, range: 'bytes=100-199')
|
|
64
76
|
|
|
65
77
|
rio.seek(100)
|
|
66
78
|
expect(rio.read(100)).to be_nil
|
|
@@ -69,15 +81,24 @@ describe FormatParser::RemoteIO do
|
|
|
69
81
|
it 'does not overwrite size when the range cannot be satisfied and the response is 416' do
|
|
70
82
|
rio = described_class.new('https://images.invalid/img.jpg')
|
|
71
83
|
|
|
72
|
-
|
|
73
|
-
|
|
84
|
+
fake_resp1 = double(headers: {'Content-Range' => 'bytes 0-0/13'}, status: 206, body: 'a')
|
|
85
|
+
fake_resp2 = double(headers: {}, status: 416, body: 'You jumped off the end of the file maam')
|
|
86
|
+
|
|
87
|
+
faraday_conn = instance_double(Faraday::Connection)
|
|
88
|
+
allow(Faraday).to receive(:new).and_return(faraday_conn)
|
|
89
|
+
expect(faraday_conn).to receive(:get)
|
|
90
|
+
.with('https://images.invalid/img.jpg', nil, range: 'bytes=0-0')
|
|
91
|
+
.ordered
|
|
92
|
+
.and_return(fake_resp1)
|
|
93
|
+
expect(faraday_conn).to receive(:get)
|
|
94
|
+
.with('https://images.invalid/img.jpg', nil, range: 'bytes=100-199')
|
|
95
|
+
.ordered
|
|
96
|
+
.and_return(fake_resp2)
|
|
97
|
+
|
|
74
98
|
rio.read(1)
|
|
75
99
|
|
|
76
100
|
expect(rio.size).to eq(13)
|
|
77
101
|
|
|
78
|
-
fake_resp = double(headers: {}, status: 416, body: 'You jumped off the end of the file maam')
|
|
79
|
-
expect(Faraday).to receive(:get).with('https://images.invalid/img.jpg', nil, range: 'bytes=100-199').and_return(fake_resp)
|
|
80
|
-
|
|
81
102
|
rio.seek(100)
|
|
82
103
|
expect(rio.read(100)).to be_nil
|
|
83
104
|
|
|
@@ -88,7 +109,9 @@ describe FormatParser::RemoteIO do
|
|
|
88
109
|
rio = described_class.new('https://images.invalid/img.jpg')
|
|
89
110
|
|
|
90
111
|
fake_resp = double(headers: {}, status: 502, body: 'Guru meditation')
|
|
91
|
-
|
|
112
|
+
faraday_conn = instance_double(Faraday::Connection, get: fake_resp)
|
|
113
|
+
allow(Faraday).to receive(:new).and_return(faraday_conn)
|
|
114
|
+
expect(faraday_conn).to receive(:get).with('https://images.invalid/img.jpg', nil, range: 'bytes=100-199')
|
|
92
115
|
|
|
93
116
|
rio.seek(100)
|
|
94
117
|
expect { rio.read(100) }.to raise_error(/replied with a 502 and we might want to retry/)
|
|
@@ -100,7 +123,9 @@ describe FormatParser::RemoteIO do
|
|
|
100
123
|
expect(rio.pos).to eq(0)
|
|
101
124
|
|
|
102
125
|
fake_resp = double(headers: {'Content-Range' => 'bytes 0-0/13'}, status: 206, body: 'a')
|
|
103
|
-
|
|
126
|
+
faraday_conn = instance_double(Faraday::Connection, get: fake_resp)
|
|
127
|
+
allow(Faraday).to receive(:new).and_return(faraday_conn)
|
|
128
|
+
expect(faraday_conn).to receive(:get).with('https://images.invalid/img.jpg', nil, range: 'bytes=0-0')
|
|
104
129
|
rio.read(1)
|
|
105
130
|
|
|
106
131
|
expect(rio.pos).to eq(1)
|
metadata
CHANGED
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: format_parser
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.28.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Noah Berman
|
|
8
8
|
- Julik Tarkhanov
|
|
9
|
-
autorequire:
|
|
9
|
+
autorequire:
|
|
10
10
|
bindir: exe
|
|
11
11
|
cert_chain: []
|
|
12
|
-
date: 2021-
|
|
12
|
+
date: 2021-02-17 00:00:00.000000000 Z
|
|
13
13
|
dependencies:
|
|
14
14
|
- !ruby/object:Gem::Dependency
|
|
15
15
|
name: ks
|
|
@@ -73,6 +73,20 @@ dependencies:
|
|
|
73
73
|
- - "~>"
|
|
74
74
|
- !ruby/object:Gem::Version
|
|
75
75
|
version: '0.13'
|
|
76
|
+
- !ruby/object:Gem::Dependency
|
|
77
|
+
name: faraday_middleware
|
|
78
|
+
requirement: !ruby/object:Gem::Requirement
|
|
79
|
+
requirements:
|
|
80
|
+
- - "~>"
|
|
81
|
+
- !ruby/object:Gem::Version
|
|
82
|
+
version: '0.14'
|
|
83
|
+
type: :runtime
|
|
84
|
+
prerelease: false
|
|
85
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
86
|
+
requirements:
|
|
87
|
+
- - "~>"
|
|
88
|
+
- !ruby/object:Gem::Version
|
|
89
|
+
version: '0.14'
|
|
76
90
|
- !ruby/object:Gem::Dependency
|
|
77
91
|
name: measurometer
|
|
78
92
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -183,10 +197,10 @@ executables:
|
|
|
183
197
|
extensions: []
|
|
184
198
|
extra_rdoc_files: []
|
|
185
199
|
files:
|
|
200
|
+
- ".github/workflows/main.yml"
|
|
186
201
|
- ".gitignore"
|
|
187
202
|
- ".rspec"
|
|
188
203
|
- ".rubocop.yml"
|
|
189
|
-
- ".travis.yml"
|
|
190
204
|
- CHANGELOG.md
|
|
191
205
|
- CODE_OF_CONDUCT.md
|
|
192
206
|
- CONTRIBUTING.md
|
|
@@ -280,7 +294,7 @@ licenses:
|
|
|
280
294
|
- MIT (Hippocratic)
|
|
281
295
|
metadata:
|
|
282
296
|
allowed_push_host: https://rubygems.org
|
|
283
|
-
post_install_message:
|
|
297
|
+
post_install_message:
|
|
284
298
|
rdoc_options: []
|
|
285
299
|
require_paths:
|
|
286
300
|
- lib
|
|
@@ -296,7 +310,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
296
310
|
version: '0'
|
|
297
311
|
requirements: []
|
|
298
312
|
rubygems_version: 3.0.3
|
|
299
|
-
signing_key:
|
|
313
|
+
signing_key:
|
|
300
314
|
specification_version: 4
|
|
301
315
|
summary: A library for efficient parsing of file metadata
|
|
302
316
|
test_files: []
|