baza.rb 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +2 -2
- data/lib/baza-rb/fake.rb +11 -0
- data/lib/baza-rb/version.rb +1 -1
- data/lib/baza-rb.rb +135 -102
- data/test/test_baza-rb.rb +13 -1
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ebcb57db3953278c8d402cc38ae954e982062cfc4498e301bc0495a7b3cba95a
|
4
|
+
data.tar.gz: 0a6acf7107e149f8515911542460266b95b825df586576713734648751f2de56
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a40b4c3655a48f0449cb0247f1dd1045344b95868330121b6ce462d874aec6117b1490d12819ceac9afc29217e0352d4d30d2c6ff78c1031761946bb3cf89654
|
7
|
+
data.tar.gz: 7a479de17acf3e82f60327fa7c131605bebf2a0ae2dbfd1fde80ac08a3bcfafd2d40b252ef4d23845de083867a5955244d7079ce5526574d94c1fb9814e62cf8
|
data/Gemfile.lock
CHANGED
@@ -107,7 +107,7 @@ GEM
|
|
107
107
|
regexp_parser (2.10.0)
|
108
108
|
retries (0.0.5)
|
109
109
|
rexml (3.4.1)
|
110
|
-
rubocop (1.76.
|
110
|
+
rubocop (1.76.1)
|
111
111
|
json (~> 2.3)
|
112
112
|
language_server-protocol (~> 3.17.0.2)
|
113
113
|
lint_roller (~> 1.1.0)
|
@@ -118,7 +118,7 @@ GEM
|
|
118
118
|
rubocop-ast (>= 1.45.0, < 2.0)
|
119
119
|
ruby-progressbar (~> 1.7)
|
120
120
|
unicode-display_width (>= 2.4.0, < 4.0)
|
121
|
-
rubocop-ast (1.45.
|
121
|
+
rubocop-ast (1.45.1)
|
122
122
|
parser (>= 3.3.7.2)
|
123
123
|
prism (~> 1.4)
|
124
124
|
rubocop-minitest (0.38.1)
|
data/lib/baza-rb/fake.rb
CHANGED
@@ -123,6 +123,17 @@ class BazaRb::Fake
|
|
123
123
|
true
|
124
124
|
end
|
125
125
|
|
126
|
+
# Find a single durable.
|
127
|
+
#
|
128
|
+
# @param [String] jname The name of the job on the server
|
129
|
+
# @param [String] file The path to the file to upload
|
130
|
+
# @return [Integer] Always returns 42 as the fake durable ID
|
131
|
+
def durable_find(jname, file)
|
132
|
+
assert_name(jname)
|
133
|
+
assert_file(file)
|
134
|
+
42
|
135
|
+
end
|
136
|
+
|
126
137
|
# Place a single durable file on the server.
|
127
138
|
#
|
128
139
|
# @param [String] jname The name of the job on the server
|
data/lib/baza-rb/version.rb
CHANGED
data/lib/baza-rb.rb
CHANGED
@@ -158,25 +158,7 @@ class BazaRb
|
|
158
158
|
data = ''
|
159
159
|
elapsed(@loog) do
|
160
160
|
Tempfile.open do |file|
|
161
|
-
|
162
|
-
request = Typhoeus::Request.new(
|
163
|
-
home.append('pull').append("#{id}.fb").to_s,
|
164
|
-
method: :get,
|
165
|
-
headers: headers.merge(
|
166
|
-
'Accept' => 'application/zip, application/factbase'
|
167
|
-
),
|
168
|
-
accept_encoding: 'gzip',
|
169
|
-
connecttimeout: @timeout,
|
170
|
-
timeout: @timeout
|
171
|
-
)
|
172
|
-
request.on_body do |chunk|
|
173
|
-
f.write(chunk)
|
174
|
-
end
|
175
|
-
retry_it do
|
176
|
-
request.run
|
177
|
-
end
|
178
|
-
checked(request.response)
|
179
|
-
end
|
161
|
+
download(home.append('pull').append("#{id}.fb").to_s, file.path)
|
180
162
|
data = File.binread(file)
|
181
163
|
throw :"Pulled #{data.bytesize} bytes of job ##{id} factbase at #{@host}"
|
182
164
|
end
|
@@ -388,28 +370,34 @@ class BazaRb
|
|
388
370
|
raise 'The "file" of the durable is nil' if file.nil?
|
389
371
|
raise "The file '#{file}' is absent" unless File.exist?(file)
|
390
372
|
id = nil
|
391
|
-
|
392
|
-
|
393
|
-
|
394
|
-
|
395
|
-
|
396
|
-
|
397
|
-
|
398
|
-
'
|
399
|
-
|
400
|
-
|
401
|
-
|
402
|
-
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
|
408
|
-
|
409
|
-
|
410
|
-
|
411
|
-
|
373
|
+
Tempfile.open do |f|
|
374
|
+
File.write(f.path, 'placeholder')
|
375
|
+
elapsed(@loog) do
|
376
|
+
ret =
|
377
|
+
retry_it do
|
378
|
+
checked(
|
379
|
+
Typhoeus::Request.post(
|
380
|
+
home.append('durables').append('place').to_s,
|
381
|
+
body: {
|
382
|
+
'_csrf' => csrf,
|
383
|
+
'jname' => jname,
|
384
|
+
'file' => File.basename(file),
|
385
|
+
'zip' => File.open(f, 'rb')
|
386
|
+
},
|
387
|
+
headers:,
|
388
|
+
connecttimeout: @timeout,
|
389
|
+
timeout: @timeout
|
390
|
+
),
|
391
|
+
302
|
392
|
+
)
|
393
|
+
end
|
394
|
+
id = ret.headers['X-Zerocracy-DurableId'].to_i
|
395
|
+
throw :"Durable ##{id} (#{file}) placed for job \"#{jname}\" at #{@host}"
|
396
|
+
end
|
412
397
|
end
|
398
|
+
durable_lock(id, user_agent)
|
399
|
+
durable_save(id, file)
|
400
|
+
durable_unlock(id, user_agent)
|
413
401
|
id
|
414
402
|
end
|
415
403
|
|
@@ -446,26 +434,8 @@ class BazaRb
|
|
446
434
|
raise 'The ID of the durable is nil' if id.nil?
|
447
435
|
raise 'The ID of the durable must be a positive integer' unless id.positive?
|
448
436
|
raise 'The "file" of the durable is nil' if file.nil?
|
449
|
-
FileUtils.mkdir_p(File.dirname(file))
|
450
437
|
elapsed(@loog) do
|
451
|
-
|
452
|
-
request = Typhoeus::Request.new(
|
453
|
-
home.append('durables').append(id).to_s,
|
454
|
-
method: :get,
|
455
|
-
headers: headers.merge(
|
456
|
-
'Accept' => 'application/octet-stream'
|
457
|
-
),
|
458
|
-
connecttimeout: @timeout,
|
459
|
-
timeout: @timeout
|
460
|
-
)
|
461
|
-
request.on_body do |chunk|
|
462
|
-
f.write(chunk)
|
463
|
-
end
|
464
|
-
retry_it do
|
465
|
-
request.run
|
466
|
-
end
|
467
|
-
checked(request.response)
|
468
|
-
end
|
438
|
+
download(home.append('durables').append(id).to_s, file)
|
469
439
|
throw :"Durable ##{id} loaded #{File.size(file)} bytes from #{@host}"
|
470
440
|
end
|
471
441
|
end
|
@@ -639,55 +609,28 @@ class BazaRb
|
|
639
609
|
# @return [Boolean] TRUE if a job was successfully popped, FALSE if queue is empty
|
640
610
|
# @raise [ServerFailure] If the pop operation fails
|
641
611
|
def pop(owner, zip)
|
642
|
-
raise 'The "zip" of the job is nil' if zip.nil?
|
643
612
|
success = false
|
644
|
-
FileUtils.rm_f(zip)
|
645
|
-
job = nil
|
646
613
|
elapsed(@loog) do
|
647
|
-
|
648
|
-
|
649
|
-
|
650
|
-
|
651
|
-
|
652
|
-
|
653
|
-
|
654
|
-
headers: headers.merge(
|
655
|
-
'Accept' => 'application/octet-stream',
|
656
|
-
'Range' => "bytes=#{f.size}-"
|
614
|
+
uri = home.append('pop').add(owner:)
|
615
|
+
ret =
|
616
|
+
retry_it do
|
617
|
+
checked(
|
618
|
+
Typhoeus::Request.get(
|
619
|
+
uri.to_s,
|
620
|
+
headers:
|
657
621
|
),
|
658
|
-
|
659
|
-
timeout: @timeout
|
622
|
+
[204, 302]
|
660
623
|
)
|
661
|
-
request.on_body do |chunk|
|
662
|
-
f.write(chunk)
|
663
|
-
end
|
664
|
-
retry_it do
|
665
|
-
request.run
|
666
|
-
end
|
667
|
-
ret = request.response
|
668
|
-
checked(ret, [200, 204, 206])
|
669
|
-
success = ret.code != 204
|
670
|
-
break unless ret.code == 206
|
671
|
-
job = ret.headers['X-Zerocracy-JobId']
|
672
|
-
raise 'Job ID is not returned in X-Zerocracy-JobId' if job.nil?
|
673
|
-
raise "Job ID returned in X-Zerocracy-JobId is not valid (#{job.inspect})" unless job.match?(/^[0-9]+$/)
|
674
|
-
_, v = ret.headers['Content-Range'].split
|
675
|
-
range, total = v.split('/')
|
676
|
-
raise "Total size is not valid (#{total.inspect})" unless total.match?(/^\*|[0-9]+$/)
|
677
|
-
b, e = range.split('-')
|
678
|
-
raise "Range is not valid (#{range.inspect})" unless e.match?(/^[0-9]+$/)
|
679
|
-
len = ret.headers['Content-Length'].to_i
|
680
|
-
unless len.zero?
|
681
|
-
raise "Range size (#{range.inspect}) is not equal to Content-Length" unless len - 1 == e.to_i - b.to_i
|
682
|
-
raise "Range end (#{range.inspect}) is not equal to #{f.size}" if e.to_i != f.size - 1
|
683
|
-
end
|
684
|
-
break if e.to_i == total.to_i - 1
|
685
624
|
end
|
686
|
-
|
687
|
-
unless success
|
625
|
+
if ret.code == 204
|
688
626
|
FileUtils.rm_f(zip)
|
689
|
-
throw :"Nothing to pop at #{
|
627
|
+
throw :"Nothing to pop at #{uri}"
|
690
628
|
end
|
629
|
+
job = ret.headers['X-Zerocracy-JobId']
|
630
|
+
raise 'Job ID is not returned in X-Zerocracy-JobId' if job.nil?
|
631
|
+
raise "Job ID returned in X-Zerocracy-JobId is not valid (#{job.inspect})" unless job.match?(/^[0-9]+$/)
|
632
|
+
download(uri.add(job:), zip)
|
633
|
+
success = true
|
691
634
|
throw :"Popped #{File.size(zip)} bytes in ZIP archive at #{@host}"
|
692
635
|
end
|
693
636
|
success
|
@@ -794,9 +737,13 @@ class BazaRb
|
|
794
737
|
|
795
738
|
private
|
796
739
|
|
740
|
+
def user_agent
|
741
|
+
"baza.rb #{BazaRb::VERSION}"
|
742
|
+
end
|
743
|
+
|
797
744
|
def headers
|
798
745
|
{
|
799
|
-
'User-Agent' =>
|
746
|
+
'User-Agent' => user_agent,
|
800
747
|
'Connection' => 'close',
|
801
748
|
'X-Zerocracy-Token' => @token
|
802
749
|
}
|
@@ -883,4 +830,90 @@ class BazaRb
|
|
883
830
|
@loog.error(msg)
|
884
831
|
raise ServerFailure, msg
|
885
832
|
end
|
833
|
+
|
834
|
+
# Download file via GET, in ranges.
|
835
|
+
# @param [String] uri The URI
|
836
|
+
# @param [String] file The path to save to
|
837
|
+
def download(uri, file)
|
838
|
+
raise 'The "file" is nil' if file.nil?
|
839
|
+
FileUtils.mkdir_p(File.dirname(file))
|
840
|
+
elapsed(@loog) do
|
841
|
+
File.open(file, 'wb+') do |f|
|
842
|
+
loop do
|
843
|
+
request = Typhoeus::Request.new(
|
844
|
+
uri.to_s,
|
845
|
+
method: :get,
|
846
|
+
headers: headers.merge(
|
847
|
+
'Accept' => 'application/octet-stream',
|
848
|
+
'Range' => "bytes=#{f.size}-"
|
849
|
+
),
|
850
|
+
connecttimeout: @timeout,
|
851
|
+
timeout: @timeout
|
852
|
+
)
|
853
|
+
request.on_body do |chunk|
|
854
|
+
f.write(chunk)
|
855
|
+
end
|
856
|
+
retry_it do
|
857
|
+
request.run
|
858
|
+
end
|
859
|
+
ret = request.response
|
860
|
+
checked(ret, [200, 206])
|
861
|
+
break if ret.code == 200
|
862
|
+
_, v = ret.headers['Content-Range'].split
|
863
|
+
range, total = v.split('/')
|
864
|
+
raise "Total size is not valid (#{total.inspect})" unless total.match?(/^\*|[0-9]+$/)
|
865
|
+
b, e = range.split('-')
|
866
|
+
raise "Range is not valid (#{range.inspect})" unless e.match?(/^[0-9]+$/)
|
867
|
+
len = ret.headers['Content-Length'].to_i
|
868
|
+
unless len.zero?
|
869
|
+
raise "Range size (#{range.inspect}) is not equal to Content-Length" unless len - 1 == e.to_i - b.to_i
|
870
|
+
raise "Range end (#{range.inspect}) is not equal to #{f.size}" if e.to_i != f.size - 1
|
871
|
+
end
|
872
|
+
break if e.to_i == total.to_i - 1
|
873
|
+
end
|
874
|
+
end
|
875
|
+
throw :"Downloaded #{File.size(file)} bytes from #{uri}"
|
876
|
+
end
|
877
|
+
end
|
878
|
+
|
879
|
+
# Upload file via PUT, in ranges.
|
880
|
+
# @param [String] uri The URI
|
881
|
+
# @param [String] file The path to save to
|
882
|
+
# @param [Hash] headers Hash of HTTP headers
|
883
|
+
def upload(uri, file, headers = {})
|
884
|
+
raise 'The "file" is nil' if file.nil?
|
885
|
+
raise 'The "file" does not exist' unless File.exist?(file)
|
886
|
+
params = {
|
887
|
+
connecttimeout: @timeout,
|
888
|
+
timeout: @timeout,
|
889
|
+
body: data,
|
890
|
+
headers: headers.merge(
|
891
|
+
'Content-Type' => 'application/octet-stream'
|
892
|
+
)
|
893
|
+
}
|
894
|
+
max = 1_000_000
|
895
|
+
chunk = 0
|
896
|
+
elapsed(@loog) do
|
897
|
+
params[:headers]['X-Zerocracy-Chunk'] = chunk.to_s
|
898
|
+
loop do
|
899
|
+
File.open(file, 'rb') do |f|
|
900
|
+
f.seek(max * chunk)
|
901
|
+
data = f.read(max)
|
902
|
+
data = '' if data.nil?
|
903
|
+
params[:headers]['Content-Length'] = data.bytesize
|
904
|
+
retry_it do
|
905
|
+
checked(
|
906
|
+
Typhoeus::Request.put(
|
907
|
+
uri.to_s,
|
908
|
+
@compress ? zipped(params) : params
|
909
|
+
)
|
910
|
+
)
|
911
|
+
end
|
912
|
+
break if data.empty?
|
913
|
+
end
|
914
|
+
chunk += 1
|
915
|
+
end
|
916
|
+
throw :"Uploaded #{File.size(file)} bytes to #{uri} in #{chunk + 1} chunk(s)"
|
917
|
+
end
|
918
|
+
end
|
886
919
|
end
|
data/test/test_baza-rb.rb
CHANGED
@@ -94,7 +94,7 @@ class TestBazaRb < Minitest::Test
|
|
94
94
|
skip('We are offline') unless we_are_online
|
95
95
|
Dir.mktmpdir do |dir|
|
96
96
|
file = File.join(dir, "#{fake_name}.bin")
|
97
|
-
File.binwrite(file, 'hello')
|
97
|
+
File.binwrite(file, 'hello, world!' * 100_000)
|
98
98
|
jname = fake_name
|
99
99
|
refute(LIVE.durable_find(jname, File.basename(file)))
|
100
100
|
id = LIVE.durable_place(jname, file)
|
@@ -179,6 +179,11 @@ class TestBazaRb < Minitest::Test
|
|
179
179
|
stub_request(:post, 'https://example.org/durables/place').to_return(
|
180
180
|
status: 302, headers: { 'X-Zerocracy-DurableId' => '42' }
|
181
181
|
)
|
182
|
+
stub_request(:get, %r{https://example\.org/durables/42/lock})
|
183
|
+
.to_return(status: 302)
|
184
|
+
stub_request(:get, %r{https://example\.org/durables/42/unlock})
|
185
|
+
.to_return(status: 302)
|
186
|
+
stub_request(:put, 'https://example.org/durables/42').to_return(status: 200)
|
182
187
|
Dir.mktmpdir do |dir|
|
183
188
|
file = File.join(dir, 'test.bin')
|
184
189
|
File.binwrite(file, 'hello')
|
@@ -212,6 +217,13 @@ class TestBazaRb < Minitest::Test
|
|
212
217
|
job = 4242
|
213
218
|
stub_request(:get, 'https://example.org/pop')
|
214
219
|
.with(query: { owner: })
|
220
|
+
.to_return(
|
221
|
+
status: 302,
|
222
|
+
headers: { 'X-Zerocracy-JobId' => job },
|
223
|
+
body: ''
|
224
|
+
)
|
225
|
+
stub_request(:get, 'https://example.org/pop')
|
226
|
+
.with(query: { job: })
|
215
227
|
.to_return(
|
216
228
|
status: 206,
|
217
229
|
headers: { 'Content-Range' => 'bytes 0-0/*', 'X-Zerocracy-JobId' => job, 'Content-Length' => 0 },
|