bkblz 0.1.12 → 0.1.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: 67343da9ff4b13623a47edc49f3d2d00b68d1d4f
4
- data.tar.gz: 8ae1e66f110945ab489e2f05667fddc6d922b0af
2
+ SHA256:
3
+ metadata.gz: 7f294d136a3d0f442f0b416e5f7b6559d675f6f0ab041eee9429bdcbece338eb
4
+ data.tar.gz: caac5d40546eb5de9e41ebb1e63ceece8a91dd33d94f7a6c995fa6dc8bd2cf5c
5
5
  SHA512:
6
- metadata.gz: 224f37da1dd4adad73ca36becd793769110d278bc22230271260ffd797cffc47e1cc532154daface01e701baf98c0df197c381591668f48c61cadc2f51213695
7
- data.tar.gz: 996efe932b47415ad4a8fcdac2cc5283620510c147fbca8e1229b923489f1f6d840284f414c4719e6e53838bd07293dfa7473bbe29a3b4b2aa5a63c4c905cf80
6
+ metadata.gz: d20fe9ccad91955038065f733f5f503a0552f4397c92256ff137d8793d0b992f6e30a8cdb0085906113f5588cfe2f840f8a887055f39714baecc45f20b346726
7
+ data.tar.gz: a424d090a142412fde903cc7a1dc510fd705c7a88767e279e70d1d10b428231ccbc11155c5ca7b72d960a5c12ada2f88776a0765c9fe93067373cf493d77fdbd
data/README.rb CHANGED
@@ -16,10 +16,14 @@ Currently the gem supports the following V1 API calls:
16
16
  * b2_delete_bucket
17
17
  * b2_delete_file_version
18
18
  * b2_get_file_info
19
+ * b2_get_upload_part_url
20
+ * b2_finish_large_file
19
21
  * b2_list_buckets
20
22
  * b2_list_file_names
21
23
  * b2_list_file_versions
24
+ * b2_start_large_file
22
25
  * b2_upload_file
26
+ * b2_upload_part
23
27
  * b2_download_file_by_id
24
28
  * b2_download_file_by_name
25
29
 
@@ -193,6 +197,8 @@ def run_readme
193
197
  byte_range_download = session.send(
194
198
  Bkblz::V1::DownloadFileByNameRequest.new bucket, file_name, bytes).to_model
195
199
  Bkblz.log.info "file bytes: #{byte_range_download.body}"
200
+
201
+ # TODO: add examples for uploading large files by parts... see lib/bkblz/task/upload_file
196
202
  rescue => e
197
203
  Bkblz.log.error "there was an error: #{e}"
198
204
  Bkblz.log.error e.backtrace.join "\n"
@@ -9,7 +9,9 @@ module Bkblz
9
9
 
10
10
  :log_device => :stderr, # [:stdout, :stderr, :devnull, path, fd]
11
11
  :log_level => :warn, # [:debug, :info, :warn, :error, :fatal, (-6..-1)]
12
- :log_colorize => true
12
+ :log_colorize => true,
13
+
14
+ :large_file_max_chunk_size => 1e8, # 100MB
13
15
  }.freeze
14
16
 
15
17
  attr_reader *CONFIG_VARS.keys, :config_map
@@ -19,15 +19,6 @@ module Bkblz
19
19
  task_param :gzip
20
20
 
21
21
  def run_internal(session, params)
22
- file_body = if params[:file_path]
23
- f = ::File.new(params[:file_path], "r")
24
- f.read
25
- elsif params[:file_body]
26
- params[:file_body]
27
- else
28
- raise 'missing either :file_body or :file_path param'
29
- end
30
-
31
22
  file_name = if params[:file_name]
32
23
  params[:file_name]
33
24
  elsif params[:file_path]
@@ -42,13 +33,74 @@ module Bkblz
42
33
  Time.now
43
34
  end.to_i * 1000
44
35
 
36
+ bucket = find_bucket_by_name session, params[:bucket_name]
37
+ size = file_size(params)
38
+
39
+ file_data = {
40
+ :file_name => file_name,
41
+ :file_mtime => file_mtime,
42
+ :file_size => size,
43
+ :bucket => bucket,
44
+ }
45
+
46
+ if size > config.large_file_max_chunk_size
47
+ upload_large_file session, params, **file_data
48
+ else
49
+ upload_file session, params, **file_data
50
+ end
51
+ end
52
+
53
+ private
54
+ def file_size(params)
55
+ if params[:file_path]
56
+ ::File.new(params[:file_path]).size
57
+ elsif params[:file_body]
58
+ params[:file_body].size
59
+ end
60
+ end
61
+
62
+ def upload_large_file(session, params, bucket:, file_name:, file_mtime:, **file_data)
63
+ start_large_file_info = session.send(Bkblz::V1::StartLargeFileRequest.new(
64
+ bucket.bucket_id, file_name, file_mtime)).to_model
65
+ file_id = start_large_file_info.file_id
66
+
67
+ upload_part_auth = session.send(Bkblz::V1::GetUploadPartUrlRequest.new(file_id)).to_model
68
+
69
+ actual_size = file_data[:file_size]
70
+ chunk_size = config.large_file_max_chunk_size
71
+ num_chunks = (actual_size / chunk_size.to_f).ceil
72
+
73
+ file_io = if params[:file_path]
74
+ ::File.new(params[:file_path], "rb")
75
+ else
76
+ raise 'only file_path is supported for large file uploads'
77
+ end
78
+
79
+ upload_part_infos = (0..num_chunks - 1).map do |chunk_i|
80
+ session.send(Bkblz::V1::UploadPartRequest.new(
81
+ upload_part_auth, file_io, chunk_i, chunk_size)).to_model
82
+ end
83
+
84
+ file_info = session.send(
85
+ Bkblz::V1::FinishLargeFileRequest.new(file_id, upload_part_infos)).to_model
86
+ end
87
+
88
+ def upload_file(session, params, bucket:, file_name:, file_mtime:, **file_data)
89
+ file_body = if params[:file_path]
90
+ f = ::File.new(params[:file_path], "r")
91
+ f.read
92
+ elsif params[:file_body]
93
+ params[:file_body]
94
+ else
95
+ raise 'missing either :file_body or :file_path param'
96
+ end
97
+
45
98
  if params[:gzip]
46
99
  # https://ruby-doc.org/stdlib-2.4.2/libdoc/zlib/rdoc/Zlib.html#method-c-gzip
47
100
  file_body = Zlib.gzip file_body, level: Zlib::DEFAULT_COMPRESSION
48
101
  file_name << ".gz" unless file_name =~ /\.gz$/
49
102
  end
50
103
 
51
- bucket = find_bucket_by_name session, params[:bucket_name]
52
104
  upload_auth = session.send(
53
105
  Bkblz::V1::GetUploadUrlRequest.new bucket.bucket_id).to_model
54
106
 
@@ -7,13 +7,17 @@ require_relative "request"
7
7
  require_relative "error_response"
8
8
 
9
9
  require_relative "authorize_account"
10
- require_relative "list_buckets"
11
10
  require_relative "create_bucket"
12
11
  require_relative "delete_bucket"
12
+ require_relative "delete_file_version"
13
+ require_relative "download_file"
14
+ require_relative "finish_large_file"
13
15
  require_relative "get_file_info"
16
+ require_relative "get_upload_part_url"
14
17
  require_relative "get_upload_url"
15
- require_relative "upload_file"
18
+ require_relative "list_buckets"
16
19
  require_relative "list_file_names"
17
20
  require_relative "list_file_versions"
18
- require_relative "delete_file_version"
19
- require_relative "download_file"
21
+ require_relative "start_large_file"
22
+ require_relative "upload_file"
23
+ require_relative "upload_part"
@@ -0,0 +1,30 @@
1
+ require "digest/sha1"
2
+
3
+ module Bkblz
4
+ module V1
5
+
6
+ class FinishLargeFileResponse < Response
7
+ response_model Model::FileInfo
8
+ end
9
+
10
+ class FinishLargeFileRequest < Request
11
+
12
+ response_class FinishLargeFileResponse
13
+ url_suffix "/b2api/v1/b2_finish_large_file"
14
+
15
+ def initialize(file_id, file_part_infos)
16
+ sha1_sums = file_part_infos.sort { |a, b| a.part_number <=> b.part_number }.map do |info|
17
+ info.content_sha1
18
+ end
19
+ @body = {
20
+ :file_id => file_id,
21
+ :part_sha1_array => sha1_sums,
22
+ }
23
+ end
24
+
25
+ def build_request(session)
26
+ session.create_post url(session), @body
27
+ end
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,22 @@
1
+ module Bkblz
2
+ module V1
3
+
4
+ class GetUploadPartUrlResponse < Response
5
+ response_model Model::UploadPartAuth
6
+ end
7
+
8
+ class GetUploadPartUrlRequest < Request
9
+
10
+ response_class GetUploadPartUrlResponse
11
+ url_suffix "/b2api/v1/b2_get_upload_part_url"
12
+
13
+ def initialize(file_id)
14
+ @body = {:file_id => file_id}
15
+ end
16
+
17
+ def build_request(session)
18
+ session.create_post url(session), @body
19
+ end
20
+ end
21
+ end
22
+ end
@@ -15,18 +15,26 @@ module Bkblz
15
15
  :body, :content_length, :content_type, :file_id, :file_name, :sha1, :x_bz_info
16
16
  ]
17
17
 
18
- # Returned by upload_file
18
+ # Returned by upload_file, finish_large_file, start_large_file
19
19
  FileInfo = Model.define *[
20
- :account_id, :bucket_id, :content_length, :content_sha1, :content_type,
20
+ :account_id, :action, :bucket_id, :content_length, :content_sha1, :content_type,
21
21
  :file_id, :file_info, :file_name
22
22
  ]
23
23
 
24
+ # Returned by upload_file
25
+ FilePartInfo = Model.define *[
26
+ :file_id, :part_number, :content_length, :content_sha1, :upload_timestamp,
27
+ ]
28
+
24
29
  # Returned by delete_file_version
25
30
  PartialFileInfo = Model.define :file_id, :file_name
26
31
 
27
32
  # Returned by get_upload_url
28
33
  UploadAuth = Model.define :bucket_id, :upload_url, :authorization_token
29
34
 
35
+ # Returned by get_upload_part_url
36
+ UploadPartAuth = Model.define :file_id, :upload_url, :authorization_token
37
+
30
38
  # Possibly returned by any request
31
39
  Error = Model.define :status, :code, :message
32
40
  end
@@ -0,0 +1,40 @@
1
+ require "digest/sha1"
2
+
3
+ module Bkblz
4
+ module V1
5
+
6
+ class StartLargeFileResponse < Response
7
+ response_model Model::FileInfo
8
+ end
9
+
10
+ class StartLargeFileRequest < Request
11
+
12
+ response_class StartLargeFileResponse
13
+ url_suffix "/b2api/v1/b2_start_large_file"
14
+
15
+ def initialize(bucket_id, file_name, last_modified_millis=nil,
16
+ file_sha=nil, content_type='b2/x-auto', **file_info)
17
+ # Both of the following are recommended here:
18
+ # https://www.backblaze.com/b2/docs/b2_start_large_file.html
19
+ if last_modified_millis
20
+ # must be a string
21
+ file_info[:src_last_modified_millis] = last_modified_millis.to_s
22
+ end
23
+ if file_sha
24
+ file_info[:large_file_sha1] = file_sha
25
+ end
26
+
27
+ @body = {
28
+ :bucket_id => bucket_id,
29
+ :file_name => file_name,
30
+ :content_type => content_type,
31
+ :file_info => file_info
32
+ }
33
+ end
34
+
35
+ def build_request(session)
36
+ session.create_post url(session), @body
37
+ end
38
+ end
39
+ end
40
+ end
@@ -0,0 +1,53 @@
1
+ require "digest/sha1"
2
+
3
+ module Bkblz
4
+ module V1
5
+
6
+ class UploadPartResponse < Response
7
+ response_model Model::FilePartInfo
8
+ end
9
+
10
+ class UploadPartRequest < Request
11
+
12
+ response_class UploadPartResponse
13
+
14
+ REQUIRED_HEADERS = {
15
+ :"Authorization" => nil,
16
+ :"Content-Length" => nil,
17
+ :"Content-Type" => 'application/octet-stream',
18
+ :"X-Bz-Part-Number" => nil, # a value in [1..10000]
19
+ :"X-Bz-Content-Sha1" => nil
20
+ }
21
+
22
+ ##
23
+ # @param {chunk_number} is a value in [0...9999]
24
+ def initialize(upload_part_auth, io, chunk_number, chunk_size)
25
+ @upload_url = upload_part_auth.upload_url
26
+ @body_chunk = read_chunk(io, chunk_number, chunk_size)
27
+ @headers = REQUIRED_HEADERS.dup
28
+
29
+ part_number = chunk_number + 1
30
+ @headers[:"Authorization"] = upload_part_auth.authorization_token
31
+ @headers[:"Content-Length"] = @body_chunk.size
32
+ @headers[:"X-Bz-Part-Number"] = part_number
33
+ @headers[:"X-Bz-Content-Sha1"] = Digest::SHA1.hexdigest @body_chunk
34
+ end
35
+
36
+ def build_request(session)
37
+ session.create_post @upload_url, @body_chunk, @headers
38
+ end
39
+
40
+ private
41
+ def read_chunk(io, chunk_number, chunk_size)
42
+ unless io.is_a?(IO)
43
+ raise 'only IO type is supported for upload_part'
44
+ end
45
+
46
+ byte = chunk_number * chunk_size
47
+ # https://ruby-doc.org/core-2.5/IO.html#method-i-seek
48
+ io.seek byte, IO::SEEK_SET
49
+ io.read(chunk_size)
50
+ end
51
+ end
52
+ end
53
+ end
@@ -1,3 +1,3 @@
1
1
  module Bkblz
2
- VERSION = "0.1.12"
2
+ VERSION = "0.1.13"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: bkblz
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.12
4
+ version: 0.1.13
5
5
  platform: ruby
6
6
  authors:
7
7
  - Erick Johnson
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-11-30 00:00:00.000000000 Z
11
+ date: 2019-01-09 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: thor
@@ -105,7 +105,9 @@ files:
105
105
  - lib/bkblz/v1/delete_file_version.rb
106
106
  - lib/bkblz/v1/download_file.rb
107
107
  - lib/bkblz/v1/error_response.rb
108
+ - lib/bkblz/v1/finish_large_file.rb
108
109
  - lib/bkblz/v1/get_file_info.rb
110
+ - lib/bkblz/v1/get_upload_part_url.rb
109
111
  - lib/bkblz/v1/get_upload_url.rb
110
112
  - lib/bkblz/v1/list_buckets.rb
111
113
  - lib/bkblz/v1/list_file_names.rb
@@ -115,7 +117,9 @@ files:
115
117
  - lib/bkblz/v1/request.rb
116
118
  - lib/bkblz/v1/response.rb
117
119
  - lib/bkblz/v1/session.rb
120
+ - lib/bkblz/v1/start_large_file.rb
118
121
  - lib/bkblz/v1/upload_file.rb
122
+ - lib/bkblz/v1/upload_part.rb
119
123
  - lib/bkblz/version.rb
120
124
  homepage: https://github.com/erickj/bkblz
121
125
  licenses:
@@ -137,7 +141,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
137
141
  version: '0'
138
142
  requirements: []
139
143
  rubyforge_project:
140
- rubygems_version: 2.6.13
144
+ rubygems_version: 2.7.7
141
145
  signing_key:
142
146
  specification_version: 4
143
147
  summary: Bkblz GEM for the Backblaze B2 API. https://www.backblaze.com/b2/docs/