bkblz 0.1.12 → 0.1.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 67343da9ff4b13623a47edc49f3d2d00b68d1d4f
-  data.tar.gz: 8ae1e66f110945ab489e2f05667fddc6d922b0af
+SHA256:
+  metadata.gz: 7f294d136a3d0f442f0b416e5f7b6559d675f6f0ab041eee9429bdcbece338eb
+  data.tar.gz: caac5d40546eb5de9e41ebb1e63ceece8a91dd33d94f7a6c995fa6dc8bd2cf5c
 SHA512:
-  metadata.gz: 224f37da1dd4adad73ca36becd793769110d278bc22230271260ffd797cffc47e1cc532154daface01e701baf98c0df197c381591668f48c61cadc2f51213695
-  data.tar.gz: 996efe932b47415ad4a8fcdac2cc5283620510c147fbca8e1229b923489f1f6d840284f414c4719e6e53838bd07293dfa7473bbe29a3b4b2aa5a63c4c905cf80
+  metadata.gz: d20fe9ccad91955038065f733f5f503a0552f4397c92256ff137d8793d0b992f6e30a8cdb0085906113f5588cfe2f840f8a887055f39714baecc45f20b346726
+  data.tar.gz: a424d090a142412fde903cc7a1dc510fd705c7a88767e279e70d1d10b428231ccbc11155c5ca7b72d960a5c12ada2f88776a0765c9fe93067373cf493d77fdbd

data/README.rb

@@ -16,10 +16,14 @@ Currently the gem supports the following V1 API calls:
   * b2_delete_bucket
   * b2_delete_file_version
   * b2_get_file_info
+  * b2_get_upload_part_url
+  * b2_finish_large_file
   * b2_list_buckets
   * b2_list_file_names
   * b2_list_file_versions
+  * b2_start_large_file
   * b2_upload_file
+  * b2_upload_part
   * b2_download_file_by_id
   * b2_download_file_by_name
 
@@ -193,6 +197,8 @@ def run_readme
       byte_range_download = session.send(
         Bkblz::V1::DownloadFileByNameRequest.new bucket, file_name, bytes).to_model
       Bkblz.log.info "file bytes: #{byte_range_download.body}"
+
+      # TODO: add examples for uploading large files by parts... see lib/bkblz/task/upload_file
     rescue => e
       Bkblz.log.error "there was an error: #{e}"
       Bkblz.log.error e.backtrace.join "\n"

data/lib/bkblz/config.rb

@@ -9,7 +9,9 @@ module Bkblz
 
       :log_device => :stderr, # [:stdout, :stderr, :devnull, path, fd]
       :log_level => :warn, # [:debug, :info, :warn, :error, :fatal, (-6..-1)]
-      :log_colorize => true
+      :log_colorize => true,
+
+      :large_file_max_chunk_size => 1e8, # 100MB
     }.freeze
 
     attr_reader *CONFIG_VARS.keys, :config_map

data/lib/bkblz/task/upload_file.rb

@@ -19,15 +19,6 @@ module Bkblz
       task_param :gzip
 
       def run_internal(session, params)
-        file_body = if params[:file_path]
-                      f = ::File.new(params[:file_path], "r")
-                      f.read
-                    elsif params[:file_body]
-                      params[:file_body]
-                    else
-                      raise 'missing either :file_body or :file_path param'
-                    end
-
         file_name = if params[:file_name]
                       params[:file_name]
                     elsif params[:file_path]
@@ -42,13 +33,74 @@ module Bkblz
                        Time.now
                      end.to_i * 1000
 
+        bucket = find_bucket_by_name session, params[:bucket_name]
+        size = file_size(params)
+
+        file_data = {
+          :file_name => file_name,
+          :file_mtime => file_mtime,
+          :file_size => size,
+          :bucket => bucket,
+        }
+
+        if size > config.large_file_max_chunk_size
+          upload_large_file session, params, **file_data
+        else
+          upload_file session, params, **file_data
+        end
+      end
+
+      private
+      def file_size(params)
+        if params[:file_path]
+          ::File.new(params[:file_path]).size
+        elsif params[:file_body]
+          params[:file_body].size
+        end
+      end
+
+      def upload_large_file(session, params, bucket:, file_name:, file_mtime:, **file_data)
+        start_large_file_info = session.send(Bkblz::V1::StartLargeFileRequest.new(
+            bucket.bucket_id, file_name, file_mtime)).to_model
+        file_id = start_large_file_info.file_id
+
+        upload_part_auth = session.send(Bkblz::V1::GetUploadPartUrlRequest.new(file_id)).to_model
+
+        actual_size = file_data[:file_size]
+        chunk_size = config.large_file_max_chunk_size
+        num_chunks = (actual_size / chunk_size.to_f).ceil
+
+        file_io = if params[:file_path]
+                    ::File.new(params[:file_path], "rb")
+                  else
+                    raise 'only file_path is supported for large file uploads'
+                  end
+
+        upload_part_infos = (0..num_chunks - 1).map do |chunk_i|
+          session.send(Bkblz::V1::UploadPartRequest.new(
+              upload_part_auth, file_io, chunk_i, chunk_size)).to_model
+        end
+
+        file_info = session.send(
+          Bkblz::V1::FinishLargeFileRequest.new(file_id, upload_part_infos)).to_model
+      end
+
+      def upload_file(session, params, bucket:, file_name:, file_mtime:, **file_data)
+        file_body = if params[:file_path]
+                      f = ::File.new(params[:file_path], "r")
+                      f.read
+                    elsif params[:file_body]
+                      params[:file_body]
+                    else
+                      raise 'missing either :file_body or :file_path param'
+                    end
+
         if params[:gzip]
           # https://ruby-doc.org/stdlib-2.4.2/libdoc/zlib/rdoc/Zlib.html#method-c-gzip
           file_body = Zlib.gzip file_body, level: Zlib::DEFAULT_COMPRESSION
           file_name << ".gz" unless file_name =~ /\.gz$/
         end
 
-        bucket = find_bucket_by_name session, params[:bucket_name]
         upload_auth = session.send(
           Bkblz::V1::GetUploadUrlRequest.new bucket.bucket_id).to_model
 

data/lib/bkblz/v1/all.rb

@@ -7,13 +7,17 @@ require_relative "request"
 require_relative "error_response"
 
 require_relative "authorize_account"
-require_relative "list_buckets"
 require_relative "create_bucket"
 require_relative "delete_bucket"
+require_relative "delete_file_version"
+require_relative "download_file"
+require_relative "finish_large_file"
 require_relative "get_file_info"
+require_relative "get_upload_part_url"
 require_relative "get_upload_url"
-require_relative "upload_file"
+require_relative "list_buckets"
 require_relative "list_file_names"
 require_relative "list_file_versions"
-require_relative "delete_file_version"
-require_relative "download_file"
+require_relative "start_large_file"
+require_relative "upload_file"
+require_relative "upload_part"

data/lib/bkblz/v1/finish_large_file.rb

@@ -0,0 +1,30 @@
+require "digest/sha1"
+
+module Bkblz
+  module V1
+
+    class FinishLargeFileResponse < Response
+      response_model Model::FileInfo
+    end
+
+    class FinishLargeFileRequest < Request
+
+      response_class FinishLargeFileResponse
+      url_suffix "/b2api/v1/b2_finish_large_file"
+
+      def initialize(file_id, file_part_infos)
+        sha1_sums = file_part_infos.sort { |a, b| a.part_number <=> b.part_number }.map do |info|
+          info.content_sha1
+        end
+        @body = {
+          :file_id => file_id,
+          :part_sha1_array => sha1_sums,
+        }
+      end
+
+      def build_request(session)
+        session.create_post url(session), @body
+      end
+    end
+  end
+end

data/lib/bkblz/v1/get_upload_part_url.rb

@@ -0,0 +1,22 @@
+module Bkblz
+  module V1
+
+    class GetUploadPartUrlResponse < Response
+      response_model Model::UploadPartAuth
+    end
+
+    class GetUploadPartUrlRequest < Request
+
+      response_class GetUploadPartUrlResponse
+      url_suffix "/b2api/v1/b2_get_upload_part_url"
+
+      def initialize(file_id)
+        @body = {:file_id => file_id}
+      end
+
+      def build_request(session)
+        session.create_post url(session), @body
+      end
+    end
+  end
+end

data/lib/bkblz/v1/models.rb

@@ -15,18 +15,26 @@ module Bkblz
         :body, :content_length, :content_type, :file_id, :file_name, :sha1, :x_bz_info
       ]
 
-      # Returned by upload_file
+      # Returned by upload_file, finish_large_file, start_large_file
       FileInfo = Model.define *[
-        :account_id, :bucket_id, :content_length, :content_sha1, :content_type,
+        :account_id, :action, :bucket_id, :content_length, :content_sha1, :content_type,
         :file_id, :file_info, :file_name
       ]
 
+      # Returned by upload_file
+      FilePartInfo = Model.define *[
+        :file_id, :part_number, :content_length, :content_sha1, :upload_timestamp,
+      ]
+
       # Returned by delete_file_version
       PartialFileInfo = Model.define :file_id, :file_name
 
       # Returned by get_upload_url
       UploadAuth = Model.define :bucket_id, :upload_url, :authorization_token
 
+      # Returned by get_upload_part_url
+      UploadPartAuth = Model.define :file_id, :upload_url, :authorization_token
+
       # Possibly returned by any request
       Error = Model.define :status, :code, :message
     end

data/lib/bkblz/v1/start_large_file.rb

@@ -0,0 +1,40 @@
+require "digest/sha1"
+
+module Bkblz
+  module V1
+
+    class StartLargeFileResponse < Response
+      response_model Model::FileInfo
+    end
+
+    class StartLargeFileRequest < Request
+
+      response_class StartLargeFileResponse
+      url_suffix "/b2api/v1/b2_start_large_file"
+
+      def initialize(bucket_id, file_name, last_modified_millis=nil,
+          file_sha=nil, content_type='b2/x-auto', **file_info)
+        # Both of the following are recommended here:
+        # https://www.backblaze.com/b2/docs/b2_start_large_file.html
+        if last_modified_millis
+          # must be a string
+          file_info[:src_last_modified_millis] = last_modified_millis.to_s
+        end
+        if file_sha
+          file_info[:large_file_sha1] = file_sha
+        end
+
+        @body = {
+          :bucket_id => bucket_id,
+          :file_name => file_name,
+          :content_type => content_type,
+          :file_info => file_info
+        }
+      end
+
+      def build_request(session)
+        session.create_post url(session), @body
+      end
+    end
+  end
+end

data/lib/bkblz/v1/upload_part.rb

@@ -0,0 +1,53 @@
+require "digest/sha1"
+
+module Bkblz
+  module V1
+
+    class UploadPartResponse < Response
+      response_model Model::FilePartInfo
+    end
+
+    class UploadPartRequest < Request
+
+      response_class UploadPartResponse
+
+      REQUIRED_HEADERS = {
+                          :"Authorization" => nil,
+                          :"Content-Length" => nil,
+                          :"Content-Type" => 'application/octet-stream',
+                          :"X-Bz-Part-Number" => nil, # a value in [1..10000]
+                          :"X-Bz-Content-Sha1" => nil
+      }
+
+      ##
+      # @param {chunk_number} is a value in [0...9999]
+      def initialize(upload_part_auth, io, chunk_number, chunk_size)
+        @upload_url = upload_part_auth.upload_url
+        @body_chunk = read_chunk(io, chunk_number, chunk_size)
+        @headers = REQUIRED_HEADERS.dup
+
+        part_number = chunk_number + 1
+        @headers[:"Authorization"] = upload_part_auth.authorization_token
+        @headers[:"Content-Length"] = @body_chunk.size
+        @headers[:"X-Bz-Part-Number"] = part_number
+        @headers[:"X-Bz-Content-Sha1"] = Digest::SHA1.hexdigest @body_chunk
+      end
+
+      def build_request(session)
+        session.create_post @upload_url, @body_chunk, @headers
+      end
+
+      private
+      def read_chunk(io, chunk_number, chunk_size)
+        unless io.is_a?(IO)
+          raise 'only IO type is supported for upload_part'
+        end
+
+        byte = chunk_number * chunk_size
+        # https://ruby-doc.org/core-2.5/IO.html#method-i-seek
+        io.seek byte, IO::SEEK_SET
+        io.read(chunk_size)
+      end
+    end
+  end
+end

data/lib/bkblz/version.rb

@@ -1,3 +1,3 @@
 module Bkblz
-  VERSION = "0.1.12"
+  VERSION = "0.1.13"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bkblz
 version: !ruby/object:Gem::Version
-  version: 0.1.12
+  version: 0.1.13
 platform: ruby
 authors:
 - Erick Johnson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-30 00:00:00.000000000 Z
+date: 2019-01-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -105,7 +105,9 @@ files:
 - lib/bkblz/v1/delete_file_version.rb
 - lib/bkblz/v1/download_file.rb
 - lib/bkblz/v1/error_response.rb
+- lib/bkblz/v1/finish_large_file.rb
 - lib/bkblz/v1/get_file_info.rb
+- lib/bkblz/v1/get_upload_part_url.rb
 - lib/bkblz/v1/get_upload_url.rb
 - lib/bkblz/v1/list_buckets.rb
 - lib/bkblz/v1/list_file_names.rb
@@ -115,7 +117,9 @@ files:
 - lib/bkblz/v1/request.rb
 - lib/bkblz/v1/response.rb
 - lib/bkblz/v1/session.rb
+- lib/bkblz/v1/start_large_file.rb
 - lib/bkblz/v1/upload_file.rb
+- lib/bkblz/v1/upload_part.rb
 - lib/bkblz/version.rb
 homepage: https://github.com/erickj/bkblz
 licenses:
@@ -137,7 +141,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.13
+rubygems_version: 2.7.7
 signing_key: 
 specification_version: 4
 summary: Bkblz GEM for the Backblaze B2 API. https://www.backblaze.com/b2/docs/