bkblz 0.1.9 → 0.1.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 276cd7b864f9a13fd8a32ea5839514f041afbd02
-  data.tar.gz: c87c7242a87241a7b6e48f3bef5aab55117ef37b
+SHA256:
+  metadata.gz: 47f032205c5b3348111f4cf170501fa08952ba7817cdc0c3ae06fba4c68c2a7d
+  data.tar.gz: d5a9dbae5c9fef54cbdc0e03520d283fe7634fc1b01cb537c81731ee0113d338
 SHA512:
-  metadata.gz: 5ac0e9aa47318c48169be640350dede4d116d83b15980520b99289e19ea9b1ae7a2ae79c5a24a8e3c0e92f3dc7733e311888630b099ff150705a293a77715d10
-  data.tar.gz: 53ac8463bd04b6a747f853837f825a6cb1908c1c55cefa837c472e0a83f575f12933969ed540ffb40fa9aad7c7d6a8f64f9553eca91ae4e3330c4366087c9956
+  metadata.gz: 3ac101a0d32a08d63ec18a156cc087b9401267ddaa37554fb4eadad5700f2a042b498953493d2c2c44127474665d6da688913814ab2620af20f17906d3fe7d50
+  data.tar.gz: 10bcde19f56e2488cb19879af0cbf46caa1a2221c987fd4cebf6237ad9c34d5b15671b5fa3142d21125325b5d36162969ab74d8e8bb0462c74362eda34a707ca

data/.gitignore

@@ -7,3 +7,4 @@
 /pkg/
 /spec/reports/
 /tmp/
+bkblz-*\.gem

data/README.rb

@@ -1,6 +1,12 @@
 =begin
 
-This is the bkblz ruby gem, a library for the Backblaze B2 cloud
+tldr:
+$ gem install bkblz
+$ bkblz help
+
+More...
+
+This is the bkblz Ruby gem, a library for the Backblaze B2 cloud
 storage API: https://www.backblaze.com/b2/docs/
 
 Currently the gem supports the following V1 API calls:
@@ -10,10 +16,14 @@ Currently the gem supports the following V1 API calls:
   * b2_delete_bucket
   * b2_delete_file_version
   * b2_get_file_info
+  * b2_get_upload_part_url
+  * b2_finish_large_file
   * b2_list_buckets
   * b2_list_file_names
   * b2_list_file_versions
+  * b2_start_large_file
   * b2_upload_file
+  * b2_upload_part
   * b2_download_file_by_id
   * b2_download_file_by_name
 
@@ -187,6 +197,8 @@ def run_readme
       byte_range_download = session.send(
         Bkblz::V1::DownloadFileByNameRequest.new bucket, file_name, bytes).to_model
       Bkblz.log.info "file bytes: #{byte_range_download.body}"
+
+      # TODO: add examples for uploading large files by parts... see lib/bkblz/task/upload_file
     rescue => e
       Bkblz.log.error "there was an error: #{e}"
       Bkblz.log.error e.backtrace.join "\n"

data/bin/bkblz

@@ -109,18 +109,28 @@ module Bkblz
   class File < Thor
     include Helper
 
-    desc "upload <bucket_name> <local_file_path>", "upload a local file to B2"
+    desc "upload <bucket_name> <local_file_path>", "upload a local file to B2" +
+      " if <local_file_path> is \"-\" (0x2d), then upload will read from STDIN."
     option :file_name, :desc => "remote file name, if not given the basename" +
-      "of the local file is used"
+      " of the local file is used"
+    option :gzip, :type => :boolean, :desc => "automatically gzip files"
     def upload(bucket_name, local_file_path)
       parse_opts
 
+      params = {
+        :bucket_name => bucket_name,
+        :file_name => options[:file_name],
+        :gzip => options[:gzip]
+      }
+
+      if local_file_path == "-"
+        params[:file_body] = STDIN.read
+      else
+        params[:file_path] = local_file_path
+      end
+
       file_info = run_task do
-        Bkblz::Task::UploadFile.run Bkblz.config, {
-          :bucket_name => bucket_name,
-          :file_path => local_file_path,
-          :file_name => options[:file_name]
-        }
+        Bkblz::Task::UploadFile.run Bkblz.config, params
       end
 
       print_header "File Info"
@@ -129,6 +139,7 @@ module Bkblz
 
     desc "downloadbyname <bucket> <name> <dir>", "downloads file by <bucket> " +
       "and <name> to directory <dir>"
+    option :strip_prefix, :desc => "prefix to strip from filename when writing locally"
     def downloadbyname(bucket_name, file_name, dir_path)
       parse_opts
 
@@ -137,7 +148,8 @@ module Bkblz
           :file_name => file_name,
           :bucket_name => bucket_name,
           :dir_path => dir_path,
-          :use_filename => true
+          :use_filename => true,
+          :strip_prefix => options[:strip_prefix]
         }
       end
 
@@ -147,6 +159,7 @@ module Bkblz
 
     # TODO(erick): Rename to downloadbyid
     desc "download <id> <dir>", "downloads file <id> to directory <dir>"
+    option :strip_prefix, :desc => "prefix to strip from filename when writing locally"
     def download(file_id, dir_path)
       parse_opts
 
@@ -154,7 +167,8 @@ module Bkblz
         Bkblz::Task::DownloadFile.run Bkblz.config, {
           :file_id => file_id,
           :dir_path => dir_path,
-          :use_filename => options[:by_name]
+          :use_filename => false,
+          :strip_prefix => options[:strip_prefix]
         }
       end
 
@@ -218,7 +232,8 @@ module Bkblz
   class Cli < Thor
     include Helper
 
-    class_option :config, :desc => "path to a yml config file", :aliases => ["-c"]
+    class_option :config, :desc => "path to a yml config file", :aliases => ["-c"],
+    :default => ::File.expand_path('~/etc/bkblz/bkblz.yml')
 
     desc "bucket SUBCOMMAND ...ARGS", "manage backblaze buckets"
     subcommand "bucket", Bucket

data/lib/bkblz/config.rb

@@ -9,7 +9,9 @@ module Bkblz
 
       :log_device => :stderr, # [:stdout, :stderr, :devnull, path, fd]
       :log_level => :warn, # [:debug, :info, :warn, :error, :fatal, (-6..-1)]
-      :log_colorize => true
+      :log_colorize => true,
+
+      :large_file_max_chunk_size => 1e8, # 100MB
     }.freeze
 
     attr_reader *CONFIG_VARS.keys, :config_map

data/lib/bkblz/task/download_file.rb

@@ -5,6 +5,7 @@ module Bkblz
     class DownloadFile < BaseTask
 
       task_param :dir_path, :required => true
+      task_param :strip_prefix
 
       # for downalod by id
       task_param :file_id
@@ -33,7 +34,12 @@ module Bkblz
           raise "unable to write to directory %s" % dir_path
         end
 
-        f_path = ::File.join dir_path, download_file_info.file_name
+        out_file_name = download_file_info.file_name
+        unless params[:strip_prefix].nil?
+          out_file_name.gsub! params[:strip_prefix], ""
+        end
+
+        f_path = ::File.join dir_path, out_file_name
         if ::File.exists?(f_path) && !::File.writable?(f_path)
           raise "unable to write to existing file: %s" % f_path
         end

data/lib/bkblz/task/upload_file.rb

@@ -1,27 +1,112 @@
+require 'zlib'
+
 module Bkblz
   module Task
     class UploadFile < BaseTask
 
       task_param :bucket_name, :required => true
-      task_param :file_path, :required => true
-      task_param :file_name # Overrides local file_path if given
 
-      # TODO(erick): Change file_path to a byte string, let the task be agnostic
-      # to where the data comes from. (Probably v0.2).
+      # Either file_body or file_path is required
+      task_param :file_body
+
+      task_param :file_path
+
+      # Overrides local file_path if given
+      task_param :file_name
+
+      # Use gzip default compression before writing to b2 the file name will be
+      # updated by appending ".gz"
+      task_param :gzip
+
       def run_internal(session, params)
-        f = ::File.new(params[:file_path], "r")
+        file_name = if params[:file_name]
+                      params[:file_name]
+                    elsif params[:file_path]
+                      ::File.basename(params[:file_path])
+                    else
+                      raise 'missing either :file_name or :file_path param'
+                    end
+
+        file_mtime = if params[:file_path]
+                       ::File.mtime(params[:file_path])
+                     else
+                       Time.now
+                     end.to_i * 1000
 
         bucket = find_bucket_by_name session, params[:bucket_name]
+        size = file_size(params)
+
+        file_data = {
+          :file_name => file_name,
+          :file_mtime => file_mtime,
+          :file_size => size,
+          :bucket => bucket,
+        }
+
+        if size > config.large_file_max_chunk_size
+          upload_large_file session, params, **file_data
+        else
+          upload_file session, params, **file_data
+        end
+      end
+
+      private
+      def file_size(params)
+        if params[:file_path]
+          ::File.new(params[:file_path]).size
+        elsif params[:file_body]
+          params[:file_body].size
+        end
+      end
+
+      def upload_large_file(session, params, bucket:, file_name:, file_mtime:, **file_data)
+        start_large_file_info = session.send(Bkblz::V1::StartLargeFileRequest.new(
+            bucket.bucket_id, file_name, file_mtime)).to_model
+        file_id = start_large_file_info.file_id
+
+        upload_part_auth = session.send(Bkblz::V1::GetUploadPartUrlRequest.new(file_id)).to_model
+
+        actual_size = file_data[:file_size]
+        chunk_size = config.large_file_max_chunk_size
+        num_chunks = (actual_size / chunk_size.to_f).ceil
+
+        file_io = if params[:file_path]
+                    ::File.new(params[:file_path], "rb")
+                  else
+                    raise 'only file_path is supported for large file uploads'
+                  end
+
+        upload_part_infos = (0..num_chunks - 1).map do |chunk_i|
+          session.send(Bkblz::V1::UploadPartRequest.new(
+              upload_part_auth, file_io, chunk_i, chunk_size)).to_model
+        end
+
+        file_info = session.send(
+          Bkblz::V1::FinishLargeFileRequest.new(file_id, upload_part_infos)).to_model
+      end
+
+      def upload_file(session, params, bucket:, file_name:, file_mtime:, **file_data)
+        file_body = if params[:file_path]
+                      f = ::File.new(params[:file_path], "r")
+                      f.read
+                    elsif params[:file_body]
+                      params[:file_body]
+                    else
+                      raise 'missing either :file_body or :file_path param'
+                    end
+
+        if params[:gzip]
+          # https://ruby-doc.org/stdlib-2.4.2/libdoc/zlib/rdoc/Zlib.html#method-c-gzip
+          file_body = Zlib.gzip file_body, level: Zlib::DEFAULT_COMPRESSION
+          file_name << ".gz" unless file_name =~ /\.gz$/
+        end
+
         upload_auth = session.send(
           Bkblz::V1::GetUploadUrlRequest.new bucket.bucket_id).to_model
 
-        file_name = params[:file_name] || ::File.basename(f.path)
-        file_body = f.read
-        mtime_millis = f.mtime.to_i * 1000
-
         upload_file_info = session.send(
-          Bkblz::V1::UploadFileRequest.new upload_auth, file_body, file_name,
-                                           nil, mtime_millis).to_model
+          Bkblz::V1::UploadFileRequest.new(
+            upload_auth, file_body, file_name, nil, file_mtime)).to_model
       end
     end
   end

data/lib/bkblz/v1/all.rb

@@ -7,13 +7,17 @@ require_relative "request"
 require_relative "error_response"
 
 require_relative "authorize_account"
-require_relative "list_buckets"
 require_relative "create_bucket"
 require_relative "delete_bucket"
+require_relative "delete_file_version"
+require_relative "download_file"
+require_relative "finish_large_file"
 require_relative "get_file_info"
+require_relative "get_upload_part_url"
 require_relative "get_upload_url"
-require_relative "upload_file"
+require_relative "list_buckets"
 require_relative "list_file_names"
 require_relative "list_file_versions"
-require_relative "delete_file_version"
-require_relative "download_file"
+require_relative "start_large_file"
+require_relative "upload_file"
+require_relative "upload_part"

data/lib/bkblz/v1/finish_large_file.rb

@@ -0,0 +1,30 @@
+require "digest/sha1"
+
+module Bkblz
+  module V1
+
+    class FinishLargeFileResponse < Response
+      response_model Model::FileInfo
+    end
+
+    class FinishLargeFileRequest < Request
+
+      response_class FinishLargeFileResponse
+      url_suffix "/b2api/v1/b2_finish_large_file"
+
+      def initialize(file_id, file_part_infos)
+        sha1_sums = file_part_infos.sort { |a, b| a.part_number <=> b.part_number }.map do |info|
+          info.content_sha1
+        end
+        @body = {
+          :file_id => file_id,
+          :part_sha1_array => sha1_sums,
+        }
+      end
+
+      def build_request(session)
+        session.create_post url(session), @body
+      end
+    end
+  end
+end

data/lib/bkblz/v1/get_upload_part_url.rb

@@ -0,0 +1,22 @@
+module Bkblz
+  module V1
+
+    class GetUploadPartUrlResponse < Response
+      response_model Model::UploadPartAuth
+    end
+
+    class GetUploadPartUrlRequest < Request
+
+      response_class GetUploadPartUrlResponse
+      url_suffix "/b2api/v1/b2_get_upload_part_url"
+
+      def initialize(file_id)
+        @body = {:file_id => file_id}
+      end
+
+      def build_request(session)
+        session.create_post url(session), @body
+      end
+    end
+  end
+end

data/lib/bkblz/v1/models.rb

@@ -15,18 +15,26 @@ module Bkblz
         :body, :content_length, :content_type, :file_id, :file_name, :sha1, :x_bz_info
       ]
 
-      # Returned by upload_file
+      # Returned by upload_file, finish_large_file, start_large_file
       FileInfo = Model.define *[
-        :account_id, :bucket_id, :content_length, :content_sha1, :content_type,
+        :account_id, :action, :bucket_id, :content_length, :content_sha1, :content_type,
         :file_id, :file_info, :file_name
       ]
 
+      # Returned by upload_file
+      FilePartInfo = Model.define *[
+        :file_id, :part_number, :content_length, :content_sha1, :upload_timestamp,
+      ]
+
       # Returned by delete_file_version
       PartialFileInfo = Model.define :file_id, :file_name
 
       # Returned by get_upload_url
       UploadAuth = Model.define :bucket_id, :upload_url, :authorization_token
 
+      # Returned by get_upload_part_url
+      UploadPartAuth = Model.define :file_id, :upload_url, :authorization_token
+
       # Possibly returned by any request
       Error = Model.define :status, :code, :message
     end

data/lib/bkblz/v1/start_large_file.rb

@@ -0,0 +1,40 @@
+require "digest/sha1"
+
+module Bkblz
+  module V1
+
+    class StartLargeFileResponse < Response
+      response_model Model::FileInfo
+    end
+
+    class StartLargeFileRequest < Request
+
+      response_class StartLargeFileResponse
+      url_suffix "/b2api/v1/b2_start_large_file"
+
+      def initialize(bucket_id, file_name, last_modified_millis=nil,
+          file_sha=nil, content_type='b2/x-auto', **file_info)
+        # Both of the following are recommended here:
+        # https://www.backblaze.com/b2/docs/b2_start_large_file.html
+        if last_modified_millis
+          # must be a string
+          file_info[:src_last_modified_millis] = last_modified_millis.to_s
+        end
+        if file_sha
+          file_info[:large_file_sha1] = file_sha
+        end
+
+        @body = {
+          :bucket_id => bucket_id,
+          :file_name => file_name,
+          :content_type => content_type,
+          :file_info => file_info
+        }
+      end
+
+      def build_request(session)
+        session.create_post url(session), @body
+      end
+    end
+  end
+end

data/lib/bkblz/v1/upload_part.rb

@@ -0,0 +1,53 @@
+require "digest/sha1"
+
+module Bkblz
+  module V1
+
+    class UploadPartResponse < Response
+      response_model Model::FilePartInfo
+    end
+
+    class UploadPartRequest < Request
+
+      response_class UploadPartResponse
+
+      REQUIRED_HEADERS = {
+                          :"Authorization" => nil,
+                          :"Content-Length" => nil,
+                          :"Content-Type" => 'application/octet-stream',
+                          :"X-Bz-Part-Number" => nil, # a value in [1..10000]
+                          :"X-Bz-Content-Sha1" => nil
+      }
+
+      ##
+      # @param {chunk_number} is a value in [0...9999]
+      def initialize(upload_part_auth, io, chunk_number, chunk_size)
+        @upload_url = upload_part_auth.upload_url
+        @body_chunk = read_chunk(io, chunk_number, chunk_size)
+        @headers = REQUIRED_HEADERS.dup
+
+        part_number = chunk_number + 1
+        @headers[:"Authorization"] = upload_part_auth.authorization_token
+        @headers[:"Content-Length"] = @body_chunk.size
+        @headers[:"X-Bz-Part-Number"] = part_number
+        @headers[:"X-Bz-Content-Sha1"] = Digest::SHA1.hexdigest @body_chunk
+      end
+
+      def build_request(session)
+        session.create_post @upload_url, @body_chunk, @headers
+      end
+
+      private
+      def read_chunk(io, chunk_number, chunk_size)
+        unless io.is_a?(IO)
+          raise 'only IO type is supported for upload_part'
+        end
+
+        byte = chunk_number * chunk_size
+        # https://ruby-doc.org/core-2.5/IO.html#method-i-seek
+        io.seek byte, IO::SEEK_SET
+        io.read(chunk_size)
+      end
+    end
+  end
+end

data/lib/bkblz/version.rb

@@ -1,3 +1,3 @@
 module Bkblz
-  VERSION = "0.1.9"
+  VERSION = "0.1.14"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bkblz
 version: !ruby/object:Gem::Version
-  version: 0.1.9
+  version: 0.1.14
 platform: ruby
 authors:
 - Erick Johnson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-27 00:00:00.000000000 Z
+date: 2020-11-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -105,7 +105,9 @@ files:
 - lib/bkblz/v1/delete_file_version.rb
 - lib/bkblz/v1/download_file.rb
 - lib/bkblz/v1/error_response.rb
+- lib/bkblz/v1/finish_large_file.rb
 - lib/bkblz/v1/get_file_info.rb
+- lib/bkblz/v1/get_upload_part_url.rb
 - lib/bkblz/v1/get_upload_url.rb
 - lib/bkblz/v1/list_buckets.rb
 - lib/bkblz/v1/list_file_names.rb
@@ -115,7 +117,9 @@ files:
 - lib/bkblz/v1/request.rb
 - lib/bkblz/v1/response.rb
 - lib/bkblz/v1/session.rb
+- lib/bkblz/v1/start_large_file.rb
 - lib/bkblz/v1/upload_file.rb
+- lib/bkblz/v1/upload_part.rb
 - lib/bkblz/version.rb
 homepage: https://github.com/erickj/bkblz
 licenses:
@@ -136,8 +140,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.13
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Bkblz GEM for the Backblaze B2 API. https://www.backblaze.com/b2/docs/