backup-backblaze 0.1.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 705a6fd53f7173dc6f8209bd8dd889ad49098f43
+  data.tar.gz: f633db6bde19d0cba7c95cd84b49225f0fb74647
+SHA512:
+  metadata.gz: 6a945e4dc4597f54f5b15056f6cbb5edd6e0f4494f3412951d47dee96aab2c0d173a97b36d1992d36d94302cd26b8c51504845ac97b12ed01588219cbf0f02b6
+  data.tar.gz: a6864d6aa9d249b432c4e0ed61f778434ae61ae3f4e6eca2b01319a81b3390c9e7473ebbe63fc6c64020d3c123690fe7aebba9eb17e0756513af85ae3b799906

data/.gitignore

@@ -0,0 +1,13 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+Gemfile.lock

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.rvmrc

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+# This is an RVM Project .rvmrc file, used to automatically load the ruby
+# development environment upon cd'ing into the directory
+
+# First we specify our desired <ruby>[@<gemset>], the @gemset name is optional,
+# Only full ruby name is supported here, for short names use:
+#     echo "rvm use 2.5.0@b2" > .rvmrc
+#environment_id="ruby-2.5.0@b2"
+environment_id="ruby-2.3.6@backup-backblaze"
+
+# Uncomment the following lines if you want to verify rvm version per project
+# rvmrc_rvm_version="1.29.3 (master)" # 1.10.1 seems like a safe start
+# eval "$(echo ${rvm_version}.${rvmrc_rvm_version} | __rvm_awk -F. '{print "[[ "$1*65536+$2*256+$3" -ge "$4*65536+$5*256+$6" ]]"}' )" || {
+#   echo "This .rvmrc file requires at least RVM ${rvmrc_rvm_version}, aborting loading."
+#   return 1
+# }
+
+# First we attempt to load the desired environment directly from the environment
+# file. This is very fast and efficient compared to running through the entire
+# CLI and selector. If you want feedback on which environment was used then
+# insert the word 'use' after --create as this triggers verbose mode.
+if [[ -d "${rvm_path:-$HOME/.rvm}/environments"
+  && -s "${rvm_path:-$HOME/.rvm}/environments/$environment_id" ]]
+then
+  \. "${rvm_path:-$HOME/.rvm}/environments/$environment_id"
+  for __hook in "${rvm_path:-$HOME/.rvm}/hooks/after_use"*
+  do
+    if [[ -f "${__hook}" && -x "${__hook}" && -s "${__hook}" ]]
+    then \. "${__hook}" || true
+    fi
+  done
+  unset __hook
+  if (( ${rvm_use_flag:=1} >= 2 )) # display only when forced
+  then
+    if [[ $- == *i* ]] # check for interactive shells
+    then printf "%b" "Using: $(tput setaf 2 2>/dev/null)$GEM_HOME$(tput sgr0 2>/dev/null)\n" # show the user the ruby and gemset they are using in green
+    else printf "%b" "Using: $GEM_HOME\n" # don't use colors in non-interactive shells
+    fi
+  fi
+else
+  # If the environment file has not yet been created, use the RVM CLI to select.
+  rvm --create  "$environment_id" || {
+    echo "Failed to create RVM environment '${environment_id}'."
+    return 1
+  }
+fi

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.5.0
+before_install: gem install bundler -v 1.15.1

data/Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in backup-backblaze.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 John Anderson
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,60 @@
+# Backup::Backblaze
+
+Plugin for the [Backup](https://github.com/backup/backup) gem to use [Backblaze](https://www.backblaze.com/) as storage.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'backup-backblaze'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install backup-backblaze
+
+## Usage
+
+  Add a storage block something like this to your usual Backup DSL file:
+
+    # BackBlaze must be a string here, not a class name. Because it's defined external to Backup gem.
+    store_with 'BackBlaze' do |server|
+      # from backblaze ui
+      server.account_id = 'deadbeefdead'
+      server.app_key    = 'c27111357f682232c9943f6e63e98f916722c975e4'
+
+      # bucket name must be globally unique (yes, really).
+      # create buckets on the backblaze website. app_key must have access.
+      server.bucket     = 'your_globally_unique_bucket_name'
+
+      # path defaults to '/'
+      server.path       = '/whatever/you_like'
+      server.keep       = 3
+
+      # minimum is 5mb, default is 100mb. Leave at default unless you have a good reason.
+      # server.part_size = 5000000
+    end
+
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/djellemah/backup-backblaze.
+
+## Acknowledgements
+
+Thanks to [NETSTOCK](https://www.netstock.co/) for funding development.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/backup-backblaze.gemspec

@@ -0,0 +1,39 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'backup/backblaze/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'backup-backblaze'
+  spec.version       = Backup::Backblaze::VERSION
+  spec.authors       = ['John Anderson']
+  spec.email         = ['panic@semiosix.com']
+
+  spec.summary       = %q{Backup plugin for BackBlaze}
+  spec.description   = %q{BackBlaze provides cloud storage. This makes it available to the Backup gem.}
+  spec.homepage      = 'http://github.com/djellemah/backup-backblaze'
+  spec.license       = 'MIT'
+
+  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
+  # to allow pushing to a single host or delete this section to allow pushing to any host.
+  if spec.respond_to?(:metadata)
+    spec.metadata['allowed_push_host'] = 'https://rubygems.org'
+  else
+    raise 'RubyGems 2.0 or newer is required to protect against public gem pushes.'
+  end
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.15'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency 'pry'
+
+  spec.add_dependency 'backup'
+  spec.add_dependency 'excon'
+end

data/bin/console

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "backup/backblaze"
+
+# You can add fixtures and/or initialization code here to make experimenting # with your gem easier.
+require "pry"
+Pry.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/backup/backblaze.rb

@@ -0,0 +1,9 @@
+require 'backup'
+
+module Backup
+  module Backblaze
+  end
+end
+
+require_relative 'backblaze/version'
+require_relative 'backblaze/back_blaze'

data/lib/backup/backblaze/account.rb

@@ -0,0 +1,160 @@
+require 'base64'
+require 'excon'
+require 'json'
+
+require_relative 'hash_wrap'
+
+module Backup
+  module Backblaze
+    class Account
+      def initialize account_id:, app_key:
+        @account_id = account_id
+        @app_key = app_key
+        auth!
+      end
+
+      attr_reader :account_id, :app_key, :body
+
+      class NotFound < RuntimeError; end
+
+      def auth!
+        # first call b2_authorize_account to get an account_auth_token
+        encoded = Base64.strict_encode64 "#{account_id}:#{app_key}"
+        rsp = Excon.get \
+          'https://api.backblazeb2.com/b2api/v1/b2_authorize_account',
+          headers: {'Authorization' => "Basic #{encoded}"},
+          expects: 200
+
+        # this has to stick around because it has various important data
+        @body = HashWrap.from_json rsp.body
+
+        unless body.allowed.capabilities.include? 'writeFiles'
+          raise "app_key #{app_key} does not have write access to account #{account_id}"
+        end
+      end
+
+      def auth_headers
+        {headers: {'Authorization' => authorization_token}}
+      end
+
+      def api_url
+        body.apiUrl or raise NotFound, 'apiUrl'
+      end
+
+      def authorization_token
+        body.authorizationToken or raise NotFound, 'authorizationToken'
+      end
+
+      def minimum_part_size
+        # why b2 has this as well as minimumPartSize ¯\_(ツ)_/¯
+        body.absoluteMinimumPartSize
+      end
+
+      def recommended_part_size
+        body.recommendedPartSize
+      end
+
+      # The following is leaning towards Bucket.new account, bucket_id/bucket_name
+
+      # returns [upload_url, auth_token]
+      # Several files can be uploaded to one url.
+      # But uploading files in parallel requires one upload url per thread.
+      def upload_url bucket_id:
+        # get the upload url for a specific bucket id. Buckets can be named.
+        body = {bucketId: bucket_id }
+        rsp = Excon.post \
+          "#{api_url}/b2api/v1/b2_get_upload_url",
+          **auth_headers,
+          body: body.to_json,
+          expects: 200
+
+        hw = HashWrap.from_json rsp.body
+        return hw.uploadUrl, hw.authorizationToken
+      end
+
+      # return id for given name, or nil if no such named bucket
+      def bucket_id bucket_name:
+        rsp = Excon.post \
+          "#{api_url}/b2api/v1/b2_list_buckets",
+          **auth_headers,
+          body: {bucketName: bucket_name, accountId: account_id}.to_json,
+          expects: 200
+
+        buckets = (JSON.parse rsp.body)['buckets']
+        found = buckets.find do |ha|
+          ha['bucketName'] == bucket_name
+        end
+        found&.dig 'bucketId' or raise NotFound, "no bucket named #{bucket_name}"
+      end
+
+      # Hurhur
+      def bucket_list bucket_id: nil
+        b2_list_buckets bucketId: bucket_id, accountId: account_id
+      end
+
+      def b2_list_buckets body
+        rsp = Excon.post \
+          "#{api_url}/b2api/v1/b2_list_buckets",
+          **auth_headers,
+          body: body.select{|_,v|v}.to_json,
+          expects: 200
+
+        HashWrap.from_json rsp
+      end
+
+      # This might be dangerous because large number of file names might come back.
+      # But I'm not worrying about that now. Maybe later. Anyway, that's what
+      # nextFile and startFile are for.
+      def files bucket_name
+        rsp = Excon.post \
+          "#{api_url}/b2api/v1/b2_list_file_names",
+          **auth_headers,
+          body: {bucketId: (bucket_id bucket_name: bucket_name)}.to_json,
+          expects: 200
+
+        # ignoring the top-level {files:, nextFileName:} structure
+        files_hash = (JSON.parse rsp.body)['files']
+
+        # ignoring the top-level {files:, nextFileName:} structure
+        files_hash.map do |file_info_hash|
+          HashWrap.new file_info_hash
+        end
+      end
+
+      # This is mostly used to get a fileId for a given fileName
+      def file_info bucket_name, filename
+        # It's too much of a PITA to make this Excon call in only one place
+        rsp = Excon.post \
+          "#{api_url}/b2api/v1/b2_list_file_names",
+          **auth_headers,
+          body: {bucketId: (bucket_id bucket_name: bucket_name), maxFileCount: 1, startFileName: filename}.to_json,
+          expects: 200
+
+        files_hash = (JSON.parse rsp.body)['files']
+
+        raise NotFound, "#{filename} not found" unless files_hash.size == 1
+
+        HashWrap.new files_hash.first
+      end
+
+      # delete the named file in the named bucket
+      # https://www.backblaze.com/b2/docs/b2_delete_file_version.html
+      def delete_file bucket_name, filename
+        # lookup fileId from given filename
+        info = file_info bucket_name, filename
+
+        # delete the fileId
+        Excon.post \
+          "#{api_url}/b2api/v1/b2_delete_file_version",
+          **auth_headers,
+          body: {fileName: filename, fileId: info.fileId}.to_json,
+          expects: 200
+
+      # ignore 400 with body containing "code": "file_not_present"
+      rescue Excon::Errors::BadRequest => ex
+        hw = HashWrap.from_json ex.response.body
+        raise unless hw.code == 'file_not_present'
+      end
+    end
+  end
+end

data/lib/backup/backblaze/back_blaze.rb

@@ -0,0 +1,124 @@
+require 'excon'
+require 'base64'
+require 'json'
+require 'pathname'
+
+require_relative 'upload_file.rb'
+require_relative 'upload_large_file.rb'
+require_relative 'account.rb'
+require_relative 'retry.rb'
+
+require 'backup/storage/base'
+
+# module naming like this is required by Backup to find the storage
+module Backup
+  module Storage
+    # Different naming to module
+    class BackBlaze < Base
+      include Backup::Storage::Cycler
+
+      class ConfigurationError < Backup::Error; end
+
+      # Values specified in Model DSL:
+      # - API credentials
+      # - bucket name
+      REQUIRED_ATTRS = %i[account_id app_key bucket]
+      attr_accessor *REQUIRED_ATTRS
+
+      # - part size for large files
+      attr_accessor :part_size
+
+      def initialize(model, storage_id = nil)
+        super
+        @path ||= '/'
+        check_configuration
+      end
+
+      protected
+
+      def check_configuration
+        not_specified = REQUIRED_ATTRS.reject{|name| send name}
+        if not_specified.any?
+          raise ConfigurationError, "#{not_specified.join(", ")} required"
+        end
+
+        if part_size && part_size < account.minimum_part_size
+          raise ConfigurationError, "part_size must be > #{account.minimum_part_size}"
+        end
+      end
+
+      def remote_relative_pathname
+        @remote_relative_pathname ||= Pathname.new(remote_path).relative_path_from(root)
+      end
+
+      def root; @root ||= Pathname.new '/'; end
+      def tmp_dir; @tmp_dir ||= Pathname.new Config.tmp_path; end
+
+      def working_part_size
+        @working_part_size ||= part_size || account.recommended_part_size
+      end
+
+      def transfer!
+        bucket_id = account.bucket_id bucket_name: bucket
+
+        package.filenames.each do |filename|
+          dst = (remote_relative_pathname + filename).to_s
+          src_pathname = tmp_dir + filename
+
+          upload =
+          if src_pathname.size > working_part_size * 2.5 || src_pathname.size > 5 * 10**9
+            Logger.info "Storing Large '#{dst}'"
+            ::Backup::Backblaze::UploadLargeFile.new \
+              src: src_pathname,
+              dst: dst,
+              authorization_token: account.authorization_token,
+              url: account.api_url,
+              part_size: working_part_size,
+              bucket_id: bucket_id
+          else
+            Logger.info "Storing '#{dst}'"
+
+            # TODO could upload several files in parallel with several of these token_provider
+            token_provider = ::Backup::Backblaze::Retry::TokenProvider.new do
+              account.upload_url bucket_id: bucket_id
+            end
+
+            ::Backup::Backblaze::UploadFile.new \
+              src: src_pathname.to_s,
+              dst: dst,
+              token_provider: token_provider
+          end
+
+          hash_wrap = upload.call
+
+          Logger.info "'#{dst}' stored at #{hash_wrap.fileName}"
+        end
+      end
+
+      # Called by the Cycler.
+      # Any error raised will be logged as a warning.
+      def remove!(package)
+        Logger.info "Removing backup package dated #{package.time}"
+
+        # workaround for stoopid design in Backup
+        package_remote_relative_pathname = Pathname.new(remote_path(package)).relative_path_from(root)
+
+        package.filenames.each do |filename|
+          dst = (package_remote_relative_pathname + filename).to_s
+          Logger.info "Removing file #{dst}"
+          account.delete_file bucket, dst
+        end
+      end
+
+      protected
+
+      def account
+        @account ||= begin
+          account_deets = {account_id: account_id}
+          Logger.info "Account login for #{account_deets.inspect}"
+          ::Backup::Backblaze::Account.new account_id: account_id, app_key: app_key
+        end
+      end
+    end
+  end
+end

data/lib/backup/backblaze/hash_wrap.rb

@@ -0,0 +1,43 @@
+module Backup
+  module Backblaze
+    # Intended as a quick-n-dirty way to deep-wrap json objects.
+    # If it doesn't work for you, rather than scope-creeping this consider: Hash, OpenStruct, a class, etc.
+    class HashWrap
+      def initialize( hash )
+        @hash = hash
+      end
+
+      def method_missing(meth, *args, &blk)
+        value = @hash.fetch meth.to_s do |_key|
+          @hash.fetch meth do |_key|
+            super
+          end
+        end
+        __wrap value
+      end
+
+      private def __wrap value
+        case value
+        when Hash
+          self.class.new value
+        when Array
+          value.map do |item|
+            __wrap item
+          end
+        else
+          value
+        end
+      end
+
+      def to_h
+        # no, you can't have a copy of this hash to mess with
+        @hash.dup
+      end
+
+      # really a convenience method
+      def self.from_json json
+        new JSON.parse json
+      end
+    end
+  end
+end

data/lib/backup/backblaze/retry.rb

@@ -0,0 +1,78 @@
+module Backup
+  module Backblaze
+    module Retry
+      MAX_RETRIES = 3
+
+      # Use the url and token returned by the next_url_token block, until we get
+      # a reset indicating that we need a new url and token.
+      class TokenProvider
+        def initialize &next_url_token
+          @next_url_token = next_url_token
+          reset
+        end
+
+        attr_reader :upload_url, :file_auth_token
+
+        def reset
+          @upload_url, @file_auth_token = @next_url_token.call
+          self
+        end
+      end
+
+      class TooManyRetries < RuntimeError; end
+
+      # Try up to retries times to call the upload_blk. Recursive.
+      #
+      # Various errors (passed through from Excon) coming out of upload_blk will
+      # be caught. When an error is caught, :reset method called on
+      # token_provider.
+      #
+      # Return whatever upload_blk returns
+      def retry_upload retries, token_provider, &upload_blk
+        raise TooManyRetries, "max retries is #{MAX_RETRIES}" unless retries < MAX_RETRIES
+        sleep retries ** 2 # exponential backoff for retries > 0
+
+        # Called by all the rescue blocks that want to retry.
+        # Mainly so we don't make stoopid errors - like leaving out the +1 for one of the calls :-|
+        retry_lambda = lambda do
+          retry_upload retries + 1, token_provider.reset, &upload_blk
+        end
+
+        begin
+          upload_blk.call token_provider, retries
+        rescue Excon::Errors::Error => ex
+          # The most convenient place to log this
+          Backup::Logger.info ex.message
+          raise
+        end
+
+      # Recoverable errors details sourced from:
+      #   https://www.backblaze.com/b2/docs/integration_checklist.html
+      #   https://www.backblaze.com/b2/docs/uploading.html
+
+      # socket-related, 408, and 429
+      rescue Excon::Errors::SocketError, Excon::Errors::Timeout, Excon::Errors::RequestTimeout, Excon::Errors::TooManyRequests
+        retry_lambda.call
+
+      # some 401
+      rescue Excon::Errors::Unauthorized => ex
+        hw = HashWrap.from_json ex.response.body
+        case hw.code
+        when 'bad_auth_token', 'expired_auth_token'
+          retry_lambda.call
+        else
+          raise
+        end
+
+      # 500-599 where the BackBlaze "code" doesn't matter
+      rescue Excon::Errors::HTTPStatusError => ex
+        if (500..599) === ex.response.status
+          retry_lambda.call
+        else
+          raise
+        end
+
+      end
+    end
+  end
+end

data/lib/backup/backblaze/upload_file.rb

@@ -0,0 +1,111 @@
+require 'digest'
+
+require_relative 'retry.rb'
+
+module Backup
+  module Backblaze
+    # calculates sha1 and uploads file
+    # Of course, this entire class is an atomic failure, because the underlying file could change at any point.
+    #
+    # dst can contain / for namespaces
+    class UploadFile
+      def initialize src:, dst:, token_provider:, content_type: nil
+        @src = src
+        @dst = dst
+        @content_type = content_type
+        @token_provider = token_provider
+      end
+
+      attr_reader :src, :dst, :token_provider, :content_type
+
+      def headers
+        # headers all have to be strings, otherwise excon & Net::HTTP choke :-|
+        {
+          'X-Bz-File-Name'                     => (URI.encode dst.encode 'UTF-8'),
+          'X-Bz-Content-Sha1'                  => sha1_digest,
+          'Content-Length'                     => content_length.to_s,
+          'Content-Type'                       => content_type,
+
+          # optional
+          'X-Bz-Info-src_last_modified_millis' => last_modified_millis.to_s,
+          'X-Bz-Info-b2-content-disposition'   => content_disposition,
+        }.select{|k,v| v}
+      end
+
+      def content_type
+        @content_type || 'b2/x-auto'
+      end
+
+      # No idea what has to be in here
+      def content_disposition
+      end
+
+      def content_length
+        File.size src
+      end
+
+      def sha1
+        @sha1 = Digest::SHA1.file src
+      end
+
+      def sha1_digest
+        @sha1_digest = sha1.hexdigest
+      end
+
+      def last_modified_millis
+        @last_modified_millis ||= begin
+          time = File.lstat(src).mtime
+          time.tv_sec * 1000 + time.tv_usec / 1000
+        end
+      end
+
+      include Retry
+
+      # upload with incorrect sha1 responds with
+      #
+      # {"code"=>"bad_request", "message"=>"Sha1 did not match data received", "status"=>400}
+      #
+      # Normal response
+      #
+      #{"accountId"=>"d765e276730e",
+      # "action"=>"upload",
+      # "bucketId"=>"dd8786b5eef2c7d66743001e",
+      # "contentLength"=>6144,
+      # "contentSha1"=>"5ba6cf1b3b3a088d73941052f60e78baf05d91fd",
+      # "contentType"=>"application/octet-stream",
+      # "fileId"=>"4_zdd8786b5eef2c7d66743001e_f1096f3027e0b1927_d20180725_m115148_c002_v0001095_t0047",
+      # "fileInfo"=>{"src_last_modified_millis"=>"1532503455580"},
+      # "fileName"=>"test_file",
+      # "uploadTimestamp"=>1532519508000}
+      def call
+        retry_upload 0, token_provider do |token_provider, retries|
+          Backup::Logger.info "#{src} retry #{retries}"
+          rsp = Excon.post \
+            token_provider.upload_url,
+            headers: (headers.merge 'Authorization' => token_provider.file_auth_token),
+            body: (File.read src),
+            expects: 200
+
+          HashWrap.from_json rsp.body
+        end
+      end
+
+      # Seems this doesn't work. Fails with
+      #
+      # 400 Missing header: Content-Length
+      #
+      # Probably because chunked encoding doesn't send an initial Content-Length
+      private def excon_stream_upload( upload )
+        File.open src do |io|
+          chunker = lambda do
+            # Excon.defaults[:chunk_size] defaults to 1048576, ie 1MB
+            # to_s will convert the nil received after everything is read to the final empty chunk
+            io.read(Excon.defaults[:chunk_size]).to_s
+          end
+
+          Excon.post url, headers: headers, :request_block => chunker, debug_request: true, debug_response: true, instrumentor: Excon::StandardInstrumentor
+        end
+      end
+    end
+  end
+end

data/lib/backup/backblaze/upload_large_file.rb

@@ -0,0 +1,173 @@
+require 'digest'
+require_relative 'hash_wrap'
+require_relative 'retry'
+
+module Backup
+  module Backblaze
+    # Upload a large file in several parts.
+    class UploadLargeFile
+      # src is a Pathname
+      # dst is a String
+      def initialize src:, dst:, authorization_token:, content_type: nil, url:, part_size:, bucket_id:
+        @src = src
+        @dst = dst
+        @authorization_token = authorization_token
+        @content_type = content_type
+        @url = url
+        @part_size = part_size
+        @bucket_id = bucket_id
+      end
+
+      attr_reader :src, :dst, :authorization_token, :url, :content_type, :part_size, :bucket_id
+
+      # same as account
+      def auth_headers
+        # only cos the double {{}} is a quite ugly :-p
+        Hash headers: {'Authorization' => authorization_token}
+      end
+
+      def content_type
+        @content_type || 'b2/x-auto'
+      end
+
+      # No idea what has to be in here
+      def content_disposition
+      end
+
+      def content_length
+        src.size
+      end
+
+      def last_modified_millis
+        @last_modified_millis ||= begin
+          time = File.lstat(src).mtime
+          time.tv_sec * 1000 + time.tv_usec / 1000
+        end
+      end
+
+      # https://www.backblaze.com/b2/docs/b2_start_large_file.html
+      # definitely need fileInfo back from this. Maybe also uploadTimestamp not sure yet.
+      def b2_start_large_file
+        # Unlike in UploadFile, it's OK to use symbols here cos to_json converts them to strings
+        body = {
+          bucketId: bucket_id,
+          fileName: dst,
+          contentType: content_type,
+          fileInfo: {
+            src_last_modified_millis: last_modified_millis.to_s,
+            'b2-content-disposition': content_disposition
+            # this seems to be optional, and is hard to calculate for large file up
+            # front. So don't send it.
+            # large_file_sha1: sha1_digest,
+          }.select{|k,v| v}
+        }
+
+        rsp = Excon.post \
+          "#{url}/b2api/v1/b2_start_large_file",
+          **auth_headers,
+          body: body.to_json,
+          expects: 200
+
+        HashWrap.from_json rsp.body
+      end
+
+      def file_id
+        @file_id ||= b2_start_large_file.fileId
+      end
+
+      def b2_get_upload_part_url
+        rsp = Excon.post \
+          "#{url}/b2api/v1/b2_get_upload_part_url",
+          **auth_headers,
+          body: {fileId: file_id}.to_json,
+          expects: 200
+
+        hash = JSON.parse rsp.body
+        return hash.values_at 'uploadUrl', 'authorizationToken'
+      end
+
+      # NOTE Is there a way to stream this instead of loading multiple 100M chunks
+      # into memory? No, backblaze does not allow parts to use chunked encoding.
+      def b2_upload_part sequence, upload_url, file_auth_token, &log_block
+        # read length, offset
+        bytes = src.read part_size, part_size * sequence
+
+        # return nil if the read comes back as a nil, ie no bytes read
+        return if bytes.nil? || bytes.empty?
+
+        # This is a bit weird. But not so weird that it needs fixing.
+        log_block.call
+
+        headers = {
+          # not the same as the auth_headers value
+          'Authorization' => file_auth_token,
+          # cos backblaze wants 1-based, but we want 0-based for reading file
+          'X-Bz-Part-Number' => sequence + 1,
+          'Content-Length' => bytes.length,
+          'X-Bz-Content-Sha1' => (sha = Digest::SHA1.hexdigest bytes),
+        }
+
+        # Yes, this is a different pattern to the other Excon.post calls ¯\_(ツ)_/¯
+        rsp = Excon.post \
+          upload_url,
+          headers: headers,
+          body: bytes,
+          expects: 200
+
+        # 200 response will be
+        # fileId The unique ID for this file.
+        # partNumber Which part this is.
+        # contentLength The number of bytes stored in the part.
+        # contentSha1 The SHA1 of the bytes stored in the part.
+
+        # return for the sha collection
+        sha
+      end
+
+      def b2_finish_large_file shas
+        rsp = Excon.post \
+          "#{url}/b2api/v1/b2_finish_large_file",
+          **auth_headers,
+          body: {fileId: file_id, partSha1Array: shas }.to_json,
+          expects: 200
+
+        HashWrap.from_json rsp.body
+      end
+
+      # 10000 is backblaze specified max number of parts
+      MAX_PARTS = 10000
+
+      include Retry
+
+      def call
+        if src.size > part_size * MAX_PARTS
+          raise Error, "File #{src.to_s} has size #{src.size} which is larger than part_size * MAX_PARTS #{part_size * MAX_PARTS}. Try increasing part_size in model."
+        end
+
+        # TODO could have multiple threads here, each would need a separate token_provider
+        token_provider = TokenProvider.new &method(:b2_get_upload_part_url)
+        shas = (0...MAX_PARTS).each_with_object [] do |sequence, shas|
+          sha = retry_upload 0, token_provider do |token_provider, retries|
+            # return sha
+            b2_upload_part sequence, token_provider.upload_url, token_provider.file_auth_token do
+              Backup::Logger.info "#{src} trying part #{sequence + 1} of #{(src.size / part_size.to_r).ceil} retry #{retries}"
+            end
+          end
+
+          # sha will come back as nil once the file is done.
+          if sha
+            shas << sha
+            Backup::Logger.info "#{src} stored part #{sequence + 1} with #{sha}"
+          else
+            break shas
+          end
+        end
+
+        # finish up, log and return the response
+        hash_wrap = b2_finish_large_file shas
+        Backup::Logger.info "#{src} finished"
+        hash_wrap
+      end
+    end
+  end
+end

data/lib/backup/backblaze/version.rb

@@ -0,0 +1,5 @@
+module Backup
+  module Backblaze
+    VERSION = '0.1.2'
+  end
+end

metadata

@@ -0,0 +1,149 @@
+--- !ruby/object:Gem::Specification
+name: backup-backblaze
+version: !ruby/object:Gem::Version
+  version: 0.1.2
+platform: ruby
+authors:
+- John Anderson
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-08-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: backup
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: excon
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: BackBlaze provides cloud storage. This makes it available to the Backup
+  gem.
+email:
+- panic@semiosix.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".rvmrc"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- backup-backblaze.gemspec
+- bin/console
+- bin/setup
+- lib/backup/backblaze.rb
+- lib/backup/backblaze/account.rb
+- lib/backup/backblaze/back_blaze.rb
+- lib/backup/backblaze/hash_wrap.rb
+- lib/backup/backblaze/retry.rb
+- lib/backup/backblaze/upload_file.rb
+- lib/backup/backblaze/upload_large_file.rb
+- lib/backup/backblaze/version.rb
+homepage: http://github.com/djellemah/backup-backblaze
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.14
+signing_key: 
+specification_version: 4
+summary: Backup plugin for BackBlaze
+test_files: []