fog-backblaze 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e72a3f8088a51fed32e1b1efc3d18772ff03348b
+  data.tar.gz: 635bf11d1a64069912521d3c179fb5e46e67e3ec
+SHA512:
+  metadata.gz: d3af95365d4d87cd941c016e18ebf326c9bcdbebb3d7440b29e1f8bc0557cc416c1fd9d973bc5e9cdf3a0091123bdff9b8c933d7c41e53ec25b682a421585e3c
+  data.tar.gz: 86a1546b2f45cf95bb0f79a93c97b3eaa3201b3d3f9655bd264cf13cc539d0c9e882d892392ab6d8f45e7cde3041bcaec7aa2bfe34d142ab7877e0a99b3bed6b

data/.gitignore

@@ -0,0 +1,8 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.4.0
+before_install: gem install bundler -v 1.15.1

data/Gemfile

@@ -0,0 +1,8 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in fog-backblaze.gemspec
+gemspec
+
+gem "bundler", ">= 1.15"
+gem "rake", "~> 12.0"
+gem "minitest", ">= 5.0"

data/Gemfile.lock

@@ -0,0 +1,34 @@
+PATH
+  remote: .
+  specs:
+    fog-backblaze (0.1.1)
+      fog-core
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    builder (3.2.3)
+    excon (0.61.0)
+    fog-core (2.1.0)
+      builder
+      excon (~> 0.58)
+      formatador (~> 0.2)
+      mime-types
+    formatador (0.2.5)
+    mime-types (3.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2016.0521)
+    minitest (5.11.3)
+    rake (12.3.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (>= 1.15)
+  fog-backblaze!
+  minitest (>= 5.0)
+  rake (~> 12.0)
+
+BUNDLED WITH
+   1.16.1

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 Pavel Evstigneev
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,72 @@
+# Fog::Backblaze
+
+Integration library for gem fog and [Backblaze B2 Cloud Storage](https://www.backblaze.com/b2/cloud-storage.html)
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'fog-backblaze'
+```
+
+Or install it with gem:
+
+```sh
+gem install fog-backblaze
+```
+
+## Usage
+
+```ruby
+require "fog/backblaze"
+
+connection = Fog::Storage.new(
+  provider: 'backblaze',
+  b2_account_id: '123456',
+  b2_account_token: 'aaaaabbbbbccccddddeeeeeffffff111112222223333',
+
+  # optional, used to make some operations faster
+  b2_bucket_name: 'app-test',
+  b2_bucket_id: '6ec42006ec42006ec42',
+
+  logger: Logger.new(STDOUT).tap {|l|
+    l.formatter = proc {|severity, datetime, progname, msg|
+      "#{severity.to_s[0]} - #{datetime.strftime("%T.%L")}: #{msg}\n"
+    }
+  },
+
+  token_cache: 'file.txt'
+)
+```
+
+See [example](examples/example.rb) for more details
+
+## Adding b2_bucket_id
+
+Most of internal operations requires `bucketId` field, to get right value, fog-backblaze will make API request.
+Usually applications use only one bucket and it's id never change (it may change only if we delete bucket and create new one with same name).
+We can eliminate this API request by setting `b2_bucket_id` attribute.
+
+How to get `b2_bucket_id`:
+```ruby
+p connection._get_bucket_id(bucket_name)
+```
+
+## Token Cache
+
+Each request requires authentication token, it comes from `b2_authorize_account` response.
+
+Let's say we want to upload a files, then it will make 4 requests inernally:
+
+1. `b2_authorize_account` - valid for 24 hours
+2. `b2_list_buckets` - to get bucket_id value can be optimized with `:b2_bucket_id` field (should not change)
+3. `b2_get_upload_url` - valid for 24 hours
+4. Send data to URL from step 3
+
+Results of steps 1, 2, 3 can be re-used by saving in TokenCache. It acts as general cachin interface with few predefined implementations:
+
+* In memory store `token_cache: :memory` (default)
+* JSON file store `token_cache: 'file.txt'`
+* Null store (will not cache anything) `token_cache: false` or `token_cache: Fog::Backblaze::TokenCache::NullTokenCache.new`
+* Create your custom, see [token_cache.rb](lib/fog/backblaze/token_cache.rb) for examples

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/*_test.rb"]
+end
+
+task :default => :test

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "fog/backblaze"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/examples/example.rb

@@ -0,0 +1,70 @@
+require 'logger'
+require 'pp'
+
+$:.push(File.expand_path("../../lib", __FILE__))
+
+require_relative "../lib/fog/backblaze"
+
+if !ENV['B2_ACCOUNT_ID'] || ENV['B2_ACCOUNT_ID'] == ""
+  puts "Missing env B2_ACCOUNT_ID"
+  exit 1
+end
+
+if !ENV['B2_ACCOUNT_TOKEN'] || ENV['B2_ACCOUNT_TOKEN'] == ""
+  puts "Missing env B2_ACCOUNT_TOKEN"
+  exit 1
+end
+
+connection = Fog::Storage.new(
+  provider: 'backblaze',
+  b2_account_id: ENV['B2_ACCOUNT_ID'],
+  b2_account_token: ENV['B2_ACCOUNT_TOKEN'],
+
+  #b2_bucket_name: ENV['B2_BUCKET'],
+  #b2_bucket_id: '111222333444',
+
+  logger: ENV['DEBUG'] ? Logger.new(STDOUT) : nil,
+  token_cache: :memory
+)
+
+connection.delete_bucket("fog-smoke-test") rescue nil
+
+puts "Put a bucket..."
+puts "----------------"
+pp connection.put_bucket("fog-smoke-test", public: true).json
+
+puts
+puts "Get the bucket..."
+puts "-----------------"
+pp connection.get_bucket("fog-smoke-test").json
+
+puts
+puts "Put a test file..."
+puts "---------------"
+pp connection.put_object("fog-smoke-test", "my file", "THISISATESTFILE").json
+
+puts
+puts "Get the test file..."
+puts "---------------"
+p connection.get_object("fog-smoke-test", "my file")
+
+puts
+puts "Head file..."
+puts "---------------"
+pp connection.head_object("fog-smoke-test", "my file").headers
+
+puts
+puts "Object URL..."
+puts "---------------"
+p connection.get_object_url("fog-smoke-test", "my file")
+
+puts
+puts "Delete the test file..."
+puts "---------------"
+pp connection.delete_object("fog-smoke-test", "my file").json
+
+puts
+puts "Delete the bucket..."
+puts "------------------"
+pp connection.delete_bucket("fog-smoke-test").json
+puts

data/fog-backblaze.gemspec

@@ -0,0 +1,21 @@
+# coding: utf-8
+
+require_relative 'lib/fog/backblaze/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "fog-backblaze"
+  spec.version       = Fog::Backblaze::VERSION
+  spec.authors       = ["Pavel Evstigneev"]
+  spec.email         = ["pavel.evst@gmail.com"]
+
+  spec.summary       = "Module for the 'fog' gem to support Blackblade B2 stogate."
+  spec.description   = "Blackblade B2 stogate client for 'fog' gem, can be used for working with files and buckets. E.g. carrierwave uploads"
+  spec.homepage      = "https://github.com/fog/fog-backblaze"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "fog-core", ">= 1.40", "<3"
+end

data/lib/fog/backblaze.rb

@@ -0,0 +1,16 @@
+require_relative 'backblaze/version'
+require_relative 'backblaze/token_cache'
+require_relative 'backblaze/json_response'
+
+require 'fog/core'
+require 'json'
+
+#require_relative 'backblaze/storage'
+
+module Fog
+  module Backblaze
+    extend Fog::Provider
+    service(:storage, "Storage")
+
+  end
+end

data/lib/fog/backblaze/json_response.rb

@@ -0,0 +1,25 @@
+module Fog
+  module Backblaze
+    module JSONReponse
+
+      attr_writer :json
+
+      def raw_body
+        @body
+      end
+
+      def json
+        @json ||= JSON.parse(raw_body)
+      end
+
+      def assign_json_body!
+        self.body = json
+      end
+
+      def josn_response?
+        headers['Content-Type'].start_with?("application/json")
+      end
+
+    end
+  end
+end

data/lib/fog/backblaze/models/directories.rb

@@ -0,0 +1,15 @@
+class Fog::Storage::Backblaze::Directories < Fog::Collection
+  model Fog::Storage::Backblaze::Directory
+
+  def all
+    data = service.list_buckets
+    load(data)
+  end
+
+  def get(name)
+    list_response = service.list_buckets
+    bucket = list_response.json['buckets'].detect {|bucket| bucket['bucketName'] == name }
+    return new(bucket) if bucket
+  end
+
+end

data/lib/fog/backblaze/models/directory.rb

@@ -0,0 +1,37 @@
+class Fog::Storage::Backblaze::Directory < Fog::Model
+  identity :key, aliases: %w(bucketName)
+
+  attribute :bucket_id, aliases: 'bucketId'
+  attribute :bucket_info, aliases: 'bucketInfo'
+  attribute :bucket_type, aliases: 'bucketType'
+
+  def destroy
+    requires :key
+    service.delete_bucket(key)
+    true
+  #rescue Fog::Errors::NotFound
+  #  false
+  end
+
+  def save
+    requires :key
+    options = {}
+
+    response = service.put_bucket(key, options)
+
+    attributes[:bucket_id] = response.json['bucketId']
+    attributes[:bucket_type] = response.json['bucketType']
+    attributes[:bucket_info] = response.json['bucketInfo']
+
+    true
+  end
+
+  def files
+    @files ||= Fog::Storage::Backblaze::Files.new(directory: self, service: service)
+  end
+
+  def public?
+    attributes[:bucket_type] == "allPublic"
+  end
+
+end

data/lib/fog/backblaze/models/file.rb

@@ -0,0 +1,52 @@
+class Fog::Storage::Backblaze::File < Fog::Model
+
+  identity :file_name, aliases: %w{fileName key}
+
+  attribute :content_length, aliases: 'contentLength'
+  attribute :content_type, aliases: 'contentType'
+  attribute :file_id, aliases: 'fileId'
+  attribute :file_info, aliases: 'fileInfo'
+  attribute :upload_timestamp, aliases: 'uploadTimestamp'
+
+  attr_accessor :directory
+
+  # TODO: read content from cloud on demand
+  def body
+    attributes[:body] #||= file_id && (file = collection.get(identity)) ? file.body : ""
+  end
+
+  def body=(new_body)
+    attributes[:body] = new_body
+  end
+
+  alias_method :key, :file_name
+  alias_method :key=, :file_name=
+
+  def save(options = {})
+    requires :body, :directory, :key
+
+    options[:content_type] = content_type if content_type
+
+    data = service.put_object(directory.key, key, body, options)
+
+    merge_attributes(data.headers.reject { |key, _value| %w(contentLength contentType).include?(key) })
+
+    self.content_length = Fog::Storage.get_body_size(body)
+    self.content_type ||= Fog::Storage.get_content_type(body)
+
+    true
+  end
+
+  def public_url
+    requires :directory, :key
+
+    service.get_object_url(directory.key, key)
+  end
+
+  # TODO
+  #def url(expires)
+  #  requires :key
+  #  collection.get_https_url(key, expires)
+  #end
+
+end

data/lib/fog/backblaze/models/files.rb

@@ -0,0 +1,65 @@
+class Fog::Storage::Backblaze::Files < Fog::Collection
+  model Fog::Storage::Backblaze::File
+
+  attribute :directory
+  #attribute :common_prefixes, :aliases => "CommonPrefixes"
+  #attribute :delimiter,       :aliases => "Delimiter"
+  #attribute :page_token,      :aliases => %w(pageToken page_token)
+  #attribute :max_results,     :aliases => ["MaxKeys", "max-keys"]
+  #attribute :prefix,          :aliases => "Prefix"
+
+  model Fog::Storage::Backblaze::File
+
+  def all(options = {})
+    requires :directory
+
+    body = service.list_objects(directory.key, options).body
+    load(body["files"] || [])
+  end
+
+  def get(file_name)
+    requires :directory
+
+    file_response = service.get_object(directory.key, file_name)
+    file_data = _headers_to_attrs(file_response)
+
+    new(file_data)
+  end
+
+  def new(attributes = {})
+    requires :directory
+    super({directory: directory}.merge!(attributes))
+  end
+
+  # TODO: download url for private buckets
+  def get_https_url(file_name, expires, options)
+    requires :directory
+    service.get_object_https_url(directory.key, file_name, expires, options)
+  end
+
+  # TODO: download url for private buckets
+  def head_url(file_name, expires, options = {})
+    requires :directory
+    service.get_object_https_url(directory.key, key, expires, options)
+  end
+
+  def head(file_name, options = {})
+    requires :directory
+    data = service.head_object(directory.key, file_name, options)
+    file_data = _headers_to_attrs(file_response)
+    new(file_data)
+  rescue Excon::Errors::NotFound
+    nil
+  end
+
+  def _headers_to_attrs(file_response)
+    {
+      fileName:        file_response.headers['x-bz-file-name'],
+      fileId:          file_response.headers['x-bz-file-id'],
+      uploadTimestamp: file_response.headers['X-Bz-Upload-Timestamp'],
+      contentType:     file_response.headers['Content-Type'],
+      contentLength:   file_response.headers['Content-Length']
+    }
+  end
+
+end

data/lib/fog/backblaze/storage.rb

@@ -0,0 +1,426 @@
+require 'json'
+require 'digest'
+require 'cgi'
+
+class Fog::Storage::Backblaze < Fog::Service
+  requires :b2_account_id, :b2_account_token
+  recognizes :b2_bucket_name, :b2_bucket_id, :token_cache, :logger
+
+  model_path 'fog/backblaze/models'
+  model       :directory
+  collection  :directories
+  model       :file
+  collection  :files
+
+  class Mock
+    #include Integrity
+
+    def self.data
+      @data ||= Hash.new do |hash, key|
+        hash[key] = {}
+      end
+    end
+
+    def self.reset
+      @data = nil
+    end
+
+    def initialize(options={})
+      @b2_account_id = options[:b2_account_id]
+      @b2_account_token = options[:b2_account_token]
+      @path = '/v1/AUTH_1234'
+      @containers = {}
+    end
+
+    def data
+      self.class.data[@softlayer_username]
+    end
+
+    def reset_data
+      self.class.data.delete(@softlayer_username)
+    end
+
+    def change_account(account)
+      @original_path ||= @path
+      version_string = @original_path.split('/')[1]
+      @path = "/#{version_string}/#{account}"
+    end
+
+    def reset_account_name
+      @path = @original_path
+    end
+
+  end
+
+  class Real
+    attr_reader :token_cache, :options
+
+    def initialize(options = {})
+      @options = options
+      @logger = @options[:logger] || begin
+        require 'logger'
+        Logger.new("/dev/null")
+      end
+
+      @token_cache = if options[:token_cache].nil? || options[:token_cache] == :memory
+        Fog::Backblaze::TokenCache.new
+      elsif options[:token_cache] === false
+        Fog::Backblaze::TokenCache::NullTokenCache.new
+      elsif token_cache.is_a?(Fog::Backblaze::TokenCache)
+        token_cache
+      else
+        Fog::Backblaze::TokenCache::FileTokenCache.new(options[:token_cache])
+      end
+    end
+
+    def logger
+      @logger
+    end
+
+    ## Buckets
+
+    def put_bucket(key, extra_options = {})
+      options = {
+        accountId: @options[:b2_account_id],
+        bucketType: extra_options.delete(:public) ? 'allPublic' : 'allPrivate',
+        bucketName: key,
+      }.merge(extra_options)
+
+      response = b2_command(:b2_create_bucket, body: options)
+
+      if response.status >= 400
+        raise Fog::Errors::Error, "Failed put_bucket, status = #{response.status} #{response.body}"
+      end
+
+      if cached = @token_cache.buckets
+        @token_cache.buckets = cached.merge(key => response.json)
+      else
+        @token_cache.buckets = {key => response.json}
+      end
+
+      response
+    end
+
+    def list_buckets
+      response = b2_command(:b2_list_buckets, body: {accountId: @options[:b2_account_id]})
+
+      response
+    end
+
+    def get_bucket(bucket_name)
+      reponse = list_buckets
+      bucket = reponse.json['buckets'].detect do |bucket|
+        bucket['bucketName'] == bucket_name
+      end
+
+      unless bucket
+        raise Fog::Errors::NotFound, "No bucket with name: #{bucket_name}, " +
+          "found: #{reponse.json['buckets'].map {|b| b['bucketName']}.join(", ")}"
+      end
+
+      reponse.body = bucket
+      reponse.json = bucket
+      return reponse
+    end
+
+    def delete_bucket(bucket_name, options = {})
+      bucket_id = _get_bucket_id(bucket_name)
+
+      unless bucket_id
+        raise Fog::Errors::NotFound, "Can not bucket #{bucket_name}"
+      end
+
+      response = b2_command(:b2_delete_bucket,
+        body: {
+          bucketId: bucket_id,
+          accountId: @options[:b2_account_id]
+        }
+      )
+
+      if !options[:is_retrying]
+        if response.status == 400 && response.json['message'] =~ /Bucket .+ does not exist/
+          logger.info("Try drop cache and try again")
+          @token_cache.buckets = nil
+          return delete_bucket(bucket_name, is_retrying: true)
+        end
+      end
+
+      if response.status >= 400
+        raise Fog::Errors::Error, "Failed delete_bucket, status = #{response.status} #{response.body}"
+      end
+
+      if cached = @token_cache.buckets
+        cached.delete(bucket_name)
+        @token_cache.buckets = cached
+      end
+
+      response
+    end
+
+    ## Objects
+
+    def list_objects(bucket_name, options = {})
+      bucket_id = _get_bucket_id(bucket_name)
+
+      unless bucket_id
+        raise Fog::Errors::NotFound, "Can not bucket #{bucket_name}"
+      end
+
+      b2_command(:b2_list_file_names, body: {
+        bucketId: bucket_id,
+        maxFileCount: 10_000
+      }.merge(options))
+    end
+
+    def head_object(bucket_name, file_path)
+      file_url = get_object_url(bucket_name, file_path)
+
+      result = b2_command(nil,
+        method: :head,
+        url: file_url
+      )
+
+      if result.status == 404
+        raise Fog::Errors::NotFound, "Can not find #{file_path.inspect} in bucket #{bucket_name}"
+      end
+
+      if result.status >= 400
+        raise Fog::Errors::NotFound, "Backblaze respond with status = #{result.status} - #{result.reason_phrase}"
+      end
+
+      result
+    end
+
+    # TODO: handle options
+    def put_object(bucket_name, file_path, content, options = {})
+      upload_url = @token_cache.fetch("upload_url/#{bucket_name}") do
+        bucket_id = _get_bucket_id(bucket_name)
+        unless bucket_id
+          raise Fog::Errors::NotFound, "Can not find bucket #{bucket_name.inspect}"
+        end
+        result = b2_command(:b2_get_upload_url, body: {bucketId: _get_bucket_id(bucket_name)})
+        result.json
+      end
+
+      extra_headers = {}
+      if options[:content_type]
+        extra_headers['Content-Type'] = options[:content_type]
+      end
+      if options[:last_modified]
+        extra_headers['X-Bz-Info-src_last_modified_millis'] = options[:last_modified]
+      end
+      if options[:content_disposition]
+        extra_headers['X-Bz-Info-b2-content-disposition'] = options[:content_disposition]
+      end
+
+      response = b2_command(nil,
+        url: upload_url['uploadUrl'],
+        body: content,
+        headers: {
+          'Authorization': upload_url['authorizationToken'],
+          'Content-Type': 'b2/x-auto',
+          'X-Bz-File-Name': "#{_esc_file(file_path)}",
+          'X-Bz-Content-Sha1': Digest::SHA1.hexdigest(content)
+        }.merge(extra_headers)
+      )
+
+      if response.json['fileId'] == nil
+        raise Fog::Errors::Error, "Failed put_object, status = #{response.status} #{response.body}"
+      end
+
+      response
+    end
+
+    def get_object_url(bucket_name, file_path)
+      "#{auth_response['downloadUrl']}/file/#{CGI.escape(bucket_name)}/#{_esc_file(file_path)}"
+    end
+
+    alias_method :get_object_https_url, :get_object_url
+
+    def get_public_object_url(bucket_name, file_path, options = {})
+      bucket_id = _get_bucket_id(bucket_name)
+
+      unless bucket_id
+        raise Fog::Errors::NotFound, "Can not bucket #{bucket_name}"
+      end
+
+      result = b2_command(:b2_get_download_authorization, body: {
+        bucketId: bucket_id,
+        fileNamePrefix: file_path,
+        validDurationInSeconds: 604800
+      }.merge(options))
+
+      if result.status == 404
+        raise Fog::Errors::NotFound, "Can not find #{file_path.inspect} in bucket #{bucket_name}"
+      end
+
+      if result.status >= 400
+        raise Fog::Errors::NotFound, "Backblaze respond with status = #{result.status} - #{result.reason_phrase}"
+      end
+
+      "#{get_object_url(bucket_name, file_path)}?Authorization=#{result.json['authorizationToken']}"
+    end
+
+    def get_object(bucket_name, file_name)
+      file_url = get_object_url(bucket_name, file_name)
+
+      result = b2_command(nil,
+        method: :get,
+        url: file_url
+      )
+
+      if result.status == 404
+        raise Fog::Errors::NotFound, "Can not find #{file_name.inspect} in bucket #{bucket_name}"
+      end
+
+      return result
+    end
+
+    def delete_object(bucket_name, file_name)
+      version_ids = _get_object_version_ids(bucket_name, file_name)
+
+      if version_ids.size == 0
+        raise Fog::Errors::NotFound, "Can not find #{file_name} in in bucket #{bucket_name}"
+      end
+
+      logger.info("Deleting #{version_ids.size} versions of #{file_name}")
+
+      last_response = nil
+      version_ids.each do |version_id|
+        last_response = b2_command(:b2_delete_file_version, body: {
+          fileName: file_name,
+          fileId: version_id
+        })
+      end
+
+      last_response
+    end
+
+    def _get_object_version_ids(bucket_name, file_name)
+      response = b2_command(:b2_list_file_versions,
+        body: {
+          startFileName: file_name,
+          prefix: file_name,
+          bucketId: _get_bucket_id(bucket_name),
+          maxFileCount: 1000
+        }
+      )
+
+      if response.json['files']
+        version_ids = []
+        response.json['files'].map do |file_version|
+          version_ids << file_version['fileId'] if file_version['fileName'] == file_name
+        end
+        version_ids
+      else
+        []
+      end
+    end
+
+    def _get_bucket_id(bucket_name)
+      if @options[:b2_bucket_name] == bucket_name && @options[:b2_bucket_id]
+        return @options[:b2_bucket_id]
+      else
+        cached = @token_cache && @token_cache.buckets
+
+        if cached && cached[bucket_name]
+          return cached[bucket_name]['bucketId']
+        else
+          fetched = _cached_buchets_hash(force_fetch: !!cached)
+          return fetched[bucket_name] && fetched[bucket_name]['bucketId']
+        end
+      end
+    end
+
+    def _cached_buchets_hash(force_fetch: false)
+
+      if !force_fetch && cached = @token_cache.buckets
+        cached
+      end
+
+      buckets_hash = {}
+      list_buckets.json['buckets'].each do |bucket|
+        buckets_hash[bucket['bucketName']] = bucket
+      end
+
+      @token_cache.buckets = buckets_hash
+
+      buckets_hash
+    end
+
+    def auth_response
+      #return @auth_response.json if @auth_response
+
+      if cached = @token_cache.auth_response
+        logger.info("get token from cache")
+        return cached
+      end
+
+      @auth_response = json_req(:get, "https://api.backblazeb2.com/b2api/v1/b2_authorize_account",
+        headers: {
+          "Authorization" => "Basic " + Base64.strict_encode64("#{@options[:b2_account_id]}:#{@options[:b2_account_token]}")
+        },
+        persistent: false
+      )
+
+      if @auth_response.status >= 400
+        raise Fog::Errors::Error, "Authentication error: #{@auth_response.json['message']} (status = #{@auth_response.status})\n#{@auth_response.body}"
+      end
+
+      @token_cache.auth_response = @auth_response.json
+
+      @auth_response.json
+    end
+
+    def b2_command(command, options = {})
+      auth_response = self.auth_response
+      options[:headers] ||= {}
+      options[:headers]['Authorization'] ||= auth_response['authorizationToken']
+
+      if options[:body] && !options[:body].is_a?(String)
+        options[:body] = JSON.generate(options[:body])
+      end
+
+      request_url = options.delete(:url) || "#{auth_response['apiUrl']}/b2api/v1/#{command}"
+
+      #pp [:b2_command, request_url, options]
+
+      json_req(options.delete(:method) || :post, request_url, options)
+    end
+
+    def json_req(method, url, options = {})
+      start_time = Time.now.to_f
+      logger.info("Req #{method.to_s.upcase} #{url}")
+      logger.debug(options.to_s)
+
+      if !options.has_key?(:persistent) || options[:persistent] == true
+        @connections ||= {}
+        full_path = [URI.parse(url).request_uri, URI.parse(url).fragment].compact.join("#")
+        host_url = url.sub(full_path, "")
+        connection = @connections[host_url] ||= Excon.new(host_url, persistent: true)
+        http_response = connection.send(method, options.merge(path: full_path, idempotent: true))
+      else
+        http_response = Excon.send(method, url, options)
+      end
+
+      http_response.extend(Fog::Backblaze::JSONReponse)
+      http_response.assign_json_body! if http_response.josn_response?
+
+      http_response
+    ensure
+      status = http_response && http_response.status
+      logger.info("Done #{method.to_s.upcase} #{url} = #{status} (#{(Time.now.to_f - start_time).round(3)} sec)")
+      logger.debug(http_response.headers) if http_response
+      logger.debug(http_response.body) if http_response
+    end
+
+    def reset_token_cache
+      @token_cache.reset
+    end
+
+    def _esc_file(file_name)
+      CGI.escape(file_name).gsub('%2F', '/')
+    end
+  end
+end
+

data/lib/fog/backblaze/token_cache.rb

@@ -0,0 +1,134 @@
+# Each request must have authentication header, recieved from b2_authorize_account
+# Authentication token is active for 24 hours, same as uploadUrl. To make things faster we keep it in cache
+#
+# Avaliable cache storages: file, memory, null
+#
+# To create own cache storage (stored in redis for example)
+#
+#   class RedisTokenCache < Fog::Backblaze::TokenCache
+#     def initialize(redis_url)
+#       @redis = Redis.new(redis_url)
+#       super()
+#     end
+#     def load_data
+#       raw_data = @redis.get("b2_token_cache")
+#       raw_data ? JSON.parse(raw_data) : {}
+#     end
+#     def save_data
+#       @redis.set("b2_token_cache", JSON.pretty_generate(@data))
+#     end
+#   end
+#   
+#   Fog::Storage.new(provider: 'backblaze', ..., token_cache: RedisTokenCache.new)
+#
+
+module Fog
+  module Backblaze
+  end
+end
+
+class Fog::Backblaze::TokenCache
+
+  def initialize
+    @data = load_data || {}
+  end
+
+  def load_data
+  end
+
+  def save_data
+  end
+
+  TTLS = {
+    auth_response: 3600 * 24,
+    buckets: 3600 * 24,
+    upload_url: 3600 * 24
+  }
+
+  def fetch(field)
+    if result = access_part(field)
+      result
+    else
+      result = yield
+      write_part(field, result, TTLS[field])
+      save_data
+      result
+    end
+  end
+
+  def auth_response
+    access_part(:auth_response)
+  end
+
+  def auth_response=(value)
+    write_part(:auth_response, value, 3600 * 24)
+    save_data
+  end
+
+  def buckets
+    access_part(:buckets)
+  end
+
+  def buckets=(value)
+    write_part(:buckets, value, 3600 * 24)
+    save_data
+  end
+
+  def upload_url
+    
+  end
+
+  def access_part(name)
+    name = name.to_s
+    if @data[name] && ::DateTime.parse(@data[name]['expires_at']).to_time > ::Time.now
+      @data[name]['value']
+    end
+  end
+
+  def write_part(name, value, ttl = 3600)
+    ttl = 3600 if ttl.nil?
+    name = name.to_s
+    if value.nil?
+      @data.delete(name)
+    else
+      @data[name] = {
+        'value' => value,
+        'expires_at' => ::Time.at(::Time.now + ttl - 1).to_s
+      }
+    end
+  end
+
+  def reset
+    @data = {}
+    save_data
+  end
+
+  # stored in local file
+  class FileTokenCache < Fog::Backblaze::TokenCache
+
+    def initialize(file)
+      @file = file
+      super()
+    end
+
+    def load_data
+      if File.exist?(@file)
+        JSON.parse(File.open(@file, 'rb', &:read))
+      else
+        {}
+      end
+    end
+
+    def save_data
+      File.open(@file, 'wb') do |f|
+        f.write(JSON.pretty_generate(@data) + "\n")
+      end
+    end
+  end
+
+  # black hole, always clean cache
+  class NullTokenCache < Fog::Backblaze::TokenCache
+    def write_part(name, value, ttl = 3600)
+    end
+  end
+end

data/lib/fog/backblaze/version.rb

@@ -0,0 +1,5 @@
+module Fog
+  module Backblaze
+    VERSION = "0.1.1"
+  end
+end

metadata

@@ -0,0 +1,84 @@
+--- !ruby/object:Gem::Specification
+name: fog-backblaze
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Pavel Evstigneev
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-03-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fog-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.40'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.40'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3'
+description: Blackblade B2 stogate client for 'fog' gem, can be used for working with
+  files and buckets. E.g. carrierwave uploads
+email:
+- pavel.evst@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- examples/example.rb
+- fog-backblaze.gemspec
+- lib/fog/backblaze.rb
+- lib/fog/backblaze/json_response.rb
+- lib/fog/backblaze/models/directories.rb
+- lib/fog/backblaze/models/directory.rb
+- lib/fog/backblaze/models/file.rb
+- lib/fog/backblaze/models/files.rb
+- lib/fog/backblaze/storage.rb
+- lib/fog/backblaze/token_cache.rb
+- lib/fog/backblaze/version.rb
+homepage: https://github.com/fog/fog-backblaze
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.8
+signing_key: 
+specification_version: 4
+summary: Module for the 'fog' gem to support Blackblade B2 stogate.
+test_files: []