fog-backblaze 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e72a3f8088a51fed32e1b1efc3d18772ff03348b
-  data.tar.gz: 635bf11d1a64069912521d3c179fb5e46e67e3ec
+  metadata.gz: c206d62c779b21fd9771105907c940dec19a1ea0
+  data.tar.gz: 35c054e6d149c3b09ab1f3521643bc255187f5c5
 SHA512:
-  metadata.gz: d3af95365d4d87cd941c016e18ebf326c9bcdbebb3d7440b29e1f8bc0557cc416c1fd9d973bc5e9cdf3a0091123bdff9b8c933d7c41e53ec25b682a421585e3c
-  data.tar.gz: 86a1546b2f45cf95bb0f79a93c97b3eaa3201b3d3f9655bd264cf13cc539d0c9e882d892392ab6d8f45e7cde3041bcaec7aa2bfe34d142ab7877e0a99b3bed6b
+  metadata.gz: a1858a028b04d845479b515cf3a9b30cf44ca3cde4265095fb40b64e4154593260615bbb6685c20b4fdc0bdcd4096188639c4a858e2791d0cded17453df78886
+  data.tar.gz: 9655f45406f1bd4ff75c35155f46d294bb6ed26b7e6b7f10d7291218107bad1f59f2cfd3f1866bfe9b60535c7ee6edc68697719115060e64fa05e76369574360

data/.gitignore

@@ -1,8 +1,23 @@
-/.bundle/
-/.yardoc
-/_yardoc/
-/coverage/
-/doc/
-/pkg/
-/spec/reports/
-/tmp/
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+gemfiles/*.lock

data/.travis.yml

@@ -2,4 +2,5 @@ sudo: false
 language: ruby
 rvm:
   - 2.4.0
+  - 2.5.0
 before_install: gem install bundler -v 1.15.1

data/Rakefile

@@ -4,7 +4,7 @@ require "rake/testtask"
 Rake::TestTask.new(:test) do |t|
   t.libs << "test"
   t.libs << "lib"
-  t.test_files = FileList["test/**/*_test.rb"]
+  t.test_files = FileList["test/**/*_spec.rb"]
 end
 
 task :default => :test

data/examples/example.rb

@@ -23,8 +23,7 @@ connection = Fog::Storage.new(
   #b2_bucket_name: ENV['B2_BUCKET'],
   #b2_bucket_id: '111222333444',
 
-  logger: ENV['DEBUG'] ? Logger.new(STDOUT) : nil,
-  token_cache: :memory
+  logger: ENV['DEBUG'] ? Logger.new(STDOUT) : nil
 )
 
 connection.delete_bucket("fog-smoke-test") rescue nil

data/lib/fog/backblaze/json_response.rb

@@ -1,6 +1,6 @@
 module Fog
   module Backblaze
-    module JSONReponse
+    module JSONResponse
 
       attr_writer :json
 
@@ -9,7 +9,7 @@ module Fog
       end
 
       def json
-        @json ||= JSON.parse(raw_body)
+        @json ||= ::JSON.parse(raw_body)
       end
 
       def assign_json_body!

data/lib/fog/backblaze/token_cache.rb

@@ -12,7 +12,7 @@
 #     end
 #     def load_data
 #       raw_data = @redis.get("b2_token_cache")
-#       raw_data ? JSON.parse(raw_data) : {}
+#       raw_data ? ::JSON.parse(raw_data) : {}
 #     end
 #     def save_data
 #       @redis.set("b2_token_cache", JSON.pretty_generate(@data))
@@ -22,11 +22,6 @@
 #   Fog::Storage.new(provider: 'backblaze', ..., token_cache: RedisTokenCache.new)
 #
 
-module Fog
-  module Backblaze
-  end
-end
-
 class Fog::Backblaze::TokenCache
 
   def initialize
@@ -113,7 +108,7 @@ class Fog::Backblaze::TokenCache
 
     def load_data
       if File.exist?(@file)
-        JSON.parse(File.open(@file, 'rb', &:read))
+        ::JSON.parse(File.open(@file, 'rb', &:read))
       else
         {}
       end

data/lib/fog/backblaze/version.rb

@@ -1,5 +1,5 @@
 module Fog
   module Backblaze
-    VERSION = "0.1.1"
+    VERSION = "0.1.2"
   end
 end

data/lib/fog/backblaze.rb

@@ -1,16 +1,16 @@
-require_relative 'backblaze/version'
-require_relative 'backblaze/token_cache'
-require_relative 'backblaze/json_response'
-
 require 'fog/core'
 require 'json'
 
-#require_relative 'backblaze/storage'
-
 module Fog
   module Backblaze
+    autoload :JSONResponse, File.expand_path("../backblaze/json_response", __FILE__)
+    autoload :TokenCache, File.expand_path("../backblaze/token_cache", __FILE__)
+
     extend Fog::Provider
     service(:storage, "Storage")
+  end
 
+  module Storage
+    autoload :Backblaze, File.expand_path("../storage/backblaze", __FILE__)
   end
 end

data/lib/fog/storage/backblaze/mock.rb

@@ -0,0 +1,36 @@
+class Fog::Storage::Backblaze::Mock
+  def self.data
+    @data ||= Hash.new do |hash, key|
+      hash[key] = {}
+    end
+  end
+
+  def self.reset
+    @data = nil
+  end
+
+  def initialize(options={})
+    @b2_account_id = options[:b2_account_id]
+    @b2_account_token = options[:b2_account_token]
+    @path = '/v1/AUTH_1234'
+    @containers = {}
+  end
+
+  def data
+    self.class.data[@softlayer_username]
+  end
+
+  def reset_data
+    self.class.data.delete(@softlayer_username)
+  end
+
+  def change_account(account)
+    @original_path ||= @path
+    version_string = @original_path.split('/')[1]
+    @path = "/#{version_string}/#{account}"
+  end
+
+  def reset_account_name
+    @path = @original_path
+  end
+end

data/lib/fog/{backblaze → storage/backblaze}/models/directories.rb

@@ -3,7 +3,7 @@ class Fog::Storage::Backblaze::Directories < Fog::Collection
 
   def all
     data = service.list_buckets
-    load(data)
+    load(data.body['buckets'])
   end
 
   def get(name)

data/lib/fog/storage/backblaze/models/directory.rb

@@ -0,0 +1,61 @@
+class Fog::Storage::Backblaze::Directory < Fog::Model
+  identity :key, aliases: %w(bucketName)
+
+  attribute :bucket_id#,       aliases: 'bucketId'
+  attribute :bucket_info#,     aliases: 'bucketInfo'
+  attribute :bucket_type#,     aliases: 'bucketType'
+  attribute :cors_rules#,      aliases: 'corsRules'
+  attribute :lifecycle_rules#, aliases: 'lifecycleRules'
+  attribute :revision
+
+  alias_method :name, :key
+
+  def destroy
+    requires :key
+    response = service.delete_bucket(key)
+    return response.status < 400
+  #rescue Fog::Errors::NotFound
+  #  false
+  end
+
+  def save
+    requires :key
+    options = {}
+
+    options[:bucketInfo]     = bucket_info     if bucket_info
+    options[:bucketType]     = bucket_type     if bucket_type
+    options[:lifecycleRules] = lifecycle_rules if lifecycle_rules
+    options[:corsRules]      = cors_rules      if cors_rules
+
+    if attributes[:bucket_id]
+      options[:bucketId] = attributes[:bucket_id]
+      response = service.update_bucket(key, options)
+    else
+      response = service.put_bucket(key, options)
+    end
+
+    attributes[:bucket_id] = response.json['bucketId']
+    attributes[:bucket_type] = response.json['bucketType']
+    attributes[:bucket_info] = response.json['bucketInfo']
+    attributes[:revision] = response.json['revision']
+    attributes[:lifecycle_rules] = response.json['lifecycleRules']
+    attributes[:cors_rules] = response.json['corsRules']
+
+    true
+  end
+
+  def files
+    @files ||= Fog::Storage::Backblaze::Files.new(directory: self, service: service)
+  end
+
+  def public?
+    attributes[:bucket_type] == "allPublic"
+  end
+
+  alias_method :public, :public?
+
+  def public=(value)
+    self.bucket_type = value ? 'allPublic' : 'allPrivate'
+  end
+
+end

data/lib/fog/{backblaze → storage/backblaze}/models/file.rb

@@ -1,6 +1,6 @@
 class Fog::Storage::Backblaze::File < Fog::Model
 
-  identity :file_name, aliases: %w{fileName key}
+  identity :file_name, aliases: %w{fileName key name}
 
   attribute :content_length, aliases: 'contentLength'
   attribute :content_type, aliases: 'contentType'
@@ -10,6 +10,8 @@ class Fog::Storage::Backblaze::File < Fog::Model
 
   attr_accessor :directory
 
+  alias_method :name, :file_name
+
   # TODO: read content from cloud on demand
   def body
     attributes[:body] #||= file_id && (file = collection.get(identity)) ? file.body : ""
@@ -37,6 +39,14 @@ class Fog::Storage::Backblaze::File < Fog::Model
     true
   end
 
+  def destroy
+    requires :key
+    response = service.delete_object(directory.key, key)
+    return response.status < 400
+  #rescue Fog::Errors::NotFound
+  #  false
+  end
+
   def public_url
     requires :directory, :key
 

data/lib/fog/storage/backblaze/real.rb

@@ -0,0 +1,433 @@
+class Fog::Storage::Backblaze::Real
+  attr_reader :token_cache, :options
+
+  def initialize(options = {})
+    @options = options
+    @logger = @options[:logger] || begin
+      require 'logger'
+      Logger.new("/dev/null")
+    end
+
+    @token_cache = if options[:token_cache].nil? || options[:token_cache] == :memory
+      Fog::Backblaze::TokenCache.new
+    elsif options[:token_cache] === false
+      Fog::Backblaze::TokenCache::NullTokenCache.new
+    elsif token_cache.is_a?(Fog::Backblaze::TokenCache)
+      token_cache
+    else
+      Fog::Backblaze::TokenCache::FileTokenCache.new(options[:token_cache])
+    end
+  end
+
+  def logger
+    @logger
+  end
+
+  ## Buckets
+
+  # call b2_create_bucket
+  def put_bucket(bucket_name, extra_options = {})
+    options = {
+      accountId: @options[:b2_account_id],
+      bucketType: extra_options.delete(:public) ? 'allPublic' : 'allPrivate',
+      bucketName: bucket_name,
+    }.merge(extra_options)
+
+    response = b2_command(:b2_create_bucket, body: options)
+
+    if response.status >= 400
+      raise Fog::Errors::Error, "Failed put_bucket, status = #{response.status} #{response.body}"
+    end
+
+    if cached = @token_cache.buckets
+      @token_cache.buckets = cached.merge(bucket_name => response.json)
+    else
+      @token_cache.buckets = {bucket_name => response.json}
+    end
+
+    response
+  end
+
+  # call b2_update_bucket
+  # if options[:bucketId] presents, then bucket_name is option
+  def update_bucket(bucket_name, extra_options)
+    options = {
+      accountId: @options[:b2_account_id],
+      bucketId: extra_options[:bucketId] || _get_bucket_id!(bucket_name),
+    }
+    if extra_options.has_key?(:public)
+      options[:bucketType] = extra_options.delete(:public) ? 'allPublic' : 'allPrivate'
+    end
+    options.merge!(extra_options)
+
+    response = b2_command(:b2_update_bucket, body: options)
+
+    if response.status >= 400
+      raise Fog::Errors::Error, "Failed update_bucket, status = #{response.status} #{response.body}"
+    end
+
+    if cached = @token_cache.buckets
+      @token_cache.buckets = cached.merge(bucket_name => response.json)
+    else
+      @token_cache.buckets = {bucket_name => response.json}
+    end
+
+    response
+  end
+
+  # call b2_list_buckets
+  def list_buckets
+    response = b2_command(:b2_list_buckets, body: {accountId: @options[:b2_account_id]})
+
+    response
+  end
+
+  # call b2_list_buckets
+  def get_bucket(bucket_name)
+    response = list_buckets
+    bucket = response.json['buckets'].detect do |bucket|
+      bucket['bucketName'] == bucket_name
+    end
+
+    unless bucket
+      raise Fog::Errors::NotFound, "No bucket with name: #{bucket_name}, " +
+        "found buckets: #{response.json['buckets'].map {|b| b['bucketName']}.join(", ")}"
+    end
+
+    response.body = bucket
+    response.json = bucket
+    return response
+  end
+
+  # call b2_delete_bucket
+  def delete_bucket(bucket_name, options = {})
+    bucket_id = _get_bucket_id!(bucket_name)
+
+    response = b2_command(:b2_delete_bucket,
+      body: {
+        bucketId: bucket_id,
+        accountId: @options[:b2_account_id]
+      }
+    )
+
+    if !options[:is_retrying]
+      if response.status == 400 && response.json['message'] =~ /Bucket .+ does not exist/
+        logger.info("Try drop cache and try again")
+        @token_cache.buckets = nil
+        return delete_bucket(bucket_name, is_retrying: true)
+      end
+    end
+
+    if response.status >= 400
+      raise Fog::Errors::Error, "Failed delete_bucket, status = #{response.status} #{response.body}"
+    end
+
+    if cached = @token_cache.buckets
+      #cached.delete(bucket_name)
+      #@token_cache.buckets = cached
+      @token_cache.buckets = nil
+    end
+
+    response
+  end
+
+  ## Objects
+
+  # call b2_list_file_names
+  def list_objects(bucket_name, options = {})
+    bucket_id = _get_bucket_id!(bucket_name)
+
+    b2_command(:b2_list_file_names, body: {
+      bucketId: bucket_id,
+      maxFileCount: 10_000
+    }.merge(options))
+  end
+
+  def head_object(bucket_name, file_path)
+    file_url = get_object_url(bucket_name, file_path)
+
+    result = b2_command(nil,
+      method: :head,
+      url: file_url
+    )
+
+    if result.status == 404
+      raise Fog::Errors::NotFound, "Can not find #{file_path.inspect} in bucket #{bucket_name}"
+    end
+
+    if result.status >= 400
+      raise Fog::Errors::NotFound, "Backblaze respond with status = #{result.status} - #{result.reason_phrase}"
+    end
+
+    result
+  end
+
+  # call b2_get_upload_url
+  #
+  #   connection.put_object("a-bucket", "/some_file.txt", string_or_io, options)
+  #
+  # Possible options:
+  # * content_type
+  # * last_modified - time object or number of miliseconds
+  # * content_disposition
+  # * extra_headers - hash, list of custom headers
+  def put_object(bucket_name, file_path, content, options = {})
+    upload_url = @token_cache.fetch("upload_url/#{bucket_name}") do
+      bucket_id = _get_bucket_id!(bucket_name)
+      result = b2_command(:b2_get_upload_url, body: {bucketId: bucket_id})
+      result.json
+    end
+
+    if content.is_a?(IO)
+      content = content.read
+    end
+
+    extra_headers = {}
+    if options[:content_type]
+      extra_headers['Content-Type'] = options[:content_type]
+    end
+
+    if options[:last_modified]
+      value = if options[:last_modified].is_a?(::Time)
+        (options[:last_modified].to_f * 1000).round
+      else
+        value
+      end
+      extra_headers['X-Bz-Info-src_last_modified_millis'] = value
+    end
+
+    if options[:content_disposition]
+      extra_headers['X-Bz-Info-b2-content-disposition'] = options[:content_disposition]
+    end
+
+    if options[:extra_headers]
+      options[:extra_headers].each do |key, value|
+        extra_headers["X-Bz-Info-#{key}"] = value
+      end
+    end
+
+    response = b2_command(nil,
+      url: upload_url['uploadUrl'],
+      body: content,
+      headers: {
+        'Authorization': upload_url['authorizationToken'],
+        'Content-Type': 'b2/x-auto',
+        'X-Bz-File-Name': "#{_esc_file(file_path)}",
+        'X-Bz-Content-Sha1': Digest::SHA1.hexdigest(content)
+      }.merge(extra_headers)
+    )
+
+    if response.json['fileId'] == nil
+      raise Fog::Errors::Error, "Failed put_object, status = #{response.status} #{response.body}"
+    end
+
+    response
+  end
+
+  # generates url regardless if bucket is private or not
+  def get_object_url(bucket_name, file_path)
+    "#{auth_response['downloadUrl']}/file/#{CGI.escape(bucket_name)}/#{_esc_file(file_path)}"
+  end
+
+  alias_method :get_object_https_url, :get_object_url
+
+  # call b2_get_download_authorization
+  def get_public_object_url(bucket_name, file_path, options = {})
+    bucket_id = _get_bucket_id!(bucket_name)
+
+    result = b2_command(:b2_get_download_authorization, body: {
+      bucketId: bucket_id,
+      fileNamePrefix: file_path,
+      validDurationInSeconds: 604800
+    }.merge(options))
+
+    if result.status == 404
+      raise Fog::Errors::NotFound, "Can not find #{file_path.inspect} in bucket #{bucket_name}"
+    end
+
+    if result.status >= 400
+      raise Fog::Errors::NotFound, "Backblaze respond with status = #{result.status} - #{result.reason_phrase}"
+    end
+
+    "#{get_object_url(bucket_name, file_path)}?Authorization=#{result.json['authorizationToken']}"
+  end
+
+  def get_object(bucket_name, file_name)
+    file_url = get_object_url(bucket_name, file_name)
+
+    response = b2_command(nil,
+      method: :get,
+      url: file_url
+    )
+
+    if response.status == 404
+      raise Fog::Errors::NotFound, "Can not find #{file_name.inspect} in bucket #{bucket_name}"
+    end
+
+    if response.status > 400
+      raise Fog::Errors::Error, "Failed get_object, status = #{response.status} #{response.body}"
+    end
+
+    return response
+  end
+
+  # call b2_delete_file_version
+  def delete_object(bucket_name, file_name)
+    version_ids = _get_object_version_ids(bucket_name, file_name)
+
+    if version_ids.size == 0
+      raise Fog::Errors::NotFound, "Can not find #{file_name} in in bucket #{bucket_name}"
+    end
+
+    logger.info("Deleting #{version_ids.size} versions of #{file_name}")
+
+    last_response = nil
+    version_ids.each do |version_id|
+      last_response = b2_command(:b2_delete_file_version, body: {
+        fileName: file_name,
+        fileId: version_id
+      })
+    end
+
+    last_response
+  end
+
+  def _get_object_version_ids(bucket_name, file_name)
+    response = b2_command(:b2_list_file_versions,
+      body: {
+        startFileName: file_name,
+        prefix: file_name,
+        bucketId: _get_bucket_id!(bucket_name),
+        maxFileCount: 1000
+      }
+    )
+
+    if response.status >= 400
+      raise Fog::Errors::Error, "Fetch error: #{response.json['message']} (status = #{response.status})"
+    end
+
+    if response.json['files']
+      version_ids = []
+      response.json['files'].map do |file_version|
+        version_ids << file_version['fileId'] if file_version['fileName'] == file_name
+      end
+      version_ids
+    else
+      []
+    end
+  end
+
+  def _get_bucket_id(bucket_name)
+    if @options[:b2_bucket_name] == bucket_name && @options[:b2_bucket_id]
+      return @options[:b2_bucket_id]
+    else
+      cached = @token_cache && @token_cache.buckets
+
+      if cached && cached[bucket_name]
+        return cached[bucket_name]['bucketId']
+      else
+        fetched = _cached_buchets_hash(force_fetch: !!cached)
+        return fetched[bucket_name] && fetched[bucket_name]['bucketId']
+      end
+    end
+  end
+
+  def _get_bucket_id!(bucket_name)
+    bucket_id = _get_bucket_id(bucket_name)
+    unless bucket_id
+      raise Fog::Errors::NotFound, "Can not find bucket #{bucket_name}"
+    end
+
+    return bucket_id
+  end
+
+  def _cached_buchets_hash(force_fetch: false)
+
+    if !force_fetch && cached = @token_cache.buckets
+      cached
+    end
+
+    buckets_hash = {}
+    list_buckets.json['buckets'].each do |bucket|
+      buckets_hash[bucket['bucketName']] = bucket
+    end
+
+    @token_cache.buckets = buckets_hash
+
+    buckets_hash
+  end
+
+  def auth_response
+    #return @auth_response.json if @auth_response
+
+    if cached = @token_cache.auth_response
+      logger.info("get token from cache")
+      return cached
+    end
+
+    @auth_response = json_req(:get, "https://api.backblazeb2.com/b2api/v1/b2_authorize_account",
+      headers: {
+        "Authorization" => "Basic " + Base64.strict_encode64("#{@options[:b2_account_id]}:#{@options[:b2_account_token]}")
+      },
+      persistent: false
+    )
+
+    if @auth_response.status >= 400
+      raise Fog::Errors::Error, "Authentication error: #{@auth_response.json['message']} (status = #{@auth_response.status})\n#{@auth_response.body}"
+    end
+
+    @token_cache.auth_response = @auth_response.json
+
+    @auth_response.json
+  end
+
+  def b2_command(command, options = {})
+    auth_response = self.auth_response
+    options[:headers] ||= {}
+    options[:headers]['Authorization'] ||= auth_response['authorizationToken']
+
+    if options[:body] && !options[:body].is_a?(String)
+      options[:body] = JSON.generate(options[:body])
+    end
+
+    request_url = options.delete(:url) || "#{auth_response['apiUrl']}/b2api/v1/#{command}"
+
+    #pp [:b2_command, request_url, options]
+
+    json_req(options.delete(:method) || :post, request_url, options)
+  end
+
+  def json_req(method, url, options = {})
+    start_time = Time.now.to_f
+    logger.info("Req #{method.to_s.upcase} #{url}")
+    logger.debug(options.to_s)
+
+    if !options.has_key?(:persistent) || options[:persistent] == true
+      @connections ||= {}
+      full_path = [URI.parse(url).request_uri, URI.parse(url).fragment].compact.join("#")
+      host_url = url.sub(full_path, "")
+      connection = @connections[host_url] ||= Excon.new(host_url, persistent: true)
+      http_response = connection.send(method, options.merge(path: full_path, idempotent: true))
+    else
+      http_response = Excon.send(method, url, options)
+    end
+
+    http_response.extend(Fog::Backblaze::JSONResponse)
+    http_response.assign_json_body! if http_response.josn_response?
+
+    http_response
+  ensure
+    status = http_response && http_response.status
+    logger.info("Done #{method.to_s.upcase} #{url} = #{status} (#{(Time.now.to_f - start_time).round(3)} sec)")
+    logger.debug("Response Headers: #{http_response.headers}") if http_response
+    logger.debug("Response Body: #{http_response.body}") if http_response
+  end
+
+  def reset_token_cache
+    @token_cache.reset
+  end
+
+  def _esc_file(file_name)
+    CGI.escape(file_name).gsub('%2F', '/')
+  end
+end

data/lib/fog/storage/backblaze.rb

@@ -0,0 +1,17 @@
+require 'json'
+require 'digest'
+require 'cgi'
+
+class Fog::Storage::Backblaze < Fog::Service
+  requires :b2_account_id, :b2_account_token
+  recognizes :b2_bucket_name, :b2_bucket_id, :token_cache, :logger
+
+  model_path 'fog/storage/backblaze/models'
+  model       :directory
+  collection  :directories
+  model       :file
+  collection  :files
+
+  autoload :Mock, File.expand_path("../backblaze/mock", __FILE__)
+  autoload :Real, File.expand_path("../backblaze/real", __FILE__)
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fog-backblaze
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Pavel Evstigneev
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-03-27 00:00:00.000000000 Z
+date: 2018-05-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fog-core
@@ -41,7 +41,6 @@ files:
 - ".gitignore"
 - ".travis.yml"
 - Gemfile
-- Gemfile.lock
 - LICENSE
 - README.md
 - Rakefile
@@ -51,13 +50,15 @@ files:
 - fog-backblaze.gemspec
 - lib/fog/backblaze.rb
 - lib/fog/backblaze/json_response.rb
-- lib/fog/backblaze/models/directories.rb
-- lib/fog/backblaze/models/directory.rb
-- lib/fog/backblaze/models/file.rb
-- lib/fog/backblaze/models/files.rb
-- lib/fog/backblaze/storage.rb
 - lib/fog/backblaze/token_cache.rb
 - lib/fog/backblaze/version.rb
+- lib/fog/storage/backblaze.rb
+- lib/fog/storage/backblaze/mock.rb
+- lib/fog/storage/backblaze/models/directories.rb
+- lib/fog/storage/backblaze/models/directory.rb
+- lib/fog/storage/backblaze/models/file.rb
+- lib/fog/storage/backblaze/models/files.rb
+- lib/fog/storage/backblaze/real.rb
 homepage: https://github.com/fog/fog-backblaze
 licenses: []
 metadata: {}

data/Gemfile.lock

@@ -1,34 +0,0 @@
-PATH
-  remote: .
-  specs:
-    fog-backblaze (0.1.1)
-      fog-core
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    builder (3.2.3)
-    excon (0.61.0)
-    fog-core (2.1.0)
-      builder
-      excon (~> 0.58)
-      formatador (~> 0.2)
-      mime-types
-    formatador (0.2.5)
-    mime-types (3.1)
-      mime-types-data (~> 3.2015)
-    mime-types-data (3.2016.0521)
-    minitest (5.11.3)
-    rake (12.3.1)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  bundler (>= 1.15)
-  fog-backblaze!
-  minitest (>= 5.0)
-  rake (~> 12.0)
-
-BUNDLED WITH
-   1.16.1

data/lib/fog/backblaze/models/directory.rb

@@ -1,37 +0,0 @@
-class Fog::Storage::Backblaze::Directory < Fog::Model
-  identity :key, aliases: %w(bucketName)
-
-  attribute :bucket_id, aliases: 'bucketId'
-  attribute :bucket_info, aliases: 'bucketInfo'
-  attribute :bucket_type, aliases: 'bucketType'
-
-  def destroy
-    requires :key
-    service.delete_bucket(key)
-    true
-  #rescue Fog::Errors::NotFound
-  #  false
-  end
-
-  def save
-    requires :key
-    options = {}
-
-    response = service.put_bucket(key, options)
-
-    attributes[:bucket_id] = response.json['bucketId']
-    attributes[:bucket_type] = response.json['bucketType']
-    attributes[:bucket_info] = response.json['bucketInfo']
-
-    true
-  end
-
-  def files
-    @files ||= Fog::Storage::Backblaze::Files.new(directory: self, service: service)
-  end
-
-  def public?
-    attributes[:bucket_type] == "allPublic"
-  end
-
-end

data/lib/fog/backblaze/storage.rb

@@ -1,426 +0,0 @@
-require 'json'
-require 'digest'
-require 'cgi'
-
-class Fog::Storage::Backblaze < Fog::Service
-  requires :b2_account_id, :b2_account_token
-  recognizes :b2_bucket_name, :b2_bucket_id, :token_cache, :logger
-
-  model_path 'fog/backblaze/models'
-  model       :directory
-  collection  :directories
-  model       :file
-  collection  :files
-
-  class Mock
-    #include Integrity
-
-    def self.data
-      @data ||= Hash.new do |hash, key|
-        hash[key] = {}
-      end
-    end
-
-    def self.reset
-      @data = nil
-    end
-
-    def initialize(options={})
-      @b2_account_id = options[:b2_account_id]
-      @b2_account_token = options[:b2_account_token]
-      @path = '/v1/AUTH_1234'
-      @containers = {}
-    end
-
-    def data
-      self.class.data[@softlayer_username]
-    end
-
-    def reset_data
-      self.class.data.delete(@softlayer_username)
-    end
-
-    def change_account(account)
-      @original_path ||= @path
-      version_string = @original_path.split('/')[1]
-      @path = "/#{version_string}/#{account}"
-    end
-
-    def reset_account_name
-      @path = @original_path
-    end
-
-  end
-
-  class Real
-    attr_reader :token_cache, :options
-
-    def initialize(options = {})
-      @options = options
-      @logger = @options[:logger] || begin
-        require 'logger'
-        Logger.new("/dev/null")
-      end
-
-      @token_cache = if options[:token_cache].nil? || options[:token_cache] == :memory
-        Fog::Backblaze::TokenCache.new
-      elsif options[:token_cache] === false
-        Fog::Backblaze::TokenCache::NullTokenCache.new
-      elsif token_cache.is_a?(Fog::Backblaze::TokenCache)
-        token_cache
-      else
-        Fog::Backblaze::TokenCache::FileTokenCache.new(options[:token_cache])
-      end
-    end
-
-    def logger
-      @logger
-    end
-
-    ## Buckets
-
-    def put_bucket(key, extra_options = {})
-      options = {
-        accountId: @options[:b2_account_id],
-        bucketType: extra_options.delete(:public) ? 'allPublic' : 'allPrivate',
-        bucketName: key,
-      }.merge(extra_options)
-
-      response = b2_command(:b2_create_bucket, body: options)
-
-      if response.status >= 400
-        raise Fog::Errors::Error, "Failed put_bucket, status = #{response.status} #{response.body}"
-      end
-
-      if cached = @token_cache.buckets
-        @token_cache.buckets = cached.merge(key => response.json)
-      else
-        @token_cache.buckets = {key => response.json}
-      end
-
-      response
-    end
-
-    def list_buckets
-      response = b2_command(:b2_list_buckets, body: {accountId: @options[:b2_account_id]})
-
-      response
-    end
-
-    def get_bucket(bucket_name)
-      reponse = list_buckets
-      bucket = reponse.json['buckets'].detect do |bucket|
-        bucket['bucketName'] == bucket_name
-      end
-
-      unless bucket
-        raise Fog::Errors::NotFound, "No bucket with name: #{bucket_name}, " +
-          "found: #{reponse.json['buckets'].map {|b| b['bucketName']}.join(", ")}"
-      end
-
-      reponse.body = bucket
-      reponse.json = bucket
-      return reponse
-    end
-
-    def delete_bucket(bucket_name, options = {})
-      bucket_id = _get_bucket_id(bucket_name)
-
-      unless bucket_id
-        raise Fog::Errors::NotFound, "Can not bucket #{bucket_name}"
-      end
-
-      response = b2_command(:b2_delete_bucket,
-        body: {
-          bucketId: bucket_id,
-          accountId: @options[:b2_account_id]
-        }
-      )
-
-      if !options[:is_retrying]
-        if response.status == 400 && response.json['message'] =~ /Bucket .+ does not exist/
-          logger.info("Try drop cache and try again")
-          @token_cache.buckets = nil
-          return delete_bucket(bucket_name, is_retrying: true)
-        end
-      end
-
-      if response.status >= 400
-        raise Fog::Errors::Error, "Failed delete_bucket, status = #{response.status} #{response.body}"
-      end
-
-      if cached = @token_cache.buckets
-        cached.delete(bucket_name)
-        @token_cache.buckets = cached
-      end
-
-      response
-    end
-
-    ## Objects
-
-    def list_objects(bucket_name, options = {})
-      bucket_id = _get_bucket_id(bucket_name)
-
-      unless bucket_id
-        raise Fog::Errors::NotFound, "Can not bucket #{bucket_name}"
-      end
-
-      b2_command(:b2_list_file_names, body: {
-        bucketId: bucket_id,
-        maxFileCount: 10_000
-      }.merge(options))
-    end
-
-    def head_object(bucket_name, file_path)
-      file_url = get_object_url(bucket_name, file_path)
-
-      result = b2_command(nil,
-        method: :head,
-        url: file_url
-      )
-
-      if result.status == 404
-        raise Fog::Errors::NotFound, "Can not find #{file_path.inspect} in bucket #{bucket_name}"
-      end
-
-      if result.status >= 400
-        raise Fog::Errors::NotFound, "Backblaze respond with status = #{result.status} - #{result.reason_phrase}"
-      end
-
-      result
-    end
-
-    # TODO: handle options
-    def put_object(bucket_name, file_path, content, options = {})
-      upload_url = @token_cache.fetch("upload_url/#{bucket_name}") do
-        bucket_id = _get_bucket_id(bucket_name)
-        unless bucket_id
-          raise Fog::Errors::NotFound, "Can not find bucket #{bucket_name.inspect}"
-        end
-        result = b2_command(:b2_get_upload_url, body: {bucketId: _get_bucket_id(bucket_name)})
-        result.json
-      end
-
-      extra_headers = {}
-      if options[:content_type]
-        extra_headers['Content-Type'] = options[:content_type]
-      end
-      if options[:last_modified]
-        extra_headers['X-Bz-Info-src_last_modified_millis'] = options[:last_modified]
-      end
-      if options[:content_disposition]
-        extra_headers['X-Bz-Info-b2-content-disposition'] = options[:content_disposition]
-      end
-
-      response = b2_command(nil,
-        url: upload_url['uploadUrl'],
-        body: content,
-        headers: {
-          'Authorization': upload_url['authorizationToken'],
-          'Content-Type': 'b2/x-auto',
-          'X-Bz-File-Name': "#{_esc_file(file_path)}",
-          'X-Bz-Content-Sha1': Digest::SHA1.hexdigest(content)
-        }.merge(extra_headers)
-      )
-
-      if response.json['fileId'] == nil
-        raise Fog::Errors::Error, "Failed put_object, status = #{response.status} #{response.body}"
-      end
-
-      response
-    end
-
-    def get_object_url(bucket_name, file_path)
-      "#{auth_response['downloadUrl']}/file/#{CGI.escape(bucket_name)}/#{_esc_file(file_path)}"
-    end
-
-    alias_method :get_object_https_url, :get_object_url
-
-    def get_public_object_url(bucket_name, file_path, options = {})
-      bucket_id = _get_bucket_id(bucket_name)
-
-      unless bucket_id
-        raise Fog::Errors::NotFound, "Can not bucket #{bucket_name}"
-      end
-
-      result = b2_command(:b2_get_download_authorization, body: {
-        bucketId: bucket_id,
-        fileNamePrefix: file_path,
-        validDurationInSeconds: 604800
-      }.merge(options))
-
-      if result.status == 404
-        raise Fog::Errors::NotFound, "Can not find #{file_path.inspect} in bucket #{bucket_name}"
-      end
-
-      if result.status >= 400
-        raise Fog::Errors::NotFound, "Backblaze respond with status = #{result.status} - #{result.reason_phrase}"
-      end
-
-      "#{get_object_url(bucket_name, file_path)}?Authorization=#{result.json['authorizationToken']}"
-    end
-
-    def get_object(bucket_name, file_name)
-      file_url = get_object_url(bucket_name, file_name)
-
-      result = b2_command(nil,
-        method: :get,
-        url: file_url
-      )
-
-      if result.status == 404
-        raise Fog::Errors::NotFound, "Can not find #{file_name.inspect} in bucket #{bucket_name}"
-      end
-
-      return result
-    end
-
-    def delete_object(bucket_name, file_name)
-      version_ids = _get_object_version_ids(bucket_name, file_name)
-
-      if version_ids.size == 0
-        raise Fog::Errors::NotFound, "Can not find #{file_name} in in bucket #{bucket_name}"
-      end
-
-      logger.info("Deleting #{version_ids.size} versions of #{file_name}")
-
-      last_response = nil
-      version_ids.each do |version_id|
-        last_response = b2_command(:b2_delete_file_version, body: {
-          fileName: file_name,
-          fileId: version_id
-        })
-      end
-
-      last_response
-    end
-
-    def _get_object_version_ids(bucket_name, file_name)
-      response = b2_command(:b2_list_file_versions,
-        body: {
-          startFileName: file_name,
-          prefix: file_name,
-          bucketId: _get_bucket_id(bucket_name),
-          maxFileCount: 1000
-        }
-      )
-
-      if response.json['files']
-        version_ids = []
-        response.json['files'].map do |file_version|
-          version_ids << file_version['fileId'] if file_version['fileName'] == file_name
-        end
-        version_ids
-      else
-        []
-      end
-    end
-
-    def _get_bucket_id(bucket_name)
-      if @options[:b2_bucket_name] == bucket_name && @options[:b2_bucket_id]
-        return @options[:b2_bucket_id]
-      else
-        cached = @token_cache && @token_cache.buckets
-
-        if cached && cached[bucket_name]
-          return cached[bucket_name]['bucketId']
-        else
-          fetched = _cached_buchets_hash(force_fetch: !!cached)
-          return fetched[bucket_name] && fetched[bucket_name]['bucketId']
-        end
-      end
-    end
-
-    def _cached_buchets_hash(force_fetch: false)
-
-      if !force_fetch && cached = @token_cache.buckets
-        cached
-      end
-
-      buckets_hash = {}
-      list_buckets.json['buckets'].each do |bucket|
-        buckets_hash[bucket['bucketName']] = bucket
-      end
-
-      @token_cache.buckets = buckets_hash
-
-      buckets_hash
-    end
-
-    def auth_response
-      #return @auth_response.json if @auth_response
-
-      if cached = @token_cache.auth_response
-        logger.info("get token from cache")
-        return cached
-      end
-
-      @auth_response = json_req(:get, "https://api.backblazeb2.com/b2api/v1/b2_authorize_account",
-        headers: {
-          "Authorization" => "Basic " + Base64.strict_encode64("#{@options[:b2_account_id]}:#{@options[:b2_account_token]}")
-        },
-        persistent: false
-      )
-
-      if @auth_response.status >= 400
-        raise Fog::Errors::Error, "Authentication error: #{@auth_response.json['message']} (status = #{@auth_response.status})\n#{@auth_response.body}"
-      end
-
-      @token_cache.auth_response = @auth_response.json
-
-      @auth_response.json
-    end
-
-    def b2_command(command, options = {})
-      auth_response = self.auth_response
-      options[:headers] ||= {}
-      options[:headers]['Authorization'] ||= auth_response['authorizationToken']
-
-      if options[:body] && !options[:body].is_a?(String)
-        options[:body] = JSON.generate(options[:body])
-      end
-
-      request_url = options.delete(:url) || "#{auth_response['apiUrl']}/b2api/v1/#{command}"
-
-      #pp [:b2_command, request_url, options]
-
-      json_req(options.delete(:method) || :post, request_url, options)
-    end
-
-    def json_req(method, url, options = {})
-      start_time = Time.now.to_f
-      logger.info("Req #{method.to_s.upcase} #{url}")
-      logger.debug(options.to_s)
-
-      if !options.has_key?(:persistent) || options[:persistent] == true
-        @connections ||= {}
-        full_path = [URI.parse(url).request_uri, URI.parse(url).fragment].compact.join("#")
-        host_url = url.sub(full_path, "")
-        connection = @connections[host_url] ||= Excon.new(host_url, persistent: true)
-        http_response = connection.send(method, options.merge(path: full_path, idempotent: true))
-      else
-        http_response = Excon.send(method, url, options)
-      end
-
-      http_response.extend(Fog::Backblaze::JSONReponse)
-      http_response.assign_json_body! if http_response.josn_response?
-
-      http_response
-    ensure
-      status = http_response && http_response.status
-      logger.info("Done #{method.to_s.upcase} #{url} = #{status} (#{(Time.now.to_f - start_time).round(3)} sec)")
-      logger.debug(http_response.headers) if http_response
-      logger.debug(http_response.body) if http_response
-    end
-
-    def reset_token_cache
-      @token_cache.reset
-    end
-
-    def _esc_file(file_name)
-      CGI.escape(file_name).gsub('%2F', '/')
-    end
-  end
-end
-