blobby 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 4a0b96e9f498791cc7df4c6dfe2a16fe3333495f
+  data.tar.gz: c45aa9cff2b917fed75bb174a7ca02482b39ff4e
+SHA512:
+  metadata.gz: 6e1643a9675acfdfa5a64ea97a8d1e7fd98557f150b243dd75838be5a5ddbe9a7b8e8a312b5c4560b0097eef0f99423c94a1a5e359e5b5b7b7f3365d5c71fc61
+  data.tar.gz: 5a010f2432a9a6dd3caa22e5eed35b8b6986180c5b6ace5ca2d49c4b2f52d9d1100a8965913c4d693d6464d49f8c3c9a86cf3598526755ba86f15e4a38ea5b58

data/.gitignore

@@ -0,0 +1,20 @@
+*.gem
+*.rbc
+.bundle
+.config
+.rbenv-*
+.ruby-version
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+.rvmrc

data/.rspec

@@ -0,0 +1 @@
+--color

data/.rubocop.yml

@@ -0,0 +1,47 @@
+Eval:
+  Exclude:
+    - "Rakefile"
+
+Metrics/AbcSize:
+  Enabled: false
+
+Metrics/LineLength:
+  Max: 120
+
+Metrics/MethodLength:
+  Max: 30
+
+Style/ClassAndModuleChildren:
+  EnforcedStyle: nested
+  Exclude:
+    - "spec/**/*"
+
+Style/Documentation:
+  Exclude:
+    - "spec/**/*"
+
+Style/EmptyLinesAroundBlockBody:
+  Enabled: false
+
+Style/EmptyLinesAroundClassBody:
+  EnforcedStyle: empty_lines
+
+Style/EmptyLinesAroundModuleBody:
+  Enabled: false
+
+Style/Encoding:
+  EnforcedStyle: when_needed
+  Enabled: true
+
+Style/FileName:
+  Exclude:
+    - "bin/*"
+
+Style/HashSyntax:
+  EnforcedStyle: hash_rockets
+
+Style/StringLiterals:
+  EnforcedStyle: double_quotes
+
+Style/WordArray:
+  Enabled: false

data/.travis.yml

@@ -0,0 +1,5 @@
+language: ruby
+rvm:
+  - 2.0.0
+  - 2.1.5
+  - 2.2.0

data/Gemfile

@@ -0,0 +1,10 @@
+source "https://rubygems.org"
+
+# Runtime dependencies in blobby.gemspec
+gemspec
+
+# Development/test dependencies below
+gem "rake", "~> 10.0"
+gem "rspec", "~> 3.1"
+gem "sinatra"
+gem "sham_rack", "~> 1.3.5"

data/README.md

@@ -0,0 +1,43 @@
+# Blobby
+
+[![Build Status](https://travis-ci.org/realestate-com-au/blobby.svg?branch=master)](https://travis-ci.org/realestate-com-au/blobby)
+
+This gem provides a standard interface for storing big chunks of data.
+
+## Usage
+
+It supports popular BLOBs operations such as reading:
+
+    store["key"].read
+
+writing:
+
+    store["key"].write("some content")
+    store["key"].write(File.open("kitty.png"))
+
+checking for existance:
+
+    store["key"].exists?
+
+and even deleting:
+
+    store["key"].delete
+
+This gem provides several "store" implementations:
+
+    # on disk
+    Blobby::FilesystemStore.new("/big/data")
+
+    # in memory
+    Blobby::InMemoryStore.new
+
+    # generic HTTP
+    Blobby::HttpStore.new("http://attachment-store/objects")
+
+    # fake success
+    Blobby::FakeSuccessStore.new
+
+Other gems provide additional implementations:
+
+    # gem "blobby-s3"
+    Blobby::S3Store.new("mybucket")

data/Rakefile

@@ -0,0 +1,13 @@
+#!/usr/bin/env rake
+
+require "bundler"
+
+Bundler::GemHelper.install_tasks
+
+require "rspec/core/rake_task"
+
+task "default" => "spec"
+
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.rspec_opts = ["--colour"]
+end

data/blobby.gemspec

@@ -0,0 +1,20 @@
+# -*- encoding: utf-8 -*-
+
+$LOAD_PATH << File.expand_path("../lib", __FILE__)
+require "blobby/version"
+
+Gem::Specification.new do |gem|
+
+  gem.authors       = ["Mike Williams"]
+  gem.email         = ["mdub@dogbiscuit.org"]
+  gem.summary       = "Various ways of storing BLOBs"
+  gem.homepage      = "https://github.com/realestate-com.au/blobby"
+
+  gem.files         = `git ls-files`.split($OUTPUT_RECORD_SEPARATOR)
+  gem.executables   = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "blobby"
+  gem.require_paths = ["lib"]
+  gem.version       = Blobby::VERSION
+
+end

data/lib/blobby/composite_store.rb

@@ -0,0 +1,64 @@
+module Blobby
+
+  # Compose a number of stores.
+  #
+  # Writes go to all stores. Reads use the first store to respond.
+  #
+  class CompositeStore
+
+    def initialize(stores)
+      @stores = stores
+    end
+
+    def [](key)
+      KeyConstraint.must_allow!(key)
+      objects = stores.map { |store| store[key] }
+      StoredObject.new(objects)
+    end
+
+    def available?
+      stores.all?(&:available?)
+    end
+
+    private
+
+    attr_reader :stores
+
+    class StoredObject
+
+      def initialize(objects)
+        @objects = objects
+      end
+
+      def exists?
+        objects.any?(&:exists?)
+      end
+
+      def read(&block)
+        objects.each do |o|
+          return o.read(&block) if o.exists?
+        end
+        nil
+      end
+
+      def write(content)
+        content = content.read if content.respond_to?(:read)
+        objects.each do |o|
+          o.write(content)
+        end
+        nil
+      end
+
+      def delete
+        objects.all?(&:delete)
+      end
+
+      private
+
+      attr_reader :objects
+
+    end
+
+  end
+
+end

data/lib/blobby/fake_success_store.rb

@@ -0,0 +1,52 @@
+require "blobby/key_constraint"
+
+module Blobby
+
+  # A BLOB store that is always successful.
+  #
+  class FakeSuccessStore
+
+    def available?
+      true
+    end
+
+    def [](key)
+      KeyConstraint.must_allow!(key)
+      StoredObject.new
+    end
+
+    class StoredObject
+
+      def exists?
+        true
+      end
+
+      def read
+        image_path = Pathname(File.dirname(__FILE__)) + "placeholder.png"
+        image_path.open("rb") do |io|
+          if block_given?
+            while chunk = io.read(512)
+              yield chunk
+            end
+            nil
+          else
+            io.read
+          end
+        end
+      rescue Errno::ENOENT
+        nil
+      end
+
+      def write(_content)
+        nil
+      end
+
+      def delete
+        true
+      end
+
+    end
+
+  end
+
+end

data/lib/blobby/filesystem_store.rb

@@ -0,0 +1,133 @@
+require "blobby/key_constraint"
+require "fileutils"
+require "forwardable"
+require "pathname"
+require "tempfile"
+
+module Blobby
+
+  # A BLOB store backed by a file-system.
+  #
+  class FilesystemStore
+
+    def initialize(dir, options = {}, &sharding_strategy)
+      @dir = Pathname(dir)
+      @umask = options[:umask] || File.umask
+      @sharding_strategy = sharding_strategy || noop_sharding_strategy
+    end
+
+    attr_reader :dir
+    attr_reader :umask
+
+    def available?
+      dir.directory? && dir.readable? && dir.writable?
+    end
+
+    def [](key)
+      KeyConstraint.must_allow!(key)
+      relative_path = @sharding_strategy.call(key)
+      StoredObject.new(dir + relative_path, umask)
+    end
+
+    private
+
+    def noop_sharding_strategy
+      ->(key) { key }
+    end
+
+    class StoredObject
+
+      def initialize(path, umask)
+        @path = path
+        @umask = umask
+      end
+
+      extend Forwardable
+
+      def_delegator :@path, :exist?, :exists?
+
+      def read
+        @path.open("rb") do |io|
+          if block_given?
+            while chunk = io.read(512)
+              yield chunk
+            end
+            nil
+          else
+            io.read
+          end
+        end
+      rescue Errno::ENOENT
+        nil
+      end
+
+      def write(content)
+        atomic_create(@path) do |out|
+          if content.respond_to?(:read)
+            FileUtils.copy_stream(content, out)
+          else
+            out << content
+          end
+        end
+        nil
+      end
+
+      def delete
+        !!FileUtils.rm(@path)
+      rescue Errno::ENOENT
+        false
+      end
+
+      private
+
+      def apply_umask(mode)
+        mode & ~@umask
+      end
+
+      def using_default_umask?
+        @umask == File.umask
+      end
+
+      RAND_MAX = ("F" * 10).to_i(16)
+
+      def tmp_name
+        sprintf("tmp-%X", rand(RAND_MAX))
+      end
+
+      def atomic_create(store_path)
+        store_dir = store_path.parent
+        tmp_path  = store_dir + tmp_name
+
+        tmp = nil
+        begin
+          tmp = tmp_path.open(File::CREAT | File::EXCL | File::WRONLY, 0666)
+          tmp.binmode
+        rescue Errno::ENOENT => e
+          FileUtils.mkdir_p(store_dir.to_s, :mode => apply_umask(0777))
+          retry
+        end
+
+        begin
+          yield tmp
+          tmp.chmod(apply_umask(0666)) unless using_default_umask?
+        ensure
+          tmp.close
+        end
+
+        first_try = true
+        begin
+          tmp_path.rename(store_path)
+        rescue Errno::ESTALE => e
+          raise unless first_try
+          first_try = false
+          now = Time.now
+          File.utime(now, now, store_dir.to_s)
+          retry
+        end
+      end
+
+    end
+
+  end
+
+end

data/lib/blobby/http_store.rb

@@ -0,0 +1,135 @@
+require "blobby/key_constraint"
+require "net/http"
+
+module Blobby
+
+  # A BLOB store backed by HTTP.
+  #
+  class HttpStore
+
+    def initialize(base_url, options = {})
+      @base_url = base_url
+      @base_url += "/" unless @base_url.end_with?("/")
+      @max_retries = options.fetch(:max_retries, 2)
+    end
+
+    attr_reader :base_url
+    attr_reader :max_retries
+
+    def available?
+      with_http_connection do
+        true
+      end
+    rescue
+      false
+    end
+
+    def [](key)
+      KeyConstraint.must_allow!(key)
+      StoredObject.new(self, key)
+    end
+
+    def base_uri
+      URI(base_url)
+    end
+
+    def with_http_connection
+      remaining_retry_intervals = retry_intervals(max_retries)
+      begin
+        Net::HTTP.start(base_uri.host, base_uri.port) do |http|
+          yield http, base_uri.path
+        end
+      rescue *retryable_exceptions => e
+        raise e if remaining_retry_intervals.empty?
+        sleep(remaining_retry_intervals.shift) && retry
+      end
+    end
+
+    protected
+
+    def retryable_exceptions
+      [EOFError, Errno::ECONNRESET]
+    end
+
+    def retry_intervals(n)
+      # exponential backoff: [0.5, 1, 2, 4, 8, ...]
+      scaling_factor = (0.5 + Kernel.rand * 0.1) # a little random avoids throbbing
+      Array.new(n) { |i| (2**i) * scaling_factor }
+    end
+
+    class StoredObject
+
+      def initialize(store, key)
+        @store = store
+        @key = key
+      end
+
+      attr_reader :key
+
+      def exists?
+        with_http_connection do |http, path|
+          response = http.head(path)
+          response.code == "200"
+        end
+      end
+
+      def read(&block)
+        with_http_connection do |http, path|
+          http.request_get(path) do |response|
+            case response
+            when Net::HTTPNotFound then
+              return nil
+            when Net::HTTPSuccess then
+              if block_given?
+                response.read_body(&block)
+                return nil
+              else
+                return response.read_body
+              end
+            end
+            response.error!
+          end
+        end
+      end
+
+      def write(content)
+        content = content.read if content.respond_to?(:read)
+        with_http_connection do |http, path|
+          put = Net::HTTP::Put.new(path)
+          put.body = content
+          put["Content-Type"] = "application/octet-stream"
+          response = http.request(put)
+          response.error! unless response.is_a?(Net::HTTPSuccess)
+          true
+        end
+        nil
+      end
+
+      def delete
+        with_http_connection do |http, path|
+          delete = Net::HTTP::Delete.new(path)
+          response = http.request(delete)
+          case response
+          when Net::HTTPSuccess then
+            true
+          when Net::HTTPNotFound then
+            false
+          else
+            response.error!
+          end
+        end
+      end
+
+      private
+
+      def with_http_connection
+        @store.with_http_connection do |http, base_path|
+          yield http, base_path + key
+        end
+      end
+
+    end
+
+  end
+
+end