fragmenter 0.5.0

data/.gitignore

@@ -0,0 +1,9 @@
+*.gem
+*.rbc
+.bundle
+.config
+Gemfile.lock
+doc/
+bin/
+vendor/ruby
+tmp

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/HISTORY.md

@@ -0,0 +1,3 @@
+# 0.5.0
+
+* Initial release!

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Dscout, Inc
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,51 @@
+# Fragmenter
+
+Fragmenter stores and rebuilds binary data in a distributed fashion. The only
+engine currently provided is Redis.
+
+## Why Fragments?
+
+It alleviates the problems posed by uploading large blocks of data from slow
+clients, notably mobile apps, by allowing the device to send multiple smaller
+blocks of data independently. Once all of the smaller blocks have been received
+they can quickly be rebuilt into the original file on the server.
+
+## Requirements
+
+Fragmenter is tested on Ruby 1.9.3, but any ruby implementation with 1.9 syntax
+should be supported.
+
+Redis 2.0 or greater is required and version 2.6 is recommended.
+
+## Installation
+
+    $ gem install fragmenter
+
+## Getting Started
+
+### Configuration
+
+    Fragmenter.configure do |config|
+      config.redis      = $redis
+      config.logger     = Rails.logger
+      config.expiration = 2.days.to_i
+    end
+
+### Usage
+
+    fragmenter = Fragmenter::Base.new(record)
+
+    fragmenter.store(binary_data, number: 1, total: 12, content_type: 'image/jpeg')
+    fragmenter.complete? # => false
+
+    fragmenter.store(binary_data, number: 12, total: 12, content_type: 'image/jpeg')
+    fragmenter.complete? # => true
+
+    rebuilt = fragmenter.rebuild # => binary data
+    fragmenter.clean!
+
+More detailed examples will be added soon.
+
+## License
+
+Please see LICENSE for licensing details.

data/Rakefile

@@ -0,0 +1,7 @@
+require 'bundler/setup'
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new :spec
+
+task :default => :spec

data/fragmenter.gemspec

@@ -0,0 +1,21 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'fragmenter/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = 'fragmenter'
+  gem.version       = Fragmenter::VERSION
+  gem.authors       = ['Parker Selbert']
+  gem.email         = ['parker@sorentwo.com']
+  gem.description   = %q{Fragmentize and rebuild data}
+  gem.summary       = %q{Fragmentize and rebuild data}
+
+  gem.files         = `git ls-files`.split($/)
+  gem.test_files    = gem.files.grep(%r{^(spec)/})
+  gem.require_paths = ['lib']
+
+  gem.add_dependency 'redis', '~> 3.0.0'
+
+  gem.add_development_dependency 'rspec', '~> 2.11.0'
+end

data/lib/fragmenter.rb

@@ -0,0 +1,37 @@
+require 'logger'
+require 'redis'
+require 'fragmenter/version'
+require 'fragmenter/base'
+require 'fragmenter/redis'
+
+module Fragmenter
+  def self.configure(&block)
+    yield self
+  end
+
+  def self.logger
+    @logger ||= Logger.new(STDOUT).tap do |logger|
+      logger.level = Logger::INFO
+    end
+  end
+
+  def self.logger=(logger)
+    @logger = logger
+  end
+
+  def self.redis
+    @redis ||= ::Redis.new
+  end
+
+  def self.redis=(redis)
+    @redis = redis
+  end
+
+  def self.expiration=(expiration)
+    @expiration = expiration
+  end
+
+  def self.expiration
+    @expiration || 60 * 60 * 24
+  end
+end

data/lib/fragmenter/base.rb

@@ -0,0 +1,27 @@
+module Fragmenter
+  class Base
+    extend Forwardable
+
+    attr_reader :object, :engine
+
+    delegate clean!:    :engine
+    delegate complete?: :engine
+    delegate fragments: :engine
+    delegate meta:      :engine
+    delegate rebuild:   :engine
+    delegate store:     :engine
+
+    def initialize(object, engine_class = Fragmenter::Redis)
+      @object = object
+      @engine = engine_class.new(self)
+    end
+
+    def key
+      [object.class.to_s.downcase, object.id].join('-')
+    end
+
+    def as_json
+      engine.meta.merge('fragments' => engine.fragments)
+    end
+  end
+end

data/lib/fragmenter/fragment.rb

@@ -0,0 +1,50 @@
+module Fragmenter
+  class Fragment
+    attr_reader :blob, :options
+
+    def initialize(blob, options)
+      @blob    = blob
+      @options = options
+    end
+
+    def number
+      @number ||= options.fetch(:number, 1).to_i
+    end
+
+    def total
+      @total ||= options.fetch(:total, 1).to_i
+    end
+
+    def content_type
+      @content_type ||= options[:content_type] || 'application/octet-stream'
+    end
+
+    def padded_number
+      digits = total.to_s.length
+
+      "%0#{digits}d" % number.to_s
+    end
+
+    def valid?
+      valid_blob? && valid_number? && valid_total? && valid_content_type?
+    end
+
+    private
+
+    def valid_blob?
+      blob.size > 0
+    end
+
+    def valid_number?
+      number.kind_of?(Integer) && number > 0
+    end
+
+    def valid_total?
+      total.kind_of?(Integer) && total > 0 && total >= number
+    end
+
+    def valid_content_type?
+      content_type =~ /\w+\/\w+/
+    end
+  end
+end

data/lib/fragmenter/redis.rb

@@ -0,0 +1,102 @@
+require 'fragmenter/fragment'
+
+module Fragmenter
+  class Redis
+    extend Forwardable
+
+    delegate expiration: Fragmenter
+    delegate logger:     Fragmenter
+    delegate redis:      Fragmenter
+
+    attr_reader :fragmenter
+
+    def initialize(fragmenter)
+      @fragmenter = fragmenter
+    end
+
+    def store_key
+      fragmenter.key
+    end
+
+    def meta_key
+      [store_key, 'options'].join('-')
+    end
+
+    def store(blob, options)
+      fragment = Fragmenter::Fragment.new(blob, options)
+
+      if fragment.valid?
+        persist_fragment(fragment)
+      else
+        false
+      end
+    end
+
+    def meta
+      redis.hgetall meta_key
+    end
+
+    def fragments
+      redis.hkeys(store_key).sort
+    end
+
+    def complete?
+      redis.hlen(store_key).to_s == redis.hget(meta_key, :total)
+    end
+
+    def rebuild
+      benchmark_rebuilding do
+        redis.hmget(store_key, *fragments).join('')
+      end
+    rescue ::Redis::CommandError
+      log 'Failure rebuilding, most likely there are no fragments to rebuild'
+
+      ''
+    end
+
+    def clean!
+      redis.del store_key, meta_key
+    end
+
+    private
+
+    def log(message)
+      logger.info "Fragmenter: #{message}"
+    end
+
+    def persist_fragment(fragment)
+      benchmark_persistence(fragment) do
+        redis.multi do
+          redis.hset store_key, fragment.padded_number, fragment.blob
+          redis.hset meta_key, :content_type, fragment.content_type
+          redis.hset meta_key, :total, fragment.total
+
+          redis.expire store_key, expiration
+          redis.expire meta_key, expiration
+        end
+      end
+    end
+
+    def benchmark_persistence(fragment, &block)
+      log %(Storing #{fragment.number}/#{fragment.total}...)
+      start_time = Time.now
+
+      yield
+
+      end_time = Time.now
+      log %(Stored (#{end_time - start_time}) #{fragment.number}/#{fragment.total} #{fragment.blob.size} bytes)
+    end
+
+    def benchmark_rebuilding(&block)
+      log %(Rebuilding #{fragments.length} fragments...)
+      start_time = Time.now
+
+      rebuilt = yield
+
+      end_time = Time.now
+      log %(Rebuilt (#{end_time - start_time}) #{fragments.length} fragments #{rebuilt.size} bytes)
+
+      rebuilt
+    end
+  end
+end

data/lib/fragmenter/version.rb

@@ -0,0 +1,3 @@
+module Fragmenter
+  VERSION = '0.5.0'
+end

data/spec/fragmenter/base_spec.rb

@@ -0,0 +1,43 @@
+require 'fragmenter/base'
+
+describe Fragmenter::Base do
+  let(:object)       { mock('object', id: 1001) }
+  let(:engine_class) { mock('engine_class', new: engine) }
+  let(:engine)       { mock('engine') }
+
+  subject { described_class.new(object, engine_class) }
+
+  describe '#key' do
+    it 'composes a key from the object class and id value' do
+      subject.key.should match(/[a-z]+-\d+/)
+    end
+  end
+
+  describe 'engine delegation' do
+    let(:blob)    { '0101' }
+    let(:headers) { {} }
+
+    it 'delegates #store to the storage engine' do
+      engine.should_receive(:store).with(blob, headers)
+
+      subject.store(blob, headers)
+    end
+
+    it 'delegates #fragments to the storage engine' do
+      engine.should_receive(:fragments)
+      subject.fragments
+    end
+  end
+
+  describe '#as_json' do
+    it 'merges the stored meta and fragments' do
+      engine.stub('meta' => { 'content_type' => 'application/octet-stream' },
+                  'fragments' => ['1', '2'])
+
+      subject.as_json.tap do |json|
+        json.should have_key('content_type')
+        json.should have_key('fragments')
+      end
+    end
+  end
+end

data/spec/fragmenter/fragment_spec.rb

@@ -0,0 +1,58 @@
+require 'fragmenter/fragment'
+
+describe Fragmenter::Fragment do
+  let(:blob) { '1010101' }
+
+  describe '#number' do
+    it 'defaults the number to 1' do
+      described_class.new(blob, {}).number.should == 1
+    end
+  end
+
+  describe '#total' do
+    it 'defaults the total to 1' do
+      described_class.new(blob, {}).total.should == 1
+    end
+  end
+
+  describe '#content_type' do
+    it 'defaults the content_type to a binary format' do
+      described_class.new(blob, {}).content_type.should == 'application/octet-stream'
+    end
+  end
+
+  describe '#padded_number' do
+    it 'zero pads the number with as many zeros as the total has places' do
+      described_class.new(blob, number: 1, total: 2000).padded_number.should == '0001'
+    end
+  end
+
+  describe '#valid?' do
+    it 'is valid with a complete blob and sensible options' do
+      described_class.new(blob, number: 1, total: 2).should be_valid
+    end
+
+    it 'is not valid with an empty blob' do
+      described_class.new('', number: 1, total: 2).should_not be_valid
+    end
+
+    it 'is not valid without an integer part number greater than 1' do
+      described_class.new(blob, number: -1, total: 2).should_not be_valid
+      described_class.new(blob, number: 'one', total: 2).should_not be_valid
+    end
+
+    it 'is not valid without an integer part total' do
+      described_class.new(blob, number: 1, total: -2).should_not be_valid
+      described_class.new(blob, number: 1, total: 'two').should_not be_valid
+    end
+
+    it 'is not valid when the number is greater the total' do
+      described_class.new(blob, number: 2, total: 1).should_not be_valid
+      described_class.new(blob, number: 2, total: 2).should be_valid
+    end
+
+    it 'is not valid without a content type resembling a mime type' do
+      described_class.new(blob, content_type: 'jpg').should_not be_valid
+    end
+  end
+end

data/spec/fragmenter/redis_spec.rb

@@ -0,0 +1,153 @@
+require 'fragmenter'
+require 'fragmenter/redis'
+
+describe Fragmenter::Redis do
+  let(:blob_1)     { '00010110' }
+  let(:blob_2)     { '11101110' }
+  let(:fragmenter) { mock(:fragmenter, key: 'abcdefg') }
+  let(:redis)      { Fragmenter.redis }
+
+  subject(:engine) { described_class.new(fragmenter) }
+
+  before do
+    Fragmenter.logger = Logger.new('/dev/null')
+  end
+
+  after do
+    redis.del engine.store_key, engine.meta_key
+  end
+
+  describe '#store_key' do
+    it 'delegates store key to the fragmenter key' do
+      engine.store_key.should == fragmenter.key
+    end
+  end
+
+  describe '#meta_key' do
+    it 'combines the base store key with options' do
+      engine.meta_key.should include(fragmenter.key)
+      engine.meta_key.should include('options')
+    end
+  end
+
+  describe '#store' do
+    it 'does not store empty blobs' do
+      engine.store('', number: 1, total: 2).should be_false
+    end
+
+    it 'writes the provided blob to the fragmenter key and the provided number' do
+      engine.store(blob_1, number: 1, total: 48)
+
+      redis.hget(fragmenter.key, '01').should == blob_1
+    end
+
+    it 'overwrites existing data at the key + number location' do
+      engine.store(blob_1, number: 1, total: 48)
+      engine.store(blob_2, number: 1, total: 48)
+
+      redis.hget(fragmenter.key, '01').should == blob_2
+    end
+
+    it 'sets the fragment to expire' do
+      engine.store(blob_1, number: 1, total: 48)
+
+      redis.ttl(engine.store_key).should == Fragmenter.expiration
+    end
+
+    it 'stores meta-data in a spearate key' do
+      subject.store(blob_1, number: 1, total: 48)
+
+      redis.exists(engine.meta_key).should be_true
+    end
+
+    it 'sets the meta to expire' do
+      engine.store(blob_1, number: 1, total: 48)
+
+      redis.ttl(engine.meta_key).should == Fragmenter.expiration
+    end
+
+    it 'defaults the stored content-type to application/octet-stream' do
+      subject.store(blob_1, number: 1, total: 48)
+
+      redis.hget(engine.meta_key, :content_type).should == 'application/octet-stream'
+    end
+  end
+
+  describe '#meta' do
+    it 'returns an empty hash when nothing has been stored' do
+      engine.meta.should == {}
+    end
+
+    it 'returns the accumulated metadata when data has been stored' do
+      engine.store(blob_1, content_type: 'image/jpeg', number: 1, total: 2)
+      engine.meta.should eq(
+        'content_type' => 'image/jpeg',
+        'total' => '2'
+      )
+    end
+  end
+
+  describe '#fragments' do
+    context 'without any fragments' do
+      it 'returns an empty array' do
+        engine.fragments.should == []
+      end
+    end
+
+    context 'when fragments have been stored' do
+      before do
+        engine.store(blob_2, number: 3, total: 30)
+        engine.store(blob_1, number: 1, total: 30)
+      end
+
+      it 'returns an array of the stored fragment indecies' do
+        engine.fragments.should == ['01', '03']
+      end
+    end
+  end
+
+  describe '#complete?' do
+    it 'is incomplete if the number of stored fragments does not match the total' do
+      engine.store(blob_1, number: 1, total: 2)
+
+      engine.should_not be_complete
+    end
+
+    it 'is complete if the stored fragments matches all values between 1 and the total' do
+      engine.store(blob_1, number: 1, total: 2)
+      engine.store(blob_2, number: 2, total: 2)
+
+      engine.should be_complete
+    end
+  end
+
+  describe '#rebuild' do
+    before do
+      engine.store(blob_1, number: 1, total: 2)
+      engine.store(blob_2, number: 2, total: 2)
+    end
+
+    it 'returns the aggregated values from all stored fragments' do
+      engine.rebuild.should == [blob_1, blob_2].join('')
+    end
+
+    it 'returns nothing when no fragments are present' do
+      redis.del engine.store_key
+
+      engine.rebuild.should == ''
+    end
+  end
+
+  describe '#clean!' do
+    before do
+      engine.store(blob_1, number: 1, total: 2)
+    end
+
+    it 'deletes the storage and meta data' do
+      engine.clean!
+
+      redis.exists(engine.meta_key).should be_false
+      redis.exists(engine.store_key).should be_false
+    end
+  end
+end

data/spec/fragmenter_spec.rb

@@ -0,0 +1,68 @@
+require 'fragmenter'
+
+describe Fragmenter do
+  after do
+    Fragmenter.redis      = nil
+    Fragmenter.logger     = nil
+    Fragmenter.expiration = nil
+  end
+
+  describe '.logger' do
+    it 'attempts to instantiate a standard logger to STDOUT' do
+      Fragmenter.logger.should be_instance_of(Logger)
+      Fragmenter.logger.level.should == Logger::INFO
+    end
+  end
+
+  describe '.logger=' do
+    it 'stores the logger instance on the module' do
+      logger = mock(:logger)
+
+      Fragmenter.logger = logger
+      Fragmenter.logger.should be(logger)
+    end
+  end
+
+  describe '.redis' do
+    it 'attempts to create a redis connection with default values' do
+      Fragmenter.redis.should be_instance_of(Redis)
+    end
+  end
+
+  describe '.redis=' do
+    it 'stores the redis instance on the module' do
+      redis = mock(:redis)
+
+      Fragmenter.redis = redis
+      Fragmenter.redis.should be(redis)
+    end
+  end
+
+  describe '.expiration' do
+    it 'defaults expiration to one day' do
+      Fragmenter.expiration.should == 86400
+    end
+  end
+
+  describe '.expiration=' do
+    it 'stoires the expiration value on the module' do
+      Fragmenter.expiration = 10000
+      Fragmenter.expiration.should eq(10000)
+    end
+  end
+
+  describe '.configure' do
+    let(:redis)  { mock(:redis) }
+    let(:logger) { mock(:logger) }
+
+    it 'allows customization via passing a block' do
+      Fragmenter.configure do |config|
+        config.redis  = redis
+        config.logger = logger
+      end
+
+      Fragmenter.redis.should == redis
+      Fragmenter.logger.should == logger
+    end
+  end
+end

metadata

@@ -0,0 +1,105 @@
+--- !ruby/object:Gem::Specification
+name: fragmenter
+version: !ruby/object:Gem::Version
+  version: 0.5.0
+  prerelease: 
+platform: ruby
+authors:
+- Parker Selbert
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-10-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.11.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.11.0
+description: Fragmentize and rebuild data
+email:
+- parker@sorentwo.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- HISTORY.md
+- LICENSE.txt
+- README.md
+- Rakefile
+- fragmenter.gemspec
+- lib/fragmenter.rb
+- lib/fragmenter/base.rb
+- lib/fragmenter/fragment.rb
+- lib/fragmenter/redis.rb
+- lib/fragmenter/version.rb
+- spec/fragmenter/base_spec.rb
+- spec/fragmenter/fragment_spec.rb
+- spec/fragmenter/redis_spec.rb
+- spec/fragmenter_spec.rb
+- spec/spec_helper.rb
+homepage: 
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -818571163033621587
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -818571163033621587
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Fragmentize and rebuild data
+test_files:
+- spec/fragmenter/base_spec.rb
+- spec/fragmenter/fragment_spec.rb
+- spec/fragmenter/redis_spec.rb
+- spec/fragmenter_spec.rb
+- spec/spec_helper.rb