jferris-hashpipe 0.0.5.0.1247583913

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2007 Justin S. Leitgeb
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,71 @@
+= HashPipe
+
+Transparently maintains an attribute of an ActiveRecord-backed model in a separate key-value data store such 
+as Amazon S3, memcached, Tokyo Cabinet, or the filesystem.
+
+== Description
+
+HashPipe makes it easy to use an external key-value based data storage system alongside a traditional ActiveRecord-
+backed model.  In this sense, it acts as a pipe from the ActiveRecord model to a hash - a hash pipe!
+
+It functions by hooking into the native callbacks provided by ActiveRecord models.  HashPipe keeps track of when the 
+hashed attribute has been written to, and flushes it out to the external data source on saves of the ActiveRecord
+model.  Hashed attributes are lazy-loaded, so they aren't pulled from the external data source until they're requested.
+Finally, when the ActiveRecord model is deleted, the hashed attributes for that model are also deleted automatically.
+
+HashPipe supports transparent gzip compression/decompression of attribute data and serialization/deserialization 
+using Marshal in case you wish to store Ruby objects in the attribute.
+
+The backend of HashPipe is implemented using Moneta, which is a universal library for key-value storage systems.  
+This means that it supports all of the backend storage types supported by this common library, and that new 
+backends are added regularly.
+
+== Use Cases
+
+There are certain cases where it makes more sense to store data objects in a separate storage system from the RDBMS:
+
+* Certain parts of your data set require frequent reads and writes, and need to be fast without the overhead introduced by the ACID properties of traditional relational database systems
+* You store large objects, and don't want to worry about increasing the size of your RDBMS when the size grows out of the space you've allocated
+
+= Quick Start
+
+Getting started with HashPipe is easy.
+
+* Install the gem 'hashpipe' available from github as jsl-hashpipe.  If you're using a recent version of Rails you may 
+  want to add a line like the following to your environment.rb:
+
+    config.gem "jsl-hashpipe", :lib => 'hashpipe', :source => 'http://gems.github.com'
+
+* If you wanted to serialize an attribute 'elephant' in model Foo, you would call hattr :elephant in your model.  
+  For example, class 'Foo' may look like:
+
+    class Foo < ActiveRecord::Base
+      hattr :elephant
+    end
+
+* Configure hashpipe by creating a file hashpipe.yml in your projects' config directory according to the 
+  instructions in the 'Configuration' section of this document
+
+* After following the above steps, saving to the model and retrieving from it saves to the backend store 
+  defined in your options.  Likewise, deleting the model triggers a callback to delete the element in the 
+  associated store.
+
+== Configuration
+
+HashPipe supports the backends supported by Moneta.  Configure the backend that you want to use in your YAML 
+configuration file 'hashpipe.yml':
+
+  test:
+    moneta_klass: Moneta::Memory
+
+  development:
+    moneta_klass: Moneta::Memcache
+    moneta_options: 
+      server: localhost:1978
+
+You can define a default backend globally, and override it in individual models as needed (e.g., store one 
+attribute in the filesystem and another in S3).
+
+== Author
+
+Justin S. Leitgeb, <justin@phq.org>

data/Rakefile

@@ -0,0 +1,47 @@
+require 'rubygems'
+require 'spec'
+
+require 'rake'
+require 'spec/rake/spectask'
+require 'rake/rdoctask'
+
+desc 'Test the plugin.'
+Spec::Rake::SpecTask.new(:spec) do |t|
+  t.spec_opts  = ["--format", "specdoc", "--colour"]
+  t.libs << 'lib'
+  t.verbose = true
+end
+
+desc "Run all the tests"
+task :default => :spec
+
+desc 'Generate documentation for the hashpipe plugin.'
+Rake::RDocTask.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'HashPipe'
+  rdoc.options << '--line-numbers' << '--inline-source'
+  rdoc.rdoc_files.include('README')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+gem_spec = Gem::Specification.new do |s|
+  s.name     = "hashpipe"
+  s.version  = "0.0.5.0.#{Time.now.to_i}"
+  s.summary  = "ActiveRecord plugin to save content to a pluggable, hash-style backend"
+  s.email    = "justin@phq.org"
+  s.homepage = "http://github.com/jsl/hashpipe"
+  s.description = "HashPipe connects an AR-backed model to a key-value store"
+  s.has_rdoc = true
+  s.authors  = ["Justin Leitgeb"]
+  s.files    = FileList['[A-Z]*', '{lib,spec}/**/*.rb', 'init.rb', 'config/hashpipe.yml']
+  s.test_files = FileList['spec/**/*.rb']
+  s.add_dependency("wycats-moneta", ["> 0.0.0"])
+  s.add_dependency("activesupport", ["> 0.0.0"])
+end
+
+desc "Generate a gemspec file"
+task :gemspec do
+  File.open("#{gem_spec.name}.gemspec", 'w') do |f|
+    f.write gem_spec.to_yaml
+  end
+end

data/config/hashpipe.yml

@@ -0,0 +1,23 @@
+test:
+  storage: filesystem
+  s3:
+    bucket: test_archived_attributes
+    access_key: your access key
+    secret_key: your secret key
+    protocol: https
+
+development:
+  storage: s3
+  s3:
+    bucket: development_archived_attributes
+    access_key: your access key
+    secret_key: your secret key
+
+anotherenv:
+  storage: filesystem
+  filesystem:
+    archive_root: /tmp/archived_attributes
+        
+qa:
+  storage: memcache
+  namespace: hashit

data/init.rb

@@ -0,0 +1 @@
+require 'hashpipe'

data/lib/hashpipe/archived_attribute.rb

@@ -0,0 +1,60 @@
+require 'activesupport'
+require 'moneta'
+
+module HashPipe
+
+  class ArchivedAttribute
+    attr_reader :name, :backend
+    attr_accessor :scope
+
+    def initialize(name, scope, backend, opts = {})
+      @name    = name
+      @scope   = scope
+      @dirty   = false
+      @options = HashPipe::GlobalConfiguration.instance.to_hash.merge(opts)
+      @backend = backend
+    end
+
+    def value
+      val = defined?(@stashed_value) ? @stashed_value : backend[key]
+      val = compress? && !val.nil? ? Zlib::Inflate.inflate(val) : val
+      val = marshal? ? Marshal.load(val) : val
+    end
+
+    def value=(other)
+      other = marshal? ? Marshal.dump(other) : other
+      other = compress? && !other.nil? ? Zlib::Deflate.deflate(other) : other
+      @stashed_value = other
+      @dirty = true
+    end
+
+    def dirty?
+      @dirty
+    end
+
+    def save
+      backend[key] = @stashed_value if self.dirty?
+      @dirty = false
+    end
+
+    def destroy
+      backend.delete(key)
+    end
+
+    def options
+      @options
+    end
+
+    def key
+      [scope, name].join('_')
+    end
+
+    [:marshal, :compress].each do |sym|
+      define_method("#{sym}?") do                   # def marshal?
+        options[sym].nil? ? false : options[sym]    #   options[:marshal].nil? ? false : options[:marshal]
+      end                                           # end
+    end
+
+  end
+
+end

data/lib/hashpipe/global_configuration.rb

@@ -0,0 +1,43 @@
+module HashPipe
+
+  # Singleton class for reading the defaults archived attribute configuration
+  # for this environment.
+  class GlobalConfiguration
+    include Singleton
+
+    DEFAULTS = HashWithIndifferentAccess.new({
+      :moneta_klass => 'Moneta::Memory',
+      :marshal      => false,
+      :compress     => false
+    })
+    
+    def [](val)
+      config[val]
+    end
+
+    def to_hash
+      config
+    end
+
+    def to_s
+      config.inspect
+    end
+
+    private
+
+    def config
+      @config ||= HashWithIndifferentAccess.new(
+        DEFAULTS.merge(load_yaml_configuration)
+      )
+
+      @config.dup
+    end
+
+    def load_yaml_configuration
+      YAML.load_file(
+        File.join( RAILS_ROOT, 'config', 'hashpipe.yml' )
+      )[RAILS_ENV]
+    end
+  end
+
+end

data/lib/hashpipe.rb

@@ -0,0 +1,96 @@
+require File.expand_path(File.join(File.dirname(__FILE__), %w[hashpipe archived_attribute] ))
+require File.expand_path(File.join(File.dirname(__FILE__), %w[hashpipe global_configuration] ))
+
+module HashPipe
+
+  def self.included(base)
+    base.extend(SingletonMethods)
+  end
+
+  module SingletonMethods
+
+    def hattr(*args)
+      attribute = args.first
+
+      options = args.extract_options!
+      options.reverse_merge! :marshalled => false
+
+      if archived_attribute_definitions.nil?
+        write_inheritable_attribute(:archived_attribute_definitions, {})
+
+        after_save :save_archived_attributes
+        before_destroy :destroy_archived_attributes
+      end
+
+      archived_attribute_definitions[attribute] = options
+
+      include InstanceMethods
+
+      define_method attribute do
+        archive_stash_for(attribute).value
+      end
+
+      define_method "#{attribute}=" do |value|
+        archive_stash_for(attribute).value = value
+      end
+    end
+
+    # Returns the attachment definitions defined by each call to
+    # has_attached_file.
+    def archived_attribute_definitions
+      read_inheritable_attribute(:archived_attribute_definitions)
+    end
+
+    def backend
+      @backend ||= initialize_cache_klass(HashPipe::GlobalConfiguration.instance[:moneta_klass])
+    end
+
+    def initialize_cache_klass(cache_klass)
+      require_moneta_library_for(cache_klass)
+      klass_const = cache_klass.respond_to?(:constantize) ? cache_klass.constantize : cache_klass
+      klass_const.new HashPipe::GlobalConfiguration.instance[:moneta_options]
+    end
+
+    def require_moneta_library_for(cache_klass)
+      require cache_klass.to_s.gsub(/::/, '/').downcase
+    end
+  end
+
+  module InstanceMethods
+    def archive_stash_for(attribute)
+      @_archived_attribute_stashes ||= {}
+      @_archived_attribute_stashes[attribute] ||= ArchivedAttribute.new(
+        attribute,
+        archived_attribute_scope,
+        self.class.backend,
+        self.class.archived_attribute_definitions[attribute]
+      )
+    end
+
+    def archived_attribute_scope
+      "#{self.class.table_name}_#{id}"
+    end
+
+    def each_archived_stash
+      self.class.archived_attribute_definitions.each do |name, definition|
+        yield(name, archive_stash_for(name))
+      end
+    end
+
+    def save_archived_attributes
+      each_archived_stash do |name, stash|
+        stash.scope = archived_attribute_scope
+        stash.save
+      end
+    end
+
+    def destroy_archived_attributes
+      each_archived_stash do |name, stash|
+        stash.destroy
+      end
+    end
+
+  end
+end
+
+ActiveRecord::Base.send(:include, HashPipe)

data/spec/hashpipe/archived_attribute_spec.rb

@@ -0,0 +1,170 @@
+require File.join(File.dirname(__FILE__), %w[ .. spec_helper ])
+
+module ArchivedAttributeHelpers
+  def build_attribute(opts = {})
+    opts[:name]    ||= :glorp
+    opts[:backend] ||= Moneta::Memory.new
+    opts[:scope]   ||= 'unique-id'
+    HashPipe::ArchivedAttribute.new(opts.delete(:name),
+                                    opts.delete(:scope),
+                                    opts.delete(:backend),
+                                    opts)
+  end
+
+  def clone_attribute(attribute)
+    build_attribute(:marshal  => attribute.marshal?,
+                    :compress => attribute.compress?,
+                    :backend  => attribute.backend,
+                    :scope    => attribute.scope,
+                    :name     => attribute.name)
+  end
+
+  def backend
+    subject.backend
+  end
+end
+
+describe HashPipe::ArchivedAttribute do
+
+  include ArchivedAttributeHelpers
+
+  before { @attribute = build_attribute }
+
+  subject { @attribute }
+
+  it "should not be dirty when a value has not been set" do
+    should_not be_dirty
+  end
+
+  it "should join the scope and the attribute name as a hash key" do
+    subject.key.should == "#{subject.scope}_#{subject.name}"
+  end
+
+  it "should allow the scope to be set" do
+    new_scope = 'another scope'
+    subject.scope = new_scope
+    subject.scope.should == new_scope
+  end
+
+  it "should use the provided backend" do
+    backend = Moneta::Memory.new
+    build_attribute(:backend => backend).backend.should == backend
+  end
+
+  it "should return when the value has been set" do
+    subject.value = 'stuff'
+    should be_dirty
+  end
+
+  it "should not be dirty after save is called" do
+    subject.value = 'stuff'
+    subject.save
+    should_not be_dirty
+  end
+
+  it "should add itself to the backend when saved" do
+    value = 'a value'
+
+    backend.should_not have_key(subject.key)
+
+    subject.value = value
+    subject.save
+
+    backend[subject.key].should == value
+  end
+
+  it "should retrieve itself from the backend" do
+    value = 'a value'
+    backend[subject.key] = value
+
+    subject.value.should == value
+  end
+
+  it "should remove itself from the backend when destroyed" do
+    backend[subject.key] = 'value'
+    subject.destroy
+    backend.should_not have_key(subject.key)
+  end
+
+  it "should correctly store a marshalled, compressed value" do
+    value = 'the value'
+    attribute = build_attribute(:marshal => true, :compress => true)
+    attribute.value = value
+    attribute.save
+
+    clone_attribute(attribute).value.should == value
+  end
+end
+
+
+describe HashPipe::ArchivedAttribute, "when marshal is on" do
+
+  include ArchivedAttributeHelpers
+
+  before { @attribute = build_attribute(:marshal => true) }
+  subject { @attribute }
+
+  it "should be marshalled" do
+    subject.marshal?.should be
+  end
+
+  it "should store a marshalled value" do
+    stored = 'stored value'
+    actual = 'a value'
+
+    Marshal.expects(:dump).with(actual).returns(stored)
+
+    subject.value = actual
+    subject.save
+
+    backend[subject.key].should == stored
+  end
+
+  it "should retrieve a marshalled value" do
+    stored = 'stored value'
+    actual = 'a value'
+    backend[subject.key] = stored
+
+    Marshal.expects(:load).with(stored).returns(actual)
+
+    subject.value.should == actual
+  end
+
+  it "should correctly store a nil value" do
+    subject.value = nil
+    subject.save
+    clone_attribute(subject).value.should be_nil
+  end
+end
+
+describe HashPipe::ArchivedAttribute, "when compression is on" do
+
+  include ArchivedAttributeHelpers
+
+  before { @attribute = build_attribute(:compress => true) }
+  subject { @attribute }
+
+  it "should be compressed" do
+    subject.compress?.should be
+  end
+
+  it "should store a compressed value" do
+    value = 'a value'
+    subject.value = value
+    subject.save
+    backend[subject.key].should == Zlib::Deflate.deflate(value)
+  end
+
+  it "should retrieve a compressed value" do
+    value = 'a value'
+    backend[subject.key] = Zlib::Deflate.deflate(value)
+    subject.value.should == value
+  end
+
+  it "should correctly store a nil value" do
+    subject.value = nil
+    subject.save
+    clone_attribute(subject).value.should be_nil
+  end
+end
+

data/spec/hashpipe/global_configuration_spec.rb

@@ -0,0 +1,72 @@
+require File.join(File.dirname(__FILE__), %w[ .. spec_helper ])
+
+describe HashPipe::GlobalConfiguration do
+  before do
+    @conf = HashPipe::GlobalConfiguration.instance
+  end
+
+  describe "defaults" do
+    it "should set default marshal value to false" do
+      HashPipe::GlobalConfiguration::DEFAULTS[:marshal].should == false
+    end
+    
+    it "should set gzip value to false" do
+      HashPipe::GlobalConfiguration::DEFAULTS[:marshal].should == false
+    end
+  end
+
+  it "should read default access key from the configuration file" do
+    @conf[:s3]['access_key'].should == 'your access key'
+  end
+
+  it "should the default secret key from configuration" do
+    @conf[:s3]['secret_key'].should == 'your secret key'
+  end
+
+  it "should read the default bucket from the configuration file" do
+    @conf[:s3]['bucket'].should == 'test_archived_attributes'
+  end
+
+  describe "#to_s" do
+    it "should return a string" do
+      @conf.to_s.should be_an_instance_of(String)
+    end
+  end
+
+  describe "hash cloning" do
+    it "should be able to alter a Hash without affecting the original object" do
+      conf = @conf.to_hash
+      previous = @conf[:storage]
+      conf[:storage] = 'foo'
+      @conf[:storage].should == previous
+    end
+
+    it "should not affect deeply nested attributes when values are changed" do
+      conf = @conf.to_hash
+      previous = @conf[:s3][:protocol]
+      conf[:s3][:protocol] = 'puddle'
+      @conf[:s3][:protocol].should == previous
+    end
+  end
+
+  describe "#to_hash" do
+    it "should return an instance of Hash" do
+      @conf.to_hash.should be_an_instance_of(HashWithIndifferentAccess)
+    end
+  end
+
+  describe "when an option is not specified in the yaml config file" do
+    it "should have a section for s3 options in hash" do
+      @conf[:s3].should_not be_nil
+    end
+
+    it "should default to https for protocol" do
+      @conf[:s3][:protocol].should == 'https'
+    end
+
+    it "should set default storage attribute to :filesystem" do
+      @conf[:storage].should == 'filesystem'
+    end
+  end
+
+end

data/spec/hashpipe_spec.rb

@@ -0,0 +1,57 @@
+require File.join(File.dirname(__FILE__),  'spec_helper')
+
+ActiveRecord::Base.establish_connection(
+  :adapter => 'sqlite3',
+  :dbfile => ":memory:"
+)
+
+ActiveRecord::Migration.verbose = false
+
+ActiveRecord::Base.silence do
+  ActiveRecord::Schema.define do
+    create_table :stories do |table|
+      table.string :title
+      table.string :content_key
+      table.string :description_key
+    end
+  end
+end
+
+class Story < ActiveRecord::Base
+  hattr :content
+  hattr :description, { :marshalled => true }
+end
+
+describe HashPipe do
+  before do
+    @lamb_text    =  'Baaah!'
+    @bear_struct  =  OpenStruct.new(:blah => 'arg')
+    @lamb_story   =  Story.create!(:title => 'lamb story', :content => @lamb_text)
+    @bear_story   =  Story.create!(:title => 'bear story', :content => 'Raaaar!', :description => @bear_struct)
+  end
+
+  def refind(instance)
+    instance.class.find(instance.id)
+  end
+
+  it "should retrieve textual content" do
+    refind(@lamb_story).content.should == @lamb_text
+  end
+
+  it "should load marshalled objects" do
+    refind(@bear_story).description.should == @bear_struct
+  end
+
+  [:save_archived_attributes, :destroy_archived_attributes].each do |sym|
+    it "should respond to attached storage method #{sym}" do
+      Story.new.should respond_to(sym)
+    end
+  end
+
+  describe "#destroy_archived_attributes" do
+    it "should be called on destruction of object" do
+      @bear_story.expects(:destroy_archived_attributes).at_least_once
+      @bear_story.destroy
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,19 @@
+require 'rubygems'
+require 'moneta'
+require 'moneta/memory'
+require 'ostruct'
+require 'mocha'
+require 'spec'
+require 'activerecord'
+require 'zlib'
+
+Spec::Runner.configure do |config|
+  config.mock_with(:mocha)
+end
+
+$: << File.join(File.dirname(__FILE__), '..', 'lib')
+require File.join(File.dirname(__FILE__), %w[.. init])
+
+RAILS_DEFAULT_LOGGER = Logger.new(STDOUT) unless defined?(RAILS_DEFAULT_LOGGER)
+RAILS_ROOT = File.join(File.dirname(__FILE__), '..')  unless defined?(RAILS_ROOT)
+RAILS_ENV = 'test' unless defined?(RAILS_ENV)

metadata

@@ -0,0 +1,86 @@
+--- !ruby/object:Gem::Specification 
+name: jferris-hashpipe
+version: !ruby/object:Gem::Version 
+  version: 0.0.5.0.1247583913
+platform: ruby
+authors: 
+- Justin Leitgeb
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-07-13 21:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: wycats-moneta
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">"
+      - !ruby/object:Gem::Version 
+        version: 0.0.0
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: activesupport
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">"
+      - !ruby/object:Gem::Version 
+        version: 0.0.0
+    version: 
+description: HashPipe connects an AR-backed model to a key-value store
+email: justin@phq.org
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- LICENSE
+- Rakefile
+- README.rdoc
+- lib/hashpipe/archived_attribute.rb
+- lib/hashpipe/global_configuration.rb
+- lib/hashpipe.rb
+- spec/hashpipe/archived_attribute_spec.rb
+- spec/hashpipe/global_configuration_spec.rb
+- spec/hashpipe_spec.rb
+- spec/spec_helper.rb
+- init.rb
+- config/hashpipe.yml
+has_rdoc: true
+homepage: http://github.com/jsl/hashpipe
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 3
+summary: ActiveRecord plugin to save content to a pluggable, hash-style backend
+test_files: 
+- spec/hashpipe/archived_attribute_spec.rb
+- spec/hashpipe/global_configuration_spec.rb
+- spec/hashpipe_spec.rb
+- spec/spec_helper.rb