harmonize 0.0.1

data/Gemfile

@@ -0,0 +1,13 @@
+source "http://rubygems.org"
+
+gem 'rails', '3.0.9'
+gem 'hashie'
+
+group :development, :test, :console do
+  gem 'sqlite3'
+  gem 'rspec-rails', '~> 2'
+  gem 'ruby-debug19', :require => 'ruby-debug'
+  gem 'capybara', '~> 1'
+  gem 'database_cleaner'
+  gem 'factory_girl_rails'
+end

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2011 YOURNAME
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,153 @@
+# harmonize
+
+hahr-muh-nahyz -
+
+1.  to bring into harmony,  accord, or agreement: to harmonize one's views with the new situation.
+2.  Music . to accompany with appropriate harmony.
+3.  to be in agreement in action, sense, or feeling: Though of different political parties, all the delegates harmonized on civil rights.
+4.  to sing in harmony.
+
+harmonize is a rails 3 engine that allows one to harmonize entire scopes of an ActiveRecord model with arbitrary external data sources.
+
+harmonize works by allowing one to setup feeds from external data called "sources". harmonize applies the "source" feeds to sets of ActiveRecord instances called "targets". These sets of "source" and "target" data are then handed of to a "strategy" to determine how to use the source to modify the target. When applying a "strategy" harmonize creates a Harmonize::Log which has_many Harmonize::Modifications. These records are then used to report information about the harmonize process.
+
+## Simple Example
+
+Lets pretend we work for a company that has a large list of stores. This list of stores in maintained in a database we don't have access to directly. The database administrator wrote a script to export the data in CSV format and upload it to our application. In our application we have a Store model but we always want it to be in harmony with the latest CSV file uploaded by the administrator. Every store has a unique name and an address, and the upstream database uses the name as the primary_key.
+
+    class Store < ActiveRecord::Base
+      validates :name, :address, :presence => true
+
+      # Setup a harmonizer
+      harmonize do |config|
+        config.key = :name
+      end
+
+      # By default harmonize expects a class method to be implemented
+      # that knows how to provide the source data feed
+      def self.harmonize_source_default
+        # parse the csv file and return
+        # an collection of hash like objects
+      end
+    end
+
+With our Store model wired up as above, we will get a new class method on our model called "harmonize_default!". When we call harmonize_default! harmonize will use the default strategy to harmonize the source records with the target records. In order to understand what actions are taken to bring the targets into harmony, we need to understand Harmonize::Strategies, but that is getting us ahead of ourselves. First lets look at how we configure harmonize.
+
+## Harmonize::Configuration
+
+Each call to harmonize creates a Harmonize::Configuration instance that defines how to configure a harmonizer. Here is an example using all the currently available configuration options using non-default settings:
+
+    class Store < ActiveRecord::Base
+      harmonize do |config|
+        config.harmonizer_name    = :custom_name
+        config.key                = :the_key
+        config.source             = lambda do
+          get_latest_source_data_feed!
+        end
+        config.target             = lambda do
+          where(:active => true)
+        end
+        config.strategy           = YourCustomStrategy
+        config.strategy_arguments = { :options => 'needed', :to => 'initialize', :your => 'custom strategy' }
+      end
+    end
+
+### Harmonizer::Configuration.harmonizer_name
+
+harmonize uses the configured harmonize_name as the name of the harmonizer being configured. Each harmonize_name may only be used once. This allows the harmonize method to be called more than once per model.
+
+The default harmonizer_name is :default
+This option is used to name special methods used by harmonize. The harmonization method is named with the following convention: "harmonize\_#{harmonizer\_name}!".
+
+### Harmonizer::Configuration.key
+
+harmonize uses the configured key to determine what attribute to use to find existing target records.
+
+The default key is :id
+
+### Harmonizer::Configuration.source
+
+harmonize uses the configured source to gather the latest set of source records. This can be set to a lambda or any other callable object. The only requirement is that it returns a collection of hash like objects.
+
+The default source is a lambda
+This lambda calls the default source method who's name is determined by the harmonizer_name. The convention is: "harmonizer\_source\_#{harmonizer_name}".
+
+### Harmonizer::Configuration.target
+
+harmonize uses the configured target to gather the latest set of target records. This can be set to a lambda or any other callable object. The only requirement is that it returns an ActiveRecord::Relation
+
+The default target is a lambda. This lambda returns an ActiveRecord::Relation via the ActiveRecord::Base#scoped method. Hint...all (named) scopes return ActiveRecord::Relation instances.
+
+### Harmonizer::Configuration.strategy
+
+harmonize uses the configured strategy to determine which Harmonize::Strategies::Strategy subclass to use when harmonizing. harmonize uses this setting as well as the strategy_arguments setting to create an instance of the Strategy subclass.
+
+The default strategy is Harmonize::Strategies::BasicCrudStrategy
+
+### Harmonizer::Configuration.target
+
+harmonize uses the configured strategy_arguments to determine which arguments to use when initializing the set Harmonize::Strategies::Strategy subclass.
+
+The default strategy is {}.
+
+## Harmonize::Strategies
+
+harmonize figures out how to use your source data to modify your target by using a Harmonize::Strategies::Strategy subclass. A strategy is responsible for determining when and how to create, update, and destroy your target records based on the source records.
+
+The default harmonize strategy is Harmonize::Strategies::BasicCrudStrategy. This strategy assumes your source data to be a "source of truth" which reflects exactly how the entire target record set should look. Here are the steps it takes to achieve harmony:
+
+1.  For each record in the source, use the configured "key" to find out if a record with that key already exists.
+2.  If an existing record was found in target, update it with any attributes provided by source
+3.  If no existing record was found in target, create a new record with any attributes provided by source
+4.  Save the record and create a Harmonize::Modification record making if we created or updated
+5.  If save has errors, update the Harmonize::Modification record to reflect that and store the error messages
+6.  Save the key of the modified (or errored) record in a collection
+7.  After all source records have been harmonized, destory any target record with a key not in the modified keys collection
+8.  Save Harmonize::Log for reporting the actions took in this harmonization
+
+Currently this is the only strategy provided by harmonize, but more will be added when I need them or you send them to me as a pull request.
+
+## Installation
+
+Add harmonize to the gem file for your rails application:
+
+    gem 'harmonize'
+
+Run the migration generator in your rails application root:
+
+    rails generate harmonize:migration
+
+Configure your model to use harmonize and implement your source:
+
+    class MyModel < ActiveRecord::Base
+      harmonize
+
+      def self.harmonize_source_default
+        # a collection of hash like objects
+      end
+    end
+
+Use, report bugs, fix them, and send pull requests!
+
+## Plans
+
+## TODO
+
+*  Maybe move key from Configuration to a strategy_argument as it is not a configuration option really, but a way to change stratgey behaviour.
+
+## Contributors
+
+*  Bram Swenson <bram@craniumisjar.com>
+
+## Notes
+
+Check out the rspec suite for more details about how harmonize works: [BasicCrudStrategy Spec](http://github.com/bramswenson/harmonize/blob/master/spec/lib/harmonize/strategies/basic_crud_strategy_spec.rb)
+
+Please let me know how you use harmonize!
+
+Thanks in advance!
+
+## License
+
+This project rocks and uses MIT-LICENSE.
+

data/Rakefile

@@ -0,0 +1,31 @@
+# encoding: UTF-8
+require 'rubygems'
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+require 'rake'
+require 'rspec/core'
+require 'rspec/core/rake_task'
+require 'bundler/gem_tasks'
+
+namespace :test do
+  RSpec::Core::RakeTask.new(:spec)
+  desc 'Setup the test database'
+  task :dbsetup do
+    results = %x(
+      cd spec/dummy &&
+      rm db/*.sqlite3 &&
+      RAILS_ENV=test rake db:migrate
+    )
+    puts "dbsetup: #{results}" unless results == ''
+  end
+
+  desc 'Setup the test database and run rspec'
+  task :run => %w( test:dbsetup test:spec )
+end
+
+task :default => 'test:run'
+

data/app/models/harmonize/log.rb

@@ -0,0 +1,16 @@
+module Harmonize
+  class Log < ActiveRecord::Base
+    set_table_name :harmonize_logs
+
+    validates :key, :class_name, :harmonizer_name, :strategy, :strategy_arguments, :presence => true
+
+    ModificationOptions = { :class_name => "Harmonize::Modification", :foreign_key => :harmonize_log_id }
+
+    has_many :modifications, ModificationOptions.merge(:inverse_of => :log, :dependent => :destroy)
+    has_many :created,       ModificationOptions.merge(:conditions => { :modification_type => 'create' })
+    has_many :updated,       ModificationOptions.merge(:conditions => { :modification_type => 'update' })
+    has_many :destroyed,     ModificationOptions.merge(:conditions => { :modification_type => 'destroy'})
+    has_many :errored,       ModificationOptions.merge(:conditions => { :modification_type => 'error'  })
+
+  end
+end

data/app/models/harmonize/modification.rb

@@ -0,0 +1,18 @@
+module Harmonize
+  class Modification < ActiveRecord::Base
+    set_table_name :harmonize_modifications
+
+    validates :harmonize_log_id, :modification_type, :presence => true
+    validates :modification_type, :inclusion => %w( create update destroy error )
+
+    serialize :instance_errors
+
+    belongs_to :log, :class_name => "Harmonize::Log",
+               :foreign_key => :harmonize_log_id, :inverse_of => :modifications
+
+    scope :created,   where(:modification_type => 'create')
+    scope :updated,   where(:modification_type => 'update')
+    scope :destroyed, where(:modification_type => 'destroy')
+    scope :errored,   where(:modification_type => 'error')
+  end
+end

data/lib/harmonize/base.rb

@@ -0,0 +1,95 @@
+module Harmonize
+
+  module Base
+
+    def self.included(base)
+      base.send :extend,  ClassMethods
+      base.send :include, InstanceMethods
+    end
+
+    module ClassMethods
+
+      # Enable harmonization on a set of records
+      def harmonize(options = {})
+        configuration = Harmonize::Configuration.new(options)
+        yield(configuration) if block_given?
+        configure_harmonizer(configuration)
+      end
+
+      def harmonizers
+        @harmonizers ||= {}
+      end
+
+      def harmonize!(harmonizer_name)
+        raise UnknownHarmonizerName.new(harmonizer_name) unless harmonizers.has_key?(harmonizer_name)
+        harmonizer = harmonizers[harmonizer_name]
+        harmonize_log = create_harmonize_log(harmonizer_name)
+        strategy = harmonizer.strategy.new(*harmonizer.strategy_arguments)
+        strategy.harmonize(:harmonizer => harmonizer, :harmonize_log => harmonize_log)
+        close_harmonize_log(harmonize_log)
+      end
+
+      def remove_harmonizer!(harmonizer_name = :default)
+        harmonizers.delete(harmonizer_name)
+      end
+
+      private
+
+        def close_harmonize_log(harmonize_log)
+          harmonize_log.end = DateTime.now
+          harmonize_log.save!
+          harmonize_log
+        end
+
+        def create_harmonize_log(harmonizer_name)
+          harmonizer = harmonizers[harmonizer_name]
+          Harmonize::Log.create!(
+            :start => DateTime.now,
+            :class_name => self.class.name,
+            :harmonizer_name => harmonizer_name,
+            :key => harmonizer.key,
+            :strategy => harmonizer.strategy.inspect,
+            :strategy_arguments => harmonizer.strategy_arguments.inspect,
+          )
+        end
+
+        def default_harmonizer_options(harmonizer_name)
+          Harmonize::Configuration.new({
+            :source => lambda{ harmonizer_source_method(harmonizer_name) },
+            :target => lambda{ scoped }
+          })
+        end
+
+        def harmonizer_source_method(harmonizer_name)
+          method_name = "harmonizer_source_#{harmonizer_name}".to_sym
+          raise HarmonizerSourceUndefined.new(harmonizer_name) unless respond_to?(method_name)
+          send(method_name)
+        end
+
+        def validate_harmonizer_configuration(configuration)
+          configuration.reverse_merge(default_harmonizer_options(configuration.harmonizer_name))
+        end
+
+        def setup_harmonizer_method(harmonizer_name)
+          self.class.instance_eval do
+            define_method "harmonize_#{harmonizer_name}!" do
+              send(:harmonize!, harmonizer_name)
+            end
+          end
+        end
+
+        def configure_harmonizer(configuration)
+          configuration = validate_harmonizer_configuration(configuration)
+          raise DuplicateHarmonizerName.new(configuration.harmonizer_name.to_s) if harmonizers.has_key?(configuration.harmonizer_name)
+          harmonizers[configuration.harmonizer_name] = configuration
+          setup_harmonizer_method(configuration.harmonizer_name)
+        end
+
+    end
+
+    module InstanceMethods
+    end
+
+  end
+end
+

data/lib/harmonize/configuration.rb

@@ -0,0 +1,10 @@
+module Harmonize
+  class Configuration < Hashie::Dash
+    property :source
+    property :target
+    property :key,                :default => :id
+    property :harmonizer_name,    :default => :default
+    property :strategy,           :default => Harmonize::Strategies::BasicCrudStrategy
+    property :strategy_arguments, :default => Hash.new
+  end
+end

data/lib/harmonize/engine.rb

@@ -0,0 +1,4 @@
+module Harmonize
+  class Engine < ::Rails::Engine
+  end
+end

data/lib/harmonize/errors.rb

@@ -0,0 +1,7 @@
+module Harmonize
+  class HarmonizeError < StandardError ; end
+  class DuplicateHarmonizerName < HarmonizeError ; end
+  class UnknownHarmonizerName < HarmonizeError ; end
+  class HarmonizerSourceUndefined < HarmonizeError ; end
+  class HarmonizerTargetUndefined < HarmonizeError ; end
+end

data/lib/harmonize/gemdata.rb

@@ -0,0 +1,15 @@
+module Harmonize
+  module Gemdata
+    Name    = 'harmonize'
+    Version = '0.0.1'
+    Authors = [ 'Bram Swenson' ]
+    Email   = [ 'bram@craniumisajar.com' ]
+    Summary = <<-END_SUMMARY
+Bring entire sets of models into harmony with external sources.
+    END_SUMMARY
+    Description = <<-END_DESCRIPTION
+Bring entire sets of models into harmony with external sources.
+    END_DESCRIPTION
+    Homepage = 'http://github.com/bramswenson/harmonize'
+  end
+end

data/lib/harmonize/strategies/basic_crud_strategy.rb

@@ -0,0 +1,69 @@
+module Harmonize
+  module Strategies
+
+    class BasicCrudStrategy < Strategy
+
+      def find_target_instance(key, value)
+        target_relation = targets.where(key => value)
+        instance = target_relation.first rescue nil
+        instance ||= target_relation.build
+        instance
+      end
+
+      def create_modification(instance)
+        unless instance.new_record?
+          harmonize_log.modifications.create!(:modification_type => 'update', :before_time => DateTime.now)
+        else
+          harmonize_log.modifications.create!(:modification_type => 'create', :before_time => DateTime.now)
+        end
+      end
+
+      def harmonize_target_instance!(target_instance, source, modification)
+        # TODO: this is a mess
+        error_message = nil
+        errored  = false
+        begin
+          unless target_instance.update_attributes(source)
+            errored = true
+            error_message = target_instance.errors.full_messages
+          end
+        rescue ActiveRecord::ActiveRecordError, ActiveRecord::UnknownAttributeError => e
+          errored = true
+          error_message = e.message
+        end
+        unless errored
+          modification.update_attributes!(:after_time => DateTime.now, :instance_id => target_instance.id)
+        else
+          modification.update_attributes!(:modification_type => 'error', :instance_errors => error_message)
+        end
+      end
+
+      def create_and_update_targets
+        sources.each.inject([]) do |keys, source|
+          target_instance = find_target_instance(harmonizer.key, source[harmonizer.key])
+          modification = create_modification(target_instance)
+          harmonize_target_instance!(target_instance, source, modification)
+          keys << source[harmonizer.key]
+        end
+      end
+
+      def destroy_targets_not_found_in_source(touched_keys)
+        # if we didn't get any touched_keys, destroy everything in targets scope
+        destroy_scope = touched_keys.empty? ? targets : targets.where("#{harmonizer.key} NOT IN (?)", touched_keys)
+        destroy_scope.find_each do |instance|
+          modification = harmonize_log.modifications.build(
+            :modification_type => 'destroy', :before_time => DateTime.now,
+            :instance_id => instance.id
+          )
+          instance.destroy
+          modification.update_attributes!(:after_time => DateTime.now)
+        end
+      end
+
+      def harmonize!
+        touched_keys = create_and_update_targets
+        destroy_targets_not_found_in_source(touched_keys)
+      end
+    end
+  end
+end

data/lib/harmonize/strategies/strategy.rb

@@ -0,0 +1,31 @@
+module Harmonize
+  module Strategies
+
+    class Strategy
+      attr_accessor :harmonizer, :harmonize_log, :sources, :targets
+
+      def initialize(attributes = {})
+        update_attributes(attributes)
+      end
+
+      def harmonize(attributes)
+        update_attributes(attributes)
+        self.sources = harmonizer.source.call
+        self.targets = harmonizer.target.call
+        harmonize!
+      end
+
+      def harmonize!
+      end
+
+      private
+
+        def update_attributes(attributes)
+          attributes.each_pair do |k, v|
+            self.send("#{k}=", v)
+          end
+        end
+
+    end
+  end
+end

data/lib/harmonize/strategies.rb

@@ -0,0 +1,6 @@
+module Harmonize
+  module Strategies
+    autoload :Strategy,          'harmonize/strategies/strategy'
+    autoload :BasicCrudStrategy, 'harmonize/strategies/basic_crud_strategy'
+  end
+end

data/lib/harmonize.rb

@@ -0,0 +1,9 @@
+require 'hashie/dash'
+require 'harmonize/errors'
+module Harmonize
+  require 'harmonize/engine' if defined?(Rails) && Rails::VERSION::MAJOR == 3
+  autoload :Base,          'harmonize/base'
+  autoload :Strategies,    'harmonize/strategies'
+  autoload :Configuration, 'harmonize/configuration'
+end
+ActiveRecord::Base.send :include, Harmonize::Base if defined?(ActiveRecord::Base)

data/lib/rails/generators/harmonize/migration_generator.rb

@@ -0,0 +1,29 @@
+require 'rails/generators'
+require 'rails/generators/migration'
+
+module Harmonize
+
+  class MigrationGenerator < Rails::Generators::Base
+    include Rails::Generators::Migration
+
+    def self.source_root
+      File.join(File.dirname(__FILE__), 'templates')
+    end
+
+    def self.next_migration_number(dirname) #:nodoc:
+      if ActiveRecord::Base.timestamped_migrations
+        Time.now.utc.strftime("%Y%m%d%H%M%S")
+      else
+        "%.3d" % (current_migration_number(dirname) + 1)
+      end
+    end
+
+    def create_migration_file
+      f = File.join(File.dirname(__FILE__), 'templates', 'migration.rb')
+      migration_template f, 'db/migrate/create_harmonize_tables.rb'
+    end
+
+  end
+
+end
+

data/lib/rails/generators/harmonize/templates/migration.rb

@@ -0,0 +1,34 @@
+class CreateHarmonizeTables < ActiveRecord::Migration
+  def self.up
+    create_table :harmonize_logs do |t|
+      t.string    :key
+      t.string    :class_name
+      t.string    :harmonizer_name
+      t.string    :strategy
+      t.string    :strategy_arguments
+      t.datetime  :start
+      t.datetime  :end
+      t.timestamps
+    end
+    add_index :harmonize_logs, [ :class_name, :harmonizer_name ]
+
+    create_table :harmonize_modifications do |t|
+      t.integer  :harmonize_log_id
+      t.integer  :instance_id
+      t.string   :modification_type
+      t.datetime :before_time
+      t.datetime :after_time
+      t.text     :instance_errors
+      t.timestamps
+    end
+    add_index :harmonize_modifications, :harmonize_log_id
+    add_index :harmonize_modifications, [ :harmonize_log_id, :instance_id ], :name => 'index_harmonize_modification_with_id'
+    add_index :harmonize_modifications, [ :harmonize_log_id, :instance_id, :modification_type ], :name => 'index_harmonize_modification_with_id_and_type'
+  end
+
+  def self.down
+    drop_table :harmonize_logs
+    drop_table :harmonize_modifications
+  end
+end
+

metadata

@@ -0,0 +1,88 @@
+--- !ruby/object:Gem::Specification 
+name: harmonize
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 0.0.1
+platform: ruby
+authors: 
+- Bram Swenson
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-06-26 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: hashie
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  prerelease: false
+  version_requirements: *id001
+description: |
+  Bring entire sets of models into harmony with external sources.
+
+email: 
+- bram@craniumisajar.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- app/models/harmonize/log.rb
+- app/models/harmonize/modification.rb
+- lib/harmonize/base.rb
+- lib/harmonize/configuration.rb
+- lib/harmonize/engine.rb
+- lib/harmonize/errors.rb
+- lib/harmonize/gemdata.rb
+- lib/harmonize/strategies/basic_crud_strategy.rb
+- lib/harmonize/strategies/strategy.rb
+- lib/harmonize/strategies.rb
+- lib/harmonize.rb
+- lib/rails/generators/harmonize/migration_generator.rb
+- lib/rails/generators/harmonize/templates/migration.rb
+- MIT-LICENSE
+- Rakefile
+- Gemfile
+- README.md
+has_rdoc: true
+homepage: http://github.com/bramswenson/harmonize
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 2398958541788060618
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.6.1
+signing_key: 
+specification_version: 3
+summary: Bring entire sets of models into harmony with external sources.
+test_files: []
+