sourced_attributes 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 60d367ee4b743bbcc4ee67e653cafe5b53b9ffc0
+  data.tar.gz: 771bf3f0feb097ea84927bc7a0583c83ecf9197f
+SHA512:
+  metadata.gz: e8fe5020e6977c061132b683f01d76366dd89776d8428e35e3c9dda7e9167c035bbd695ae389c49281ece5636dbe618dcc13160ad834bece8031c90f339852bf
+  data.tar.gz: ba2d5172d0e37c1e342940452fb425ff4e4b6cfb061810eeff59194fb8aa3ebbbbb8a4c4caa2873b8011f4b7f1c53678c4e9ac767b4406dfe117173eaaebf9e6

data/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2016 Jon Egeland
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/lib/sourced_attributes.rb

@@ -0,0 +1,3 @@
+require 'sourced_attributes/sourced_attributes'
+require 'sourced_attributes/dsl'
+require 'sourced_atributes//source'

data/lib/sourced_attributes/dsl.rb

@@ -0,0 +1,52 @@
+module SourcedAttributes
+  module DSL
+    # Set options specific to this Source instance.
+    def configure options={}
+      @config.merge! options
+    end
+
+    # Set the primary key that this Source will use to find records to update.
+    def primary_key local, opts={}
+      @primary_key = { local: local, source: opts[:source] || local }
+    end
+
+    # Short-hand for defining attributes whose local names map directly to
+    # field names in the source data.
+    def attributes *args
+      args.each{ |arg| @attribute_map[arg] = arg }
+    end
+
+    # Define an attribute whose local name is different from its name in the
+    # source data.
+    def aliased_attribute local_name, source_name
+      @attribute_map[local_name] = source_name
+    end
+
+    def complex_attribute local_name, &block
+      attributes local_name
+      @complex_attributes[local_name] = block
+    end
+
+    # Conditional attributes only get updated when the block is true. If no
+    # block is given, a default block checking for the presence of the
+    # attribute in the source data will be used
+    def conditional_attribute local_name, &block
+      attributes local_name
+      if block_given?
+        @conditional_attributes[local_name] = block
+      else
+        @conditional_attributes[local_name] = ->(record) { record[local_name] }
+      end
+    end
+
+    # Define an association whose value comes from the source data.
+    # `primary_key` here is the primary key to use for the associated table.
+    # `source_key` is the key to pick out of the source data.
+    def association name, options={}
+      options[:name] ||= name
+      options[:source_key] ||= options[:name]
+      options[:preload] ||= false
+      @associations << options
+    end
+  end
+end

data/lib/sourced_attributes/source.rb

@@ -0,0 +1,206 @@
+module SourcedAttributes
+  class Source
+    include ::SourcedAttributes::DSL
+
+    # A map of all aliases for concrete Source objects for use by the factory
+    # methods.
+    @@subclasses = {}
+
+    class << self
+      # A factory for creating instances of Source subclasses based on the
+      # parameterized name passed in and the list of registered subclasses.
+      def create name, opts, klass
+        (@@subclasses[name] or Source).new klass, opts
+      end
+
+      # Subclasses of Source should register aliases with the factory through
+      # this method.
+      def register_source name
+        @@subclasses[name] = self
+      end
+    end
+
+    # The default values for source-agnostic options that can be overridden by
+    # including new values in the Hash argument to `sources_attributes_from`.
+    DEFAULT_OPTIONS = {
+      save: true,
+      create_new: true,
+      batch_size: nil
+    }
+
+    def initialize klass, opts={}
+      # The model this source is working on.
+      @klass = klass
+      # A configuration hash for source-agnostic options, passed in as a Hash
+      # from the arguments given to the `sources_attributes_from` helper.
+      @options = DEFAULT_OPTIONS.merge(opts)
+      # A generic configuration hash for source-specific options, managed
+      # through the `configure` helper.
+      @config = {}
+      # The primary key that this source will use to find records to update.
+      # `local` is the alias of the primary key in the locally, while `source`
+      # is the alias of the primary key in the source data.
+      @primary_key = { local: :id, source: :id }
+      # A mapping of attributes on the model to field names from the source.
+      @attribute_map = {}
+      # A mapping of attributes which require special preparation to Procs
+      # which can perform that preparation, provided by the configuration.
+      @complex_attributes = {}
+      # A mapping of attributes which are only to be updated when a given
+      # condition is met to Procs which represent that condition.
+      @conditional_attributes = {}
+      # A list of associations that this source will update. Each entry is a
+      # hash, containing the keys :name, :primary_key, :preload.
+      @associations = []
+      # The most recent set of data from the source, formatted as a Hash using
+      # the :primary_key values as keys. Updated by `refresh` and used by
+      # `apply` to update records.
+      @source_data = []
+      # The last-retrieved set of data from the source, formatted the same way
+      # as @source_data.
+      @previous_data = []
+      # The records that that the source data will affect. Updated by
+      # `refresh_affected_records`.
+      @affected_records = {}
+    end
+
+    # Given an attribute name and a primary key, resolve the value to be given
+    # to that attribute using the configuration supplied through the DSL.
+    def resolve_attribute_for_datum attribute, datum
+      # The alias of this attribute in the source data
+      source_name = @attribute_map[attribute]
+      # Complex Attributes are evaluated with the datum as a parameter
+      if @complex_attributes.has_key?(attribute)
+        @complex_attributes[attribute].call(datum)
+      else
+        datum[source_name]
+      end
+    end
+
+    # Create a Hash from the @source_data array, keyed by @primary_key. If
+    # @source_data is already a Hash, assume it has already been indexed and do
+    # nothing.
+    def ensure_indexed_source_data
+      return if @source_data.is_a?(Hash)
+      @source_data = @source_data.inject({}) do |hash, datum|
+        hash[datum[@primary_key[:source]]] = datum
+        hash
+      end
+    end
+
+    # Create new instances of @klass for every key that does not already have
+    # an instance associated with it.
+    def create_new_records
+      @source_data.keys.each do |pk|
+        # TODO: Add an option for creating/not creating new records
+        @affected_records[pk] ||= @klass.new(@primary_key[:local] => pk)
+      end
+    end
+
+    # Fill @affected_records with all records affected by the current set of
+    # source data, creating new records for any keys which do not yet exist.
+    def refresh_affected_records
+      # Ensure that the source data is indexed by primary key...
+      ensure_indexed_source_data
+      # ...so that it can be skimmed to find existing records.
+      @affected_records = @klass \
+        .where(@primary_key[:local] => @source_data.keys) \
+        .index_by(&@primary_key[:local])
+      # Then create new objects for the remaining data
+      create_new_records if @options[:create_new]
+    end
+
+    # Apply the attribute map to the source data for the given record
+    def mapped_attributes_for pk
+      source = @source_data[pk]
+      @attribute_map.inject({}) do |hash, (attribute,_)|
+        # Only apply conditional attributes if they're condition is met
+        if @conditional_attributes.has_key?(attribute)
+          next hash unless @conditional_attributes[attribute].call(source)
+        end
+        hash[attribute] = resolve_attribute_for_datum(attribute, source)
+        hash
+      end
+    end
+
+    # Load into memory all key-value pairs needed to fully associate all
+    # records that a source handles.
+    def preload_association_data
+      @associations.each do |assoc|
+        # Get the model that this association references.
+        reflection = @klass.reflect_on_association(assoc[:name])
+        # Determine which key-value pairs should be preloaded.
+        values = @source_data.map{ |key, datum| datum[assoc[:source_key]] }
+        # Load the data, keyed by the local primary key
+        assoc[:data] = reflection.klass \
+          .where(assoc[:primary_key] => values) \
+          .select(assoc[:primary_key], reflection.klass.primary_key) \
+          .index_by(&assoc[:primary_key])
+      end
+    end
+
+    # Apply the current set of source data to the attributes for the given
+    # primary key.
+    def apply_attributes_to pk, record
+      # Map the attributes from the source data to their local counterparts
+      # and apply it to the record
+      record.assign_attributes(mapped_attributes_for(pk))
+    end
+
+    # Apply the current set of source data to the associations for the given
+    # primary key.
+    def apply_associations_to pk, record
+      @associations.each do |config|
+        # Get the model that this association references
+        reflection = @klass.reflect_on_association(config[:name])
+        # The associated records are already loaded, but need to be plucked out
+        # of their containing hash
+        associated_records = config[:data][@source_data[pk][config[:source_key]]]
+        # Apply the updated association to the record
+        record.assign_attributes(config[:name] => associated_records)
+      end
+    end
+
+    # Perform all of the operations related to updating a sourced record
+    def update_record pk, record
+      # Update attributes
+      apply_attributes_to(pk, record)
+      # Update associations
+      apply_associations_to(pk, record)
+      # Save the record if it should be
+      record.save if (@options[:save] && !@options[:batch_size])
+    end
+
+    # Apply the current set of source data to the records it affects
+    def apply
+      # Make sure the source data is up-to-date
+      refresh
+      # Make sure the source data is indexed by the primary key
+      ensure_indexed_source_data
+      # Make sure that all of the affected records are loaded
+      refresh_affected_records
+      # Preload all key-value pairs needed to fully associate this record.
+      preload_association_data
+      # Wrap all of the updates in a single transaction
+      @klass.transaction do
+        if @options[:batch_size]
+          @affected_records.each_slice(@options[:batch_size]) do |batch|
+            @klass.import batch.map{ |pk, record| update_record(pk, record); record }
+          end
+        else
+          @affected_records.each do |pk, record|
+            update_record pk, record
+          end
+        end
+      end
+    end
+
+
+    # #
+    # Abstract Methods for Subclasses
+    # #
+
+    # Talk to the data source to refresh the contents of @source_data.
+    def refresh; raise :subclass_responsiblity; end
+  end
+end

data/lib/sourced_attributes/sourced_attributes.rb

@@ -0,0 +1,25 @@
+module SourcedAttributes
+  def self.included base
+    base.extend ClassMethods
+  end
+
+  module ClassMethods
+    # Configure a new Source for a model.
+    def sources_attributes_from source_name, **opts, &block
+      @sources ||= {}
+      @sources[source_name] ||= Source.create(source_name, opts, self)
+      @sources[source_name].instance_eval(&block)
+      @sources[source_name]
+    end
+
+    # Apply all Sources to the model. If `source_name` is specified, only apply
+    # changes from that Source.
+    def update_sourced_attributes source_name=nil
+      if source_name
+        @sources[source_name].apply
+      else
+        @sources.values.each(&:apply)
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,49 @@
+--- !ruby/object:Gem::Specification
+name: sourced_attributes
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Jon Egeland
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-05-16 00:00:00.000000000 Z
+dependencies: []
+description: A DSL for aggregating data from multiple sources into ActiveRecord models.
+  Includes options for aliases, predicates, associations, and complex attributes.
+email: audiobahn404@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- lib/sourced_attributes.rb
+- lib/sourced_attributes/dsl.rb
+- lib/sourced_attributes/source.rb
+- lib/sourced_attributes/sourced_attributes.rb
+homepage: http://github.com/audiobahn404/sourced_attributes
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.2.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: A DSL for aggregating data from multiple sources into ActiveRecord models.
+test_files: []