drive_time 0.0.3

data/lib/drive_time/converters/worksheet_converter.rb

@@ -0,0 +1,262 @@
+module DriveTime
+
+  class WorksheetConverter
+
+    attr_accessor :row_map
+
+    class NoClassWithTitleError < StandardError; end
+    class NoFieldNameError < StandardError; end
+
+    class NoKeyError < StandardError; end
+    class PolymorphicAssociationError < StandardError; end
+
+    def initialize(model_store, class_name_map, loader, namespace)
+      @class_name_map = class_name_map
+      @model_store = model_store
+      @loader = loader
+      @namespace = namespace
+      @field_expander = FieldExpander.new(@loader)
+    end
+
+    def convert(worksheet)
+      Logger.log_as_header "Converting worksheet: #{worksheet.title}"
+      # Use the spreadsheet name unless 'map_to_class' is set
+      class_name = DriveTime.class_name_from_title(worksheet.title)
+      class_name = @class_name_map.resolve_mapped_from_original class_name
+      Logger.debug "Converting Worksheet #{worksheet.title} to class #{class_name}"
+      # Check class exists - better we know immediately
+      begin
+        clazz = namespaced_class_name class_name
+      rescue StandardError => error
+        raise NoClassWithTitleError, "Worksheet named #{worksheet.title} doesn't exists as class #{class_name}"
+      end
+      rows = worksheet.rows.dup
+      # Remove the first row and use it for field-names
+      fields = rows.shift.map{ |row| row }
+      # Reject rows of only empty strings (empty cells).
+      rows.reject! {|row| row.all?(&:empty?)}
+
+      rows.each do |row|
+        generate_model_from_row clazz, worksheet.mapping, fields, row
+      end
+
+      Logger.log_as_header "Conversion Complete. Woot Woot."
+    end
+
+    protected
+
+    def generate_model_from_row(clazz, mapping, fields, row)
+      Logger.log_as_header "Converting row to class: #{clazz.name}"
+      # Create a hash of field-names and row cell values
+      build_row_map(fields, row)
+
+      # If a row has been marked as not complete, ignore it
+      if @row_map[:complete] == 'No'
+        Logger.debug 'Row marked as not complete. Ignoring'
+        return
+      end
+
+      model_key = build_id_for_model mapping
+      Logger.log_as_sub_header "Model Key: #{model_key}"
+      # Build hash ready to pass into model
+      model_fields = row_fields_to_model_fields mapping, model_key
+
+      # Set the model key as an attribute on the model
+      if mapping[:key_to]
+        model_fields[mapping[:key_to]] = model_key
+      end
+
+      Logger.debug "Creating Model of class '#{clazz.name.to_s}' with Model Fields #{model_fields.to_s}"
+      # Create new model
+      model = clazz.new(model_fields, without_protection: true)
+      # Add its associations
+      add_associations(model, mapping)
+      # Store the model using its ID
+      @model_store.add_model model, model_key, clazz
+    end
+
+    # Convert worksheet row into hash
+    def build_row_map(fields, row)
+      @row_map = HashWithIndifferentAccess.new
+      Logger.log_as_sub_header "Mapping fields to cells"
+      row.dup.each_with_index do |cell, index|
+        # Sanitise
+        field_name = DriveTime.underscore_from_text fields[index]
+        @row_map[field_name] = row[index]
+        Logger.debug "- #{field_name} -> #{row[index]}"
+      end
+    end
+
+    # Run through the mapping's fields and convert them
+    def row_fields_to_model_fields(mapping, model_key)
+      model_fields = HashWithIndifferentAccess.new
+      # Run through the mapping, pulling values from row_map as required
+      if mapping[:fields]
+        mapping[:fields].each do |field_mapping|
+          field_name = field_mapping[:name]
+          mapped_to_field_name = field_mapping[:map_to]
+          unless field_name
+            raise NoFieldNameError "Missing Field: Name for field: #{value} in mapping: #{mapping}"
+          end
+
+          field_value = @row_map[field_name]
+          # Check for token pattern: {{some_value}}
+          match = /\{\{(.*?)\}\}/.match(field_value)
+          if match
+            field_value = @field_expander.expand(match[1], model_key)
+          end
+
+          # Check for Boolean values
+          if field_value
+            downcased = field_value.downcase
+            if downcased == 'y' || downcased == 'yes' || downcased == 'true'
+              field_value = true
+            elsif downcased == 'n' || downcased == 'no' || downcased == 'false'
+              field_value = false
+            end
+          end
+
+          # Make sure any empty cells give us nil (rather than an empty string)
+          field_value = nil if field_value.blank?
+          model_fields[mapped_to_field_name || field_name] = field_value
+
+        end
+      end
+      return model_fields
+    end
+
+    # TODO: Refactor this big ugly beast
+    def add_associations(model, mapping)
+      associations_mapping = mapping[:associations]
+      if associations_mapping
+        Logger.log_as_sub_header "Adding associations to model "
+
+        # Loop through any associations defined in the mapping for this model
+        associations_mapping.each do |association_mapping|
+          # Use the spreadsheet name unless 'map_to_class' is set
+          if !association_mapping[:polymorphic]
+            association_class_name = @class_name_map.resolve_mapped_from_original association_mapping[:name].classify
+          else
+            possible_class_names = association_mapping[:name]
+            # The classname will be taken from the type collumn
+            association_class_name = DriveTime::class_name_from_title @row_map[:type]
+            # if !possible_class_names.include? association_class_name.underscore
+            #     raise PolymorphicAssociationError, "Mapping for polymorphic associations: #{possible_class_names.inspect} doesn't include #{association_class_name}"
+            # end
+          end
+          # Get class reference using class name
+          begin
+            clazz = namespaced_class_name association_class_name
+          rescue
+            raise NoClassWithTitleError, "Association defined in worksheet doesn't exist as class: #{association_class_name}"
+          end
+
+          # Assemble associated model instances to satisfy this association
+          associated_models = gather_associated_models(association_mapping, association_class_name, clazz)
+          set_associations_on_model(model, associated_models, association_mapping, association_class_name)
+        end
+      end
+    end
+
+    def gather_associated_models(association_mapping, association_class_name, clazz)
+      associated_models = []
+      if association_mapping[:builder]
+        associated_models = associated_models_from_builder association_mapping, association_class_name
+      else # It's a single text value, so convert it to an ID
+        association_class = @class_name_map.resolve_original_from_mapped(association_class_name).underscore
+        if !association_mapping[:polymorphic]
+          association_id = @row_map[association_class]
+        else
+          association_id = @row_map[association_mapping[:polymorphic][:association]]
+        end
+        raise MissingAssociationError, "No field #{association_class_name.underscore} to satisfy association" if !association_id
+        if association_id.length > 0 || association_mapping[:required] == true
+          associated_models << model_for_id(association_id, clazz)
+        end
+      end
+      return associated_models
+    end
+
+    def set_associations_on_model(model, associated_models, association_mapping, association_class_name)
+      # We now have one or more associated_models to set as associations on our model
+      associated_models.each do |associated_model|
+        Logger.debug " - Associated Model: #{associated_model}"
+        unless association_mapping[:inverse] == true
+          association_name = association_class_name.underscore
+          if association_mapping[:singular] == true
+            Logger.debug "   - Adding association #{associated_model} to #{model}::#{association_name}"
+            # Set the association
+            model.send("#{association_name}=", associated_model)
+          else
+            association_name = association_name.pluralize
+            model_associations = model.send(association_name)
+            Logger.debug "   - Adding association #{associated_model} to #{model}::#{association_name}"
+            # Push the association
+            model_associations << associated_model
+          end
+        else # The relationship is actually inverted, so save the model as an association on the associated_model
+          model_name = model.class.name.split('::').last.underscore
+          if association_mapping[:singular] == true
+            Logger.debug "   - Adding association #{model} to #{associated_model}::#{association_name}"
+            associated_model.send model_name, model
+          else
+            model_name = model_name.pluralize
+            model_associations = associated_model.send model_name
+            Logger.debug "   - Adding association #{model} to #{associated_model}::#{model_name}"
+            model_associations << model
+          end
+        end
+      end
+    end
+
+    def associated_models_from_builder(association_mapping, class_name)
+      associated_models = []
+      if association_mapping[:builder] == 'multi' # It's a multi value, find a matching cell and split its value by comma
+        cell_value = @row_map[class_name.underscore.pluralize]
+        raise MissingAssociationError "No field #{class_name.underscore.pluralize} to satisfy multi association" if !cell_value && association_mapping[:optional] != true
+        components = cell_value.split ','
+        components.each do |component|
+          associated_models << model_for_id(component, namespaced_class_name(class_name))
+        end
+      elsif association_mapping[:builder] == 'use_fields' # Use column names as values if cell contains 'yes' or 'y'
+        association_mapping[:field_names].each do |field_name|
+          cell_value = @row_map[field_name]
+          if DriveTime.is_affirmative? cell_value
+            associated_models << model_for_id(field_name, namespaced_class_name(class_name))
+          end
+        end
+      end
+      return associated_models
+    end
+
+    def model_for_id(value, clazz)
+      model_key = DriveTime.underscore_from_text value
+      return @model_store.get_model clazz, model_key
+    end
+
+    def build_id_for_model(mapping)
+      raise NoKeyError, 'All mappings must declare a key' unless mapping.has_key? :key
+      key_node = mapping[:key]
+      if key_node.is_a? Hash
+        if key_node[:builder] == 'join'
+          key = JoinBuilder.new.build key_node[:from_fields], @row_map
+        elsif key_node[:builder] == 'name'
+          key = NameBuilder.new.build key_node[:from_fields], @row_map
+        else
+          raise "No builder for key on worksheet #{mapping[:title]}"
+        end
+      else # If it's a string, it refers to a spreadsheet column
+        key_attribute = key_node
+        # Is there a column
+        key = @row_map[key_attribute]
+        raise NoFieldNameError, "No column #{key_attribute} on worksheet #{mapping[:title]}" if !key
+        DriveTime.underscore_from_text key
+      end
+    end
+
+    def namespaced_class_name class_name
+      class_name = "#{@namespace}::#{class_name}" unless @namespace.blank?
+      class_name.constantize
+    end
+  end
+end

data/lib/drive_time/field_expander.rb

@@ -0,0 +1,73 @@
+module DriveTime
+
+  class TokenExpansionError < StandardError; end
+
+  class FieldExpander
+
+
+
+    def initialize(loader)
+      @loader = loader
+    end
+
+    def expand(value, model_key)
+      filename = model_key
+      # Check for token
+      match = /\[(.*?)\]/.match(value)
+      # Expand token into file
+      # Is there a different filename defined in hard brackets [file_name]
+      if match
+        filename = match[1]
+        value = value.split('[').first
+      end
+
+      if value == 'expand_file'
+        file = @loader.load_file_direct(filename+'.txt');
+        if file.blank?
+          raise TokenExpansionError, "Missing file named: #{filename} when expanding from value: #{value} in model: #{model_key}"
+        end
+        value = expand_file(file)
+      elsif value == 'expand_spreadsheet'
+        spreadsheet = @loader.load_spreadsheet_direct(filename)
+        if spreadsheet.blank?
+          raise TokenExpansionError, "Missing spreadsheet named: #{filename} when expanding from value: #{value} in model: #{model_key}"
+        end
+        # Use first Worksheet
+        worksheet = spreadsheet.worksheets[0]
+        value = expand_worksheet(worksheet)
+      else
+        raise TokenExpansionError, "Don't know how to expand the value #{value} for model: #{model_key}"
+      end
+      return value
+    end
+
+    protected
+
+    # Build a JSON object from the columns
+    def expand_worksheet(worksheet)
+      rows = worksheet.rows.dup
+
+      # Take the first row which will be the column names and use the cell value as the field name
+      fields = rows.shift.map{ |row|  row[/\w+/] }
+
+      instances = []
+      # Reject rows of only empty strings (empty cells).
+      rows.reject! {|row| row.all?(&:empty?)}
+
+      rows.each do |row|
+        key_value_pairs = [fields.dup, row.dup].transpose
+        hash = Hash[*key_value_pairs.flatten]
+        instances.push(hash)
+      end
+
+      root = {objects: instances }
+      return root.to_json.to_s
+    end
+
+    def expand_file(file)
+      return file.download_to_string()
+    end
+
+  end
+
+end

data/lib/drive_time/loader.rb

@@ -0,0 +1,100 @@
+module DriveTime
+
+  # Load a Spreadsheet from Google Drive
+  class Loader
+
+    class SpreadsheetNotFoundError < StandardError; end
+    class WorksheetNotFoundError < StandardError; end
+
+    def initialize
+      begin_session
+    end
+
+    def load_file_direct(title)
+      return @session.file_by_title title
+    end
+
+    def load_spreadsheet_direct(title)
+      @session.spreadsheet_by_title title
+    end
+
+    def load_spreadsheet(title, use_cache=true)
+      cached_directory = ENV['CACHED_DIR']
+      spreadsheet_name = "#{title}.yml"
+      spreadsheet_file_path = File.join(cached_directory, spreadsheet_name) if cached_directory
+      spreadsheet = nil
+      # Try and pull the file from the cache
+      if cached_directory && use_cache
+
+        if File.exist? spreadsheet_file_path
+          File.open(spreadsheet_file_path, 'r') do |file|
+            Logger.info "Pulling spreadsheet '#{title}' from cache"
+            spreadsheet = YAML::load(file)
+          end
+        end
+      end
+
+      # If we haven't loaded a spreadsheet from cache, get it from drive
+      unless spreadsheet
+        Logger.info "Loading spreadsheet '#{title}' from Drive"
+        spreadsheet = @session.spreadsheet_by_title(title)
+
+        raise SpreadsheetNotFoundError, "Spreadsheet #{title} not found" if spreadsheet.nil?
+        # Save the file to cache directory
+        if cached_directory && use_cache
+          # Save the spreadsheet
+          Logger.info "Saving spreadsheet '#{title}' to cache"
+          File.open(spreadsheet_file_path, 'w') do |file|
+            file.puts YAML::dump(spreadsheet)
+          end
+
+          # Save its worksheets
+          spreadsheet.worksheets.each do |worksheet|
+            Logger.info "Saving worksheet '#{worksheet.title}' to cache"
+            # Force worksheet to down of cells data
+            #worksheet.reload
+            worksheet_file_path = File.join(cached_directory, worksheet.title) + '.yml'
+            File.open(worksheet_file_path, 'w') do |file|
+              file.puts YAML::dump(worksheet)
+            end
+          end
+        end
+      end
+
+      return spreadsheet
+    end
+
+    def load_worksheet_from_spreadsheet(spreadsheet, title, use_cache=true)
+      cached_directory = ENV['CACHED_DIR']
+      worksheet_name = "#{title}.yml"
+
+      worksheet = nil
+
+      # Get the worksheet from the cache
+      if cached_directory && use_cache
+        worksheet_file_path = File.join(cached_directory, title)
+        if File.exist? worksheet_file_path
+          File.open(worksheet_file_path, 'r') do |file|
+            Logger.info "Pulling worksheet '#{title}' from cache"
+            worksheet = YAML::load(file)
+          end
+        end
+      end
+
+      # If we don't yet have a worksheet, pull it from Google Drive via the Spreadsheet
+      unless worksheet
+        worksheet = spreadsheet.worksheet_by_title title
+        raise WorksheetNotFoundError, "Worksheet '#{title}'' not found in spreadsheet '#{spreadsheet.title}'" if worksheet.nil?
+      end
+
+      return worksheet
+    end
+
+    protected
+
+    def begin_session
+      @session = GoogleDrive.login( ENV['GOOGLE_USERNAME'], ENV['GOOGLE_PASSWORD'])
+    end
+
+  end
+end

data/lib/drive_time/logging.rb

@@ -0,0 +1,19 @@
+require "log4r"
+
+module Log4r
+  class Logger
+    def log_as_header(message)
+      puts "\n"
+      info "=============================================================================="
+      info "#{message}"
+      info '=============================================================================='
+    end
+
+    def log_as_sub_header(message)
+      puts "\n" if self.level <= DEBUG
+      debug "--------------------------------------------------------------------------------"
+      debug "  #{message}"
+      debug '--------------------------------------------------------------------------------'
+    end
+  end
+end

data/lib/drive_time/model_store.rb

@@ -0,0 +1,73 @@
+require 'log4r'
+
+module DriveTime
+
+  include Log4r
+
+  # Store model instances by class and link
+  # This way we can look them up as needed to
+  # satisfy dependencies and avoid duplication
+  class ModelStore
+
+    # Errors
+    class NoModelsOfClassInStoreError < StandardError; end
+    class NoModelOfClassWithKeyInStoreError < StandardError; end
+    class ModelAddedTwiceError < StandardError; end
+
+    def initialize(log_level=Log4r::INFO)
+      @store = {}
+
+      # Set up logging
+      formatter = Log4r::PatternFormatter.new(:pattern => "[%c] %M")
+      outputter = Log4r::Outputter.stdout
+      outputter.formatter = formatter
+
+      @logger = Log4r::Logger.new ' Model Store '
+      @logger.level = log_level
+      @logger.outputters = outputter
+    end
+
+    # Store the model by class to avoid key collisions
+    def add_model(instance, key, clazz)
+      class_string = clazz.to_s
+      # Sanitise key
+      key = DriveTime.underscore_from_text(key)
+      @logger.debug "Adding model with key #{key} of class #{clazz}"
+      if !@store[class_string]
+        @store[class_string] = {}
+      elsif @store[class_string][key]
+        raise ModelAddedTwiceError, "#{instance} has already been added to model store"
+      end
+      @store[class_string][key] = instance
+    end
+
+    def get_model(clazz, key)
+      @logger.debug "Request for model with key #{key} of class #{clazz}"
+
+      models_for_class = @store[clazz.to_s]
+      # Are there any classes of this type in the store?
+      if models_for_class.nil?
+        raise NoModelsOfClassInStoreError, "No classes of type: #{clazz} in model store"
+      end
+
+      # Is there an instance
+      model = models_for_class[key]
+
+      if !model
+        raise NoModelOfClassWithKeyInStoreError, "No model of class #{clazz} with a key of #{key} in model store"
+      end
+
+      return model
+    end
+
+    def save_all
+      @store.each do |key, models|
+        models.each do |key, model|
+          model.save!
+        end
+      end
+    end
+
+  end
+
+end

data/lib/drive_time/version.rb

@@ -0,0 +1,3 @@
+module DriveTime
+  VERSION = "0.0.3"
+end

data/lib/drive_time.rb

@@ -0,0 +1,71 @@
+require "drive_time/version"
+
+require "log4r"
+require 'google_drive'
+require 'deep_end'
+require 'active_support'
+require 'active_support/inflector'
+require 'active_support/core_ext/hash'
+require 'active_support/core_ext/module'
+
+require 'drive_time/model_store'
+require 'drive_time/field_expander'
+require 'drive_time/bi_directional_hash'
+require 'drive_time/loader'
+require 'drive_time/class_name_map'
+require 'drive_time/builders/join_builder'
+require 'drive_time/builders/name_builder'
+require 'drive_time/converters/spreadsheets_converter'
+require 'drive_time/converters/worksheet_converter'
+require 'drive_time/logging'
+
+module DriveTime
+
+  mattr_accessor :log_level
+
+  class MissingAssociationError < StandardError; end
+
+  include ActiveSupport::Inflector
+  include Log4r
+
+  # Set up logging
+  formatter = PatternFormatter.new(:pattern => "[%c] %M")
+  outputter = Outputter.stdout
+  outputter.formatter = formatter
+
+  @@log_level = INFO
+  # Create constants for loggers - available in inner classes
+  Logger = Log4r::Logger.new ' Primary     '
+  Logger.level = @@log_level
+  Logger.outputters = outputter
+
+  # Store the mapping on the spreadsheets and worksheets
+  class GoogleDrive::Spreadsheet
+    attr_accessor :mapping
+  end
+
+  class GoogleDrive::Worksheet
+    attr_accessor :mapping
+  end
+
+  def self.underscore_from_text(text)
+    text.strip.downcase.parameterize('_')
+  end
+
+  def self.class_name_from_title(title)
+    self.underscore_from_text(title).classify
+  end
+
+  def self.is_affirmative?(value)
+    return false if !value
+    value.
+      strip.
+      downcase == 'yes' || value.downcase == 'y'
+  end
+
+  def self.log_level=(log_level)
+    @@log_level = log_level
+    Logger.level = log_level
+  end
+
+end

data/spec/drive_time/bi_directional_hash_spec.rb

@@ -0,0 +1,54 @@
+require 'spec_helper'
+
+module DriveTime
+
+  describe 'BiDirectionalHash' do
+
+    before(:each) do
+      @hash = BiDirectionalHash.new
+    end
+
+    it 'should correctly declare if a value exists for a key' do
+      @hash.insert('A', 1);
+      @hash.insert('B', 2);
+      @hash.insert(3, 'A');
+
+      @hash.has_value_for_key('A').should be_true
+      @hash.has_value_for_key('B').should be_true
+      @hash.has_value_for_key(3).should be_true
+      @hash.has_value_for_key('Nope').should be_false
+    end
+
+    it 'should correctly declare if a key exists for a value' do
+      @hash.insert('A', 1);
+      @hash.insert('B', 2);
+      @hash.insert(3, 'A');
+
+      @hash.has_key_for_value(1).should be_true
+      @hash.has_key_for_value(2).should be_true
+      @hash.has_key_for_value('A').should be_true
+      @hash.has_key_for_value('Nope').should be_false
+    end
+
+    it 'should allow values to be looked up using keys' do
+      @hash.insert('A', 1);
+      @hash.insert('B', 2);
+      @hash.insert(3, 'A');
+
+      @hash.value_for_key('A').should == 1
+      @hash.value_for_key('B').should == 2
+      @hash.value_for_key(3).should == 'A'
+    end
+
+    it 'should allow keys to be looked up using values' do
+      @hash.insert('A', 1);
+      @hash.insert('B', 2);
+      @hash.insert(3, 'A');
+
+      @hash.key_for_value(1).should == 'A'
+      @hash.key_for_value(2).should == 'B'
+      @hash.key_for_value('A').should == 3
+    end
+
+  end
+end