ingestor 0.1.1

data/ingestor.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'ingestor/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "ingestor"
+  gem.version       = Ingestor::VERSION
+  gem.authors       = ["Cory O'Daniel"]
+  gem.email         = ["github@coryodaniel.com"]
+  gem.description   = "Ingesting local and remote data files into ActiveRecord"
+  gem.summary       = "Ingesting local and remote data files into ActiveRecord"
+  gem.homepage      = "http://github.com/coryodaniel/ingestor"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+  gem.add_dependency "docile"
+  gem.add_dependency "rubyzip"
+  gem.add_dependency "thor"
+  gem.add_dependency "activesupport", '>= 3.2.0'
+end

data/lib/ingestor.rb

@@ -0,0 +1,37 @@
+require 'docile'
+require 'open-uri'
+require 'csv'
+require 'logger'
+require 'zip/zipfilesystem'
+require 'ingestor/version'
+require 'ingestor/proxy'
+require 'ingestor/dsl'
+
+#require 'debugger'
+
+module Ingestor
+  LOG = Logger.new(STDOUT)
+  LOG.level = Logger::WARN
+  class << self
+    def parsers
+      @parsers ||= {}
+    end
+    def register_parser(kind, klass)
+      parsers[kind] = klass
+    end
+
+    def parser_for(kind)
+      raise Exception, "No parser for type #{kind}" if parsers[kind].nil?
+      parsers[kind]
+    end  
+  end
+end
+
+def ingest(filename, &block)
+  options = Ingestor::Dsl.new
+  options.file = filename
+  proxy = Docile.dsl_eval(options, &block).build.start!
+end
+
+require 'ingestor/parser/base'
+require 'ingestor/parser/plain_text'

data/lib/ingestor/dsl.rb

@@ -0,0 +1,110 @@
+module Ingestor
+  class Dsl
+    class InvalidBlockSpecification < Exception;end;
+    def initialize(*args)
+      @options = {}
+      
+      includes_header(false)
+      compressed(false)
+      parser :plain_text
+      parser_options({})
+      working_directory '/tmp/ingestor'
+    end
+
+    def options
+      @options
+    end
+
+    # the file to retrieve
+    def file=(v);                     @file = v;end;    
+
+    # When set to true sample will get the file and print out the first
+    # set of raw values
+    def sample(v)
+      @options[:sample] = v
+    end
+
+    # where the file will be moved locally for processing when it is compressed or a remote file.
+    # local files will not use working directory
+    def working_directory(v)
+      @options[:working_directory] = v
+    end
+
+    # set parser, default :plain_text
+    def parser(v)
+      @options[:parser] = v
+    end
+
+    # set options
+    def parser_options(v)
+      @options[:parser_options] = v
+    end
+
+    # skip first line?
+    def includes_header(v);           @options[:includes_header] = v;end;
+        
+    # if the remote file is compressed, this will decompress it.
+    def compressed(v);                @options[:compressed] = v;end;
+
+    # Takes an array of values (a line/entry/node) and should return an
+    # ActiveModel type object
+    #
+    # You do not need to set the attributes here, than is handled by #processor
+    # 
+    # update or create:
+    # finder{|attrs| User.where(id: attrs[:id]).first || User.new}
+    #
+    # create:
+    # finder{|attrs| User.new}
+    # @required
+    def finder(&block)
+      if !block_given? || block.arity != 1
+        raise InvalidBlockSpecification, "finder proc should have an arity of 1 (Array: values)"
+      end      
+      @options[:finder] = block
+    end
+
+    # How to process an entry in a file. The default takes the values and passes them to the record returned
+    #  by your finder and calls update attributes
+    # Proc should receive two parameters
+    #   attrs - Hash, mapped attributs for this record
+    #   record - ~ActiveRecord:Base, record found by #finder
+    def processor(&block)
+      if !block_given? || block.arity != 2
+        raise InvalidBlockSpecification, "processor proc should have an arity of 2 (Array: values, ~ActiveRecord: record)"
+      end      
+      @options[:processor] = block
+    end
+
+    # Processing performed on the attributes before being passed to [+finder+]
+    def before(&block)
+      if !block_given? || block.arity != 1
+        raise InvalidBlockSpecification, "before proc should have an arity of 1 (Array: values)"
+      end      
+      @options[:before] = block
+    end
+
+    # Processing performed on the record AFTER being passing to [+processor+]
+    def after(&block)
+      if !block_given? || block.arity != 1
+        raise InvalidBlockSpecification, "after proc should have an arity of 1 (~ActiveRecord: record)"
+      end      
+      @options[:after] = block
+    end
+
+    # This method is called for each entry in the document
+    # Block should receive 'values' (array for plain text, hash for all others) and return a hash
+    #   of ActiveModel attribute name to value
+    #
+    def map_attributes(&block)
+      if !block_given? || block.arity != 1
+        raise InvalidBlockSpecification, "after proc should have an arity of 1 (Hash|Array: values)"
+      end      
+      @options[:map_attributes] = block
+    end
+
+    def build
+      Ingestor::Proxy.new(@file, @options)
+    end    
+  end
+end

data/lib/ingestor/parser/base.rb

@@ -0,0 +1,28 @@
+module Ingestor
+  module Parser
+    module Base
+      def self.included(base)
+        base.extend(ClassMethods)
+      end
+      
+      module ClassMethods;end;
+  
+      def initialize(proxy,document)
+        @proxy    = proxy
+        @document = document
+      end
+      
+      def options(opts)
+        @options = opts
+      end
+
+      def sample!
+        raise Exception, "#sample! not implemented"
+      end
+
+      def process!
+        raise Exception, "#process! not implemented"
+      end
+    end
+  end
+end

data/lib/ingestor/parser/csv.rb

@@ -0,0 +1,8 @@
+module Ingestor
+  module Parser
+    class Csv
+    end
+  end
+end
+
+Ingestor.register_parser :csv, Ingestor::Parser::Csv

data/lib/ingestor/parser/json.rb

@@ -0,0 +1,8 @@
+module Ingestor
+  module Parser
+    class Json
+    end
+  end
+end
+
+Ingestor.register_parser :json, Ingestor::Parser::Json

data/lib/ingestor/parser/plain_text.rb

@@ -0,0 +1,44 @@
+module Ingestor
+  module Parser
+    class PlainText
+      include Ingestor::Parser::Base
+
+      def options(opts={})
+        @options = {
+          delimiter: '|',
+          line_processor: nil
+        }.merge(opts)
+      end
+
+      def process!
+        while line = @document.gets do
+          line.chomp!    
+          attrs   = @proxy.options[:map_attributes].call( process_line(line) )
+          @proxy.process_entry attrs
+        end
+      end
+
+      def sample!
+        line = @document.gets
+        line.chomp!
+
+        puts line
+      end
+
+      protected
+      # Runs the default line processor or line processor provided to options
+      def process_line(line)
+        if @options[:line_processor]
+          @options[:line_processor].call(line)
+        else
+          default_line_processor(line)
+        end
+      end
+      def default_line_processor(line)
+        line.split(@options[:delimiter])
+      end       
+    end
+  end
+end
+
+Ingestor.register_parser :plain_text, Ingestor::Parser::PlainText

data/lib/ingestor/parser/xml.rb

@@ -0,0 +1,37 @@
+#   #require 'open-uri'
+#   #http://nokogiri.org/tutorials/parsing_an_html_xml_document.html
+#   #doc = Nokogiri::HTML(open("http://www.threescompany.com/"))
+
+require 'nokogiri'
+require 'active_support/core_ext/hash/conversions'
+
+module Ingestor
+  module Parser
+    class Xml
+      include Ingestor::Parser::Base      
+      def options(opts={})
+        @options = {
+          encoding: nil,
+          xpath: nil
+        }.merge(opts)
+      end
+
+      def sample!
+        doc = Nokogiri::XML(@document, nil, @options[:encoding])
+        puts Hash.from_xml( doc.xpath(@options[:xpath]).first.to_s )
+      end      
+
+      def process!
+        doc = Nokogiri::XML(@document, nil, @options[:encoding])
+        
+        doc.xpath(@options[:xpath]).each do |node|
+          node_attrs = Hash.from_xml(node.to_s)
+          attrs   = @proxy.options[:map_attributes].call( node_attrs )
+          @proxy.process_entry attrs
+        end
+      end
+    end
+  end
+end
+
+Ingestor.register_parser :xml, Ingestor::Parser::Xml

data/lib/ingestor/proxy.rb

@@ -0,0 +1,113 @@
+module Ingestor
+  Proxy = Struct.new(:file, :options) do
+    def document
+      @document
+    end
+    
+    def header
+      @header
+    end
+
+    def remote?
+      file =~ /http(s)?|ftp/i
+    end
+
+    def local?
+      !remote?
+    end
+
+    def working_directory
+      options[:working_directory]
+    end
+
+    def compressed?; options[:compressed]; end;
+
+    # for debugging, testing
+    def continue_from(line_num)
+      @document.rewind
+      @document.drop( line_num -1 ).take(1)
+    end
+
+    def finder
+      options[:finder]
+    end
+
+    def start!
+      load
+      Ingestor::LOG.warn("No #finder specified") if !finder
+      @header = @document.gets.strip if options[:includes_header]
+
+      parser = Ingestor.parser_for( options[:parser] ).new(self, @document)
+      parser.options( options[:parser_options] )
+
+      unless options[:sample]
+        parser.process!
+      else
+        parser.sample!
+      end
+            
+      self
+    end
+
+    # To be called from Parsers, send a attributes, get a record
+    def process_entry( attrs )
+      options[:before].call(attrs) if options[:before]
+            
+      record = finder ? finder.call(attrs) : nil
+
+      if record && record.class.ancestors.count{|r| r.to_s =~ /ActiveModel/} > 0            
+        process_record(attrs,record)
+        options[:after].call(record) if options[:after]
+        record
+      else
+        Ingestor::LOG.warn("Processing skipped, ActiveModel type record not returned for #{attrs}")
+      end
+    end
+
+    def process_record(attrs,record)
+      options[:processor] ? options[:processor].call(attrs, record) : default_processor(attrs, record)
+    end
+
+    def default_processor(attrs,record)
+      record.update_attributes( attrs, without_protection: true )
+    end
+
+    def load_remote
+      Ingestor::LOG.debug("Remote file detected #{file}...")
+      @document = Tempfile.new("local", working_directory)
+      @document.binmode if compressed?
+
+      open( file, 'rb' ) do |remote_file|
+        Ingestor::LOG.debug("Downloading #{file}...")
+        @document.write remote_file.read
+        @document.rewind
+      end
+    end
+
+    # When loading compressed files the assumption is that if there is more than one
+    # that the files are chunked, they will be put together and treated as one large file
+    def load_compressed
+      Ingestor::LOG.debug("Compressed file detected #{file}...")
+      @tempfile     = @document
+      @document = Tempfile.new("decompressed", working_directory)
+      @document.binmode
+      
+      Zip::ZipFile.open(@tempfile.path) do |zipfile|
+        zipfile.each do |entry|
+          istream = entry.get_input_stream
+          @document.write istream.read
+        end
+      end
+      @document.rewind
+    end
+
+    def load
+      Dir.mkdir(working_directory, 0777) unless Dir.exists?(working_directory)
+
+      load_remote if remote?
+      load_compressed if compressed?
+
+      @document ||= File.new( file )
+    end
+  end
+end

data/lib/ingestor/tasks.rb

@@ -0,0 +1,15 @@
+# require this file to load the tasks
+require 'rake'
+
+# noop 
+=begin
+This is here as a start point for adding rake tasks that can be 'required' by another project
+Just add: require 'ingestor/tasks' to your Rakefile
+=end
+
+namespace :ingestor do
+  desc "Example task"
+  task :example do
+    puts "I'm a task"
+  end
+end

data/lib/ingestor/version.rb

@@ -0,0 +1,3 @@
+module Ingestor
+  VERSION = "0.1.1"
+end

data/samples/animals.csv

@@ -0,0 +1,7 @@
+common name, domesticated, family, subfamily, genus
+chicken, true, Phasianidae, Phasianidae, Gallus
+brown bear, false, Ursidae,,Ursus
+cow, true, Bovidae, Bovinae, Bos
+dog, true, Canidae,,Canis
+cat,true, Felidae,,Felis
+platypus,false,Ornithorhynchidae,,Ornithorhynchus