chicago-etl 0.0.9 → 0.0.10

data/VERSION

@@ -1 +1 @@
-0.0.9
+0.0.10

data/chicago-etl.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "chicago-etl"
-  s.version = "0.0.9"
+  s.version = "0.0.10"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Roland Swingler"]
-  s.date = "2013-02-19"
+  s.date = "2013-02-25"
   s.description = "ETL tools for Chicago"
   s.email = "roland.swingler@gmail.com"
   s.extra_rdoc_files = [
@@ -63,6 +63,7 @@ Gem::Specification.new do |s|
     "spec/etl/screens/out_of_bounds_spec.rb",
     "spec/etl/sequel/dependant_tables_spec.rb",
     "spec/etl/sequel/filter_to_etl_batch_spec.rb",
+    "spec/etl/sequel/load_data_infile_expression_spec.rb",
     "spec/etl/sequel/load_data_infile_spec.rb",
     "spec/etl/sink_spec.rb",
     "spec/etl/table_builder_spec.rb",

data/lib/chicago/etl/sequel/load_data_infile.rb

@@ -1,15 +1,137 @@
 module Chicago
   module ETL
     module SequelExtensions
+      # @api private
+      class LoadDataInfileExpression
+        attr_reader :path, :table, :columns, :ignore, :character_set
+
+        def initialize(path, table, columns, opts={})
+          @path    = path
+          @table   = table
+          @columns = columns
+          @ignore  = opts[:ignore]
+          @update  = opts[:update]
+          @set     = opts[:set] || {}
+          @character_set = opts[:character_set] || "utf8"
+          if opts[:format] == :csv
+            @field_terminator = ","
+            @enclosed_by = '"'
+            @escaped_by = '"'
+          end
+        end
+
+        def replace?
+          @update == :replace
+        end
+
+        def ignore?
+          @update == :ignore
+        end
+
+        def to_sql(db)
+          @db = db
+          [load_fragment,
+           replace_fragment,
+           table_fragment,
+           character_set_fragment,
+           field_terminator_fragment,
+           field_enclosure_fragment,
+           escape_fragment,
+           ignore_fragment, 
+           column_fragment,
+           set_fragment].compact.join(" ")
+        end
+
+        private
+
+        def load_fragment
+          "LOAD DATA INFILE '#{path}'"
+        end
+
+        def replace_fragment
+          @update.to_s.upcase if replace? || ignore?
+        end
+
+        def table_fragment
+          "INTO TABLE `#{table}`"
+        end
+
+        def character_set_fragment
+          "CHARACTER SET '#{character_set}'"
+        end
+
+        def field_terminator_fragment
+          "FIELDS TERMINATED BY '#{@field_terminator}'" if @field_terminator
+        end
+
+        def field_enclosure_fragment
+          "OPTIONALLY ENCLOSED BY '#{@enclosed_by}'" if @enclosed_by
+        end
+
+        def escape_fragment
+          "ESCAPED BY '#{@escaped_by}'" if @escaped_by
+        end
+
+        def ignore_fragment
+          "IGNORE #{ignore} LINES" if ignore
+        end
+
+        def column_fragment
+          "(" + columns.map {|c| format_column(c) }.join(",") + ")"
+        end
+
+        def set_fragment
+          unless @set.empty?
+            "SET " + @set.map do |k, v|
+              "#{@db.literal(k)} = #{@db.literal(v)}"
+            end.join(", ")
+          end
+        end
+
+        def format_column(column)
+          column.to_s[0] == "@" ? column : "`#{column}`"
+        end
+      end
+
       module LoadDataInfile
-        # Loads the CSV data columns in filepath into this dataset's table.
-        def load_csv_infile(filepath, columns)
-          execute_dui(load_csv_infile_sql(filepath, columns))
+        # Load data in file specified at path.
+        #
+        # Columns is a list of columns to load - column names starting
+        # with an @ symbol will be treated as variables.
+        #
+        # By default, this will generate a REPLACE INTO TABLE
+        # statement.
+        #
+        # Options:
+        # :ignore - the number of lines to ignore in the source file
+        # :update - nil, :ignore or :replace
+        # :set - a hash specifying autopopulation of columns
+        # :character_set - the character set of the file, UTF8 default
+        # :format - either nil or :csv
+        def load_infile(path, columns, options={})
+          execute_dui(load_infile_sql(filepath, columns, options))
+        end
+
+        def load_infile_sql(path, columns, options={})
+          replacement = opts[:insert_ignore] ? :ignore : :replace
+          options = {:update => replacement}.merge(options)
+          LoadDataInfileExpression.new(path, 
+                                       opts[:from].first, 
+                                       columns, 
+                                       options).
+            to_sql(db)
+        end
+
+        # Loads the CSV data columns in path into this dataset's
+        # table.
+        #
+        # See load_infile for more options.
+        def load_csv_infile(path, columns, options={})
+          execute_dui(load_csv_infile_sql(filepath, columns, options))
         end
         
-        def load_csv_infile_sql(filepath, columns)
-          replacement = opts[:insert_ignore] ? "IGNORE" : "REPLACE"
-          "LOAD DATA INFILE '#{filepath}' #{replacement} INTO TABLE `#{opts[:from].first}` CHARACTER SET 'utf8' FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"' ESCAPED BY '\"' (`#{columns.join('`,`')}`);"
+        def load_csv_infile_sql(path, columns, options={})
+          load_infile_sql(path, columns, options.merge(:format => :csv))
         end
       end
     end

data/spec/etl/sequel/load_data_infile_expression_spec.rb

@@ -0,0 +1,56 @@
+require 'spec_helper'
+
+describe Chicago::ETL::SequelExtensions::LoadDataInfileExpression do
+  it "loads the data in the file into the table" do
+    described_class.new("bar.csv", :foo, ['bar', 'quux']).
+      to_sql(TEST_DB).should include("LOAD DATA INFILE 'bar.csv' INTO TABLE `foo`")
+  end
+
+  it "loads the data with replacment" do
+    described_class.new("bar.csv", :foo, ['bar', 'quux'],
+                        :update => :replace).
+      to_sql(TEST_DB).should include("REPLACE INTO TABLE")
+  end
+
+  it "loads the data ignoring rows" do
+    described_class.new("bar.csv", :foo, ['bar', 'quux'], :update => :ignore).
+      to_sql(TEST_DB).should include("IGNORE INTO TABLE")
+  end
+
+  it "should be in UTF-8 character set by default" do
+    described_class.new("bar.csv", :foo, ['bar', 'quux']).
+      to_sql(TEST_DB).should include("CHARACTER SET 'utf8'")
+  end
+
+  it "may be in other character sets" do
+    described_class.new("bar.csv", :foo, ['bar', 'quux'], :character_set => "ascii").
+      to_sql(TEST_DB).should include("CHARACTER SET 'ascii'")
+  end
+
+  it "should load columns" do
+    described_class.new("bar.csv", :foo, ['bar', 'quux']).
+      to_sql(TEST_DB).should include("(`bar`,`quux`)")
+  end
+
+  it "should load into variables if column begins with @" do
+    described_class.new("bar.csv", :foo, ['@bar', 'quux']).
+      to_sql(TEST_DB).should include("(@bar,`quux`)")
+  end
+
+  it "can ignore lines" do
+    described_class.new("bar.csv", :foo, ['bar', 'quux'], :ignore => 2).
+      to_sql(TEST_DB).should include("IGNORE 2 LINES")
+  end
+
+  it "can be in csv format" do
+    described_class.new("bar.csv", :foo, ['bar', 'quux'], :format => :csv).
+      to_sql(TEST_DB).should include("FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"' ESCAPED BY '\"'")
+  end
+
+  it "can set column values" do
+    described_class.new("bar.csv", :foo, ['@bar', 'quux'], 
+                        :set => {:bar => :unhex.sql_function("@bar".lit),
+                        :etl_batch_id => 3}).
+      to_sql(TEST_DB).should include("SET `bar` = unhex(@bar), `etl_batch_id` = 3")
+  end
+end

data/spec/etl/sequel/load_data_infile_spec.rb

@@ -26,7 +26,7 @@ describe Chicago::ETL::SequelExtensions::LoadDataInfile do
   end
 
   it "loads into the columns specified" do
-    @sql.should include("(`bar`,`baz`);")
+    @sql.should include("(`bar`,`baz`)")
   end
 
   it "can ignore instead of replacing rows" do

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: chicago-etl
 version: !ruby/object:Gem::Version 
-  hash: 13
+  hash: 11
   prerelease: 
   segments: 
   - 0
   - 0
-  - 9
-  version: 0.0.9
+  - 10
+  version: 0.0.10
 platform: ruby
 authors: 
 - Roland Swingler
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2013-02-19 00:00:00 Z
+date: 2013-02-25 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   version_requirements: &id001 !ruby/object:Gem::Requirement 
@@ -200,6 +200,7 @@ files:
 - spec/etl/screens/out_of_bounds_spec.rb
 - spec/etl/sequel/dependant_tables_spec.rb
 - spec/etl/sequel/filter_to_etl_batch_spec.rb
+- spec/etl/sequel/load_data_infile_expression_spec.rb
 - spec/etl/sequel/load_data_infile_spec.rb
 - spec/etl/sink_spec.rb
 - spec/etl/table_builder_spec.rb