GFunk911-dataload 0.3.4 → 0.8.0

data/VERSION.yml

@@ -1,4 +1,4 @@
 --- 
-:patch: 4
+:patch: 0
 :major: 0
-:minor: 3
+:minor: 8

data/lib/dataload/batch_insert.rb

@@ -0,0 +1,18 @@
+class BatchInsert
+  include FromHash
+  attr_accessor :rows, :table_name
+  fattr(:column_names) { rows.first.sorted_column_names }
+  fattr(:values_sql) do
+    "VALUES " + rows.map { |x| x.insert_values_sql }.join(", ")
+  end
+  fattr(:columns_sql) do
+    "(" + column_names.join(", ") + ")"
+  end
+  fattr(:insert_sql) do
+    "INSERT into #{table_name} #{columns_sql} #{values_sql};"
+  end
+  def insert!
+    ActiveRecord::Base.connection.execute(insert_sql) 
+  end
+end
+    

data/lib/dataload/column.rb

@@ -0,0 +1,7 @@
+class Column
+  include FromHash
+  attr_accessor :target_name, :blk
+  def target_value(row)
+    row.instance_eval(&blk)
+  end
+end 

data/lib/dataload/dsl/master_loader_dsl.rb

@@ -0,0 +1,28 @@
+class MasterLoaderDSL
+  fattr(:master) { MasterLoader.instance }
+  def initialize(&b)
+    @blk = b
+    instance_eval(&b)
+  end
+  def database(ops)
+    master.db_ops = ops
+  end
+  def load_order(*tables)
+    master.raw_table_load_order = tables.flatten
+  end
+  def delete_order(*tables)
+    master.raw_table_delete_order = tables.flatten
+  end
+  def block_size(n)
+    master.block_size = n
+  end
+  def run!
+    master.run!
+  end
+end
+
+def master_dataload(&b)
+  handle_errors do
+    MasterLoaderDSL.new(&b).run!
+  end
+end

data/lib/dataload/dsl/table_loader_dsl.rb

@@ -0,0 +1,52 @@
+class Both
+  include FromHash
+  fattr(:objs) { [] }
+  def method_missing(sym,*args,&b)
+    objs.each { |x| x.send(sym,*args,&b) }
+  end
+end
+
+class TableLoaderDSL
+  fattr(:table_name) { loader.table_name }
+  fattr(:loader) { TableLoader.new }
+  fattr(:manager) { TableManager.new }
+  fattr(:both) { Both.new(:objs => [loader,manager]) }
+  def master
+    MasterLoader.instance
+  end
+  def initialize(&b)
+    @blk = b
+    instance_eval(&@blk)
+    master.add(self)
+  end
+  def column(name,type,&blk)
+    blk ||= lambda { |x| x.send(name) }
+    loader.columns << Column.new(:target_name => name, :blk => blk)
+  end
+  def method_missing(sym,*args,&b)
+    if [:string, :text, :integer, :float, :decimal, :datetime, :timestamp, :time, :date, :binary, :boolean].include?(sym)
+      column(args.first,sym,&b)
+    else
+      super(sym,*args,&b)
+    end
+  end
+  def source(file)
+    loader.source_filename = file
+  end
+  def table(name)
+    both.table_name = name
+  end
+  def run!
+    manager.delete_rows! if @delete_existing_rows
+    loader.load!
+  end
+  def delimiter(x)
+    loader.delimiter = x
+  end
+end
+
+def table_dataload(&b)
+  handle_errors do
+    TableLoaderDSL.new(&b)
+  end
+end

data/lib/dataload/ext/active_record.rb

@@ -0,0 +1,9 @@
+class Hash
+  def sorted_column_names
+    keys.sort_by { |x| x.to_s }
+  end
+  def insert_values_sql
+    res = sorted_column_names.map { |x| "'#{self[x]}'" }.join(",")
+    "(#{res})"
+  end
+end

data/lib/dataload/ext/enumerator.rb

@@ -0,0 +1,3 @@
+def enum(*args,&b)
+  Enumerable::Enumerator.new(*args,&b)
+end

data/lib/dataload/ext/faster_csv.rb

@@ -0,0 +1,25 @@
+class FasterCSV::Row
+  def method_missing(sym,*args,&b)
+    if self[sym.to_s]
+      self[sym.to_s].safe_to_num
+    else
+      super(sym,*args,&b)
+    end
+  end
+end
+
+class FasterCSV
+  def self.each(*args,&b)
+    foreach(*args,&b)
+  end
+end
+
+class Object
+  def safe_to_num
+    if self =~ /^\d+$/
+      to_i
+    else
+      self
+    end
+  end
+end

data/lib/dataload/master_loader.rb

@@ -0,0 +1,35 @@
+class MasterLoader
+  include Singleton
+  attr_accessor_nn :raw_table_load_order, :db_ops, :block_size
+  fattr(:raw_table_delete_order) { raw_table_load_order.reverse }
+  fattr(:tables_in_load_order) do
+    raw_table_load_order.map { |x| table_hash[x.to_s] }
+  end
+  fattr(:tables_in_delete_order) do
+    raw_table_delete_order.map { |x| table_hash[x.to_s] }
+  end
+  fattr(:table_hash) { {} }
+  def add(tl)
+    self.table_hash[tl.table_name.to_s] = tl
+  end
+  def delete_rows!
+    tables_in_delete_order.each { |t| t.manager.delete_rows! }
+  end
+  def load_rows!
+    tables_in_load_order.each { |t| t.loader.load! }
+  end
+  def run!
+    tables_in_load_order.each do |t| 
+      t.loader.block_size = block_size 
+    end
+    tm("MasterLoader run") do
+      connect!
+      delete_rows!
+      load_rows!
+    end
+  end
+  def connect!
+    ActiveRecord::Base.establish_connection(db_ops)
+    Dataload.log "Established Connection"
+  end
+end

data/lib/dataload/migration.rb

@@ -0,0 +1,18 @@
+class DataloadMigration < ActiveRecord::Migration
+  class << self
+    attr_accessor :cols, :table_name, :b
+    include FromHash
+  end
+  def self.new_migration(ops,&b)
+    cls = Class.new(DataloadMigration)
+    cls.from_hash(ops)
+    cls.b = b
+    cls.class_eval do
+      def self.up
+        instance_eval(&b)
+        Dataload.log "Created table #{table_name}"
+      end
+    end
+    cls
+  end
+end

data/lib/dataload/sample.rb

@@ -1,24 +1,56 @@
 require 'rubygems'
-require 'dataload'
+require File.dirname(__FILE__) + '/../dataload'
 
 #setup the sample source file
-source_filename = File.dirname(__FILE__) + "/sample_source.csv"
+source_filename = File.dirname(__FILE__) + "/../../tmp/sample_source.csv"
+db_path = File.dirname(__FILE__) + "/../../tmp/sample.sqlite3"
+
 source_text = <<EOF
-name,age,city,state
 Bob Smith,24,Atlanta,GA
 Jane Doe,35,Buffalo,NY
 Evan Stein,31,Princeton,NJ
 EOF
+source_text = "name,age,city,state\n" + (1..10000).map { source_text }.join
 File.create(source_filename,source_text)
 
 #load into a database, creating the table if needed
-dataload do
+table_dataload do
+  # csv file the data is being sourced from
   source source_filename
-  database :adapter => 'sqlite3', :database => "db.sqlite3", :timeout => 5000
+  
+  # database/table the data should be loaded into.
+  # the table will be created if it does not already exist
+  #database :adapter => 'sqlite3', :database => db_path, :timeout => 5000
+  #database :adapter => 'sqlserver', :host => '192.168.1.49', :username => 'pci-tae', :password => 'fgfgf', :database => 'fgfgfgf'
   table 'people'
+  
+  #field delimiter in source file
+  delimiter ","
+    
+  # columns in the new table
+  # available types are string, text, integer, float, decimal, datetime, timestamp, time, date, binary, boolean
+  #
+  # The first argument is the name of the new column
+  # The block describes the value to be populated
+  #
+  # Example: string(:full_name) { name } 
+  # This creates a field 'full_name' of type string in the new table, and populates it with the name field from the csv
+  #
+  # Example: boolean(:is_tall) { height_in_inches.to_i > 74 }
+  # Creates a field 'is_tall' and populates with true if the height_in_inches field in the csv is greater than 74
+  #
+  # A column without a block just passes through the same field in the csv
+  # integer(:age) creates an integer field 'age' in the new table, populated with the age field in the csv
   string(:full_name) { name }
   string(:first_name) { name.split[0] }
   string(:last_name) { name.split[1] }
   integer(:age)
   string(:city_state) { "#{city}, #{state}" }
+end
+
+master_dataload do
+  #database :adapter => 'sqlite3', :database => db_path, :timeout => 5000
+  database :adapter => 'mysql', :database => 'dataload_test', :username => 'root'
+  load_order :people
+  block_size 1000
 end

data/lib/dataload/table_loader.rb

@@ -0,0 +1,72 @@
+require 'rubygems'
+begin
+  require "/Code/mharris_ext/lib/mharris_ext"
+rescue
+  require 'mharris_ext'
+end
+require 'fastercsv'
+require 'activerecord'
+require 'facets/enumerable'
+
+%w(migration column table_module batch_insert).each { |x| require File.dirname(__FILE__) + "/#{x}" }
+Dir[File.dirname(__FILE__) + "/ext/*.rb"].each { |x| require x }
+
+class Object
+  def fattr_tm(name,&b)
+    fattr(name) do
+      tm(name) do
+        instance_eval(&b)
+      end
+    end
+  end
+end
+      
+class TableLoader
+  include TableModule
+  attr_accessor_nn :source_filename
+  fattr(:delimiter) { "," }
+  fattr(:block_size) { 1000 }
+  fattr(:columns) { [] }
+  fattr(:source_row_groups) do
+    e = enum(FasterCSV,:foreach,source_filename,:headers => true, :col_sep => delimiter)
+    enum(e,:each_by,block_size)
+  end
+  def target_hash_for_row(row)
+    columns.inject({}) { |h,col| h.merge(col.target_name => col.target_value(row)) }
+  end
+  def target_hashes(rows)
+    rows.map { |x| target_hash_for_row(x) }
+  end
+  def target_hash_groups
+    source_row_groups.each_with_index do |rows,i|
+      yield(target_hashes(rows),i*block_size+rows.size)
+    end
+  end
+  def load!
+    migrate!
+    Dataload.log "Starting load of table '#{table_name}'"
+    total = 0
+    target_hash_groups do |hs,num_inserted|
+      BatchInsert.new(:rows => hs, :table_name => table_name).insert!
+      Dataload.log "Inserted #{block_size} rows into table '#{table_name}'.  Total of #{num_inserted} rows inserted."
+      total = num_inserted
+    end
+    Dataload.log "Finished load of table '#{table_name}'.  Loaded #{total} rows."
+  end
+end
+
+module TableCreation
+  fattr(:migration) do
+    DataloadMigration.new_migration(:cols => columns, :table_name => table_name) do
+      create_table table_name do |t|
+        cols.each do |col|
+          t.column col.target_name, :string
+        end
+      end
+    end
+  end
+  def migrate!
+    migration.migrate(:up) unless ar_cls.table_exists?
+  end
+end
+TableLoader.send(:include,TableCreation)

data/lib/dataload/table_manager.rb

@@ -0,0 +1,9 @@
+class TableManager
+  include TableModule
+  def delete_rows!
+    return unless ar_cls.table_exists?
+    Dataload.log "Deleting #{ar_cls.count} rows from table '#{table_name}'"
+    ar_cls.connection.execute("DELETE from #{table_name}")
+    Dataload.log "Deleted rows from table '#{table_name}'"
+  end
+end

data/lib/dataload/table_module.rb

@@ -0,0 +1,7 @@
+module TableModule
+  attr_accessor_nn :table_name
+  fattr(:ar_cls) do
+    Class.new(ActiveRecord::Base).tap { |x| x.set_table_name(table_name) }
+  end
+end
+  

data/lib/dataload.rb

@@ -1 +1,34 @@
-require File.dirname(__FILE__) + "/dataload/loader"
+def handle_errors
+  yield
+rescue => exp
+  msg = [exp.message,exp.backtrace.join("\n")].join("\n")
+  Dataload.log msg
+  puts exp.message
+  raise "Error occured and logged.  Exiting."
+end
+
+require File.dirname(__FILE__) + "/dataload/table_loader"
+require File.dirname(__FILE__) + "/dataload/table_manager"
+require File.dirname(__FILE__) + "/dataload/master_loader"
+Dir[File.dirname(__FILE__) + "/dataload/dsl/*.rb"].each { |x| require x }
+
+class Dataload
+  class << self
+    fattr(:logger) { DataloadLogger.new }
+    def log(str)
+      logger.log(str)
+    end
+  end
+end
+
+class DataloadLogger
+  def log(str)
+    File.append(filename,"#{Time.now.short_dt} #{str}\n")
+  end
+  fattr(:filename) do
+    t = Time.now.strftime("%Y%m%d%H%M%S")
+    res = File.expand_path("dataload_#{t}.log")
+    puts "Logging to #{res}"
+    res
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: GFunk911-dataload
 version: !ruby/object:Gem::Version 
-  version: 0.3.4
+  version: 0.8.0
 platform: ruby
 authors: 
 - Mike Harris
@@ -52,8 +52,19 @@ extra_rdoc_files:
 - README.rdoc
 - LICENSE
 files: 
-- lib/dataload/loader.rb
+- lib/dataload/batch_insert.rb
+- lib/dataload/column.rb
+- lib/dataload/dsl/master_loader_dsl.rb
+- lib/dataload/dsl/table_loader_dsl.rb
+- lib/dataload/ext/active_record.rb
+- lib/dataload/ext/enumerator.rb
+- lib/dataload/ext/faster_csv.rb
+- lib/dataload/master_loader.rb
+- lib/dataload/migration.rb
 - lib/dataload/sample.rb
+- lib/dataload/table_loader.rb
+- lib/dataload/table_manager.rb
+- lib/dataload/table_module.rb
 - lib/dataload.rb
 - spec/dataload_spec.rb
 - spec/spec_helper.rb

data/lib/dataload/loader.rb

@@ -1,126 +0,0 @@
-require 'rubygems'
-require 'mharris_ext'
-require 'fastercsv'
-require 'activerecord'
-
-class FasterCSV::Row
-  def method_missing(sym,*args,&b)
-    if self[sym.to_s]
-      self[sym.to_s]
-    else
-      super(sym,*args,&b)
-    end
-  end
-end
-
-class Loader
-  fattr(:columns) { [] }
-  attr_accessor :source_filename, :db_ops, :table_name
-  fattr(:source_rows) do
-    res = []
-    FasterCSV.foreach(source_filename, :headers => true) do |row|
-      res << row
-    end
-    res
-  end
-  def target_hash_for_row(row)
-    h = {}
-    columns.each do |col|
-      h[col.target_name] = col.target_value(row)
-    end
-    h
-  end
-  def target_hashes
-    source_rows.map { |x| target_hash_for_row(x) }
-  end
-  def target_column_names
-    columns.map { |x| x.target_name }
-  end
-  def new_struct
-    Struct.new(*target_column_names)
-  end
-  fattr(:migration) do
-    raise "must define table" unless table_name
-    cls = Class.new(ActiveRecord::Migration)
-    class << cls
-      attr_accessor :cols, :table_name
-    end
-    cls.cols = columns
-    cls.table_name = table_name
-    puts "Table: #{table_name}"
-    cls.class_eval do
-      def self.up
-        create_table table_name do |t|
-          cols.each do |col|
-            t.column col.target_name, :string
-          end
-        end
-      end
-    end
-    cls
-  end
-  fattr(:ar) do
-    cls = Class.new(ActiveRecord::Base)
-    cls.send(:set_table_name, table_name)
-    cls
-  end
-  def migrate!
-    ar.find(:first)
-  rescue => exp
-    puts "find failed"
-    puts exp.inspect
-    migration.migrate(:up)
-  end
-  fattr(:ar_objects) do
-    target_hashes.map { |h| ar.new(h) }
-  end
-  def load!
-    ActiveRecord::Base.establish_connection(db_ops)
-    migrate!
-    ar_objects.each { |x| x.save! }
-  end
-end
-
-class Column
-  include FromHash
-  attr_accessor :target_name, :blk
-  def target_value(row)
-    if blk.arity == 1
-      blk.call(row)
-    else
-      row.instance_eval(&blk)
-    end
-  end
-end 
-
-class LoaderDSL
-  fattr(:loader) { Loader.new }
-  def column(name,type,&blk)
-    blk ||= lambda { |x| x.send(name) }
-    loader.columns << Column.new(:target_name => name, :blk => blk)
-  end
-  def method_missing(sym,*args,&b)
-    if [:string, :text, :integer, :float, :decimal, :datetime, :timestamp, :time, :date, :binary, :boolean].include?(sym)
-      column(args.first,sym,&b)
-    else
-      super(sym,*args,&b)
-    end
-  end
-  def source(file)
-    loader.source_filename = file
-  end
-  def database(ops)
-    loader.db_ops = ops
-  end
-  def table(name)
-    loader.table_name = name
-  end
-end
-
-def dataload(&b)
-  dsl = LoaderDSL.new
-  dsl.instance_eval(&b)
-  dsl.loader.load!
-  puts "Row Count: " + dsl.loader.ar.find(:all).size.to_s
-end
-