GFunk911-dataload 0.9.1 → 0.9.2

data/VERSION.yml

@@ -1,4 +1,4 @@
 --- 
-:patch: 1
+:patch: 2
 :major: 0
 :minor: 9

data/lib/dataload.rb

@@ -22,13 +22,20 @@ class Dataload
 end
 
 class DataloadLogger
+  fattr(:first) { true }
   def log(str)
-    File.append(filename,"#{Time.now.short_dt} #{str}\n")
+    File.create(current_filename,"") if first
+    File.append(timestamp_filename,"#{Time.now.short_dt} #{str}\n")
+    File.append(current_filename,"#{Time.now.short_dt} #{str}\n")
+    self.first = false
   end
-  fattr(:filename) do
+  fattr(:timestamp_filename) do
     t = Time.now.strftime("%Y%m%d%H%M%S")
     res = File.expand_path("dataload_#{t}.log")
     puts "Logging to #{res}"
     res
   end
+  fattr(:current_filename) do
+    File.expand_path("dataload.log")
+  end
 end

data/lib/dataload/batch_insert.rb

@@ -1,6 +1,6 @@
 class StandardBatchInsert
   include FromHash
-  attr_accessor :rows, :table_name
+  attr_accessor :rows, :table_name, :ar_class, :block_size
   fattr(:column_names) { rows.first.sorted_column_names }
   fattr(:values_sql) do
     "VALUES " + rows.map { |x| x.insert_values_sql }.join(", ")
@@ -15,6 +15,28 @@ class StandardBatchInsert
     ActiveRecord::Base.connection.execute(insert_sql) 
   end
 end
+
+class SqlServerBatchInsert < StandardBatchInsert
+  fattr(:load_file_str) do
+    cols = ar_class.columns.map { |x| x.name }
+    rows.map do |row|
+      #raise cols.inspect + "|" + row.inspect
+      cols.map { |c| row[c.to_sym] }.join("|")
+    end.join("______")
+  end
+  fattr(:load_file_path) { File.expand_path("temp_load_file.txt") }
+  def write_load_file!
+    File.create(load_file_path,load_file_str)
+    Dataload.log "Temp file for bulk insert created at #{load_file_path}"
+  end
+  def insert_sql
+    "BULK INSERT #{table_name} FROM '#{load_file_path}' WITH (FIELDTERMINATOR = '|', ROWTERMINATOR = '______', BATCHSIZE = #{block_size})"
+  end
+  def insert!
+    write_load_file!
+    ActiveRecord::Base.connection.execute(insert_sql)
+  end
+end
     
 class OracleBatchInsert < StandardBatchInsert
   fattr(:insert_sql) do
@@ -28,10 +50,14 @@ end
 
 class BatchInsert
   def self.get_class
-    if %w(oci oci8 oracle).include?(MasterLoader.instance.db_ops[:adapter])
+    adapter = MasterLoader.instance.db_ops[:adapter].to_s
+    if %w(oci oci8 oracle).include?(adapter)
       OracleBatchInsert
+    elsif adapter == 'sqlserver'
+      SqlServerBatchInsert
     else
       StandardBatchInsert
     end
   end
-end
+end
+

data/lib/dataload/master_loader.rb

@@ -29,6 +29,11 @@ class MasterLoader
     end
   end
   def connect!
+    if db_ops[:adapter].to_s == 'sqlserver'
+      gem 'activerecord-sqlserver-adapter'
+      require 'active_record/connection_adapters/sqlserver_adapter'
+      require File.dirname(__FILE__) + "/sqlserver_ext"
+    end
     ActiveRecord::Base.establish_connection(db_ops)
     Dataload.log "Established Connection"
   end

data/lib/dataload/sqlserver_ext.rb

@@ -0,0 +1,40 @@
+#Dir["C:/Ruby/lib/ruby/gems/1.8/gems/activerecord-sqlserver-adapter*/**/*.rb"].each { |x| puts x; require x }
+
+module ActiveRecord
+  class Base
+    class << self
+      alias_method :orig_sqlserver_connection, :sqlserver_connection
+      #puts "redeffing sqlserver_connection"
+      def sqlserver_connection(config) #:nodoc:
+        #puts "in redeffed sqlserver connection"
+        require_library_or_gem 'dbi' unless self.class.const_defined?(:DBI)
+        
+        config = config.symbolize_keys
+
+        mode        = config[:mode] ? config[:mode].to_s.upcase : 'ADO'
+        username    = config[:username] ? config[:username].to_s : 'sa'
+        password    = config[:password] ? config[:password].to_s : ''
+        autocommit  = config.key?(:autocommit) ? config[:autocommit] : true
+        if mode == "ODBC"
+          raise ArgumentError, "Missing DSN. Argument ':dsn' must be set in order for this adapter to work." unless config.has_key?(:dsn)
+          dsn       = config[:dsn]
+          driver_url = "DBI:ODBC:#{dsn}"
+        else
+          raise ArgumentError, "Missing Database. Argument ':database' must be set in order for this adapter to work." unless config.has_key?(:database)
+          database  = config[:database]
+          host      = config[:host] ? config[:host].to_s : 'localhost'
+          if username =~ /PCI/
+            driver_url = "DBI:ADO:Provider=SQLOLEDB;Data Source=#{host};Initial Catalog=#{database};Integrated Security=SSPI;"
+          else
+            driver_url = "DBI:ADO:Provider=SQLOLEDB;Data Source=#{host};Initial Catalog=#{database};User ID=#{username};Password=#{password};"
+          end
+        end
+        #puts "DriverURL: #{driver_url}"
+        conn      = DBI.connect(driver_url, username, password)
+        conn["AutoCommit"] = autocommit
+        ConnectionAdapters::SQLServerAdapter.new(conn, logger, [driver_url, username, password])
+      end
+    end
+  end
+end
+    

data/lib/dataload/table_loader.rb

@@ -23,10 +23,20 @@ class TableLoader
   attr_accessor_nn :source_filename
   fattr(:delimiter) { "," }
   fattr(:block_size) { 1000 }
+  def master
+    MasterLoader.instance
+  end
+  def effective_block_size
+    if master.db_ops[:adapter].to_s == 'sqlserver' 
+      100000
+    else
+      block_size
+    end
+  end
   fattr(:columns) { [] }
   fattr(:source_row_groups) do
     e = enum(FasterCSV,:foreach,source_filename,:headers => true, :col_sep => delimiter)
-    enum(e,:each_by,block_size)
+    enum(e,:each_by,effective_block_size)
   end
   def target_hash_for_row(row)
     columns.inject({}) { |h,col| h.merge(col.target_name => col.target_value(row)) }
@@ -36,7 +46,7 @@ class TableLoader
   end
   def target_hash_groups
     source_row_groups.each_with_index do |rows,i|
-      yield(target_hashes(rows),i*block_size+rows.size)
+      yield(target_hashes(rows),i*effective_block_size+rows.size)
     end
   end
   def load!
@@ -44,8 +54,8 @@ class TableLoader
     Dataload.log "Starting load of table '#{table_name}'"
     total = 0
     target_hash_groups do |hs,num_inserted|
-      BatchInsert.get_class.new(:rows => hs, :table_name => table_name).insert!
-      Dataload.log "Inserted #{block_size} rows into table '#{table_name}'.  Total of #{num_inserted} rows inserted."
+      BatchInsert.get_class.new(:rows => hs, :table_name => table_name, :ar_class => ar_cls, :block_size => block_size).insert!
+      Dataload.log "Inserted #{num_inserted - total} rows into table '#{table_name}'.  Total of #{num_inserted} rows inserted."
       total = num_inserted
     end
     Dataload.log "Finished load of table '#{table_name}'.  Loaded #{total} rows."

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: GFunk911-dataload
 version: !ruby/object:Gem::Version 
-  version: 0.9.1
+  version: 0.9.2
 platform: ruby
 authors: 
 - Mike Harris
@@ -61,6 +61,7 @@ files:
 - lib/dataload/ext/faster_csv.rb
 - lib/dataload/master_loader.rb
 - lib/dataload/migration.rb
+- lib/dataload/sqlserver_ext.rb
 - lib/dataload/table_loader.rb
 - lib/dataload/table_manager.rb
 - lib/dataload/table_module.rb
@@ -73,6 +74,7 @@ files:
 - LICENSE
 - README.rdoc
 - VERSION.yml
+- tmp
 has_rdoc: true
 homepage: http://github.com/GFunk911/dataload
 post_install_message: