jsuchal-activerecord-fast-import 0.1.0 → 0.1.1

data/README.markdown

@@ -0,0 +1,66 @@
+# activerecord-fast-import
+
+Loads data from text files into tables using fast native MySQL [LOAD DATA INFILE](http://dev.mysql.com/doc/refman/5.1/en/load-data.html) query.
+
+## Examples
+
+### Loading data from tab delimited log file
+
+Suppose you have an ActiveRecord model LogEntry defined as `LogEntry happened_at:datetime url:string` and a log file with tab delimited columns like this:
+
+    2009-09-30 12:32:43<tab>http://github.com/
+    2009-09-30 13:36:13<tab>http://facebook.com/
+
+To import data from this log file, you have to use
+
+    LogEntry.fast_import('huge.log')
+
+That's it! 
+
+Of course in real world you will also need more advanced features. Read on...
+
+
+### Changing delimiters and ignoring some rows
+
+Of course not all log files are delimited by tabs and newlines. Just pass custom delimiters to options. If you want to ignore first 10 lines, just use `:ignore_lines`
+
+    import_options = {
+        :fields_terminated_by => ',',
+        :lines_terminated_by => ';',
+        :ignore_lines => 10
+    }
+
+### Changing order of columns and ignoring columns
+
+Now, imagine you want to import data from a huge log file with following format:
+
+    http://github.com/<tab>Mozilla<tab>2009-09-30 12:32:43
+    http://facebook.com/<tab>Opera<tab>2009-09-30 13:36:13
+
+It is clear that columns are in different order and we even want to ignore the second column. Let's do it
+
+    import_options = {:columns => ["url", "@dummy", "happened_at"]}
+    LogEntry.fast_import('huge.log', import_options)
+
+The special `@dummy` loads that column into a local variable and when unused (in a transformation) is just ignored.
+
+### Transforming data
+
+Now imagine we have a log file like this:
+
+    2009-09-30 12:32:43<tab>http://github.com/<tab>image.jpg
+    2009-09-30 13:36:13<tab>http://facebook.com/<tab>styles/default.css
+
+We want to concatenate those two columns into one.
+
+    import_options = {
+        :columns => ["happened_at", "@domain", "@file"],
+        :mapping => { :url => "CONCAT(@domain, @file)" }
+    }
+    LogEntry.fast_import('huge.log', import_options)
+
+Of course you can use any of those shiny [MySQL functions](http://dev.mysql.com/doc/refman/5.1/en/functions.html).
+
+## Copyright
+
+Copyright (c) 2009 Jan Suchal. See LICENSE for details.

data/VERSION

@@ -1 +1 @@
-0.1.0
+0.1.1

data/activerecord-fast-import.gemspec

@@ -5,7 +5,7 @@
 
 Gem::Specification.new do |s|
   s.name = %q{activerecord-fast-import}
-  s.version = "0.1.0"
+  s.version = "0.1.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Jan Suchal"]
@@ -14,13 +14,13 @@ Gem::Specification.new do |s|
   s.email = %q{johno@jsmf.net}
   s.extra_rdoc_files = [
     "LICENSE",
-     "README.rdoc"
+     "README.markdown"
   ]
   s.files = [
     ".document",
      ".gitignore",
      "LICENSE",
-     "README.rdoc",
+     "README.markdown",
      "Rakefile",
      "VERSION",
      "activerecord-fast-import.gemspec",

data/lib/activerecord-fast-import.rb

@@ -1,53 +1,53 @@
-module ActiveRecord
-  class Base
-    # Deletes all rows in table very fast, but without calling +destroy+ method
-    # nor any hooks.
-    def self.truncate_table
-      connection.execute("TRUNCATE TABLE #{quoted_table_name}")
-    end
-
-    # Disables key updates for model table
-    def self.disable_keys
-      connection.execute("ALTER TABLE #{quoted_table_name} DISABLE KEYS")
-    end
-
-    # Enables key updates for model table
-    def self.enable_keys
-      connection.execute("ALTER TABLE #{quoted_table_name} ENABLE KEYS")
-    end
-
-    # Loads data from file using MySQL native LOAD DATA INFILE query, disabling
-    # key updates for even faster import speed
-    #
-    # ==== Parameters
-    # * +file+ the file to load
-    # * +options+ (see <tt>load_data_infile?</tt>)
-    def self.fast_import(file, options = {})
-      disable_keys
-      load_data_infile(file, options)
-      enable_keys
-    end
-
-    # Loads data from file using MySQL native LOAD DATA INFILE query
-    #
-    # ==== Parameters
-    # * +file+ the file to import
-    # * +options+
-    def self.load_data_infile(file, options = {})
-      sql = "LOAD DATA LOCAL INFILE '#{file}' INTO TABLE #{quoted_table_name} "
-      sql << "FIELDS TERMINATED BY '#{options[:fields_terminated_by]}' " if options[:fields_terminated_by]
-      sql << "LINES TERMINATED BY '#{options[:lines_terminated_by]}' " if options[:lines_terminated_by]
-      sql << "IGNORE #{options[:ignore_lines]} LINES " if options[:ignore_lines]
-      sql << "(" + options[:columns].join(', ') + ") " if options[:columns]
-      if options[:mapping]
-        mappings = []
-        options[:mapping].each_pair do |column, mapping|
-          mappings << "#{column} = #{mapping}"
-        end
-        sql << "SET #{mappings.join(', ')} " if mappings.size > 0
-      end
-      sql << ";"
-      connection.execute(sql)
-    end
-  end
+module ActiveRecord #:nodoc:
+  class Base
+    # Deletes all rows in table very fast, but without calling +destroy+ method
+    # nor any hooks.
+    def self.truncate_table
+      connection.execute("TRUNCATE TABLE #{quoted_table_name}")
+    end
+
+    # Disables key updates for model table
+    def self.disable_keys
+      connection.execute("ALTER TABLE #{quoted_table_name} DISABLE KEYS")
+    end
+
+    # Enables key updates for model table
+    def self.enable_keys
+      connection.execute("ALTER TABLE #{quoted_table_name} ENABLE KEYS")
+    end
+
+    # Loads data from file using MySQL native LOAD DATA INFILE query, disabling
+    # key updates for even faster import speed
+    #
+    # ==== Parameters
+    # * +file+ the file to load
+    # * +options+ (see <tt>load_data_infile</tt>)
+    def self.fast_import(file, options = {})
+      disable_keys
+      load_data_infile(file, options)
+      enable_keys
+    end
+
+    # Loads data from file using MySQL native LOAD DATA INFILE query
+    #
+    # ==== Parameters
+    # * +file+ the file to import
+    # * +options+
+    def self.load_data_infile(file, options = {})
+      sql = "LOAD DATA LOCAL INFILE '#{file}' INTO TABLE #{quoted_table_name} "
+      sql << "FIELDS TERMINATED BY '#{options[:fields_terminated_by]}' " if options[:fields_terminated_by]
+      sql << "LINES TERMINATED BY '#{options[:lines_terminated_by]}' " if options[:lines_terminated_by]
+      sql << "IGNORE #{options[:ignore_lines]} LINES " if options[:ignore_lines]
+      sql << "(" + options[:columns].join(', ') + ") " if options[:columns]
+      if options[:mapping]
+        mappings = []
+        options[:mapping].each_pair do |column, mapping|
+          mappings << "#{column} = #{mapping}"
+        end
+        sql << "SET #{mappings.join(', ')} " if mappings.size > 0
+      end
+      sql << ";"
+      connection.execute(sql)
+    end
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: jsuchal-activerecord-fast-import
 version: !ruby/object:Gem::Version 
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors: 
 - Jan Suchal
@@ -40,12 +40,12 @@ extensions: []
 
 extra_rdoc_files: 
 - LICENSE
-- README.rdoc
+- README.markdown
 files: 
 - .document
 - .gitignore
 - LICENSE
-- README.rdoc
+- README.markdown
 - Rakefile
 - VERSION
 - activerecord-fast-import.gemspec

data/README.rdoc

@@ -1,5 +0,0 @@
-= activerecord-fast-import
-
-== Copyright
-
-Copyright (c) 2009 Jan Suchal. See LICENSE for details.