usda-nutrient-database 2.0.0 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 55eaf87a81739b7b4da9d592c67eca5fcd857801
-  data.tar.gz: e42ac18382b777910c4cbb6ca0546a48aaa3a48f
+  metadata.gz: 35f7dffdf433c3c075779ed74ffc85775784d873
+  data.tar.gz: 1290cbc6ab3f2abedf992dd1605e44132fe308a0
 SHA512:
-  metadata.gz: 616a437099608f3b36179de590f12b81b97e1996e926a8d83b75118175508e51147db803560792a8b572f69252be0352b7c23e5e6f338fd30136c73009a330f3
-  data.tar.gz: dedcb6ef862b1034fa15086f76e8059ff689a1c6f06f626392cc0790761541c22bdec009a922034e8dd184ccda6abb1b36117689242aa293fbdddac219448b85
+  metadata.gz: 0e2db9e3b9a216a9d0c3f1bebc8f0f9a642f1c9d7bbe0c970a70a2a6481159b14e465f39d1861966eed9e7420c909519e7b8b08e24f4b1fd87b70c2d2bb5f5be
+  data.tar.gz: f9319fd8b497e4df8b827d51bf2a69b0ffafb37b6e41ac24ac239dc540828830791bfc8edbd07b1fceeca610d91a826b053abd727c2541a98f9b4a7c67a7ac92

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## 2.1.0
+
+- Made it work with `activerecord-import` gem to speed up import times by 30x
+  [#16](https://github.com/mattbeedle/usda-nutrient-database/pull/16)
+
 ## 2.0.0
 
 - Removed IDs from tables which don't have them in USDA data. Instead opted to

data/README.md

@@ -47,8 +47,43 @@ If you're using rails then copy the migrations across:
 rake usda_nutrient_database_engine:install:migrations
 ```
 
+## Configuration
+
+```ruby
+UsdaNutrientDatabase.configure do |config|
+  config.batch_size = 20000 # import batch size, if using activerecord-import
+  config.perform_logging = true # default false
+  config.logger = Rails.logger # default Logger.new(STDOUT)
+  config.usda_version = 'sr25' # default sr28
+end
+```
+
 ## Usage
 
+### Importing with UPSERT (takes around 2 minutes)
+
+If you are running MySQL => 5.6 or PostgreSQL => 9.5 then you're in luck, you
+can use UPSERT (insert or update) to speed up imports x30. To do this you're
+going to need to install
+[activerecord-import](https://github.com/zdennis/activerecord-import)
+
+```ruby
+require 'activerecord-import/base'
+ActiveRecord::Import.require_adapter('postgresql')
+
+# You may want to disable logging during this process to avoid dumping huge SQL
+# strings in to your logs
+ActiveRecord::Base.logger = Logger.new('/dev/null')
+```
+
+Now run the rake task which will import everything in around 2 minutes.
+```
+rake usda:import
+```
+
+
+### Importing without UPSERT (takes 60+ minutes)
+
 Import the latest data with the import task:
 ```
 rake usda:import
@@ -60,6 +95,8 @@ tasks:
 rake -T usda
 ```
 
+### Models
+
 Use the models to query and profit:
 ```
 UsdaNutrientDatabase::FoodGroup

data/lib/usda-nutrient-database.rb

@@ -32,10 +32,18 @@ module UsdaNutrientDatabase
       end
     end
 
+    def activerecord_import?
+      defined?(ActiveRecord::Import)
+    end
+
     def configuration
       @configuration ||= UsdaNutrientDatabase::Configuration.new
     end
 
+    def batch_size
+      @batch_fize ||= configuration.batch_size
+    end
+
     def usda_version
       @usda_version ||= configuration.usda_version
     end

data/lib/usda_nutrient_database/configuration.rb

@@ -1,9 +1,14 @@
 module UsdaNutrientDatabase
   class Configuration
     attr_accessor :logger
-    attr_writer   :perform_logging,
+    attr_writer :batch_size,
+      :perform_logging,
       :usda_version
 
+    def batch_size
+      @batch_size ||= 10000
+    end
+
     def logger
       @logger ||= Logger.new(STDOUT)
     end

data/lib/usda_nutrient_database/import/base.rb

@@ -5,24 +5,35 @@ module UsdaNutrientDatabase
     class Base
       def initialize(directory)
         @directory = directory
+        @objects_to_import = []
       end
 
       def import
         log_import_started
         CSV.open(file_location, 'r:iso-8859-1:utf-8', csv_options) do |csv|
-          csv.each { |row| extract_row(row) }
+          csv.each { |row| objects_to_import << extract_row(row) }
         end
+        ar_import? ? save_objects : objects_to_import.each(&:save)
       end
 
       private
 
-      attr_reader :directory
+      attr_reader :directory,
+        :objects_to_import
+
+      def ar_import?
+        UsdaNutrientDatabase.activerecord_import?
+      end
 
       def extract_row(row)
-        build_object(apply_typecasts(row)).save
+        build_object(apply_typecasts(row))
       end
 
       def build_object(row)
+        ar_import? ? build_values_array(row) : build_ar_object(row)
+      end
+
+      def build_ar_object(row)
         find_or_initialize(row).tap do |object|
           columns.each_with_index do |column, index|
             object.send("#{column}=", row[index])
@@ -30,6 +41,10 @@ module UsdaNutrientDatabase
         end
       end
 
+      def build_values_array(row)
+        columns.map.with_index { |_, index| row[index] }
+      end
+
       def columns
         raise NotImplementedError
       end
@@ -53,6 +68,10 @@ module UsdaNutrientDatabase
       def csv_options
         { col_sep: '^', quote_char: '~' }
       end
+
+      def save_objects
+        raise NotImplementedError
+      end
     end
   end
 end

data/lib/usda_nutrient_database/import/food_groups.rb

@@ -19,6 +19,16 @@ module UsdaNutrientDatabase
       def log_import_started
         UsdaNutrientDatabase.log 'Importing food groups'
       end
+
+      def save_objects
+        UsdaNutrientDatabase::FoodGroup.import(columns, objects_to_import, {
+          validate: false,
+          on_duplicate_key_update: {
+            conflict_target: :code,
+            columns: %i(description)
+          }
+        })
+      end
     end
   end
 end

data/lib/usda_nutrient_database/import/foods.rb

@@ -32,6 +32,16 @@ module UsdaNutrientDatabase
           :protein_factor, :fat_factor, :carbohydrate_factor
         ]
       end
+
+      def save_objects
+        UsdaNutrientDatabase::Food.import(columns, objects_to_import, {
+          validate: false,
+          on_duplicate_key_update: {
+            conflict_target: :nutrient_databank_number,
+            columns: columns
+          }
+        })
+      end
     end
   end
 end

data/lib/usda_nutrient_database/import/foods_nutrients.rb

@@ -4,6 +4,11 @@ module UsdaNutrientDatabase
 
       private
 
+      def apply_typecasts(row)
+        row[8] = row[8] == 'Y'
+        row
+      end
+
       def columns
         [
           :nutrient_databank_number, :nutrient_number, :nutrient_value,
@@ -15,8 +20,9 @@ module UsdaNutrientDatabase
       end
 
       def find_or_initialize(row)
-        UsdaNutrientDatabase::FoodsNutrient.find_or_initialize_by(
-          nutrient_databank_number: row[0], nutrient_number: row[1]
+        UsdaNutrientDatabase::FoodsNutrient.new(
+          nutrient_databank_number: row[0],
+          nutrient_number: row[3]
         )
       end
 
@@ -27,6 +33,26 @@ module UsdaNutrientDatabase
       def log_import_started
         UsdaNutrientDatabase.log 'Importing foods_nutrients'
       end
+
+      def save_objects
+        options = {
+          batch_size: UsdaNutrientDatabase.batch_size,
+          validate: false
+        }
+        if UsdaNutrientDatabase::FoodsNutrient.exists?
+          options.merge!(
+            on_duplicate_key_update: {
+              conflict_target: %i(nutrient_databank_number nutrient_number),
+              columns: columns
+            }
+          )
+        end
+        UsdaNutrientDatabase::FoodsNutrient.import(
+          columns,
+          objects_to_import,
+          options
+        )
+      end
     end
   end
 end

data/lib/usda_nutrient_database/import/footnotes.rb

@@ -25,6 +25,16 @@ module UsdaNutrientDatabase
       def log_import_started
         UsdaNutrientDatabase.log 'Importing footnotes'
       end
+
+      def save_objects
+        UsdaNutrientDatabase::Footnote.import(columns, objects_to_import, {
+          validate: false,
+          on_duplicate_key_update: {
+            conflict_target: %i(nutrient_databank_number nutrient_number footnote_number),
+            columns: columns
+          }
+        })
+      end
     end
   end
 end

data/lib/usda_nutrient_database/import/nutrients.rb

@@ -23,6 +23,16 @@ module UsdaNutrientDatabase
       def log_import_started
         UsdaNutrientDatabase.log 'Importing nutrients'
       end
+
+      def save_objects
+        UsdaNutrientDatabase::Nutrient.import(columns, objects_to_import, {
+          validate: false,
+          on_duplicate_key_update: {
+            conflict_target: :nutrient_number,
+            columns: columns
+          }
+        })
+      end
     end
   end
 end

data/lib/usda_nutrient_database/import/source_codes.rb

@@ -13,12 +13,22 @@ module UsdaNutrientDatabase
       end
 
       def log_import_started
-        UsdaNutrientDatabase.log 'Source code import started'
+        UsdaNutrientDatabase.log 'Import source codes'
       end
 
       def filename
         'SRC_CD.txt'
       end
+
+      def save_objects
+        UsdaNutrientDatabase::SourceCode.import(columns, objects_to_import, {
+          validate: false,
+          on_duplicate_key_update: {
+            conflict_target: :code,
+            columns: columns
+          }
+        })
+      end
     end
   end
 end

data/lib/usda_nutrient_database/import/weights.rb

@@ -26,6 +26,16 @@ module UsdaNutrientDatabase
       def log_import_started
         UsdaNutrientDatabase.log 'Importing weights'
       end
+
+      def save_objects
+        UsdaNutrientDatabase::Weight.import(columns, objects_to_import, {
+          validate: false,
+          on_duplicate_key_update: {
+            conflict_target: %i(nutrient_databank_number sequence_number),
+            columns: columns
+          }
+        })
+      end
     end
   end
 end

data/lib/usda_nutrient_database/version.rb

@@ -1,3 +1,3 @@
 module UsdaNutrientDatabase
-  VERSION = '2.0.0'
+  VERSION = '2.1.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: usda-nutrient-database
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.1.0
 platform: ruby
 authors:
 - Matt Beedle