usda-nutrient-database 1.4.1 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b828c4a6e16e2136f5eda8372159bc822ddcff44
-  data.tar.gz: 5ee0841df5cc1b4fa1b835a821662b481298477a
+  metadata.gz: 0939aaca9f37d86998fb685c9dfbcd672c483b01
+  data.tar.gz: 1adb4ebe955fa5cd4b6f17ea4f851cb72f7d456e
 SHA512:
-  metadata.gz: 24eb084caf4d83da0b2422895ac7d8c79b1159ac2116676935f3d3281ac87d183ce56d888fab4c209e683f56d8645c7ce3166fc29def97122cdd71ed29217363
-  data.tar.gz: 395783ce1ef08de5f0a6841049598e046a7f6a43d973b608c1ecfe7d9217509ffe9ec57df005eaa938646e07158192a59284f16377f0859709d674e62baeb8a7
+  metadata.gz: 2a52d264629a668a674b364aab1b286ee2300f3501ab8909892572cc46c355026de6c9e4c94e384414edaac35f0a52fa2de055eef9565449fef27ddac2a9c4cd
+  data.tar.gz: fafe5a7ecd2dcedcb16e816b346418668ecee60c07f5b489aafd44669e7dc1420fc77440a6b69f58c57494f4d073ecffb3a8ef37923aab04a4da3576cf973966

data/lib/usda-nutrient-database.rb

@@ -32,10 +32,18 @@ module UsdaNutrientDatabase
       end
     end
 
+    def activerecord_import?
+      defined?(ActiveRecord::Import)
+    end
+
     def configuration
       @configuration ||= UsdaNutrientDatabase::Configuration.new
     end
 
+    def batch_size
+      @batch_fize ||= configuration.batch_size
+    end
+
     def usda_version
       @usda_version ||= configuration.usda_version
     end

data/lib/usda_nutrient_database/configuration.rb

@@ -1,9 +1,14 @@
 module UsdaNutrientDatabase
   class Configuration
     attr_accessor :logger
-    attr_writer   :perform_logging,
+    attr_writer :batch_size,
+      :perform_logging,
       :usda_version
 
+    def batch_size
+      @batch_size ||= 10000
+    end
+
     def logger
       @logger ||= Logger.new(STDOUT)
     end

data/lib/usda_nutrient_database/import/base.rb

@@ -5,24 +5,35 @@ module UsdaNutrientDatabase
     class Base
       def initialize(directory)
         @directory = directory
+        @objects_to_import = []
       end
 
       def import
         log_import_started
         CSV.open(file_location, 'r:iso-8859-1:utf-8', csv_options) do |csv|
-          csv.each { |row| extract_row(row) }
+          csv.each { |row| objects_to_import << extract_row(row) }
         end
+        ar_import? ? save_objects : objects_to_import.each(&:save)
       end
 
       private
 
-      attr_reader :directory
+      attr_reader :directory,
+        :objects_to_import
+
+      def ar_import?
+        UsdaNutrientDatabase.activerecord_import?
+      end
 
       def extract_row(row)
-        build_object(apply_typecasts(row)).save!
+        build_object(apply_typecasts(row))
       end
 
       def build_object(row)
+        ar_import? ? build_values_array(row) : build_ar_object(row)
+      end
+
+      def build_ar_object(row)
         find_or_initialize(row).tap do |object|
           columns.each_with_index do |column, index|
             object.send("#{column}=", row[index])
@@ -30,6 +41,10 @@ module UsdaNutrientDatabase
         end
       end
 
+      def build_values_array(row)
+        columns.map.with_index { |_, index| row[index] }
+      end
+
       def columns
         raise NotImplementedError
       end
@@ -53,6 +68,10 @@ module UsdaNutrientDatabase
       def csv_options
         { col_sep: '^', quote_char: '~' }
       end
+
+      def save_objects
+        raise NotImplementedError
+      end
     end
   end
 end

data/lib/usda_nutrient_database/import/food_groups.rb

@@ -19,6 +19,16 @@ module UsdaNutrientDatabase
       def log_import_started
         UsdaNutrientDatabase.log 'Importing food groups'
       end
+
+      def save_objects
+        UsdaNutrientDatabase::FoodGroup.import(columns, objects_to_import, {
+          validate: false,
+          on_duplicate_key_update: {
+            conflict_target: :code,
+            columns: %i(description)
+          }
+        })
+      end
     end
   end
 end

data/lib/usda_nutrient_database/import/foods.rb

@@ -32,6 +32,16 @@ module UsdaNutrientDatabase
           :protein_factor, :fat_factor, :carbohydrate_factor
         ]
       end
+
+      def save_objects
+        UsdaNutrientDatabase::Food.import(columns, objects_to_import, {
+          validate: false,
+          on_duplicate_key_update: {
+            conflict_target: :nutrient_databank_number,
+            columns: columns
+          }
+        })
+      end
     end
   end
 end

data/lib/usda_nutrient_database/import/foods_nutrients.rb

@@ -4,6 +4,11 @@ module UsdaNutrientDatabase
 
       private
 
+      def apply_typecasts(row)
+        row[8] = row[8] == 'Y'
+        row
+      end
+
       def columns
         [
           :nutrient_databank_number, :nutrient_number, :nutrient_value,
@@ -15,8 +20,9 @@ module UsdaNutrientDatabase
       end
 
       def find_or_initialize(row)
-        UsdaNutrientDatabase::FoodsNutrient.find_or_initialize_by(
-          nutrient_databank_number: row[0], nutrient_number: row[1]
+        UsdaNutrientDatabase::FoodsNutrient.new(
+          nutrient_databank_number: row[0],
+          nutrient_number: row[3]
         )
       end
 
@@ -27,6 +33,26 @@ module UsdaNutrientDatabase
       def log_import_started
         UsdaNutrientDatabase.log 'Importing foods_nutrients'
       end
+
+      def save_objects
+        options = {
+          batch_size: UsdaNutrientDatabase.batch_size,
+          validate: false
+        }
+        if UsdaNutrientDatabase::FoodsNutrient.exists?
+          options.merge!(
+            on_duplicate_key_update: {
+              conflict_target: %i(nutrient_databank_number nutrient_number),
+              columns: columns
+            }
+          )
+        end
+        UsdaNutrientDatabase::FoodsNutrient.import(
+          columns,
+          objects_to_import,
+          options
+        )
+      end
     end
   end
 end

data/lib/usda_nutrient_database/import/footnotes.rb

@@ -25,6 +25,16 @@ module UsdaNutrientDatabase
       def log_import_started
         UsdaNutrientDatabase.log 'Importing footnotes'
       end
+
+      def save_objects
+        UsdaNutrientDatabase::Footnote.import(columns, objects_to_import, {
+          validate: false,
+          on_duplicate_key_update: {
+            conflict_target: %i(nutrient_databank_number nutrient_number footnote_number),
+            columns: columns
+          }
+        })
+      end
     end
   end
 end

data/lib/usda_nutrient_database/import/nutrients.rb

@@ -23,6 +23,16 @@ module UsdaNutrientDatabase
       def log_import_started
         UsdaNutrientDatabase.log 'Importing nutrients'
       end
+
+      def save_objects
+        UsdaNutrientDatabase::Nutrient.import(columns, objects_to_import, {
+          validate: false,
+          on_duplicate_key_update: {
+            conflict_target: :nutrient_number,
+            columns: columns
+          }
+        })
+      end
     end
   end
 end

data/lib/usda_nutrient_database/import/source_codes.rb

@@ -13,12 +13,22 @@ module UsdaNutrientDatabase
       end
 
       def log_import_started
-        UsdaNutrientDatabase.log 'Source code import started'
+        UsdaNutrientDatabase.log 'Import source codes'
       end
 
       def filename
         'SRC_CD.txt'
       end
+
+      def save_objects
+        UsdaNutrientDatabase::SourceCode.import(columns, objects_to_import, {
+          validate: false,
+          on_duplicate_key_update: {
+            conflict_target: :code,
+            columns: columns
+          }
+        })
+      end
     end
   end
 end

data/lib/usda_nutrient_database/import/weights.rb

@@ -26,6 +26,16 @@ module UsdaNutrientDatabase
       def log_import_started
         UsdaNutrientDatabase.log 'Importing weights'
       end
+
+      def save_objects
+        UsdaNutrientDatabase::Weight.import(columns, objects_to_import, {
+          validate: false,
+          on_duplicate_key_update: {
+            conflict_target: %i(nutrient_databank_number sequence_number),
+            columns: columns
+          }
+        })
+      end
     end
   end
 end

data/lib/usda_nutrient_database/version.rb

@@ -1,3 +1,3 @@
 module UsdaNutrientDatabase
-  VERSION = '1.4.1'
+  VERSION = '1.5.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: usda-nutrient-database
 version: !ruby/object:Gem::Version
-  version: 1.4.1
+  version: 1.5.0
 platform: ruby
 authors:
 - Matt Beedle