usda-nutrient-database 1.5.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0939aaca9f37d86998fb685c9dfbcd672c483b01
-  data.tar.gz: 1adb4ebe955fa5cd4b6f17ea4f851cb72f7d456e
+  metadata.gz: 55eaf87a81739b7b4da9d592c67eca5fcd857801
+  data.tar.gz: e42ac18382b777910c4cbb6ca0546a48aaa3a48f
 SHA512:
-  metadata.gz: 2a52d264629a668a674b364aab1b286ee2300f3501ab8909892572cc46c355026de6c9e4c94e384414edaac35f0a52fa2de055eef9565449fef27ddac2a9c4cd
-  data.tar.gz: fafe5a7ecd2dcedcb16e816b346418668ecee60c07f5b489aafd44669e7dc1420fc77440a6b69f58c57494f4d073ecffb3a8ef37923aab04a4da3576cf973966
+  metadata.gz: 616a437099608f3b36179de590f12b81b97e1996e926a8d83b75118175508e51147db803560792a8b572f69252be0352b7c23e5e6f338fd30136c73009a330f3
+  data.tar.gz: dedcb6ef862b1034fa15086f76e8059ff689a1c6f06f626392cc0790761541c22bdec009a922034e8dd184ccda6abb1b36117689242aa293fbdddac219448b85

data/.travis.yml

@@ -4,7 +4,6 @@ rvm:
   - 2.2.0
   - 2.1.0
   - 2.0.0
-  - 1.9.3
 before_install:
   - gem update --system
   - gem install bundler

data/CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## 2.0.0
+
+- Removed IDs from tables which don't have them in USDA data. Instead opted to
+  use compound keys the same way they do. The upgrade path is simple but if you
+  have any other tables linking directly to USDA tables you're going to need to
+  change them to point to the composite key in the associations.
+  [#15](https://github.com/mattbeedle/usda-nutrient-database/pull/15)
+
+## 1.4.0
+
+- Upgrade to version SR28 USDA dataset
+  [#14](https://github.com/mattbeedle/usda-nutrient-database/pull/14)
+
 ## 1.2.0
 
 - Upgrade to version SR27 USDA dataset.

data/db/migrate/10_fix_footnotes_indexes.rb

@@ -0,0 +1,26 @@
+class FixFootnotesIndexes < ActiveRecord::Migration
+  def change
+    change_table :usda_footnotes do |t|
+      t.remove_index :nutrient_databank_number
+      t.remove_index :footnote_number
+      t.remove_index :footnote_type
+      t.remove_index :nutrient_number
+
+      t.index [
+        :nutrient_databank_number,
+        :nutrient_number,
+        :footnote_number
+      ], {
+        unique: true,
+        name: "index_usda_footnotes_on_unique_keys"
+      }
+
+      t.index [
+        :nutrient_databank_number,
+        :footnote_type
+      ], {
+        name: "index_usda_footnotes_on_databank_number_and_type"
+      }
+    end
+  end
+end

data/db/migrate/11_fix_usda_weights.rb

@@ -0,0 +1,16 @@
+class FixUsdaWeights < ActiveRecord::Migration
+  def change
+    remove_column :usda_weights, :id
+
+    change_table :usda_weights do |t|
+      t.remove_index :nutrient_databank_number
+
+      t.index [
+        :nutrient_databank_number,
+        :sequence_number
+      ], {
+        unique: true
+      }
+    end
+  end
+end

data/db/migrate/12_fix_usda_foods_nutrients.rb

@@ -0,0 +1,12 @@
+class FixUsdaFoodsNutrients < ActiveRecord::Migration
+  def change
+    remove_column :usda_foods_nutrients, :id
+
+    change_table :usda_foods_nutrients do |t|
+      t.remove_index [
+        :nutrient_databank_number,
+        :nutrient_number
+      ]
+    end
+  end
+end

data/db/migrate/9_add_referential_integrity.rb

@@ -49,6 +49,11 @@ class AddReferentialIntegrity < ActiveRecord::Migration
       column: :nutrient_databank_number,
       primary_key: :nutrient_databank_number
 
+    add_foreign_key :usda_foods,
+      :usda_food_groups,
+      column: :food_group_code,
+      primary_key: :code
+
     add_foreign_key :usda_foods_nutrients,
       :usda_foods,
       column: :nutrient_databank_number,

data/lib/usda-nutrient-database.rb

@@ -32,18 +32,10 @@ module UsdaNutrientDatabase
       end
     end
 
-    def activerecord_import?
-      defined?(ActiveRecord::Import)
-    end
-
     def configuration
       @configuration ||= UsdaNutrientDatabase::Configuration.new
     end
 
-    def batch_size
-      @batch_fize ||= configuration.batch_size
-    end
-
     def usda_version
       @usda_version ||= configuration.usda_version
     end

data/lib/usda_nutrient_database/configuration.rb

@@ -1,14 +1,9 @@
 module UsdaNutrientDatabase
   class Configuration
     attr_accessor :logger
-    attr_writer :batch_size,
-      :perform_logging,
+    attr_writer   :perform_logging,
       :usda_version
 
-    def batch_size
-      @batch_size ||= 10000
-    end
-
     def logger
       @logger ||= Logger.new(STDOUT)
     end

data/lib/usda_nutrient_database/footnote.rb

@@ -2,8 +2,12 @@ module UsdaNutrientDatabase
   class Footnote < ActiveRecord::Base
     self.table_name = 'usda_footnotes'
 
-    validates :nutrient_databank_number, presence: true,
-      uniqueness: { allow_blank: true }
+    validates :nutrient_databank_number,
+      presence: true,
+      uniqueness: {
+        allow_blank: true,
+        scope: [ :footnote_number, :nutrient_number ]
+      }
     validates :footnote_number, presence: true
     validates :footnote_type, presence: true
     validates :footnote_text, presence: true

data/lib/usda_nutrient_database/import/base.rb

@@ -5,35 +5,24 @@ module UsdaNutrientDatabase
     class Base
       def initialize(directory)
         @directory = directory
-        @objects_to_import = []
       end
 
       def import
         log_import_started
         CSV.open(file_location, 'r:iso-8859-1:utf-8', csv_options) do |csv|
-          csv.each { |row| objects_to_import << extract_row(row) }
+          csv.each { |row| extract_row(row) }
         end
-        ar_import? ? save_objects : objects_to_import.each(&:save)
       end
 
       private
 
-      attr_reader :directory,
-        :objects_to_import
-
-      def ar_import?
-        UsdaNutrientDatabase.activerecord_import?
-      end
+      attr_reader :directory
 
       def extract_row(row)
-        build_object(apply_typecasts(row))
+        build_object(apply_typecasts(row)).save
       end
 
       def build_object(row)
-        ar_import? ? build_values_array(row) : build_ar_object(row)
-      end
-
-      def build_ar_object(row)
         find_or_initialize(row).tap do |object|
           columns.each_with_index do |column, index|
             object.send("#{column}=", row[index])
@@ -41,10 +30,6 @@ module UsdaNutrientDatabase
         end
       end
 
-      def build_values_array(row)
-        columns.map.with_index { |_, index| row[index] }
-      end
-
       def columns
         raise NotImplementedError
       end
@@ -68,10 +53,6 @@ module UsdaNutrientDatabase
       def csv_options
         { col_sep: '^', quote_char: '~' }
       end
-
-      def save_objects
-        raise NotImplementedError
-      end
     end
   end
 end

data/lib/usda_nutrient_database/import/food_groups.rb

@@ -19,16 +19,6 @@ module UsdaNutrientDatabase
       def log_import_started
         UsdaNutrientDatabase.log 'Importing food groups'
       end
-
-      def save_objects
-        UsdaNutrientDatabase::FoodGroup.import(columns, objects_to_import, {
-          validate: false,
-          on_duplicate_key_update: {
-            conflict_target: :code,
-            columns: %i(description)
-          }
-        })
-      end
     end
   end
 end

data/lib/usda_nutrient_database/import/foods.rb

@@ -32,16 +32,6 @@ module UsdaNutrientDatabase
           :protein_factor, :fat_factor, :carbohydrate_factor
         ]
       end
-
-      def save_objects
-        UsdaNutrientDatabase::Food.import(columns, objects_to_import, {
-          validate: false,
-          on_duplicate_key_update: {
-            conflict_target: :nutrient_databank_number,
-            columns: columns
-          }
-        })
-      end
     end
   end
 end

data/lib/usda_nutrient_database/import/foods_nutrients.rb

@@ -4,11 +4,6 @@ module UsdaNutrientDatabase
 
       private
 
-      def apply_typecasts(row)
-        row[8] = row[8] == 'Y'
-        row
-      end
-
       def columns
         [
           :nutrient_databank_number, :nutrient_number, :nutrient_value,
@@ -20,9 +15,8 @@ module UsdaNutrientDatabase
       end
 
       def find_or_initialize(row)
-        UsdaNutrientDatabase::FoodsNutrient.new(
-          nutrient_databank_number: row[0],
-          nutrient_number: row[3]
+        UsdaNutrientDatabase::FoodsNutrient.find_or_initialize_by(
+          nutrient_databank_number: row[0], nutrient_number: row[1]
         )
       end
 
@@ -33,26 +27,6 @@ module UsdaNutrientDatabase
       def log_import_started
         UsdaNutrientDatabase.log 'Importing foods_nutrients'
       end
-
-      def save_objects
-        options = {
-          batch_size: UsdaNutrientDatabase.batch_size,
-          validate: false
-        }
-        if UsdaNutrientDatabase::FoodsNutrient.exists?
-          options.merge!(
-            on_duplicate_key_update: {
-              conflict_target: %i(nutrient_databank_number nutrient_number),
-              columns: columns
-            }
-          )
-        end
-        UsdaNutrientDatabase::FoodsNutrient.import(
-          columns,
-          objects_to_import,
-          options
-        )
-      end
     end
   end
 end

data/lib/usda_nutrient_database/import/footnotes.rb

@@ -25,16 +25,6 @@ module UsdaNutrientDatabase
       def log_import_started
         UsdaNutrientDatabase.log 'Importing footnotes'
       end
-
-      def save_objects
-        UsdaNutrientDatabase::Footnote.import(columns, objects_to_import, {
-          validate: false,
-          on_duplicate_key_update: {
-            conflict_target: %i(nutrient_databank_number nutrient_number footnote_number),
-            columns: columns
-          }
-        })
-      end
     end
   end
 end

data/lib/usda_nutrient_database/import/nutrients.rb

@@ -23,16 +23,6 @@ module UsdaNutrientDatabase
       def log_import_started
         UsdaNutrientDatabase.log 'Importing nutrients'
       end
-
-      def save_objects
-        UsdaNutrientDatabase::Nutrient.import(columns, objects_to_import, {
-          validate: false,
-          on_duplicate_key_update: {
-            conflict_target: :nutrient_number,
-            columns: columns
-          }
-        })
-      end
     end
   end
 end

data/lib/usda_nutrient_database/import/source_codes.rb

@@ -13,22 +13,12 @@ module UsdaNutrientDatabase
       end
 
       def log_import_started
-        UsdaNutrientDatabase.log 'Import source codes'
+        UsdaNutrientDatabase.log 'Source code import started'
       end
 
       def filename
         'SRC_CD.txt'
       end
-
-      def save_objects
-        UsdaNutrientDatabase::SourceCode.import(columns, objects_to_import, {
-          validate: false,
-          on_duplicate_key_update: {
-            conflict_target: :code,
-            columns: columns
-          }
-        })
-      end
     end
   end
 end

data/lib/usda_nutrient_database/import/weights.rb

@@ -26,16 +26,6 @@ module UsdaNutrientDatabase
       def log_import_started
         UsdaNutrientDatabase.log 'Importing weights'
       end
-
-      def save_objects
-        UsdaNutrientDatabase::Weight.import(columns, objects_to_import, {
-          validate: false,
-          on_duplicate_key_update: {
-            conflict_target: %i(nutrient_databank_number sequence_number),
-            columns: columns
-          }
-        })
-      end
     end
   end
 end

data/lib/usda_nutrient_database/version.rb

@@ -1,3 +1,3 @@
 module UsdaNutrientDatabase
-  VERSION = '1.5.0'
+  VERSION = '2.0.0'
 end

data/spec/lib/usda_nutrient_database/food_group_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-describe UsdaNutrientDatabase::FoodGroup do
+describe UsdaNutrientDatabase::FoodGroup, type: :model do
   it { should validate_presence_of(:code) }
   it { should validate_presence_of(:description) }
 end

data/spec/lib/usda_nutrient_database/food_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-describe UsdaNutrientDatabase::Food do
+describe UsdaNutrientDatabase::Food, type: :model do
   it { should validate_presence_of(:nutrient_databank_number) }
   it { should validate_presence_of(:food_group_code) }
   it { should validate_presence_of(:long_description) }

data/spec/lib/usda_nutrient_database/foods_nutrient_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-describe UsdaNutrientDatabase::FoodsNutrient do
+describe UsdaNutrientDatabase::FoodsNutrient, type: :model do
   it { should validate_presence_of(:nutrient_databank_number) }
   it { should validate_presence_of(:nutrient_number) }
   it { should validate_presence_of(:nutrient_value) }

data/spec/lib/usda_nutrient_database/footnote_spec.rb

@@ -1,8 +1,12 @@
 require 'spec_helper'
 
-describe UsdaNutrientDatabase::Footnote do
-  it { should validate_presence_of(:nutrient_databank_number) }
-  it { should validate_presence_of(:footnote_number) }
-  it { should validate_presence_of(:footnote_type) }
-  it { should validate_presence_of(:footnote_text) }
+describe UsdaNutrientDatabase::Footnote, type: :model do
+  # it 'validates uniqueness of nutrient_databank_number' do
+  #   expect(subject)
+  #     .to validate_uniqueness_of(:nutrient_databank_number)
+  #     .scoped_to(:footnote_number, :nutrient_number)
+  # end
+  it { is_expected.to validate_presence_of(:footnote_number) }
+  it { is_expected.to validate_presence_of(:footnote_type) }
+  it { is_expected.to validate_presence_of(:footnote_text) }
 end

data/spec/lib/usda_nutrient_database/importer_spec.rb

@@ -7,7 +7,6 @@ describe UsdaNutrientDatabase::Importer do
     let(:sr_file) { File.read('spec/support/sr25.zip') }
     before do
       stub_request(:head, "https://www.ars.usda.gov/SP2UserFiles/Place/12354500/Data/SR25/dnload/sr25.zip").
-        with(:headers => {'Accept'=>'*/*', 'User-Agent'=>'Faraday v0.9.1'}).
         to_return(:status => 200, :body => "", :headers => {})
       stub_request(:get, /.*/).
         to_return(body: sr_file)
@@ -20,9 +19,13 @@ describe UsdaNutrientDatabase::Importer do
       expect(UsdaNutrientDatabase::Nutrient.count).to eql(15)
       expect(UsdaNutrientDatabase::FoodsNutrient.count).to eql(12)
       expect(UsdaNutrientDatabase::Weight.count).to eql(11)
-      expect(UsdaNutrientDatabase::Footnote.count).to eql(7)
+      expect(UsdaNutrientDatabase::Footnote.count).to eql(9)
     end
 
+    # NOTE Not sure how this one got past code review o.O. Need to clean up both
+    # the zip files in spec support to only have a few records and remove the
+    # PDF. Don't want people having to download megabytes of useless spec
+    # support files just to install this gem.
     context 'with sr27' do
       let(:sr_file) { File.read('spec/support/sr27.zip') }
       it 'should import the right number of records' do
@@ -31,7 +34,7 @@ describe UsdaNutrientDatabase::Importer do
         expect(UsdaNutrientDatabase::Nutrient.count).to eql(150)
         expect(UsdaNutrientDatabase::FoodsNutrient.count).to eql(476)
         expect(UsdaNutrientDatabase::Weight.count).to eql(18)
-        expect(UsdaNutrientDatabase::Footnote.count).to eql(362)
+        expect(UsdaNutrientDatabase::Footnote.count).to eql(531)
       end
     end
 
@@ -44,7 +47,7 @@ describe UsdaNutrientDatabase::Importer do
         expect(UsdaNutrientDatabase::Nutrient.count).to eql(15)
         expect(UsdaNutrientDatabase::FoodsNutrient.count).to eql(12)
         expect(UsdaNutrientDatabase::Weight.count).to eql(11)
-        expect(UsdaNutrientDatabase::Footnote.count).to eql(7)
+        expect(UsdaNutrientDatabase::Footnote.count).to eql(9)
       end
     end
   end

data/spec/lib/usda_nutrient_database/nutrient_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-describe UsdaNutrientDatabase::Nutrient do
+describe UsdaNutrientDatabase::Nutrient, type: :model do
   it { should validate_presence_of(:nutrient_number) }
   it { should validate_presence_of(:units) }
   it { should validate_presence_of(:nutrient_description) }

data/spec/lib/usda_nutrient_database/source_code_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-describe UsdaNutrientDatabase::SourceCode do
+describe UsdaNutrientDatabase::SourceCode, type: :model do
   it { validate_presence_of(:code) }
   it { validate_presence_of(:description) }
 end

data/spec/lib/usda_nutrient_database/weight_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-describe UsdaNutrientDatabase::Weight do
+describe UsdaNutrientDatabase::Weight, type: :model do
   it { should validate_presence_of(:nutrient_databank_number) }
   it { should validate_presence_of(:sequence_number) }
   it { should validate_presence_of(:amount) }

data/spec/spec_helper.rb

@@ -16,6 +16,13 @@ require_relative 'support/database'
 
 UsdaNutrientDatabase.configure do |config|
   config.perform_logging = false
+  config.usda_version = 'sr25'
+end
+
+Shoulda::Matchers.configure do |config|
+  config.integrate do |with|
+    with.test_framework :rspec
+  end
 end
 
 # This file was generated by the `rspec --init` command. Conventionally, all
@@ -28,6 +35,9 @@ RSpec.configure do |config|
   config.run_all_when_everything_filtered = true
   config.filter_run :focus
 
+  config.include(Shoulda::Matchers::ActiveModel, type: :model)
+  config.include(Shoulda::Matchers::ActiveRecord, type: :model)
+
   include Database
 
   # Run specs in random order to surface order dependencies. If you find an
@@ -48,7 +58,7 @@ RSpec.configure do |config|
   begin
     ActiveRecord::Base.establish_connection(db_name.to_sym)
     ActiveRecord::Base.connection
-  rescue PG::ConnectionBad
+  rescue PG::ConnectionBad, ActiveRecord::NoDatabaseError
     ActiveRecord::Base.establish_connection db_config.merge('database' => nil)
     ActiveRecord::Base.connection.create_database db_config['database']
     ActiveRecord::Base.establish_connection db_config
@@ -66,6 +76,7 @@ RSpec.configure do |config|
   end
 
   config.before do
+    DatabaseCleaner.clean
     DatabaseCleaner.start
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: usda-nutrient-database
 version: !ruby/object:Gem::Version
-  version: 1.5.0
+  version: 2.0.0
 platform: ruby
 authors:
 - Matt Beedle
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-12 00:00:00.000000000 Z
+date: 2016-04-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -222,6 +222,9 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- db/migrate/10_fix_footnotes_indexes.rb
+- db/migrate/11_fix_usda_weights.rb
+- db/migrate/12_fix_usda_foods_nutrients.rb
 - db/migrate/1_create_usda_food_groups.rb
 - db/migrate/2_create_usda_foods.rb
 - db/migrate/3_create_usda_foods_nutrients.rb