usda-nutrient-database 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/usda-nutrient-database.rb +3 -1
- data/lib/usda_nutrient_database/configuration.rb +5 -0
- data/lib/usda_nutrient_database/footnote.rb +2 -1
- data/lib/usda_nutrient_database/import/base.rb +15 -0
- data/lib/usda_nutrient_database/import/downloader.rb +1 -4
- data/lib/usda_nutrient_database/import/food_groups.rb +24 -11
- data/lib/usda_nutrient_database/import/foods.rb +28 -19
- data/lib/usda_nutrient_database/import/foods_nutrients.rb +16 -12
- data/lib/usda_nutrient_database/import/footnotes.rb +14 -13
- data/lib/usda_nutrient_database/import/nutrients.rb +26 -11
- data/lib/usda_nutrient_database/import/weights.rb +27 -11
- data/lib/usda_nutrient_database/importer.rb +8 -21
- data/lib/usda_nutrient_database/version.rb +1 -1
- data/lib/usda_nutrient_database/weight.rb +4 -2
- data/spec/lib/usda_nutrient_database/importer_spec.rb +12 -0
- data/spec/spec_helper.rb +4 -0
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fd3a9157bd4c59d9a33a7b0d360fbd09558904b4
|
4
|
+
data.tar.gz: 3222844968dbf95a113c76a480870ae0cdb77cb1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 439158006e6be1b8a20dafee9d5994a33e6d470dbab00dfbb1cc3a12fb583e181484c9b8cbdec5670cd92696420470ba03fdce0796fd3bb687725de8689688a0
|
7
|
+
data.tar.gz: 199f3b62e2d32de1b3803b5cd60b96a8c9cae44a09860ea7261e4e12d6bf7dc3937a2ea1c8b633021be49f939cd80503ab6404c6086bfd51aa4e093769d49f23
|
@@ -24,7 +24,9 @@ module UsdaNutrientDatabase
|
|
24
24
|
attr_writer :configuration
|
25
25
|
|
26
26
|
def log(message, level = :debug)
|
27
|
-
configuration.
|
27
|
+
if configuration.perform_logging?
|
28
|
+
configuration.logger.send(level, message)
|
29
|
+
end
|
28
30
|
end
|
29
31
|
|
30
32
|
def configuration
|
@@ -2,7 +2,8 @@ module UsdaNutrientDatabase
|
|
2
2
|
class Footnote < ActiveRecord::Base
|
3
3
|
self.table_name = 'usda_footnotes'
|
4
4
|
|
5
|
-
validates :nutrient_databank_number, presence: true
|
5
|
+
validates :nutrient_databank_number, presence: true,
|
6
|
+
uniqueness: { allow_blank: true }
|
6
7
|
validates :footnote_number, presence: true
|
7
8
|
validates :footnote_type, presence: true
|
8
9
|
validates :footnote_text, presence: true
|
@@ -1,3 +1,5 @@
|
|
1
|
+
require 'csv'
|
2
|
+
|
1
3
|
module UsdaNutrientDatabase
|
2
4
|
module Import
|
3
5
|
class Base
|
@@ -7,8 +9,21 @@ module UsdaNutrientDatabase
|
|
7
9
|
@directory = directory
|
8
10
|
end
|
9
11
|
|
12
|
+
def import
|
13
|
+
log_import_started
|
14
|
+
CSV.open(
|
15
|
+
"#{directory}/#{filename}", 'r:iso-8859-1:utf-8', csv_options
|
16
|
+
) do |csv|
|
17
|
+
csv.each { |row| extract_row(row) }
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
10
21
|
private
|
11
22
|
|
23
|
+
def filename
|
24
|
+
raise NotImplementedError
|
25
|
+
end
|
26
|
+
|
12
27
|
def csv_options
|
13
28
|
{ col_sep: '^', quote_char: '~' }
|
14
29
|
end
|
@@ -49,10 +49,7 @@ module UsdaNutrientDatabase
|
|
49
49
|
end
|
50
50
|
|
51
51
|
def connection
|
52
|
-
@connection ||= Faraday.new(url: 'http://www.ars.usda.gov')
|
53
|
-
faraday.response :logger
|
54
|
-
faraday.adapter Faraday.default_adapter
|
55
|
-
end
|
52
|
+
@connection ||= Faraday.new(url: 'http://www.ars.usda.gov')
|
56
53
|
end
|
57
54
|
end
|
58
55
|
end
|
@@ -1,19 +1,32 @@
|
|
1
|
-
require 'csv'
|
2
|
-
|
3
1
|
module UsdaNutrientDatabase
|
4
2
|
module Import
|
5
|
-
class FoodGroups <
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
)
|
11
|
-
|
12
|
-
|
13
|
-
|
3
|
+
class FoodGroups < Base
|
4
|
+
|
5
|
+
private
|
6
|
+
|
7
|
+
def extract_row(row)
|
8
|
+
build_food_group(row).save
|
9
|
+
end
|
10
|
+
|
11
|
+
def build_food_group(row)
|
12
|
+
UsdaNutrientDatabase::FoodGroup.new.tap do |food_group|
|
13
|
+
columns.each_with_index do |column, index|
|
14
|
+
food_group.send("#{column}=", row[index])
|
14
15
|
end
|
15
16
|
end
|
16
17
|
end
|
18
|
+
|
19
|
+
def columns
|
20
|
+
@columns ||= %w(code description)
|
21
|
+
end
|
22
|
+
|
23
|
+
def filename
|
24
|
+
'FD_GROUP.txt'
|
25
|
+
end
|
26
|
+
|
27
|
+
def log_import_started
|
28
|
+
UsdaNutrientDatabase.log 'Importing food groups'
|
29
|
+
end
|
17
30
|
end
|
18
31
|
end
|
19
32
|
end
|
@@ -3,28 +3,37 @@
|
|
3
3
|
module UsdaNutrientDatabase
|
4
4
|
module Import
|
5
5
|
class Foods < Base
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
survey: row[6],
|
18
|
-
refuse_description: row[7],
|
19
|
-
percentage_refuse: row[8],
|
20
|
-
nitrogen_factor: row[9],
|
21
|
-
protein_factor: row[10],
|
22
|
-
fat_factor: row[11],
|
23
|
-
carbohydrate_factor: row[12]
|
24
|
-
)
|
6
|
+
|
7
|
+
private
|
8
|
+
|
9
|
+
def extract_row(row)
|
10
|
+
build_food(row).save
|
11
|
+
end
|
12
|
+
|
13
|
+
def build_food(row)
|
14
|
+
UsdaNutrientDatabase::Food.new.tap do |food|
|
15
|
+
columns.each_with_index do |column, index|
|
16
|
+
food.send("#{column}=", row[index])
|
25
17
|
end
|
26
18
|
end
|
27
19
|
end
|
20
|
+
|
21
|
+
def log_import_started
|
22
|
+
UsdaNutrientDatabase.log 'Importing foods'
|
23
|
+
end
|
24
|
+
|
25
|
+
def filename
|
26
|
+
'FOOD_DES.txt'
|
27
|
+
end
|
28
|
+
|
29
|
+
def columns
|
30
|
+
[
|
31
|
+
:nutrient_databank_number, :food_group_code, :long_description,
|
32
|
+
:short_description, :common_names, :manufacturer_name, :survey,
|
33
|
+
:refuse_description, :percentage_refuse, :nitrogen_factor,
|
34
|
+
:protein_factor, :fat_factor, :carbohydrate_factor
|
35
|
+
]
|
36
|
+
end
|
28
37
|
end
|
29
38
|
end
|
30
39
|
end
|
@@ -1,14 +1,6 @@
|
|
1
1
|
module UsdaNutrientDatabase
|
2
2
|
module Import
|
3
3
|
class FoodsNutrients < Base
|
4
|
-
def import
|
5
|
-
UsdaNutrientDatabase.log 'Importing foods_nutrients'
|
6
|
-
CSV.open(
|
7
|
-
"#{directory}/NUT_DATA.txt", 'r:iso-8859-1:utf-8', csv_options
|
8
|
-
) do |csv|
|
9
|
-
csv.each { |row| extract_row(row) }
|
10
|
-
end
|
11
|
-
end
|
12
4
|
|
13
5
|
private
|
14
6
|
|
@@ -23,11 +15,23 @@ module UsdaNutrientDatabase
|
|
23
15
|
end
|
24
16
|
|
25
17
|
def extract_row(row)
|
26
|
-
|
27
|
-
|
28
|
-
|
18
|
+
build_foods_nutrient(row).save
|
19
|
+
end
|
20
|
+
|
21
|
+
def build_foods_nutrient(row)
|
22
|
+
UsdaNutrientDatabase::FoodsNutrient.new.tap do |foods_nutrient|
|
23
|
+
columns.each_with_index do |column, index|
|
24
|
+
foods_nutrient.send("#{column}=", row[index])
|
25
|
+
end
|
29
26
|
end
|
30
|
-
|
27
|
+
end
|
28
|
+
|
29
|
+
def filename
|
30
|
+
'NUT_DATA.txt'
|
31
|
+
end
|
32
|
+
|
33
|
+
def log_import_started
|
34
|
+
UsdaNutrientDatabase.log 'Importing foods_nutrients'
|
31
35
|
end
|
32
36
|
end
|
33
37
|
end
|
@@ -1,34 +1,35 @@
|
|
1
1
|
module UsdaNutrientDatabase
|
2
2
|
module Import
|
3
3
|
class Footnotes < Base
|
4
|
-
def import
|
5
|
-
CSV.open(
|
6
|
-
"#{directory}/FOOTNOTE.txt", 'r:iso-8859-1:utf-8', csv_options
|
7
|
-
) do |csv|
|
8
|
-
csv.each { |row| extract_row(row) }
|
9
|
-
end
|
10
|
-
end
|
11
4
|
|
12
5
|
private
|
13
6
|
|
14
7
|
def extract_row(row)
|
15
|
-
|
8
|
+
build_footnote(row).save
|
16
9
|
end
|
17
10
|
|
18
|
-
def
|
19
|
-
|
20
|
-
columns.each_with_index do |
|
21
|
-
|
11
|
+
def build_footnote(row)
|
12
|
+
UsdaNutrientDatabase::Footnote.new.tap do |footnote|
|
13
|
+
columns.each_with_index do |column, index|
|
14
|
+
footnote.send("#{column}=", row[index])
|
22
15
|
end
|
23
16
|
end
|
24
17
|
end
|
25
18
|
|
19
|
+
def filename
|
20
|
+
'FOOTNOTE.txt'
|
21
|
+
end
|
22
|
+
|
26
23
|
def columns
|
27
|
-
[
|
24
|
+
@columns ||= [
|
28
25
|
:nutrient_databank_number, :footnote_number, :footnote_type,
|
29
26
|
:nutrient_number, :footnote_text
|
30
27
|
]
|
31
28
|
end
|
29
|
+
|
30
|
+
def log_import_started
|
31
|
+
UsdaNutrientDatabase.log 'Importing footnotes'
|
32
|
+
end
|
32
33
|
end
|
33
34
|
end
|
34
35
|
end
|
@@ -1,20 +1,35 @@
|
|
1
1
|
module UsdaNutrientDatabase
|
2
2
|
module Import
|
3
3
|
class Nutrients < Base
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
)
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
)
|
4
|
+
|
5
|
+
private
|
6
|
+
|
7
|
+
def extract_row(row)
|
8
|
+
build_nutrient(row).save
|
9
|
+
end
|
10
|
+
|
11
|
+
def build_nutrient(row)
|
12
|
+
UsdaNutrientDatabase::Nutrient.new.tap do |nutrient|
|
13
|
+
columns.each_with_index do |column, index|
|
14
|
+
nutrient.send("#{column}=", row[index])
|
15
15
|
end
|
16
16
|
end
|
17
17
|
end
|
18
|
+
|
19
|
+
def columns
|
20
|
+
@columns ||= [
|
21
|
+
:nutrient_number, :units, :tagname, :nutrient_description,
|
22
|
+
:number_decimal_places, :sort_record_order
|
23
|
+
]
|
24
|
+
end
|
25
|
+
|
26
|
+
def filename
|
27
|
+
'NUTR_DEF.txt'
|
28
|
+
end
|
29
|
+
|
30
|
+
def log_import_started
|
31
|
+
UsdaNutrientDatabase.log 'Importing nutrients'
|
32
|
+
end
|
18
33
|
end
|
19
34
|
end
|
20
35
|
end
|
@@ -1,20 +1,36 @@
|
|
1
1
|
module UsdaNutrientDatabase
|
2
2
|
module Import
|
3
3
|
class Weights < Base
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
)
|
4
|
+
|
5
|
+
private
|
6
|
+
|
7
|
+
def extract_row(row)
|
8
|
+
build_weight(row).save
|
9
|
+
end
|
10
|
+
|
11
|
+
def build_weight(row)
|
12
|
+
UsdaNutrientDatabase::Weight.new.tap do |weight|
|
13
|
+
columns.each_with_index do |column, index|
|
14
|
+
weight.send("#{column}=", row[index])
|
15
15
|
end
|
16
16
|
end
|
17
17
|
end
|
18
|
+
|
19
|
+
def filename
|
20
|
+
'WEIGHT.txt'
|
21
|
+
end
|
22
|
+
|
23
|
+
def columns
|
24
|
+
@columns ||= [
|
25
|
+
:nutrient_databank_number, :sequence_number, :amount,
|
26
|
+
:measurement_description, :gram_weight, :num_data_points,
|
27
|
+
:standard_deviation
|
28
|
+
]
|
29
|
+
end
|
30
|
+
|
31
|
+
def log_import_started
|
32
|
+
UsdaNutrientDatabase.log 'Importing weights'
|
33
|
+
end
|
18
34
|
end
|
19
35
|
end
|
20
36
|
end
|
@@ -9,38 +9,25 @@ module UsdaNutrientDatabase
|
|
9
9
|
|
10
10
|
def import
|
11
11
|
downloader.download_and_unzip
|
12
|
-
|
13
|
-
food_importer.import
|
14
|
-
nutrient_importer.import
|
15
|
-
foods_nutrient_importer.import
|
16
|
-
weights_importer.import
|
12
|
+
importer_names.each { |importer_name| importer_for(importer_name).import }
|
17
13
|
ensure
|
18
14
|
downloader.cleanup
|
19
15
|
end
|
20
16
|
|
21
17
|
private
|
22
18
|
|
23
|
-
def
|
24
|
-
|
19
|
+
def importer_names
|
20
|
+
[
|
21
|
+
'Foods', 'Nutrients', 'FoodsNutrients', 'FoodGroups', 'Weights',
|
22
|
+
'Footnotes'
|
23
|
+
]
|
25
24
|
end
|
26
25
|
|
27
|
-
def
|
28
|
-
UsdaNutrientDatabase::Import
|
29
|
-
end
|
30
|
-
|
31
|
-
def foods_nutrient_importer
|
32
|
-
UsdaNutrientDatabase::Import::FoodsNutrients.
|
26
|
+
def importer_for(importer_name)
|
27
|
+
"UsdaNutrientDatabase::Import::#{importer_name}".constantize.
|
33
28
|
new("#{directory}/#{version}")
|
34
29
|
end
|
35
30
|
|
36
|
-
def food_group_importer
|
37
|
-
UsdaNutrientDatabase::Import::FoodGroups.new("#{directory}/#{version}")
|
38
|
-
end
|
39
|
-
|
40
|
-
def weights_importer
|
41
|
-
UsdaNutrientDatabase::Import::Weights.new("#{directory}/#{version}")
|
42
|
-
end
|
43
|
-
|
44
31
|
def downloader
|
45
32
|
UsdaNutrientDatabase::Import::Downloader.new(directory, version)
|
46
33
|
end
|
@@ -2,8 +2,10 @@ module UsdaNutrientDatabase
|
|
2
2
|
class Weight < ActiveRecord::Base
|
3
3
|
self.table_name = 'usda_weights'
|
4
4
|
|
5
|
-
validates :nutrient_databank_number, presence: true
|
6
|
-
|
5
|
+
validates :nutrient_databank_number, presence: true,
|
6
|
+
uniqueness: { scope: :sequence_number }
|
7
|
+
validates :sequence_number, presence: true,
|
8
|
+
uniqueness: { scope: :nutrient_databank_number }
|
7
9
|
validates :amount, presence: true
|
8
10
|
validates :measurement_description, presence: true
|
9
11
|
validates :gram_weight, presence: true
|
@@ -15,5 +15,17 @@ describe UsdaNutrientDatabase::Importer do
|
|
15
15
|
it { expect(UsdaNutrientDatabase::Nutrient.count).to eql(15) }
|
16
16
|
it { expect(UsdaNutrientDatabase::FoodsNutrient.count).to eql(12) }
|
17
17
|
it { expect(UsdaNutrientDatabase::Weight.count).to eql(11) }
|
18
|
+
it { expect(UsdaNutrientDatabase::Footnote.count).to eql(7) }
|
19
|
+
|
20
|
+
context 'importing twice' do
|
21
|
+
before { importer.import }
|
22
|
+
|
23
|
+
it { expect(UsdaNutrientDatabase::FoodGroup.count).to eql(25) }
|
24
|
+
it { expect(UsdaNutrientDatabase::Food.count).to eql(16) }
|
25
|
+
it { expect(UsdaNutrientDatabase::Nutrient.count).to eql(15) }
|
26
|
+
it { expect(UsdaNutrientDatabase::FoodsNutrient.count).to eql(12) }
|
27
|
+
it { expect(UsdaNutrientDatabase::Weight.count).to eql(11) }
|
28
|
+
it { expect(UsdaNutrientDatabase::Footnote.count).to eql(7) }
|
29
|
+
end
|
18
30
|
end
|
19
31
|
end
|
data/spec/spec_helper.rb
CHANGED
@@ -10,6 +10,10 @@ require 'usda-nutrient-database'
|
|
10
10
|
require 'shoulda-matchers'
|
11
11
|
require 'webmock/rspec'
|
12
12
|
|
13
|
+
UsdaNutrientDatabase.configure do |config|
|
14
|
+
config.perform_logging = false
|
15
|
+
end
|
16
|
+
|
13
17
|
# This file was generated by the `rspec --init` command. Conventionally, all
|
14
18
|
# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
|
15
19
|
# Require this file using `require "spec_helper"` to ensure that it is only
|