revs-utils 1.0.14 → 1.0.15
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +8 -8
- data/Gemfile.lock +3 -3
- data/config/manifest_headers.yml +12 -2
- data/lib/revs-utils/version.rb +1 -1
- data/lib/revs-utils.rb +55 -19
- data/spec/revs-utils_spec.rb +22 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
MDA0MDNjYzdmYThhMTQzNDNkYzdkYjY4YWE4ZGI5MDIyY2JkZGI1MQ==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
OTYwODZjNTIzN2IzZDdmMTA3Mjg3MTA0N2FjNGE0MjNiZjMwN2MwNQ==
|
7
7
|
SHA512:
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
ZTA5MDU0OTgzNWIwNjdjZWI0M2RmOTg1OGM5MDk4Y2I3ZmU5YWE5YjVhYWQ2
|
10
|
+
NTQ1NWExMmJiNmU5MGNhMzUzYTdhMzk4ZWYwNzkyNTNlYzljNTVmOTBmN2Vl
|
11
|
+
OTdiMTg2MjQ3YWUxOGQ3YWMxYzk5MmVlNTA1YjkwM2U2ZjRjNzA=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
NTBmNTczNzZhZDI2YzI5OTg0YjMyOWQyOGMyMDJlYjM1ZDBmZGUxYzEzNGI1
|
14
|
+
YjllMzEyMzkxZWJhNzk3MGNkZjRhNjcyZDljOGFlMDhkNDJkZDY3NjBhYTc0
|
15
|
+
Nzg0YTJkYWU1MTlmNjViMGU2ODcyNTk1MGM5ZDVkZGQ2YmU2MWQ=
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
revs-utils (1.0.
|
4
|
+
revs-utils (1.0.15)
|
5
5
|
actionpack (~> 3)
|
6
6
|
chronic
|
7
7
|
countries (= 0.9.2)
|
@@ -42,14 +42,14 @@ GEM
|
|
42
42
|
rake (>= 0.8.7)
|
43
43
|
rest-client
|
44
44
|
mime-types (2.0)
|
45
|
-
multi_json (1.
|
45
|
+
multi_json (1.11.0)
|
46
46
|
rack (1.4.5)
|
47
47
|
rack-cache (1.2)
|
48
48
|
rack (>= 0.4)
|
49
49
|
rack-test (0.6.3)
|
50
50
|
rack (>= 1.0)
|
51
51
|
rake (10.1.0)
|
52
|
-
rdf (1.1.
|
52
|
+
rdf (1.1.11)
|
53
53
|
link_header (~> 0.0, >= 0.0.8)
|
54
54
|
rest-client (1.6.7)
|
55
55
|
mime-types (>= 1.16)
|
data/config/manifest_headers.yml
CHANGED
@@ -30,5 +30,15 @@ metadata:
|
|
30
30
|
hide: hide
|
31
31
|
format: format
|
32
32
|
collection_name: collection_name
|
33
|
-
|
34
|
-
|
33
|
+
known_formats:
|
34
|
+
- black-and-white film
|
35
|
+
- color film
|
36
|
+
- slides
|
37
|
+
- foldouts
|
38
|
+
- pamphlets
|
39
|
+
- oversize items
|
40
|
+
- photographic prints
|
41
|
+
- black-and-white negatives
|
42
|
+
- color negatives
|
43
|
+
- black-and-white transparencies
|
44
|
+
- color transparencies
|
data/lib/revs-utils/version.rb
CHANGED
data/lib/revs-utils.rb
CHANGED
@@ -14,7 +14,7 @@ REVS_LC_TERMS_FILENAME=File.join(PROJECT_ROOT,'files','revs-lc-marque-terms.obj'
|
|
14
14
|
REVS_MANIFEST_HEADERS_FILEPATH = File.join(PROJECT_ROOT,'config',"manifest_headers.yml")
|
15
15
|
REGISTER = "register"
|
16
16
|
METADATA = "metadata"
|
17
|
-
|
17
|
+
FORMATS = "known_formats"
|
18
18
|
|
19
19
|
module Revs
|
20
20
|
module Utils
|
@@ -28,6 +28,9 @@ module Revs
|
|
28
28
|
REVS_MANIFEST_HEADERS_FILE = File.open(REVS_MANIFEST_HEADERS_FILEPATH)
|
29
29
|
REVS_MANIFEST_HEADERS = YAML.load( REVS_MANIFEST_HEADERS_FILE)
|
30
30
|
|
31
|
+
def revs_known_formats
|
32
|
+
get_manifest_section(FORMATS)
|
33
|
+
end
|
31
34
|
|
32
35
|
def get_manifest_section(section)
|
33
36
|
return REVS_MANIFEST_HEADERS[section]
|
@@ -67,40 +70,59 @@ module Revs
|
|
67
70
|
sources = Array.new
|
68
71
|
files.each do |file|
|
69
72
|
file.each do |row|
|
70
|
-
#Make sure the
|
73
|
+
#Make sure the sourceid and filename are the same
|
71
74
|
fname = row[get_manifest_section(REGISTER)['filename']].chomp(File.extname(row[get_manifest_section(REGISTER)['filename']]))
|
72
75
|
return false if row[get_manifest_section(REGISTER)['sourceid']] != fname
|
73
76
|
sources << row[get_manifest_section(REGISTER)['sourceid']]
|
74
|
-
end
|
75
|
-
|
76
|
-
|
77
|
-
|
77
|
+
end
|
78
78
|
end
|
79
79
|
return sources.uniq.size == sources.size
|
80
80
|
|
81
81
|
end
|
82
|
-
|
83
|
-
|
82
|
+
|
84
83
|
#Pass this function a CSV file and it will return true if the proper headers are there and each entry has the required fields filled in
|
85
84
|
def valid_to_register(file_path)
|
86
|
-
|
87
85
|
file = read_csv_with_headers(file_path)
|
86
|
+
return check_valid_to_register(file)
|
87
|
+
end
|
88
|
+
|
89
|
+
#Pass this function a CSV file and it will return true if the proper headers are there and each entry has the required fields filled in.
|
90
|
+
def valid_for_metadata(file_path)
|
91
|
+
file = read_csv_with_headers(file_path)
|
92
|
+
return check_headers(file)
|
93
|
+
end
|
94
|
+
|
95
|
+
# pass in csv data and it will tell if you everything is safe to register based on having labels, unique sourceIDs and filenames matching sourceIDs
|
96
|
+
def check_valid_to_register(csv_data)
|
88
97
|
#Make sure all the required headers are there
|
89
|
-
return false if not get_manifest_section(REGISTER).values-
|
90
|
-
|
98
|
+
return false if not get_manifest_section(REGISTER).values-csv_data[0].keys == []
|
99
|
+
sources=Array.new
|
91
100
|
#Make sure all files have entries for those required headers
|
92
|
-
|
101
|
+
csv_data.each do |row|
|
93
102
|
get_manifest_section(REGISTER).keys.each do |header| # label should be there as a column but does not always need a value
|
94
103
|
return false if header.downcase !='label' && row[header].blank? #Alternatively consider row[header].class != String or row[header].size <= 0
|
95
104
|
end
|
105
|
+
fname = row[get_manifest_section(REGISTER)['filename']].chomp(File.extname(row[get_manifest_section(REGISTER)['filename']]))
|
106
|
+
return false if row[get_manifest_section(REGISTER)['sourceid']] != fname
|
107
|
+
sources << row[get_manifest_section(REGISTER)['sourceid']]
|
96
108
|
end
|
97
|
-
|
109
|
+
return sources.uniq.size == sources.size
|
98
110
|
end
|
99
111
|
|
100
|
-
#
|
101
|
-
def
|
102
|
-
|
103
|
-
|
112
|
+
# looks at certain metadata fields in manifest to confirm validity (such as dates and formats)
|
113
|
+
def check_metadata(csv_data)
|
114
|
+
bad_rows=0
|
115
|
+
csv_data.each do |row|
|
116
|
+
valid_date=revs_is_valid_datestring?(row[get_manifest_section(METADATA)['year']] || row[get_manifest_section(METADATA)['date']])
|
117
|
+
valid_format=revs_is_valid_format?(row[get_manifest_section(METADATA)['format']])
|
118
|
+
bad_rows+=1 unless (valid_date && valid_format)
|
119
|
+
end
|
120
|
+
return bad_rows
|
121
|
+
end
|
122
|
+
|
123
|
+
# pass in csv data from a file read in and it will tell you if the headers are valid
|
124
|
+
def check_headers(csv_data)
|
125
|
+
file_headers=csv_data[0].keys.reject(&:blank?).collect(&:downcase)
|
104
126
|
#The file doesn't need to have all the metadata values, it just can't have headers that aren't used for metadata or registration
|
105
127
|
if file_headers.include?('date') && file_headers.include?('year') # can't have both date and year
|
106
128
|
return false
|
@@ -110,7 +132,7 @@ module Revs
|
|
110
132
|
return file_headers-get_manifest_section(METADATA).values-get_manifest_section(REGISTER).values == []
|
111
133
|
end
|
112
134
|
end
|
113
|
-
|
135
|
+
|
114
136
|
def clean_collection_name(name)
|
115
137
|
return "" if name.blank? || name.nil?
|
116
138
|
name=name.to_s
|
@@ -147,7 +169,13 @@ module Revs
|
|
147
169
|
return row
|
148
170
|
end
|
149
171
|
|
150
|
-
#
|
172
|
+
# checks to see if we have a valid format
|
173
|
+
def revs_is_valid_format?(format)
|
174
|
+
formats=format.split("|").collect{|f| f.strip}
|
175
|
+
!formats.collect {|f| revs_known_formats.include?(f)}.uniq.include?(false)
|
176
|
+
end
|
177
|
+
|
178
|
+
# check a single format and fix some common issues
|
151
179
|
def revs_check_format(format)
|
152
180
|
return revs_check_formats([format]).first
|
153
181
|
end
|
@@ -237,6 +265,14 @@ module Revs
|
|
237
265
|
date_string.to_s.strip.scan(/\D/).empty? and (starting_year..Date.today.year).include?(date_string.to_i)
|
238
266
|
end
|
239
267
|
|
268
|
+
# tell us if the incoming datestring supplied in the manifest column is a valid date, year or list of years
|
269
|
+
def revs_is_valid_datestring?(date_string)
|
270
|
+
return true if date_string.nil? || date_string.empty?
|
271
|
+
is_full_date=(get_full_date(date_string) != false)
|
272
|
+
is_year=!parse_years(date_string).empty?
|
273
|
+
return is_year || is_full_date
|
274
|
+
end
|
275
|
+
|
240
276
|
# tell us if the string passed is in is a full date of the format M/D/YYYY or m-d-yyyy or m-d-yy or M/D/YY, and returns the date object if it is valid
|
241
277
|
def get_full_date(date_string)
|
242
278
|
begin
|
data/spec/revs-utils_spec.rb
CHANGED
@@ -108,7 +108,27 @@ describe "Revs-Utils" do
|
|
108
108
|
@revs.is_valid_year?('1700').should be_false # too old! no cars even existed yet
|
109
109
|
@revs.is_valid_year?('1700',1600).should be_true # unless we allow it to be ok
|
110
110
|
end
|
111
|
-
|
111
|
+
|
112
|
+
it "should indicate if we have unknown formats" do
|
113
|
+
@revs.revs_is_valid_format?('slides').should be_true
|
114
|
+
@revs.revs_is_valid_format?('slide').should be_false
|
115
|
+
@revs.revs_is_valid_format?('slides | slide').should be_false
|
116
|
+
@revs.revs_is_valid_format?('slides | black-and-white negatives').should be_true
|
117
|
+
@revs.revs_is_valid_format?('black-and-white-negatives').should be_false
|
118
|
+
@revs.revs_is_valid_format?('black-and-white negatives').should be_true
|
119
|
+
end
|
120
|
+
|
121
|
+
it "should indicate if we have a valid datestring" do
|
122
|
+
@revs.revs_is_valid_datestring?('1959').should be_true
|
123
|
+
@revs.revs_is_valid_datestring?('bogus').should be_false
|
124
|
+
@revs.revs_is_valid_datestring?('').should be_true
|
125
|
+
@revs.revs_is_valid_datestring?(nil).should be_true
|
126
|
+
@revs.revs_is_valid_datestring?([]).should be_true
|
127
|
+
@revs.revs_is_valid_datestring?('2/2/1950').should be_true
|
128
|
+
@revs.revs_is_valid_datestring?('2/31/1950').should be_false
|
129
|
+
@revs.revs_is_valid_datestring?('2/2/50').should be_true
|
130
|
+
@revs.revs_is_valid_datestring?('195x').should be_true
|
131
|
+
end
|
112
132
|
|
113
133
|
it "should lookup the country correctly" do
|
114
134
|
@revs.revs_get_country('USA').should == "United States"
|
@@ -137,6 +157,7 @@ describe "Revs-Utils" do
|
|
137
157
|
it "should parse 1950s correctly" do
|
138
158
|
|
139
159
|
@revs.parse_years('1950s').should == ['1950','1951','1952','1953','1954','1955','1956','1957','1958','1959']
|
160
|
+
@revs.parse_years("1950's").should == ['1950','1951','1952','1953','1954','1955','1956','1957','1958','1959']
|
140
161
|
|
141
162
|
end
|
142
163
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: revs-utils
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.15
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Peter Mangiafico
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-04-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: countries
|