marc2linkeddata 0.1.2 → 0.1.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (5) hide show
  1. checksums.yaml +4 -4
  2. data/bin/marcAuthority2LD +7 -16
  3. data/lib/includes.rb +4 -2
  4. data/marc2linkeddata.gemspec +14 -13
  5. metadata +64 -50
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 30923ebbb08cf2eb45cbe20a79bfc115fb8f695a
4
- data.tar.gz: 1a98f477c2f8c9b61b4efb1f8c2a4d171862f2f4
3
+ metadata.gz: c7a2edf3d7a7a2abc392d18d1ec431d1b17152c8
4
+ data.tar.gz: 1ff8daa2a0924649f0f5da2868bb3a97e7360365
5
5
  SHA512:
6
- metadata.gz: c92c1369d3e39d46df6f712a94c27189c848c36b04c529fe1d9cfd2e341da9a1c8fe56f974fb7c011672fb59369572f9427307c077de9b1db0d29f6ced7a7dce
7
- data.tar.gz: bf3e016ec9a3b01c6ed7b1bede449ee1c7791fb34fca1150d1e1efde09eb6f94436692e7e4461f32b9458ebd34890b16428bff7e532d8a0aac69a01a13e3a170
6
+ metadata.gz: 62ebd8cb6a8e7531e70cfc2915962523bf8428f018cea6352e21001452998c37b2cbba666ca55cf80f2a59f3634f2c38078cad00a78d20e1f4d4d8cf2b3d2a1a
7
+ data.tar.gz: 44cf240042696b4f6f4833c2ab5a15f904aa856f6e3f7694e8488fb4016d45a8977f3ef1f1babd1463c00d73c6f2df286e520e025793d647bdcd3e6b0f996d3b
data/bin/marcAuthority2LD CHANGED
@@ -139,12 +139,6 @@ def marc_file2ld(marc_filename)
139
139
  ld_file.close
140
140
  end
141
141
 
142
- def thread_wait(threads)
143
- threads.each{|t| t.join}
144
- threads.delete_if {|t| t.status == false}
145
- threads.delete_if {|t| t.status.nil? }
146
- end
147
-
148
142
 
149
143
  # ---------------------------------------------------------------------
150
144
  # MAIN
@@ -183,19 +177,16 @@ marc_files.each do |path|
183
177
  output_path = File.join(path.dirname.to_s, output_dir)
184
178
  Dir.mkdir(output_path, 0775) unless File.directory? output_path
185
179
  auth_records = marc_authority_records(path.to_s)
186
- progress = ProgressBar.create(:total => auth_records.length, :format => '%a %f |%b>>%i| %P%% %t')
187
180
  if CONFIG.threads
188
- threads = []
189
- auth_records.each do |r|
190
- thread_wait(threads) while threads.length >= CONFIG.thread_limit
191
- t = Thread.new { marc_record2turtle(r, output_path) }
192
- t.abort_on_exception = true
193
- threads << t
194
- progress.increment # increment progress although thread is still running
181
+ # Allow Parallel to automatically determine the optimal concurrency model.
182
+ # Note that :in_threads crashed and :in_processes worked.
183
+ # Parallel.each(auth_records, :progress => 'Records: ', :in_threads=>CONFIG.thread_limit) do |r|
184
+ # Parallel.each(auth_records, :progress => 'Records: ', :in_processes=>CONFIG.thread_limit) do |r|
185
+ Parallel.each(auth_records, :progress => 'Records: ') do |r|
186
+ marc_record2turtle(r, output_path)
195
187
  end
196
- # Ensure any remaining threads complete
197
- threads.each{|t| t.join}
198
188
  else
189
+ progress = ProgressBar.create(:total => auth_records.length, :format => '%a %f |%b>>%i| %P%% %t')
199
190
  auth_records.each do |r|
200
191
  marc_record2turtle(r, output_path)
201
192
  progress.increment
data/lib/includes.rb CHANGED
@@ -3,9 +3,12 @@ Dotenv.load
3
3
 
4
4
  require 'addressable/uri'
5
5
  require 'json'
6
- require 'rest_client'
6
+ require 'rest-client'
7
7
  RestClient.proxy = ENV['http_proxy'] unless ENV['http_proxy'].nil?
8
+ require 'ruby-progressbar'
9
+
8
10
  require 'thread'
11
+ require 'parallel'
9
12
 
10
13
  require 'marc'
11
14
  require 'linkeddata'
@@ -14,7 +17,6 @@ require 'rdf/mongo'
14
17
 
15
18
  require 'pry'
16
19
  require 'pry-doc'
17
- require 'ruby-progressbar'
18
20
 
19
21
  require_relative 'marc2linkeddata/configuration'
20
22
 
data/marc2linkeddata.gemspec CHANGED
@@ -4,7 +4,7 @@ $:.unshift lib unless $:.include?(lib)
4
4
 
5
5
  Gem::Specification.new do |s|
6
6
  s.name = 'marc2linkeddata'
7
- s.version = '0.1.2'
7
+ s.version = '0.1.3'
8
8
  s.licenses = ['Apache-2.0']
9
9
 
10
10
  # mysql and bson_ext only install on MRI (c-ruby)
@@ -19,35 +19,36 @@ Gem::Specification.new do |s|
19
19
  #s.required_rubygems_version = '>= 1.3.6'
20
20
  # s.required_ruby_version = '>= 2.1.0'
21
21
 
22
- s.add_dependency 'addressable'
23
- s.add_dependency 'linkeddata'
24
- s.add_dependency 'marc'
25
- s.add_dependency 'rest-client'
26
- s.add_dependency 'ruby-progressbar'
22
+ s.add_dependency 'addressable', '~> 2.3'
23
+ s.add_dependency 'linkeddata', '~> 1.0'
24
+ s.add_dependency 'marc', '~> 1.0'
25
+ s.add_dependency 'parallel', '~> 1.0'
26
+ s.add_dependency 'rest-client', '~> 1.0'
27
+ s.add_dependency 'ruby-progressbar', '~> 1.0'
27
28
 
28
29
  # DB clients
29
- s.add_dependency 'mysql' # not for jruby
30
- s.add_dependency 'sequel'
30
+ s.add_dependency 'mysql', '~> 2.0' # not for jruby
31
+ s.add_dependency 'sequel', '~> 4.0'
31
32
 
32
33
  # Use ENV for config
33
- s.add_dependency 'dotenv'
34
+ s.add_dependency 'dotenv', '~> 1.0'
34
35
 
35
36
  # ruby gem for RDF on 4store, see https://github.com/emk/rdf-4store
36
37
  s.add_dependency 'rdf-4store'
37
38
  # ruby gem for RDF on allegrograph, see https://github.com/emk/rdf-agraph
38
39
  s.add_dependency 'rdf-agraph'
39
40
  # ruby gem for RDF on mongodb, see https://rubygems.org/gems/rdf-mongo
40
- s.add_dependency 'bson_ext' # not for jruby
41
- s.add_dependency 'rdf-mongo'
41
+ s.add_dependency 'bson_ext', '~> 1.0' # not for jruby
42
+ s.add_dependency 'rdf-mongo', '~> 1.0'
42
43
 
43
44
  # cache simple RDF on redis
44
45
  s.add_dependency 'hiredis'
45
- s.add_dependency 'redis'
46
+ s.add_dependency 'redis', '~> 3.0'
46
47
 
47
48
  # Use pry for console and debug config
48
49
  s.add_dependency 'pry'
49
50
  s.add_dependency 'pry-doc'
50
- s.add_development_dependency 'rspec'
51
+ s.add_development_dependency 'rspec', '~> 3.0'
51
52
 
52
53
  s.files = `git ls-files`.split($/)
53
54
  dev_files = %w(.gitignore bin/setup.sh bin/test.sh)
metadata CHANGED
@@ -1,127 +1,141 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: marc2linkeddata
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.2
4
+ version: 0.1.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Darren Weber
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-02-17 00:00:00.000000000 Z
11
+ date: 2015-02-18 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: addressable
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
- - - ">="
17
+ - - "~>"
18
18
  - !ruby/object:Gem::Version
19
- version: '0'
19
+ version: '2.3'
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
- - - ">="
24
+ - - "~>"
25
25
  - !ruby/object:Gem::Version
26
- version: '0'
26
+ version: '2.3'
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: linkeddata
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
- - - ">="
31
+ - - "~>"
32
32
  - !ruby/object:Gem::Version
33
- version: '0'
33
+ version: '1.0'
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
- - - ">="
38
+ - - "~>"
39
39
  - !ruby/object:Gem::Version
40
- version: '0'
40
+ version: '1.0'
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: marc
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
- - - ">="
45
+ - - "~>"
46
46
  - !ruby/object:Gem::Version
47
- version: '0'
47
+ version: '1.0'
48
48
  type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
- - - ">="
52
+ - - "~>"
53
53
  - !ruby/object:Gem::Version
54
- version: '0'
54
+ version: '1.0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: parallel
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '1.0'
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '1.0'
55
69
  - !ruby/object:Gem::Dependency
56
70
  name: rest-client
57
71
  requirement: !ruby/object:Gem::Requirement
58
72
  requirements:
59
- - - ">="
73
+ - - "~>"
60
74
  - !ruby/object:Gem::Version
61
- version: '0'
75
+ version: '1.0'
62
76
  type: :runtime
63
77
  prerelease: false
64
78
  version_requirements: !ruby/object:Gem::Requirement
65
79
  requirements:
66
- - - ">="
80
+ - - "~>"
67
81
  - !ruby/object:Gem::Version
68
- version: '0'
82
+ version: '1.0'
69
83
  - !ruby/object:Gem::Dependency
70
84
  name: ruby-progressbar
71
85
  requirement: !ruby/object:Gem::Requirement
72
86
  requirements:
73
- - - ">="
87
+ - - "~>"
74
88
  - !ruby/object:Gem::Version
75
- version: '0'
89
+ version: '1.0'
76
90
  type: :runtime
77
91
  prerelease: false
78
92
  version_requirements: !ruby/object:Gem::Requirement
79
93
  requirements:
80
- - - ">="
94
+ - - "~>"
81
95
  - !ruby/object:Gem::Version
82
- version: '0'
96
+ version: '1.0'
83
97
  - !ruby/object:Gem::Dependency
84
98
  name: mysql
85
99
  requirement: !ruby/object:Gem::Requirement
86
100
  requirements:
87
- - - ">="
101
+ - - "~>"
88
102
  - !ruby/object:Gem::Version
89
- version: '0'
103
+ version: '2.0'
90
104
  type: :runtime
91
105
  prerelease: false
92
106
  version_requirements: !ruby/object:Gem::Requirement
93
107
  requirements:
94
- - - ">="
108
+ - - "~>"
95
109
  - !ruby/object:Gem::Version
96
- version: '0'
110
+ version: '2.0'
97
111
  - !ruby/object:Gem::Dependency
98
112
  name: sequel
99
113
  requirement: !ruby/object:Gem::Requirement
100
114
  requirements:
101
- - - ">="
115
+ - - "~>"
102
116
  - !ruby/object:Gem::Version
103
- version: '0'
117
+ version: '4.0'
104
118
  type: :runtime
105
119
  prerelease: false
106
120
  version_requirements: !ruby/object:Gem::Requirement
107
121
  requirements:
108
- - - ">="
122
+ - - "~>"
109
123
  - !ruby/object:Gem::Version
110
- version: '0'
124
+ version: '4.0'
111
125
  - !ruby/object:Gem::Dependency
112
126
  name: dotenv
113
127
  requirement: !ruby/object:Gem::Requirement
114
128
  requirements:
115
- - - ">="
129
+ - - "~>"
116
130
  - !ruby/object:Gem::Version
117
- version: '0'
131
+ version: '1.0'
118
132
  type: :runtime
119
133
  prerelease: false
120
134
  version_requirements: !ruby/object:Gem::Requirement
121
135
  requirements:
122
- - - ">="
136
+ - - "~>"
123
137
  - !ruby/object:Gem::Version
124
- version: '0'
138
+ version: '1.0'
125
139
  - !ruby/object:Gem::Dependency
126
140
  name: rdf-4store
127
141
  requirement: !ruby/object:Gem::Requirement
@@ -154,30 +168,30 @@ dependencies:
154
168
  name: bson_ext
155
169
  requirement: !ruby/object:Gem::Requirement
156
170
  requirements:
157
- - - ">="
171
+ - - "~>"
158
172
  - !ruby/object:Gem::Version
159
- version: '0'
173
+ version: '1.0'
160
174
  type: :runtime
161
175
  prerelease: false
162
176
  version_requirements: !ruby/object:Gem::Requirement
163
177
  requirements:
164
- - - ">="
178
+ - - "~>"
165
179
  - !ruby/object:Gem::Version
166
- version: '0'
180
+ version: '1.0'
167
181
  - !ruby/object:Gem::Dependency
168
182
  name: rdf-mongo
169
183
  requirement: !ruby/object:Gem::Requirement
170
184
  requirements:
171
- - - ">="
185
+ - - "~>"
172
186
  - !ruby/object:Gem::Version
173
- version: '0'
187
+ version: '1.0'
174
188
  type: :runtime
175
189
  prerelease: false
176
190
  version_requirements: !ruby/object:Gem::Requirement
177
191
  requirements:
178
- - - ">="
192
+ - - "~>"
179
193
  - !ruby/object:Gem::Version
180
- version: '0'
194
+ version: '1.0'
181
195
  - !ruby/object:Gem::Dependency
182
196
  name: hiredis
183
197
  requirement: !ruby/object:Gem::Requirement
@@ -196,16 +210,16 @@ dependencies:
196
210
  name: redis
197
211
  requirement: !ruby/object:Gem::Requirement
198
212
  requirements:
199
- - - ">="
213
+ - - "~>"
200
214
  - !ruby/object:Gem::Version
201
- version: '0'
215
+ version: '3.0'
202
216
  type: :runtime
203
217
  prerelease: false
204
218
  version_requirements: !ruby/object:Gem::Requirement
205
219
  requirements:
206
- - - ">="
220
+ - - "~>"
207
221
  - !ruby/object:Gem::Version
208
- version: '0'
222
+ version: '3.0'
209
223
  - !ruby/object:Gem::Dependency
210
224
  name: pry
211
225
  requirement: !ruby/object:Gem::Requirement
@@ -238,16 +252,16 @@ dependencies:
238
252
  name: rspec
239
253
  requirement: !ruby/object:Gem::Requirement
240
254
  requirements:
241
- - - ">="
255
+ - - "~>"
242
256
  - !ruby/object:Gem::Version
243
- version: '0'
257
+ version: '3.0'
244
258
  type: :development
245
259
  prerelease: false
246
260
  version_requirements: !ruby/object:Gem::Requirement
247
261
  requirements:
248
- - - ">="
262
+ - - "~>"
249
263
  - !ruby/object:Gem::Version
250
- version: '0'
264
+ version: '3.0'
251
265
  description: Utilities for translation of Marc21 records to linked open data.
252
266
  email:
253
267
  - dlweber@stanford.edu