marc2linkeddata 0.1.2 → 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. checksums.yaml +4 -4
  2. data/bin/marcAuthority2LD +7 -16
  3. data/lib/includes.rb +4 -2
  4. data/marc2linkeddata.gemspec +14 -13
  5. metadata +64 -50
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 30923ebbb08cf2eb45cbe20a79bfc115fb8f695a
4
- data.tar.gz: 1a98f477c2f8c9b61b4efb1f8c2a4d171862f2f4
3
+ metadata.gz: c7a2edf3d7a7a2abc392d18d1ec431d1b17152c8
4
+ data.tar.gz: 1ff8daa2a0924649f0f5da2868bb3a97e7360365
5
5
  SHA512:
6
- metadata.gz: c92c1369d3e39d46df6f712a94c27189c848c36b04c529fe1d9cfd2e341da9a1c8fe56f974fb7c011672fb59369572f9427307c077de9b1db0d29f6ced7a7dce
7
- data.tar.gz: bf3e016ec9a3b01c6ed7b1bede449ee1c7791fb34fca1150d1e1efde09eb6f94436692e7e4461f32b9458ebd34890b16428bff7e532d8a0aac69a01a13e3a170
6
+ metadata.gz: 62ebd8cb6a8e7531e70cfc2915962523bf8428f018cea6352e21001452998c37b2cbba666ca55cf80f2a59f3634f2c38078cad00a78d20e1f4d4d8cf2b3d2a1a
7
+ data.tar.gz: 44cf240042696b4f6f4833c2ab5a15f904aa856f6e3f7694e8488fb4016d45a8977f3ef1f1babd1463c00d73c6f2df286e520e025793d647bdcd3e6b0f996d3b
data/bin/marcAuthority2LD CHANGED
@@ -139,12 +139,6 @@ def marc_file2ld(marc_filename)
139
139
  ld_file.close
140
140
  end
141
141
 
142
- def thread_wait(threads)
143
- threads.each{|t| t.join}
144
- threads.delete_if {|t| t.status == false}
145
- threads.delete_if {|t| t.status.nil? }
146
- end
147
-
148
142
 
149
143
  # ---------------------------------------------------------------------
150
144
  # MAIN
@@ -183,19 +177,16 @@ marc_files.each do |path|
183
177
  output_path = File.join(path.dirname.to_s, output_dir)
184
178
  Dir.mkdir(output_path, 0775) unless File.directory? output_path
185
179
  auth_records = marc_authority_records(path.to_s)
186
- progress = ProgressBar.create(:total => auth_records.length, :format => '%a %f |%b>>%i| %P%% %t')
187
180
  if CONFIG.threads
188
- threads = []
189
- auth_records.each do |r|
190
- thread_wait(threads) while threads.length >= CONFIG.thread_limit
191
- t = Thread.new { marc_record2turtle(r, output_path) }
192
- t.abort_on_exception = true
193
- threads << t
194
- progress.increment # increment progress although thread is still running
181
+ # Allow Parallel to automatically determine the optimal concurrency model.
182
+ # Note that :in_threads crashed and :in_processes worked.
183
+ # Parallel.each(auth_records, :progress => 'Records: ', :in_threads=>CONFIG.thread_limit) do |r|
184
+ # Parallel.each(auth_records, :progress => 'Records: ', :in_processes=>CONFIG.thread_limit) do |r|
185
+ Parallel.each(auth_records, :progress => 'Records: ') do |r|
186
+ marc_record2turtle(r, output_path)
195
187
  end
196
- # Ensure any remaining threads complete
197
- threads.each{|t| t.join}
198
188
  else
189
+ progress = ProgressBar.create(:total => auth_records.length, :format => '%a %f |%b>>%i| %P%% %t')
199
190
  auth_records.each do |r|
200
191
  marc_record2turtle(r, output_path)
201
192
  progress.increment
data/lib/includes.rb CHANGED
@@ -3,9 +3,12 @@ Dotenv.load
3
3
 
4
4
  require 'addressable/uri'
5
5
  require 'json'
6
- require 'rest_client'
6
+ require 'rest-client'
7
7
  RestClient.proxy = ENV['http_proxy'] unless ENV['http_proxy'].nil?
8
+ require 'ruby-progressbar'
9
+
8
10
  require 'thread'
11
+ require 'parallel'
9
12
 
10
13
  require 'marc'
11
14
  require 'linkeddata'
@@ -14,7 +17,6 @@ require 'rdf/mongo'
14
17
 
15
18
  require 'pry'
16
19
  require 'pry-doc'
17
- require 'ruby-progressbar'
18
20
 
19
21
  require_relative 'marc2linkeddata/configuration'
20
22
 
data/marc2linkeddata.gemspec CHANGED
@@ -4,7 +4,7 @@ $:.unshift lib unless $:.include?(lib)
4
4
 
5
5
  Gem::Specification.new do |s|
6
6
  s.name = 'marc2linkeddata'
7
- s.version = '0.1.2'
7
+ s.version = '0.1.3'
8
8
  s.licenses = ['Apache-2.0']
9
9
 
10
10
  # mysql and bson_ext only install on MRI (c-ruby)
@@ -19,35 +19,36 @@ Gem::Specification.new do |s|
19
19
  #s.required_rubygems_version = '>= 1.3.6'
20
20
  # s.required_ruby_version = '>= 2.1.0'
21
21
 
22
- s.add_dependency 'addressable'
23
- s.add_dependency 'linkeddata'
24
- s.add_dependency 'marc'
25
- s.add_dependency 'rest-client'
26
- s.add_dependency 'ruby-progressbar'
22
+ s.add_dependency 'addressable', '~> 2.3'
23
+ s.add_dependency 'linkeddata', '~> 1.0'
24
+ s.add_dependency 'marc', '~> 1.0'
25
+ s.add_dependency 'parallel', '~> 1.0'
26
+ s.add_dependency 'rest-client', '~> 1.0'
27
+ s.add_dependency 'ruby-progressbar', '~> 1.0'
27
28
 
28
29
  # DB clients
29
- s.add_dependency 'mysql' # not for jruby
30
- s.add_dependency 'sequel'
30
+ s.add_dependency 'mysql', '~> 2.0' # not for jruby
31
+ s.add_dependency 'sequel', '~> 4.0'
31
32
 
32
33
  # Use ENV for config
33
- s.add_dependency 'dotenv'
34
+ s.add_dependency 'dotenv', '~> 1.0'
34
35
 
35
36
  # ruby gem for RDF on 4store, see https://github.com/emk/rdf-4store
36
37
  s.add_dependency 'rdf-4store'
37
38
  # ruby gem for RDF on allegrograph, see https://github.com/emk/rdf-agraph
38
39
  s.add_dependency 'rdf-agraph'
39
40
  # ruby gem for RDF on mongodb, see https://rubygems.org/gems/rdf-mongo
40
- s.add_dependency 'bson_ext' # not for jruby
41
- s.add_dependency 'rdf-mongo'
41
+ s.add_dependency 'bson_ext', '~> 1.0' # not for jruby
42
+ s.add_dependency 'rdf-mongo', '~> 1.0'
42
43
 
43
44
  # cache simple RDF on redis
44
45
  s.add_dependency 'hiredis'
45
- s.add_dependency 'redis'
46
+ s.add_dependency 'redis', '~> 3.0'
46
47
 
47
48
  # Use pry for console and debug config
48
49
  s.add_dependency 'pry'
49
50
  s.add_dependency 'pry-doc'
50
- s.add_development_dependency 'rspec'
51
+ s.add_development_dependency 'rspec', '~> 3.0'
51
52
 
52
53
  s.files = `git ls-files`.split($/)
53
54
  dev_files = %w(.gitignore bin/setup.sh bin/test.sh)
metadata CHANGED
@@ -1,127 +1,141 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: marc2linkeddata
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.2
4
+ version: 0.1.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Darren Weber
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-02-17 00:00:00.000000000 Z
11
+ date: 2015-02-18 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: addressable
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
- - - ">="
17
+ - - "~>"
18
18
  - !ruby/object:Gem::Version
19
- version: '0'
19
+ version: '2.3'
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
- - - ">="
24
+ - - "~>"
25
25
  - !ruby/object:Gem::Version
26
- version: '0'
26
+ version: '2.3'
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: linkeddata
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
- - - ">="
31
+ - - "~>"
32
32
  - !ruby/object:Gem::Version
33
- version: '0'
33
+ version: '1.0'
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
- - - ">="
38
+ - - "~>"
39
39
  - !ruby/object:Gem::Version
40
- version: '0'
40
+ version: '1.0'
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: marc
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
- - - ">="
45
+ - - "~>"
46
46
  - !ruby/object:Gem::Version
47
- version: '0'
47
+ version: '1.0'
48
48
  type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
- - - ">="
52
+ - - "~>"
53
53
  - !ruby/object:Gem::Version
54
- version: '0'
54
+ version: '1.0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: parallel
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '1.0'
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '1.0'
55
69
  - !ruby/object:Gem::Dependency
56
70
  name: rest-client
57
71
  requirement: !ruby/object:Gem::Requirement
58
72
  requirements:
59
- - - ">="
73
+ - - "~>"
60
74
  - !ruby/object:Gem::Version
61
- version: '0'
75
+ version: '1.0'
62
76
  type: :runtime
63
77
  prerelease: false
64
78
  version_requirements: !ruby/object:Gem::Requirement
65
79
  requirements:
66
- - - ">="
80
+ - - "~>"
67
81
  - !ruby/object:Gem::Version
68
- version: '0'
82
+ version: '1.0'
69
83
  - !ruby/object:Gem::Dependency
70
84
  name: ruby-progressbar
71
85
  requirement: !ruby/object:Gem::Requirement
72
86
  requirements:
73
- - - ">="
87
+ - - "~>"
74
88
  - !ruby/object:Gem::Version
75
- version: '0'
89
+ version: '1.0'
76
90
  type: :runtime
77
91
  prerelease: false
78
92
  version_requirements: !ruby/object:Gem::Requirement
79
93
  requirements:
80
- - - ">="
94
+ - - "~>"
81
95
  - !ruby/object:Gem::Version
82
- version: '0'
96
+ version: '1.0'
83
97
  - !ruby/object:Gem::Dependency
84
98
  name: mysql
85
99
  requirement: !ruby/object:Gem::Requirement
86
100
  requirements:
87
- - - ">="
101
+ - - "~>"
88
102
  - !ruby/object:Gem::Version
89
- version: '0'
103
+ version: '2.0'
90
104
  type: :runtime
91
105
  prerelease: false
92
106
  version_requirements: !ruby/object:Gem::Requirement
93
107
  requirements:
94
- - - ">="
108
+ - - "~>"
95
109
  - !ruby/object:Gem::Version
96
- version: '0'
110
+ version: '2.0'
97
111
  - !ruby/object:Gem::Dependency
98
112
  name: sequel
99
113
  requirement: !ruby/object:Gem::Requirement
100
114
  requirements:
101
- - - ">="
115
+ - - "~>"
102
116
  - !ruby/object:Gem::Version
103
- version: '0'
117
+ version: '4.0'
104
118
  type: :runtime
105
119
  prerelease: false
106
120
  version_requirements: !ruby/object:Gem::Requirement
107
121
  requirements:
108
- - - ">="
122
+ - - "~>"
109
123
  - !ruby/object:Gem::Version
110
- version: '0'
124
+ version: '4.0'
111
125
  - !ruby/object:Gem::Dependency
112
126
  name: dotenv
113
127
  requirement: !ruby/object:Gem::Requirement
114
128
  requirements:
115
- - - ">="
129
+ - - "~>"
116
130
  - !ruby/object:Gem::Version
117
- version: '0'
131
+ version: '1.0'
118
132
  type: :runtime
119
133
  prerelease: false
120
134
  version_requirements: !ruby/object:Gem::Requirement
121
135
  requirements:
122
- - - ">="
136
+ - - "~>"
123
137
  - !ruby/object:Gem::Version
124
- version: '0'
138
+ version: '1.0'
125
139
  - !ruby/object:Gem::Dependency
126
140
  name: rdf-4store
127
141
  requirement: !ruby/object:Gem::Requirement
@@ -154,30 +168,30 @@ dependencies:
154
168
  name: bson_ext
155
169
  requirement: !ruby/object:Gem::Requirement
156
170
  requirements:
157
- - - ">="
171
+ - - "~>"
158
172
  - !ruby/object:Gem::Version
159
- version: '0'
173
+ version: '1.0'
160
174
  type: :runtime
161
175
  prerelease: false
162
176
  version_requirements: !ruby/object:Gem::Requirement
163
177
  requirements:
164
- - - ">="
178
+ - - "~>"
165
179
  - !ruby/object:Gem::Version
166
- version: '0'
180
+ version: '1.0'
167
181
  - !ruby/object:Gem::Dependency
168
182
  name: rdf-mongo
169
183
  requirement: !ruby/object:Gem::Requirement
170
184
  requirements:
171
- - - ">="
185
+ - - "~>"
172
186
  - !ruby/object:Gem::Version
173
- version: '0'
187
+ version: '1.0'
174
188
  type: :runtime
175
189
  prerelease: false
176
190
  version_requirements: !ruby/object:Gem::Requirement
177
191
  requirements:
178
- - - ">="
192
+ - - "~>"
179
193
  - !ruby/object:Gem::Version
180
- version: '0'
194
+ version: '1.0'
181
195
  - !ruby/object:Gem::Dependency
182
196
  name: hiredis
183
197
  requirement: !ruby/object:Gem::Requirement
@@ -196,16 +210,16 @@ dependencies:
196
210
  name: redis
197
211
  requirement: !ruby/object:Gem::Requirement
198
212
  requirements:
199
- - - ">="
213
+ - - "~>"
200
214
  - !ruby/object:Gem::Version
201
- version: '0'
215
+ version: '3.0'
202
216
  type: :runtime
203
217
  prerelease: false
204
218
  version_requirements: !ruby/object:Gem::Requirement
205
219
  requirements:
206
- - - ">="
220
+ - - "~>"
207
221
  - !ruby/object:Gem::Version
208
- version: '0'
222
+ version: '3.0'
209
223
  - !ruby/object:Gem::Dependency
210
224
  name: pry
211
225
  requirement: !ruby/object:Gem::Requirement
@@ -238,16 +252,16 @@ dependencies:
238
252
  name: rspec
239
253
  requirement: !ruby/object:Gem::Requirement
240
254
  requirements:
241
- - - ">="
255
+ - - "~>"
242
256
  - !ruby/object:Gem::Version
243
- version: '0'
257
+ version: '3.0'
244
258
  type: :development
245
259
  prerelease: false
246
260
  version_requirements: !ruby/object:Gem::Requirement
247
261
  requirements:
248
- - - ">="
262
+ - - "~>"
249
263
  - !ruby/object:Gem::Version
250
- version: '0'
264
+ version: '3.0'
251
265
  description: Utilities for translation of Marc21 records to linked open data.
252
266
  email:
253
267
  - dlweber@stanford.edu