marc2linkeddata 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/marcAuthority2LD +7 -16
- data/lib/includes.rb +4 -2
- data/marc2linkeddata.gemspec +14 -13
- metadata +64 -50
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c7a2edf3d7a7a2abc392d18d1ec431d1b17152c8
|
4
|
+
data.tar.gz: 1ff8daa2a0924649f0f5da2868bb3a97e7360365
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 62ebd8cb6a8e7531e70cfc2915962523bf8428f018cea6352e21001452998c37b2cbba666ca55cf80f2a59f3634f2c38078cad00a78d20e1f4d4d8cf2b3d2a1a
|
7
|
+
data.tar.gz: 44cf240042696b4f6f4833c2ab5a15f904aa856f6e3f7694e8488fb4016d45a8977f3ef1f1babd1463c00d73c6f2df286e520e025793d647bdcd3e6b0f996d3b
|
data/bin/marcAuthority2LD
CHANGED
@@ -139,12 +139,6 @@ def marc_file2ld(marc_filename)
|
|
139
139
|
ld_file.close
|
140
140
|
end
|
141
141
|
|
142
|
-
def thread_wait(threads)
|
143
|
-
threads.each{|t| t.join}
|
144
|
-
threads.delete_if {|t| t.status == false}
|
145
|
-
threads.delete_if {|t| t.status.nil? }
|
146
|
-
end
|
147
|
-
|
148
142
|
|
149
143
|
# ---------------------------------------------------------------------
|
150
144
|
# MAIN
|
@@ -183,19 +177,16 @@ marc_files.each do |path|
|
|
183
177
|
output_path = File.join(path.dirname.to_s, output_dir)
|
184
178
|
Dir.mkdir(output_path, 0775) unless File.directory? output_path
|
185
179
|
auth_records = marc_authority_records(path.to_s)
|
186
|
-
progress = ProgressBar.create(:total => auth_records.length, :format => '%a %f |%b>>%i| %P%% %t')
|
187
180
|
if CONFIG.threads
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
progress.increment # increment progress although thread is still running
|
181
|
+
# Allow Parallel to automatically determine the optimal concurrency model.
|
182
|
+
# Note that :in_threads crashed and :in_processes worked.
|
183
|
+
# Parallel.each(auth_records, :progress => 'Records: ', :in_threads=>CONFIG.thread_limit) do |r|
|
184
|
+
# Parallel.each(auth_records, :progress => 'Records: ', :in_processes=>CONFIG.thread_limit) do |r|
|
185
|
+
Parallel.each(auth_records, :progress => 'Records: ') do |r|
|
186
|
+
marc_record2turtle(r, output_path)
|
195
187
|
end
|
196
|
-
# Ensure any remaining threads complete
|
197
|
-
threads.each{|t| t.join}
|
198
188
|
else
|
189
|
+
progress = ProgressBar.create(:total => auth_records.length, :format => '%a %f |%b>>%i| %P%% %t')
|
199
190
|
auth_records.each do |r|
|
200
191
|
marc_record2turtle(r, output_path)
|
201
192
|
progress.increment
|
data/lib/includes.rb
CHANGED
@@ -3,9 +3,12 @@ Dotenv.load
|
|
3
3
|
|
4
4
|
require 'addressable/uri'
|
5
5
|
require 'json'
|
6
|
-
require '
|
6
|
+
require 'rest-client'
|
7
7
|
RestClient.proxy = ENV['http_proxy'] unless ENV['http_proxy'].nil?
|
8
|
+
require 'ruby-progressbar'
|
9
|
+
|
8
10
|
require 'thread'
|
11
|
+
require 'parallel'
|
9
12
|
|
10
13
|
require 'marc'
|
11
14
|
require 'linkeddata'
|
@@ -14,7 +17,6 @@ require 'rdf/mongo'
|
|
14
17
|
|
15
18
|
require 'pry'
|
16
19
|
require 'pry-doc'
|
17
|
-
require 'ruby-progressbar'
|
18
20
|
|
19
21
|
require_relative 'marc2linkeddata/configuration'
|
20
22
|
|
data/marc2linkeddata.gemspec
CHANGED
@@ -4,7 +4,7 @@ $:.unshift lib unless $:.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |s|
|
6
6
|
s.name = 'marc2linkeddata'
|
7
|
-
s.version = '0.1.
|
7
|
+
s.version = '0.1.3'
|
8
8
|
s.licenses = ['Apache-2.0']
|
9
9
|
|
10
10
|
# mysql and bson_ext only install on MRI (c-ruby)
|
@@ -19,35 +19,36 @@ Gem::Specification.new do |s|
|
|
19
19
|
#s.required_rubygems_version = '>= 1.3.6'
|
20
20
|
# s.required_ruby_version = '>= 2.1.0'
|
21
21
|
|
22
|
-
s.add_dependency 'addressable'
|
23
|
-
s.add_dependency 'linkeddata'
|
24
|
-
s.add_dependency 'marc'
|
25
|
-
s.add_dependency '
|
26
|
-
s.add_dependency '
|
22
|
+
s.add_dependency 'addressable', '~> 2.3'
|
23
|
+
s.add_dependency 'linkeddata', '~> 1.0'
|
24
|
+
s.add_dependency 'marc', '~> 1.0'
|
25
|
+
s.add_dependency 'parallel', '~> 1.0'
|
26
|
+
s.add_dependency 'rest-client', '~> 1.0'
|
27
|
+
s.add_dependency 'ruby-progressbar', '~> 1.0'
|
27
28
|
|
28
29
|
# DB clients
|
29
|
-
s.add_dependency 'mysql' # not for jruby
|
30
|
-
s.add_dependency 'sequel'
|
30
|
+
s.add_dependency 'mysql', '~> 2.0' # not for jruby
|
31
|
+
s.add_dependency 'sequel', '~> 4.0'
|
31
32
|
|
32
33
|
# Use ENV for config
|
33
|
-
s.add_dependency 'dotenv'
|
34
|
+
s.add_dependency 'dotenv', '~> 1.0'
|
34
35
|
|
35
36
|
# ruby gem for RDF on 4store, see https://github.com/emk/rdf-4store
|
36
37
|
s.add_dependency 'rdf-4store'
|
37
38
|
# ruby gem for RDF on allegrograph, see https://github.com/emk/rdf-agraph
|
38
39
|
s.add_dependency 'rdf-agraph'
|
39
40
|
# ruby gem for RDF on mongodb, see https://rubygems.org/gems/rdf-mongo
|
40
|
-
s.add_dependency 'bson_ext' # not for jruby
|
41
|
-
s.add_dependency 'rdf-mongo'
|
41
|
+
s.add_dependency 'bson_ext', '~> 1.0' # not for jruby
|
42
|
+
s.add_dependency 'rdf-mongo', '~> 1.0'
|
42
43
|
|
43
44
|
# cache simple RDF on redis
|
44
45
|
s.add_dependency 'hiredis'
|
45
|
-
s.add_dependency 'redis'
|
46
|
+
s.add_dependency 'redis', '~> 3.0'
|
46
47
|
|
47
48
|
# Use pry for console and debug config
|
48
49
|
s.add_dependency 'pry'
|
49
50
|
s.add_dependency 'pry-doc'
|
50
|
-
s.add_development_dependency 'rspec'
|
51
|
+
s.add_development_dependency 'rspec', '~> 3.0'
|
51
52
|
|
52
53
|
s.files = `git ls-files`.split($/)
|
53
54
|
dev_files = %w(.gitignore bin/setup.sh bin/test.sh)
|
metadata
CHANGED
@@ -1,127 +1,141 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: marc2linkeddata
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Darren Weber
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-02-
|
11
|
+
date: 2015-02-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: addressable
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - "
|
17
|
+
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '
|
19
|
+
version: '2.3'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- - "
|
24
|
+
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '
|
26
|
+
version: '2.3'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: linkeddata
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- - "
|
31
|
+
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: '0'
|
33
|
+
version: '1.0'
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- - "
|
38
|
+
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: '0'
|
40
|
+
version: '1.0'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: marc
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- - "
|
45
|
+
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: '0'
|
47
|
+
version: '1.0'
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- - "
|
52
|
+
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: '0'
|
54
|
+
version: '1.0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: parallel
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '1.0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '1.0'
|
55
69
|
- !ruby/object:Gem::Dependency
|
56
70
|
name: rest-client
|
57
71
|
requirement: !ruby/object:Gem::Requirement
|
58
72
|
requirements:
|
59
|
-
- - "
|
73
|
+
- - "~>"
|
60
74
|
- !ruby/object:Gem::Version
|
61
|
-
version: '0'
|
75
|
+
version: '1.0'
|
62
76
|
type: :runtime
|
63
77
|
prerelease: false
|
64
78
|
version_requirements: !ruby/object:Gem::Requirement
|
65
79
|
requirements:
|
66
|
-
- - "
|
80
|
+
- - "~>"
|
67
81
|
- !ruby/object:Gem::Version
|
68
|
-
version: '0'
|
82
|
+
version: '1.0'
|
69
83
|
- !ruby/object:Gem::Dependency
|
70
84
|
name: ruby-progressbar
|
71
85
|
requirement: !ruby/object:Gem::Requirement
|
72
86
|
requirements:
|
73
|
-
- - "
|
87
|
+
- - "~>"
|
74
88
|
- !ruby/object:Gem::Version
|
75
|
-
version: '0'
|
89
|
+
version: '1.0'
|
76
90
|
type: :runtime
|
77
91
|
prerelease: false
|
78
92
|
version_requirements: !ruby/object:Gem::Requirement
|
79
93
|
requirements:
|
80
|
-
- - "
|
94
|
+
- - "~>"
|
81
95
|
- !ruby/object:Gem::Version
|
82
|
-
version: '0'
|
96
|
+
version: '1.0'
|
83
97
|
- !ruby/object:Gem::Dependency
|
84
98
|
name: mysql
|
85
99
|
requirement: !ruby/object:Gem::Requirement
|
86
100
|
requirements:
|
87
|
-
- - "
|
101
|
+
- - "~>"
|
88
102
|
- !ruby/object:Gem::Version
|
89
|
-
version: '0'
|
103
|
+
version: '2.0'
|
90
104
|
type: :runtime
|
91
105
|
prerelease: false
|
92
106
|
version_requirements: !ruby/object:Gem::Requirement
|
93
107
|
requirements:
|
94
|
-
- - "
|
108
|
+
- - "~>"
|
95
109
|
- !ruby/object:Gem::Version
|
96
|
-
version: '0'
|
110
|
+
version: '2.0'
|
97
111
|
- !ruby/object:Gem::Dependency
|
98
112
|
name: sequel
|
99
113
|
requirement: !ruby/object:Gem::Requirement
|
100
114
|
requirements:
|
101
|
-
- - "
|
115
|
+
- - "~>"
|
102
116
|
- !ruby/object:Gem::Version
|
103
|
-
version: '0'
|
117
|
+
version: '4.0'
|
104
118
|
type: :runtime
|
105
119
|
prerelease: false
|
106
120
|
version_requirements: !ruby/object:Gem::Requirement
|
107
121
|
requirements:
|
108
|
-
- - "
|
122
|
+
- - "~>"
|
109
123
|
- !ruby/object:Gem::Version
|
110
|
-
version: '0'
|
124
|
+
version: '4.0'
|
111
125
|
- !ruby/object:Gem::Dependency
|
112
126
|
name: dotenv
|
113
127
|
requirement: !ruby/object:Gem::Requirement
|
114
128
|
requirements:
|
115
|
-
- - "
|
129
|
+
- - "~>"
|
116
130
|
- !ruby/object:Gem::Version
|
117
|
-
version: '0'
|
131
|
+
version: '1.0'
|
118
132
|
type: :runtime
|
119
133
|
prerelease: false
|
120
134
|
version_requirements: !ruby/object:Gem::Requirement
|
121
135
|
requirements:
|
122
|
-
- - "
|
136
|
+
- - "~>"
|
123
137
|
- !ruby/object:Gem::Version
|
124
|
-
version: '0'
|
138
|
+
version: '1.0'
|
125
139
|
- !ruby/object:Gem::Dependency
|
126
140
|
name: rdf-4store
|
127
141
|
requirement: !ruby/object:Gem::Requirement
|
@@ -154,30 +168,30 @@ dependencies:
|
|
154
168
|
name: bson_ext
|
155
169
|
requirement: !ruby/object:Gem::Requirement
|
156
170
|
requirements:
|
157
|
-
- - "
|
171
|
+
- - "~>"
|
158
172
|
- !ruby/object:Gem::Version
|
159
|
-
version: '0'
|
173
|
+
version: '1.0'
|
160
174
|
type: :runtime
|
161
175
|
prerelease: false
|
162
176
|
version_requirements: !ruby/object:Gem::Requirement
|
163
177
|
requirements:
|
164
|
-
- - "
|
178
|
+
- - "~>"
|
165
179
|
- !ruby/object:Gem::Version
|
166
|
-
version: '0'
|
180
|
+
version: '1.0'
|
167
181
|
- !ruby/object:Gem::Dependency
|
168
182
|
name: rdf-mongo
|
169
183
|
requirement: !ruby/object:Gem::Requirement
|
170
184
|
requirements:
|
171
|
-
- - "
|
185
|
+
- - "~>"
|
172
186
|
- !ruby/object:Gem::Version
|
173
|
-
version: '0'
|
187
|
+
version: '1.0'
|
174
188
|
type: :runtime
|
175
189
|
prerelease: false
|
176
190
|
version_requirements: !ruby/object:Gem::Requirement
|
177
191
|
requirements:
|
178
|
-
- - "
|
192
|
+
- - "~>"
|
179
193
|
- !ruby/object:Gem::Version
|
180
|
-
version: '0'
|
194
|
+
version: '1.0'
|
181
195
|
- !ruby/object:Gem::Dependency
|
182
196
|
name: hiredis
|
183
197
|
requirement: !ruby/object:Gem::Requirement
|
@@ -196,16 +210,16 @@ dependencies:
|
|
196
210
|
name: redis
|
197
211
|
requirement: !ruby/object:Gem::Requirement
|
198
212
|
requirements:
|
199
|
-
- - "
|
213
|
+
- - "~>"
|
200
214
|
- !ruby/object:Gem::Version
|
201
|
-
version: '0'
|
215
|
+
version: '3.0'
|
202
216
|
type: :runtime
|
203
217
|
prerelease: false
|
204
218
|
version_requirements: !ruby/object:Gem::Requirement
|
205
219
|
requirements:
|
206
|
-
- - "
|
220
|
+
- - "~>"
|
207
221
|
- !ruby/object:Gem::Version
|
208
|
-
version: '0'
|
222
|
+
version: '3.0'
|
209
223
|
- !ruby/object:Gem::Dependency
|
210
224
|
name: pry
|
211
225
|
requirement: !ruby/object:Gem::Requirement
|
@@ -238,16 +252,16 @@ dependencies:
|
|
238
252
|
name: rspec
|
239
253
|
requirement: !ruby/object:Gem::Requirement
|
240
254
|
requirements:
|
241
|
-
- - "
|
255
|
+
- - "~>"
|
242
256
|
- !ruby/object:Gem::Version
|
243
|
-
version: '0'
|
257
|
+
version: '3.0'
|
244
258
|
type: :development
|
245
259
|
prerelease: false
|
246
260
|
version_requirements: !ruby/object:Gem::Requirement
|
247
261
|
requirements:
|
248
|
-
- - "
|
262
|
+
- - "~>"
|
249
263
|
- !ruby/object:Gem::Version
|
250
|
-
version: '0'
|
264
|
+
version: '3.0'
|
251
265
|
description: Utilities for translation of Marc21 records to linked open data.
|
252
266
|
email:
|
253
267
|
- dlweber@stanford.edu
|