dwc-archive 0.5.0 → 0.5.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +1 -0
- data/VERSION +1 -1
- data/features/step_definitions/dwc-reader_steps.rb +3 -4
- data/lib/dwc-archive.rb +13 -0
- data/lib/dwc-archive/ingester.rb +8 -4
- metadata +8 -5
data/.gitignore
CHANGED
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.5.
|
1
|
+
0.5.1
|
@@ -143,21 +143,20 @@ end
|
|
143
143
|
Then /^I can read its core content using block$/ do
|
144
144
|
res = []
|
145
145
|
@dwc.core.ignore_headers.should be_true
|
146
|
-
|
146
|
+
read_result = @dwc.core.read(200) do |r, err|
|
147
147
|
res << [r.size, err.size]
|
148
148
|
end
|
149
|
-
res << [tail_data.size, tail_errors.size]
|
150
149
|
res.should == [[198,2],[200,0],[186,1]]
|
150
|
+
read_result[0].size.should == 186
|
151
151
|
end
|
152
152
|
|
153
153
|
Then /^I can read extensions content using block$/ do
|
154
154
|
res = []
|
155
155
|
ext = @dwc.extensions[0]
|
156
156
|
ext.ignore_headers.should be_true
|
157
|
-
|
157
|
+
ext.read(200) do |r, err|
|
158
158
|
res << [r.size, err.size]
|
159
159
|
end
|
160
|
-
res << [tail_data.size, tail_errors.size]
|
161
160
|
res.should == [[1,0]]
|
162
161
|
end
|
163
162
|
|
data/lib/dwc-archive.rb
CHANGED
@@ -16,6 +16,7 @@ else
|
|
16
16
|
end
|
17
17
|
end
|
18
18
|
|
19
|
+
require 'logger'
|
19
20
|
require 'dwc-archive/ingester'
|
20
21
|
require 'dwc-archive/errors'
|
21
22
|
require 'dwc-archive/expander'
|
@@ -51,6 +52,18 @@ class DarwinCore
|
|
51
52
|
end
|
52
53
|
end
|
53
54
|
|
55
|
+
def self.logger
|
56
|
+
@@logger ||= Logger.new(nil)
|
57
|
+
end
|
58
|
+
|
59
|
+
def self.logger=(logger)
|
60
|
+
@@logger = logger
|
61
|
+
end
|
62
|
+
|
63
|
+
def self.logger_reset
|
64
|
+
@@logger = Logger.new(nil)
|
65
|
+
end
|
66
|
+
|
54
67
|
def initialize(dwc_path, tmp_dir = DEFAULT_TMP_DIR)
|
55
68
|
@archive = DarwinCore::Archive.new(dwc_path, tmp_dir)
|
56
69
|
@core = DarwinCore::Core.new(@archive)
|
data/lib/dwc-archive/ingester.rb
CHANGED
@@ -12,12 +12,16 @@ class DarwinCore
|
|
12
12
|
CSV.open(@file_path, args).each_with_index do |r, i|
|
13
13
|
index_fix = 0; next if @ignore_headers && i == 0
|
14
14
|
min_size > r.size ? errors << r : process_csv_row(res, errors, r)
|
15
|
-
if
|
16
|
-
|
17
|
-
|
18
|
-
|
15
|
+
if (i + index_fix) % batch_size == 0
|
16
|
+
DarwinCore.logger.info("%s| Ingested %s records" % [self.object_id, (i + index_fix)])
|
17
|
+
if block_given?
|
18
|
+
yield [res, errors]
|
19
|
+
res = []
|
20
|
+
errors = []
|
21
|
+
end
|
19
22
|
end
|
20
23
|
end
|
24
|
+
yield [res, errors] if block_given?
|
21
25
|
[res, errors]
|
22
26
|
end
|
23
27
|
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: dwc-archive
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 9
|
5
5
|
prerelease: false
|
6
6
|
segments:
|
7
7
|
- 0
|
8
8
|
- 5
|
9
|
-
-
|
10
|
-
version: 0.5.
|
9
|
+
- 1
|
10
|
+
version: 0.5.1
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Dmitry Mozzherin
|
@@ -66,8 +66,9 @@ dependencies:
|
|
66
66
|
version_requirements: *id003
|
67
67
|
description: Darwin Core Archive is the current standard exchange format for GLobal Names Architecture modules. This gem makes it easy to incorporate files in Darwin Core Archive format into a ruby project.
|
68
68
|
email: dmozzherin at gmail dot com
|
69
|
-
executables:
|
70
|
-
|
69
|
+
executables:
|
70
|
+
- preparse.rb
|
71
|
+
- t
|
71
72
|
extensions: []
|
72
73
|
|
73
74
|
extra_rdoc_files:
|
@@ -118,6 +119,8 @@ files:
|
|
118
119
|
- spec/lib/ruby_extenstions_spec.rb
|
119
120
|
- spec/spec.opts
|
120
121
|
- spec/spec_helper.rb
|
122
|
+
- bin/preparse.rb
|
123
|
+
- bin/t
|
121
124
|
has_rdoc: true
|
122
125
|
homepage: http://github.com/GlobalNamesArchitecture/dwc-archive
|
123
126
|
licenses: []
|