cdmbl 0.2.19 → 0.2.20
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/cdmbl/etl_worker.rb +6 -1
- data/lib/cdmbl/tasks/etl.rake +2 -1
- data/lib/cdmbl/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 3d015ba13c6af1dbce4fdd610f1445c1e7a62050
|
|
4
|
+
data.tar.gz: 5374bec7147146fb4c69b1728013bb05ca51462e
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 77de9a18808ce929ad3ed8a1e920d62d2300e85608d5ea7afad8402a3d0e4cd31593c942264521d97ea17f455f6d0ca2c73822fd70744bf9154d87646dabdf51
|
|
7
|
+
data.tar.gz: 3a98ae96bf1dac23a5af699f4a057d658fbcd7b9c4100ca5b9122c61a93e0abc1705022880cfd17df8642ceccb5cf28173f7b6ce88312b6664f107812b5cb427
|
data/lib/cdmbl/etl_worker.rb
CHANGED
|
@@ -3,14 +3,17 @@ module CDMBL
|
|
|
3
3
|
class ETLWorker
|
|
4
4
|
attr_reader :solr_config,
|
|
5
5
|
:etl_config,
|
|
6
|
+
:batch_size,
|
|
6
7
|
:is_recursive,
|
|
7
8
|
:identifiers,
|
|
8
9
|
:deletables,
|
|
9
10
|
:sets
|
|
10
11
|
|
|
11
12
|
include Sidekiq::Worker
|
|
13
|
+
|
|
12
14
|
def perform(solr_config,
|
|
13
15
|
etl_config,
|
|
16
|
+
batch_size = 10,
|
|
14
17
|
is_recursive = true,
|
|
15
18
|
identifiers = [],
|
|
16
19
|
deletables = [],
|
|
@@ -22,6 +25,7 @@ module CDMBL
|
|
|
22
25
|
@identifiers = identifiers
|
|
23
26
|
@deletables = deletables
|
|
24
27
|
@sets = sets
|
|
28
|
+
@batch_size = batch_size
|
|
25
29
|
|
|
26
30
|
if !identifiers.empty?
|
|
27
31
|
load!
|
|
@@ -39,11 +43,12 @@ module CDMBL
|
|
|
39
43
|
|
|
40
44
|
def ingest_batches!
|
|
41
45
|
sent_deleted = false
|
|
42
|
-
extraction.local_identifiers.each_slice(
|
|
46
|
+
extraction.local_identifiers.each_slice(batch_size) do |ids|
|
|
43
47
|
delete_ids = (sent_deleted == false) ? extraction.deletable_ids : []
|
|
44
48
|
CDMBL::LoaderNotification.call!(ids, delete_ids)
|
|
45
49
|
ETLWorker.perform_async(solr_config,
|
|
46
50
|
etl_config,
|
|
51
|
+
batch_size,
|
|
47
52
|
is_recursive,
|
|
48
53
|
ids,
|
|
49
54
|
delete_ids,
|
data/lib/cdmbl/tasks/etl.rake
CHANGED
|
@@ -6,7 +6,8 @@ namespace :cdmbl do
|
|
|
6
6
|
solr_config = { url: args[solr_url] }
|
|
7
7
|
etl_config = { oai_endpoint: args[:oai_endpoint], cdm_endpoint: args[:cdm_endpoint], minimum_date: args[:minimum_date] }
|
|
8
8
|
etl_config = (args[:resumption_token]) ? etl_cofig.merge(args[:resumption_token]) : etl_config
|
|
9
|
-
|
|
9
|
+
batch_size = (args[:batch_size]) ? args[:batch_size] : 10
|
|
10
|
+
CDMBL::ETLWorker.perform_async(solr_config, etl_config, batch_size)
|
|
10
11
|
end
|
|
11
12
|
end
|
|
12
13
|
|
data/lib/cdmbl/version.rb
CHANGED