maltese 0.1.6 → 0.1.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (14) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile.lock +1 -1
  3. data/lib/maltese/cli.rb +3 -3
  4. data/lib/maltese/sitemap.rb +7 -6
  5. data/lib/maltese/version.rb +1 -1
  6. data/spec/cli_spec.rb +4 -12
  7. data/spec/fixtures/vcr_cassettes/Maltese_CLI/sitemap/should_succeed.yml +21 -21
  8. data/spec/fixtures/vcr_cassettes/Maltese_Sitemap/push_data/should_report_if_there_are_no_works_returned_by_the_Datacite_Solr_API.yml +13 -13
  9. data/spec/fixtures/vcr_cassettes/Maltese_Sitemap/push_data/should_report_if_there_are_works_returned_by_the_Datacite_Solr_API.yml +129 -15
  10. data/spec/fixtures/vcr_cassettes/Maltese_Sitemap/queue_jobs/should_report_if_there_are_works_returned_by_the_Datacite_Solr_API.yml +18 -132
  11. data/spec/spec_helper.rb +1 -1
  12. metadata +1 -3
  13. data/lib/maltese/utils.rb +0 -87
  14. data/public/sitemap.xml.gz +0 -0
data/spec/fixtures/vcr_cassettes/Maltese_Sitemap/push_data/should_report_if_there_are_no_works_returned_by_the_Datacite_Solr_API.yml CHANGED
@@ -10,23 +10,23 @@ http_interactions:
10
10
  User-Agent:
11
11
  - fog-core/1.37.0
12
12
  X-Amz-Date:
13
- - 20170226T200314Z
13
+ - 20170226T231704Z
14
14
  X-Amz-Content-Sha256:
15
15
  - e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
16
16
  Authorization:
17
17
  - AWS4-HMAC-SHA256 Credential=<AWS_ACCESS_KEY_ID>/20170226/us-east-1/s3/aws4_request,
18
- SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=8bf9b050f46afa1bcb96a9caf5a1400824b680c9c43b98a2e283d3c56b5c01cb
18
+ SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=e7ab88d4f355ede1b9a312b6ad3d72fdfd1c921c87d229a45b758576115d5c1f
19
19
  response:
20
20
  status:
21
21
  code: 200
22
22
  message: ''
23
23
  headers:
24
24
  X-Amz-Id-2:
25
- - S8Z0IuaR+OA96lrRz/wO6ujQrKDHEZQLUgrQJlz2Eq9DlOLQ1eIkV+8Zc9M/ecyWwXIAyGO3TCg=
25
+ - ZQuecF0xSq/j7wEmiyzLd5SdChGL+GP6rs2rB/6dVutYOjLO/16M9pB4vwHJTeE9FXx+xkGgGHI=
26
26
  X-Amz-Request-Id:
27
- - EA318DDA19F6A296
27
+ - B4BF21A1FAEAF07C
28
28
  Date:
29
- - Sun, 26 Feb 2017 20:03:15 GMT
29
+ - Sun, 26 Feb 2017 23:17:06 GMT
30
30
  Content-Type:
31
31
  - application/xml
32
32
  Server:
@@ -35,12 +35,12 @@ http_interactions:
35
35
  encoding: UTF-8
36
36
  string: |-
37
37
  <?xml version="1.0" encoding="UTF-8"?>
38
- <ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Owner><ID>8170cc616e9d1833db4bade0c7af0010f04464343a1004b67a00fbcb41e361de</ID><DisplayName>infrastructure</DisplayName></Owner><Buckets><Bucket><Name>api.datacite.org</Name><CreationDate>2017-02-09T07:05:20.000Z</CreationDate></Bucket><Bucket><Name>api.test.datacite.org</Name><CreationDate>2017-02-09T07:05:21.000Z</CreationDate></Bucket><Bucket><Name>archive.datacite.org</Name><CreationDate>2016-12-26T07:02:24.000Z</CreationDate></Bucket><Bucket><Name>archive.test.datacite.org</Name><CreationDate>2017-01-13T19:04:30.000Z</CreationDate></Bucket><Bucket><Name>assets.datacite.org</Name><CreationDate>2016-04-29T17:51:32.000Z</CreationDate></Bucket><Bucket><Name>assets.test.datacite.org</Name><CreationDate>2017-01-13T18:47:02.000Z</CreationDate></Bucket><Bucket><Name>blog.datacite.org</Name><CreationDate>2015-09-26T08:59:21.000Z</CreationDate></Bucket><Bucket><Name>blog.test.datacite.org</Name><CreationDate>2017-01-13T18:54:23.000Z</CreationDate></Bucket><Bucket><Name>crosscite.org</Name><CreationDate>2016-01-05T22:11:03.000Z</CreationDate></Bucket><Bucket><Name>data.datacite.org</Name><CreationDate>2016-08-29T17:11:58.000Z</CreationDate></Bucket><Bucket><Name>data.test.datacite.org</Name><CreationDate>2017-01-13T20:17:34.000Z</CreationDate></Bucket><Bucket><Name>datacite.org</Name><CreationDate>2016-01-04T21:53:46.000Z</CreationDate></Bucket><Bucket><Name>lagotto.io</Name><CreationDate>2016-01-10T10:12:40.000Z</CreationDate></Bucket><Bucket><Name>ops.datacite.org</Name><CreationDate>2016-06-03T19:38:04.000Z</CreationDate></Bucket><Bucket><Name>papertrail.datacite.org</Name><CreationDate>2016-01-04T23:25:19.000Z</CreationDate></Bucket><Bucket><Name>pidapalooza.org</Name><CreationDate>2016-06-16T07:28:25.000Z</CreationDate></Bucket><Bucket><Name>schema.datacite.org</Name><CreationDate>2015-09-26T08:01:48.000Z</CreationDate></Bucket><Bucket><Name>schema.test.datacite.org</Name><CreationDate>2017-01-10T11:47:55.000Z</CreationDate></Bucket><Bucket><Name>search.datacite.org</Name><CreationDate>2017-02-26T20:00:32.000Z</CreationDate></Bucket><Bucket><Name>search.test.datacite.org</Name><CreationDate>2017-02-26T20:00:32.000Z</CreationDate></Bucket><Bucket><Name>sitemaps.datacite.org</Name><CreationDate>2017-01-12T21:41:18.000Z</CreationDate></Bucket><Bucket><Name><FOG_DIRECTORY></Name><CreationDate>2017-01-27T07:13:20.000Z</CreationDate></Bucket><Bucket><Name>stats.datacite.org</Name><CreationDate>2016-08-29T10:21:19.000Z</CreationDate></Bucket><Bucket><Name>stats.test.datacite.org</Name><CreationDate>2017-01-13T20:17:34.000Z</CreationDate></Bucket><Bucket><Name>test.datacite.org</Name><CreationDate>2017-01-08T21:50:53.000Z</CreationDate></Bucket><Bucket><Name>www.crosscite.org</Name><CreationDate>2017-01-09T10:18:11.000Z</CreationDate></Bucket><Bucket><Name>www.datacite.org</Name><CreationDate>2015-09-26T08:01:48.000Z</CreationDate></Bucket><Bucket><Name>www.test.datacite.org</Name><CreationDate>2017-01-13T19:50:11.000Z</CreationDate></Bucket></Buckets></ListAllMyBucketsResult>
38
+ <ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Owner><ID>8170cc616e9d1833db4bade0c7af0010f04464343a1004b67a00fbcb41e361de</ID><DisplayName>infrastructure</DisplayName></Owner><Buckets><Bucket><Name>api.datacite.org</Name><CreationDate>2017-02-09T07:05:20.000Z</CreationDate></Bucket><Bucket><Name>api.test.datacite.org</Name><CreationDate>2017-02-09T07:05:21.000Z</CreationDate></Bucket><Bucket><Name>archive.datacite.org</Name><CreationDate>2016-12-26T07:02:24.000Z</CreationDate></Bucket><Bucket><Name>archive.test.datacite.org</Name><CreationDate>2017-01-13T19:04:30.000Z</CreationDate></Bucket><Bucket><Name>assets.datacite.org</Name><CreationDate>2016-04-29T17:51:32.000Z</CreationDate></Bucket><Bucket><Name>assets.test.datacite.org</Name><CreationDate>2017-01-13T18:47:02.000Z</CreationDate></Bucket><Bucket><Name>blog.datacite.org</Name><CreationDate>2015-09-26T08:59:21.000Z</CreationDate></Bucket><Bucket><Name>blog.test.datacite.org</Name><CreationDate>2017-01-13T18:54:23.000Z</CreationDate></Bucket><Bucket><Name>crosscite.org</Name><CreationDate>2016-01-05T22:11:03.000Z</CreationDate></Bucket><Bucket><Name>data.datacite.org</Name><CreationDate>2016-08-29T17:11:58.000Z</CreationDate></Bucket><Bucket><Name>data.test.datacite.org</Name><CreationDate>2017-01-13T20:17:34.000Z</CreationDate></Bucket><Bucket><Name>datacite.org</Name><CreationDate>2016-01-04T21:53:46.000Z</CreationDate></Bucket><Bucket><Name>lagotto.io</Name><CreationDate>2016-01-10T10:12:40.000Z</CreationDate></Bucket><Bucket><Name>ops.datacite.org</Name><CreationDate>2016-06-03T19:38:04.000Z</CreationDate></Bucket><Bucket><Name>papertrail.datacite.org</Name><CreationDate>2016-01-04T23:25:19.000Z</CreationDate></Bucket><Bucket><Name>pidapalooza.org</Name><CreationDate>2016-06-16T07:28:25.000Z</CreationDate></Bucket><Bucket><Name>schema.datacite.org</Name><CreationDate>2015-09-26T08:01:48.000Z</CreationDate></Bucket><Bucket><Name>schema.test.datacite.org</Name><CreationDate>2017-01-10T11:47:55.000Z</CreationDate></Bucket><Bucket><Name>search.datacite.org</Name><CreationDate>2017-02-26T20:00:32.000Z</CreationDate></Bucket><Bucket><Name>search.test.datacite.org</Name><CreationDate>2017-02-26T20:00:32.000Z</CreationDate></Bucket><Bucket><Name>stats.datacite.org</Name><CreationDate>2016-08-29T10:21:19.000Z</CreationDate></Bucket><Bucket><Name>stats.test.datacite.org</Name><CreationDate>2017-01-13T20:17:34.000Z</CreationDate></Bucket><Bucket><Name>test.datacite.org</Name><CreationDate>2017-01-08T21:50:53.000Z</CreationDate></Bucket><Bucket><Name>www.crosscite.org</Name><CreationDate>2017-01-09T10:18:11.000Z</CreationDate></Bucket><Bucket><Name>www.datacite.org</Name><CreationDate>2015-09-26T08:01:48.000Z</CreationDate></Bucket><Bucket><Name>www.test.datacite.org</Name><CreationDate>2017-01-13T19:50:11.000Z</CreationDate></Bucket></Buckets></ListAllMyBucketsResult>
39
39
  http_version:
40
40
  recorded_at: Tue, 07 Apr 2015 22:00:00 GMT
41
41
  - request:
42
42
  method: put
43
- uri: https://s3-eu-west-1.amazonaws.com/search.datacite.org/sitemaps/sitemap.xml.gz
43
+ uri: https://s3-eu-west-1.amazonaws.com/search.datacite.org/sitemaps-test/sitemap.xml.gz
44
44
  body:
45
45
  encoding: US-ASCII
46
46
  string: ''
@@ -52,7 +52,7 @@ http_interactions:
52
52
  X-Amz-Acl:
53
53
  - public-read
54
54
  X-Amz-Date:
55
- - 20170226T200315Z
55
+ - 20170226T231706Z
56
56
  X-Amz-Content-Sha256:
57
57
  - STREAMING-AWS4-HMAC-SHA256-PAYLOAD
58
58
  X-Amz-Decoded-Content-Length:
@@ -60,20 +60,20 @@ http_interactions:
60
60
  Authorization:
61
61
  - AWS4-HMAC-SHA256 Credential=<AWS_ACCESS_KEY_ID>/20170226/eu-west-1/s3/aws4_request,
62
62
  SignedHeaders=content-type;host;x-amz-acl;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length,
63
- Signature=d2d072afd0b1056cbce31ac662b97015c35ec964439d1ac909cdf04cd4afc2cd
63
+ Signature=283085acb7b37369e262e1b0c5e4c38e2e7bac57843fc4410466dc9b3e36fe65
64
64
  response:
65
65
  status:
66
66
  code: 200
67
67
  message: ''
68
68
  headers:
69
69
  X-Amz-Id-2:
70
- - ICzSpRQYl/ZPkip6g45LNrcoMaZKUvBEBCRt3zoOAFiDb/mP6t7AqfQ9o3rtBBMPPpTu8Ez3I7M=
70
+ - Sxubg7JQjr/97ZhF/T7dDhmbsAVOVIqrmMN0ZdpAO6aXm8O/io0MOGGiubgWhUw3C9lJYjsGprc=
71
71
  X-Amz-Request-Id:
72
- - D1EDFD25AF752851
72
+ - ED7CC1EEB6DFF0E0
73
73
  Date:
74
- - Sun, 26 Feb 2017 20:03:15 GMT
74
+ - Sun, 26 Feb 2017 23:17:06 GMT
75
75
  Etag:
76
- - '"03822532fa15267acf77702062e41664"'
76
+ - '"cec15d6b7a72b73c565fc102e43f98b2"'
77
77
  Content-Length:
78
78
  - '0'
79
79
  Server:
data/spec/fixtures/vcr_cassettes/Maltese_Sitemap/push_data/should_report_if_there_are_works_returned_by_the_Datacite_Solr_API.yml CHANGED
@@ -10,37 +10,151 @@ http_interactions:
10
10
  User-Agent:
11
11
  - fog-core/1.37.0
12
12
  X-Amz-Date:
13
- - 20170226T200311Z
13
+ - 20150407T220000Z
14
14
  X-Amz-Content-Sha256:
15
15
  - e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
16
16
  Authorization:
17
- - AWS4-HMAC-SHA256 Credential=<AWS_ACCESS_KEY_ID>/20170226/us-east-1/s3/aws4_request,
18
- SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=3a46b615f7e02160ab9979693aab49192bced1b21f5fea390796391299828f53
17
+ - AWS4-HMAC-SHA256 Credential=<AWS_ACCESS_KEY_ID>/20150407/us-east-1/s3/aws4_request,
18
+ SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=2fe3044f0e0fd2b7ad2caeb9e6af67ab3de66969b6bbe67cf4e6d53621e33391
19
19
  response:
20
20
  status:
21
- code: 200
21
+ code: 403
22
22
  message: ''
23
23
  headers:
24
+ X-Amz-Request-Id:
25
+ - AFF611ADFC0BB4FF
24
26
  X-Amz-Id-2:
25
- - H+N9UJPnLIZlf3HFJyzanLRYuYCCEnhQpbkDM4ktSe9x3fYtZT5QyBM0C9vv50OcmC9EvGOGLEs=
27
+ - "/bNnD4tgFXPfaowrE4eiO7FozZYXEYgHiENHDySPcRCX35tWZjANdnIBsCC9Q6kkixEiusHKQBU="
28
+ Content-Type:
29
+ - application/xml
30
+ Date:
31
+ - Sun, 26 Feb 2017 23:17:02 GMT
32
+ Server:
33
+ - AmazonS3
34
+ body:
35
+ encoding: UTF-8
36
+ string: |-
37
+ <?xml version="1.0" encoding="UTF-8"?>
38
+ <Error><Code>RequestTimeTooSkewed</Code><Message>The difference between the request time and the current time is too large.</Message><RequestTime>20150407T220000Z</RequestTime><ServerTime>2017-02-26T23:17:04Z</ServerTime><MaxAllowedSkewMilliseconds>900000</MaxAllowedSkewMilliseconds><RequestId>AFF611ADFC0BB4FF</RequestId><HostId>/bNnD4tgFXPfaowrE4eiO7FozZYXEYgHiENHDySPcRCX35tWZjANdnIBsCC9Q6kkixEiusHKQBU=</HostId></Error>
39
+ http_version:
40
+ recorded_at: Tue, 07 Apr 2015 22:00:00 GMT
41
+ - request:
42
+ method: get
43
+ uri: https://s3.amazonaws.com/
44
+ body:
45
+ encoding: US-ASCII
46
+ string: ''
47
+ headers:
48
+ User-Agent:
49
+ - fog-core/1.37.0
50
+ X-Amz-Date:
51
+ - 20150407T220000Z
52
+ X-Amz-Content-Sha256:
53
+ - e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
54
+ Authorization:
55
+ - AWS4-HMAC-SHA256 Credential=<AWS_ACCESS_KEY_ID>/20150407/us-east-1/s3/aws4_request,
56
+ SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=2fe3044f0e0fd2b7ad2caeb9e6af67ab3de66969b6bbe67cf4e6d53621e33391
57
+ response:
58
+ status:
59
+ code: 403
60
+ message: ''
61
+ headers:
26
62
  X-Amz-Request-Id:
27
- - 123F33BA70B9D0D2
63
+ - 6C3AF366E95921DC
64
+ X-Amz-Id-2:
65
+ - NExE51oG2se2Z7H3VsKE+brqZWFRNxa4+GtaSv+ik5coyoqkoIc8ntppA0iWmbK/zBFJp4eq4x0=
66
+ Content-Type:
67
+ - application/xml
28
68
  Date:
29
- - Sun, 26 Feb 2017 20:03:14 GMT
69
+ - Sun, 26 Feb 2017 23:17:03 GMT
70
+ Server:
71
+ - AmazonS3
72
+ body:
73
+ encoding: UTF-8
74
+ string: |-
75
+ <?xml version="1.0" encoding="UTF-8"?>
76
+ <Error><Code>RequestTimeTooSkewed</Code><Message>The difference between the request time and the current time is too large.</Message><RequestTime>20150407T220000Z</RequestTime><ServerTime>2017-02-26T23:17:04Z</ServerTime><MaxAllowedSkewMilliseconds>900000</MaxAllowedSkewMilliseconds><RequestId>6C3AF366E95921DC</RequestId><HostId>NExE51oG2se2Z7H3VsKE+brqZWFRNxa4+GtaSv+ik5coyoqkoIc8ntppA0iWmbK/zBFJp4eq4x0=</HostId></Error>
77
+ http_version:
78
+ recorded_at: Tue, 07 Apr 2015 22:00:00 GMT
79
+ - request:
80
+ method: get
81
+ uri: https://s3.amazonaws.com/
82
+ body:
83
+ encoding: US-ASCII
84
+ string: ''
85
+ headers:
86
+ User-Agent:
87
+ - fog-core/1.37.0
88
+ X-Amz-Date:
89
+ - 20150407T220000Z
90
+ X-Amz-Content-Sha256:
91
+ - e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
92
+ Authorization:
93
+ - AWS4-HMAC-SHA256 Credential=<AWS_ACCESS_KEY_ID>/20150407/us-east-1/s3/aws4_request,
94
+ SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=2fe3044f0e0fd2b7ad2caeb9e6af67ab3de66969b6bbe67cf4e6d53621e33391
95
+ response:
96
+ status:
97
+ code: 403
98
+ message: ''
99
+ headers:
100
+ X-Amz-Request-Id:
101
+ - 330970DE0017172F
102
+ X-Amz-Id-2:
103
+ - 0z7s2hCXs5OjxgbjnpbDPuQt9r6spEdLnK+5sosrKFY8q9gyJxZ1gQivArIFwMSRrHFGwUjayt4=
30
104
  Content-Type:
31
105
  - application/xml
106
+ Date:
107
+ - Sun, 26 Feb 2017 23:17:03 GMT
108
+ Server:
109
+ - AmazonS3
110
+ body:
111
+ encoding: UTF-8
112
+ string: |-
113
+ <?xml version="1.0" encoding="UTF-8"?>
114
+ <Error><Code>RequestTimeTooSkewed</Code><Message>The difference between the request time and the current time is too large.</Message><RequestTime>20150407T220000Z</RequestTime><ServerTime>2017-02-26T23:17:04Z</ServerTime><MaxAllowedSkewMilliseconds>900000</MaxAllowedSkewMilliseconds><RequestId>330970DE0017172F</RequestId><HostId>0z7s2hCXs5OjxgbjnpbDPuQt9r6spEdLnK+5sosrKFY8q9gyJxZ1gQivArIFwMSRrHFGwUjayt4=</HostId></Error>
115
+ http_version:
116
+ recorded_at: Tue, 07 Apr 2015 22:00:00 GMT
117
+ - request:
118
+ method: get
119
+ uri: https://s3.amazonaws.com/
120
+ body:
121
+ encoding: US-ASCII
122
+ string: ''
123
+ headers:
124
+ User-Agent:
125
+ - fog-core/1.37.0
126
+ X-Amz-Date:
127
+ - 20150407T220000Z
128
+ X-Amz-Content-Sha256:
129
+ - e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
130
+ Authorization:
131
+ - AWS4-HMAC-SHA256 Credential=<AWS_ACCESS_KEY_ID>/20150407/us-east-1/s3/aws4_request,
132
+ SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=2fe3044f0e0fd2b7ad2caeb9e6af67ab3de66969b6bbe67cf4e6d53621e33391
133
+ response:
134
+ status:
135
+ code: 403
136
+ message: ''
137
+ headers:
138
+ X-Amz-Request-Id:
139
+ - B51EF66A02713CF1
140
+ X-Amz-Id-2:
141
+ - G6gHwBkRjHX7vinr62hb+Y93oAgmT/moREnayTOWuDSJN4yATUkTUasqVwpdNKuJQhuZOWJiutI=
142
+ Content-Type:
143
+ - application/xml
144
+ Date:
145
+ - Sun, 26 Feb 2017 23:17:04 GMT
32
146
  Server:
33
147
  - AmazonS3
34
148
  body:
35
149
  encoding: UTF-8
36
150
  string: |-
37
151
  <?xml version="1.0" encoding="UTF-8"?>
38
- <ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Owner><ID>8170cc616e9d1833db4bade0c7af0010f04464343a1004b67a00fbcb41e361de</ID><DisplayName>infrastructure</DisplayName></Owner><Buckets><Bucket><Name>api.datacite.org</Name><CreationDate>2017-02-09T07:05:20.000Z</CreationDate></Bucket><Bucket><Name>api.test.datacite.org</Name><CreationDate>2017-02-09T07:05:21.000Z</CreationDate></Bucket><Bucket><Name>archive.datacite.org</Name><CreationDate>2016-12-26T07:02:24.000Z</CreationDate></Bucket><Bucket><Name>archive.test.datacite.org</Name><CreationDate>2017-01-13T19:04:30.000Z</CreationDate></Bucket><Bucket><Name>assets.datacite.org</Name><CreationDate>2016-04-29T17:51:32.000Z</CreationDate></Bucket><Bucket><Name>assets.test.datacite.org</Name><CreationDate>2017-01-13T18:47:02.000Z</CreationDate></Bucket><Bucket><Name>blog.datacite.org</Name><CreationDate>2015-09-26T08:59:21.000Z</CreationDate></Bucket><Bucket><Name>blog.test.datacite.org</Name><CreationDate>2017-01-13T18:54:23.000Z</CreationDate></Bucket><Bucket><Name>crosscite.org</Name><CreationDate>2016-01-05T22:11:03.000Z</CreationDate></Bucket><Bucket><Name>data.datacite.org</Name><CreationDate>2016-08-29T17:11:58.000Z</CreationDate></Bucket><Bucket><Name>data.test.datacite.org</Name><CreationDate>2017-01-13T20:17:34.000Z</CreationDate></Bucket><Bucket><Name>datacite.org</Name><CreationDate>2016-01-04T21:53:46.000Z</CreationDate></Bucket><Bucket><Name>lagotto.io</Name><CreationDate>2016-01-10T10:12:40.000Z</CreationDate></Bucket><Bucket><Name>ops.datacite.org</Name><CreationDate>2016-06-03T19:38:04.000Z</CreationDate></Bucket><Bucket><Name>papertrail.datacite.org</Name><CreationDate>2016-01-04T23:25:19.000Z</CreationDate></Bucket><Bucket><Name>pidapalooza.org</Name><CreationDate>2016-06-16T07:28:25.000Z</CreationDate></Bucket><Bucket><Name>schema.datacite.org</Name><CreationDate>2015-09-26T08:01:48.000Z</CreationDate></Bucket><Bucket><Name>schema.test.datacite.org</Name><CreationDate>2017-01-10T11:47:55.000Z</CreationDate></Bucket><Bucket><Name>search.datacite.org</Name><CreationDate>2017-02-26T20:00:32.000Z</CreationDate></Bucket><Bucket><Name>search.test.datacite.org</Name><CreationDate>2017-02-26T20:00:32.000Z</CreationDate></Bucket><Bucket><Name>sitemaps.datacite.org</Name><CreationDate>2017-01-12T21:41:18.000Z</CreationDate></Bucket><Bucket><Name><FOG_DIRECTORY></Name><CreationDate>2017-01-27T07:13:20.000Z</CreationDate></Bucket><Bucket><Name>stats.datacite.org</Name><CreationDate>2016-08-29T10:21:19.000Z</CreationDate></Bucket><Bucket><Name>stats.test.datacite.org</Name><CreationDate>2017-01-13T20:17:34.000Z</CreationDate></Bucket><Bucket><Name>test.datacite.org</Name><CreationDate>2017-01-08T21:50:53.000Z</CreationDate></Bucket><Bucket><Name>www.crosscite.org</Name><CreationDate>2017-01-09T10:18:11.000Z</CreationDate></Bucket><Bucket><Name>www.datacite.org</Name><CreationDate>2015-09-26T08:01:48.000Z</CreationDate></Bucket><Bucket><Name>www.test.datacite.org</Name><CreationDate>2017-01-13T19:50:11.000Z</CreationDate></Bucket></Buckets></ListAllMyBucketsResult>
152
+ <Error><Code>RequestTimeTooSkewed</Code><Message>The difference between the request time and the current time is too large.</Message><RequestTime>20150407T220000Z</RequestTime><ServerTime>2017-02-26T23:17:05Z</ServerTime><MaxAllowedSkewMilliseconds>900000</MaxAllowedSkewMilliseconds><RequestId>B51EF66A02713CF1</RequestId><HostId>G6gHwBkRjHX7vinr62hb+Y93oAgmT/moREnayTOWuDSJN4yATUkTUasqVwpdNKuJQhuZOWJiutI=</HostId></Error>
39
153
  http_version:
40
154
  recorded_at: Tue, 07 Apr 2015 22:00:00 GMT
41
155
  - request:
42
156
  method: put
43
- uri: https://s3-eu-west-1.amazonaws.com/search.datacite.org/sitemaps/sitemap.xml.gz
157
+ uri: https://s3-eu-west-1.amazonaws.com/search.datacite.org/sitemaps-test/sitemap.xml.gz
44
158
  body:
45
159
  encoding: US-ASCII
46
160
  string: ''
@@ -52,7 +166,7 @@ http_interactions:
52
166
  X-Amz-Acl:
53
167
  - public-read
54
168
  X-Amz-Date:
55
- - 20170226T200314Z
169
+ - 20170226T231704Z
56
170
  X-Amz-Content-Sha256:
57
171
  - STREAMING-AWS4-HMAC-SHA256-PAYLOAD
58
172
  X-Amz-Decoded-Content-Length:
@@ -60,20 +174,20 @@ http_interactions:
60
174
  Authorization:
61
175
  - AWS4-HMAC-SHA256 Credential=<AWS_ACCESS_KEY_ID>/20170226/eu-west-1/s3/aws4_request,
62
176
  SignedHeaders=content-type;host;x-amz-acl;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length,
63
- Signature=5e9ed31b30d3738d53ac102329a9262ce9537c3438d23f327a19c28a64ad4957
177
+ Signature=7cb583308e13da8753991703d16f56513c29cceb683bdd4f04f993d6794faac3
64
178
  response:
65
179
  status:
66
180
  code: 200
67
181
  message: ''
68
182
  headers:
69
183
  X-Amz-Id-2:
70
- - hSpbSucDCoD/Zg9N6F8hkLniDZxVMUJEBVKMHgmSUFTWxn8xe3m3iW/IeRebnGOuPOATFcrB9+U=
184
+ - aYVn0IlwhiMfeiIgZngv3IsDHTkIWoLlONhJK8WE9yNHbL2rnrlQIS8K6wqFN0JmfKt0PdXXd68=
71
185
  X-Amz-Request-Id:
72
- - 3780F30B8C8A6511
186
+ - 61DAFAF29518E861
73
187
  Date:
74
- - Sun, 26 Feb 2017 20:03:14 GMT
188
+ - Sun, 26 Feb 2017 23:17:05 GMT
75
189
  Etag:
76
- - '"b87f68e059f36ee355b259fb511313b6"'
190
+ - '"157fe411131d9f960f73c9195ee315bd"'
77
191
  Content-Length:
78
192
  - '0'
79
193
  Server:
data/spec/fixtures/vcr_cassettes/Maltese_Sitemap/queue_jobs/should_report_if_there_are_works_returned_by_the_Datacite_Solr_API.yml CHANGED
@@ -19,7 +19,7 @@ http_interactions:
19
19
  Server:
20
20
  - openresty/1.11.2.2
21
21
  Date:
22
- - Sun, 26 Feb 2017 20:03:09 GMT
22
+ - Sun, 26 Feb 2017 23:17:53 GMT
23
23
  Content-Type:
24
24
  - application/json;charset=UTF-8
25
25
  Connection:
@@ -30,7 +30,7 @@ http_interactions:
30
30
  - GET, POST, OPTIONS
31
31
  body:
32
32
  encoding: UTF-8
33
- string: '{"responseHeader":{"status":0,"QTime":0},"response":{"numFound":2521,"start":0,"docs":[]}}
33
+ string: '{"responseHeader":{"status":0,"QTime":1},"response":{"numFound":2521,"start":0,"docs":[]}}
34
34
 
35
35
  '
36
36
  http_version:
@@ -54,7 +54,7 @@ http_interactions:
54
54
  Server:
55
55
  - openresty/1.11.2.2
56
56
  Date:
57
- - Sun, 26 Feb 2017 20:03:09 GMT
57
+ - Sun, 26 Feb 2017 23:17:53 GMT
58
58
  Content-Type:
59
59
  - application/json;charset=UTF-8
60
60
  Connection:
@@ -80,151 +80,37 @@ http_interactions:
80
80
  User-Agent:
81
81
  - fog-core/1.37.0
82
82
  X-Amz-Date:
83
- - 20150407T220001Z
83
+ - 20170226T231704Z
84
84
  X-Amz-Content-Sha256:
85
85
  - e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
86
86
  Authorization:
87
- - AWS4-HMAC-SHA256 Credential=<AWS_ACCESS_KEY_ID>/20150407/us-east-1/s3/aws4_request,
88
- SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=63ae9b360414ffd780af6bec925edc820e5991b83ce6eab4d03861342340d476
87
+ - AWS4-HMAC-SHA256 Credential=<AWS_ACCESS_KEY_ID>/20170226/us-east-1/s3/aws4_request,
88
+ SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=e7ab88d4f355ede1b9a312b6ad3d72fdfd1c921c87d229a45b758576115d5c1f
89
89
  response:
90
90
  status:
91
- code: 403
91
+ code: 200
92
92
  message: ''
93
93
  headers:
94
- X-Amz-Request-Id:
95
- - F1F1935B9D093F3B
96
94
  X-Amz-Id-2:
97
- - sGMBvTChZ4S+owf6quTgVg2rHd+0Vw4+0KomLLJHZziKUgbCDRItUwATwjsIT3Wldz/LNPs58jw=
98
- Content-Type:
99
- - application/xml
100
- Date:
101
- - Sun, 26 Feb 2017 20:03:09 GMT
102
- Server:
103
- - AmazonS3
104
- body:
105
- encoding: UTF-8
106
- string: |-
107
- <?xml version="1.0" encoding="UTF-8"?>
108
- <Error><Code>RequestTimeTooSkewed</Code><Message>The difference between the request time and the current time is too large.</Message><RequestTime>20150407T220001Z</RequestTime><ServerTime>2017-02-26T20:03:11Z</ServerTime><MaxAllowedSkewMilliseconds>900000</MaxAllowedSkewMilliseconds><RequestId>F1F1935B9D093F3B</RequestId><HostId>sGMBvTChZ4S+owf6quTgVg2rHd+0Vw4+0KomLLJHZziKUgbCDRItUwATwjsIT3Wldz/LNPs58jw=</HostId></Error>
109
- http_version:
110
- recorded_at: Tue, 07 Apr 2015 22:00:00 GMT
111
- - request:
112
- method: get
113
- uri: https://s3.amazonaws.com/
114
- body:
115
- encoding: US-ASCII
116
- string: ''
117
- headers:
118
- User-Agent:
119
- - fog-core/1.37.0
120
- X-Amz-Date:
121
- - 20150407T220001Z
122
- X-Amz-Content-Sha256:
123
- - e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
124
- Authorization:
125
- - AWS4-HMAC-SHA256 Credential=<AWS_ACCESS_KEY_ID>/20150407/us-east-1/s3/aws4_request,
126
- SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=63ae9b360414ffd780af6bec925edc820e5991b83ce6eab4d03861342340d476
127
- response:
128
- status:
129
- code: 403
130
- message: ''
131
- headers:
95
+ - YW0AEI4DrC2+XZqK3P9yS7wmBhYI8WSadoF94kNZhhz7Cd2KaMYcxP+s8J/QtKWEX1OLrSdgAxQ=
132
96
  X-Amz-Request-Id:
133
- - C9A6B93BA12A87BD
134
- X-Amz-Id-2:
135
- - pbCD9EJXef72LT9E3rF9Kg/Z1WwRDTZOmXOndbVm/rVbN0W2QnJyhordjReIFVGFsMG3y9cFXME=
136
- Content-Type:
137
- - application/xml
97
+ - E4F3A2FFAADFF9A2
138
98
  Date:
139
- - Sun, 26 Feb 2017 20:03:10 GMT
140
- Server:
141
- - AmazonS3
142
- body:
143
- encoding: UTF-8
144
- string: |-
145
- <?xml version="1.0" encoding="UTF-8"?>
146
- <Error><Code>RequestTimeTooSkewed</Code><Message>The difference between the request time and the current time is too large.</Message><RequestTime>20150407T220001Z</RequestTime><ServerTime>2017-02-26T20:03:11Z</ServerTime><MaxAllowedSkewMilliseconds>900000</MaxAllowedSkewMilliseconds><RequestId>C9A6B93BA12A87BD</RequestId><HostId>pbCD9EJXef72LT9E3rF9Kg/Z1WwRDTZOmXOndbVm/rVbN0W2QnJyhordjReIFVGFsMG3y9cFXME=</HostId></Error>
147
- http_version:
148
- recorded_at: Tue, 07 Apr 2015 22:00:00 GMT
149
- - request:
150
- method: get
151
- uri: https://s3.amazonaws.com/
152
- body:
153
- encoding: US-ASCII
154
- string: ''
155
- headers:
156
- User-Agent:
157
- - fog-core/1.37.0
158
- X-Amz-Date:
159
- - 20150407T220001Z
160
- X-Amz-Content-Sha256:
161
- - e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
162
- Authorization:
163
- - AWS4-HMAC-SHA256 Credential=<AWS_ACCESS_KEY_ID>/20150407/us-east-1/s3/aws4_request,
164
- SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=63ae9b360414ffd780af6bec925edc820e5991b83ce6eab4d03861342340d476
165
- response:
166
- status:
167
- code: 403
168
- message: ''
169
- headers:
170
- X-Amz-Request-Id:
171
- - 04FBF6BEDA1B20A8
172
- X-Amz-Id-2:
173
- - XuOHzBTtiAFfNV4wZEED7xsUWIxn11sYx5uM47rWoQjfeDnW1TH4/+5FBTsjGBeen1XJ4h1ywbA=
99
+ - Sun, 26 Feb 2017 23:17:55 GMT
174
100
  Content-Type:
175
101
  - application/xml
176
- Date:
177
- - Sun, 26 Feb 2017 20:03:10 GMT
178
- Server:
179
- - AmazonS3
180
- body:
181
- encoding: UTF-8
182
- string: |-
183
- <?xml version="1.0" encoding="UTF-8"?>
184
- <Error><Code>RequestTimeTooSkewed</Code><Message>The difference between the request time and the current time is too large.</Message><RequestTime>20150407T220001Z</RequestTime><ServerTime>2017-02-26T20:03:12Z</ServerTime><MaxAllowedSkewMilliseconds>900000</MaxAllowedSkewMilliseconds><RequestId>04FBF6BEDA1B20A8</RequestId><HostId>XuOHzBTtiAFfNV4wZEED7xsUWIxn11sYx5uM47rWoQjfeDnW1TH4/+5FBTsjGBeen1XJ4h1ywbA=</HostId></Error>
185
- http_version:
186
- recorded_at: Tue, 07 Apr 2015 22:00:00 GMT
187
- - request:
188
- method: get
189
- uri: https://s3.amazonaws.com/
190
- body:
191
- encoding: US-ASCII
192
- string: ''
193
- headers:
194
- User-Agent:
195
- - fog-core/1.37.0
196
- X-Amz-Date:
197
- - 20150407T220001Z
198
- X-Amz-Content-Sha256:
199
- - e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
200
- Authorization:
201
- - AWS4-HMAC-SHA256 Credential=<AWS_ACCESS_KEY_ID>/20150407/us-east-1/s3/aws4_request,
202
- SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=63ae9b360414ffd780af6bec925edc820e5991b83ce6eab4d03861342340d476
203
- response:
204
- status:
205
- code: 403
206
- message: ''
207
- headers:
208
- X-Amz-Request-Id:
209
- - F355B67176738D33
210
- X-Amz-Id-2:
211
- - wKrrJy/Rz8Yaa/XUjwpcGb//ae93pPZoRKnA+vbOEnJH8zKA13p4G/0u8zhu7lhqcpV+kSK1lgY=
212
- Content-Type:
213
- - application/xml
214
- Date:
215
- - Sun, 26 Feb 2017 20:03:11 GMT
216
102
  Server:
217
103
  - AmazonS3
218
104
  body:
219
105
  encoding: UTF-8
220
106
  string: |-
221
107
  <?xml version="1.0" encoding="UTF-8"?>
222
- <Error><Code>RequestTimeTooSkewed</Code><Message>The difference between the request time and the current time is too large.</Message><RequestTime>20150407T220001Z</RequestTime><ServerTime>2017-02-26T20:03:12Z</ServerTime><MaxAllowedSkewMilliseconds>900000</MaxAllowedSkewMilliseconds><RequestId>F355B67176738D33</RequestId><HostId>wKrrJy/Rz8Yaa/XUjwpcGb//ae93pPZoRKnA+vbOEnJH8zKA13p4G/0u8zhu7lhqcpV+kSK1lgY=</HostId></Error>
108
+ <ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Owner><ID>8170cc616e9d1833db4bade0c7af0010f04464343a1004b67a00fbcb41e361de</ID><DisplayName>infrastructure</DisplayName></Owner><Buckets><Bucket><Name>api.datacite.org</Name><CreationDate>2017-02-09T07:05:20.000Z</CreationDate></Bucket><Bucket><Name>api.test.datacite.org</Name><CreationDate>2017-02-09T07:05:21.000Z</CreationDate></Bucket><Bucket><Name>archive.datacite.org</Name><CreationDate>2016-12-26T07:02:24.000Z</CreationDate></Bucket><Bucket><Name>archive.test.datacite.org</Name><CreationDate>2017-01-13T19:04:30.000Z</CreationDate></Bucket><Bucket><Name>assets.datacite.org</Name><CreationDate>2016-04-29T17:51:32.000Z</CreationDate></Bucket><Bucket><Name>assets.test.datacite.org</Name><CreationDate>2017-01-13T18:47:02.000Z</CreationDate></Bucket><Bucket><Name>blog.datacite.org</Name><CreationDate>2015-09-26T08:59:21.000Z</CreationDate></Bucket><Bucket><Name>blog.test.datacite.org</Name><CreationDate>2017-01-13T18:54:23.000Z</CreationDate></Bucket><Bucket><Name>crosscite.org</Name><CreationDate>2016-01-05T22:11:03.000Z</CreationDate></Bucket><Bucket><Name>data.datacite.org</Name><CreationDate>2016-08-29T17:11:58.000Z</CreationDate></Bucket><Bucket><Name>data.test.datacite.org</Name><CreationDate>2017-01-13T20:17:34.000Z</CreationDate></Bucket><Bucket><Name>datacite.org</Name><CreationDate>2016-01-04T21:53:46.000Z</CreationDate></Bucket><Bucket><Name>lagotto.io</Name><CreationDate>2016-01-10T10:12:40.000Z</CreationDate></Bucket><Bucket><Name>ops.datacite.org</Name><CreationDate>2016-06-03T19:38:04.000Z</CreationDate></Bucket><Bucket><Name>papertrail.datacite.org</Name><CreationDate>2016-01-04T23:25:19.000Z</CreationDate></Bucket><Bucket><Name>pidapalooza.org</Name><CreationDate>2016-06-16T07:28:25.000Z</CreationDate></Bucket><Bucket><Name>schema.datacite.org</Name><CreationDate>2015-09-26T08:01:48.000Z</CreationDate></Bucket><Bucket><Name>schema.test.datacite.org</Name><CreationDate>2017-01-10T11:47:55.000Z</CreationDate></Bucket><Bucket><Name>search.datacite.org</Name><CreationDate>2017-02-26T20:00:32.000Z</CreationDate></Bucket><Bucket><Name>search.test.datacite.org</Name><CreationDate>2017-02-26T20:00:32.000Z</CreationDate></Bucket><Bucket><Name>stats.datacite.org</Name><CreationDate>2016-08-29T10:21:19.000Z</CreationDate></Bucket><Bucket><Name>stats.test.datacite.org</Name><CreationDate>2017-01-13T20:17:34.000Z</CreationDate></Bucket><Bucket><Name>test.datacite.org</Name><CreationDate>2017-01-08T21:50:53.000Z</CreationDate></Bucket><Bucket><Name>www.crosscite.org</Name><CreationDate>2017-01-09T10:18:11.000Z</CreationDate></Bucket><Bucket><Name>www.datacite.org</Name><CreationDate>2015-09-26T08:01:48.000Z</CreationDate></Bucket><Bucket><Name>www.test.datacite.org</Name><CreationDate>2017-01-13T19:50:11.000Z</CreationDate></Bucket></Buckets></ListAllMyBucketsResult>
223
109
  http_version:
224
110
  recorded_at: Tue, 07 Apr 2015 22:00:00 GMT
225
111
  - request:
226
112
  method: put
227
- uri: https://s3-eu-west-1.amazonaws.com/search.datacite.org/sitemaps/sitemap.xml.gz
113
+ uri: https://s3-eu-west-1.amazonaws.com/search.datacite.org/sitemaps-test/sitemap.xml.gz
228
114
  body:
229
115
  encoding: US-ASCII
230
116
  string: ''
@@ -236,7 +122,7 @@ http_interactions:
236
122
  X-Amz-Acl:
237
123
  - public-read
238
124
  X-Amz-Date:
239
- - 20170226T200311Z
125
+ - 20170226T231755Z
240
126
  X-Amz-Content-Sha256:
241
127
  - STREAMING-AWS4-HMAC-SHA256-PAYLOAD
242
128
  X-Amz-Decoded-Content-Length:
@@ -244,20 +130,20 @@ http_interactions:
244
130
  Authorization:
245
131
  - AWS4-HMAC-SHA256 Credential=<AWS_ACCESS_KEY_ID>/20170226/eu-west-1/s3/aws4_request,
246
132
  SignedHeaders=content-type;host;x-amz-acl;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length,
247
- Signature=fc9847746cf3557422ed8f4c4d09b4821308a4a0d8771251319825fc101709f5
133
+ Signature=bf87c1e1f8d8e4953520591f838baf2c4581ad1ad6a6ae8e16279127ef6a00f4
248
134
  response:
249
135
  status:
250
136
  code: 200
251
137
  message: ''
252
138
  headers:
253
139
  X-Amz-Id-2:
254
- - LeNgCLMk5pL4uCWFjKtu3wOeB+afq/Uok4Nx35VKCR1u4rjHgQKaUJuArueJoMUG9sefHkHvkiM=
140
+ - aJuVURJE59LNw6VCI9dzyr4lcmXDmSqBP53P3qV/ROVTu/L2+DiqePGctgUf8AK2s5djv1h9xHg=
255
141
  X-Amz-Request-Id:
256
- - 37FEBFDC5AE39977
142
+ - 43C955CD826AF51D
257
143
  Date:
258
- - Sun, 26 Feb 2017 20:03:12 GMT
144
+ - Sun, 26 Feb 2017 23:17:56 GMT
259
145
  Etag:
260
- - '"5cf4ae06eba5f9e40621802f0e630252"'
146
+ - '"7427ddae1f77df30471b29fba613264e"'
261
147
  Content-Length:
262
148
  - '0'
263
149
  Server:
data/spec/spec_helper.rb CHANGED
@@ -1,4 +1,4 @@
1
- ENV["FOG_DIRECTORY"] = "sitemaps.test.datacite.org"
1
+ ENV['RACK'] = "test"
2
2
 
3
3
  require 'bundler/setup'
4
4
  Bundler.setup
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: maltese
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.6
4
+ version: 0.1.7
5
5
  platform: ruby
6
6
  authors:
7
7
  - Martin Fenner
@@ -252,10 +252,8 @@ files:
252
252
  - lib/maltese.rb
253
253
  - lib/maltese/cli.rb
254
254
  - lib/maltese/sitemap.rb
255
- - lib/maltese/utils.rb
256
255
  - lib/maltese/version.rb
257
256
  - maltese.gemspec
258
- - public/sitemap.xml.gz
259
257
  - spec/cli_spec.rb
260
258
  - spec/fixtures/sitemap.json
261
259
  - spec/fixtures/sitemap_nil.json
data/lib/maltese/utils.rb DELETED
@@ -1,87 +0,0 @@
1
- module Maltese
2
- module Utils
3
- # load ENV variables from container environment if json file exists
4
- # see https://github.com/phusion/baseimage-docker#envvar_dumps
5
- env_json_file = "/etc/container_environment.json"
6
- if File.size?(env_json_file).to_i > 2
7
- env_vars = JSON.parse(File.read(env_json_file))
8
- env_vars.each { |k, v| ENV[k] = v }
9
- end
10
-
11
- def queue_jobs(options={})
12
- options[:offset] = options[:offset].to_i || 0
13
- options[:rows] = options[:rows].presence || job_batch_size
14
-
15
- total = get_total(options)
16
-
17
- if total > 0
18
- puts process_data(options.merge(total: total))
19
- else
20
- puts "No works found for date range #{from_date} - #{until_date}."
21
- end
22
-
23
- # return number of works queued
24
- total
25
- end
26
-
27
- def get_total(options={})
28
- query_url = get_query_url(options.merge(rows: 0))
29
- result = Maremma.get(query_url, options)
30
- result.body.fetch("data", {}).fetch("response", {}).fetch("numFound", 0)
31
- end
32
-
33
- def get_query_url(options={})
34
- updated = "updated:[#{from_date}T00:00:00Z TO #{until_date}T23:59:59Z]"
35
- fq = "#{updated} AND has_metadata:true AND is_active:true"
36
-
37
- params = { q: "*:*",
38
- fq: fq,
39
- start: options[:offset],
40
- rows: options[:rows],
41
- fl: "doi,updated",
42
- sort: "updated asc",
43
- wt: "json" }
44
- url + URI.encode_www_form(params)
45
- end
46
-
47
- def process_data(options = {})
48
- options[:start_time] = Time.now
49
-
50
- # walk through paginated results
51
- total_pages = (options[:total].to_f / job_batch_size).ceil
52
-
53
- (0...total_pages).each do |page|
54
- options[:offset] = page * job_batch_size
55
- data = get_data(options.merge(timeout: timeout))
56
- parse_data(data)
57
- end
58
-
59
- push_data(options)
60
- end
61
-
62
- def get_data(options={})
63
- query_url = get_query_url(options)
64
- Maremma.get(query_url, options)
65
- end
66
-
67
- def parse_data(result, options={})
68
- return result.body.fetch("errors") if result.body.fetch("errors", nil).present?
69
-
70
- items = result.body.fetch("data", {}).fetch('response', {}).fetch('docs', nil)
71
- Array(items).each do |item|
72
- loc = "/works/" + item.fetch("doi")
73
- sitemap.add loc, changefreq: "monthly", lastmod: item.fetch("updated")
74
- end
75
- sitemap.sitemap.link_count
76
- end
77
-
78
- def push_data(options={})
79
- # sync time with AWS S3 before uploading
80
- fog_storage.sync_clock
81
-
82
- sitemap.finalize!
83
- time_taken = Time.now - options[:start_time]
84
- sitemap.sitemap_index.stats_summary(:time_taken => time_taken)
85
- end
86
- end
87
- end
data/public/sitemap.xml.gz DELETED
Binary file