datahen 0.16.1 → 0.16.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/datahen/scraper/batch_parser.rb +2 -0
- data/lib/datahen/version.rb +1 -1
- metadata +5 -5
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 8945724e5d11f40eba22a9ffca7ca7d024b8565dae4ba1d3da9e486eac262575
|
|
4
|
+
data.tar.gz: 867319a0c6358c593951e6241b24d9c3fd9dcb759ce1287eeb640ade0c23e69a
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: bad8dea41951df061c84934fa63a0594bc8caf22b8665c3d6746bb29ca2821103ecd3814a9a9eba020009e8165390900b1c45a1fd0a7175fb7ea52f7a077fdab
|
|
7
|
+
data.tar.gz: 74342e01eaa21a590ef998219282fb40342cde4bf8db58617d24583bfa00b967df3f8c38e41c9b92868e90113b7cc5f6346fb2062b446edd3bb4612e053c5da7
|
|
@@ -334,6 +334,7 @@ module Datahen
|
|
|
334
334
|
Parallel.each(dequeue, in_threads: (worker_count)) do |page|
|
|
335
335
|
parser_file = self.parsers[page['page_type']]
|
|
336
336
|
begin
|
|
337
|
+
self.repeat_puts("Parsing page with GID #{page['gid']}")
|
|
337
338
|
puts Datahen::Scraper::Parser.exec_parser_by_page(
|
|
338
339
|
parser_file,
|
|
339
340
|
page,
|
|
@@ -342,6 +343,7 @@ module Datahen
|
|
|
342
343
|
nil,
|
|
343
344
|
keep_outputs
|
|
344
345
|
)
|
|
346
|
+
self.repeat_puts("Finish parsing page with GID #{page['gid']}")
|
|
345
347
|
rescue Parallel::Kill => e
|
|
346
348
|
puts "[Worker #{Parallel.worker_number}]: Someone tried to kill Parallel!!!"
|
|
347
349
|
rescue Parallel::Break => e
|
data/lib/datahen/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: datahen
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.16.
|
|
4
|
+
version: 0.16.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Parama Danoesubroto
|
|
8
|
-
autorequire:
|
|
8
|
+
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2021-
|
|
11
|
+
date: 2021-08-11 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: thor
|
|
@@ -276,7 +276,7 @@ metadata:
|
|
|
276
276
|
allowed_push_host: https://rubygems.org
|
|
277
277
|
homepage_uri: https://datahen.com
|
|
278
278
|
source_code_uri: https://github.com/DataHenOfficial/datahen-ruby
|
|
279
|
-
post_install_message:
|
|
279
|
+
post_install_message:
|
|
280
280
|
rdoc_options: []
|
|
281
281
|
require_paths:
|
|
282
282
|
- lib
|
|
@@ -292,7 +292,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
292
292
|
version: '0'
|
|
293
293
|
requirements: []
|
|
294
294
|
rubygems_version: 3.0.3
|
|
295
|
-
signing_key:
|
|
295
|
+
signing_key:
|
|
296
296
|
specification_version: 4
|
|
297
297
|
summary: DataHen toolbelt for developers
|
|
298
298
|
test_files: []
|