vore 0.2.6-arm64-linux → 0.3.0-arm64-linux

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 72fc70a334c49c0cd3d49fa18a0754e593024555276c128503f6d2897c3bb25a
4
- data.tar.gz: 47d4b811f806876617702d1d599848eaa3948f4ebae715a686d05919a37a9f1c
3
+ metadata.gz: 7c3033ea973b09cb40e841e5be9403126d3aa9acc1c82bcd6ef718086b03b397
4
+ data.tar.gz: e3fb4483dfa4b3cc0406c5e99ed623cade66f05b50204121a5325307b74a8abc
5
5
  SHA512:
6
- metadata.gz: 5b6d5a89270fdf1b4630cc09ea1e7e0e9622b3af61a5965ba48718a2b6b2706d5984f59ffa510a8b192910604e7a1ff79f3b373466f3181d3f69ebd714bad3e3
7
- data.tar.gz: 9906bcb9a83a4ea7c12de3be523fa3b534a6e0faeb2bf4dcc586a3e097c4efc5fd511d091641a902b4b050be6cff65e25809fb453298b587074aeda8ae3f932d
6
+ metadata.gz: 2cde527795fe34275c0a8ad93218d38674b01197eae21337d898809eb7da05d38fbdeaa405082cfd4d4e1ba2184703190f337544aa979f640830a0172e5ee732
7
+ data.tar.gz: 5a867187798343e9ba6736937ef54f9feee35537421652969d4a3558c538d183ca422a25fc1b9d4f909e7e99e9a113262f257915c0f9f000e51afe553149177d
@@ -5,5 +5,9 @@ module Vole
5
5
  DEFAULT_SANITIZATION_CONFIG = Selma::Sanitizer::Config::RELAXED.dup.merge({
6
6
  allow_doctype: false,
7
7
  })
8
+
9
+ DEFAULT_OPTIONS = {
10
+ delay: 3500,
11
+ }
8
12
  end
9
13
  end
data/lib/vore/crawler.rb CHANGED
@@ -8,9 +8,11 @@ module Vore
8
8
  PLATFORM = [:cpu, :os].map { |m| Gem::Platform.local.send(m) }.join("-")
9
9
  FILE_SEPERATOR = PLATFORM.include?("windows") ? File::ALT_SEPARATOR : File::SEPARATOR
10
10
 
11
+ attr_reader :output_dir
12
+
11
13
  # Creates a crawler
12
14
  # denylist: Sets a denylist filter, allows a regexp, string or array of either to be matched.
13
- def initialize(denylist: /a^/, sanitization_config: Vole::Configuration::DEFAULT_SANITIZATION_CONFIG)
15
+ def initialize(denylist: /a^/, sanitization_config: Vole::Configuration::DEFAULT_SANITIZATION_CONFIG, options: Vole::Configuration::DEFAULT_OPTIONS)
14
16
  @denylist_regexp = Regexp.union(denylist)
15
17
 
16
18
  @content_extractor = Vole::Handlers::ContentExtractor.new
@@ -18,6 +20,7 @@ module Vore
18
20
  ext = PLATFORM.include?("windows") ? ".exe" : ""
19
21
  @executable = File.expand_path([__FILE__, "..", "..", "..", "exe", "vore-spider#{ext}"].join(FILE_SEPERATOR))
20
22
  @parent_output_dir = "tmp/vore"
23
+ @options = options
21
24
 
22
25
  return if File.exist?(@executable)
23
26
 
@@ -26,16 +29,10 @@ module Vore
26
29
  end
27
30
 
28
31
  def scrape_each_page(website, &block)
29
- output_dir = "#{@parent_output_dir}/#{website.gsub(/[^a-zA-Z0-9]/, "_").squeeze("_")}"
32
+ @output_dir = "#{@parent_output_dir}/#{website.gsub(/[^a-zA-Z0-9]/, "_").squeeze("_")}"
30
33
  Vore.logger.info("Vore started crawling #{website}, outputting to #{output_dir}")
31
34
 
32
- output = %x(#{@executable} \
33
- --user-agent #{user_agent} \
34
- --delay 3500 \
35
- --url #{website} \
36
- download \
37
- -t \
38
- #{output_dir})
35
+ output = run_command(website, delay: @options[:delay])
39
36
 
40
37
  Vore.logger.info("Vore finished crawling #{website}: #{output}")
41
38
 
@@ -54,7 +51,6 @@ module Vore
54
51
  rewritten_html_file = ""
55
52
 
56
53
  if html_file.empty?
57
- Vore.logger.warn("HTML file empty: #{path}")
58
54
  results[:pages_unprocessed] += 1
59
55
  results[:unprocessed_pages] << path
60
56
  next
@@ -91,6 +87,16 @@ module Vore
91
87
  # crawl_site(site)
92
88
  # end
93
89
 
90
+ def run_command(website, delay: 3500)
91
+ %x(#{@executable} \
92
+ --user-agent #{user_agent} \
93
+ --delay #{delay} \
94
+ --url #{website} \
95
+ download \
96
+ -t \
97
+ #{@output_dir})
98
+ end
99
+
94
100
  def user_agent
95
101
  "'Mozilla/5.0 (compatible; Vore/#{Vore::VERSION}; +https://github.com/gjtorikian/vore)'"
96
102
  end
@@ -20,7 +20,6 @@ module Vole
20
20
 
21
21
  def handle_element(element)
22
22
  if element.tag_name == "pre" ||
23
- element.tag_name == "code" ||
24
23
  element.tag_name == "form" ||
25
24
  element.tag_name == "style" ||
26
25
  element.tag_name == "noscript" ||
data/lib/vore/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Vore
4
- VERSION = "0.2.6"
4
+ VERSION = "0.3.0"
5
5
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: vore
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.6
4
+ version: 0.3.0
5
5
  platform: arm64-linux
6
6
  authors:
7
7
  - Garen J. Torikian