vore 0.2.5-arm64-darwin → 0.2.8-arm64-darwin

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: bc0dadc7e35ecbe250d467f65c9c24443ed535523d4874476b27628ac680fabf
4
- data.tar.gz: 83a30b479e3d23585e8f001dc9879402b97f4a921d22ef0644925a484822168a
3
+ metadata.gz: cea3bfc2d1ad6b14200a35b52eaa739bb14b2f6d5ace761be3d77a53366f19b2
4
+ data.tar.gz: '03861e5e9ccdbca488f9397bc59f480f48e6d62e03b4cf40ee652e3cc57745da'
5
5
  SHA512:
6
- metadata.gz: aa3f33ea776f0b4775f650dacaaa350d9a9c0f4fa05cdf2380c2626f46c30d0cb6440972a6eab472e65ec86de1dc771bd9792d183ef5830a5fddc02716805c73
7
- data.tar.gz: 5e3b7fb092abb1d341ac4514ea171affb6df9eace0d6a0183ef477ec3ae3f1c97a33517c05f09720bed51c0219bd2eb819f6653c613e683d45c37e3fec677d25
6
+ metadata.gz: b7c448fce4a1c6b26b6bff99fbe91053588f56b86b077567913902b4c3405d805d56e41ae430d302fbe2c8d834b50514a945f7361b8996c992d33355cdd7269a
7
+ data.tar.gz: aa7a8facb68f637b2642b4ee1fbb7437b4cf78ede8f31c8505e9d8141f45bdb4ef689f80ca498af4eb681cd6d5c6c3cbbdba8059f8890ad9fcb37eac5a480095
data/exe/vore-spider CHANGED
Binary file
data/lib/vore/crawler.rb CHANGED
@@ -8,6 +8,8 @@ module Vore
8
8
  PLATFORM = [:cpu, :os].map { |m| Gem::Platform.local.send(m) }.join("-")
9
9
  FILE_SEPERATOR = PLATFORM.include?("windows") ? File::ALT_SEPARATOR : File::SEPARATOR
10
10
 
11
+ attr_reader :output_dir
12
+
11
13
  # Creates a crawler
12
14
  # denylist: Sets a denylist filter, allows a regexp, string or array of either to be matched.
13
15
  def initialize(denylist: /a^/, sanitization_config: Vole::Configuration::DEFAULT_SANITIZATION_CONFIG)
@@ -26,16 +28,10 @@ module Vore
26
28
  end
27
29
 
28
30
  def scrape_each_page(website, &block)
29
- output_dir = "#{@parent_output_dir}/#{website.gsub(/[^a-zA-Z0-9]/, "_").squeeze("_")}"
31
+ @output_dir = "#{@parent_output_dir}/#{website.gsub(/[^a-zA-Z0-9]/, "_").squeeze("_")}"
30
32
  Vore.logger.info("Vore started crawling #{website}, outputting to #{output_dir}")
31
33
 
32
- output = %x(#{@executable} \
33
- --user-agent #{user_agent} \
34
- --delay 3500 \
35
- --url #{website} \
36
- download \
37
- -t \
38
- #{output_dir})
34
+ output = run_command(website, @output_dir)
39
35
 
40
36
  Vore.logger.info("Vore finished crawling #{website}: #{output}")
41
37
 
@@ -51,6 +47,7 @@ module Vore
51
47
  results[:pages_visited] += 1
52
48
 
53
49
  html_file = File.read(path).force_encoding("UTF-8")
50
+ rewritten_html_file = ""
54
51
 
55
52
  if html_file.empty?
56
53
  results[:pages_unprocessed] += 1
@@ -58,7 +55,13 @@ module Vore
58
55
  next
59
56
  end
60
57
 
61
- rewritten_html_file = @selma.rewrite(html_file)
58
+ begin
59
+ rewritten_html_file = @selma.rewrite(html_file)
60
+ rescue StandardError => e
61
+ Vore.logger.warn("Error rewriting #{path}: #{e}")
62
+ results[:pages_unprocessed] += 1
63
+ next
64
+ end
62
65
 
63
66
  # drops the first 3 parts of the path, which are "tmp", "vore", and the site name
64
67
  url_path = path.split(FILE_SEPERATOR)[3..].join("/")
@@ -83,6 +86,16 @@ module Vore
83
86
  # crawl_site(site)
84
87
  # end
85
88
 
89
+ def run_command(website, output_dir)
90
+ %x(#{@executable} \
91
+ --user-agent #{user_agent} \
92
+ --delay 3500 \
93
+ --url #{website} \
94
+ download \
95
+ -t \
96
+ #{output_dir})
97
+ end
98
+
86
99
  def user_agent
87
100
  "'Mozilla/5.0 (compatible; Vore/#{Vore::VERSION}; +https://github.com/gjtorikian/vore)'"
88
101
  end
@@ -20,7 +20,6 @@ module Vole
20
20
 
21
21
  def handle_element(element)
22
22
  if element.tag_name == "pre" ||
23
- element.tag_name == "code" ||
24
23
  element.tag_name == "form" ||
25
24
  element.tag_name == "style" ||
26
25
  element.tag_name == "noscript" ||
data/lib/vore/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Vore
4
- VERSION = "0.2.5"
4
+ VERSION = "0.2.8"
5
5
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: vore
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.5
4
+ version: 0.2.8
5
5
  platform: arm64-darwin
6
6
  authors:
7
7
  - Garen J. Torikian