snapcrawl 0.5.3 → 0.5.4.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1238ab663146a888fc002379efaae0abaa72f02fc9ef7954bffb79e7dad4b07d
-  data.tar.gz: c340cf0b5d1675158077257007f0d58883e9d3e0418f4c0551039fa2e09c6df8
+  metadata.gz: 3ae3bce2bddf47cad311373025e31b28e09acc5ee9e6d9a265794ad8ae355989
+  data.tar.gz: 2d6bb777cac25e0f7817e5385249c2009de50bf7bdb75dea71dc7435cfe65120
 SHA512:
-  metadata.gz: 5e1c2ca21bbfa5471d58fce0e20d61237a87e14157b156fec0e0ee0b07d8c2a169dd38bffda1c99292bc737fc5cf25c64811e2650de958756c29fd630ca87721
-  data.tar.gz: fd7d8e18393e00274f3c145c103e03c0572f7113fab98b7cf3572e99f77d16b88628be0b3050bf98fba311fb05db974453377cb96e8bbd567794426b2a83c099
+  metadata.gz: f54f93475622850ed12de2325b49e61e661e57d3d1372cf2086673453bc4f9c9a6878d0ef3d9551b4c96064231b1dea529cc44151ab915c6e383fc5fb0e4ecce
+  data.tar.gz: 8839db491de4274076d6ae9c7f801faa96f21bb3364335c0e4f8cb4bf28d1dfde5f0f8dff401715cffef259a9dbbecbd808079a5ca205ea13e3f14988a83c82d

data/bin/snapcrawl

@@ -10,21 +10,16 @@ include Colsole
 
 begin
   CLI.new.call ARGV
-
 rescue MissingPhantomJS => e
-  message = "Cannot find phantomjs executable in the path, please install it first."
-  say! "\n\n!undred!#{e.class}!txtrst!\n#{message}"
+  message = 'Cannot find phantomjs executable in the path, please install it first.'
+  say! "\n\nru`#{e.class}`\n#{message}"
   exit 2
-
-rescue MissingImageMagick=> e
-  message = "Cannot find convert (ImageMagick) executable in the path, please install it first."
-  say! "\n\n!undred!#{e.class}!txtrst!\n#{message}"
+rescue MissingImageMagick => e
+  message = 'Cannot find convert (ImageMagick) executable in the path, please install it first.'
+  say! "\n\nru`#{e.class}`\n#{message}"
   exit 3
-
 rescue => e
   puts e.backtrace.reverse if ENV['DEBUG']
-  say! "\n!undred!#{e.class}!txtrst!\n#{e.message}"
+  say! "\nru`#{e.class}`\n#{e.message}"
   exit 1
-
 end
-

data/lib/snapcrawl/cli.rb

@@ -7,13 +7,11 @@ module Snapcrawl
     include Colsole
     using StringRefinements
     using PairSplit
-    
+
     def call(args = [])
-      begin
-        execute Docopt::docopt(docopt, version: VERSION, argv: args)
-      rescue Docopt::Exit => e
-        puts e.message
-      end
+      execute Docopt.docopt(docopt, version: VERSION, argv: args)
+    rescue Docopt::Exit => e
+      puts e.message
     end
 
   private
@@ -26,7 +24,7 @@ module Snapcrawl
       apply_tweaks tweaks if tweaks
 
       Dependencies.verify
-      
+
       $logger.debug 'initializing cli'
       FileUtils.mkdir_p Config.snaps_dir
 
@@ -37,11 +35,11 @@ module Snapcrawl
     end
 
     def docopt
-      @doc ||= File.read docopt_path
+      @docopt ||= File.read docopt_path
     end
 
     def docopt_path
-      File.expand_path "templates/docopt.txt", __dir__
+      File.expand_path 'templates/docopt.txt', __dir__
     end
 
     def apply_tweaks(tweaks)
@@ -50,6 +48,5 @@ module Snapcrawl
         $logger.level = value if key == 'log_level'
       end
     end
-
   end
 end

data/lib/snapcrawl/config.rb

@@ -7,19 +7,19 @@ module Snapcrawl
       def load(file = nil)
         reset!
         push defaults
-        
+
         return unless file
 
-        file = "#{file}.yml" unless file =~ /\.ya?ml$/
+        file = "#{file}.yml" unless /\.ya?ml$/.match?(file)
 
         # FIXME: Cannot use logger here due to the "chicken and egg" with
         #        Config. The $logger is available, but it was not yet fully
         #        configured with log_level etc.
         if File.exist? file
-          # $logger.debug "loading config file !txtgrn!#{file}"
+          # $logger.debug "loading config file g`#{file}`"
           push file
         else
-          # $logger.debug "creating config file !txtgrn!#{file}"
+          # $logger.debug "creating config file g`#{file}`"
           create_config file
         end
       end
@@ -28,20 +28,20 @@ module Snapcrawl
 
       def defaults
         {
-          depth: 1,
-          width: 1280,
-          height: 0,
-          cache_life: 86400,
-          cache_dir: 'cache',
-          snaps_dir: 'snaps',
-          name_template: '%{url}',
-          url_whitelist: nil,
-          url_blacklist: nil,
-          css_selector: nil,
-          log_level: 1,
-          log_color: 'auto',
+          depth:                 1,
+          width:                 1280,
+          height:                0,
+          cache_life:            86_400,
+          cache_dir:             'cache',
+          snaps_dir:             'snaps',
+          name_template:         '%{url}',
+          url_whitelist:         nil,
+          url_blacklist:         nil,
+          css_selector:          nil,
+          log_level:             1,
+          log_color:             'auto',
           skip_ssl_verification: false,
-          screenshot_delay: nil
+          screenshot_delay:      nil,
         }
       end
 
@@ -55,7 +55,6 @@ module Snapcrawl
       def config_template
         File.expand_path 'templates/config.yml', __dir__
       end
-
     end
   end
 end

data/lib/snapcrawl/crawler.rb

@@ -7,10 +7,10 @@ module Snapcrawl
     attr_reader :url
 
     def initialize(url)
-      $logger.debug "initializing crawler with !txtgrn!#{url}"
-      
+      $logger.debug "initializing crawler with g`#{url}`"
+
       config_for_display = Config.settings.dup
-      config_for_display['name_template'] = '%%{url}' 
+      config_for_display['name_template'] = '%%{url}'
 
       $logger.debug "config #{config_for_display}"
       @url = url
@@ -25,27 +25,27 @@ module Snapcrawl
   private
 
     def process_todo
-      $logger.debug "processing queue: !txtgrn!#{todo.count} remaining"
+      $logger.debug "processing queue: g`#{todo.count} remaining`"
 
       url, page = todo.shift
       done.push url
 
-      if process_page page
-        register_sub_pages page.pages if page.depth < Config.depth
-      end
+      return unless process_page page
+
+      register_sub_pages page.pages if page.depth < Config.depth
     end
 
     def register_sub_pages(pages)
       pages.each do |sub_page|
-        next if todo.has_key?(sub_page) or done.include?(sub_page)
-        
-        if Config.url_whitelist and sub_page.path !~ /#{Config.url_whitelist}/
-          $logger.debug "ignoring !undpur!#{sub_page.url}!txtrst!, reason: whitelist"
+        next if todo.has_key?(sub_page) || done.include?(sub_page)
+
+        if Config.url_whitelist && sub_page.path !~ (/#{Config.url_whitelist}/)
+          $logger.debug "ignoring mu`#{sub_page.url}`, reason: whitelist"
           next
         end
 
-        if Config.url_blacklist and sub_page.path =~ /#{Config.url_blacklist}/
-          $logger.debug "ignoring !undpur!#{sub_page.url}!txtrst!, reason: blacklist"
+        if Config.url_blacklist && sub_page.path =~ (/#{Config.url_blacklist}/)
+          $logger.debug "ignoring mu`#{sub_page.url}`, reason: blacklist"
           next
         end
 
@@ -56,9 +56,9 @@ module Snapcrawl
     def process_page(page)
       outfile = "#{Config.snaps_dir}/#{Config.name_template}.png" % { url: page.url.to_slug }
 
-      $logger.info "processing !undpur!#{page.url}!txtrst!, depth: #{page.depth}"
+      $logger.info "processing mu`#{page.url}`, depth: #{page.depth}"
 
-      if !page.valid?
+      unless page.valid?
         $logger.debug "page #{page.path} is invalid, aborting process"
         return false
       end
@@ -66,7 +66,7 @@ module Snapcrawl
       if file_fresh? outfile
         $logger.info "screenshot for #{page.path} already exists"
       else
-        $logger.info "!bldgrn!capturing screenshot for #{page.path}"
+        $logger.info "gb`capturing screenshot for #{page.path}`"
         save_screenshot page, outfile
       end
 
@@ -76,11 +76,11 @@ module Snapcrawl
     def save_screenshot(page, outfile)
       page.save_screenshot outfile
     rescue => e
-      $logger.error "screenshot error on !undpur!#{page.path}!txtrst! - !txtred!#{e.class}!txtrst!: #{e.message}"
+      $logger.error "screenshot error on mu`#{page.path}` - r`#{e.class}`: #{e.message}"
     end
 
     def file_fresh?(file)
-      Config.cache_life > 0 and File.exist?(file) and file_age(file) < Config.cache_life
+      Config.cache_life.positive? and File.exist?(file) and file_age(file) < Config.cache_life
     end
 
     def file_age(file)
@@ -94,6 +94,5 @@ module Snapcrawl
     def done
       @done ||= []
     end
-
   end
 end

data/lib/snapcrawl/dependencies.rb

@@ -4,18 +4,18 @@ module Snapcrawl
   class Dependencies
     class << self
       include Colsole
-      
+
       def verify
         return if @verified
 
-        $logger.debug 'verifying !txtgrn!phantomjs!txtrst! is present'
-        raise MissingPhantomJS unless command_exist? "phantomjs"
+        $logger.debug 'verifying g`phantomjs` is present'
+        raise MissingPhantomJS unless command_exist? 'phantomjs'
 
-        $logger.debug 'verifying !txtgrn!imagemagick!txtrst! is present'
-        raise MissingImageMagick unless command_exist? "convert"
+        $logger.debug 'verifying g`imagemagick` is present'
+        raise MissingImageMagick unless command_exist? 'convert'
 
         @verified = true
       end
     end
   end
-end
+end

data/lib/snapcrawl/log_helpers.rb

@@ -5,18 +5,18 @@ module Snapcrawl
     include Colsole
 
     SEVERITY_COLORS = {
-      'INFO' => :txtblu,
-      'WARN' => :txtylw,
-      'ERROR' => :txtred,
-      'FATAL' => :txtred,
-      'DEBUG' => :txtcyn
+      'INFO'  => :b,
+      'WARN'  => :y,
+      'ERROR' => :r,
+      'FATAL' => :r,
+      'DEBUG' => :c,
     }
-    
+
     def log_formatter
       proc do |severity, _time, _prog, message|
         severity_color = SEVERITY_COLORS[severity]
-        line = "!#{severity_color}!#{severity.rjust 5}!txtrst! : #{message}\n"
-        use_colors? ? colorize(line) : strip_color_markers(line)
+        line = "#{severity_color}`#{severity.rjust 5}` : #{message}\n"
+        use_colors? ? colorize(line) : strip_colors(line)
       end
     end
 
@@ -25,11 +25,12 @@ module Snapcrawl
     end
 
     def tty?
-      ENV['TTY'] == 'on' ? true : ENV['TTY'] == 'off' ? false : $stdout.tty?
-    end
-
-    def strip_color_markers(text)
-      text.gsub(/\!([a-z]{6})\!/, '')
+      case ENV['TTY']
+      when 'on' then true
+      when 'off' then false
+      else
+        $stdout.tty?
+      end
     end
   end
 end

data/lib/snapcrawl/page.rb

@@ -10,11 +10,12 @@ module Snapcrawl
 
     attr_reader :url, :depth
 
-    EXTENSION_BLACKLIST = "png|gif|jpg|pdf|zip"
-    PROTOCOL_BLACKLIST = "mailto|tel"
+    EXTENSION_BLACKLIST = 'png|gif|jpg|pdf|zip'
+    PROTOCOL_BLACKLIST = 'mailto|tel'
 
     def initialize(url, depth: 0)
-      @url, @depth = url.protocolize, depth
+      @url = url.protocolize
+      @depth = depth
     end
 
     def valid?
@@ -31,18 +32,21 @@ module Snapcrawl
 
     def links
       return nil unless valid?
+
       doc = Nokogiri::HTML http_response.body
       normalize_links doc.css('a')
     end
 
     def pages
       return nil unless valid?
-      links.map { |link| Page.new link, depth: depth+1 }
+
+      links.map { |link| Page.new link, depth: depth + 1 }
     end
 
     def save_screenshot(outfile)
       return false unless valid?
-      Screenshot.new(url).save "#{outfile}"
+
+      Screenshot.new(url).save outfile
     end
 
   private
@@ -54,16 +58,14 @@ module Snapcrawl
     def http_response!
       response = cache.get(url) { HTTParty.get url, httparty_options }
 
-      if !response.success?
-        $logger.warn "http error on !undpur!#{url}!txtrst!, code: !txtylw!#{response.code}!txtrst!, message: #{response.message.strip}"
+      unless response.success?
+        $logger.warn "http error on mu`#{url}`, code: y`#{response.code}`, message: #{response.message.strip}"
       end
 
       response
-
     rescue => e
-      $logger.error "http error on !undpur!#{url}!txtrst! - !txtred!#{e.class}!txtrst!: #{e.message}"
+      $logger.error "http error on mu`#{url}` - r`#{e.class}`: #{e.message}"
       nil
-
     end
 
     def httparty_options
@@ -89,8 +91,8 @@ module Snapcrawl
       return nil if link.empty?
 
       # Remove links to specific extensions and protocols
-      return nil if link =~ /\.(#{EXTENSION_BLACKLIST})(\?.*)?$/
-      return nil if link =~ /^(#{PROTOCOL_BLACKLIST}):/
+      return nil if /\.(#{EXTENSION_BLACKLIST})(\?.*)?$/o.match?(link)
+      return nil if /^(#{PROTOCOL_BLACKLIST}):/o.match?(link)
 
       # Strip spaces
       link.strip!
@@ -99,12 +101,13 @@ module Snapcrawl
       begin
         link = Addressable::URI.join(url, link).to_s.dup
       rescue => e
-        $logger.warn "!txtred!#{e.class}!txtrst!: #{e.message} on #{path} (link: #{link})"
+        $logger.warn "r`#{e.class}`: #{e.message} on #{path} (link: #{link})"
         return nil
       end
 
       # Keep only links in our base domain
       return nil unless link.include? site
+
       link
     end
 

data/lib/snapcrawl/refinements/pair_split.rb

@@ -2,21 +2,24 @@ module Snapcrawl
   module PairSplit
     refine Array do
       def pair_split
-        map do |pair|
+        false_values = %w[no false]
+        true_values = %w[yes true]
+
+        to_h do |pair|
           key, value = pair.split '='
-          
-          value = if value =~ /^\d+$/
-            value.to_i 
-          elsif ['no', 'false'].include? value
+
+          value = if /^\d+$/.match?(value)
+            value.to_i
+          elsif false_values.include? value
             false
-          elsif ['yes', 'true'].include? value
+          elsif true_values.include? value
             true
           else
             value
           end
 
           [key, value]
-        end.to_h
+        end
       end
     end
   end

data/lib/snapcrawl/refinements/string_refinements.rb

@@ -6,7 +6,7 @@ module Snapcrawl
       end
 
       def protocolize
-        self =~ /^http/ ? self : "http://#{self}"
+        /^http/.match?(self) ? self : "http://#{self}"
       end
     end
   end

data/lib/snapcrawl/screenshot.rb

@@ -27,10 +27,10 @@ module Snapcrawl
       hide_output do
         webshot.capture url, image_path, webshot_options do |magick|
           magick.combine_options do |c|
-            c.background "white"
+            c.background 'white'
             c.gravity 'north'
             c.quality 100
-            c.extent Config.height > 0 ? "#{Config.width}x#{Config.height}" : "#{Config.width}x"
+            c.extent Config.height.positive? ? "#{Config.width}x#{Config.height}" : "#{Config.width}x"
           end
         end
       end
@@ -38,9 +38,9 @@ module Snapcrawl
 
     def webshot_options
       result = { allowed_status_codes: [404, 401, 403] }
-      
-      if Config.selector
-        result[:selector] = Config.selector
+
+      if Config.css_selector
+        result[:selector] = Config.css_selector
         result[:full] = false
       end
 
@@ -55,16 +55,19 @@ module Snapcrawl
       @webshot ||= Webshot::Screenshot.instance
     end
 
-    # The webshot gem messes with stdout/stderr streams so we keep it in 
+    # The webshot gem messes with stdout/stderr streams so we keep it in
     # check by using this method. Also, in some sites (e.g. uown.co) it
     # prints some output to stdout, this is why we override $stdout for
     # the duration of the run.
     def hide_output
-      keep_stdout, keep_stderr = $stdout, $stderr
-      $stdout, $stderr = StringIO.new, StringIO.new
+      keep_stdout = $stdout
+      keep_stderr = $stderr
+      $stdout = StringIO.new
+      $stderr = StringIO.new
       yield
     ensure
-      $stdout, $stderr = keep_stdout, keep_stderr
+      $stdout = keep_stdout
+      $stderr = keep_stderr
     end
   end
 end

data/lib/snapcrawl/version.rb

@@ -1,3 +1,3 @@
 module Snapcrawl
-  VERSION = "0.5.3"
-end
+  VERSION = '0.5.4.rc1'
+end

metadata

@@ -1,127 +1,133 @@
 --- !ruby/object:Gem::Specification
 name: snapcrawl
 version: !ruby/object:Gem::Version
-  version: 0.5.3
+  version: 0.5.4.rc1
 platform: ruby
 authors:
 - Danny Ben Shitrit
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-03-29 00:00:00.000000000 Z
+date: 2023-07-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: colsole
+  name: addressable
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: '2.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: '2.7'
 - !ruby/object:Gem::Dependency
-  name: docopt
+  name: colsole
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: 0.8.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: 0.8.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
 - !ruby/object:Gem::Dependency
-  name: nokogiri
+  name: docopt
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '0.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '0.6'
 - !ruby/object:Gem::Dependency
-  name: webshot
+  name: httparty
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '0.21'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '0.21'
 - !ruby/object:Gem::Dependency
-  name: httparty
+  name: lightly
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.18'
+        version: '0.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.18'
+        version: '0.3'
 - !ruby/object:Gem::Dependency
-  name: addressable
+  name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.7'
+        version: '1.10'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.7'
+        version: '1.10'
 - !ruby/object:Gem::Dependency
-  name: lightly
+  name: sting
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: '0.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: '0.4'
 - !ruby/object:Gem::Dependency
-  name: sting
+  name: webshot
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: '0.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: '0.1'
 description: Snapcrawl is a command line utility for crawling a website and saving
   screenshots.
 email: db@dannyben.com
@@ -150,8 +156,9 @@ files:
 homepage: https://github.com/DannyBen/snapcrawl
 licenses:
 - MIT
-metadata: {}
-post_install_message: 
+metadata:
+  rubygems_mfa_required: 'true'
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -159,15 +166,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.3'
+      version: '3.0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubygems_version: 3.2.3
-signing_key: 
+rubygems_version: 3.4.17
+signing_key:
 specification_version: 4
 summary: Crawl a website and take screenshots (CLI + Library)
 test_files: []