snapcrawl 0.2.8 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2500895cfd465dd633ac9650fc702f87b684ae4a81aa2feb979a16fad17bedc2
-  data.tar.gz: 042f0e14f8139d3b33bdd0061731c7973f7c445e57e2c474d32c3b226d333acb
+  metadata.gz: e0dba7d3febfccf88e22194125387c6219cf500f0cb0e00fe4ed343a57fe9034
+  data.tar.gz: d05440bccd6ddde9773b24c425f3420b1970c3985805f6eb685171237e382cda
 SHA512:
-  metadata.gz: 7603f985467c47f26b5bbf8c3cebf5654de708d0f7039c0268b4c4824089c259c17b687be482ed3e5813f44fef2274407542c76e3291f064aed1b06c2af1837a
-  data.tar.gz: 3246ea728fa90fc65da1732964492da9383ae0c99a47746d1194f5c991b2efadd333c69478e2c97fd99274acb47fc38773285a53c9fd5505e7a605af08f58cf2
+  metadata.gz: 71f65748af4339bf606765e2f1ea0aab90f4099ff986ce59318c6c966363ca6386a92d08ce175373cfa8e83c096af844983da2cde22ccbbebd2ab151514b6e72
+  data.tar.gz: a641b038a18114117375249d260895d73d08cd2cf47c7f668641c0891cd4b4723db80d4313a6a64cd89de02be0ea72aa9fa7b15f74d1ba6e0724017486c2d9c4

data/lib/snapcrawl/crawler.rb

@@ -1,10 +1,11 @@
 require 'colsole'
 require 'docopt'
 require 'fileutils'
+require 'httparty'
 require 'nokogiri'
-require 'open-uri'
 require 'ostruct'
 require 'pstore'
+require 'uri'
 require 'webshot'
 
 module Snapcrawl
@@ -131,15 +132,18 @@ module Snapcrawl
       say "       !txtblu!Crawl!!txtrst! Extracting links... "
 
       begin
-        doc = Nokogiri::HTML open url
-        links = doc.css('a')
-        links = normalize_links links
-        @store.transaction { @store[url] = links }
-        say "done"
-      rescue OpenURI::HTTPError => e
-        links = []
-        say "!txtred!FAILED"
-        say "!txtred!  !    HTTP Error: #{e.message} at #{url}"
+        response = HTTParty.get url
+        if response.success?
+          doc = Nokogiri::HTML response.body
+          links = doc.css('a')
+          links = normalize_links links
+          @store.transaction { @store[url] = links }
+          say "done"
+        else
+          links = []
+          say "!txtred!FAILED"
+          say "!txtred!  !    HTTP Error: #{response.code} #{response.message.strip} at #{url}"
+        end
       end
       links
     end
@@ -181,7 +185,7 @@ module Snapcrawl
 
       links_array = []
 
-      links.each_with_index do |link|
+      links.each do |link|
         link = link.attribute('href').to_s
 
         # Remove #hash
@@ -191,10 +195,12 @@ module Snapcrawl
         # Remove links to specific extensions and protocols
         next if link =~ /\.(#{extensions})(\?.*)?$/
         next if link =~ /^(#{beginnings})/
-        
-        # Add the base domain to relative URLs
-        link = link =~ /^http/ ? link : "#{@opts.base}#{link}" 
-        link = "http://#{link}" unless link =~ /^http/
+
+        # Strip spaces
+        link.strip!
+
+        # Convert relative links to absolute
+        link = URI.join( @opts.base, link ).to_s
 
         # Keep only links in our base domain
         next unless link.include? @opts.base

data/lib/snapcrawl/version.rb

@@ -1,3 +1,3 @@
 module Snapcrawl
-  VERSION = "0.2.8"
+  VERSION = "0.3.0"
 end

data/lib/snapcrawl.rb

@@ -1,4 +1,6 @@
 require 'snapcrawl/version'
 require 'snapcrawl/crawler'
 
+require 'byebug' if ENV['BYEBUG']
+
 self.extend Snapcrawl

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: snapcrawl
 version: !ruby/object:Gem::Version
-  version: 0.2.8
+  version: 0.3.0
 platform: ruby
 authors:
 - Danny Ben Shitrit
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-06-14 00:00:00.000000000 Z
+date: 2019-09-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: colsole
@@ -72,6 +72,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.17'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.17'
 description: Snapcrawl is a command line utility for crawling a website and saving
   screenshots.
 email: db@dannyben.com
@@ -105,7 +119,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.0.4
 signing_key: 
 specification_version: 4
 summary: Crawl a website and take screenshots (CLI + Library)