elsmore 0.1.7 → 0.2.0

data/lib/elsmore/resource.rb

@@ -1,8 +1,8 @@
-require 'open-uri'
+require 'httparty'
 
 module Elsmore
   class Resource
-    attr_accessor :url, :filename
+    attr_accessor :url, :filename, :emitter
 
     def initialize url, parent
       self.url = Elsmore::Url.new(url, parent)
@@ -10,12 +10,13 @@ module Elsmore
 
     def write!
       writer = Elsmore::Writer.new(self)
+      writer.emitter = emitter
       writer.write
       self.filename = writer.canonical_filename
     end
 
     def data
-      @data ||= open(url.canonical_url).read
+      @data ||= HTTParty.get(url.canonical_url)
     end
   end
 end

data/lib/elsmore/rewriter.rb

@@ -1,6 +1,6 @@
 module Elsmore
   class Rewriter
-    attr_accessor :resource
+    attr_accessor :resource, :emitter
 
     def initialize resource
       self.resource = resource
@@ -20,7 +20,7 @@ module Elsmore
     end
 
     def write_css
-      resource.doc.xpath('//link[rel=stylesheet]').each do |element|
+      resource.doc.xpath('//link[@rel="stylesheet"]').each do |element|
         write_element(element, 'href')
       end
     end
@@ -41,7 +41,10 @@ module Elsmore
       return unless element.attribute(key)
       url = element.attribute(key).value
       _resource = Elsmore::Resource.new(url, resource.url)
+      _resource.emitter = emitter
       _resource.write!
+
+      emitter.log("# Rewriting #{url} => #{_resource.filename}") if url != _resource.filename
       element.attribute(key).value = _resource.filename
     end
 
@@ -49,8 +52,15 @@ module Elsmore
       resource.doc.xpath('//a').each do |element|
         return unless element.attribute('href')
         href = element.attribute('href').value
+
         url = Elsmore::Url.new(href, resource.url)
-        element.attribute('href').value = url.absolute_path_or_external_url
+        if url.valid
+          new_url = url.absolute_path_or_external_url
+        else
+          new_url = href
+        end
+        emitter.log("# Rewriting #{href} => #{new_url}") if href != new_url
+        element.attribute('href').value = new_url
       end
     end
   end

data/lib/elsmore/scraper.rb

@@ -1,49 +1,65 @@
 module Elsmore
   class Scraper
-    attr_accessor :emitter
+    attr_accessor :emitter, :unprocessed, :processed, :invalid, :unprocessed_urls, :valid_domains
 
     def initialize initial_url
       seed = Elsmore::Document.new(initial_url)
 
-      @valid_domains = [seed.url.host]
-      @unprocessed = [seed]
-      @processed = []
-      @invalid = []
-    end
+      self.valid_domains = [seed.url.host]
 
-    def run
-      while !@unprocessed.empty?
-        document = @unprocessed.shift
-        next if @processed.include?(document.url.canonical_url)
-        emitter.dot
+      self.unprocessed = [seed]
+      self.unprocessed_urls = [seed.url.canonical_url]
 
-        enqueue(document.links)
-        document.rewrite
-        document.write!
+      self.processed = []
+      self.invalid = []
+    end
 
-        @processed << document.url.canonical_url
+    def run
+      while !unprocessed.empty?
+        document = unprocessed.shift
+        process document
       end
 
       {
-        processed: @processed,
-        invalid: @invalid
+        processed: processed,
+        invalid: invalid
       }
     end
 
     private
 
+    def process document
+      emitter.log(document.url.canonical_url.colorize(:green))
+
+      document.emitter = emitter
+
+      enqueue(document.links)
+      document.rewrite
+      document.write!
+
+      processed << document.url.canonical_url
+    end
+
     def enqueue links
       links.each_with_index do |document, index|
-        if !document.url.valid
-          emitter.unsure
-          @invalid << document.url.raw_url
-          next
-        end
-
-        next if !@valid_domains.include?(document.url.host)
-        next if @processed.include?(document.url.canonical_url)
-        @unprocessed << document
+        next unless valid?(document)
+        next if !valid_domains.include?(document.url.host)
+        next if processed.include?(document.url.canonical_url)
+        next if unprocessed_urls.include?(document.url.canonical_url)
+
+        emitter.log("> Enqueued: #{document.url.canonical_url}")
+
+        unprocessed << document
+        unprocessed_urls << document.url.canonical_url
+      end
+    end
+
+    def valid?(document)
+      if !document.url.valid && !self.invalid.include?(document.url.raw_url)
+        emitter.warning("> Invalid URL: #{document.url.raw_url}")
+        invalid << document.url.raw_url
       end
+      document.url.valid
     end
   end
 end

data/lib/elsmore/version.rb

@@ -1,3 +1,3 @@
 module Elsmore
-  VERSION = '0.1.7'
+  VERSION = '0.2.0'
 end

data/lib/elsmore/writer.rb

@@ -1,6 +1,6 @@
 module Elsmore
   class Writer
-    attr_accessor :resource
+    attr_accessor :resource, :emitter
 
     def initialize resource
       self.resource = resource
@@ -24,6 +24,9 @@ module Elsmore
     def write_file
       return if File.exist?(full_filename)
       ensure_directory full_filename
+
+      emitter.log("! Saving #{full_filename}")
+
       File.open(full_filename, 'w') do |file|
         file.write(resource.data)
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: elsmore
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.2.0
 platform: ruby
 authors:
 - Cristiano Betta
@@ -111,6 +111,79 @@ files:
 - Rakefile
 - bin/elsmore
 - elsmore.gemspec
+- hackference.co.uk/assets/css/site.css
+- hackference.co.uk/assets/css/slicknav.css
+- hackference.co.uk/assets/css/tito.css
+- hackference.co.uk/assets/img/algolia.png
+- hackference.co.uk/assets/img/andrew-faraday.jpg
+- hackference.co.uk/assets/img/conf.png
+- hackference.co.uk/assets/img/contentful.jpg
+- hackference.co.uk/assets/img/cristiano-betta.jpg
+- hackference.co.uk/assets/img/dan-jenkins.png
+- hackference.co.uk/assets/img/daniel-knell.jpg
+- hackference.co.uk/assets/img/etiene-dalcol.jpg
+- hackference.co.uk/assets/img/felienne-hermans.jpg
+- hackference.co.uk/assets/img/hack.png
+- hackference.co.uk/assets/img/hugh-rawlinson.jpg
+- hackference.co.uk/assets/img/improbable.png
+- hackference.co.uk/assets/img/jessica-rose-cartoon.png
+- hackference.co.uk/assets/img/jonathan-kingsley.jpeg
+- hackference.co.uk/assets/img/logo-small.png
+- hackference.co.uk/assets/img/martin-splitt.jpg
+- hackference.co.uk/assets/img/microsoft.png
+- hackference.co.uk/assets/img/mozilla.png
+- hackference.co.uk/assets/img/nexmo.png
+- hackference.co.uk/assets/img/packt.png
+- hackference.co.uk/assets/img/pebble.png
+- hackference.co.uk/assets/img/proactive.png
+- hackference.co.uk/assets/img/pusher.png
+- hackference.co.uk/assets/img/remy-sharp.jpg
+- hackference.co.uk/assets/img/sam-wierema.jpg
+- hackference.co.uk/assets/img/samathy-barratt.jpg
+- hackference.co.uk/assets/img/soledad.png
+- hackference.co.uk/assets/img/technical-team-solutions.png
+- hackference.co.uk/assets/img/terence-eden.png
+- hackference.co.uk/cdn-cgi/l/email-protection/index.html
+- hackference.co.uk/cdn-cgi/scripts/cf.common.js
+- hackference.co.uk/cdn-cgi/scripts/zepto.min.js
+- hackference.co.uk/cdn-cgi/styles/cf.errors.css
+- hackference.co.uk/cdnjs.cloudflare.com/ajax/libs/SlickNav/1.0.7/jquery.slicknav.min.js
+- hackference.co.uk/code-of-conduct.html
+- hackference.co.uk/code.jquery.com/jquery-1.12.3.min.js
+- hackference.co.uk/conference.html
+- hackference.co.uk/contact.html
+- hackference.co.uk/credits.html
+- hackference.co.uk/hackathon.html
+- hackference.co.uk/index.html
+- hackference.co.uk/js.tito.io/v1/index.html
+- hackference.co.uk/maxcdn.bootstrapcdn.com/bootstrap/3.3.6/css/bootstrap.min.css
+- hackference.co.uk/maxcdn.bootstrapcdn.com/font-awesome/4.3.0/css/font-awesome.min.css
+- hackference.co.uk/platform.twitter.com/js/tweet.00c2bb81d4f3f312a9a021c715a1c9dc.js
+- hackference.co.uk/platform.twitter.com/widgets.js
+- hackference.co.uk/speakers/andrew-faraday.html
+- hackference.co.uk/speakers/dan-jenkins.html
+- hackference.co.uk/speakers/daniel-knell.html
+- hackference.co.uk/speakers/etiene-dalcol.html
+- hackference.co.uk/speakers/felienne-hermans.html
+- hackference.co.uk/speakers/hugh-rawlinson.html
+- hackference.co.uk/speakers/jonathan-kingsley.html
+- hackference.co.uk/speakers/martin-splitt.html
+- hackference.co.uk/speakers/remy-sharp.html
+- hackference.co.uk/speakers/sam-wierema.html
+- hackference.co.uk/speakers/samathy-barratt.html
+- hackference.co.uk/speakers/soledad-penades.html
+- hackference.co.uk/speakers/terence-eden.html
+- hackference.co.uk/sponsors/algolia.html
+- hackference.co.uk/sponsors/contentful.html
+- hackference.co.uk/sponsors/improbable.html
+- hackference.co.uk/sponsors/index.html
+- hackference.co.uk/sponsors/microsoft.html
+- hackference.co.uk/sponsors/nexmo.html
+- hackference.co.uk/sponsors/packt.html
+- hackference.co.uk/sponsors/pebble.html
+- hackference.co.uk/sponsors/pusher.html
+- hackference.co.uk/sponsors/sponsor-us.html
+- hackference.co.uk/tickets.html
 - lib/elsmore.rb
 - lib/elsmore/command.rb
 - lib/elsmore/document.rb