elsmore 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7a1918ab7c969ec280e5fc1f7c6aebbee87fa19e
+  data.tar.gz: 5344fc276d3dec1cfabe38481dba0b5703369b74
+SHA512:
+  metadata.gz: 591510236ccc9fba7eacd7b3d03a08b861a84a8d970ded0c3ca2e6289d49973799eca4e31653345d3b52e04b4d7129de361f87ea95c140094442f9c456edb872
+  data.tar.gz: 41c518b723dbc0c945060c27e351396e21cf198174ae3b57b22950f98b247ea8a37770d21a19733782d76fb0e32a71d7ae9114a77061c268cde06947996a0d4f

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2016 Cristiano Betta
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1 @@
+# Work in Progress

data/elsmore.gemspec

@@ -0,0 +1,22 @@
+require File.expand_path('lib/elsmore/version', File.dirname(__FILE__))
+
+Gem::Specification.new do |s|
+  s.name        = 'elsmore'
+  s.version     =  Elsmore::VERSION
+  s.summary     = "Backup too for Hackference"
+  s.description = "Backup too for Hackference"
+  s.authors     = ["Cristiano Betta", "Mike Elsmore"]
+  s.email       = ['cbetta@gmail.com', "mike@hackference.co.uk"]
+  s.files       = Dir.glob('{lib,spec}/**/*') + %w(LICENSE README.md elsmore.gemspec)
+  s.homepage    = 'https://github.com/cbetta/elsmore'
+  s.license     = 'MIT'
+  s.require_path = 'lib'
+
+  s.add_development_dependency('rake')
+
+  s.add_dependency('commander')
+  s.add_dependency('httparty')
+  s.add_dependency('oga')
+  s.add_dependency('awesome_print')
+  s.add_dependency('colorize')
+end

data/lib/elsmore/command.rb

@@ -0,0 +1,43 @@
+require 'commander'
+
+module Elsmore
+  class Command
+    include Commander::Methods
+
+    attr_accessor :emitter
+
+    def initialize
+      self.emitter = Elsmore::Emitter.new
+    end
+
+    def run
+      program :name, 'elsmore'
+      program :version, Elsmore::VERSION
+      program :description, 'A convenient scraper for archiving sites'
+      program :help, 'Author', 'Cristiano Betta <cristiano@betta.io>'
+
+      command :snap do |c|
+        c.syntax = 'spider <url> [options]'
+        c.description = 'Spiders a URL within from the given page, sticking within the original domain'
+        c.action do |args, options|
+          scraper = Elsmore::Scraper.new(args.first)
+          scraper.emitter = emitter
+          result = scraper.run
+
+          emitter.newline
+          emitter.newline
+          emitter.say "Processed"
+          emitter.pretty result[:processed]
+          emitter.newline
+          emitter.say "Could not be processed"
+          emitter.pretty result[:invalid]
+        end
+      end
+      alias_command :'go fetch', :'snap'
+      default_command :snap
+
+
+      run!
+    end
+  end
+end

data/lib/elsmore/document.rb

@@ -0,0 +1,41 @@
+require 'httparty'
+require 'oga'
+
+module Elsmore
+  class Document
+    attr_accessor :url
+
+    def initialize url, parent = nil
+      self.url = Elsmore::Url.new(url, parent)
+    end
+
+    def links
+      @links ||= begin
+        doc.xpath('//a').map do |element|
+          element.attribute('href')
+        end.compact.map(&:value).map do |href|
+          Elsmore::Document.new(href, url)
+        end.compact
+      end
+    end
+
+    def data
+      doc.to_xml
+    end
+
+    def write!
+      Elsmore::Writer.new(self).write
+    end
+
+    def rewrite
+      Elsmore::Rewriter.new(self).rewrite
+    end
+
+    def doc
+      @doc ||= begin
+        html = HTTParty.get(url.canonical_url)
+        Oga.parse_html(html)
+      end
+    end
+  end
+end

data/lib/elsmore/emitter.rb

@@ -0,0 +1,30 @@
+require 'awesome_print'
+require 'colorize'
+
+module Elsmore
+  class Emitter
+    def newline
+      say "\n"
+    end
+
+    def say value
+      puts value
+    end
+
+    def dot
+      print ".".colorize(:green)
+    end
+
+    def x
+      print "x".colorize(:red)
+    end
+
+    def unsure
+      print "?".colorize(:blue)
+    end
+
+    def pretty value
+      ap value
+    end
+  end
+end

data/lib/elsmore/resource.rb

@@ -0,0 +1,21 @@
+require 'open-uri'
+
+module Elsmore
+  class Resource
+    attr_accessor :url, :filename
+
+    def initialize url, parent
+      self.url = Elsmore::Url.new(url, parent)
+    end
+
+    def write!
+      writer = Elsmore::Writer.new(self)
+      writer.write
+      self.filename = writer.canonical_filename
+    end
+
+    def data
+      @data ||= open(url.canonical_url).read
+    end
+  end
+end

data/lib/elsmore/rewriter.rb

@@ -0,0 +1,57 @@
+module Elsmore
+  class Rewriter
+    attr_accessor :resource
+
+    def initialize resource
+      self.resource = resource
+    end
+
+    def rewrite
+      rewrite_linked_resources
+      rewrite_links
+    end
+
+    private
+
+    def rewrite_linked_resources
+      write_css
+      write_js
+      write_images
+    end
+
+    def write_css
+      resource.doc.xpath('//link[rel=stylesheet]').each do |element|
+        write_element(element, 'href')
+      end
+    end
+
+    def write_images
+      resource.doc.xpath('//img').each do |element|
+        write_element(element, 'src')
+      end
+    end
+
+    def write_js
+      resource.doc.xpath('//script').each do |element|
+        write_element(element, 'src')
+      end
+    end
+
+    def write_element element, key
+      return unless element.attribute(key)
+      url = element.attribute(key).value
+      _resource = Elsmore::Resource.new(url, resource.url)
+      _resource.write!
+      element.attribute(key).value = _resource.filename
+    end
+
+    def rewrite_links
+      resource.doc.xpath('//a').each do |element|
+        return unless element.attribute('href')
+        href = element.attribute('href').value
+        url = Elsmore::Url.new(href, resource.url)
+        element.attribute('href').value = url.absolute_path_or_external_url
+      end
+    end
+  end
+end

data/lib/elsmore/scraper.rb

@@ -0,0 +1,49 @@
+module Elsmore
+  class Scraper
+    attr_accessor :emitter
+
+    def initialize initial_url
+      seed = Elsmore::Document.new(initial_url)
+
+      @valid_domains = [seed.url.host]
+      @unprocessed = [seed]
+      @processed = []
+      @invalid = []
+    end
+
+    def run
+      while !@unprocessed.empty?
+        document = @unprocessed.shift
+        next if @processed.include?(document.url.canonical_url)
+        emitter.dot
+
+        enqueue(document.links)
+        document.rewrite
+        document.write!
+
+        @processed << document.url.canonical_url
+      end
+
+      {
+        processed: @processed,
+        invalid: @invalid
+      }
+    end
+
+    private
+
+    def enqueue links
+      links.each_with_index do |document, index|
+        if !document.url.valid
+          emitter.unsure
+          @invalid << document.url.raw_url
+          next
+        end
+
+        next if !@valid_domains.include?(document.url.host)
+        next if @processed.include?(document.url.canonical_url)
+        @unprocessed << document
+      end
+    end
+  end
+end

data/lib/elsmore/url.rb

@@ -0,0 +1,63 @@
+module Elsmore
+  class Url
+    attr_accessor :parent, :raw_url, :valid, :uri
+
+    def initialize raw_url, parent
+      self.raw_url = raw_url.strip
+      self.parent = parent
+      self.valid = true
+
+      sanitize_string
+      parse_uri
+    end
+
+    def host
+      uri.host
+    end
+
+    def parent_host
+      return unless parent
+      parent.host
+    end
+
+    def scheme
+      uri.scheme
+    end
+
+    def canonical_url
+      uri.to_s
+    end
+
+    def absolute_path_or_external_url
+      if parent && parent.host == host
+        uri.path
+      else
+        canonical_url
+      end
+    end
+
+    private
+
+    def sanitize_string
+      if raw_url.start_with?('//')
+        self.raw_url = "http:#{raw_url}"
+      elsif raw_url.start_with?('/')
+        self.raw_url = "#{parent.scheme}://#{parent.host}#{raw_url}"
+      elsif !(raw_url.start_with?('http://') || raw_url.start_with?('https://'))
+        if parent
+          self.raw_url = URI.join(parent.canonical_url, raw_url).to_s
+        else
+          self.raw_url = "http://#{raw_url}"
+        end
+      end
+    end
+
+    def parse_uri
+      self.uri = URI.parse(raw_url)
+      uri.path = "/" if uri.path.empty?
+      uri.fragment = nil
+    rescue
+      self.valid = false
+    end
+  end
+end

data/lib/elsmore/version.rb

@@ -0,0 +1,3 @@
+module Elsmore
+  VERSION = '0.1.1'
+end

data/lib/elsmore/writer.rb

@@ -0,0 +1,71 @@
+module Elsmore
+  class Writer
+    attr_accessor :resource
+
+    def initialize resource
+      self.resource = resource
+    end
+
+    def write
+      write_file
+    end
+
+    def canonical_filename
+      if host == parent_host
+        "#{filename}"
+      else
+        "/#{host}#{filename}"
+      end
+    end
+
+
+    private
+
+    def write_file
+      return if File.exist?(full_filename)
+      ensure_directory full_filename
+      File.open(full_filename, 'w') do |file|
+        file.write(resource.data)
+      end
+    end
+
+    def full_filename
+      @full_filename ||= begin
+        if host == parent_host || !parent_host
+          "#{host}#{filename}"
+        else
+          "#{parent_host}/#{host}#{filename}"
+        end
+      end
+    end
+
+    def filename
+      @filename ||= begin
+        path = resource.url.uri.path
+        if path.end_with?('/')
+          return path + 'index.html'
+        elsif !path.split('/').last.include?(".")
+          return path + '/index.html'
+        end
+        path
+      end
+    end
+
+    def ensure_directory filename
+      dir = File.dirname(filename)
+      unless File.directory?(dir)
+        FileUtils.mkdir_p(dir)
+      end
+    end
+
+    def host
+      resource.url.host
+    end
+
+    def parent_host
+      resource.url.parent_host
+    end
+
+
+  end
+end

data/lib/elsmore.rb

@@ -0,0 +1,9 @@
+require 'elsmore/version'
+require 'elsmore/command'
+require 'elsmore/scraper'
+require 'elsmore/document'
+require 'elsmore/emitter'
+require 'elsmore/writer'
+require 'elsmore/url'
+require 'elsmore/resource'
+require 'elsmore/rewriter'

metadata

@@ -0,0 +1,143 @@
+--- !ruby/object:Gem::Specification
+name: elsmore
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Cristiano Betta
+- Mike Elsmore
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-10-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: commander
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: oga
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: awesome_print
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Backup too for Hackference
+email:
+- cbetta@gmail.com
+- mike@hackference.co.uk
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- elsmore.gemspec
+- lib/elsmore.rb
+- lib/elsmore/command.rb
+- lib/elsmore/document.rb
+- lib/elsmore/emitter.rb
+- lib/elsmore/resource.rb
+- lib/elsmore/rewriter.rb
+- lib/elsmore/scraper.rb
+- lib/elsmore/url.rb
+- lib/elsmore/version.rb
+- lib/elsmore/writer.rb
+homepage: https://github.com/cbetta/elsmore
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Backup too for Hackference
+test_files: []