snapshotify 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2155e4a1be045e2d84a53ff9b1732b8ebf478c62
+  data.tar.gz: f7e78a6b078025ab3ebb6325a291583286fa093f
+SHA512:
+  metadata.gz: 95b6c3096a7fe7f0c753f3254ddbe6df17a741b32e237dea7381a504664728cf88381320bb1c0df02a3a7501ead0a1c9cf048616bbf0897cdd9d8a7f4ce61816
+  data.tar.gz: b91a62e4dab128e1ccdf3d696b42b29c91cc01e3889eae10cbf22469fc7e27be1f2f101840b535bed3ba175201caa6cc2d9bc629f12594daf57249136152f495

data/.gitignore

@@ -0,0 +1,3 @@
+site
+pkg
+hackference.co.uk

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,40 @@
+PATH
+  remote: .
+  specs:
+    snapshotify (0.1.0)
+      awesome_print
+      colorize
+      commander
+      httparty
+      oga
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ansi (1.5.0)
+    ast (2.3.0)
+    awesome_print (1.8.0)
+    colorize (0.8.1)
+    commander (4.4.3)
+      highline (~> 1.7.2)
+    highline (1.7.8)
+    httparty (0.15.6)
+      multi_xml (>= 0.5.2)
+    multi_xml (0.6.0)
+    oga (2.11)
+      ast
+      ruby-ll (~> 2.1)
+    rake (12.1.0)
+    ruby-ll (2.1.2)
+      ansi
+      ast
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  rake
+  snapshotify!
+
+BUNDLED WITH
+   1.16.0.pre.3

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2016 Cristiano Betta
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,25 @@
+# Snapshotify
+
+A simple tool for backing up a full site into a static instance.
+
+## To install
+```sh
+gem install snapshotify
+```
+
+## To fetch website
+```sh
+snapshotify go fetch [website_url] [--debug]
+```
+
+## To serve
+```sh
+snapshotify serve [website_url]
+```
+
+## Todo
+
+* Add tests
+* Add code to limit hops
+* Add code to limit domains to scrape
+* Fix bugs in asset rewriting

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require 'rake/testtask'
+
+task :console do
+  exec "irb -r snapshotify -I ./lib"
+end

data/bin/snapshotify

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+require 'snapshotify'
+require 'commander'
+
+Snapshotify::Command.new.run

data/lib/snapshotify.rb

@@ -0,0 +1,9 @@
+require 'snapshotify/version'
+require 'snapshotify/command'
+require 'snapshotify/scraper'
+require 'snapshotify/document'
+require 'snapshotify/emitter'
+require 'snapshotify/writer'
+require 'snapshotify/url'
+require 'snapshotify/resource'
+require 'snapshotify/rewriter'

data/lib/snapshotify/command.rb

@@ -0,0 +1,55 @@
+require 'commander'
+
+module Snapshotify
+  class Command
+    include Commander::Methods
+
+    attr_accessor :emitter
+
+    def initialize
+      self.emitter = Snapshotify::Emitter.new
+    end
+
+    def run
+      program :name, 'snapshotify'
+      program :version, Snapshotify::VERSION
+      program :description, 'A convenient scraper for archiving sites'
+      program :help, 'Author', 'Cristiano Betta <cristiano@betta.io>'
+
+      global_option('--debug') { emitter.debug! }
+
+      command :snap do |c|
+        c.syntax = 'spider <url>'
+        c.description = 'Spiders a URL within from the given page, sticking within the original domain'
+        c.action do |args, options|
+          scraper = Snapshotify::Scraper.new(args.first)
+          scraper.emitter = emitter
+          scraper.run
+
+          emitter.newline
+          emitter.newline
+          emitter.say "Processed"
+          emitter.pretty scraper.processed
+          emitter.newline
+          emitter.say "Could not be processed"
+          emitter.pretty scraper.invalid
+
+          emitter.newline
+          emitter.say "Run 'snapshotify serve #{args.first}' to start a webserver on port 8000 with your local copy"
+        end
+      end
+      alias_command :'go fetch', :'snap'
+      default_command :snap
+
+      command :serve do |c|
+        c.syntax = 'serve <folder_name>'
+        c.description = 'Serve local folder'
+        c.action do |args, options|
+          exec "ruby -run -ehttpd ./#{ARGV[1]} -p8000"
+        end
+      end
+
+      run!
+    end
+  end
+end

data/lib/snapshotify/document.rb

@@ -0,0 +1,45 @@
+require 'httparty'
+require 'oga'
+
+module Snapshotify
+  class Document
+    attr_accessor :url, :emitter
+
+    def initialize url, parent = nil
+      self.url = Snapshotify::Url.new(url, parent)
+    end
+
+    def links
+      @links ||= begin
+        doc.xpath('//a').map do |element|
+          element.attribute('href')
+        end.compact.map(&:value).map do |href|
+          Snapshotify::Document.new(href, url)
+        end.compact
+      end
+    end
+
+    def data
+      doc.to_xml
+    end
+
+    def write!
+      writer = Snapshotify::Writer.new(self)
+      writer.emitter = emitter
+      writer.write
+    end
+
+    def rewrite
+      rewriter = Snapshotify::Rewriter.new(self)
+      rewriter.emitter = emitter
+      rewriter.rewrite
+    end
+
+    def doc
+      @doc ||= begin
+        html = HTTParty.get(url.canonical_url)
+        Oga.parse_html(html)
+      end
+    end
+  end
+end

data/lib/snapshotify/emitter.rb

@@ -0,0 +1,40 @@
+require 'awesome_print'
+require 'colorize'
+
+module Snapshotify
+  class Emitter
+    attr_accessor :debug
+
+    def initialize
+      self.debug = false
+    end
+
+    def debug!
+      self.debug = true
+    end
+
+    def newline
+      puts "\n"
+    end
+
+    def pretty value
+      ap value
+    end
+
+    def log message
+      if debug
+        puts message
+      else
+        print ".".colorize(:green)
+      end
+    end
+
+    def warning message
+      if debug
+        puts message.colorize(:yellow)
+      else
+        print ".".colorize(:yellow)
+      end
+    end
+  end
+end

data/lib/snapshotify/resource.rb

@@ -0,0 +1,45 @@
+require 'httparty'
+
+module Snapshotify
+  class Resource
+    attr_accessor :url, :filename, :emitter, :parent
+
+    def initialize url, parent
+      self.url = Snapshotify::Url.new(url, parent)
+      self.parent = parent
+    end
+
+    def write! nested_urls = false
+      process_nested_urls if nested_urls
+
+      writer = Snapshotify::Writer.new(self)
+      writer.emitter = emitter
+      writer.write
+      self.filename = writer.canonical_filename
+    end
+
+    def data
+      @data ||= HTTParty.get(url.canonical_url)
+    end
+
+    private
+
+    def process_nested_urls
+      urls = data.scan(/url\((.*?)\)/i).map do |match|
+        if match[0].start_with?('"') || match[0].start_with?("'")
+          match[0][1...-1]
+        else
+          match[0]
+        end
+      end
+
+      urls.each do |nested_url|
+        resource = Snapshotify::Resource.new(nested_url, url)
+        resource.emitter = emitter
+        resource.write!
+
+        @data.gsub!(nested_url, resource.url.resource_path)
+      end
+    end
+  end
+end

data/lib/snapshotify/rewriter.rb

@@ -0,0 +1,67 @@
+module Snapshotify
+  class Rewriter
+    attr_accessor :resource, :emitter
+
+    def initialize resource
+      self.resource = resource
+    end
+
+    def rewrite
+      rewrite_linked_resources
+      rewrite_links
+    end
+
+    private
+
+    def rewrite_linked_resources
+      write_css
+      write_js
+      write_images
+    end
+
+    def write_css
+      resource.doc.xpath('//link[@rel="stylesheet"]').each do |element|
+        write_element(element, 'href', replace: true)
+      end
+    end
+
+    def write_images
+      resource.doc.xpath('//img').each do |element|
+        write_element(element, 'src', replace: false)
+      end
+    end
+
+    def write_js
+      resource.doc.xpath('//script').each do |element|
+        write_element(element, 'src', replace: false)
+      end
+    end
+
+    def write_element element, key, options = {}
+      return unless element.attribute(key)
+      url = element.attribute(key).value
+      _resource = Snapshotify::Resource.new(url, resource.url)
+      _resource.emitter = emitter
+      _resource.write!(options[:replace])
+
+      emitter.log("# Rewriting #{url} => #{_resource.filename}") if url != _resource.filename
+      element.attribute(key).value = _resource.filename
+    end
+
+    def rewrite_links
+      resource.doc.xpath('//a').each do |element|
+        return unless element.attribute('href')
+        href = element.attribute('href').value
+
+        url = Snapshotify::Url.new(href, resource.url)
+        if url.valid
+          new_url = url.absolute_path_or_external_url
+        else
+          new_url = href
+        end
+        emitter.log("# Rewriting #{href} => #{new_url}") if href != new_url
+        element.attribute('href').value = new_url
+      end
+    end
+  end
+end

data/lib/snapshotify/scraper.rb

@@ -0,0 +1,61 @@
+module Snapshotify
+  class Scraper
+    attr_accessor :emitter, :unprocessed, :processed, :invalid, :unprocessed_urls, :valid_domains
+
+    def initialize initial_url
+      seed = Snapshotify::Document.new(initial_url)
+
+      self.valid_domains = [seed.url.host]
+
+      self.unprocessed = [seed]
+      self.unprocessed_urls = [seed.url.canonical_url]
+
+      self.processed = []
+      self.invalid = []
+    end
+
+    def run
+      while !unprocessed.empty?
+        document = unprocessed.shift
+        process document
+      end
+      self
+    end
+
+    private
+
+    def process document
+      emitter.log(document.url.canonical_url.colorize(:green))
+
+      document.emitter = emitter
+
+      enqueue(document.links)
+      document.rewrite
+      document.write!
+
+      processed << document.url.canonical_url
+    end
+
+    def enqueue links
+      links.each_with_index do |document, index|
+        next unless valid?(document)
+        next if !valid_domains.include?(document.url.host)
+        next if processed.include?(document.url.canonical_url)
+        next if unprocessed_urls.include?(document.url.canonical_url)
+
+        emitter.log("> Enqueued: #{document.url.canonical_url}")
+
+        unprocessed << document
+        unprocessed_urls << document.url.canonical_url
+      end
+    end
+
+    def valid?(document)
+      if !document.url.valid && !self.invalid.include?(document.url.raw_url)
+        emitter.warning("> Invalid URL: #{document.url.raw_url}")
+        invalid << document.url.raw_url
+      end
+      document.url.valid
+    end
+  end
+end

data/lib/snapshotify/url.rb

@@ -0,0 +1,71 @@
+module Snapshotify
+  class Url
+    attr_accessor :parent, :raw_url, :valid, :uri
+
+    def initialize raw_url, parent
+      self.raw_url = raw_url.strip
+      self.parent = parent
+      self.valid = true
+
+      sanitize_string
+      parse_uri
+    end
+
+    def host
+      uri.host
+    end
+
+    def parent_host
+      return unless parent
+      parent.host
+    end
+
+    def scheme
+      uri.scheme
+    end
+
+    def canonical_url
+      uri.to_s
+    end
+
+    def absolute_path_or_external_url
+      if parent && parent.host == host
+        uri.path
+      else
+        canonical_url
+      end
+    end
+
+    def resource_path
+      if parent && parent.host == host
+        uri.path
+      else
+        canonical_url.gsub('http:/', '').gsub('https:/', '')
+      end
+    end
+
+    private
+
+    def sanitize_string
+      if raw_url.start_with?('//')
+        self.raw_url = "http:#{raw_url}"
+      elsif raw_url.start_with?('/')
+        self.raw_url = "#{parent.scheme}://#{parent.host}#{raw_url}"
+      elsif !(raw_url.start_with?('http://') || raw_url.start_with?('https://'))
+        if parent
+          self.raw_url = URI.join(parent.canonical_url, raw_url).to_s
+        else
+          self.raw_url = "http://#{raw_url}"
+        end
+      end
+    end
+
+    def parse_uri
+      self.uri = URI.parse(raw_url)
+      uri.path = "/" if uri.path.empty?
+      uri.fragment = nil
+    rescue
+      self.valid = false
+    end
+  end
+end

data/lib/snapshotify/version.rb

@@ -0,0 +1,3 @@
+module Snapshotify
+  VERSION = '0.1.0'
+end

data/lib/snapshotify/writer.rb

@@ -0,0 +1,74 @@
+module Snapshotify
+  class Writer
+    attr_accessor :resource, :emitter
+
+    def initialize resource
+      self.resource = resource
+    end
+
+    def write
+      write_file
+    end
+
+    def canonical_filename
+      if host == parent_host
+        "#{filename}"
+      else
+        "/#{host}#{filename}"
+      end
+    end
+
+
+    private
+
+    def write_file
+      return if File.exist?(full_filename)
+      ensure_directory full_filename
+
+      emitter.log("! Saving #{full_filename}")
+
+      File.open(full_filename, 'w') do |file|
+        file.write(resource.data)
+      end
+    end
+
+    def full_filename
+      @full_filename ||= begin
+        if host == parent_host || !parent_host
+          "#{host}#{filename}"
+        else
+          "#{parent_host}/#{host}#{filename}"
+        end
+      end
+    end
+
+    def filename
+      @filename ||= begin
+        path = resource.url.uri.path
+        if path.end_with?('/')
+          return path + 'index.html'
+        elsif !path.split('/').last.include?(".")
+          return path + '/index.html'
+        end
+        path
+      end
+    end
+
+    def ensure_directory filename
+      dir = File.dirname(filename)
+      unless File.directory?(dir)
+        FileUtils.mkdir_p(dir)
+      end
+    end
+
+    def host
+      resource.url.host
+    end
+
+    def parent_host
+      resource.url.parent_host
+    end
+
+
+  end
+end

data/snapshotify.gemspec

@@ -0,0 +1,26 @@
+$:.push File.expand_path("../lib", __FILE__)
+require "snapshotify/version"
+
+Gem::Specification.new do |s|
+  s.name        = 'snapshotify'
+  s.version     =  Snapshotify::VERSION
+  s.authors     = ["Cristiano Betta"]
+  s.email       = ['cbetta@gmail.com']
+  s.homepage    = "http://github.com/cbetta/snapshotify"
+  s.summary     = "Backup tool for Hackference"
+  s.description = "Backup tool for Hackference"
+  s.license     = 'MIT'
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_development_dependency('rake')
+
+  s.add_dependency('commander')
+  s.add_dependency('httparty')
+  s.add_dependency('oga')
+  s.add_dependency('awesome_print')
+  s.add_dependency('colorize')
+end

metadata

@@ -0,0 +1,147 @@
+--- !ruby/object:Gem::Specification
+name: snapshotify
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Cristiano Betta
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-10-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: commander
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: oga
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: awesome_print
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Backup tool for Hackference
+email:
+- cbetta@gmail.com
+executables:
+- snapshotify
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- bin/snapshotify
+- lib/snapshotify.rb
+- lib/snapshotify/command.rb
+- lib/snapshotify/document.rb
+- lib/snapshotify/emitter.rb
+- lib/snapshotify/resource.rb
+- lib/snapshotify/rewriter.rb
+- lib/snapshotify/scraper.rb
+- lib/snapshotify/url.rb
+- lib/snapshotify/version.rb
+- lib/snapshotify/writer.rb
+- snapshotify.gemspec
+homepage: http://github.com/cbetta/snapshotify
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.13
+signing_key: 
+specification_version: 4
+summary: Backup tool for Hackference
+test_files: []