surgescrape 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b3d1e58be8a7ae49097955652334f148e92f0949
+  data.tar.gz: 9087e21894944b34a4a92b0ad89cf6a5cd3472b9
+SHA512:
+  metadata.gz: a07cba90208e7c06d5c638a216ca0137ae5ea425a5063ad1c92a04f10f0508ce8858bdfff54df4657e86a5698c8d4c53556e46ad7fb47e2b44eda5b2ea525773
+  data.tar.gz: 62a55f03e9ec0b6fc68d56fe99a768eb2c626170e62e24bb0efda801da63d1ca13ed5ee5b4781621ab8afe37afae2100e280552cd3aca93e07a5f0d0c9d14812

data/.rake_tasks~

@@ -0,0 +1,6 @@
+build
+clean
+clobber
+install
+install:local
+release[remote]

data/COPYING

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2016 Daniel E
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,40 @@
+# SurgeScrape - Scrape your surge.sh website's files
+
+## Installation
+
++ `[sudo] gem install surgescrape` Download & install via RubyGems
+
+## Usage
+
+`sscrape website.surge.sh` A simple really, just specify a domain
+and watch all the files download. Run each time you need to update your
+directory, the tool won't download untouched files.
+
+## What it lookes like
+
+```
+Installing new gem...
+surgescrape (0.1.0) installed.
+daniel@pancake:~/surgescrape$ sscrape 
+Usage : `sscrape website.surge.sh`
+daniel@pancake:~/Desktop/surge$ sscrape surge.sh
+Scraping the files from https://surge.sh...
+The website has different versions of your files and/or new files, your version of these will be overwritten : 
+/css/index.css
+/help/about-the-surge-cdn.html
+[...]
+/stickers/thanks.html
+/tour.html 
+
+Downloading 107 file(s)...
+Downloaded /css/index.css
+Downloaded /help/about-the-surge-cdn.html
+[...]
+Downloaded /stickers/thanks.html
+Downloaded /tour.html
+Finished!
+daniel@pancake:~/Desktop/surge$ sudo serve 80
+Serving on http://pancake:80
+```
+
+## MIT License

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/bin/setup

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install

data/bin/sscrape

@@ -0,0 +1,52 @@
+#!/usr/bin/env ruby
+# SurgeScrape - Scrape your surge.sh website's files
+# Daniel Ethridge
+
+require "surgescrape"
+include SurgeScrape
+
+usage = "Usage : `sscrape website.surge.sh`"
+version = "SurgeScrape version : 0.1.0" #{SurgeScrape::VERSION}"
+
+help = "\n#{usage}
+
+#{version}
+
+A simple utility, just specify a domain
+and watch all the files download. Run each time
+you need to update your directory, the tool won't 
+download untouched files.
+
+--------------------------------------------------
+
+If you find a bug, want extra features, or have a suggestion,
+make a branch on github : https://github.com/wlib/surgescrape\n\n"
+
+# If there are no arguments, print usage
+if ARGV.length == 0
+  puts usage
+  exit
+end
+
+# Argument "parser", though only takes one argument...
+arg = ARGV[0].downcase
+case (arg)
+  when "help", "--help", "h", "-h", "?"
+    puts help
+    exit
+  when "version", "--version", "v", "-v"
+    puts version
+    exit
+  else
+    if arg.match( /^https:\/\/.+/i )
+      scrape(arg)
+      exit
+    elsif arg.match( /^http:\/\/.+/i )
+      puts "https:// only"
+      exit
+    else
+      scrape("https://#{arg}")
+      exit
+    end
+    exit
+end

data/lib/surgescrape/version.rb

@@ -0,0 +1,3 @@
+module SurgeScrape
+  VERSION = "0.1.0"
+end

data/lib/surgescrape.rb

@@ -0,0 +1,94 @@
+#!/usr/bin/env ruby
+# "Scrape" the website based on its `/auto.appcache` file
+# Daniel Ethridge
+
+require 'net/http'
+require 'digest'
+require 'fileutils'
+
+module SurgeScrape
+  def rm_dot(path)
+    return path[1..-1]
+  end
+
+  def get(site, file)
+    uri = URI.parse("#{site}/#{file}")
+    http = Net::HTTP.new(uri.host, uri.port)
+    http.use_ssl = true
+    request = Net::HTTP::Get.new(uri.request_uri, {"Accept-Encoding" => "gzip"})
+    response = http.request(request)
+    return response.body
+  end
+
+  def get_cache_list(site)
+    cache_manifest = get(site, "auto.appcache")
+    cache = cache_manifest
+    files = cache.scan( /^\/.+\..+$/ )
+    hashes = cache.scan( /[a-f0-9]{32}/i )
+    out = {}
+    for i in 0...hashes.length
+      out[files[i].chomp] = hashes[i].chomp
+    end
+    if out.empty?
+      puts "Website does not have a `/auto.appcache` file, try a *.surge.sh URL"
+      exit
+    end
+    return out
+  end
+
+  def get_local_list()
+    files = Dir.glob("./**/*", File::FNM_DOTMATCH).select{ |e| File.file?(e) }
+    hashes = []
+    files.each { |f| hashes << Digest::MD5.file(f).hexdigest }
+    out = {}
+    for i in 0...hashes.length
+      out[files[i].chomp] = hashes[i].chomp
+    end
+    return out
+  end
+
+  def compare_lists(cache, local)
+    local.keys.each { |k| local[rm_dot(k)] = local.delete k }
+    diffs = ( cache.to_a - local.to_a ).to_h.keys
+    extra = ( local.to_a - cache.to_a ).to_h.keys - diffs
+    if ! extra.empty?
+      puts "This directory has file(s) that are not recorded on the website, \
+they will not be overwritten : \n#{extra.join("\n")} \n\n"
+    end
+    if ! diffs.empty?
+      puts "The website has different versions of your files and/or new files, \
+your version of these will be overwritten : \n#{diffs.join("\n")} \n\n"
+      return diffs
+    else
+      return nil
+    end
+  end
+
+  def get_files(site, files)
+    puts "Downloading #{files.length} file(s)..."
+    files.each do |file|
+      retrieved = get(site, file)
+      if retrieved.nil?
+        puts "#{file} could not be downloaded"
+        next
+      end
+      FileUtils.mkdir_p("./#{File.dirname(file)}") unless Dir.exists?("./#{File.dirname(file)}")
+      local = File.open("./#{file}", "w")
+      local.write retrieved
+      puts "Downloaded #{file}"
+    end
+  end
+
+  def scrape(site)
+    puts "Scraping the files from #{site}..."
+    cache = get_cache_list(site)
+    local = get_local_list()
+    files = compare_lists(cache, local)
+    if files.nil?
+      puts "Nothing to download"
+      exit
+    end
+    get_files(site, files)
+    puts "Finished!"
+  end
+end

data/surgescrape.gemspec

@@ -0,0 +1,25 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "surgescrape/version"
+
+Gem::Specification.new do |gem|
+  gem.name          = "surgescrape"
+  gem.version       = SurgeScrape::VERSION
+  gem.authors       = ["Daniel Ethridge"]
+  gem.email         = ["danielethridge@icloud.com"]
+  gem.license = "MIT"
+
+  gem.summary       = %q{Scrape your surge.sh website's files}
+  gem.homepage      = "https://github.com/wlib/surgescrape"
+
+  gem.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  gem.bindir        = "bin"
+  gem.executables   = ["sscrape"]
+  gem.require_paths = ["lib"]
+
+  gem.add_development_dependency "bundler", "~> 1.13"
+  gem.add_development_dependency "rake", "~> 10.0"
+end

metadata

@@ -0,0 +1,83 @@
+--- !ruby/object:Gem::Specification
+name: surgescrape
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Daniel Ethridge
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-12-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: 
+email:
+- danielethridge@icloud.com
+executables:
+- sscrape
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rake_tasks~"
+- COPYING
+- README.md
+- Rakefile
+- bin/setup
+- bin/sscrape
+- lib/surgescrape.rb
+- lib/surgescrape/version.rb
+- pkg/surgescrape-0.1.0.gem
+- surgescrape.gemspec
+homepage: https://github.com/wlib/surgescrape
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Scrape your surge.sh website's files
+test_files: []