staticizer 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 60dbe34b0e69038956585a2c25804422be99bf3e
-  data.tar.gz: 539fb871a2cbbb99d81b5365f942f8a04c2232bf
+  metadata.gz: 7e22029f2f0736aeb3c0afa3ea89350ff230197d
+  data.tar.gz: e9989b561397f19d8ccb0dd01f70ab358fd7c8ec
 SHA512:
-  metadata.gz: cbb0db4aa4864d060534f971a1c16d8d66d26967ae9bc04c7fe6d155339ba80b9c4fe00c30d11fef4c7a696e8f9a8441428b0fee9ae3d865671690257480141d
-  data.tar.gz: 613cb6ac0a9109443f199fa294da8c0a0106b543555402738cabe3a996b12c033d27a8414c52f79f5daa92ecd3a0a9bb0c816bf3558638e712d64909147b1474
+  metadata.gz: 2025749d7cdfa4963fde14b5fdbe82a25779486db1c9aa597eecd73d1309336dbd8ed0a2b8a42c3388804546900fa6c6ee96ef8305813fd665730727eb5fcbd4
+  data.tar.gz: e64a229031fbf15b0b85f0547ea803f67549b237b4d668244c9cad6feee5e52b200b33c032d5784bfb65ab302bffb212ebc997f2c68343fbc570591187216fa3

data/README.md

@@ -18,6 +18,20 @@ We wanted the site uploaded to S3 to respond to the *exact* same URLs (where pos
 the existing site. This way when the  site goes down incoming links from Google search 
 results etc. will still work.
 
+## TODO
+
+* Abillity to specify AWS credentials via file or environment options
+* Tests!
+* Decide what to do with URLs with query strings. Currently they are crawled and uploaded to S3, but those keys cannot be accessed. ex http://squaremill.com/file?test=1 will be uploaded with the key file?test=1, but can only be accessed by encoding the ? like this %3Ftest=1
+* Create a 404 file on S3
+* Provide the option to rewrite absolute URLs to relative urls so that hosting can work on a different domain.
+* Multithread the crawler
+* Check for too many redirects
+* Provide regex options for what urls are scraped
+* Better handling of incorrect server mime types (ex. server returns text/plain for css instead of text/css)
+* Provide more options for uploading (upload via scp, ftp, custom etc.). Split out save/uploading into an interface.
+* Handle large files in a more memory efficient way by streaming uploads/downloads
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -30,7 +44,7 @@ And then execute:
 
 Or install it yourself as:
 
-    $ gem install s3static
+    $ gem install staticizer
 
 ## Command line usage
 
@@ -92,7 +106,7 @@ This will only crawl urls in the domain squaremill.com
     )
     s.crawl
 
-## Cralwer Options
+## Crawler Options
 
 * :aws - Hash of connection options passed to aws/sdk gem
 * :filter_url - proc called to see if a discovered URL should be crawled, return nil to not crawl a url, return the url (can be modified) to crawl

data/Rakefile

@@ -1 +1,7 @@
 require "bundler/gem_tasks"
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << "tests"
+  t.test_files = FileList['tests/*_test.rb']
+end

data/lib/staticizer/crawler.rb

@@ -6,6 +6,9 @@ require 'logger'
 
 module Staticizer
   class Crawler
+    attr_reader :url_queue
+    attr_accessor :output_dir
+
     def initialize(initial_page, opts = {})
       if initial_page.nil?
         raise ArgumentError, "Initial page required"
@@ -14,7 +17,7 @@ module Staticizer
       @opts = opts.dup
       @url_queue = []
       @processed_urls = []
-      @opts[:output_dir] ||= File.expand_path("crawl/")
+      @output_dir = @opts[:output_dir] || File.expand_path("crawl/")
       @log = @opts[:logger] || Logger.new(STDOUT)
       @log.level = @opts[:log_level] || Logger::INFO
 
@@ -32,6 +35,14 @@ module Staticizer
       add_url(initial_page)
     end
 
+    def log_level
+      @log.level
+    end
+
+    def log_level=(level)
+      @log.level = level
+    end
+
     def crawl
       @log.info("Starting crawl")
       while(@url_queue.length > 0)
@@ -59,7 +70,7 @@ module Staticizer
     end
 
     def extract_css_urls(css, base_uri)
-      css.scan(/url\(([^)]+)\)/).map {|src| make_absolute(base_uri, src[0]) }
+      css.scan(/url\(['"]?(.+?)['"]?\)/).map {|src| make_absolute(base_uri, src[0]) }
     end
 
     def add_urls(urls, info = {})
@@ -69,7 +80,7 @@ module Staticizer
     def make_absolute(base_uri, href)
       URI::join(base_uri, href).to_s
     rescue StandardError => e
-      @log.error "Could not make absolute #{base_uri} - #{href}"
+      @log.error "Could not make absolute '#{base_uri}' - '#{href}' - #{e}"
     end
 
     def add_url(url, info = {})
@@ -101,7 +112,7 @@ module Staticizer
       path_segments = path.scan(%r{[^/]*/})
       filename = path.include?("/") ? path[path.rindex("/")+1..-1] : path
 
-      current = @opts[:output_dir]
+      current = @output_dir
       FileUtils.mkdir_p(current) unless File.exist?(current)
 
       # Create all the directories necessary for this file
@@ -175,7 +186,7 @@ module Staticizer
     # TODO: for AWS S3 hosting we could instead create a redirect?
     def process_redirect(url, destination_url)
       body = "<html><head><META http-equiv='refresh' content='0;URL=\"#{destination_url}\"'></head><body>You are being redirected to <a href='#{destination_url}'>#{destination_url}</a>.</body></html>"
-      save_page_to_aws(body, url)
+      save_page(body, url)
     end
 
     # Fetch a URI and save it to disk

data/lib/staticizer/version.rb

@@ -1,3 +1,3 @@
 module Staticizer
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 end

data/staticizer.gemspec

@@ -20,6 +20,7 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rake"
+  spec.add_development_dependency "webmock"
 
   spec.add_runtime_dependency 'nokogiri'
   spec.add_runtime_dependency 'aws-sdk'

data/tests/crawler_test.rb

@@ -1,15 +1,80 @@
 require 'minitest/autorun'
+require 'ostruct'
 
-# TODO!
+lib = File.expand_path(File.dirname(__FILE__) + '/../lib')
+$LOAD_PATH.unshift(lib) if File.directory?(lib) && !$LOAD_PATH.include?(lib)
+
+require 'staticizer'
 
 class TestFilePaths < MiniTest::Unit::TestCase
-  tests = {
-    "" => "index.html"
-    "/" => "index.html"
-    "/asdfdf/dfdf" => "/asdfdf/dfdf"
-    "/asdfdf/dfdf/" => "/asdfdf/dfdf" and "/asdfdf/dfdf/index.html"
-    "/asdfad/asdffd.test" => "/asdfad/asdffd.test"
-    "/?asdfsd=12312" => "/?asdfsd=12312"
-    "/asdfad/asdffd.test?123=sdff" => "/asdfad/asdffd.test?123=sdff"
-  }
+  def setup
+    @crawler = Staticizer::Crawler.new("http://test.com")
+    @crawler.log_level = Logger::FATAL
+    @fake_page = File.read(File.expand_path(File.dirname(__FILE__) + "/fake_page.html"))
+  end
+
+  def test_save_page_to_disk
+    fake_response = OpenStruct.new(:read_body => "test", :body => "test")
+    file_paths = {
+      "http://test.com" => "index.html",
+      "http://test.com/" => "index.html",
+      "http://test.com/asdfdf/dfdf" => "/asdfdf/dfdf",
+      "http://test.com/asdfdf/dfdf/" => ["/asdfdf/dfdf","/asdfdf/dfdf/index.html"],
+      "http://test.com/asdfad/asdffd.test" => "/asdfad/asdffd.test",
+      "http://test.com/?asdfsd=12312" => "/?asdfsd=12312",
+      "http://test.com/asdfad/asdffd.test?123=sdff" => "/asdfad/asdffd.test?123=sdff",
+    }
+
+    # TODO: Stub out file system using https://github.com/defunkt/fakefs?
+    outputdir = "/tmp/staticizer_crawl_test"
+    FileUtils.rm_rf(outputdir)
+    @crawler.output_dir = outputdir
+
+    file_paths.each do |k,v|
+      @crawler.save_page_to_disk(fake_response, URI.parse(k))
+      [v].flatten.each do |file|
+        expected = File.expand_path(outputdir + "/#{file}")
+        assert File.exists?(expected), "File #{expected} not created for url #{k}"
+      end
+    end
+  end
+
+  def test_save_page_to_aws
+  end
+
+  def test_add_url_with_valid_domains
+    test_url = "http://test.com/test"
+    @crawler.add_url(test_url)
+    assert(@crawler.url_queue[-1] == [test_url, {}], "URL #{test_url} not added to queue")
+  end
+
+  def test_add_url_with_filter
+  end
+
+  def test_initialize_options
+  end
+
+  def test_process_url
+  end
+
+  def test_make_absolute
+  end
+
+  def test_link_extraction
+  end
+
+  def test_href_extraction
+  end
+
+  def test_css_extraction
+  end
+
+  def test_css_url_extraction
+  end
+
+  def test_image_extraction
+  end
+
+  def test_script_extraction
+  end
 end

data/tests/fake_page.html

@@ -0,0 +1,288 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <title>Web Application Design and Development &mdash; Square Mill Labs</title>
+  <meta content="authenticity_token" name="csrf-param" />
+<meta content="LshjtNLXmjVY9NINXYQds+2Ur+jxUtqKVjjbDbVl+9w=" name="csrf-token" />
+  <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
+<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no">
+<meta property="og:type" content="website">
+<meta property="og:url" content="http://squaremill.com/">
+<meta property="og:image" content="">
+<meta name="viewport" content="width=device-width, maximum-scale=1.0, initial-scale=1.0">
+<meta name="description" content="Web Application Design and Development &mdash; Square Mill Labs">
+  <link rel="shortcut icon" type="image/png" href="http://squaremill.com/assets/icons/favicon-0fecbe6b20ff5bdf623357a3fac76b4b.png">
+  <link data-turbolinks-track="true" href="/assets/mn_application-5ddad96f16e03ad2137bf02270506e61.css" media="all" rel="stylesheet" />
+  <!--[if lt IE 9]>
+    <script src="http://html5shim.googlecode.com/svn/trunk/html5.js"></script>
+  <![endif]-->
+
+  <script type="text/javascript" src="//use.typekit.net/cjr4fwy.js"></script>
+<script type="text/javascript">try{Typekit.load();}catch(e){}</script>
+  </head>
+
+<body id="public">
+  <script type="text/javascript">
+
+  var _gaq = _gaq || [];
+  _gaq.push(['_setAccount', 'UA-30460332-1']);
+  _gaq.push(['_setDomainName', 'squaremill.com']);
+  _gaq.push(['_trackPageview']);
+
+  (function() {
+    var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
+    ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+    var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
+  })();
+
+</script>
+  
+
+  <header id="header">
+  <nav class="nav container">
+    <a class="branding" href="http://squaremill.com/" rel="home" title="Square Mill - Digital Products for Web and Mobile">
+      <img alt="Square Mill Logo" class="logo" height="16" src="/assets/m2-wordmark-black-97525464acd136ce26b77e39c7ed2ba3.png" width="128" />
+      <p class="description">
+        Digital Products for Web and Mobile
+      </p>
+</a>    <a class="menu-trigger" href="#">Menu</a>
+    <div class="main-nav">
+  <ul class="container">
+    <li><a href="/projects">Projects</a></li>
+    <li><a href="/about">About Us</a></li>
+    <li><a href="/blog">Blog</a></li>
+
+    <!-- <li class="link-biography"><a href="http://squaremill.com/#biography">People</a></li> -->
+  </ul>
+</div>
+
+  </nav>
+  
+</header>
+
+
+
+  <div id="site-content">
+
+
+
+
+    <div class="container" id="home-projects">
+    <section class="big-promo">
+  <div class="project" id="project-7">
+    <a href="/projects/bon-voyaging">
+      <div class="devices">
+        <div class="laptop device">
+          <img alt="" class="chrome" src="/assets/projects/macbook-pro-1be04a78f99b40e2c676d78fc24fcc3d.png" />
+          <div class="screenshot">
+            <img alt="" src="/uploads/project/desktop_image/7/bonvoyaging-desktop.jpg" />
+          </div>
+        </div>
+
+          <div class="handheld device">
+            <img alt="" class="chrome" src="/assets/projects/iphone5-c1ea3a16be931f1e80bacab5cfec932d.png" />
+            <div class="screenshot">
+              <img alt="" src="/uploads/project/iphone_image/7/bonvoyagin-handheld.jpg" />
+            </div>
+          </div>
+      </div>
+
+      <div class="project-description">
+        <div class="summary">
+          <h2>Bon Voyaging <i class="icon-play-sign"></i></h2>
+          <p>Bon Voyaging enables discerning travelers to expertly envision their next voyage from inspiration to exploration. Powerful search tools and a interactive javascript interface make planning trips fun.</p>
+        </div>
+      </div>
+</a>
+  </div>
+</section>
+    <section class="big-promo">
+  <div class="project" id="project-1">
+    <a href="/projects/kpcb-fellows">
+      <div class="devices">
+        <div class="laptop device">
+          <img alt="" class="chrome" src="/assets/projects/macbook-pro-1be04a78f99b40e2c676d78fc24fcc3d.png" />
+          <div class="screenshot">
+            <img alt="" src="/uploads/project/desktop_image/1/kpcb-fellows-screenshot.jpg" />
+          </div>
+        </div>
+
+          <div class="handheld device">
+            <img alt="" class="chrome" src="/assets/projects/iphone5-c1ea3a16be931f1e80bacab5cfec932d.png" />
+            <div class="screenshot">
+              <img alt="" src="/uploads/project/iphone_image/1/kpcb-fellows-iphone-screenshot.jpg" />
+            </div>
+          </div>
+      </div>
+
+      <div class="project-description">
+        <div class="summary">
+          <h2>KPCB Fellows Website and Brand <i class="icon-play-sign"></i></h2>
+          <p>The Fellows Program is a three-month work-based program that pairs top U.S. Engineering, Design and Product Design students with leading technology companies</p>
+        </div>
+      </div>
+</a>
+  </div>
+</section>
+    <section class="big-promo">
+  <div class="project" id="project-2">
+    <a href="/projects/thomson-reuters-messenger">
+      <div class="devices">
+        <div class="laptop device no-handheld">
+          <img alt="" class="chrome" src="/assets/projects/macbook-pro-1be04a78f99b40e2c676d78fc24fcc3d.png" />
+          <div class="screenshot">
+            <img alt="" src="/uploads/project/desktop_image/2/thomson-reuters-messenger-desktop.png" />
+          </div>
+        </div>
+
+      </div>
+
+      <div class="project-description">
+        <div class="summary">
+          <h2>Thomson Reuters Messenger <i class="icon-play-sign"></i></h2>
+          <p>Messenger is an html5 / javascript instant messenger application for financial professionals</p>
+        </div>
+      </div>
+</a>
+  </div>
+</section>
+    <section class="big-promo">
+  <div class="project" id="project-3">
+    <a href="/projects/kleiner-perkins-caufield-byers-digital-presence">
+      <div class="devices">
+        <div class="laptop device">
+          <img alt="" class="chrome" src="/assets/projects/macbook-pro-1be04a78f99b40e2c676d78fc24fcc3d.png" />
+          <div class="screenshot">
+            <img alt="" src="/uploads/project/desktop_image/3/kpcb-screenshot.jpg" />
+          </div>
+        </div>
+
+          <div class="handheld device">
+            <img alt="" class="chrome" src="/assets/projects/iphone5-c1ea3a16be931f1e80bacab5cfec932d.png" />
+            <div class="screenshot">
+              <img alt="" src="/uploads/project/iphone_image/3/kpcb-iphone-screenshot.jpg" />
+            </div>
+          </div>
+      </div>
+
+      <div class="project-description">
+        <div class="summary">
+          <h2>Kleiner Perkins Caufield &amp; Byers Digital Presence <i class="icon-play-sign"></i></h2>
+          <p>KPCB is a venture capital stalwart located in Silicon Valley with over 40 years of tech and science investment.</p>
+        </div>
+      </div>
+</a>
+  </div>
+</section>
+</div>
+
+<section class="clients full-width">
+  <div class="container">
+    <h2>Clients</h2>
+    <ul class="hlist">
+        <li><a href="http://kpcb.com" rel="friend" target="_blank" title="KPCB&#39;s Website"><img alt="KPCB" src="/uploads/client/image/1/home_logo_kpcb-logo.png" /></a></li>
+        <li><a href="http://thomsonreuters.com" rel="friend" target="_blank" title="Thomson Reuters&#39;s Website"><img alt="Thomson Reuters" src="/uploads/client/image/2/home_logo_thomsonreuters.png" /></a></li>
+        <li><a href="http://sumzero.com" rel="friend" target="_blank" title="SumZero&#39;s Website"><img alt="SumZero" src="/uploads/client/image/3/home_logo_sumzero.png" /></a></li>
+        <li><a href="http://marlboroughgallery.com" rel="friend" target="_blank" title="Marlborough Gallery&#39;s Website"><img alt="Marlborough Gallery" src="/uploads/client/image/4/home_logo_marlborough.png" /></a></li>
+        <li><a href="http://flurry.com" rel="friend" target="_blank" title="Flurry Analytics&#39;s Website"><img alt="Flurry Analytics" src="/uploads/client/image/8/home_logo_flurry.png" /></a></li>
+    </ul>
+  </div>
+</section>
+
+<section class="quote">
+  <blockquote>
+  <p>"Square Mill really took the time to understand our business and think strategically about how we want to engage and communicate with our entrepreneurs online. Together, their small team is responsive, nimble and efficient and has the deep design and technical chops to back it up."</p>
+  <small><a href="http://kpcb.com/partner/christina-lee" rel="friend" title="Christina Lee, Operating Partner at KPCB">Christina Lee</a>, <em>Operating Partner at KPCB</em></small>
+</blockquote>
+</section>
+
+
+
+
+  </div>
+
+  <footer id="footer">
+  <section class="container">
+    <a class="logo" href="http://squaremill.com/" rel="home">
+      <img alt="Square Mill Logo" height="64" src="/assets/md-logo-black-942423ecfd86c43ec6f13f163ea03f97.png" width="64" />
+</a>    <div class="main-nav">
+  <ul class="container">
+    <li><a href="/projects">Projects</a></li>
+    <li><a href="/about">About Us</a></li>
+    <li><a href="/blog">Blog</a></li>
+
+    <!-- <li class="link-biography"><a href="http://squaremill.com/#biography">People</a></li> -->
+  </ul>
+</div>
+
+  </section>
+
+  <p class="copyright">
+    &copy; 2014 Square Mill Labs, LLC. All rights reserved.
+  </p>
+</footer>
+
+  <script type="text/javascript" src="http://code.jquery.com/jquery-2.0.0.js"></script>
+  <script type="text/javascript" src="http://code.jquery.com/jquery-migrate-1.1.1.js"></script>
+  <script src="/assets/mn_application-82f6787dca307be34ec0c9fa6b7ba7d4.js"></script>
+  <script>
+  $(document).ready(function() {
+
+    var controller = $.superscrollorama({
+      triggerAtCenter: true,
+      playoutAnimations: true
+    });
+
+    if ( $(window).width() >= 767 ) {
+      controller.addTween('#project-7', 
+        TweenMax.from($('#project-7'), .7, {
+          css:{"opacity":"0"},
+          onComplete: function(){
+            $('#project-7').toggleClass('active-in')
+          }
+        }), 
+        300, // duration of scroll in pixel units
+        -100, // scroll offset (from center of viewport)
+        true
+        );
+      controller.addTween('#project-1', 
+        TweenMax.from($('#project-1'), .7, {
+          css:{"opacity":"0"},
+          onComplete: function(){
+            $('#project-1').toggleClass('active-in')
+          }
+        }), 
+        300, // duration of scroll in pixel units
+        -100, // scroll offset (from center of viewport)
+        true
+        );
+      controller.addTween('#project-2', 
+        TweenMax.from($('#project-2'), .7, {
+          css:{"opacity":"0"},
+          onComplete: function(){
+            $('#project-2').toggleClass('active-in')
+          }
+        }), 
+        300, // duration of scroll in pixel units
+        -100, // scroll offset (from center of viewport)
+        true
+        );
+      controller.addTween('#project-3', 
+        TweenMax.from($('#project-3'), .7, {
+          css:{"opacity":"0"},
+          onComplete: function(){
+            $('#project-3').toggleClass('active-in')
+          }
+        }), 
+        300, // duration of scroll in pixel units
+        -100, // scroll offset (from center of viewport)
+        true
+        );
+    }
+
+  });
+</script>
+
+</body>
+</html>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: staticizer
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Conor Hunt
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-14 00:00:00.000000000 Z
+date: 2014-01-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -38,6 +38,20 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -87,6 +101,7 @@ files:
 - lib/staticizer/version.rb
 - staticizer.gemspec
 - tests/crawler_test.rb
+- tests/fake_page.html
 homepage: https://github.com/SquareMill/staticizer
 licenses:
 - MIT