staticizer 0.0.6 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 281ee17c4f67f579faa5d1762f9677203696fd0b
-  data.tar.gz: f82db0909cd61819e3f19a43e17eb3ed552111f8
+  metadata.gz: f2ac92b2be780663c4828b685c270eccdee9106d
+  data.tar.gz: 2e31e7b3f701f431ea11e4cbb9a8296ec51475de
 SHA512:
-  metadata.gz: bc5a3463838030e46d0353bd173154ff37b92ae36294d561b4e37270b2ee4dd252b4f6e7aff36cb428b0cc1c917b02fb2ec1abbc1ce7bba85679aed428af68cb
-  data.tar.gz: 3267e1e063ff7c17c6797c7072b35456fa105f0c9af0dfc905c42e9394f7faacb2615b2829019923a9c2de91bec90a499c64b24e28e43ff3ba542f860511ea27
+  metadata.gz: d019bef5884ce685a5def86bc6840edcc95879062b01a114ceeacf8b126fb4a3a24c7ba58cef3f674c7198a422a62b397fb57687a86b55843d0b1fa3f02745a8
+  data.tar.gz: fa25013152b03b1b83c3a1347604699575951efee4dfe669c7b7be04236c75a1473258beb2ca1bd935163a6a47524b3217efd5d157f7b17779a3200cacb6d696

data/README.md

@@ -75,6 +75,8 @@ This will only crawl urls in the domain squaremill.com
 
     s = Staticizer::Crawler.new("http://squaremill.com",
       :aws => {
+        :region => "us-west-1",
+        :endpoint => "http://s3.amazonaws.com",
         :bucket_name => "www.squaremill.com",
         :secret_access_key => "HIA7T189234aADfFAdf322Vs12duRhOHy+23mc1+s",
         :access_key_id => "HJFJS5gSJHMDZDFFSSDQQ"
@@ -106,6 +108,8 @@ This will only crawl urls in the domain squaremill.com
 
     s = Staticizer::Crawler.new("http://squaremill.com",
       :aws => {
+        :region => "us-west-1",
+        :endpoint => "http://s3.amazonaws.com",
         :bucket_name => "www.squaremill.com",
         :secret_access_key => "HIA7T189234aADfFAdf322Vs12duRhOHy+23mc1+s",
         :access_key_id => "HJFJS5gSJHMDZDFFSSDQQ"
@@ -130,6 +134,7 @@ This will only crawl urls in the domain squaremill.com
 * :log_level - Log level - defaults to INFO.
 * :valid_domains - Array of domains that should be crawled. Domains not in this list will be ignored.
 * :process_body - lambda called to pre-process body of content before writing it out.
+* :skip_write - don't write retrieved files to disk or s3, just crawl the site (can be used to find 404s etc.)
 
 ## Contributing
 

data/lib/staticizer/command.rb

@@ -44,6 +44,10 @@ module Staticizer
           options[:logger] = Logger.new(v)
         end
 
+        opts.on("--skip-write [PATH]", "Don't write out files to disk or s3") do |v|
+          options[:skip_write] = true
+        end
+
         opts.on("--valid-domains x,y,z", Array, "Comma separated list of domains that should be crawled, other domains will be ignored") do |v|
           options[:valid_domains] = v
         end

data/lib/staticizer/crawler.rb

@@ -23,9 +23,8 @@ module Staticizer
 
       if @opts[:aws]
         bucket_name = @opts[:aws].delete(:bucket_name)
-        AWS.config(opts[:aws])
-        @s3_bucket = AWS::S3.new.buckets[bucket_name]
-        @s3_bucket.acl = :public_read
+        Aws.config.update(opts[:aws])
+        @s3_bucket = Aws::S3::Resource.new.bucket(bucket_name)
       end
 
       if @opts[:valid_domains].nil?
@@ -86,6 +85,7 @@ module Staticizer
       URI::join(base_uri, href).to_s
     rescue StandardError => e
       @log.error "Could not make absolute '#{base_uri}' - '#{href}' - #{e}"
+      return nil
     end
 
     def add_url(url, info = {})
@@ -103,6 +103,7 @@ module Staticizer
     end
 
     def save_page(response, uri)
+      return if @opts[:skip_write]
       if @opts[:aws]
         save_page_to_aws(response, uri)
       else
@@ -160,20 +161,23 @@ module Staticizer
       key = key.gsub(%r{^/},"")
       key = "index.html" if key == ""
       # Upload this file directly to AWS::S3
-      opts = {:acl => :public_read}
+      opts = {:acl => "public-read"}
       opts[:content_type] = response['content-type'] rescue "text/html"
       @log.info "Uploading #{key} to s3 with content type #{opts[:content_type]}"
       if response.respond_to?(:read_body)
         body = process_body(response.read_body, uri, opts)
-        @s3_bucket.objects[key].write(body, opts)
+        @s3_bucket.object(key).put(opts.merge(body: body))
       else
         body = process_body(response, uri, opts)
-        @s3_bucket.objects[key].write(body, opts)
+        @s3_bucket.object(key).put(opts.merge(body: body))
       end
     end
 
     def process_success(response, parsed_uri)
       url = parsed_uri.to_s
+      if @opts[:filter_process]
+        return if @opts[:filter_process].call(response, parsed_uri)
+      end
       case response['content-type']
       when /css/
         save_page(response, parsed_uri)
@@ -211,28 +215,35 @@ module Staticizer
       parsed_uri = URI(url)
 
       @log.debug "Fetching #{parsed_uri}"
-
+  
       # Attempt to use an already open Net::HTTP connection
       key = parsed_uri.host + parsed_uri.port.to_s
       connection = @http_connections[key]
       if connection.nil?
         connection = Net::HTTP.new(parsed_uri.host, parsed_uri.port)
+        connection.use_ssl = true if parsed_uri.scheme.downcase == "https"
         @http_connections[key] = connection
       end
 
       request = Net::HTTP::Get.new(parsed_uri.request_uri)
-      connection.request(request) do |response|
-        case response
-        when Net::HTTPSuccess
-          process_success(response, parsed_uri)
-        when Net::HTTPRedirection
-          redirect_url = response['location']
-          @log.debug "Processing redirect to #{redirect_url}"
-          process_redirect(parsed_uri, redirect_url)
-          add_url(redirect_url)
-        else
-          @log.error "Error #{response.code}:#{response.message} fetching url #{url}"
+      begin
+        connection.request(request) do |response|
+          case response
+          when Net::HTTPSuccess
+            process_success(response, parsed_uri)
+          when Net::HTTPRedirection
+            redirect_url = response['location']
+            @log.debug "Processing redirect to #{redirect_url}"
+            process_redirect(parsed_uri, redirect_url)
+            add_url(redirect_url)
+          else
+            @log.error "Error #{response.code}:#{response.message} fetching url #{url}"
+          end
         end
+      rescue OpenSSL::SSL::SSLError => e
+        @log.error "SSL Error #{e.message} fetching url #{url}"
+      rescue Errno::ECONNRESET => e
+        @log.error "Error #{e.class}:#{e.message} fetching url #{url}"
       end
     end
 

data/lib/staticizer/version.rb

@@ -1,3 +1,3 @@
 module Staticizer
-  VERSION = "0.0.6"
+  VERSION = "0.0.7"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: staticizer
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.7
 platform: ruby
 authors:
 - Conor Hunt
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-21 00:00:00.000000000 Z
+date: 2016-06-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -122,7 +122,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
 summary: A tool to create a static version of a website for hosting on S3.