wayback_archiver 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b50fa3a16a416da9fea6e7eb544173f9bd8c10ea
-  data.tar.gz: cfd8ebea5a0ed30fbc06d038a157e10961a4b5c9
+  metadata.gz: c846ec4d2baeb3bc0087387f0501d25587424563
+  data.tar.gz: c7d4b89507ba88b2504e2c409ebe81440aaa1eec
 SHA512:
-  metadata.gz: 0ab0b992bbc1da9433684c5d4f6266d603f07f73ecaa7c9c3a3fafcb1932bb30077e53d5f2e866990756943bb23638e542d7a9cee80d91e71768467042f26b68
-  data.tar.gz: 37ab8183c5738c0d1c9540b39395b25c06f80fbb5c8412848892f71a8ada68d5a0611d9fc831a427d638d350fd58c32aac27bf9d19df0eade960359b51b3b4d3
+  metadata.gz: 16f09492eebc97040f466702cd5308260524dcd18bc04e289aad9813e48f8cf43fb3a77ed636e48a1fae4932c52e57c5b3316508518f327ac754cbfe036440d6
+  data.tar.gz: 8f2842913687e9d917d5ae609c12f598919bf97079d8e9a464ea6f9c679c4ff642708c63c3ab4bf37b30336878d2aab6eef3c73665f7961d971e896aa9a39af2

data/bin/wayback_archiver

@@ -1,4 +1,12 @@
 #!/usr/bin/env ruby
 
 require 'wayback_archiver'
-WaybackArchiver.archive(ARGV[0], ARGV[1])
+
+url  = ARGV[0]
+from = ARGV[1]
+
+if from.nil?
+  WaybackArchiver.archive(url)
+else
+  WaybackArchiver.archive(url, from)
+end

data/lib/wayback_archiver.rb

@@ -1,46 +1,26 @@
-require 'net/http'
 require 'uri'
+require 'net/http'
 require 'rexml/document'
 
-module WaybackArchiver
-  WAYBACK_BASE_URL = 'https://web.archive.org/save/'
-
-  def self.archive(source, from)
-    from = from || :sitemap
-    urls = Array.new
-    case from.to_sym
-    when :sitemap
-      urls = WaybackArchiver.urls_from_sitemap("#{source}/sitemap.xml")
-    when :url
-      urls << source
-    when :file
-      urls = WaybackArchiver.urls_from_file(source)
-    end
-
-    urls.each_with_index do |url, index|
-      puts "Archiving (#{index + 1}/#{urls.length}): #{url}"
-      request_url = URI("#{WAYBACK_BASE_URL}#{url}")
-      res = Net::HTTP.get_response(request_url)
+require 'wayback_archiver/collector'
+require 'wayback_archiver/archive'
+require 'wayback_archiver/request'
 
-      puts "#{res.code} => #{res.message}"
+module WaybackArchiver
+  BASE_URL = 'https://web.archive.org/save/'
+
+  def self.archive(source, from = :sitemap)
+    urls = case from
+    when :sitemap, 'sitemap'
+      Collector.urls_from_sitemap("#{source}/sitemap.xml")
+    when :url, 'url'
+      Array(source)
+    when :file, 'file'
+      Collector.urls_from_file(source)
+    else
+      raise ArgumentError, "Unknown type: '#{from}'. Allowed types: sitemal, url, file"
     end
-  end
-
-  def self.urls_from_sitemap(url)
-    urls     = Array.new
-    xml_data = Net::HTTP.get_response(URI.parse(url)).body
-    document = REXML::Document.new(xml_data)
-
-    document.elements.each('urlset/url/loc') { |element| urls << element.text }
-    urls
-  end
-
-  def self.urls_from_file(path)
-    urls = Array.new
-    text = File.open(path).read
-    text.gsub!(/\r\n?/, "\n") # Normalize line endings
-    text.each_line { |line| urls << line.gsub(/\n/, '') }
-    urls
+    Archive.post(urls)
   end
 
 end

data/lib/wayback_archiver/archive.rb

@@ -0,0 +1,19 @@
+module WaybackArchiver
+  class Archive
+
+    def self.post(urls)
+      urls.each_with_index do |url, index|
+        request_url = "#{BASE_URL}#{url}"
+        puts "Archiving (#{index + 1}/#{urls.length}): #{url}"
+        begin
+          res = Request.get_response(request_url)
+          puts "#{res.code} => #{res.message}"
+        rescue Exception => e
+          puts "Error message: #{e.message}"
+          puts "Failed to archive: #{url}"
+        end
+      end
+    end
+
+  end
+end

data/lib/wayback_archiver/collector.rb

@@ -0,0 +1,27 @@
+module WaybackArchiver
+  class Collector
+
+    class << self
+
+      def urls_from_sitemap(url)
+        urls     = Array.new
+        xml_data = Request.get_response(url).body
+        document = REXML::Document.new(xml_data)
+
+        document.elements.each('urlset/url/loc') { |element| urls << element.text }
+        urls
+      end
+
+      def urls_from_file(path)
+        raise ArgumentError, "No such file: #{path}" unless File.exist?(path)
+        urls = Array.new
+        text = File.open(path).read
+        text.gsub!(/\r\n?/, "\n") # Normalize line endings
+        text.each_line { |line| urls << line.gsub(/\n/, '').strip }
+        urls.reject(&:empty?)
+      end
+
+    end
+
+  end
+end

data/lib/wayback_archiver/request.rb

@@ -0,0 +1,9 @@
+module WaybackArchiver
+  class Request
+
+    def self.get_response(url)
+      Net::HTTP.get_response(URI.parse(url))
+    end
+
+  end
+end

data/lib/wayback_archiver/version.rb

@@ -1,3 +1,3 @@
 module WaybackArchiver
-  VERSION = '0.0.1'
+  VERSION = '0.0.2'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wayback_archiver
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Jacob Burenstam
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-17 00:00:00.000000000 Z
+date: 2014-07-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -46,15 +46,12 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- Gemfile
-- LICENSE.txt
-- README.md
-- Rakefile
 - bin/wayback_archiver
-- lib/wayback_archiver.rb
+- lib/wayback_archiver/archive.rb
+- lib/wayback_archiver/collector.rb
+- lib/wayback_archiver/request.rb
 - lib/wayback_archiver/version.rb
-- wayback_archiver.gemspec
+- lib/wayback_archiver.rb
 homepage: https://github.com/buren/wayback_archiver
 licenses:
 - MIT
@@ -77,6 +74,6 @@ requirements: []
 rubyforge_project: 
 rubygems_version: 2.0.0
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Send URLs to Wayback Machine
 test_files: []

data/.gitignore

@@ -1,17 +0,0 @@
-*.gem
-*.rbc
-.bundle
-.config
-.yardoc
-Gemfile.lock
-InstalledFiles
-_yardoc
-coverage
-doc/
-lib/bundler/man
-pkg
-rdoc
-spec/reports
-test/tmp
-test/version_tmp
-tmp

data/Gemfile

@@ -1,4 +0,0 @@
-source 'https://rubygems.org'
-
-# Specify your gem's dependencies in wayback_archiver.gemspec
-gemspec

data/LICENSE.txt

@@ -1,22 +0,0 @@
-Copyright (c) 2014 Jacob Burenstam
-
-MIT License
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -1,30 +0,0 @@
-# WaybackArchiver
-Send URLs to [Wayback Machine](https://archive.org/web/) from [/sitemap.xml](http://www.sitemaps.org), single URL or file with URLs.
-
-## Installation
-Install the gem:
-```bash
-gem install wayback_archiver
-```
-
-## Usage
-
-```bash
-wayback_archiver http://example.com               # Send each line defined in http://example.com/sitemap.xml
-wayback_archiver http://example.com/some/path url # Only send http://example.com/some/path to Wayback Machine
-wayback_archiver /path/to/some/file               # With an URL on each line
-```
-
-View domain archive: https://web.archive.org/web/*/http://example.com
-
-
-## Contributing
-
-1. Fork it
-2. Create your feature branch (`git checkout -b my-new-feature`)
-3. Commit your changes (`git commit -am 'Add some feature'`)
-4. Push to the branch (`git push origin my-new-feature`)
-5. Create new Pull Request
-
----------
-Don't know what a sitemap is? [http://sitemaps.org](http://www.sitemaps.org)

data/Rakefile

@@ -1 +0,0 @@
-require 'bundler/gem_tasks'

data/wayback_archiver.gemspec

@@ -1,34 +0,0 @@
-# coding: utf-8
-lib = File.expand_path('../lib', __FILE__)
-$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'wayback_archiver/version'
-
-Gem::Specification.new do |spec|
-  spec.name          = 'wayback_archiver'
-  spec.version       = WaybackArchiver::VERSION
-
-  spec.required_rubygems_version = Gem::Requirement.new('>= 0') if spec.respond_to? :required_rubygems_version=
-  spec.authors       = ['Jacob Burenstam']
-  spec.email         = ['burenstam@gmail.com']
-  spec.summary       = %q{Send URLs to Wayback Machine}
-  spec.description   = %q{Send URLs to Wayback Machine. From: sitemap, file or single URL.}
-  spec.homepage      = 'https://github.com/buren/wayback_archiver'
-  spec.license       = 'MIT'
-
-  spec.files         = `git ls-files`.split($/)
-  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
-  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
-  spec.require_paths = ['lib']
-
-  if spec.respond_to? :specification_version then
-    spec.specification_version = 3
-
-    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-    else
-    end
-  else
-  end
-
-  spec.add_development_dependency 'bundler', '~> 1.3'
-  spec.add_development_dependency 'rake'
-end