wayback_archiver 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b50fa3a16a416da9fea6e7eb544173f9bd8c10ea
+  data.tar.gz: cfd8ebea5a0ed30fbc06d038a157e10961a4b5c9
+SHA512:
+  metadata.gz: 0ab0b992bbc1da9433684c5d4f6266d603f07f73ecaa7c9c3a3fafcb1932bb30077e53d5f2e866990756943bb23638e542d7a9cee80d91e71768467042f26b68
+  data.tar.gz: 37ab8183c5738c0d1c9540b39395b25c06f80fbb5c8412848892f71a8ada68d5a0611d9fc831a427d638d350fd58c32aac27bf9d19df0eade960359b51b3b4d3

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in wayback_archiver.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Jacob Burenstam
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,30 @@
+# WaybackArchiver
+Send URLs to [Wayback Machine](https://archive.org/web/) from [/sitemap.xml](http://www.sitemaps.org), single URL or file with URLs.
+
+## Installation
+Install the gem:
+```bash
+gem install wayback_archiver
+```
+
+## Usage
+
+```bash
+wayback_archiver http://example.com               # Send each line defined in http://example.com/sitemap.xml
+wayback_archiver http://example.com/some/path url # Only send http://example.com/some/path to Wayback Machine
+wayback_archiver /path/to/some/file               # With an URL on each line
+```
+
+View domain archive: https://web.archive.org/web/*/http://example.com
+
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+---------
+Don't know what a sitemap is? [http://sitemaps.org](http://www.sitemaps.org)

data/Rakefile

@@ -0,0 +1 @@
+require 'bundler/gem_tasks'

data/bin/wayback_archiver

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require 'wayback_archiver'
+WaybackArchiver.archive(ARGV[0], ARGV[1])

data/lib/wayback_archiver.rb

@@ -0,0 +1,46 @@
+require 'net/http'
+require 'uri'
+require 'rexml/document'
+
+module WaybackArchiver
+  WAYBACK_BASE_URL = 'https://web.archive.org/save/'
+
+  def self.archive(source, from)
+    from = from || :sitemap
+    urls = Array.new
+    case from.to_sym
+    when :sitemap
+      urls = WaybackArchiver.urls_from_sitemap("#{source}/sitemap.xml")
+    when :url
+      urls << source
+    when :file
+      urls = WaybackArchiver.urls_from_file(source)
+    end
+
+    urls.each_with_index do |url, index|
+      puts "Archiving (#{index + 1}/#{urls.length}): #{url}"
+      request_url = URI("#{WAYBACK_BASE_URL}#{url}")
+      res = Net::HTTP.get_response(request_url)
+
+      puts "#{res.code} => #{res.message}"
+    end
+  end
+
+  def self.urls_from_sitemap(url)
+    urls     = Array.new
+    xml_data = Net::HTTP.get_response(URI.parse(url)).body
+    document = REXML::Document.new(xml_data)
+
+    document.elements.each('urlset/url/loc') { |element| urls << element.text }
+    urls
+  end
+
+  def self.urls_from_file(path)
+    urls = Array.new
+    text = File.open(path).read
+    text.gsub!(/\r\n?/, "\n") # Normalize line endings
+    text.each_line { |line| urls << line.gsub(/\n/, '') }
+    urls
+  end
+
+end

data/lib/wayback_archiver/version.rb

@@ -0,0 +1,3 @@
+module WaybackArchiver
+  VERSION = '0.0.1'
+end

data/wayback_archiver.gemspec

@@ -0,0 +1,34 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'wayback_archiver/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'wayback_archiver'
+  spec.version       = WaybackArchiver::VERSION
+
+  spec.required_rubygems_version = Gem::Requirement.new('>= 0') if spec.respond_to? :required_rubygems_version=
+  spec.authors       = ['Jacob Burenstam']
+  spec.email         = ['burenstam@gmail.com']
+  spec.summary       = %q{Send URLs to Wayback Machine}
+  spec.description   = %q{Send URLs to Wayback Machine. From: sitemap, file or single URL.}
+  spec.homepage      = 'https://github.com/buren/wayback_archiver'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  if spec.respond_to? :specification_version then
+    spec.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+    else
+    end
+  else
+  end
+
+  spec.add_development_dependency 'bundler', '~> 1.3'
+  spec.add_development_dependency 'rake'
+end

metadata

@@ -0,0 +1,82 @@
+--- !ruby/object:Gem::Specification
+name: wayback_archiver
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Jacob Burenstam
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-07-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 'Send URLs to Wayback Machine. From: sitemap, file or single URL.'
+email:
+- burenstam@gmail.com
+executables:
+- wayback_archiver
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/wayback_archiver
+- lib/wayback_archiver.rb
+- lib/wayback_archiver/version.rb
+- wayback_archiver.gemspec
+homepage: https://github.com/buren/wayback_archiver
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.0
+signing_key: 
+specification_version: 3
+summary: Send URLs to Wayback Machine
+test_files: []