warc 1.0.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    ZTQyNTg2YWM5ODBiODFlOGE4ZjczMjM0YmZmNWViZTk0MWM0OTdhZg==
+  data.tar.gz: !binary |-
+    YjJjZWU1YzFhZWRkNmI4M2QyZDkyNDE4N2M1YWVlYmE3NjU0NjVlZQ==
+SHA512:
+  metadata.gz: !binary |-
+    MmEyNDRiMjNhMWYxNTc4OGI1OTRkYTAzYmRkNTIwNzcwMTc2Yjg2Zjc5MjAx
+    YjI4OWQ2NDYzMTQyNzkxNDcwNmUzZjBjZWU3Yzg2NzdiMmNiNGZiNDdjMTEz
+    NTJlODllYzljOGNkNzRmM2QzM2QxMWFkZGI2ZmJiZGY4YjMzOGY=
+  data.tar.gz: !binary |-
+    ODIzMDhiZWIxMTI5YTRiOWQ3NzQzMDc5MjM5MTU4MWJlNGE0OTE1ZjU1YTAz
+    ZjE5MDc5ZDhiYmRiMGQ4NzUwYTc4NWI2MDUxNzcxODM1NjY4NDAxMWI4NTc1
+    MDdkZGQwNzcyYTQxMjNkOTQyZGNiY2YyMmI5YmE5NjQzNWUwNmM=

data/.gitignore

@@ -0,0 +1,20 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+.DS_Store
+.project
+.buildpath

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in warc.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 antoine
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# Warc
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'warc'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install warc
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"

data/bin/warc

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require 'warc/cli'
+
+Warc::CLI.start

data/lib/warc.rb

@@ -0,0 +1,30 @@
+require "warc/version"
+
+# Utils
+require "warc/utils/header_hash"
+
+# Updated NET library (will be included in 2.0)
+require "warc/ext/net_http"
+
+# http tools
+require "warc/http"
+
+# Stream
+require "warc/stream"
+require "warc/stream/plain"
+require "warc/stream/gzip"
+
+# Record
+require "warc/record"
+require "warc/record/validator"
+require "warc/record/header"
+
+# Parser
+require "warc/parser"
+
+# Exception
+require "warc/exceptions"
+
+module Warc
+
+end

data/lib/warc/cli.rb

@@ -0,0 +1,23 @@
+require 'warc'
+require 'warc/proxy'
+require 'thor'
+
+module Warc
+  class CLI < Thor
+    desc "dump WARC_FILE", "Dump record headers from WARC_FILE"
+    method_option :offset, :default => 0, :type => :numeric
+    def dump(path)
+      w=Warc.open_stream(path)
+      puts "WARC filename\toffset\twarc-type\twarc-target-uri\twarc-record-id\tcontent-type\tcontent-length"
+      w.each(options[:offset]) do |record|
+        puts "#{path}\t#{record.offset}\t#{record.header['warc-type']}\t#{record.header['warc-target-uri']}\t#{record.header['warc-record-id']}\t#{record.header['content-type']}\t#{record.header.content_length}"
+      end
+    end
+
+    desc "replay WARC_FILE", "Start a HTTP proxy serving request from WARC_FILE. Dashboard available at http://warc/"
+    option :p, :default => 9292, :banner => "port"
+    def replay(warc)
+      Warc::Proxy::Replay.start(warc,options[:port])
+    end
+  end
+end

data/lib/warc/ext/net_http.rb

@@ -0,0 +1,50 @@
+require 'net/http'
+require 'net/https'
+require 'stringio'
+
+module Net
+  class BufferedIO
+    def initialize(io,debug_output = nil)
+      @read_timeout = 60
+      @rbuf = ''
+      @debug_output = debug_output
+
+      @io = case io
+      when Socket, OpenSSL::SSL::SSLSocket, StringIO, IO
+        io
+      when String
+        if !io.include?("\0") && File.exists?(io) && !File.directory?(io)
+          File.open(io, "r")
+        else
+          StringIO.new(io)
+        end
+      end
+      raise "Unable to create fake socket from #{io}" unless @io
+    end
+  end
+  
+  class HTTP
+    class << self
+      def socket_type_with_warc
+        FakeWeb::StubSocket
+      end
+      alias_method :socket_type, :socket_type_with_warc
+    end
+  end
+end
+
+module Warc
+  class StubSocket #:nodoc:
+
+    def initialize(*args)
+    end
+
+    def closed?
+      @closed ||= true
+    end
+
+    def readuntil(*args)
+    end
+
+  end
+end

data/lib/warc/http.rb

@@ -0,0 +1,35 @@
+module Warc
+  class HTTP
+    def self.get(uri)
+      url = URI(uri)
+      net_res = Net::HTTP.get_response(url)
+
+      record = ::Warc::Record.new
+      record.header["WARC-Type"] = "response"
+      record.header.date = net_res.to_hash["date"][0]
+      record.header["WARC-Target-URI"] = url.to_s
+      record.header["Content-Type"] = "application/http;msgtype=response"
+
+      headers = String.new
+      headers << "HTTP/#{net_res.http_version} #{net_res.code} #{net_res.message}\r\n"
+      net_res.to_hash.each {|key,value| headers << "#{key}: #{value[0].to_s}\r\n"}
+
+      record.content = "#{headers}\r\n#{net_res.body}"
+      return record,net_res
+    end
+
+    def self.archive(uri,stream)
+      stream = case stream
+      when ::Warc::Stream
+        stream
+      when String
+        ::Warc::Stream::Gzip.new(stream)
+      end
+
+      record,response = self.get(uri)
+      stream.write_record(record)
+      return response
+    end
+
+  end
+end

data/lib/warc/parser.rb

@@ -0,0 +1,24 @@
+module Warc
+  class Parser
+    def parse(stream)
+      # Find next item
+      loop do
+        offset = stream.tell
+        version_line = stream.readline
+        break if version_line.chomp("\r\n") == "WARC/1.0"
+      end
+
+      # Prepare to read headers
+      rec = Warc::Record.new
+
+      while m = /^(.*): (.*)/.match(stream.readline)
+        rec.header[m.captures[0]] = m.captures[1].chomp("\r")
+      end
+
+      rec.content = stream.read(rec.header.content_length)
+
+      #stream.seek(rec.header["content-length"].to_i,IO::SEEK_CUR)
+      return rec
+    end
+  end
+end

data/lib/warc/proxy.rb

@@ -0,0 +1 @@
+require 'warc/proxy/proxy'

data/lib/warc/proxy/css/main.css

@@ -0,0 +1,45 @@
+body {
+	font-family: Helvetica;
+}
+
+h1 {
+font-size: 40px;
+}
+
+h2 {
+	font-size:20px;
+	margin:10px;
+	margin-top:25px;
+}
+
+ul {
+	list-style: none;
+  font-size:12px;
+}
+
+#wrapper {
+width:600px;
+margin:0px auto;
+}
+
+#editor {
+
+}
+
+#header {
+  background-color: black;
+  color: white;
+  height: 300px;
+  border-radius:200px;
+  -moz-border-radius:200px;
+  -webkit-border-radius: 200px;
+  width: 300px;
+  padding:25px;
+  margin:0 auto;
+  text-align:center;
+}
+
+iframe {
+  width:100%;
+  height:400px;
+}

data/lib/warc/proxy/proxy.rb

@@ -0,0 +1,85 @@
+require 'rack'
+require 'sinatra/base'
+
+module Warc
+  module Proxy
+    class Replay < Sinatra::Base
+      set :public_dir, File.expand_path('..', __FILE__) # set up the static dir (with images/js/css inside)
+      set :views,  File.expand_path('../views', __FILE__) # set up the views dir
+
+      disable :protection
+
+      before do
+        headers["Access-ConDrol-Allow-Origin"] = "*"
+      end
+
+      get "/" do
+        @size = @index.size
+        erb :index
+      end
+
+      get "/record/*" do
+        id = params[:splat].join('/')
+        record = @index.record
+        @@session[request.ip] = Time.parse(record.header["WARC-Date"])
+        redirect to(record.header["WARC-Target-URI"])
+      end
+
+      def initialize(app=nil,warc=nil)
+        super(app)
+        @warc = ::Warc.open_stream(warc)
+        @index = {}
+        puts "Building index"
+        @warc.each do |record|
+          if record.header["warc-type"] == "response"
+            @index[record.header.uri] = record.offset
+          end
+        end
+        puts "Indexing done"
+      end
+
+      def call(env)
+        # Send to Sinatra app
+        if env["HTTP_HOST"] == "warc"
+          super(env)
+          # Or serve from archive
+        else
+          serve(env)
+        end
+      end
+
+      def serve(env)
+        uri = "http://#{env['HTTP_HOST']}#{env['REQUEST_URI']}"
+        if @index.key?(uri)
+          record = @warc.record(@index[uri])
+          return http_response(record)
+        else
+          return [404,{"Content-Type" => "text/html"},["not found"]]
+        end
+      end
+
+      def http_response(record)
+        io = StringIO.new(record.content)
+        headers = {}
+        /^HTTP\/(\d\.\d) (\d++) (.*)/.match(io.readline)
+        code = $2
+
+        while m = /^(.*): (.*)/.match(io.readline)
+          headers[m.captures[0]] = m.captures[1].chomp("\r")
+        end
+        [code,headers,io]
+      end
+
+      def self.start(warc,port)
+        # Run the app!
+        app = Rack::Builder.new {
+          run Warc::Proxy::Replay.new(nil,warc)
+        }
+        puts "Starting proxy server on port #{port}"
+        ::Rack::Server.start(:app => app,:Port => port,:server=>:thin)
+      end
+    end
+
+
+  end
+end

data/lib/warc/proxy/views/index.erb

@@ -0,0 +1,16 @@
+<div id="wrapper">
+<div id="header">
+<h1>WARC</h1>
+<h2>Web ARChive</h2>
+<p>Currently serving <b><%= @size %> records</b></p>
+</div>
+
+<div>
+	<h2>Records:</h2>
+	<ul>
+		<% @index.collect do |uri,offset| %>
+		<li><a href="<%=uri%>"><%=uri%></a></li>
+		<% end %>
+	</ul>
+</div>
+</div>

data/lib/warc/proxy/views/layout.erb

@@ -0,0 +1,9 @@
+<!DOCTYPE html>
+<html>
+<head>    
+	<link href="/css/main.css" rel="stylesheet" type="text/css">
+</head>
+<body>
+	<%= yield %>
+</body>
+</html>

data/lib/warc/record.rb

@@ -0,0 +1,59 @@
+module Warc
+  class Record
+    VERSION = "WARC/1.0"
+
+    attr_accessor :content, :offset
+    attr_reader :header
+    def initialize(h={},content=nil)
+      @content=content
+      case h
+      when Hash
+        @header = Header.new(self,h)
+      when WEBrick::HTTPResponse
+        @header = Header.new(self)
+        @header["WARC-Type"] = "response"
+        @header["WARC-Target-URI"] = h.request_uri.to_s
+        @header["Content-Type"] = "application/http;msgtype=response"
+        #@header["WARC-IP-Address"]
+        body,crfl = String.new,"\r\n"
+        body  << h.status_line
+        h.header.each do |k,v|
+          body << "#{k}: #{v}" + crfl
+        end
+        body  << crfl + h.body
+        self.content = body
+        self.header.block_digest
+        @header["WARC-Payload-Digest"] = self.header.compute_digest(h.body)
+      end
+    end
+    
+    def to_http
+      if @header["Content-Type"] == "application/http;msgtype=response"
+        url = @header["WARC-Target-URI"]
+        socket = Net::BufferedIO.new(content)
+        r=Net::HTTPResponse.read_new(socket)
+        r.reading_body(socket,true) {}
+        return r
+      end
+    end
+
+    def dump_to(out)
+      #
+      #      warc-file    = 1*warc-record
+      #      warc-record  = header CRLF
+      #                     block CRLF CRLF
+      #      header       = version CRLF
+      #                     warc-fields
+      #      version      = "WARC/0.16" CRLF
+      #      warc-fields  = *named-field CRLF
+      #      block        = *OCTET
+      #
+      crfl = "\r\n"
+
+      out.write(VERSION + crfl)
+      out.write(self.header.to_s)
+      out.write(crfl)
+      out.write(self.content + crfl*2)
+    end
+  end
+end