warc 1.0.0

data/spec/spec_helper.rb

@@ -0,0 +1,27 @@
+require 'warc'
+
+def fixture(path)
+  File.expand_path(File.join(File.dirname(__FILE__),'fixtures',path))
+end
+
+def read_fixture(path)
+  File.read(fixture(path))
+end
+
+SAMPLES = {
+  "http://www.imdb.com/" => "http_imdb"
+}
+
+unless ENV['LIVE_TEST']
+  begin
+    require 'fakeweb'
+
+    FakeWeb.allow_net_connect = false
+    SAMPLES.each do |url,path|
+      FakeWeb.register_uri(:get,url,:response => read_fixture(path))
+    end
+  rescue LoadError
+    puts "Could not load FakeWeb, these tests will hit IMDB.com"
+    puts "You can run `gem install fakeweb` to stub out the responses."
+  end
+end

data/spec/warc/http_spec.rb

@@ -0,0 +1,9 @@
+require 'spec_helper.rb'
+
+describe Warc::HTTP do
+  it "should be able to return a HTTPResponse object from a warc record" do
+    record = ::Warc.open_stream(fixture('criterion.warc.gz')).to_a[2]
+    record.to_http.class.should eq (Net::HTTPOK)
+  end
+  
+end

data/spec/warc/record/header_spec.rb

@@ -0,0 +1,37 @@
+require 'spec_helper.rb'
+
+describe Warc::Record::Header do
+  context "when instansiated with some value" do
+    before(:each) do
+      @header=Warc::Record.new({
+        "WARC-Type"=> "response",
+        "WARC-Record-ID" => "<record-1>",
+        "WARC-Date" => "2000-01-02T03:04:05Z",
+        "Content-Length" => "10"
+      }).header
+    end
+    
+    subject {@header}
+
+    its(:type) { should eq "response"}
+    it "should have attributes for mandatory fields" do
+      @header.type.should eq "response"
+      @header.record_id.should eq "<record-1>"
+      @header.date.should eq "2000-01-02T03:04:05Z"
+      @header.content_length.should eq 10
+    end
+
+    it "should be case-insensitive to field names" do
+      ["WARC-Type", "warc-type", "WARC-TYPE"].each do |key|
+        @header[key].should eq "response"
+      end
+    end
+
+    context "when warc-content-length empty" do
+      before {@header.delete("content-length")}
+      it "should assume a content-lenght of 0" do
+        @header.content_length.should eq 0
+      end
+    end
+  end
+end

data/spec/warc/record_spec.rb

@@ -0,0 +1,20 @@
+require 'spec_helper.rb'
+
+describe Warc::Record do
+  before(:each) do
+    @rec = Warc::Record.new
+    @rec.content = "asdf"
+    @rec.header.replace({
+      "WARC-Type"=> "response",
+      "WARC-Date" => "2000-01-02T03:04:05Z"
+    })
+  end
+
+  it "should have a header" do
+    @rec.respond_to?(:header).should eq true
+  end
+  
+  it "should compute content-length" do
+    @rec.header.content_length.should eq(4)
+  end
+end

data/spec/warc/stream/gzip_spec.rb

@@ -0,0 +1,46 @@
+require 'spec_helper.rb'
+
+describe Warc::Stream::Gzip do
+  context "compressed file single entry" do
+    before(:each) do
+      @warc=Warc.open_stream(fixture('criterion.warc.gz'))
+      @record = @warc.first
+    end
+
+    it "should read the content" do
+      @record.content.length.should eq @record.header.content_length
+    end
+
+    it "should parse all headers" do
+      @record.header.length.should eq(7)
+    end
+
+    it "should return the headers as an hash" do
+      @record.header.should eq({
+        "WARC-Type" => "warcinfo",
+        "Content-Type" => "application/warc-fields",
+        "WARC-Date" => "2012-09-13T22:52:52Z",
+        "WARC-Record-ID" => "<urn:uuid:671787C3-3C00-4256-8C5C-386A4D8F7468>",
+        "WARC-Filename" => "criterion.warc.gz",
+        "WARC-Block-Digest" => "sha1:OX3R5RVY4LFQ6WIPTDCLTY3ABKWLXUBU",
+        "Content-Length" => "234"
+      })
+    end
+
+  end
+
+  context "compressed file mutliple entries" do
+    before(:each) do
+      @warc=Warc.open_stream(fixture('frg.warc.gz'))
+    end
+
+    it "should find all record" do
+      @warc.count.should eq 56
+    end
+  end
+  
+  context "writing record" do
+    before(:each) do
+    end
+  end
+end

data/spec/warc/stream/plain_spec.rb

@@ -0,0 +1,41 @@
+require 'spec_helper.rb'
+
+describe Warc::Stream::Plain do
+  context "single entry of an uncompressed file" do
+    before(:each) do
+      @warc=Warc.open_stream(fixture('criterion.warc'))
+      @record = @warc.first
+    end
+
+    it "should parse all headers" do
+      @record.header.length.should eq(7)
+    end
+
+    it "should return the headers as an hash" do
+      @record.header.should eq({
+        "WARC-Type" => "warcinfo",
+        "Content-Type" => "application/warc-fields",
+        "WARC-Date" => "2012-09-13T22:53:20Z",
+        "WARC-Record-ID" => "<urn:uuid:CF5083F4-FEB1-4C63-B3CB-B450AB609875>",
+        "WARC-Filename" => "criterion.warc",
+        "WARC-Block-Digest" => "sha1:XGSXIX3L7RQGXFU6XJ32NCHXCKN6BBMK",
+        "Content-Length" => "258"
+      })
+    end
+
+    it "should read the content" do
+      @record.content.length.should eq @record.header.content_length
+    end
+  end
+
+  context "multiples entries" do
+    before(:each) do
+      @warc=Warc.open_stream(fixture('frg.warc'))
+    end
+    
+    it "should find all record" do
+      @warc.count.should eq 56
+    end
+
+  end
+end

data/spec/warc/stream_spec.rb

@@ -0,0 +1,55 @@
+require 'spec_helper.rb'
+require 'fileutils'
+
+describe Warc::Stream do
+  before(:each) do
+    @rec = Warc::Record.new
+    @rec.content = "asdf asdf"
+    @rec.header.replace({
+      "WARC-Type"=> "response",
+      "WARC-Date" => "2000-01-02T03:04:05Z",
+    })
+    header = @rec.header
+  end
+  
+  it "can't be initialized" do
+    #s = Warc::Stream.new
+  end
+  
+  it "should save to multiple files" do
+    s=Warc::Stream::Plain.new('/tmp/test',:max_filesize => 10*10**6)
+    100.times do
+      r = Warc::Record.new
+      r.content = "0" * (10**6)
+      r.header.replace({"WARC-Type"=> "response","WARC-Date" => "2000-01-02T03:04:05Z"})  
+      s.write_record(r)
+    end
+    ::File.exists?('/tmp/test.000010.warc').should eq(true)
+    FileUtils.rm Dir.glob('/tmp/test.*.warc')
+  end
+  
+  it "should dump record to file" do
+    s = Warc::Stream::Plain.new('/tmp/test.plain')
+    s.write_record(@rec)
+    s.close
+  end
+  
+  it "should dump gzipped to file" do
+    s = Warc::Stream::Gzip.new('/tmp/test.gzip')
+    s.write_record(@rec)
+    s.close    
+  end
+  
+  it "should find record" do
+    stream = ::Warc.open_stream(fixture('arg.warc'))
+    uri = "http://antoineroygobeil.com/"
+    record = stream.detect do |rec|
+      rec.header["warc-target-uri"] == uri && rec.header["warc-type"] == "response"
+    end
+  end
+  
+  it "should read record at given offset" do
+    stream = ::Warc.open_stream(fixture('arg.warc'))
+    stream.record(8287).header.record_id.should eq("<urn:uuid:5D799C11-D46C-4AC8-B598-5DC9F4205C6E>")
+  end
+end

data/warc.gemspec

@@ -0,0 +1,27 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/warc/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["antoine"]
+  gem.email         = ["roygobeil.antoine@gmail.com"]
+  #gem.description   = %q{TODO: Write a gem description}
+  gem.summary       = %q{warc is a pure ruby implementation of Web ARChive file reader and writer}
+  gem.homepage      = ""
+
+  gem.license = "MIT"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "warc"
+  gem.require_paths = ["lib"]
+  gem.version       = Warc::VERSION
+  
+  gem.add_dependency("uuid")
+  gem.add_dependency("activemodel")
+  gem.add_dependency("rack")
+  gem.add_dependency("thor")
+  gem.add_dependency("rack-contrib")
+  gem.add_dependency("sinatra")
+  gem.add_dependency("thin")
+end

metadata

@@ -0,0 +1,195 @@
+--- !ruby/object:Gem::Specification
+name: warc
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- antoine
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-10-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: uuid
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activemodel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rack-contrib
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: thin
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- roygobeil.antoine@gmail.com
+executables:
+- warc
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/warc
+- lib/warc.rb
+- lib/warc/cli.rb
+- lib/warc/exceptions.rb
+- lib/warc/ext/net_http.rb
+- lib/warc/http.rb
+- lib/warc/parser.rb
+- lib/warc/proxy.rb
+- lib/warc/proxy/css/main.css
+- lib/warc/proxy/proxy.rb
+- lib/warc/proxy/views/index.erb
+- lib/warc/proxy/views/layout.erb
+- lib/warc/record.rb
+- lib/warc/record/header.rb
+- lib/warc/record/validator.rb
+- lib/warc/stream.rb
+- lib/warc/stream/gzip.rb
+- lib/warc/stream/plain.rb
+- lib/warc/utils/header_hash.rb
+- lib/warc/version.rb
+- spec/fixtures/arg.warc
+- spec/fixtures/criterion.warc
+- spec/fixtures/criterion.warc.gz
+- spec/fixtures/frg.warc
+- spec/fixtures/frg.warc.gz
+- spec/fixtures/http_imdb
+- spec/spec_helper.rb
+- spec/warc/http_spec.rb
+- spec/warc/record/header_spec.rb
+- spec/warc/record_spec.rb
+- spec/warc/stream/gzip_spec.rb
+- spec/warc/stream/plain_spec.rb
+- spec/warc/stream_spec.rb
+- warc.gemspec
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.1.9
+signing_key: 
+specification_version: 4
+summary: warc is a pure ruby implementation of Web ARChive file reader and writer
+test_files:
+- spec/fixtures/arg.warc
+- spec/fixtures/criterion.warc
+- spec/fixtures/criterion.warc.gz
+- spec/fixtures/frg.warc
+- spec/fixtures/frg.warc.gz
+- spec/fixtures/http_imdb
+- spec/spec_helper.rb
+- spec/warc/http_spec.rb
+- spec/warc/record/header_spec.rb
+- spec/warc/record_spec.rb
+- spec/warc/stream/gzip_spec.rb
+- spec/warc/stream/plain_spec.rb
+- spec/warc/stream_spec.rb