web_dump 0.0.1.0

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Marcel Massana
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,59 @@
+= web_dump
+
+Little tiny class to easily save and retrieve web pages
+
+In web related client applications, such as spiders, it is frequently necessary 
+to save pages into files with adecuate naming convention. WebDump comes to the
+rescue. It manages the details of assigning unique readable names and save files
+after URIs that have been visited. Additionally, saving data could also be
+conveniently compressed with gzip for deep web spidering. It only depends on
+telling the correct file extension when saving.
+
+Conversely, file read operation is available through convenient methods
+indicating either a pathname or a URI.
+
+== Installation
+
+  $ sudo gem install web_dump
+
+The main source repository is http://github.com/syborg/web_dump.
+
+== Usage
+
+First of all ...
+
+  require 'rubygems'
+  require 'web_dump'
+
+Instantiate an object. You may add some options that can be passed through an
+array
+
+  wd = WebDump,new :base_dir => '~/mydir', :file_ext => '.gz'
+
+`wd`, when asked to, will save all files inside expanded directory '~/mydir'
+with an appended file extension at the end '.gz' (if not overwriten later)
+
+Other options could be passed when instantiating an object.
+
+* `:file_ext => extension` (String that will be appended at the end to every filename if not changed from _save_ method)
+
+Most of them are also passed along to an UriPathname object that is created.
+
+* `:base_dir => dir_name` (directory where everything will be stored. Defaults to '~/web_dumps')
+* `:pth_sep => psep` (String that will be used to substitute '/' inside URI's path and queries (defaults to UriPathname::PTH_SEP='_|_'))
+* `:host_sep => hsep` (String that will be used separate the URI¡s hostname and path when constructing the pathname. if '/' is used, hostname will actually become a subdirectory -defaults to UriPathname::HOST_SEP='__|'-)
+* `:no_path => nopath` (String that will be used as a path placeholder when no URI's path exists, -default UriPathname::NO_PTH = '_NOPATH_'-)
+
+== Note on Patches/Pull Requests
+ 
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Commit, do not mess with rakefile, version, or history.
+  (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for topic branches.
+
+== Copyright
+
+Copyright (c) 2011 Marcel Massana. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,58 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  require './lib/web_dump/version'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "web_dump"
+    gem.summary = %Q{Saves and Retrieves data in files given an URI}
+    gem.description = %Q{Saves and Retrieves data given an URI. The filename
+      will be automatically choosed using that URI freeing the user to think
+      about that}.gsub(/\s+/,' ')
+    gem.email = "xaxaupua@gmail.com"
+    gem.homepage = "http://github.com/syborg/web_dump"
+    gem.authors = ["Marcel Massana"]
+    gem.add_dependency "uri_pathname", ">= 0"
+    # gem.add_development_dependency "thoughtbot-shoulda", ">= 0"
+    gem.version = WebDump::Version::STRING
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.pattern = 'test/**/test_*.rb'
+    test.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+  end
+end
+
+task :test => :check_dependencies
+
+task :default => :test
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "web_dump #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/lib/web_dump/version.rb

@@ -0,0 +1,16 @@
+# it's only a version number
+
+class WebDump
+
+  module Version
+
+    MAJOR = 0
+    MINOR = 0
+    PATCH = 1
+    BUILD = 0
+	
+    STRING = [MAJOR, MINOR, PATCH, BUILD].compact.join(".")
+
+  end
+
+end

data/lib/web_dump.rb

@@ -0,0 +1,104 @@
+# WebDump
+# MME 31/8/2011
+#
+# Allows saving and reading data related to URIs (i.e. pages)
+
+require 'web_dump/version'
+require 'uri'
+require 'zlib'
+require 'fileutils'
+require 'rubygems'
+require 'uri_pathname'
+
+# Allows saving and reading data related to URIs (i.e. pages)
+class WebDump
+
+  #default attributes
+  DEFAULT_ATTRS = {
+      :base_dir => '~/web_dumps',
+      :file_ext => '.html'
+    }
+
+  attr_accessor :up, *(DEFAULT_ATTRS.keys)
+
+  # initializes a WebDump object. +options+ should be a hash with options for
+  # an UriPathname object that will be internally created. Default UriPathnames
+  # options and additionally:
+  #    :base_dir => directory where everything will be stored (def. '~/web_dumps')
+  #    :file_ext => extension that will be appended to filenames (def. '.html')
+  def initialize(options = {})
+
+    attributes = DEFAULT_ATTRS.merge options if options.is_a? Hash
+    attributes.each { |k,v| instance_eval("@#{k}='#{v}'") if DEFAULT_ATTRS.keys.include?(k) }
+
+    @up=UriPathname.new attributes  # any valid option passed will be delivered
+
+  end
+
+  # saves the +content+ (String) into a file named after
+  # UriPathname#uri_to_pathname(+uri+).
+  # If +extension+ is nil initialize :file_ext option will be used:
+  #   'anything'+'.gz' ->  gzipped (less storage requirements)
+  #   other ->  as is
+  # returns a String containing the complete pathname of the file if OK else nil
+  def save(uri, content, extension = nil)
+    extension = @file_ext unless extension
+    pathname = @up.uri_to_pathname(uri,nil,extension)
+    return nil unless pathname
+    mkdir_if_not_exists(File.dirname(pathname))
+    num_bytes = nil
+    case extension
+    when /\.gz$/  # ...gz
+      File.open(pathname, 'w') do |f|
+        gz = Zlib::GzipWriter.new(f)
+        # gz.comment="#dumped with web_dump #{Version::STRING}: #{uri}" # no sembla fer res
+        num_bytes = gz.write content
+        gz.close
+      end
+    else # any other
+      File.open(pathname, 'w') do |f|
+        num_bytes = f.write(content)
+      end
+    end
+    num_bytes ? pathname : nil
+  end
+
+  # returns the stored content corresponding to file +pathname+. In case there
+  # isn't any file it returns nil.
+  def read_pathname(pathname)
+    content = nil
+    arr = @up.parse pathname
+    complete_pathname = File.expand_path(pathname)
+    extension = arr[2]
+    case extension
+    when /.gz/
+      File.open(complete_pathname, 'r') do |f|
+        gz = Zlib::GzipReader.new(f)
+        content = gz.read
+        gz.close
+      end
+    else # others as is
+      File.open(complete_pathname, 'r') do |f|
+        content = f.read
+      end
+    end
+    content
+  end
+
+  # returns the stored content corresponding to +uri+ URI. In case there
+  # isn't any file it returns nil.
+  def read_uri(uri, filext=nil)
+    filext = @file_ext unless filext
+    pathname = @up.uri_to_pathname(uri,nil,filext)
+    read_pathname(pathname)
+  end
+
+  private
+
+  # creates +directory+ if it doesn't exist
+  def mkdir_if_not_exists(directory)
+    dir = File.expand_path(directory)
+    FileUtils.mkdir_p(dir) unless (File.exist?(dir) and File.directory?(dir))
+  end
+
+end

data/test/test_web_dump.rb

@@ -0,0 +1,47 @@
+require 'test/unit'
+require 'web_dump'
+require 'fileutils'
+
+class TC_WebDump < Test::Unit::TestCase
+
+  TEST_DIR = '~/tmp/web_dump'
+
+  # called before every test
+  def setup
+    FileUtils.remove_dir(File.expand_path(TEST_DIR), true)
+    @wd = WebDump.new :base_dir => TEST_DIR
+  end
+
+  # called after every test
+  def teardown
+    FileUtils.remove_dir(File.expand_path(TEST_DIR), true)
+  end
+
+  def test_automatic_dir_and_file_creation
+    wd = WebDump.new :base_dir => TEST_DIR, :host_sep => '/'
+    pathname = wd.save 'http://www.fake.fak/fakpath', 'Hello World!'
+    assert(File.exist?(pathname))
+  end
+
+  def test_raw_file_sr_cycle
+    input = 'Hello World!'
+    uri = 'http://www.com/prova'
+    pathname = @wd.save(uri,input)
+    output = @wd.read_pathname(pathname)
+    assert_equal input, output, "retrieved through pathname"
+    output = @wd.read_uri(uri)
+    assert_equal input, output, "retrieved through uri"
+  end
+
+  def test_gzipped_file_sr_cycle
+    input = 'Hello World!'
+    uri = 'http://www.com/prova'
+    pathname = @wd.save(uri,input,".gz")
+    output = @wd.read_pathname(pathname)
+    assert_equal input, output, "retrieved through pathname"
+    output = @wd.read_uri(uri,"gz")
+    assert_equal input, output, "retrieved through uri"
+  end
+
+
+end

data/web_dump.gemspec

@@ -0,0 +1,50 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = "web_dump"
+  s.version = "0.0.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Marcel Massana"]
+  s.date = "2011-08-31"
+  s.description = "Saves and Retrieves data given an URI. The filename will be automatically choosed using that URI freeing the user to think about that"
+  s.email = "xaxaupua@gmail.com"
+  s.extra_rdoc_files = [
+    "LICENSE",
+    "README.rdoc"
+  ]
+  s.files = [
+    ".document",
+    ".goutputstream-6QBL0V",
+    ".goutputstream-6X1P0V",
+    ".goutputstream-IR2O0V",
+    ".goutputstream-TK420V",
+    "LICENSE",
+    "README.rdoc",
+    "Rakefile",
+    "lib/web_dump.rb",
+    "lib/web_dump/version.rb",
+    "test/test_web_dump.rb",
+    "web_dump.gemspec"
+  ]
+  s.homepage = "http://github.com/syborg/web_dump"
+  s.require_paths = ["lib"]
+  s.rubygems_version = "1.8.10"
+  s.summary = "Saves and Retrieves data in files given an URI"
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<uri_pathname>, [">= 0"])
+    else
+      s.add_dependency(%q<uri_pathname>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<uri_pathname>, [">= 0"])
+  end
+end
+

metadata

@@ -0,0 +1,87 @@
+--- !ruby/object:Gem::Specification 
+name: web_dump
+version: !ruby/object:Gem::Version 
+  hash: 75
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 1
+  - 0
+  version: 0.0.1.0
+platform: ruby
+authors: 
+- Marcel Massana
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-08-31 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: uri_pathname
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+description: Saves and Retrieves data given an URI. The filename will be automatically choosed using that URI freeing the user to think about that
+email: xaxaupua@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.rdoc
+files: 
+- .document
+- LICENSE
+- README.rdoc
+- Rakefile
+- lib/web_dump.rb
+- lib/web_dump/version.rb
+- test/test_web_dump.rb
+- web_dump.gemspec
+homepage: http://github.com/syborg/web_dump
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: Saves and Retrieves data in files given an URI
+test_files: []
+