web_dump 0.0.1.0 → 0.0.2.0

data/README.rdoc

@@ -20,30 +20,87 @@ The main source repository is http://github.com/syborg/web_dump.
 
 == Usage
 
+=== Intantiating
+
 First of all ...
 
   require 'rubygems'
   require 'web_dump'
 
-Instantiate an object. You may add some options that can be passed through an
-array
+Instantiating an object, you may add some options that can be passed through a Hash
 
-  wd = WebDump,new :base_dir => '~/mydir', :file_ext => '.gz'
+  wd = WebDump.new :base_dir => '~/mydir', :file_ext => '.gz'
 
 `wd`, when asked to, will save all files inside expanded directory '~/mydir'
-with an appended file extension at the end '.gz' (if not overwriten later)
+with an appended file extension '.gz' at the end (if not overwritten later)
 
-Other options could be passed when instantiating an object.
+Some options that could be passed when instantiating an object. Most of them are directly passed along to an UriPathname object that is created.
 
 * `:file_ext => extension` (String that will be appended at the end to every filename if not changed from _save_ method)
-
-Most of them are also passed along to an UriPathname object that is created.
-
 * `:base_dir => dir_name` (directory where everything will be stored. Defaults to '~/web_dumps')
 * `:pth_sep => psep` (String that will be used to substitute '/' inside URI's path and queries (defaults to UriPathname::PTH_SEP='_|_'))
 * `:host_sep => hsep` (String that will be used separate the URI¡s hostname and path when constructing the pathname. if '/' is used, hostname will actually become a subdirectory -defaults to UriPathname::HOST_SEP='__|'-)
 * `:no_path => nopath` (String that will be used as a path placeholder when no URI's path exists, -default UriPathname::NO_PTH = '_NOPATH_'-)
 
+=== Saving Web Contents
+
+You should use WebDump#save, for example:
+
+  wd.save "http://hello.world.com/hithere", data
+
+=== Retrieving Web Contents
+
+You can retrieve data using two flavoured read methods, using URIs or using
+pathnames as main argument
+
+  data = wd.read_uri(uri)
+
+or
+
+  data = wd.read_pathname(f)
+
+== Example
+
+Here is a complete example
+
+    require 'rubygems'
+    require 'open-uri'
+    require 'web_dump'
+
+    MY_URIS = [
+      'http://en.wikipedia.org/wiki/Ruby_Bridges',
+      'http://donaldfagen.com/disc_nightfly.php',
+      'http://www.rubi.cat/ajrubi/portada/index.php',
+      'http://www.google.com/cse?q=array&cx=013598269713424429640%3Ag5orptiw95w&ie=UTF-8&sa=Search'
+    ]
+
+    # all files will be saved in expanded '~/mydir' with file extension '.gz'
+    wd = WebDump.new :base_dir => '~/mydir', :file_ext => '.gz'
+
+    # Don't care about filenames while saving pages into files
+    puts "Saving data using URIs"
+    MY_URIS.each do |uri|
+      open uri do |u|
+        data = u.read
+        puts wd.save uri, data
+      end
+    end
+
+    # Possibly mocking? ... don't care about filenames while retrieving pages from files.
+    puts "\nRetrieving data using URIs"
+    MY_URIS.each do |uri|
+      data = wd.read_uri(uri)
+      puts data[0...100].gsub(/\s+/, ' ').strip
+    end
+
+    # ... or, conversely, use filenames if you need so
+    puts "\nRetrieving data using pathnames"
+    files = Dir[File.expand_path('*.gz', '~/mydir')]
+    files.each do |f|
+      data = wd.read_pathname(f)
+      puts data[0...100].gsub(/\s+/, ' ').strip
+    end
+
 == Note on Patches/Pull Requests
  
 * Fork the project.

data/examples/simple_examples.rb

@@ -0,0 +1,40 @@
+# Simple Examples to show up how to use WebDump
+# Marcel Massana 1-Sep-2011
+
+require 'rubygems'
+require 'open-uri'
+require 'web_dump'
+
+MY_URIS = [
+  'http://en.wikipedia.org/wiki/Ruby_Bridges',
+  'http://donaldfagen.com/disc_nightfly.php',
+  'http://www.rubi.cat/ajrubi/portada/index.php',
+  'http://www.google.com/cse?q=array&cx=013598269713424429640%3Ag5orptiw95w&ie=UTF-8&sa=Search'
+]
+
+# all files will be saved in expanded '~/mydir' with file extension '.gz'
+wd = WebDump.new :base_dir => '~/mydir', :file_ext => '.gz'
+
+# Don't care about filenames while saving pages into files
+puts "Saving data using URIs"
+MY_URIS.each do |uri|
+  open uri do |u|
+    data = u.read
+    puts wd.save uri, data
+  end
+end
+
+# Possibly mocking?... also don't care about filenames while retrieving pages from files.
+puts "\nRetrieving data using URIs"
+MY_URIS.each do |uri|
+  data = wd.read_uri(uri)
+  puts data[0...100].gsub(/\s+/, ' ').strip
+end
+
+# ... or, conversely, use filenames if you need so
+puts "\nRetrieving data using pathnames"
+files = Dir[File.expand_path('*.gz', '~/mydir')]
+files.each do |f|
+  data = wd.read_pathname(f)
+  puts data[0...100].gsub(/\s+/, ' ').strip
+end

data/lib/web_dump/version.rb

@@ -6,7 +6,7 @@ class WebDump
 
     MAJOR = 0
     MINOR = 0
-    PATCH = 1
+    PATCH = 2
     BUILD = 0
 	
     STRING = [MAJOR, MINOR, PATCH, BUILD].compact.join(".")

data/web_dump.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "web_dump"
-  s.version = "0.0.1.0"
+  s.version = "0.0.2.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Marcel Massana"]
-  s.date = "2011-08-31"
+  s.date = "2011-09-01"
   s.description = "Saves and Retrieves data given an URI. The filename will be automatically choosed using that URI freeing the user to think about that"
   s.email = "xaxaupua@gmail.com"
   s.extra_rdoc_files = [
@@ -18,13 +18,10 @@ Gem::Specification.new do |s|
   ]
   s.files = [
     ".document",
-    ".goutputstream-6QBL0V",
-    ".goutputstream-6X1P0V",
-    ".goutputstream-IR2O0V",
-    ".goutputstream-TK420V",
     "LICENSE",
     "README.rdoc",
     "Rakefile",
+    "examples/simple_examples.rb",
     "lib/web_dump.rb",
     "lib/web_dump/version.rb",
     "test/test_web_dump.rb",

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification 
 name: web_dump
 version: !ruby/object:Gem::Version 
-  hash: 75
+  hash: 71
   prerelease: 
   segments: 
   - 0
   - 0
-  - 1
+  - 2
   - 0
-  version: 0.0.1.0
+  version: 0.0.2.0
 platform: ruby
 authors: 
 - Marcel Massana
@@ -16,7 +16,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-08-31 00:00:00 Z
+date: 2011-09-01 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: uri_pathname
@@ -46,6 +46,7 @@ files:
 - LICENSE
 - README.rdoc
 - Rakefile
+- examples/simple_examples.rb
 - lib/web_dump.rb
 - lib/web_dump/version.rb
 - test/test_web_dump.rb