fizx-csvget 0.1.0 → 0.2.0

data/README.rdoc

@@ -25,19 +25,51 @@
       }]
     }
     > csvget --directory-prefix=./data  -A "/x" -w 5 --parselet=hn.let http://news.ycombinator.com/
-		> head data/headlines.csv
-		comments,title,time,link,score,user
-		4,Simpson's paradox: why mistrust seemingly simple statistics,2 hours ago,http://en.wikipedia.org/wiki/Simpson%27s_paradox,41,waldrews
-		67,America's unjust sex laws,2 hours ago,http://www.economist.com/opinion/displaystory.cfm?story_id=14165460,59,MikeCapone
-		23,Buy somebody lunch,3 hours ago,http://www.whattofix.com/blog/archives/2009/08/buy-somebody-lu.php,58,DanielBMarkham
-		10,A design pattern is an artifact of a missing feature in your chosen language,3 hours ago,http://www.snell-pym.org.uk/archives/2008/12/29/design-patterns/,31,bensummers
-		4,API changes in Snow Leopard,1 hour ago,http://developer.apple.com/mac/library/releasenotes/MacOSX/WhatsNewInOSX/Articles/MacOSX10_6.html#//apple_ref/doc/uid/TP40008898-SW1,14,pieter
-		16,How to run a linux based home web server,3 hours ago,http://stevehanov.ca/blog/index.php?id=73,28,RiderOfGiraffes
-		1,"OpenCL ""Hello World""",1 hour ago,"http://developer.apple.com/mac/library/documentation/Performance/Conceptual/OpenCL_MacProgGuide/Example:Hello,World/Example:Hello,World.html",8,pieter
-		15,US Senate bill allows White House to disconnect private computers from Internet,4 hours ago,http://news.cnet.com/8301-13578_3-10320096-38.html,35,drewr
-		1,Strategy: Solve Only 80 Percent of the Problem,47 minutes ago,http://highscalability.com/strategy-solve-only-80-percent-problem,6,alrex021
-		
-
+    > head data/headlines.csv
+    comments,title,time,link,score,user
+    4,Simpson's paradox: why mistrust seemingly simple statistics,2 hours ago,http://en.wikipedia.org/wiki/Simpson%27s_paradox,41,waldrews
+    67,America's unjust sex laws,2 hours ago,http://www.economist.com/opinion/displaystory.cfm?story_id=14165460,59,MikeCapone
+    23,Buy somebody lunch,3 hours ago,http://www.whattofix.com/blog/archives/2009/08/buy-somebody-lu.php,58,DanielBMarkham
+    10,A design pattern is an artifact of a missing feature in your chosen language,3 hours ago,http://www.snell-pym.org.uk/archives/2008/12/29/design-patterns/,31,bensummers
+    4,API changes in Snow Leopard,1 hour ago,http://developer.apple.com/mac/library/releasenotes/MacOSX/WhatsNewInOSX/Articles/MacOSX10_6.html#//apple_ref/doc/uid/TP40008898-SW1,14,pieter
+    16,How to run a linux based home web server,3 hours ago,http://stevehanov.ca/blog/index.php?id=73,28,RiderOfGiraffes
+    1,"OpenCL ""Hello World""",1 hour ago,"http://developer.apple.com/mac/library/documentation/Performance/Conceptual/OpenCL_MacProgGuide/Example:Hello,World/Example:Hello,World.html",8,pieter
+    15,US Senate bill allows White House to disconnect private computers from Internet,4 hours ago,http://news.cnet.com/8301-13578_3-10320096-38.html,35,drewr
+    1,Strategy: Solve Only 80 Percent of the Problem,47 minutes ago,http://highscalability.com/strategy-solve-only-80-percent-problem,6,alrex021
+    > csvget -h
+    Usage: ./bin/csvget [options] SEED_URL [SEED_URL2 ...]
+            --parselet=JSON_FILE         JSON_FILE is a parselet.
+        -w, --wait=SECONDS               wait SECONDS between retrievals.
+        -P, --directory-prefix=PREFIX    save files to PREFIX/...
+        -U, --user-agent=AGENT           identify as AGENT instead of RWget/VERSION.
+        -A, --accept-pattern=RUBY_REGEX  URLs must match RUBY_REGEX to be saved to the queue.
+            --time-limit=AMOUNT          Crawler will stop after this AMOUNT of time has passed.
+        -R, --reject-pattern=RUBY_REGEX  URLs must NOT match RUBY_REGEX to be saved to the queue.
+            --require=RUBY_SCRIPT        Will execute 'require RUBY_SCRIPT'
+            --limit-rate=RATE            limit download rate to RATE.
+            --http-proxy=URL             Proxies via URL
+            --proxy-user=USER            Sets proxy user to USER
+            --proxy-password=PASSWORD    Sets proxy password to PASSWORD
+            --fetch-class=RUBY_CLASS     Must implement fetch(uri, user_agent_string) #=> [final_redirected_url, file_object]
+            --store-class=RUBY_CLASS     Must implement put(key_string, temp_file)
+            --dupes-class=RUBY_CLASS     Must implement dupe?(uri)
+            --queue-class=RUBY_CLASS     Must implement put(key_string, depth_int) and get() #=> [key_string, depth_int]
+            --links-class=RUBY_CLASS     Must implement urls(base_uri, temp_file) #=> [uri, ...]
+        -S, --sitemap=URL                URL of a sitemap to crawl (will ignore inter-page links)
+        -V, --version
+        -Q, --quota=NUMBER               set retrieval quota to NUMBER.
+            --max-redirect=NUM           maximum redirections allowed per page.
+        -H, --span-hosts                 go to foreign hosts when recursive
+            --connect-timeout=SECS       set the connect timeout to SECS.
+        -T, --timeout=SECS               set all timeout values to SECONDS.
+        -l, --level=NUMBER               maximum recursion depth (inf or 0 for infinite).
+            --[no-]timestampize          Prepend the timestamp of when the crawl started to the directory structure.
+            --incremental-from=PREVIOUS  Build upon the indexing already saved in PREVIOUS.
+            --protocol-directories       use protocol name in directories.
+            --no-host-directories        don't create host directories.
+        -v, --[no-]verbose               Run verbosely
+        -h, --help                       Show this message
+    
 == Copyright
 
 Copyright (c) 2009 Kyle Maxwell. See LICENSE for details (MIT).

data/Rakefile

@@ -5,6 +5,7 @@ begin
   require 'jeweler'
   Jeweler::Tasks.new do |gem|
     gem.name = "csvget"
+    gem.executables = ["csvget", "jsonget"]
     gem.summary = %Q{Uses parselets and rwget to generate csv files from websites}
     gem.description = %Q{Super easy to use (but lots of dependencies :/) parser}
     gem.email = "kyle@kylemaxwell.com"

data/VERSION

@@ -1 +1 @@
-0.1.0
+0.2.0

data/bin/jsonget

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+require "rubygems"
+require File.dirname(__FILE__) + "/../lib/jsonget"
+require "rwget"
+
+parser = RWGetOptionParser.new do |opts|
+  opts.on("--parselet=JSON_FILE", "JSON_FILE is a parselet.") do |path|
+    parser.options[:parselets] ||= []
+    parser.options[:parselets] << path
+  end
+end
+parser.parse!
+
+if parser.options[:seeds].empty?
+  puts parser.usage
+  puts "    -h for options listing"
+  exit(1) 
+end
+
+parser.options[:store_class] ||= "JSONStore"
+
+controller = RWGet::Controller.new(parser.options)
+begin
+  controller.start
+ensure
+  STDERR.puts "Closing..."
+  controller.close
+end

data/csvget.gemspec

@@ -0,0 +1,63 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE
+# Instead, edit Jeweler::Tasks in Rakefile, and run `rake gemspec`
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{csvget}
+  s.version = "0.2.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Kyle Maxwell"]
+  s.date = %q{2009-08-28}
+  s.description = %q{Super easy to use (but lots of dependencies :/) parser}
+  s.email = %q{kyle@kylemaxwell.com}
+  s.executables = ["csvget", "jsonget"]
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README.rdoc"
+  ]
+  s.files = [
+    ".gitignore",
+     "LICENSE",
+     "README.rdoc",
+     "Rakefile",
+     "VERSION",
+     "bin/csvget",
+     "bin/jsonget",
+     "csvget.gemspec",
+     "lib/csvget.rb",
+     "lib/jsonget.rb",
+     "test/csvget_test.rb"
+  ]
+  s.homepage = %q{http://github.com/fizx/csvget}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.5}
+  s.summary = %q{Uses parselets and rwget to generate csv files from websites}
+  s.test_files = [
+    "test/csvget_test.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<fizx-rwget>, ["> 0.2.3"])
+      s.add_runtime_dependency(%q<fizx-parsley-ruby>, ["> 0.0.0"])
+      s.add_runtime_dependency(%q<activesupport>, ["> 0.0.0"])
+      s.add_runtime_dependency(%q<fastercsv>, [">= 1.4.0"])
+    else
+      s.add_dependency(%q<fizx-rwget>, ["> 0.2.3"])
+      s.add_dependency(%q<fizx-parsley-ruby>, ["> 0.0.0"])
+      s.add_dependency(%q<activesupport>, ["> 0.0.0"])
+      s.add_dependency(%q<fastercsv>, [">= 1.4.0"])
+    end
+  else
+    s.add_dependency(%q<fizx-rwget>, ["> 0.2.3"])
+    s.add_dependency(%q<fizx-parsley-ruby>, ["> 0.0.0"])
+    s.add_dependency(%q<activesupport>, ["> 0.0.0"])
+    s.add_dependency(%q<fastercsv>, [">= 1.4.0"])
+  end
+end

data/lib/jsonget.rb

@@ -0,0 +1,28 @@
+require "rubygems"
+require "rwget"
+require "parsley"
+require "activesupport"
+require "fileutils"
+
+class JSONStore
+  def initialize(options = {})
+    @output_folder = options[:prefix] || "."
+    FileUtils.mkdir_p(@output_folder)
+    @parselets = (options[:parselets] || []).map{|path| Parsley.new(File.read(path)) }
+    @files     = (options[:parselets] || []).map{|path| File.open("#{File.basename(path)}.json", "a") }
+  end
+
+  def put(host, tmpfile)
+    @parselets.zip(@files).each do |parselet, file|
+      begin
+        file.puts parselet.parse(:file => tmpfile.path, :output => :json) + ","
+      rescue ParsleyError => e
+        STDERR.puts "warning: #{e.message}"
+      end
+    end
+  end
+
+  def close
+    @files.map(&:close)
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: fizx-csvget
 version: !ruby/object:Gem::Version 
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors: 
 - Kyle Maxwell
@@ -10,7 +10,7 @@ bindir: bin
 cert_chain: []
 
 date: 2009-08-28 00:00:00 -07:00
-default_executable: csvget
+default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: fizx-rwget
@@ -56,6 +56,7 @@ description: Super easy to use (but lots of dependencies :/) parser
 email: kyle@kylemaxwell.com
 executables: 
 - csvget
+- jsonget
 extensions: []
 
 extra_rdoc_files: 
@@ -68,7 +69,10 @@ files:
 - Rakefile
 - VERSION
 - bin/csvget
+- bin/jsonget
+- csvget.gemspec
 - lib/csvget.rb
+- lib/jsonget.rb
 - test/csvget_test.rb
 has_rdoc: false
 homepage: http://github.com/fizx/csvget