csvget 0.0.0 → 0.4.0

data/.gitignore

@@ -0,0 +1,2 @@
+*.gem
+*.csv

data/CHANGELOG

@@ -0,0 +1,8 @@
+0.3.2 - Sept 10, 2009
+  -- Don't write headers when appending to an existing CSV file.
+0.3.0 - Sept 2, 2009
+  -- Added command-line option:
+
+      --filter=RUBY_CODE RUBY_CODE will be eval'd in context of @row.is_a?(FasterCSV::Row)
+
+  -- Added CHANGELOG

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Kyle Maxwell
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,93 @@
+= csvget (also, jsonget)
+
+== What's new in 0.3.0
+
+1. Added command-line option:
+
+    --filter=RUBY_CODE RUBY_CODE will be eval'd in context of @row.is_a?(FasterCSV::Row)
+
+    # Example:
+    ./bin/csvget --require=chronic --require=time --filter "@row['time']=Chronic.parse(@row['time']).iso8601" # ...
+
+2. Added CHANGELOG
+
+== Dependencies
+
+- http://github.com/fizx/parsley/tree/master and its dependencies.
+- Rubygems
+
+== Running on EC2
+
+    > git clone git://github.com/fizx/csvget-ec2-recipe.git
+    > cd csvget-ec2-recipe
+    > ./boot.rb
+    > ssh YOUR_INSTANCE
+
+== Local Installation
+
+1. Install the dependencies.  
+2. > gem sources -a http://gems.github.com 
+3. > sudo gem install fizx-csvget
+
+== Example Usage
+
+    > cat hn.let
+    {
+      "headlines":[{
+        "title": ".title a",
+        "link": ".title a @href",
+        "comments": "match(.subtext a:nth-child(3), '\\d+')",
+        "user": ".subtext a:nth-child(2)",
+        "score": "match(.subtext span, '\\d+')",
+        "time": "match(.subtext, '\\d+\\s+\\w+\\s+ago')"
+      }]
+    }
+    > csvget --directory-prefix=./data  -A "/x" -w 5 --parselet=hn.let http://news.ycombinator.com/
+    > head data/headlines.csv
+    comments,title,time,link,score,user
+    4,Simpson's paradox: why mistrust seemingly simple statistics,2 hours ago,http://en.wikipedia.org/wiki/Simpson%27s_paradox,41,waldrews
+    67,America's unjust sex laws,2 hours ago,http://www.economist.com/opinion/displaystory.cfm?story_id=14165460,59,MikeCapone
+    23,Buy somebody lunch,3 hours ago,http://www.whattofix.com/blog/archives/2009/08/buy-somebody-lu.php,58,DanielBMarkham
+    10,A design pattern is an artifact of a missing feature in your chosen language,3 hours ago,http://www.snell-pym.org.uk/archives/2008/12/29/design-patterns/,31,bensummers
+    4,API changes in Snow Leopard,1 hour ago,http://developer.apple.com/mac/library/releasenotes/MacOSX/WhatsNewInOSX/Articles/MacOSX10_6.html#//apple_ref/doc/uid/TP40008898-SW1,14,pieter
+    16,How to run a linux based home web server,3 hours ago,http://stevehanov.ca/blog/index.php?id=73,28,RiderOfGiraffes
+    1,"OpenCL ""Hello World""",1 hour ago,"http://developer.apple.com/mac/library/documentation/Performance/Conceptual/OpenCL_MacProgGuide/Example:Hello,World/Example:Hello,World.html",8,pieter
+    15,US Senate bill allows White House to disconnect private computers from Internet,4 hours ago,http://news.cnet.com/8301-13578_3-10320096-38.html,35,drewr
+    1,Strategy: Solve Only 80 Percent of the Problem,47 minutes ago,http://highscalability.com/strategy-solve-only-80-percent-problem,6,alrex021
+    > csvget -h
+    Usage: ./bin/csvget [options] SEED_URL [SEED_URL2 ...]
+            --parselet=JSON_FILE         JSON_FILE is a parselet.
+        -w, --wait=SECONDS               wait SECONDS between retrievals.
+        -P, --directory-prefix=PREFIX    save files to PREFIX/...
+        -U, --user-agent=AGENT           identify as AGENT instead of RWget/VERSION.
+        -A, --accept-pattern=RUBY_REGEX  URLs must match RUBY_REGEX to be saved to the queue.
+            --time-limit=AMOUNT          Crawler will stop after this AMOUNT of time has passed.
+        -R, --reject-pattern=RUBY_REGEX  URLs must NOT match RUBY_REGEX to be saved to the queue.
+            --require=RUBY_SCRIPT        Will execute 'require RUBY_SCRIPT'
+            --limit-rate=RATE            limit download rate to RATE.
+            --http-proxy=URL             Proxies via URL
+            --proxy-user=USER            Sets proxy user to USER
+            --proxy-password=PASSWORD    Sets proxy password to PASSWORD
+            --fetch-class=RUBY_CLASS     Must implement fetch(uri, user_agent_string) #=> [final_redirected_url, file_object]
+            --store-class=RUBY_CLASS     Must implement put(key_string, temp_file)
+            --dupes-class=RUBY_CLASS     Must implement dupe?(uri)
+            --queue-class=RUBY_CLASS     Must implement put(key_string, depth_int) and get() #=> [key_string, depth_int]
+            --links-class=RUBY_CLASS     Must implement urls(base_uri, temp_file) #=> [uri, ...]
+        -S, --sitemap=URL                URL of a sitemap to crawl (will ignore inter-page links)
+        -V, --version
+        -Q, --quota=NUMBER               set retrieval quota to NUMBER.
+            --max-redirect=NUM           maximum redirections allowed per page.
+        -H, --span-hosts                 go to foreign hosts when recursive
+            --connect-timeout=SECS       set the connect timeout to SECS.
+        -T, --timeout=SECS               set all timeout values to SECONDS.
+        -l, --level=NUMBER               maximum recursion depth (inf or 0 for infinite).
+            --[no-]timestampize          Prepend the timestamp of when the crawl started to the directory structure.
+            --incremental-from=PREVIOUS  Build upon the indexing already saved in PREVIOUS.
+            --protocol-directories       use protocol name in directories.
+            --no-host-directories        don't create host directories.
+        -v, --[no-]verbose               Run verbosely
+        -h, --help                       Show this message
+    
+== Copyright
+
+Copyright (c) 2009 Kyle Maxwell. See LICENSE for details (MIT).

data/Rakefile

@@ -0,0 +1,62 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "csvget"
+    gem.executables = ["csvget", "jsonget"]
+    gem.summary = %Q{Uses parselets and rwget to generate csv files from websites}
+    gem.description = %Q{Super easy to use (but lots of dependencies :/) parser}
+    gem.email = "kyle@kylemaxwell.com"
+    gem.homepage = "http://github.com/fizx/csvget"
+    gem.authors = ["Kyle Maxwell"]
+    gem.add_dependency("fizx-rwget", ["> 0.2.3"])
+    gem.add_dependency("fizx-parsley-ruby", ["> 0.0.0"])
+    gem.add_dependency("activesupport", ["> 0.0.0"])
+    gem.add_dependency("fastercsv", [">= 1.4.0"])
+    
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
+end
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/*_test.rb'
+  test.verbose = true
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.pattern = 'test/**/*_test.rb'
+    test.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+  end
+end
+
+task :test => :check_dependencies
+
+task :default => :test
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  if File.exist?('VERSION')
+    version = File.read('VERSION')
+  else
+    version = ""
+  end
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "csvget #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.4.0

data/bin/csvget

@@ -0,0 +1,34 @@
+#!/usr/bin/env ruby
+require "rubygems"
+require File.dirname(__FILE__) + "/../lib/csvget"
+require "rwget"
+
+parser = RWGetOptionParser.new do |opts|
+  opts.on("--parselet=JSON_FILE", "JSON_FILE is a parselet.") do |path|
+    parser.options[:parselets] ||= []
+    parser.options[:parselets] << path
+  end
+  
+  opts.on("--filter=RUBY_CODE", "RUBY_CODE will be eval'd in context of @row.is_a?(FasterCSV::Row)") do |filter|
+    parser.options[:filter] ||= []
+    parser.options[:filter] << filter
+  end
+  
+end
+parser.parse!
+
+if parser.options[:seeds].empty?
+  puts parser.usage
+  puts "    -h for options listing"
+  exit(1) 
+end
+
+parser.options[:store_class] ||= "CSVStore"
+
+controller = RWGet::Controller.new(parser.options)
+begin
+  controller.start
+ensure
+  STDERR.puts "Closing..."
+  controller.close
+end

data/bin/jsonget

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+require "rubygems"
+require File.dirname(__FILE__) + "/../lib/jsonget"
+require "rwget"
+
+parser = RWGetOptionParser.new do |opts|
+  opts.on("--parselet=JSON_FILE", "JSON_FILE is a parselet.") do |path|
+    parser.options[:parselets] ||= []
+    parser.options[:parselets] << path
+  end
+end
+parser.parse!
+
+if parser.options[:seeds].empty?
+  puts parser.usage
+  puts "    -h for options listing"
+  exit(1) 
+end
+
+parser.options[:store_class] ||= "JSONStore"
+
+controller = RWGet::Controller.new(parser.options)
+begin
+  controller.start
+ensure
+  STDERR.puts "Closing..."
+  controller.close
+end

data/csvget.gemspec

@@ -0,0 +1,65 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE
+# Instead, edit Jeweler::Tasks in Rakefile, and run `rake gemspec`
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{csvget}
+  s.version = "0.4.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Kyle Maxwell"]
+  s.date = %q{2009-10-16}
+  s.description = %q{Super easy to use (but lots of dependencies :/) parser}
+  s.email = %q{kyle@kylemaxwell.com}
+  s.executables = ["csvget", "jsonget"]
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README.rdoc"
+  ]
+  s.files = [
+    ".gitignore",
+     "CHANGELOG",
+     "LICENSE",
+     "README.rdoc",
+     "Rakefile",
+     "VERSION",
+     "bin/csvget",
+     "bin/jsonget",
+     "csvget.gemspec",
+     "hn.let",
+     "lib/csvget.rb",
+     "lib/jsonget.rb",
+     "test/csvget_test.rb"
+  ]
+  s.homepage = %q{http://github.com/fizx/csvget}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.4}
+  s.summary = %q{Uses parselets and rwget to generate csv files from websites}
+  s.test_files = [
+    "test/csvget_test.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<fizx-rwget>, ["> 0.2.3"])
+      s.add_runtime_dependency(%q<fizx-parsley-ruby>, ["> 0.0.0"])
+      s.add_runtime_dependency(%q<activesupport>, ["> 0.0.0"])
+      s.add_runtime_dependency(%q<fastercsv>, [">= 1.4.0"])
+    else
+      s.add_dependency(%q<fizx-rwget>, ["> 0.2.3"])
+      s.add_dependency(%q<fizx-parsley-ruby>, ["> 0.0.0"])
+      s.add_dependency(%q<activesupport>, ["> 0.0.0"])
+      s.add_dependency(%q<fastercsv>, [">= 1.4.0"])
+    end
+  else
+    s.add_dependency(%q<fizx-rwget>, ["> 0.2.3"])
+    s.add_dependency(%q<fizx-parsley-ruby>, ["> 0.0.0"])
+    s.add_dependency(%q<activesupport>, ["> 0.0.0"])
+    s.add_dependency(%q<fastercsv>, [">= 1.4.0"])
+  end
+end

data/hn.let

@@ -0,0 +1,10 @@
+{
+   "headlines":[{
+     "title": ".title a",
+     "link": ".title a @href",
+     "comments": "match(.subtext a:nth-child(3), '\\d+')",
+     "user": ".subtext a:nth-child(2)",
+     "score": "match(.subtext span, '\\d+')",
+     "time": "match(.subtext, '\\d+\\s+\\w+\\s+ago')"
+   }]
+ }

data/lib/csvget.rb

@@ -0,0 +1,54 @@
+require "rubygems"
+require "rwget"
+require "parsley"
+require "fastercsv"
+require "activesupport"
+require "fileutils"
+
+class CSVStore
+  def initialize(options = {})
+    @output_folder = options[:prefix] || "."
+    @filters = options[:filter] || []
+    FileUtils.mkdir_p(@output_folder)
+    @parselets = (options[:parselets] || []).map{|path| Parsley.new(File.read(path)) }
+    @files = {}
+    @headers = {}
+  end
+
+  def put(host, tmpfile)
+    @parselets.each do |parselet|
+      begin
+        type = (`file "#{tmpfile.path}"` =~ /xml/i) ? :xml : :html
+        output = parselet.parse(:file => tmpfile.path, :input => type)
+        walk(output)
+      rescue ParsleyError => e
+        STDERR.puts "warning: #{e.message}"
+      end
+    end
+  end
+  
+  def walk(data, prefix = nil)
+    data.each do |prefix, values|
+      values = [values] unless values.is_a?(Array)
+      file_name = File.join(@output_folder, "#{prefix}.csv")
+      h = @headers[prefix] ||= values.first.keys
+      should_write_headers = !File.exists?(file_name)
+      f = @files[prefix] ||= FasterCSV.open(file_name, "a", :headers => h, :write_headers => should_write_headers)
+      
+      values.each do |hash|
+        arr = h.inject([]) do |memo, key|
+          memo << hash[key]
+        end
+        @row = FasterCSV::Row.new(h, arr)
+        @filters.each {|filter| eval(filter) }
+        f << @row
+      end
+    end
+  end
+
+  def close
+    @files.each do |k, v|
+      v.close
+    end
+  end
+end

data/lib/jsonget.rb

@@ -0,0 +1,29 @@
+require "rubygems"
+require "rwget"
+require "parsley"
+require "activesupport"
+require "fileutils"
+
+class JSONStore
+  def initialize(options = {})
+    @output_folder = options[:prefix] || "."
+    FileUtils.mkdir_p(@output_folder)
+    @parselets = (options[:parselets] || []).map{|path| Parsley.new(File.read(path)) }
+    @files     = (options[:parselets] || []).map{|path| File.open("#{File.basename(path)}.json", "a") }
+  end
+
+  def put(host, tmpfile)
+    @parselets.zip(@files).each do |parselet, file|
+      begin
+        type = (`file "#{tmpfile.path}"` =~ /xml/i) ? :xml : :html
+        output = parselet.parse(:file => tmpfile.path, :input => type, :output => :json) + ","
+      rescue ParsleyError => e
+        STDERR.puts "warning: #{e.message}"
+      end
+    end
+  end
+
+  def close
+    @files.map(&:close)
+  end
+end

data/test/csvget_test.rb

@@ -0,0 +1,20 @@
+require "test/unit"
+require File.dirname(__FILE__) + "/../lib/csvget"
+require "fileutils"
+
+class CsvgetTest < Test::Unit::TestCase
+  include FileUtils
+  
+  # def setup
+  #   @output = {"bill-state"=>"Welcome to Google Business Solutions", "links"=>["/", "https://adwords.google.com/select/Login?sourceid=awo&subid=us-en-et-bizsol-0-biz1-all&medium=link&hl=en_US"]}
+  #   @links = ParseletLinks.new(:parselets => File.dirname(__FILE__) + "/foo.let")
+  # end
+  # 
+  # def test_bill_state
+  #   bills = File.dirname(__FILE__) + "/bills.csv"
+  #   @links.walk @output
+  #   assert_equal File.read(File.dirname(__FILE__) + "/expected.csv"), File.read(bills)
+  #   rm bills
+  # end
+end
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: csvget
 version: !ruby/object:Gem::Version 
-  version: 0.0.0
+  version: 0.4.0
 platform: ruby
 authors: 
 - Kyle Maxwell
@@ -9,27 +9,80 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-10-10 00:00:00 -07:00
+date: 2009-10-16 00:00:00 -07:00
 default_executable: 
-dependencies: []
-
-description: 
-email: 
-executables: []
-
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: fizx-rwget
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">"
+      - !ruby/object:Gem::Version 
+        version: 0.2.3
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: fizx-parsley-ruby
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">"
+      - !ruby/object:Gem::Version 
+        version: 0.0.0
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: activesupport
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">"
+      - !ruby/object:Gem::Version 
+        version: 0.0.0
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: fastercsv
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.4.0
+    version: 
+description: Super easy to use (but lots of dependencies :/) parser
+email: kyle@kylemaxwell.com
+executables: 
+- csvget
+- jsonget
 extensions: []
 
-extra_rdoc_files: []
-
-files: []
-
+extra_rdoc_files: 
+- LICENSE
+- README.rdoc
+files: 
+- .gitignore
+- CHANGELOG
+- LICENSE
+- README.rdoc
+- Rakefile
+- VERSION
+- bin/csvget
+- bin/jsonget
+- csvget.gemspec
+- hn.let
+- lib/csvget.rb
+- lib/jsonget.rb
+- test/csvget_test.rb
 has_rdoc: true
-homepage: 
+homepage: http://github.com/fizx/csvget
 licenses: []
 
 post_install_message: 
-rdoc_options: []
-
+rdoc_options: 
+- --charset=UTF-8
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
@@ -50,6 +103,6 @@ rubyforge_project:
 rubygems_version: 1.3.4
 signing_key: 
 specification_version: 3
-summary: Placeholder for a gem to be migrated later
-test_files: []
-
+summary: Uses parselets and rwget to generate csv files from websites
+test_files: 
+- test/csvget_test.rb