fizx-csvget 0.1.0

data/.gitignore

@@ -0,0 +1,2 @@
+*.gem
+*.csv

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Kyle Maxwell
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,43 @@
+= csvget
+
+== Dependencies
+
+- http://github.com/fizx/parsley/tree/master and its dependencies.
+- Rubygems
+
+== Installation
+
+1. Install the dependencies.  
+2. > gem sources -a http://gems.github.com 
+3. > sudo gem install fizx-csvget
+
+== Example Usage
+
+    > cat hn.let
+    {
+      "headlines":[{
+        "title": ".title a",
+        "link": ".title a @href",
+        "comments": "match(.subtext a:nth-child(3), '\\d+')",
+        "user": ".subtext a:nth-child(2)",
+        "score": "match(.subtext span, '\\d+')",
+        "time": "match(.subtext, '\\d+\\s+\\w+\\s+ago')"
+      }]
+    }
+    > csvget --directory-prefix=./data  -A "/x" -w 5 --parselet=hn.let http://news.ycombinator.com/
+		> head data/headlines.csv
+		comments,title,time,link,score,user
+		4,Simpson's paradox: why mistrust seemingly simple statistics,2 hours ago,http://en.wikipedia.org/wiki/Simpson%27s_paradox,41,waldrews
+		67,America's unjust sex laws,2 hours ago,http://www.economist.com/opinion/displaystory.cfm?story_id=14165460,59,MikeCapone
+		23,Buy somebody lunch,3 hours ago,http://www.whattofix.com/blog/archives/2009/08/buy-somebody-lu.php,58,DanielBMarkham
+		10,A design pattern is an artifact of a missing feature in your chosen language,3 hours ago,http://www.snell-pym.org.uk/archives/2008/12/29/design-patterns/,31,bensummers
+		4,API changes in Snow Leopard,1 hour ago,http://developer.apple.com/mac/library/releasenotes/MacOSX/WhatsNewInOSX/Articles/MacOSX10_6.html#//apple_ref/doc/uid/TP40008898-SW1,14,pieter
+		16,How to run a linux based home web server,3 hours ago,http://stevehanov.ca/blog/index.php?id=73,28,RiderOfGiraffes
+		1,"OpenCL ""Hello World""",1 hour ago,"http://developer.apple.com/mac/library/documentation/Performance/Conceptual/OpenCL_MacProgGuide/Example:Hello,World/Example:Hello,World.html",8,pieter
+		15,US Senate bill allows White House to disconnect private computers from Internet,4 hours ago,http://news.cnet.com/8301-13578_3-10320096-38.html,35,drewr
+		1,Strategy: Solve Only 80 Percent of the Problem,47 minutes ago,http://highscalability.com/strategy-solve-only-80-percent-problem,6,alrex021
+		
+
+== Copyright
+
+Copyright (c) 2009 Kyle Maxwell. See LICENSE for details (MIT).

data/Rakefile

@@ -0,0 +1,60 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "csvget"
+    gem.summary = %Q{Uses parselets and rwget to generate csv files from websites}
+    gem.description = %Q{Super easy to use (but lots of dependencies :/) parser}
+    gem.email = "kyle@kylemaxwell.com"
+    gem.homepage = "http://github.com/fizx/csvget"
+    gem.authors = ["Kyle Maxwell"]
+    gem.add_dependency("fizx-rwget", ["> 0.2.3"])
+    gem.add_dependency("fizx-parsley-ruby", ["> 0.0.0"])
+    gem.add_dependency("activesupport", ["> 0.0.0"])
+    gem.add_dependency("fastercsv", [">= 1.4.0"])
+    
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
+end
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/*_test.rb'
+  test.verbose = true
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.pattern = 'test/**/*_test.rb'
+    test.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+  end
+end
+
+task :test => :check_dependencies
+
+task :default => :test
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  if File.exist?('VERSION')
+    version = File.read('VERSION')
+  else
+    version = ""
+  end
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "csvget #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/bin/csvget

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+require "rubygems"
+require File.dirname(__FILE__) + "/../lib/csvget"
+require "rwget"
+
+parser = RWGetOptionParser.new do |opts|
+  opts.on("--parselet=JSON_FILE", "JSON_FILE is a parselet.") do |path|
+    parser.options[:parselets] ||= []
+    parser.options[:parselets] << path
+  end
+end
+parser.parse!
+
+if parser.options[:seeds].empty?
+  puts parser.usage
+  puts "    -h for options listing"
+  exit(1) 
+end
+
+parser.options[:store_class] ||= "CSVStore"
+
+controller = RWGet::Controller.new(parser.options)
+begin
+  controller.start
+ensure
+  STDERR.puts "Closing..."
+  controller.close
+end

data/lib/csvget.rb

@@ -0,0 +1,48 @@
+require "rubygems"
+require "rwget"
+require "parsley"
+require "fastercsv"
+require "activesupport"
+require "fileutils"
+
+class CSVStore
+  def initialize(options = {})
+    @output_folder = options[:prefix] || "."
+    FileUtils.mkdir_p(@output_folder)
+    @parselets = (options[:parselets] || []).map{|path| Parsley.new(File.read(path)) }
+    @files = {}
+    @headers = {}
+  end
+
+  def put(host, tmpfile)
+    @parselets.each do |parselet|
+      begin
+        output = parselet.parse(:file => tmpfile.path)
+        walk(output)
+      rescue ParsleyError => e
+        STDERR.puts "warning: #{e.message}"
+      end
+    end
+  end
+  
+  def walk(data, prefix = nil)
+    data.each do |prefix, values|
+      values = [values] unless values.is_a?(Array)
+      file_name = File.join(@output_folder, "#{prefix}.csv")
+      h = @headers[prefix] ||= values.first.keys
+      f = @files[prefix] ||= FasterCSV.open(file_name, "a", :headers => h, :write_headers => true)
+      
+      values.each do |v|
+        f << h.inject([]) do |memo, key|
+          memo << v[key]
+        end
+      end
+    end
+  end
+
+  def close
+    @files.each do |k, v|
+      v.close
+    end
+  end
+end

data/test/csvget_test.rb

@@ -0,0 +1,20 @@
+require "test/unit"
+require File.dirname(__FILE__) + "/../lib/csvget"
+require "fileutils"
+
+class CsvgetTest < Test::Unit::TestCase
+  include FileUtils
+  
+  # def setup
+  #   @output = {"bill-state"=>"Welcome to Google Business Solutions", "links"=>["/", "https://adwords.google.com/select/Login?sourceid=awo&subid=us-en-et-bizsol-0-biz1-all&medium=link&hl=en_US"]}
+  #   @links = ParseletLinks.new(:parselets => File.dirname(__FILE__) + "/foo.let")
+  # end
+  # 
+  # def test_bill_state
+  #   bills = File.dirname(__FILE__) + "/bills.csv"
+  #   @links.walk @output
+  #   assert_equal File.read(File.dirname(__FILE__) + "/expected.csv"), File.read(bills)
+  #   rm bills
+  # end
+end
+

metadata

@@ -0,0 +1,101 @@
+--- !ruby/object:Gem::Specification 
+name: fizx-csvget
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+platform: ruby
+authors: 
+- Kyle Maxwell
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-08-28 00:00:00 -07:00
+default_executable: csvget
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: fizx-rwget
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">"
+      - !ruby/object:Gem::Version 
+        version: 0.2.3
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: fizx-parsley-ruby
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">"
+      - !ruby/object:Gem::Version 
+        version: 0.0.0
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: activesupport
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">"
+      - !ruby/object:Gem::Version 
+        version: 0.0.0
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: fastercsv
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.4.0
+    version: 
+description: Super easy to use (but lots of dependencies :/) parser
+email: kyle@kylemaxwell.com
+executables: 
+- csvget
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.rdoc
+files: 
+- .gitignore
+- LICENSE
+- README.rdoc
+- Rakefile
+- VERSION
+- bin/csvget
+- lib/csvget.rb
+- test/csvget_test.rb
+has_rdoc: false
+homepage: http://github.com/fizx/csvget
+licenses: 
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: Uses parselets and rwget to generate csv files from websites
+test_files: 
+- test/csvget_test.rb