fizx-skivvies 0.1.0

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -0,0 +1,5 @@
+*.sw?
+.DS_Store
+coverage
+rdoc
+pkg

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Kyle Maxwell
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,7 @@
+= skivvies
+
+Description goes here.
+
+== Copyright
+
+Copyright (c) 2009 Kyle Maxwell. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,51 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "skivvies"
+    gem.summary = %Q{TODO}
+    gem.email = "kyle@kylemaxwell.com"
+    gem.homepage = "http://github.com/fizx/skivvies"
+    gem.authors = ["Kyle Maxwell"]
+    
+    gem.add_dependency("fizx-parsley-ruby", ["= 0.4.0"])
+    gem.add_dependency("fizx-rwget", ["= 0.4.0"])
+    gem.add_dependency("fastercsv", [">= 0.5.0"])
+  end
+
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
+end
+
+require 'spec/rake/spectask'
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+Spec::Rake::SpecTask.new(:rcov) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+
+task :default => :spec
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  if File.exist?('VERSION.yml')
+    config = YAML.load(File.read('VERSION.yml'))
+    version = "#{config[:major]}.#{config[:minor]}.#{config[:patch]}"
+  else
+    version = ""
+  end
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "skivvies #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/bin/skivvies

@@ -0,0 +1,30 @@
+#!/usr/bin/env ruby
+require File.dirname(__FILE__) + "/../lib/skivvies"
+
+parser = RWGetOptionParser.new do |opts|
+  opts.on("--parselet=JSON_FILE", "JSON_FILE must implement a parsley script") do |path|
+    parser.options[:parselets] ||= []
+    parser.options[:parselets] << path
+    parser.options[:store_class] ||= :ParseletCSVStore
+  end
+  
+  opts.on("--store_path=PATH", "PATH will contain csv files of the crawl") do |path|
+    parser.options[:store_path] = path
+  end
+end
+
+parser.parse!
+
+if parser.options[:seeds].empty?
+  puts parser.usage
+  puts "    -h for options listing"
+  exit(1) 
+end
+
+controller = RWGet::Controller.new(parser.options)
+begin
+  controller.start
+ensure
+  STDERR.puts "Closing..."
+  controller.close
+end

data/lib/parselet_csv_store.rb

@@ -0,0 +1,84 @@
+require "rubygems"
+require "parsley"
+require "faster_csv"
+require "fileutils"
+
+class ParseletCSVStore
+  include FileUtils
+  
+  SUFFIX = "skivvies.csv"
+  def initialize(options = {})
+    @options = options
+    @handles = {}
+    @path = options[:store_path] || "."
+    mkdir_p @path
+    @keys = Dir[File.join(@path, "*.#{SUFFIX}")].inject({}) do |memo, path|
+      max = 0
+      File.open(path, "r").each do |line|
+        id = line.to_i
+        max = id if id > max
+      end
+      memo[path] = max
+      memo
+    end
+    @parselets = (options[:parselets] || []).map do |path|
+      Parsley.new(File.read(path), options)
+    end
+    raise "no parselets provided" if @parselets.empty?
+  end
+  
+  def put(key, tmpfile)
+    path = tmpfile.path
+    data = @parselets.map do |parselet|
+      parselet.parse(options.merge(:file => path))
+    end
+    visit(data, "root")
+  rescue => e
+    STDERR.puts e.message
+    STDERR.puts e.backtrace.join("\n")
+  end
+
+  def close
+    @handles.values.map{|h| h.close}
+    @handles = {}
+  end
+  
+private
+
+  def visit(data, key, base = nil, base_id = nil)
+    if data.is_a?(Array)
+      data.each do |entry|
+        visit entry, key
+      end
+      return
+    end
+    
+    raise "WTF: #{data.inspect}" unless data.is_a?(Hash)
+    
+    row = []
+  
+    data.each do |k, v|
+      if v.is_a?(String)
+        row << v
+      elsif v.is_a?(Array) && v.first.is_a?(String)
+        row << v.join("|")
+      else
+        visit(v, k)
+      end
+    end
+    
+    unless row.empty?
+      row.unshift(key)
+      row.unshift(@keys[key] += 1)
+      handle(key) << row
+    end
+  end
+  
+  def handle(key)
+    @handles[key] ||= FasterCSV.open(File.join(@path, "#{key}.#{SUFFIX}"), "a")
+  end
+  
+  def max(key)
+    @keys[key] || 0
+  end
+end

data/lib/skivvies.rb

@@ -0,0 +1,3 @@
+require "rubygems"
+require "rwget"
+require File.dirname(__FILE__) + "/parselet_csv_store"

data/spec/parselet_csv_store_spec.rb

@@ -0,0 +1,5 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe Type do
+  
+end

data/spec/skivvies_spec.rb

@@ -0,0 +1,7 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe "Skivvies" do
+  it "fails" do
+    fail "hey buddy, you should probably rename this file and start specing for real"
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,9 @@
+require 'spec'
+
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'skivvies'
+
+Spec::Runner.configure do |config|
+  
+end

metadata

@@ -0,0 +1,96 @@
+--- !ruby/object:Gem::Specification 
+name: fizx-skivvies
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+platform: ruby
+authors: 
+- Kyle Maxwell
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-06-17 00:00:00 -07:00
+default_executable: skivvies
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: fizx-parsley-ruby
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - "="
+      - !ruby/object:Gem::Version 
+        version: 0.4.0
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: fizx-rwget
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - "="
+      - !ruby/object:Gem::Version 
+        version: 0.4.0
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: fastercsv
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.5.0
+    version: 
+description: 
+email: kyle@kylemaxwell.com
+executables: 
+- skivvies
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.rdoc
+files: 
+- .document
+- .gitignore
+- LICENSE
+- README.rdoc
+- Rakefile
+- VERSION
+- bin/skivvies
+- lib/parselet_csv_store.rb
+- lib/skivvies.rb
+- spec/parselet_csv_store_spec.rb
+- spec/skivvies_spec.rb
+- spec/spec_helper.rb
+has_rdoc: true
+homepage: http://github.com/fizx/skivvies
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: TODO
+test_files: 
+- spec/parselet_csv_store_spec.rb
+- spec/skivvies_spec.rb
+- spec/spec_helper.rb