monkeyshines 0.0.2 → 0.2.0

data/LICENSE.textile

@@ -5,7 +5,7 @@ title:  MIT License
 
 h1(gemheader). {{ site.gemname }} %(small):: license%
 
-<div class="toggle">
+<notextile><div class="toggle"></notextile>
 
 h2. MIT License
 
@@ -17,4 +17,4 @@ The above copyright notice and this permission notice shall be included in all c
 
 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-</div>
+<notextile></div></notextile>

data/README.textile

@@ -2,15 +2,7 @@ Monkeyshines is a tool for doing an algorithmic scrape.
 
 It's designed to handle large-scale scrapes that may exceed the capabilities of single-machine relational databases, so it plays nicely with Hadoop / Wukong, with distributed databases (MongoDB, tokyocabinet, etc.), and distributed job queue (eg "edamame/beanstalk":http://mrflip.github.com/edamame).
 
----------------------------------------------------------------------------
-
-h2. Overview
-
-A monkeyshines scraper is simple in principle:
-
-
-
-h2. Dependencies
+h2. Install
 
 This is best run standalone -- not as a gem; it's still in heavy development. I recommend cloning
 
@@ -27,22 +19,18 @@ Additionally, you'll need some of these gems:
 * extlib (0.9.12)
 * htmlentities (4.2.0)
 
-To build the gem, you'll need
-
-* git (1.2.2)
-* jeweler (1.2.1)
-* rake (0.8.7)
-* rspec (1.2.6)
-* rubyforge (1.0.4)
-* sources (0.0.1)
-
 And if you spell ruby with a 'j', you'll want
 
 * jruby-openssl (0.5.2)
 * json-jruby (1.1.7)
 
+---------------------------------------------------------------------------
+
+h2. Help!
 
+Send Monkeyshines questions to the "Infinite Monkeywrench mailing list":http://groups.google.com/group/infochimps-code
 
+---------------------------------------------------------------------------
 
 h2. Request Queue
 
@@ -53,8 +41,6 @@ Request stream can be metered using read-through, scheduled (eg cron), or test-a
 * Scheduled
 * Test and sleep. A queue of resources is cyclically polled, sleeping whenever bored.
 
-
-
 h2. Requests
 
 * Base: simple fetch and store of URI. (URI specifies immutable unique resource)

data/examples/README.textile

@@ -0,0 +1,6 @@
+
+h2. Where the hell?
+
+Most of these examples live in the "wuclan":http://mrflip.github.com/wuclan/ repository -- the directories are symlinks into that repo.
+
+Either install them from git source in adjoining directories, or just head into the wuclan examples/ directory and run stuff from there.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: monkeyshines
 version: !ruby/object:Gem::Version 
-  version: 0.0.2
+  version: 0.2.0
 platform: ruby
 authors: 
 - Philip (flip) Kromer
@@ -49,18 +49,12 @@ executables: []
 extensions: []
 
 extra_rdoc_files: 
-- LICENSE
 - LICENSE.textile
 - README.textile
 files: 
-- .document
-- .gitignore
-- LICENSE
 - LICENSE.textile
 - README.textile
-- Rakefile
-- VERSION
-- examples/.gitignore
+- examples/README.textile
 - examples/bulk_urls/scrape_bulk_urls.rb
 - examples/rename_tree/rename_hdp_tree.rb
 - examples/rename_tree/rename_ripd_tree.rb
@@ -133,8 +127,6 @@ files:
 - lib/monkeyshines/utils/union_interval.rb
 - lib/monkeyshines/utils/uri.rb
 - lib/monkeyshines/utils/uuid.rb
-- monkeyshines.gemspec
-- scrape_from_file.rb
 - spec/monkeyshines_spec.rb
 - spec/spec_helper.rb
 has_rdoc: true

data/.document

@@ -1,4 +0,0 @@
-README.textile
-lib/**/*.rb
-examples/**/*.rb
-LICENSE.textile

data/.gitignore

@@ -1,43 +0,0 @@
-\#*
-.\#*
-*~
-.DS_Store
-Icon?
-REVISION
-TAGS*
-nohup.out
-.bzr
-.hg
-.svn
-
-a.out
-*.o
-*.pyc
-*.so
-*.stackdump
-*.sw?
-*.tmproj
-*_flymake.*
-.project
-.pydevproject
-.settings
-.tasks-cache
-.yardoc
-
-/**/*DONTVERSION*
-/**/*private*
-/**/cache/*
-/**/log/*
-/**/tmp/*
-/coverage
-/doc/*
-/pkg/*
-/rdoc/*
-
-/db/*.sqlite3
-/db/sphinx
-/config/*.sphinx.conf
-/config/database.yml
-/config/sphinx.yml
-/public/stylesheets/compiled/*
-/vendor/src/**/*

data/LICENSE

@@ -1,20 +0,0 @@
-Copyright (c) 2009 Philip (flip) Kromer
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -1,105 +0,0 @@
-require 'rubygems'
-require 'rake'
-
-begin
-  require 'jeweler'
-  Jeweler::Tasks.new do |gem|
-    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
-    gem.name         = "monkeyshines"
-    gem.summary      = %Q{A simple scraper for directed scrapes of APIs, feed or structured HTML.}
-    gem.description  = %Q{A simple scraper for directed scrapes of APIs, feed or structured HTML. Plays nicely with wuclan and wukong.}
-    gem.email        = "flip@infochimps.org"
-    gem.homepage     = "http://github.com/mrflip/monkeyshines"
-    gem.authors      = ["Philip (flip) Kromer"]
-    gem.add_dependency 'addressable'
-    gem.add_dependency 'uuid'
-    gem.add_dependency 'wukong'
-  end
-  Jeweler::GemcutterTasks.new
-rescue LoadError
-  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
-end
-
-require 'spec/rake/spectask'
-Spec::Rake::SpecTask.new(:spec) do |spec|
-  spec.libs << 'lib' << 'spec'
-  spec.spec_files = FileList['spec/**/*_spec.rb']
-end
-
-Spec::Rake::SpecTask.new(:rcov) do |spec|
-  spec.libs << 'lib' << 'spec'
-  spec.pattern = 'spec/**/*_spec.rb'
-  spec.rcov = true
-end
-task :spec => :check_dependencies
-task :default => :spec
-
-begin
-  require 'reek/rake_task'
-  Reek::RakeTask.new do |t|
-    t.fail_on_error = true
-    t.verbose = false
-    t.source_files = ['lib/**/*.rb', 'examples/**/*.rb']
-  end
-rescue LoadError
-  task :reek do
-    abort "Reek is not available. In order to run reek, you must: sudo gem install reek"
-  end
-end
-
-begin
-  require 'roodi'
-  require 'roodi_task'
-  RoodiTask.new do |t|
-    t.verbose = false
-  end
-rescue LoadError
-  task :roodi do
-    abort "Roodi is not available. In order to run roodi, you must: sudo gem install roodi"
-  end
-end
-
-begin
-  require 'yard'
-  YARD::Rake::YardocTask.new do |yard|
-  end
-rescue LoadError
-  task :yardoc do
-    abort "YARD is not available. In order to run yardoc, you must: sudo gem install yard"
-  end
-end
-
-require 'rake/rdoctask'
-Rake::RDocTask.new do |rdoc|
-  require 'rdoc'
-  if File.exist?('VERSION')
-    version = File.read('VERSION')
-  else
-    version = ""
-  end
-
-  rdoc.options += [
-    '-SHN',
-    '-f', 'darkfish',  # use darkfish rdoc styler
-  ]
-  rdoc.rdoc_dir = 'rdoc'
-  rdoc.title = "edamame #{version}"
-  #
-  File.open(File.dirname(__FILE__)+'/.document').each{|line| rdoc.rdoc_files.include(line.chomp) }
-end
-
-require 'rake/rdoctask'
-Rake::RDocTask.new do |rdoc|
-  if File.exist?('VERSION.yml')
-    config = YAML.load(File.read('VERSION.yml'))
-    version = "#{config[:major]}.#{config[:minor]}.#{config[:patch]}"
-  else
-    version = ""
-  end
-
-  rdoc.rdoc_dir = 'rdoc'
-  rdoc.title = "monkeyshines #{version}"
-  rdoc.rdoc_files.include('README*')
-  rdoc.rdoc_files.include('lib/**/*.rb')
-end
-

data/VERSION

@@ -1 +0,0 @@
-0.0.2

data/examples/.gitignore

@@ -1,4 +0,0 @@
-facebook_stream
-friendster_html
-friendster_api
-myspace_api

data/monkeyshines.gemspec

@@ -1,147 +0,0 @@
-# Generated by jeweler
-# DO NOT EDIT THIS FILE
-# Instead, edit Jeweler::Tasks in Rakefile, and run `rake gemspec`
-# -*- encoding: utf-8 -*-
-
-Gem::Specification.new do |s|
-  s.name = %q{monkeyshines}
-  s.version = "0.0.2"
-
-  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.authors = ["Philip (flip) Kromer"]
-  s.date = %q{2009-10-12}
-  s.description = %q{A simple scraper for directed scrapes of APIs, feed or structured HTML. Plays nicely with wuclan and wukong.}
-  s.email = %q{flip@infochimps.org}
-  s.extra_rdoc_files = [
-    "LICENSE",
-     "LICENSE.textile",
-     "README.textile"
-  ]
-  s.files = [
-    ".document",
-     ".gitignore",
-     "LICENSE",
-     "LICENSE.textile",
-     "README.textile",
-     "Rakefile",
-     "VERSION",
-     "examples/.gitignore",
-     "examples/bulk_urls/scrape_bulk_urls.rb",
-     "examples/rename_tree/rename_hdp_tree.rb",
-     "examples/rename_tree/rename_ripd_tree.rb",
-     "examples/rss_feeds/scrape_rss_feeds.rb",
-     "examples/shorturls/README.textile",
-     "examples/shorturls/bulkdump_shorturls.rb",
-     "examples/shorturls/bulkload_shorturls.rb",
-     "examples/shorturls/extract_urls.rb",
-     "examples/shorturls/multiplex_shorturl_cache.rb",
-     "examples/shorturls/old/multidump_and_fix_shorturls.rb",
-     "examples/shorturls/old/shorturl_stats.rb",
-     "examples/shorturls/scrape_shorturls.rb",
-     "examples/shorturls/shorturl_request.rb",
-     "examples/shorturls/shorturl_sequence.rb",
-     "examples/shorturls/shorturl_start_tyrant.sh",
-     "examples/shorturls/start_shorturl_cache.sh",
-     "lib/monkeyshines.rb",
-     "lib/monkeyshines/extensions.rb",
-     "lib/monkeyshines/fetcher.rb",
-     "lib/monkeyshines/fetcher/authed_http_fetcher.rb",
-     "lib/monkeyshines/fetcher/base.rb",
-     "lib/monkeyshines/fetcher/fake_fetcher.rb",
-     "lib/monkeyshines/fetcher/http_fetcher.rb",
-     "lib/monkeyshines/fetcher/http_head_fetcher.rb",
-     "lib/monkeyshines/monitor.rb",
-     "lib/monkeyshines/monitor/chunked_store.rb",
-     "lib/monkeyshines/monitor/periodic_logger.rb",
-     "lib/monkeyshines/monitor/periodic_monitor.rb",
-     "lib/monkeyshines/options.rb",
-     "lib/monkeyshines/recursive_runner.rb",
-     "lib/monkeyshines/repository/base.rb",
-     "lib/monkeyshines/repository/s3.rb",
-     "lib/monkeyshines/request_stream.rb",
-     "lib/monkeyshines/request_stream/base.rb",
-     "lib/monkeyshines/request_stream/edamame_queue.rb",
-     "lib/monkeyshines/request_stream/klass_request_stream.rb",
-     "lib/monkeyshines/request_stream/simple_request_stream.rb",
-     "lib/monkeyshines/runner.rb",
-     "lib/monkeyshines/runner_core/options.rb",
-     "lib/monkeyshines/runner_core/parsing_runner.rb",
-     "lib/monkeyshines/scrape_job/old_paginated.rb",
-     "lib/monkeyshines/scrape_job/recursive.rb",
-     "lib/monkeyshines/scrape_request.rb",
-     "lib/monkeyshines/scrape_request/paginated.rb",
-     "lib/monkeyshines/scrape_request/raw_json_contents.rb",
-     "lib/monkeyshines/scrape_request/signed_url.rb",
-     "lib/monkeyshines/store.rb",
-     "lib/monkeyshines/store/base.rb",
-     "lib/monkeyshines/store/chunked_flat_file_store.rb",
-     "lib/monkeyshines/store/conditional_store.rb",
-     "lib/monkeyshines/store/factory.rb",
-     "lib/monkeyshines/store/flat_file_store.rb",
-     "lib/monkeyshines/store/key_store.rb",
-     "lib/monkeyshines/store/null_store.rb",
-     "lib/monkeyshines/store/read_thru_store.rb",
-     "lib/monkeyshines/store/tokyo_tdb_key_store.rb",
-     "lib/monkeyshines/store/tyrant_rdb_key_store.rb",
-     "lib/monkeyshines/store/tyrant_tdb_key_store.rb",
-     "lib/monkeyshines/utils/factory_module.rb",
-     "lib/monkeyshines/utils/filename_pattern.rb",
-     "lib/monkeyshines/utils/logger.rb",
-     "lib/monkeyshines/utils/trollop-1.14/FAQ.txt",
-     "lib/monkeyshines/utils/trollop-1.14/History.txt",
-     "lib/monkeyshines/utils/trollop-1.14/Manifest.txt",
-     "lib/monkeyshines/utils/trollop-1.14/README.txt",
-     "lib/monkeyshines/utils/trollop-1.14/Rakefile",
-     "lib/monkeyshines/utils/trollop-1.14/lib/trollop.rb",
-     "lib/monkeyshines/utils/trollop-1.14/test/test_trollop.rb",
-     "lib/monkeyshines/utils/trollop.rb",
-     "lib/monkeyshines/utils/union_interval.rb",
-     "lib/monkeyshines/utils/uri.rb",
-     "lib/monkeyshines/utils/uuid.rb",
-     "monkeyshines.gemspec",
-     "scrape_from_file.rb",
-     "spec/monkeyshines_spec.rb",
-     "spec/spec_helper.rb"
-  ]
-  s.homepage = %q{http://github.com/mrflip/monkeyshines}
-  s.rdoc_options = ["--charset=UTF-8"]
-  s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.3.5}
-  s.summary = %q{A simple scraper for directed scrapes of APIs, feed or structured HTML.}
-  s.test_files = [
-    "spec/monkeyshines_spec.rb",
-     "spec/spec_helper.rb",
-     "examples/bulk_urls/scrape_bulk_urls.rb",
-     "examples/rename_tree/rename_hdp_tree.rb",
-     "examples/rename_tree/rename_ripd_tree.rb",
-     "examples/rss_feeds/scrape_rss_feeds.rb",
-     "examples/shorturls/bulkdump_shorturls.rb",
-     "examples/shorturls/bulkload_shorturls.rb",
-     "examples/shorturls/extract_urls.rb",
-     "examples/shorturls/multiplex_shorturl_cache.rb",
-     "examples/shorturls/old/multidump_and_fix_shorturls.rb",
-     "examples/shorturls/old/shorturl_stats.rb",
-     "examples/shorturls/scrape_shorturls.rb",
-     "examples/shorturls/shorturl_request.rb",
-     "examples/shorturls/shorturl_sequence.rb"
-  ]
-
-  if s.respond_to? :specification_version then
-    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
-    s.specification_version = 3
-
-    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<addressable>, [">= 0"])
-      s.add_runtime_dependency(%q<uuid>, [">= 0"])
-      s.add_runtime_dependency(%q<wukong>, [">= 0"])
-    else
-      s.add_dependency(%q<addressable>, [">= 0"])
-      s.add_dependency(%q<uuid>, [">= 0"])
-      s.add_dependency(%q<wukong>, [">= 0"])
-    end
-  else
-    s.add_dependency(%q<addressable>, [">= 0"])
-    s.add_dependency(%q<uuid>, [">= 0"])
-    s.add_dependency(%q<wukong>, [">= 0"])
-  end
-end

data/scrape_from_file.rb

@@ -1,44 +0,0 @@
-#!/usr/bin/env ruby
-require 'rubygems'
-$: << File.dirname(__FILE__)+'/lib'
-require 'wukong'
-require 'monkeyshines'
-require 'monkeyshines/http_fetcher'
-
-request_filename = ARGV[0]
-if ! request_filename
-  warn "Please give the name of a file holding URLs to scrape"; exit
-end
-dump_filename = "/tmp/req_dump.tsv"
-
-class SimpleScrapeRequest < Struct.new(
-    :url,
-    :scraped_at, :response_code, :response_message,
-    :contents )
-end
-
-class String
-  def to_flat
-    self
-  end
-end
-
-class Monkeyshines::FlatFileStore
-  attr_accessor :file, :filename
-  def initialize filename
-    self.filename = filename
-    self.file     = File.open(filename, "w")
-  end
-  def << contents
-    p contents.to_flat
-    self.file << contents.to_flat.join("\t") + "\n"
-  end
-end
-
-fetcher = Monkeyshines::HttpFetcher.new('twitter.com')
-reqs    = Monkeyshines::FlatFileRequestStream.new(request_filename, SimpleScrapeRequest)
-store   = Monkeyshines::FlatFileStore.new(dump_filename)
-reqs.each do |scrape_request|
-  p scrape_request
-  store << fetcher.get(scrape_request)
-end