monkeyshines 0.0.2

data/lib/monkeyshines/utils/union_interval.rb

@@ -0,0 +1,52 @@
+#
+# A numeric interval
+#
+# --
+# could be done with a Range but proved annoying in practice
+# what with Range's immutability, etc.
+# ++
+#
+class UnionInterval
+  attr_accessor :min, :max
+  # initialize with set min or max values.
+  # To create an interval with no lower bound call:
+  #   UnionInterval.new(nil, 69)
+  # Pass nil (or omit) +max+ for no upper bound:
+  #   UnionInterval.new(5, nil)
+  def initialize min=nil, max=nil
+    self.min = min
+    self.max = max
+  end
+  # Expand the interval to include all the vals
+  def << vals
+    self.min = [min, vals.to_a].flatten.compact.min
+    self.max = [max, vals.to_a].flatten.compact.max
+  end
+  def + min_max
+    sum_min = [min, min_max.to_a].flatten.compact.min
+    sum_max = [max, min_max.to_a].flatten.compact.max
+    UnionInterval.new sum_min, sum_max
+  end
+  # returns span as an array:
+  #   [min, max]
+  def to_a
+    [min, max]
+  end
+  # true if the extent is defined but empty (lower bound exceeds upper bound)
+  def empty?
+    min && max && (min > max)
+  end
+  def include? val
+    val && (!min || (val >= min)) && (!max || (val <= max))
+  end
+  def size
+    return 0 unless max && min
+    max - min
+  end
+  # string conversion:
+  #   #<span:7..956734>
+  def to_s
+    "#<span:#{min}..#{max}>"
+  end
+  def inspect() to_s end
+end

data/lib/monkeyshines/utils/uri.rb

@@ -0,0 +1,70 @@
+require 'addressable/uri'
+module Addressable
+  #
+  # Add the #scrubbed and #revhost calls
+  #
+  class URI
+    #
+    # These are illegal but *are* found in URLs. We're going to let them through.
+    # Note that ' ' space is one of the tolerated miscreants.
+    #
+    URL_ILLEGAL_BUT_WHATEVER_DOOD_CHARS = '\{\}\| \^\`'
+    #
+    # These are all the characters that belong in a URL
+    #
+    PERMISSIVE_SCRUB_CHARS =
+      URL_ILLEGAL_BUT_WHATEVER_DOOD_CHARS            +
+      Addressable::URI::CharacterClasses::UNRESERVED +
+      Addressable::URI::CharacterClasses::RESERVED   + '%'
+
+    #
+    # Replace all url-insane characters by their %encoding. We don't really
+    # care here whether the URLs do anything: we just want to remove stuff that
+    # absosmurfly don't belong.
+    #
+    # This code is stolen from Addressable::URI, which unfortunately has a bug
+    # in exactly this method (fixed here). (http://addressable.rubyforge.org)
+    # Note that we are /not/ re-encoding characters like '%' -- it's assumed
+    # that the url is encoded, but perhaps poorly.
+    #
+    # In practice the illegal characters most often seen are those in
+    # RE_URL_ILLEGAL_BUT_WHATEVER_DOOD_CHARS plus
+    #   <>"\t\\
+    #
+    def self.scrub_url url
+      return url if url.blank?
+      url.gsub(/[^#{PERMISSIVE_SCRUB_CHARS}]+/) do |sequence|
+        sequence.unpack('C*').map{ |c| ("%%%02x"%c).upcase }.join("")
+      end
+    end
+
+    #
+    # +revhost+
+    # the dot-reversed host:
+    #   foo.company.com => com.company.foo
+    #
+    def revhost
+      return host unless host =~ /\./
+      host.split('.').reverse.join('.')
+    end
+
+    #
+    # The md5hash of this URI
+    #
+    # make sure to require 'digest/md5' somewhere...
+    def md5hash
+      Digest::MD5.hexdigest(self.normalize.to_s)
+    end
+
+    #
+    # +uuid+  -- RFC-4122 ver.5 uuid; guaranteed to be universally unique
+    #
+    # See http://www.faqs.org/rfcs/rfc4122.html
+    #
+    # You ned to require "monkeyshines/utils/uuid" as well...
+    #
+    def url_uuid
+      UUID.sha1_create(UUID_URL_NAMESPACE, self.normalize.to_s)
+    end
+  end
+end

data/lib/monkeyshines/utils/uuid.rb

@@ -0,0 +1,32 @@
+require 'uuidtools'
+class UUID
+
+  #
+  # A string suitable for using as a path name --
+  #
+  # Ex.
+  #   urn:uuid:3c0dce44-80a8-11dd-a897-001ff35a0a8b =>
+  #   urn_uuid/3c0dce44/80a8/11dd/a897/001ff35a0a8b
+  #
+  # It's well possible there are more perspicacious choices for points to split
+  # the string, but until we hit that limit this'll do.
+  #
+  def to_path
+    'urn_uuid/' + to_s.gsub(/[\:\-]/,'/')
+  end
+
+  def self.hex_to_str str
+    /([\da-f]{8})([\da-f]{4})([\da-f]{4})([\da-f]{4})([\da-f]{12})/.match(str).captures.join '-'
+  end
+
+
+  def self.parse_hex str
+    parse(UUID.hex_to_str(str))
+  end
+
+  # Overrides UUIDTools -- force 32 hex digits (leading zeros)
+  def hexdigest
+    "%032x" % self.to_i
+  end
+
+end

data/monkeyshines.gemspec

@@ -0,0 +1,147 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE
+# Instead, edit Jeweler::Tasks in Rakefile, and run `rake gemspec`
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{monkeyshines}
+  s.version = "0.0.2"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Philip (flip) Kromer"]
+  s.date = %q{2009-10-12}
+  s.description = %q{A simple scraper for directed scrapes of APIs, feed or structured HTML. Plays nicely with wuclan and wukong.}
+  s.email = %q{flip@infochimps.org}
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "LICENSE.textile",
+     "README.textile"
+  ]
+  s.files = [
+    ".document",
+     ".gitignore",
+     "LICENSE",
+     "LICENSE.textile",
+     "README.textile",
+     "Rakefile",
+     "VERSION",
+     "examples/.gitignore",
+     "examples/bulk_urls/scrape_bulk_urls.rb",
+     "examples/rename_tree/rename_hdp_tree.rb",
+     "examples/rename_tree/rename_ripd_tree.rb",
+     "examples/rss_feeds/scrape_rss_feeds.rb",
+     "examples/shorturls/README.textile",
+     "examples/shorturls/bulkdump_shorturls.rb",
+     "examples/shorturls/bulkload_shorturls.rb",
+     "examples/shorturls/extract_urls.rb",
+     "examples/shorturls/multiplex_shorturl_cache.rb",
+     "examples/shorturls/old/multidump_and_fix_shorturls.rb",
+     "examples/shorturls/old/shorturl_stats.rb",
+     "examples/shorturls/scrape_shorturls.rb",
+     "examples/shorturls/shorturl_request.rb",
+     "examples/shorturls/shorturl_sequence.rb",
+     "examples/shorturls/shorturl_start_tyrant.sh",
+     "examples/shorturls/start_shorturl_cache.sh",
+     "lib/monkeyshines.rb",
+     "lib/monkeyshines/extensions.rb",
+     "lib/monkeyshines/fetcher.rb",
+     "lib/monkeyshines/fetcher/authed_http_fetcher.rb",
+     "lib/monkeyshines/fetcher/base.rb",
+     "lib/monkeyshines/fetcher/fake_fetcher.rb",
+     "lib/monkeyshines/fetcher/http_fetcher.rb",
+     "lib/monkeyshines/fetcher/http_head_fetcher.rb",
+     "lib/monkeyshines/monitor.rb",
+     "lib/monkeyshines/monitor/chunked_store.rb",
+     "lib/monkeyshines/monitor/periodic_logger.rb",
+     "lib/monkeyshines/monitor/periodic_monitor.rb",
+     "lib/monkeyshines/options.rb",
+     "lib/monkeyshines/recursive_runner.rb",
+     "lib/monkeyshines/repository/base.rb",
+     "lib/monkeyshines/repository/s3.rb",
+     "lib/monkeyshines/request_stream.rb",
+     "lib/monkeyshines/request_stream/base.rb",
+     "lib/monkeyshines/request_stream/edamame_queue.rb",
+     "lib/monkeyshines/request_stream/klass_request_stream.rb",
+     "lib/monkeyshines/request_stream/simple_request_stream.rb",
+     "lib/monkeyshines/runner.rb",
+     "lib/monkeyshines/runner_core/options.rb",
+     "lib/monkeyshines/runner_core/parsing_runner.rb",
+     "lib/monkeyshines/scrape_job/old_paginated.rb",
+     "lib/monkeyshines/scrape_job/recursive.rb",
+     "lib/monkeyshines/scrape_request.rb",
+     "lib/monkeyshines/scrape_request/paginated.rb",
+     "lib/monkeyshines/scrape_request/raw_json_contents.rb",
+     "lib/monkeyshines/scrape_request/signed_url.rb",
+     "lib/monkeyshines/store.rb",
+     "lib/monkeyshines/store/base.rb",
+     "lib/monkeyshines/store/chunked_flat_file_store.rb",
+     "lib/monkeyshines/store/conditional_store.rb",
+     "lib/monkeyshines/store/factory.rb",
+     "lib/monkeyshines/store/flat_file_store.rb",
+     "lib/monkeyshines/store/key_store.rb",
+     "lib/monkeyshines/store/null_store.rb",
+     "lib/monkeyshines/store/read_thru_store.rb",
+     "lib/monkeyshines/store/tokyo_tdb_key_store.rb",
+     "lib/monkeyshines/store/tyrant_rdb_key_store.rb",
+     "lib/monkeyshines/store/tyrant_tdb_key_store.rb",
+     "lib/monkeyshines/utils/factory_module.rb",
+     "lib/monkeyshines/utils/filename_pattern.rb",
+     "lib/monkeyshines/utils/logger.rb",
+     "lib/monkeyshines/utils/trollop-1.14/FAQ.txt",
+     "lib/monkeyshines/utils/trollop-1.14/History.txt",
+     "lib/monkeyshines/utils/trollop-1.14/Manifest.txt",
+     "lib/monkeyshines/utils/trollop-1.14/README.txt",
+     "lib/monkeyshines/utils/trollop-1.14/Rakefile",
+     "lib/monkeyshines/utils/trollop-1.14/lib/trollop.rb",
+     "lib/monkeyshines/utils/trollop-1.14/test/test_trollop.rb",
+     "lib/monkeyshines/utils/trollop.rb",
+     "lib/monkeyshines/utils/union_interval.rb",
+     "lib/monkeyshines/utils/uri.rb",
+     "lib/monkeyshines/utils/uuid.rb",
+     "monkeyshines.gemspec",
+     "scrape_from_file.rb",
+     "spec/monkeyshines_spec.rb",
+     "spec/spec_helper.rb"
+  ]
+  s.homepage = %q{http://github.com/mrflip/monkeyshines}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.5}
+  s.summary = %q{A simple scraper for directed scrapes of APIs, feed or structured HTML.}
+  s.test_files = [
+    "spec/monkeyshines_spec.rb",
+     "spec/spec_helper.rb",
+     "examples/bulk_urls/scrape_bulk_urls.rb",
+     "examples/rename_tree/rename_hdp_tree.rb",
+     "examples/rename_tree/rename_ripd_tree.rb",
+     "examples/rss_feeds/scrape_rss_feeds.rb",
+     "examples/shorturls/bulkdump_shorturls.rb",
+     "examples/shorturls/bulkload_shorturls.rb",
+     "examples/shorturls/extract_urls.rb",
+     "examples/shorturls/multiplex_shorturl_cache.rb",
+     "examples/shorturls/old/multidump_and_fix_shorturls.rb",
+     "examples/shorturls/old/shorturl_stats.rb",
+     "examples/shorturls/scrape_shorturls.rb",
+     "examples/shorturls/shorturl_request.rb",
+     "examples/shorturls/shorturl_sequence.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<addressable>, [">= 0"])
+      s.add_runtime_dependency(%q<uuid>, [">= 0"])
+      s.add_runtime_dependency(%q<wukong>, [">= 0"])
+    else
+      s.add_dependency(%q<addressable>, [">= 0"])
+      s.add_dependency(%q<uuid>, [">= 0"])
+      s.add_dependency(%q<wukong>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<addressable>, [">= 0"])
+    s.add_dependency(%q<uuid>, [">= 0"])
+    s.add_dependency(%q<wukong>, [">= 0"])
+  end
+end

data/scrape_from_file.rb

@@ -0,0 +1,44 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+$: << File.dirname(__FILE__)+'/lib'
+require 'wukong'
+require 'monkeyshines'
+require 'monkeyshines/http_fetcher'
+
+request_filename = ARGV[0]
+if ! request_filename
+  warn "Please give the name of a file holding URLs to scrape"; exit
+end
+dump_filename = "/tmp/req_dump.tsv"
+
+class SimpleScrapeRequest < Struct.new(
+    :url,
+    :scraped_at, :response_code, :response_message,
+    :contents )
+end
+
+class String
+  def to_flat
+    self
+  end
+end
+
+class Monkeyshines::FlatFileStore
+  attr_accessor :file, :filename
+  def initialize filename
+    self.filename = filename
+    self.file     = File.open(filename, "w")
+  end
+  def << contents
+    p contents.to_flat
+    self.file << contents.to_flat.join("\t") + "\n"
+  end
+end
+
+fetcher = Monkeyshines::HttpFetcher.new('twitter.com')
+reqs    = Monkeyshines::FlatFileRequestStream.new(request_filename, SimpleScrapeRequest)
+store   = Monkeyshines::FlatFileStore.new(dump_filename)
+reqs.each do |scrape_request|
+  p scrape_request
+  store << fetcher.get(scrape_request)
+end

data/spec/monkeyshines_spec.rb

@@ -0,0 +1,7 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe "Monkeyshines" do
+  it "fails" do
+    fail "hey buddy, you should probably rename this file and start specing for real"
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,9 @@
+require 'spec'
+
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'monkeyshines'
+
+Spec::Runner.configure do |config|
+  
+end

metadata

@@ -0,0 +1,183 @@
+--- !ruby/object:Gem::Specification 
+name: monkeyshines
+version: !ruby/object:Gem::Version 
+  version: 0.0.2
+platform: ruby
+authors: 
+- Philip (flip) Kromer
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-10-12 00:00:00 -05:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: addressable
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: uuid
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: wukong
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+description: A simple scraper for directed scrapes of APIs, feed or structured HTML. Plays nicely with wuclan and wukong.
+email: flip@infochimps.org
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- LICENSE.textile
+- README.textile
+files: 
+- .document
+- .gitignore
+- LICENSE
+- LICENSE.textile
+- README.textile
+- Rakefile
+- VERSION
+- examples/.gitignore
+- examples/bulk_urls/scrape_bulk_urls.rb
+- examples/rename_tree/rename_hdp_tree.rb
+- examples/rename_tree/rename_ripd_tree.rb
+- examples/rss_feeds/scrape_rss_feeds.rb
+- examples/shorturls/README.textile
+- examples/shorturls/bulkdump_shorturls.rb
+- examples/shorturls/bulkload_shorturls.rb
+- examples/shorturls/extract_urls.rb
+- examples/shorturls/multiplex_shorturl_cache.rb
+- examples/shorturls/old/multidump_and_fix_shorturls.rb
+- examples/shorturls/old/shorturl_stats.rb
+- examples/shorturls/scrape_shorturls.rb
+- examples/shorturls/shorturl_request.rb
+- examples/shorturls/shorturl_sequence.rb
+- examples/shorturls/shorturl_start_tyrant.sh
+- examples/shorturls/start_shorturl_cache.sh
+- lib/monkeyshines.rb
+- lib/monkeyshines/extensions.rb
+- lib/monkeyshines/fetcher.rb
+- lib/monkeyshines/fetcher/authed_http_fetcher.rb
+- lib/monkeyshines/fetcher/base.rb
+- lib/monkeyshines/fetcher/fake_fetcher.rb
+- lib/monkeyshines/fetcher/http_fetcher.rb
+- lib/monkeyshines/fetcher/http_head_fetcher.rb
+- lib/monkeyshines/monitor.rb
+- lib/monkeyshines/monitor/chunked_store.rb
+- lib/monkeyshines/monitor/periodic_logger.rb
+- lib/monkeyshines/monitor/periodic_monitor.rb
+- lib/monkeyshines/options.rb
+- lib/monkeyshines/recursive_runner.rb
+- lib/monkeyshines/repository/base.rb
+- lib/monkeyshines/repository/s3.rb
+- lib/monkeyshines/request_stream.rb
+- lib/monkeyshines/request_stream/base.rb
+- lib/monkeyshines/request_stream/edamame_queue.rb
+- lib/monkeyshines/request_stream/klass_request_stream.rb
+- lib/monkeyshines/request_stream/simple_request_stream.rb
+- lib/monkeyshines/runner.rb
+- lib/monkeyshines/runner_core/options.rb
+- lib/monkeyshines/runner_core/parsing_runner.rb
+- lib/monkeyshines/scrape_job/old_paginated.rb
+- lib/monkeyshines/scrape_job/recursive.rb
+- lib/monkeyshines/scrape_request.rb
+- lib/monkeyshines/scrape_request/paginated.rb
+- lib/monkeyshines/scrape_request/raw_json_contents.rb
+- lib/monkeyshines/scrape_request/signed_url.rb
+- lib/monkeyshines/store.rb
+- lib/monkeyshines/store/base.rb
+- lib/monkeyshines/store/chunked_flat_file_store.rb
+- lib/monkeyshines/store/conditional_store.rb
+- lib/monkeyshines/store/factory.rb
+- lib/monkeyshines/store/flat_file_store.rb
+- lib/monkeyshines/store/key_store.rb
+- lib/monkeyshines/store/null_store.rb
+- lib/monkeyshines/store/read_thru_store.rb
+- lib/monkeyshines/store/tokyo_tdb_key_store.rb
+- lib/monkeyshines/store/tyrant_rdb_key_store.rb
+- lib/monkeyshines/store/tyrant_tdb_key_store.rb
+- lib/monkeyshines/utils/factory_module.rb
+- lib/monkeyshines/utils/filename_pattern.rb
+- lib/monkeyshines/utils/logger.rb
+- lib/monkeyshines/utils/trollop-1.14/FAQ.txt
+- lib/monkeyshines/utils/trollop-1.14/History.txt
+- lib/monkeyshines/utils/trollop-1.14/Manifest.txt
+- lib/monkeyshines/utils/trollop-1.14/README.txt
+- lib/monkeyshines/utils/trollop-1.14/Rakefile
+- lib/monkeyshines/utils/trollop-1.14/lib/trollop.rb
+- lib/monkeyshines/utils/trollop-1.14/test/test_trollop.rb
+- lib/monkeyshines/utils/trollop.rb
+- lib/monkeyshines/utils/union_interval.rb
+- lib/monkeyshines/utils/uri.rb
+- lib/monkeyshines/utils/uuid.rb
+- monkeyshines.gemspec
+- scrape_from_file.rb
+- spec/monkeyshines_spec.rb
+- spec/spec_helper.rb
+has_rdoc: true
+homepage: http://github.com/mrflip/monkeyshines
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: A simple scraper for directed scrapes of APIs, feed or structured HTML.
+test_files: 
+- spec/monkeyshines_spec.rb
+- spec/spec_helper.rb
+- examples/bulk_urls/scrape_bulk_urls.rb
+- examples/rename_tree/rename_hdp_tree.rb
+- examples/rename_tree/rename_ripd_tree.rb
+- examples/rss_feeds/scrape_rss_feeds.rb
+- examples/shorturls/bulkdump_shorturls.rb
+- examples/shorturls/bulkload_shorturls.rb
+- examples/shorturls/extract_urls.rb
+- examples/shorturls/multiplex_shorturl_cache.rb
+- examples/shorturls/old/multidump_and_fix_shorturls.rb
+- examples/shorturls/old/shorturl_stats.rb
+- examples/shorturls/scrape_shorturls.rb
+- examples/shorturls/shorturl_request.rb
+- examples/shorturls/shorturl_sequence.rb