unbreakable 0.0.1

data/.gitignore

@@ -0,0 +1,6 @@
+*.gem
+.bundle
+.yardoc
+Gemfile.lock
+doc/*
+pkg/*

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in govkit-ca.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2011 Open North Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,60 @@
+# Unbreakable
+
+Unbreakable is a Ruby gem that abstracts and bulletproofs common web scraping tasks. It forces a separation of concerns for maximum flexibility. Loose coupling allows for easier modification and re-use of component parts.
+
+# Installation
+
+    gem install unbreakable
+
+# What's the problem?
+
+A common web scraping project involves four steps. As an illustrative example, we'll scrape the language with the most articles on Wikipedia using standard command-line tools:
+
+1. Retrieve some raw HTML
+
+        # Download the list of Wikipedias
+        curl -s -o in.html http://s23.org/wikistats/wikipedias_html
+
+1. Process the raw HTML into a machine-readable format
+
+        # Extract the language with the most articles
+        grep '><td class="number">1<' in.html | sed 's/.*e">\([^<]*\).*/\1/' > out.html
+
+1. Release the data to the community through an API and/or as a download
+
+        # Upload the machine-readable data to a public server
+        curl http://pastie.org/pastes -F "paste[parser_id]=6" -F "paste[authorization]=burger" \
+          -F "paste[body]=`cat out.txt`" -s -o /dev/null -L -w "%{url_effective}"
+
+1. Use the data as you like
+
+        echo "The most popular language is `curl -s http://pastie.org/pastes/2487244/download`."
+
+In most web scraping projects, at least one step is tightly coupled to another, making modification or re-use of individual steps by the community difficult. It is especially common for authors to tailor the workflow to their specific use of the data. The coupling produces esoteric code, with the domain logic of the author's use case slipping into the otherwise generic code for retrieving and processing data. Because the scrapers are embedded in a larger project, they are often undiscoverable.
+
+Furthermore, how the first two steps store data may be incompatible with some environments. If the processor code stores data in a database, but you prefer flat files for your use case, you may have to do a long refactor.
+
+# What's the solution?
+
+Web scraping projects should write standalone downloaders, processors, APIs and apps.
+
+Retrieving should be separate from processing, if only to avoid hammering remote servers while developing or tweaking a processor. This separation also allows the community to develop multiple processors of the same raw data without duplication of effort.
+
+Standalone components are easier for the community to discover, modify and re-use, as they do not need to concern themselves with the other parts of the workflow or expose themselves to the use case of the original author.
+
+The code for retrieving and processing data should delegate the persistence of data to a storage layer. The community can then develop various, swappable storage adapters and will not be bound to any single solution.
+
+Unbreakable helps you write standalone downloaders and processor and provides an extensible persistence layer.
+
+# Getting started
+
+For now, the best way to learn how to use this gem is to read the documentation.
+
+    rake yard
+    open doc/index.html
+
+# Bugs? Questions?
+
+Unbreakable's main repository is on GitHub: [http://github.com/opennorth/unbreakable](http://github.com/opennorth/unbreakable), where your contributions, forks, bug reports, feature requests, and feedback are greatly welcomed.
+
+Copyright (c) 2011 Open North Inc., released under the MIT license

data/Rakefile

@@ -0,0 +1,16 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec
+
+begin
+  require 'yard'
+  YARD::Rake::YardocTask.new
+rescue LoadError
+  task :yard do
+    abort 'YARD is not available. In order to run yard, you must: gem install yard'
+  end
+end

data/USAGE

@@ -0,0 +1 @@
+See README.md for full usage details.

data/lib/unbreakable.rb

@@ -0,0 +1,66 @@
+require 'dragonfly'
+
+# When using this gem, you'll start by defining a {Scraper}, with methods for
+# retrieving and processing data. The data will be stored in {DataStorage};
+# this gem currently provides only a {DataStorage::FileDataStore FileDataStore}.
+# You may enhance a datastore with {Decorators} and {Observers}: for example,
+# a {Decorators::Timeout Timeout} decorator to retry on timeout with exponential
+# backoff and a {Observers::Log Log} observer which logs retrieval progress.
+# Of course, you must also define a {Processors::Transform Processor} to turn
+# your raw data into machine-readable data.
+#
+# A skeleton scraper:
+#
+#     require 'unbreakable'
+#
+#     class MyScraper < Unbreakable::Scraper
+#       def retrieve
+#         # download all the documents
+#       end
+#       def processable
+#         # return a list of documents to process
+#       end
+#     end
+#
+#     class MyProcessor < Unbreakable::Processors::Transform
+#       def perform(temp_object)
+#         # return the transformed record as a hash, array, etc.
+#       end
+#       def persist(temp_object, arg)
+#         # store the hash/array/etc. in Mongo, MySQL, YAML, etc.
+#       end
+#     end
+#
+#     scraper = MyScraper.new
+#     scraper.processor.register MyProcessor
+#     scraper.configure do |c|
+#       # configure the scraper
+#     end
+#     scraper.run(ARGV)
+#
+# Every scraper script can run as a command-line script. Try it!
+#
+#     ruby myscraper.rb
+module Unbreakable
+  autoload :Scraper, 'unbreakable/scraper'
+
+  module Processors
+    autoload :Transform, 'unbreakable/processors/transform'
+  end
+
+  module Observers
+    autoload :Observer, 'unbreakable/observers/observer'
+    autoload :Log, 'unbreakable/observers/log'
+  end
+
+  module Decorators
+    autoload :Timeout, 'unbreakable/decorators/timeout'
+  end
+
+  module DataStorage
+    autoload :FileDataStore, 'unbreakable/data_storage/file_data_store'
+  end
+
+  class UnbreakableError < StandardError; end
+  class InvalidRemoteFile < UnbreakableError; end
+end

data/lib/unbreakable/data_storage/file_data_store.rb

@@ -0,0 +1,139 @@
+require 'observer'
+
+module Unbreakable
+  module DataStorage
+    # Stores files to the filesystem. To configure:
+    #
+    #     scraper.configure do |c|
+    #       c.datastore = Unbreakable::DataStorage::FileDataStore.new(scraper,
+    #         :decorators => [:timeout],        # optional
+    #         :observers => [:log])             # optional
+    #       c.datastore.root_path = '/path/dir' # default '/var/tmp/unbreakable'
+    #       c.datastore.store_meta = true       # default false
+    #     end
+    class FileDataStore < Dragonfly::DataStorage::FileDataStore
+      include Observable
+      include Dragonfly::Loggable
+
+      # Decorators should be able to add configuration variables.
+      public_class_method :configurable_attr
+
+      # Configure the datastore to overwrite files upon repeated download.
+      #
+      #     scraper.configure do |c|
+      #       c.datastore.clobber = true # default false
+      #     end
+      #
+      # @return [Boolean, Proc, lambda] whether to overwrite files upon repeated
+      #   download
+      configurable_attr :clobber, false
+
+      # @param [Dragonfly::App] app
+      # @param [Hash] opts
+      # @option options [Module, Symbol, Array<Module, Symbol>] :decorators
+      #   a module, the name of a decorator module, or an array of such
+      # @option options [Class, Symbol, Array<Class, Symbol>] :observers
+      #   a class, the name of an observer class, or an array of such
+      def initialize(app, opts = {})
+        use_same_log_as(app)
+        use_as_fallback_config(app)
+        if opts[:decorators]
+          opts[:decorators].each do |decorator|
+            extend Symbol === decorator ? Unbreakable::Decorators.const_get(decorator.capitalize) : decorator
+          end
+        end
+        if opts[:observers]
+          opts[:observers].each do |observer|
+            add_observer Symbol === observer ? Unbreakable::Observers.const_get(observer.capitalize).new(self) : observer.new(self)
+          end
+        end
+      end
+
+      # Stores a record in the datastore. This method does lazy evaluation of
+      # the record's contents, e.g.:
+      #
+      #     defer_store(:path => 'index.html') do
+      #       open('http://www.example.com/').read
+      #     end
+      #
+      # The +open+ method is called only if the record hasn't already been
+      # downloaded or if the datastore has been configured to overwrite files
+      # upon repeated download.
+      #
+      # @param [Hash] opts
+      # @option opts [Hash] :meta any file metadata, e.g. bitrate
+      # @option opts [String] :path the relative path at which to store the file
+      # @param [Proc] block a block that yields the contents of the file
+      # @raise [Dragonfly::DataStorage::UnableToStore] if permission is denied
+      # @return [String] the relative path to the file
+      # @see [Dragonfly::DataStorage::FileDataStore#store]
+      def defer_store(opts = {}, &block)
+        meta = opts[:meta] || {}
+        relative_path = if opts[:path]
+          opts[:path]
+        else
+          filename = meta[:name] || 'file'
+          relative_path = relative_path_for(filename)
+        end
+
+        changed
+        if empty?(relative_path) or clobber?(relative_path)
+          begin
+            path = absolute(relative_path)
+            prepare_path(path)
+            string = yield_block(relative_path, &block)
+            Dragonfly::TempObject.new(string).to_file(path).close
+            store_meta_data(path, meta) if store_meta
+            notify_observers :store, relative_path, string
+            relative(path)
+          rescue InvalidRemoteFile => e
+            log.error e.message
+          rescue Errno::EACCES => e
+            raise UnableToStore, e.message
+          end
+        else
+          notify_observers :skip, relative_path
+        end
+      end
+
+      # Returns all filenames matching a pattern, if given.
+      # @param [String, Regexp] pattern a pattern to match filenames with
+      # @return [Array<String>] an array of matching filenames
+      def records(pattern = nil)
+        if pattern
+          Dir[File.join(root_path, '**', pattern)]
+        else
+          Dir[File.join(root_path, '**', '*')]
+        end.map do |absolute_path|
+          relative absolute_path
+        end
+      end
+
+    private
+
+      # @param [String] relative_path the relative path to the file
+      # @return [Boolean] whether the file is empty or non-existent
+      def empty?(relative_path)
+        path = absolute(relative_path)
+        !File.exist?(path) || File.size(path).zero?
+      end
+
+      # @param [String] relative_path the relative path to the file
+      # @return [Boolean] whether to overwrite any existing file
+      def clobber?(relative_path)
+        if clobber.respond_to? :call
+          clobber.call(relative_path)
+        else
+          !!clobber
+        end
+      end
+
+      # Yields a block.
+      # @param [String] relative_path the relative path to the file
+      # @return [String] the contents of the file
+      def yield_block(relative_path)
+        yield
+      end
+    end
+  end
+end

data/lib/unbreakable/decorators/timeout.rb

@@ -0,0 +1,47 @@
+require 'timeout'
+
+module Unbreakable
+  module Decorators
+    # Catches timeouts and retries with exponential backoff. To configure:
+    #
+    #     scraper.configure do |c|
+    #       c.datastore.retry_limit = 5 # the maximum number of retries
+    #       c.datastore.timeout_length = 60 # the timeout length
+    #     end
+    #
+    module Timeout
+      # @param object an object
+      def self.extended(obj)
+        obj.class.instance_eval do
+          configurable_attr :retry_limit, 5
+          configurable_attr :timeout_length, 60
+        end
+      end
+
+    private
+
+      # (see DataStorage::FileDataStore#yield_block)
+      def yield_block(relative_path)
+        retry_attempt = 0
+        begin
+          retry_attempt += 1
+          ::Timeout::timeout(timeout_length) do
+            super
+          end
+        rescue ::Timeout::Error
+          if retry_attempt < retry_limit
+            log.warn "Timeout on #{relative_path}, retrying in #{retry_delay} (#{retry_attempt}/#{retry_limit})"
+            sleep retry_delay
+            retry
+          else
+            log.error "Timeout on #{relative_path}, skipping"
+          end
+        end
+      end
+
+      def retry_delay(retry_attempt)
+        2 ** retry_attempt
+      end
+    end
+  end
+end

data/lib/unbreakable/observers/log.rb

@@ -0,0 +1,19 @@
+module Unbreakable
+  module Observers
+    # Logs debug messages when files are stored or skipped if the observed
+    # object has a +#log+ method.
+    class Log < Observer
+      # (see Observer#update)
+      def update(method, *args)
+        if observed.respond_to? :log
+          case method
+          when :store
+            observed.log.debug "Store #{args.first}"
+          when :skip
+            observed.log.debug "Skip #{args.first}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/unbreakable/observers/observer.rb

@@ -0,0 +1,27 @@
+module Unbreakable
+  module Observers
+    # Abstract class for observers following the Ruby
+    # {http://ruby-doc.org/stdlib/libdoc/observer/rdoc/index.html stdlib}
+    # implementation of the _Observer_ object-oriented design pattern. See
+    # {Unbreakable::Observers::Log} for an example.
+    #
+    # The following instance methods must be implemented in sub-classes:
+    #
+    # * +update+
+    class Observer
+      attr_reader :observed
+
+      # @param observed the observed object
+      def initialize(observed)
+        @observed = observed
+      end
+
+      # @param [Symbol] method the method called on the observed object
+      # @param [Array] args the arguments to the method
+      # @return [void]
+      def update(method, *args)
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/unbreakable/processors/transform.rb

@@ -0,0 +1,60 @@
+module Unbreakable
+  module Processors
+    # You may implement a transform process by subclassing this class:
+    #
+    #     require 'nokogiri'
+    #     class MyProcessor < Unbreakable::Processors::Transform
+    #       # Extracts the page title from an HTML page.
+    #       def perform(temp_object)
+    #         Nokogiri::HTML(temp_object.data).at_css('title')
+    #       end
+    #
+    #       # Saves the page title to an external database.
+    #       def persist(temp_object, arg)
+    #         MyModel.create(:title => arg)
+    #       end
+    #     end
+    #     MyScraper.processor.register MyProcessor
+    #
+    # The following instance methods must be implemented in sub-classes:
+    #
+    # * +perform+
+    # * +persist+
+    #
+    # You may also override +transform+, which calls +perform+ and +persist+ in
+    # the default implementation, but you probably won't have to.
+    class Transform
+      include Dragonfly::Configurable
+      include Dragonfly::Loggable
+
+      # +#transform+ must be defined on the subclass for Dragonfly to see it.
+      # @param [Class] subclass a subclass
+      def self.inherited(subclass)
+        subclass.class_eval do
+          # @param [Dragonfly::TempObject] temp_object
+          # @return [Dragonfly::TempObject] the same object
+          def transform(temp_object)
+            persist temp_object, perform(temp_object)
+            temp_object
+          end
+        end
+      end
+
+    private
+
+      # Transforms a record.
+      # @param [Dragonfly::TempObject] temp_object
+      # @return [Hash] the transformed record
+      def perform(temp_object)
+        raise NotImplementedError
+      end
+
+      # Persists a transformed record.
+      # @param [Dragonfly::TempObject] temp_object
+      # @param arg a transformed record
+      def persist(temp_object, arg)
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/unbreakable/scraper.rb

@@ -0,0 +1,200 @@
+require 'forwardable'
+require 'optparse'
+require 'securerandom'
+
+require 'active_support/inflector/methods'
+
+module Unbreakable
+  # You may implement a scraper by subclassing this class:
+  #
+  #     require 'open-uri'
+  #     class MyScraper < Unbreakable::Scraper
+  #       # Stores the contents of +http://www.example.com/+ in +index.html+.
+  #       def retrieve
+  #         store(:path => 'index.html'){ open('http://www.example.com/').read }
+  #       end
+  #
+  #       # Processes +index.html+.
+  #       def process
+  #         fetch('index.html').process(:transform).apply
+  #       end
+  #
+  #       # Alternatively, you can just set the files to fetch, which will be
+  #       # processed using a +:transform+ processor which you must implement.
+  #       def processable
+  #         ['index.html']
+  #       end
+  #     end
+  #
+  # To configure:
+  #
+  #     scraper.configure do |c|
+  #       c.datastore = MyDataStore.new       # default Unbreakable::DataStorage::FileDataStore.new(scraper)
+  #       c.log = Logger.new('/path/to/file') # default Logger.new(STDOUT)
+  #       c.datastore.store_meta = true       # default false
+  #     end
+  #
+  # The following instance methods must be implemented in sub-classes:
+  #
+  # * +retrieve+
+  # * +process+ or +processable+
+  class Scraper
+    extend Forwardable
+
+    def_delegators :@app, :add_child_configurable, :configure, :datastore,
+      :fetch, :log, :processor
+
+    # Initializes a Dragonfly app for storage and processing.
+    def initialize
+      @app = Dragonfly[SecureRandom.hex.to_sym]
+      # defaults to Logger.new('/var/tmp/dragonfly.log')
+      @app.log = Logger.new(STDOUT)
+      # defaults to Dragonfly::DataStorage::FileDataStore.new
+      @app.datastore = Unbreakable::DataStorage::FileDataStore.new(self)
+      # defaults to '/var/tmp/dragonfly'
+      @app.datastore.root_path = '/var/tmp/unbreakable'
+      # defaults to true
+      @app.datastore.store_meta = false
+    end
+
+    # Returns an option parser.
+    # @return [OptionParser] an option parser
+    def opts
+      if @opts.nil?
+        @opts = OptionParser.new
+        @opts.banner = <<-eos
+usage: #{@opts.program_name} [options] <command> [<args>]
+
+The most commonly used commands are:
+    retrieve  Cache remote files to the datastore for later processing
+    process   Process cached files into machine-readable data
+    config    Print the current configuration
+        eos
+
+        @opts.separator ''
+        @opts.separator 'Specific options:'
+        extract_configuration @app
+
+        @opts.separator ''
+        @opts.separator 'General options:'
+        @opts.on_tail('-h', '--help', 'Display this screen') do
+          puts @opts
+          exit
+        end
+      end
+      @opts
+    end
+
+    # Runs the command. Most often run from a command-line script as:
+    #
+    #     scraper.run(ARGV)
+    #
+    # @param [Array] args command-line arguments
+    # @note Only call this method once per scraper instance.
+    def run(args)
+      opts.parse!(args)
+      command = args.shift
+      case command
+      when 'retrieve'
+        retrieve
+      when 'process'
+        process
+      when 'config'
+        print_configuration @app
+      when nil
+        puts opts
+      else
+        opts.abort "'#{command}' is not a #{opts.program_name} command. See '#{opts.program_name} --help'."
+      end
+    end
+
+    # Stores a record in the datastore.
+    # @param [Hash] opts options to pass to the datastore
+    # @param [Proc] block a block that yields the contents of the file
+    def store(opts = {}, &block)
+      datastore.defer_store(opts, &block)
+    end
+
+    # Parses a JSON, HTML, XML, or YAML file.
+    # @param [String, Dragonfly::TempObject] temp_object_or_uid a +TempObject+ or record ID
+    # @param [String] encoding a file encoding
+    # @return the parsing, either a Ruby or +Nokogiri+ type
+    # @raise [LoadError] if the {http://nokogiri.org/ nokogiri} gem is
+    #   unavailable for parsing an HTML or XML file
+    def parse(temp_object_or_uid, encoding = 'utf-8')
+      temp_object = temp_object_or_uid.is_a?(Dragonfly::TempObject) ? temp_object_or_uid : fetch(temp_object_or_uid)
+      string = temp_object.data
+      case File.extname temp_object.path
+      when '.json'
+        begin
+          require 'yajl'
+          Yajl::Parser.parse string
+        rescue LoadError
+          require 'json'
+          JSON.parse string
+        end
+      when '.html'
+        require 'nokogiri'
+        Nokogiri::HTML string, nil, encoding
+      when '.xml'
+        require 'nokogiri'
+        Nokogiri::XML string, nil, encoding
+      when '.yml', '.yaml'
+        require 'yaml'
+        YAML.load string
+      else
+        string
+      end
+    end
+
+    # Caches remote files to the datastore for later processing.
+    def retrieve
+      raise NotImplementedError
+    end
+
+    # Processes cached files into machine-readable data.
+    def process
+      processable.each do |record|
+        fetch(record).process(:transform).apply
+      end
+    end
+
+    # Returns a list of record IDs to process.
+    # @return [Array<String>] a list of record IDs to process
+    def processable
+      raise NotImplementedError
+    end
+
+  private
+
+    # @param [#configuration] object
+    def extract_configuration(object)
+      object.default_configuration.merge(object.configuration).each do |key,value|
+        if true === value or false === value
+          @opts.on("--[no-]#{key}", "default #{value.inspect}") do |x|
+            object.send "#{key}=", x
+          end
+        elsif String === value or Fixnum === value
+          @opts.on("--#{key} ARG", "default #{value.inspect}") do |x|
+            object.send "#{key}=", x
+          end
+        elsif object != value and value.respond_to? :configuration
+          extract_configuration value
+        end
+      end
+    end
+
+    # @param [#configuration] object
+    def print_configuration(object, indent = 0)
+      indentation = ' ' * indent
+      puts "#{indentation}#{object.class.name}:"
+      object.default_configuration.merge(object.configuration).each do |key,value|
+        if true === value or false === value or String === value or Fixnum === value
+          puts "  #{indentation}#{key.to_s.ljust 25 - indent}#{value.inspect}"
+        elsif object != value and value.respond_to? :configuration
+          print_configuration value, indent + 2
+        end
+      end
+    end
+  end
+end

data/lib/unbreakable/version.rb

@@ -0,0 +1,3 @@
+module Unbreakable
+  VERSION = "0.0.1"
+end

data/spec/spec.opts

@@ -0,0 +1,5 @@
+--colour
+--format nested
+--loadby mtime
+--reverse
+--backtrace

data/spec/spec_helper.rb

@@ -0,0 +1,3 @@
+require 'rubygems'
+require 'rspec'
+require File.dirname(__FILE__) + '/../lib/unbreakable'

data/spec/unbreakable_spec.rb

@@ -0,0 +1,5 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+module Unbreakable
+  # TODO
+end

data/unbreakable.gemspec

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "unbreakable/version"
+
+Gem::Specification.new do |s|
+  s.name        = "unbreakable"
+  s.version     = Unbreakable::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Open North"]
+  s.email       = ["info@opennorth.ca"]
+  s.homepage    = "http://github.com/opennorth/unbreakable"
+  s.summary     = %q{Make your scrapers unbreakable™}
+  s.description = %q{Abstracts and bulletproofs common scraping tasks.}
+
+  s.rubyforge_project = "unbreakable"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_runtime_dependency('activesupport', '~> 3.1.0')
+  s.add_runtime_dependency('dragonfly', '~> 0.9.5')
+  s.add_development_dependency('rspec', '~> 2.6.0')
+end

metadata

@@ -0,0 +1,99 @@
+--- !ruby/object:Gem::Specification
+name: unbreakable
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Open North
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-09-07 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: &70281322437780 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *70281322437780
+- !ruby/object:Gem::Dependency
+  name: dragonfly
+  requirement: &70281322437140 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.5
+  type: :runtime
+  prerelease: false
+  version_requirements: *70281322437140
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &70281322436620 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.6.0
+  type: :development
+  prerelease: false
+  version_requirements: *70281322436620
+description: Abstracts and bulletproofs common scraping tasks.
+email:
+- info@opennorth.ca
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- USAGE
+- lib/unbreakable.rb
+- lib/unbreakable/data_storage/file_data_store.rb
+- lib/unbreakable/decorators/timeout.rb
+- lib/unbreakable/observers/log.rb
+- lib/unbreakable/observers/observer.rb
+- lib/unbreakable/processors/transform.rb
+- lib/unbreakable/scraper.rb
+- lib/unbreakable/version.rb
+- spec/spec.opts
+- spec/spec_helper.rb
+- spec/unbreakable_spec.rb
+- unbreakable.gemspec
+homepage: http://github.com/opennorth/unbreakable
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: unbreakable
+rubygems_version: 1.8.6
+signing_key: 
+specification_version: 3
+summary: Make your scrapers unbreakable™
+test_files:
+- spec/spec.opts
+- spec/spec_helper.rb
+- spec/unbreakable_spec.rb