unbreakable 0.0.3 → 0.0.4

data/lib/unbreakable.rb

@@ -6,10 +6,10 @@ require 'dragonfly'
 # You may enhance a datastore with {Decorators} and {Observers}: for example,
 # a {Decorators::Timeout Timeout} decorator to retry on timeout with exponential
 # backoff and a {Observers::Log Log} observer which logs retrieval progress.
-# Of course, you must also define a {Processors::Transform Processor} to turn
-# your raw data into machine-readable data.
+# Of course, you must also define a {Processors Processor} to turn your raw data
+# into machine-readable data.
 #
-# A skeleton scraper:
+# A simple skeleton scraper:
 #
 #     require 'unbreakable'
 #
@@ -40,7 +40,24 @@ require 'dragonfly'
 #
 # Every scraper script can run as a command-line script. Try it!
 #
-#     ruby myscraper.rb
+#     $ ruby myscraper.rb
+#     usage: irb [options] <command> [<args>]
+#
+#     The most commonly used commands are:
+#         retrieve  Cache remote files to the datastore for later processing
+#         process   Process cached files into machine-readable data
+#         config    Print the current configuration
+#
+#     Specific options:
+#             --root_path ARG              default "/var/tmp/unbreakable"
+#             --[no-]store_meta            default true
+#             --cache_duration ARG         default 31536000
+#             --fallback_mime_type ARG     default "application/octet-stream"
+#             --secret ARG                 default "secret yo"
+#             --[no-]trust_file_extensions default true
+#
+#     General options:
+#         -h, --help                       Display this screen
 module Unbreakable
   autoload :Scraper, 'unbreakable/scraper'
 

data/lib/unbreakable/processors/transform.rb

@@ -1,6 +1,51 @@
 module Unbreakable
+  # Processors are {http://markevans.github.com/dragonfly/file.Processing.html
+  # Dragonfly} processors. For example:
+  #
+  #     class MyProcessor
+  #       def coolify(temp_object, opts = {})
+  #         SomeLib.coolify(temp_object.data, opts)
+  #       end
+  #
+  #       def uglify(temp_object, ugliness)
+  #         `uglify -i #{temp_object.path} -u #{ugliness}`
+  #       end
+  #
+  #       def conditional(temp_object, format, pages)
+  #         throw :unable_to_handle unless format == :pdf
+  #         # do stuff
+  #       end
+  #
+  #     private
+  #
+  #       def my_helper_method
+  #         # do stuff
+  #       end
+  #     end
+  #     MyScraper.processor.register MyProcessor
+  #
+  # Public methods must return an object with which a +TempObject+ may be
+  # initialized (+String+, +File+, +Tempfile+, +Pathname+ or +TempObject+).
+  #
+  # You can raise +Dragonfly::Configurable::NotConfigured+ if a configurable
+  # variable is required but missing. If a variable is invalid, you can raise
+  # +Dragonfly::Configurable::BadConfigAttribute+.
+  #
+  # If a process has dependencies or conditions, then you can test for these
+  # conditions and throw +:unable_to_handle+ to skip processing.
+  #
+  # If multiple processors define a public method by the same name, the methods
+  # will be run in reverse order from the last processor to define the method
+  # until one fails to throw +:unable_to_handle+. If all raise an error, then
+  # +Dragonfly::FunctionManager::UnableToHandle+ will be thrown.
+  #
+  # As such, if you are writing a document to plain-text converter, you can
+  # write a pdftotext processor, a doctopdf processor, etc. which all define
+  # a +to_text+ public method, and use +:unable_to_handle+ to make sure the
+  # correct processor runs.
   module Processors
-    # You may implement a transform process by subclassing this class:
+    # If you are writing a simple scraper and only need one processor, you may
+    # implement a single +transform+ processor method by subclassing this class:
     #
     #     require 'nokogiri'
     #     class MyProcessor < Unbreakable::Processors::Transform
@@ -21,8 +66,10 @@ module Unbreakable
     # * +perform+
     # * +persist+
     #
-    # You may also override +transform+, which calls +perform+ and +persist+ in
-    # the default implementation, but you probably won't have to.
+    # +transform+ calls +persist+ with the output of +perform+. This makes it
+    # easy for others to subclass your processor and just change the +persist+
+    # method to change the external database, for example, while still taking
+    # advantage of the hard work done by +perform+. 
     class Transform
       include Dragonfly::Configurable
       include Dragonfly::Loggable

data/lib/unbreakable/scraper.rb

@@ -2,7 +2,7 @@ require 'forwardable'
 require 'optparse'
 require 'securerandom'
 
-require 'active_support/inflector/methods'
+require 'active_support/core_ext/class/attribute_accessors'
 
 module Unbreakable
   # You may implement a scraper by subclassing this class:
@@ -44,6 +44,9 @@ module Unbreakable
     def_delegators :@app, :add_child_configurable, :configure, :datastore,
       :fetch, :log, :processor
 
+    cattr_accessor :commands
+    @@commands = []
+
     # Initializes a Dragonfly app for storage and processing.
     def initialize
       @app = Dragonfly[SecureRandom.hex.to_sym]
@@ -73,10 +76,12 @@ The most commonly used commands are:
 
         @opts.separator ''
         @opts.separator 'Specific options:'
+        specific_options
         extract_configuration @app
 
         @opts.separator ''
         @opts.separator 'General options:'
+        general_options
         @opts.on_tail('-h', '--help', 'Display this screen') do
           puts @opts
           exit
@@ -85,6 +90,24 @@ The most commonly used commands are:
       @opts
     end
 
+    #     def specific_options
+    #       @opts.on('--echo ARG', 'Write a string to standard output') do |x|
+    #         puts x
+    #       end
+    #     end
+    #
+    # @abstract Override to add specific options to the option parser.
+    def specific_options; end
+
+    #     def general_options
+    #       @opts.on('--echo ARG', 'Write a string to standard output') do |x|
+    #         puts x
+    #       end
+    #     end
+    #
+    # @abstract Override to add general options to the option parser.
+    def general_options; end
+
     # Runs the command. Most often run from a command-line script as:
     #
     #     scraper.run(ARGV)
@@ -104,7 +127,12 @@ The most commonly used commands are:
       when nil
         puts opts
       else
-        opts.abort "'#{command}' is not a #{opts.program_name} command. See '#{opts.program_name} --help'."
+        # Allow subclasses to add more commands.
+        if self.commands.include? command.to_sym
+          send command, args
+        else
+          opts.abort "'#{command}' is not a #{opts.program_name} command. See '#{opts.program_name} --help'."
+        end
       end
     end
 
@@ -171,17 +199,18 @@ The most commonly used commands are:
 
     # @param [#configuration] object
     def extract_configuration(object)
-      object.default_configuration.merge(object.configuration).each do |key,value|
-        if true === value or false === value
-          @opts.on("--[no-]#{key}", "default #{value.inspect}") do |x|
-            object.send "#{key}=", x
+      object.config_methods.each do |meth|
+        default = object.configuration[meth] || object.default_configuration[meth]
+        if true === default or false === default
+          @opts.on("--[no-]#{meth}", "default #{default.inspect}") do |x|
+            object.configure{|c| c.send "#{meth}=", x}
           end
-        elsif String === value or Fixnum === value
-          @opts.on("--#{key} ARG", "default #{value.inspect}") do |x|
-            object.send "#{key}=", x
+        elsif String === default or Fixnum === default
+          @opts.on("--#{meth} ARG", "default #{default.inspect}") do |x|
+            object.configure{|c| c.send "#{meth}=", x}
           end
-        elsif object != value and value.respond_to? :configuration
-          extract_configuration value
+        elsif object != default and default.respond_to? :configuration
+          extract_configuration default
         end
       end
     end
@@ -190,11 +219,12 @@ The most commonly used commands are:
     def print_configuration(object, indent = 0)
       indentation = ' ' * indent
       puts "#{indentation}#{object.class.name}:"
-      object.default_configuration.merge(object.configuration).each do |key,value|
-        if true === value or false === value or String === value or Fixnum === value
-          puts "  #{indentation}#{key.to_s.ljust 25 - indent}#{value.inspect}"
-        elsif object != value and value.respond_to? :configuration
-          print_configuration value, indent + 2
+      object.config_methods.each do |meth|
+        default = object.configuration[meth] || object.default_configuration[meth]
+        if true === default or false === default or String === default or Fixnum === default
+          puts "  #{indentation}#{meth.to_s.ljust 25 - indent}#{default.inspect}"
+        elsif object != default and default.respond_to? :configuration
+          print_configuration default, indent + 2
         end
       end
     end

data/lib/unbreakable/version.rb

@@ -1,3 +1,3 @@
 module Unbreakable
-  VERSION = "0.0.3"
+  VERSION = "0.0.4"
 end

data/unbreakable.gemspec

@@ -20,6 +20,6 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   s.add_runtime_dependency('activesupport', '~> 3.1.0')
-  s.add_runtime_dependency('dragonfly', '~> 0.9.5')
+  s.add_runtime_dependency('dragonfly', '~> 0.9.8')
   s.add_development_dependency('rspec', '~> 2.6.0')
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: unbreakable
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-09-08 00:00:00.000000000Z
+date: 2011-09-09 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &70251392277560 !ruby/object:Gem::Requirement
+  requirement: &70196241006160 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,21 +21,21 @@ dependencies:
         version: 3.1.0
   type: :runtime
   prerelease: false
-  version_requirements: *70251392277560
+  version_requirements: *70196241006160
 - !ruby/object:Gem::Dependency
   name: dragonfly
-  requirement: &70251392277060 !ruby/object:Gem::Requirement
+  requirement: &70196240974480 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.9.5
+        version: 0.9.8
   type: :runtime
   prerelease: false
-  version_requirements: *70251392277060
+  version_requirements: *70196240974480
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70251392276600 !ruby/object:Gem::Requirement
+  requirement: &70196240973840 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,7 +43,7 @@ dependencies:
         version: 2.6.0
   type: :development
   prerelease: false
-  version_requirements: *70251392276600
+  version_requirements: *70196240973840
 description: Abstracts and bulletproofs common scraping tasks.
 email:
 - info@opennorth.ca