spiderfw 0.5

data/lib/spiderfw/utils/monkey/debugger.rb

@@ -0,0 +1,25 @@
+module Debugger
+    
+    class Context
+        
+        def _dump(obj)
+            ""
+        end
+        
+        def _load
+            nil
+        end
+    end
+    
+end
+
+class Binding
+    
+    def _dump(obj)
+        ""
+    end
+    
+    def _load
+        nil
+    end
+end

data/lib/spiderfw/utils/monkey/exception.rb

@@ -0,0 +1,23 @@
+require 'uuid'
+
+module UUIDException
+    
+    def uuid=(val)
+        @uuid = val
+    end
+    
+    def uuid
+        @uuid
+    end
+    
+end
+
+module UUIDExceptionMessage
+    def message
+        "#{@uuid} - #{super}"
+    end
+end
+
+class Exception
+    include UUIDException
+end

data/lib/spiderfw/utils/monkey/kernel.rb

@@ -0,0 +1,12 @@
+# Kernel monkey patch.
+
+module Kernel
+    
+    # Utility method to get a full constant, like M::N::A
+    #--
+    # (from Merb)
+    def const_get_full(const)
+        const.to_s.split("::").inject(Object) {|c1,c2| c1.const_get(c2)}
+    end
+    
+end

data/lib/spiderfw/utils/monkey/module.rb

@@ -0,0 +1,12 @@
+class Module #:nodoc:
+
+    # This is here just to be able to call this method on all constants
+    def subclass_of?(klass)
+        return false
+    end
+    
+    def parent_module(n=1)
+        return const_get_full(self.to_s.reverse.split('::', n+1)[n].reverse)
+    end
+    
+end

data/lib/spiderfw/utils/monkey/symbol.rb

@@ -0,0 +1,7 @@
+# Symbol monkey patch.
+
+class Symbol
+    def to_proc
+        proc { |obj, *args| obj.send(self, *args) }
+    end
+end

data/lib/spiderfw/utils/monkey/time.rb

@@ -0,0 +1,17 @@
+require 'date'
+require 'time'
+
+# Monkey-patched conversions to Date and DateTime
+class Time
+    def lformat(format = :default, locale=nil)
+        locale ||= Spider.locale
+        Spider::I18n.localize(locale, self, format)
+    end
+    
+    def to_date
+        ::Date.new(year, month, day)
+    end
+    def to_datetime
+        ::DateTime.civil(year, month, day, hour, min, sec, Rational(utc_offset, 86400))
+    end
+end

data/lib/spiderfw/utils/multi_level_hash.rb

@@ -0,0 +1,33 @@
+module Spider
+    
+    # Utility that makes the including Hash accept a dotted syntax for keys.
+    # The dotted syntax can be used to access and create on the fly sub-hashes. 
+    # Example:
+    #   h = MultiLevelHash.new
+    #   h['one.two.three'] = 'some val'
+    #   p h => {'one' => {'two' => {'three' => 'some val'}}}
+    #   p h['one.two.three'] => 'some val'
+    #   p h['four.five'] => nil
+    module HashDottedAccess
+        def []=(key, val)
+            parts = key.to_s.split('.', 2)
+            return super(key, val) unless parts[1]
+            self[parts[0]] ||= self.class.new
+            self[parts[0]][parts[1]] = val
+        end
+        
+        def [](key)
+            parts = key.to_s.split('.', 2)
+            return super(key) unless parts[1]
+            return self[parts[0]][parts[1]]
+        end
+    end
+        
+    # Hash including HashDottedAccess.
+    class MultiLevelHash < Hash
+        include HashDottedAccess
+        
+    end
+    
+    
+end

data/lib/spiderfw/utils/periodic_runner.rb

@@ -0,0 +1,50 @@
+module Spider
+
+    # TODO: remove
+    class PeriodicRunner #:nodoc:
+        attr_reader :runner_thread
+        
+        def initialize(sleep_time=nil)
+            @entries = []
+            @sleep_time = sleep_time
+            @mutex = Mutex.new
+            if (sleep_time)
+                @runner_thread = Thread.new{
+                    while (true)
+                        sleep(sleep_time)
+                        run
+                    end
+                }
+            end
+        end
+        
+        def add(secs, &proc)
+            proc.call
+            @mutex.synchronize {
+                @entries << {
+                    :secs => secs,
+                    :proc => proc,
+                    :last => Time.now.to_i
+                }
+            }
+        end
+        
+        def run
+            Spider::Logger.debug("Periodic_runner")
+            @mutex.synchronize {
+                Spider::Logger.debug("In synchro")
+                @entries.each do |entry|
+                    if (!entry[:last] || (entry[:last] + secs) < Time.now.to_i )
+                        proc.call
+                        entry[:last] = Time.now.to_i
+                    end
+                end
+            }
+        end
+        
+    end
+    
+
+    
+    
+end

data/lib/spiderfw/utils/rails.rb

@@ -0,0 +1,78 @@
+require 'spiderfw/model/active_record'
+
+module Spider
+    
+    class Rails
+    
+        def initialize(path, options={})
+            @path = path
+            @options = options
+            @rails_app_name = options[:app_name]
+        end
+        
+        def start
+            return if @started
+            Spider::Logger.info("Loading Rails environment at #{@path}")
+            require "#{@path}/config/environment.rb"
+            ActiveRecord::Base.instance_eval do
+                def self.spider_model
+                    @spider_model
+                end
+                def self.spider_model=(val)
+                    @spider_model = val
+                end
+            end
+            @started = true
+        end
+        
+        def started?
+            @started
+        end
+        
+        def define_models(container)
+            start unless started?
+            ar_models = []
+            
+            self.class.active_record_models.each do |mod|
+                Spider::Model::create_ar_classes(mod, container)
+            end
+            done = {}
+            # Spider::Model.ar_models.each do |mod|
+            #     mod.ar_through_models.each do |m|
+            #         next if done[m]
+            #         m.define_from_ar
+            #         done[m] = true
+            #     end
+            # end
+            
+            Spider::Model.ar_models.each do |mod|
+                #next if done[mod]
+                next unless mod.ar_defined
+                mod.rails_app_name = @rails_app_name
+                mod.define_from_ar
+                #done[mod] = true
+            end
+        end
+        
+        class <<self
+            def active_record_models(mod=nil)
+                mod ||= Object
+                res = []
+
+                mod.constants.each do |const|
+                    kl = mod.const_get(const.to_sym)
+                    if (kl.is_a?(Class))
+                        res << kl if kl < ::ActiveRecord::Base
+                    # elsif(kl.is_a?(Module))
+                    #                        res += active_record_models(kl)
+                    end
+                    
+                end
+                return res
+            end
+        end
+        
+    end
+    
+    
+end

data/lib/spiderfw/utils/rails_app.rb

@@ -0,0 +1,53 @@
+require 'spiderfw/utils/rails'
+
+module Spider
+    
+    module RailsApp
+    
+        def self.included(mod)
+            mod.extend(ClassMethods)
+            mod.instance_eval do
+                include Spider::App
+            end
+        end
+        
+        
+    
+        module ClassMethods
+        
+            def rails(path, options={})
+                @options = options
+                @rails = Spider::Rails.new(path, options)
+            end
+
+            def rails_app_name=(val)
+                @rails_app_name = val
+            end    
+            
+            def define_rails_models
+                @rails.define_models(self)
+            end
+            
+            def const_missing(name)
+                unless @options[:reload_models]
+                    local = "#{self.app.models_path}/#{Inflector.underscore(name)}.rb"
+                    local_ext = "#{self.app.models_path}/extended/#{Inflector.underscore(name)}.rb"
+                end
+                if (local && File.exist?(local))
+                    require local
+                else
+                    @rails.start
+                    define_rails_models
+                end
+                if (File.exist?(local_ext))
+                    require local_ext
+                end
+                klass = const_get(name)
+                return klass if klass
+                super
+            end
+        
+        end    
+        
+    end
+end

data/lib/spiderfw/utils/sanitizer.rb

@@ -0,0 +1,178 @@
+# Based on Sanitizer in ActionPack 2.3.4
+#Copyright (c) 2004-2009 David Heinemeier Hansson
+
+module Spider
+
+    class Sanitizer
+        def sanitize(text, options = {})
+            return text unless sanitizeable?(text)
+            tokenize(text, options).join
+        end
+
+        def sanitizeable?(text)
+            !(text.nil? || text.empty? || !text.index("<"))
+        end
+
+        protected
+        def tokenize(text, options)
+            tokenizer = HTML::Tokenizer.new(text)
+            result = []
+            while token = tokenizer.next
+                node = Node.parse(nil, 0, 0, token, false)
+                process_node node, result, options
+            end
+            result
+        end
+
+        def process_node(node, result, options)
+            result << node.to_s
+        end
+    end
+
+    class FullSanitizer < Sanitizer
+        def sanitize(text, options = {})
+            result = super
+            # strip any comments, and if they have a newline at the end (ie. line with
+            # only a comment) strip that too
+            result.gsub!(/<!--(.*?)-->[\n]?/m, "") if result
+            # Recurse - handle all dirty nested tags
+            result == text ? result : sanitize(result, options)
+        end
+
+        def process_node(node, result, options)
+            result << node.to_s if node.class == HTML::Text
+        end
+    end
+
+    class LinkSanitizer < FullSanitizer
+        cattr_accessor :included_tags, :instance_writer => false
+        self.included_tags = %w(a href)
+
+        def sanitizeable?(text)
+            !(text.nil? || text.empty? || !((text.index("<a") || text.index("<href")) && text.index(">")))
+        end
+
+        protected
+        def process_node(node, result, options)
+            result << node.to_s unless node.is_a?(HTML::Tag) && included_tags.include?(node.name) 
+        end
+    end
+
+    class WhiteListSanitizer < Sanitizer
+        [:protocol_separator, :uri_attributes, :allowed_attributes, :allowed_tags, :allowed_protocols, :bad_tags,
+            :allowed_css_properties, :allowed_css_keywords, :shorthand_css_properties].each do |attr|
+                cattr_accessor attr, :instance_writer => false
+            end
+
+            # A regular expression of the valid characters used to separate protocols like
+            # the ':' in 'http://foo.com'
+            self.protocol_separator     = /:|(&#0*58)|(&#x70)|(%|&#37;)3A/
+
+            # Specifies a Set of HTML attributes that can have URIs.
+            self.uri_attributes         = %w(href src cite action longdesc xlink:href lowsrc)
+
+            # Specifies a Set of 'bad' tags that the #sanitize helper will remove completely, as opposed
+            # to just escaping harmless tags like &lt;font&gt;
+            self.bad_tags               = %w(script)
+
+            # Specifies the default Set of tags that the #sanitize helper will allow unscathed.
+            self.allowed_tags           = %w(strong em b i p code pre tt samp kbd var sub 
+            sup dfn cite big small address hr br div span h1 h2 h3 h4 h5 h6 ul ol li dt dd abbr 
+            acronym a img blockquote del ins)
+
+            # Specifies the default Set of html attributes that the #sanitize helper will leave 
+            # in the allowed tag.
+            self.allowed_attributes     = %w(href src width height alt cite datetime title class name xml:lang abbr)
+
+            # Specifies the default Set of acceptable css properties that #sanitize and #sanitize_css will accept.
+            self.allowed_protocols      = %w(ed2k ftp http https irc mailto news gopher nntp telnet webcal xmpp callto 
+            feed svn urn aim rsync tag ssh sftp rtsp afs)
+
+            # Specifies the default Set of acceptable css keywords that #sanitize and #sanitize_css will accept.
+            self.allowed_css_properties = %w(azimuth background-color border-bottom-color border-collapse 
+            border-color border-left-color border-right-color border-top-color clear color cursor direction display 
+            elevation float font font-family font-size font-style font-variant font-weight height letter-spacing line-height
+            overflow pause pause-after pause-before pitch pitch-range richness speak speak-header speak-numeral speak-punctuation
+            speech-rate stress text-align text-decoration text-indent unicode-bidi vertical-align voice-family volume white-space
+            width)
+
+            # Specifies the default Set of acceptable css keywords that #sanitize and #sanitize_css will accept.
+            self.allowed_css_keywords   = %w(auto aqua black block blue bold both bottom brown center
+            collapse dashed dotted fuchsia gray green !important italic left lime maroon medium none navy normal
+            nowrap olive pointer purple red right solid silver teal top transparent underline white yellow)
+
+            # Specifies the default Set of allowed shorthand css properties for the #sanitize and #sanitize_css helpers.
+            self.shorthand_css_properties = %w(background border margin padding)
+
+            # Sanitizes a block of css code.  Used by #sanitize when it comes across a style attribute
+            def sanitize_css(style)
+                # disallow urls
+                style = style.to_s.gsub(/url\s*\(\s*[^\s)]+?\s*\)\s*/, ' ')
+
+                # gauntlet
+                if style !~ /^([:,;#%.\sa-zA-Z0-9!]|\w-\w|\'[\s\w]+\'|\"[\s\w]+\"|\([\d,\s]+\))*$/ ||
+                    style !~ /^(\s*[-\w]+\s*:\s*[^:;]*(;|$)\s*)*$/
+                    return ''
+                end
+
+                clean = []
+                style.scan(/([-\w]+)\s*:\s*([^:;]*)/) do |prop,val|
+                    if allowed_css_properties.include?(prop.downcase)
+                        clean <<  prop + ': ' + val + ';'
+                    elsif shorthand_css_properties.include?(prop.split('-')[0].downcase) 
+                        unless val.split().any? do |keyword|
+                            !allowed_css_keywords.include?(keyword) && 
+                            keyword !~ /^(#[0-9a-f]+|rgb\(\d+%?,\d*%?,?\d*%?\)?|\d{0,2}\.?\d{0,2}(cm|em|ex|in|mm|pc|pt|px|%|,|\))?)$/
+                        end
+                        clean << prop + ': ' + val + ';'
+                    end
+                end
+            end
+            clean.join(' ')
+        end
+
+        protected
+        def tokenize(text, options)
+            options[:parent] = []
+            options[:attributes] ||= allowed_attributes
+            options[:tags]       ||= allowed_tags
+            super
+        end
+
+        def process_node(node, result, options)
+            result << case node
+            when HTML::Tag
+                if node.closing == :close
+                    options[:parent].shift
+                else
+                    options[:parent].unshift node.name
+                end
+
+                process_attributes_for node, options
+
+                options[:tags].include?(node.name) ? node : nil
+            else
+                bad_tags.include?(options[:parent].first) ? nil : node.to_s.gsub(/</, "&lt;")
+            end
+        end
+
+        def process_attributes_for(node, options)
+            return unless node.attributes
+            node.attributes.keys.each do |attr_name|
+                value = node.attributes[attr_name].to_s
+
+                if !options[:attributes].include?(attr_name) || contains_bad_protocols?(attr_name, value)
+                    node.attributes.delete(attr_name)
+                else
+                    node.attributes[attr_name] = attr_name == 'style' ? sanitize_css(value) : CGI::escapeHTML(CGI::unescapeHTML(value))
+                end
+            end
+        end
+
+        def contains_bad_protocols?(attr_name, value)
+            uri_attributes.include?(attr_name) && 
+            (value =~ /(^[^\/:]*):|(&#0*58)|(&#x70)|(%|&#37;)3A/ && !allowed_protocols.include?(value.split(protocol_separator).first))
+        end
+    end
+
+end

data/lib/spiderfw/utils/setup_task.rb

@@ -0,0 +1,35 @@
+module Spider
+    
+    # TODO: remove?
+    class SetupTask #:nodoc:
+        
+        # FIXME: concurrency?
+        def self.current_task=(task)
+            @current_task = task
+        end
+        
+        def self.current_task
+            @current_task
+        end
+        
+        def self.inherited(subclass)
+            Spider::SetupTask.current_task = subclass
+        end
+        
+        def run
+        end
+        
+        def rollback
+            raise SetupError, "Rollback not implemented for SetupTask #{self.class}"
+        end
+        
+        def done
+            Spider::SetupTask.current_task = nil
+        end
+        
+    end
+    
+    class SetupError < RuntimeError
+    end
+    
+end

data/lib/spiderfw/utils/shared_store/file_shared_store.rb

@@ -0,0 +1,73 @@
+require 'spiderfw/utils/shared_store'
+require 'fileutils'
+
+module Spider; module Utils
+    
+    # Implementation of the SharedStore in the filesystem.
+    # This is a persistent store, accessible by different threads and processes at once.
+    class FileSharedStore < SharedStore
+        attr_reader :path
+        
+        def initialize(config={})
+            super
+            @path = config[:path]
+            if (!@path && config[:name])
+                @path = Spider.conf.get('shared_store.file.base_path')+'/'+config[:name]
+            end
+            if (!@path)
+                raise ArgumentError, "You must supply the FileSharedStore with a path, or a name a configured base path"
+            end
+            FileUtils.mkpath(@path)     
+            @sync = Sync.new
+        end
+        
+        def [](key)
+            path = map_path(key)
+            if (File.exist?(path))
+                @sync.lock(Sync::SH)
+                f = File.new(path, 'r')
+                f.flock(File::LOCK_SH)
+                data = Marshal.restore(f.read)
+                f.flock(File::LOCK_UN)
+                f.close
+                @sync.lock(Sync::UN)
+            end
+            return data
+        end
+        
+        def []=(key, value)
+            path = map_path(key)
+            @sync.lock(Sync::EX)
+            f = File.new(path, 'w')
+            f.flock(File::LOCK_EX)
+            f.puts(Marshal.dump(value))
+            f.flush
+            f.flock(File::LOCK_UN)
+            f.close
+            @sync.lock(Sync::UN)
+        end
+
+        
+        def delete(key)
+            File.unlink(map_path(key))
+        end
+        
+        def include?(key)
+            File.exist?(map_path(key))
+        end
+        
+        def map_path(key)
+            "#{@path}/key"
+        end
+        
+        def each_key
+            Dir.new(@path).each do |key|
+                next unless File.file?(@path+'/'+key)
+                yield key
+            end
+        end
+        
+        
+    end
+    
+end; end

data/lib/spiderfw/utils/shared_store/memory_shared_store.rb

@@ -0,0 +1,46 @@
+require 'spiderfw/utils/shared_store'
+
+module Spider; module Utils
+    
+    # Implementation of the SharedStore in memory.
+    # This is a transient, per-process, thread safe store. (See SharedStore about thread safety).
+    class MemorySharedStore < SharedStore
+        
+        def initialize(config={})
+            super
+            @data = {}
+            @sync = Sync.new
+        end
+        
+        def [](key)
+            @sync.lock(:SH)
+            res = @data[key]
+            @sync.lock(:UN)
+            return res
+        end
+        
+        def []=(key, value)
+            @sync.lock(:EX)
+            @data[key] = value
+            @sync.lock(:UN)
+        end
+        
+        def delete(key)
+            @sync.lock(:EX)
+            @data.delete(key)
+            @sync.lock(:UN)
+        end
+        
+        def include?(key)
+            @data.include?(key)
+        end
+        
+        def each_key
+            @data.keys.each do |k|
+                yield k
+            end
+        end
+        
+    end
+    
+end; end