waw 0.2.2 → 0.3.0

data/bin/waw-crawl

@@ -0,0 +1,26 @@
+#!/usr/bin/env ruby
+#
+#  Waw - making web development simple
+#        (see lib/waw/waw.rb for more information)
+#
+#  Copyright (c) 2010 University of Louvain, Bernard & Louis Lambeau
+#  Released under a MIT or Ruby licence
+#
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'waw'
+require 'waw/commands/command'
+require 'waw/commands/crawl_command'
+
+begin
+  r = Waw::Commands::CrawlCommand.new.run '.', ARGV
+rescue Interrupt => e
+  $stderr.puts
+  $stderr.puts "Interrupted"
+  raise e
+rescue OptionParser::ParseError => e
+  $stderr.puts e.message
+  raise e
+rescue => e
+  $stderr.puts e.message
+  raise e
+end

data/lib/waw.rb

@@ -9,13 +9,13 @@ module Waw
   
   # Requirements on gems for this version
   GEM_REQUIREMENTS = {
-    :rack  => '>= 1.1.0',
+    :rack  => '>= 1.2.1',
     :wlang => '>= 0.9.0',
     :json  => '>= 1.1.9'
   }
   
   # Waw version
-  VERSION = "0.2.2".freeze
+  VERSION = "0.3.0".freeze
   
   # Waw loading mutex
   WAW_KERNELS_LOCK = Mutex.new
@@ -96,4 +96,4 @@ require 'waw/controllers/action_controller'
 require 'waw/controllers/static_controller'
 require 'waw/controllers/no_cache'
 require 'waw/controllers/error_handler'
-require 'waw/routing'
+require 'waw/routing'

data/lib/waw/commands/command.rb

@@ -5,14 +5,14 @@ module Waw
     class Command
       
       # The verbose level
-      attr_accessor :verbose
+      attr_accessor :verbosity
       
       # Show stack traces?
       attr_accessor :trace
       
       # Creates an empty command instance
       def initialize
-        @verbose = false
+        @verbosity = 1
         @buffer = STDOUT
       end
       
@@ -35,8 +35,12 @@ module Waw
             @trace = true
           end
           
-          opt.on("--verbose", "-v", "Display extra progress as we progress") do |value|
-            @verbose = true
+          opt.on("--verbose", "Display extra info as we progress") do |value|
+            @verbosity = 2
+          end
+          
+          opt.on("--silent", "Be quiet silent") do |value|
+            @verbosity = 0
           end
           
           # No argument, shows at tail.  This will print an options summary.
@@ -55,6 +59,10 @@ module Waw
           opt.separator nil
         end
       end
+      
+      def verbose
+        verbosity < 0
+      end
 
       # Runs the command
       def run(requester_file, argv)

data/lib/waw/commands/crawl_command.rb

@@ -0,0 +1,46 @@
+require 'waw/crawler'
+module Waw
+  module Commands
+    class CrawlCommand < Command
+      
+      def banner
+        <<-EOF
+          Usage: waw-crawl [options] WEB_SITE_URI
+        EOF
+      end
+      
+      # Start command is always safe
+      def check_command_policy
+        true
+      end
+      
+      # Adds the options
+      def add_options(options)
+        @crawler = Waw::Crawler.new(nil)
+        options.on("--[no-]check-externals", "[Don't] ping any external link") do |value|
+          @crawler.check_externals = value
+        end
+        options.on("--[no-]check-img", "[Don't] check image <img src='...'>") do |value|
+          @crawler.ping_on('img/@src', value)
+        end
+        options.on("--[no-]check-link", "[Don't] check <link href='...'>") do |value|
+          @crawler.ping_on('link/@href', value)
+        end
+        options.on("--[no-]check-script", "[Don't] check <script src='...'>") do |value|
+          @crawler.ping_on('script/@src', value)
+        end
+      end
+      
+      # Runs the sub-class defined command
+      def __run(requester_file, arguments)
+        exit(nil, true) unless (arguments.size == 1)
+        @crawler.root_uri = arguments[0]
+        @crawler.listener.verbosity = @verbosity
+        @crawler.crawl
+      rescue Interrupt => ex
+        info "waw-crawl stopping now... ciao!" if verbose
+      end
+
+    end # module ProfileCommand
+  end # module Commands
+end # module Waw

data/lib/waw/commands/profile_command.rb

@@ -31,7 +31,11 @@ module Waw
         else
           puts "Visiting #{location}" if verbose
           browser.all_internal_links.each do |link|
-            visit(browser, link[:href], visited)
+            begin
+              visit(browser, link[:href], visited)
+            rescue URI::InvalidURIError => ex
+              puts "Hohoho, I've found something really wrong #{link[:href]}"
+            end
           end
         end 
       end

data/lib/waw/controllers/action/action.rb

@@ -43,6 +43,17 @@ module Waw
         "javascript:#{id}({#{buffer[2..-1]}}, '##{id}')"
       end
       
+      # Factors the ajax code for the action itself
+      def ajax_action_code
+        js = Waw::ActionController::JSGeneration.new
+        code = js.generate_js_for_action(Waw::kernel, self, "")
+        <<-EOF
+          <script type="text/javascript">
+            #{code}
+          </script>
+        EOF
+      end
+      
       # Factors the ajax code for preparing a formulary
       def ajax_form_preparer(opts = {})
         form_id = opts[:form_id] || id

data/lib/waw/controllers/action/js_generation.rb

@@ -1,6 +1,6 @@
 module Waw
   class ActionController < ::Waw::Controller
-    class JSGeneration
+    class JSGeneration < ::Waw::Controller
       
       # Header of the generated javascript file
       HEADER = <<-EOF
@@ -18,6 +18,15 @@ module Waw
         raise ConfigurationError, msg
       end
       
+      # Acts as a waw controller
+      def call(env)
+        buffer = []
+        generate_js(kernel, buffer)
+        [ 200, 
+          {'Content-Type' => 'application/javascript'},
+          buffer ]
+      end
+      
       # Start hook start callback required by Waw. Generates the javascript code
       # if the configuration variable 'code_at_startup' is true. 
       def run(waw_kernel)
@@ -97,7 +106,7 @@ module Waw
           function #{action.id}(request_data, form) {
             $.ajax({type: "POST", url: "#{action.url}", data: request_data, dataType: "json",
               error: function(data) {
-          		  window.location = '/feedback?mkey=server_error';
+          		  window.location = '/500';
           		},
           		success: function(data) {
         THEEND

data/lib/waw/controllers/static/match.rb

@@ -4,6 +4,8 @@ module Waw
       class Match
         include Waw::ScopeUtils
         
+        attr_reader :wawaccess
+        
         # Served file
         attr_reader :served_file
         
@@ -15,13 +17,14 @@ module Waw
         end
         
         # Executes on a wawaccess instance
-        def __execute
+        def __execute(env)
           instance_exec *@args, &@block
         end
         
         # Delegated to the wawaccess that created me
         def root; @wawaccess.root; end
         def folder; @wawaccess.folder; end
+        def req_path; @wawaccess.req_path; end
         
         ################################################### Callbacks proposed to .wawaccess rules
       

data/lib/waw/controllers/static/matcher.rb

@@ -0,0 +1,35 @@
+module Waw
+  class StaticController < ::Waw::Controller
+    class Matcher
+      include Waw::ScopeUtils
+      
+      # Waw access on which this matcher is defined
+      attr_reader :wawaccess
+      
+      # Matcher's predicate
+      attr_reader :predicate
+      
+      # Creates a matcher instance
+      def initialize(wawaccess, predicate)
+        @wawaccess = wawaccess
+        @predicate = predicate
+      end
+      
+      # Returns wawaccess's folder
+      def folder
+        wawaccess.folder
+      end
+      
+      # Returns requested path
+      def req_path
+        wawaccess.req_path
+      end
+      
+      # Does the matcher matches a given path?
+      def matches?(env)
+        instance_eval &predicate
+      end
+      
+    end # class Matcher
+  end # class StaticController
+end # module Waw

data/lib/waw/controllers/static/waw_access.rb

@@ -1,9 +1,8 @@
-require 'uri'
-require 'waw/controllers/static/match'
 module Waw
   class StaticController < ::Waw::Controller
     # Waw version of .htaccess files
     class WawAccess
+      include Waw::ScopeUtils
       
       # The folder which is served
       attr_accessor :folder
@@ -66,7 +65,7 @@ module Waw
       
       def recognized_pattern?(pattern)
         [FalseClass, TrueClass, String, 
-         Regexp, Waw::Validation::Validator].any?{|c| c===pattern}
+         Regexp, Waw::Validation::Validator, StaticController::Matcher].any?{|c| c===pattern}
       end
       
       # Adds a child in the hierarchy
@@ -106,6 +105,10 @@ module Waw
       
       ################################################### Utilites about paths
       
+      def req_path
+        rack_env['REQ_PATH'] || normalize_req_path(rack_env['PATH_INFO'])
+      end
+
       # Returns the real path of a file
       def realpath(file)
         File.expand_path(File.join(folder, file))
@@ -164,7 +167,8 @@ module Waw
       ################################################### .waw access rules application!
       
       # Finds the matching block inside this .wawaccess handler
-      def find_match(path)
+      def find_match(env)
+        path = env['REQ_PATH']
         @serve.each do |pattern, block|
           case pattern
             when FalseClass
@@ -186,6 +190,10 @@ module Waw
               if pattern.validate(matching_file(path))
                 return Match.new(self, path, block) 
               end
+            when StaticController::Matcher
+              if pattern.matches?(env)
+                return Match.new(self, path, block)
+              end
             else
               raise WawError, "Unrecognized wawaccess pattern #{pattern}"
           end
@@ -194,16 +202,16 @@ module Waw
       end 
       
       # Applies the rules defined here or delegate to the parent if allowed
-      def apply_rules(path)
-        if match = find_match(path)
-          match.__execute
+      def apply_rules(env)
+        if match = find_match(env)
+          match.__execute(env)
         elsif (parent and inherits)
-          parent.apply_rules(path)
+          parent.apply_rules(env)
         else
-          body = "File not found: #{path}\n"
+          body = "File not found: #{env['PATH_INFO']}\n"
           [404, {"Content-Type" => "text/plain",
-             "Content-Length" => body.size.to_s,
-             "X-Cascade" => "pass"},
+                 "Content-Length" => body.size.to_s,
+                 "X-Cascade" => "pass"},
            [body]]
         end
       end
@@ -224,10 +232,15 @@ module Waw
       end
   
       # Serves a path from a root waw access in the hierarchy
-      def do_path_serve(path)
-        path = normalize_req_path(path)
-        waw_access = (find_wawaccess_for(path) || self)
-        waw_access.apply_rules(path)
+      def do_path_serve(path, env = rack_env)
+        env['REQ_PATH'] = normalize_req_path(path)
+        waw_access = (find_wawaccess_for(env['REQ_PATH']) || self)
+        waw_access.apply_rules(env)
+      end
+      
+      # Makes a Rack standard call
+      def call(env)
+        do_path_serve(env['PATH_INFO'], env)
       end
             
     end # class WawAccess

data/lib/waw/controllers/static/waw_access_dsl.rb

@@ -9,6 +9,7 @@ module Waw
         def initialize(wawaccess)
           raise ArgumentError, "wawaccess cannot be nil" unless WawAccess===wawaccess
           @wawaccess = wawaccess
+          @matchers = {}
         end
         
         # Returns a validator that matches the root of the wawaccess tree
@@ -16,6 +17,14 @@ module Waw
           Waw::Validation.validator{|served_file| File.expand_path(served_file) == File.expand_path(@wawaccess.root.folder)}
         end
         
+        # Installs a matcher
+        def matcher(name, &predicate)
+          @matchers[name] = Matcher.new(@wawaccess, predicate)
+          (class << self; self; end).send(:define_method,name) do
+            @matchers[name]
+          end
+        end
+        
         # Starts a wawaccess file
         def wawaccess(&block)
           raise WawError, "#{@wawaccess.identifier}: missing block in wawaccess call" unless block

data/lib/waw/controllers/static_controller.rb

@@ -1,5 +1,8 @@
+require 'uri'
 require 'waw/controllers/static/waw_access'
 require 'waw/controllers/static/waw_access_dsl'
+require 'waw/controllers/static/matcher'
+require 'waw/controllers/static/match'
 module Waw
   # 
   # A waw service that serves public pages expressed in wlang wtpl format
@@ -29,8 +32,8 @@ module Waw
     ##############################################################################################
 
     # Executes the service
-    def execute(env, req, res)
-      @wawaccess.do_path_serve(env['PATH_INFO'])
+    def call(env)
+      @wawaccess.call(env)
     end
 
   end # class Controller

data/lib/waw/crawler.rb

@@ -0,0 +1,176 @@
+require 'mechanize'
+require 'waw/crawler/crawler_options'
+require 'waw/crawler/crawler_listener'
+module Waw
+  class Crawler
+    include Crawler::Options
+    
+    ###################################################################### Internal state
+    
+    # Mechanize agent instance
+    attr_reader :agent
+    
+    # Root URI to crawl
+    attr_reader :root_uri
+      
+    # Sets the root uri
+    def root_uri=(uri)
+      @root_uri = if uri.nil?
+        "127.0.0.1:9292"
+      else
+        uri.is_a?(URI) ? uri : URI::parse(uri.to_s)
+      end
+    end
+    
+    # Stack of files/pages to visit
+    attr_reader :stack
+    
+    ###################################################################### About URI visit state
+    
+    # URI statuses
+    attr_reader :uristate
+    
+    # 
+    PINGED   = 1
+    PENDING  = 2
+    CHECKING = 4
+    CHECKED  = 8
+    
+    # Marks an URI as currently pending 
+    def pending!(uri)
+      uristate[uri] |= PENDING
+    end
+
+    # Marks an URI as being pinged 
+    def pinged!(uri)
+      uristate[uri] |= PINGED
+    end
+
+    ###################################################################### Initialization
+    
+    # Creates a crawler instance on a root URI  
+    def initialize(root_uri = nil)
+      self.root_uri = root_uri
+      set_default_options
+    end
+    
+    ###################################################################### Utils
+    
+    # Returns true if a given page is internal to the website currently
+    # crawled
+    def internal_uri?(uri)
+      uri.host.nil? or ((uri.host == root_uri.host) and (uri.port == root_uri.port))
+    end
+    
+    # Resolves as an absolute URI something that has been found on
+    # a page
+    def resolve_uri(href_or_src, page)
+      URI::parse(agent.send(:resolve, href_or_src, page))
+    end
+    
+    ###################################################################### Crawling
+    
+    # Starts the crawling
+    def crawl
+      @agent = Mechanize.new
+      @uristate = Hash.new{|h,k| h[k] = 0}
+      @stack = [ agent.get(root_uri) ]
+      until stack.empty?
+        to_check = stack.shift
+        case to_check
+          when ::Mechanize::Page
+            check_web_page(to_check)
+          else
+            listener.doc_skipped(to_check)
+        end
+      end
+      @agent = nil
+      @uristate = nil
+      @stack = nil
+    end
+    
+    def crawl_all(query, referer_page)
+      referer_page.search(query).each do |loc|
+        crawl_one(loc, referer_page)
+      end
+    end
+    
+    def crawl_one(location, referer_page)
+      uri = resolve_uri(location, referer_page)
+      
+      # Bypass PENDING/CHECKING/CHECKED links
+      if uristate[uri] < PENDING
+      
+        # Mark it as PENDING now
+        pending!(uri)
+
+        # Mark as to crawl by pushing on the stack
+        if internal_uri?(uri)
+          stack.push(agent.get(uri))
+        else
+          listener.crawl_skipped(referer_page, location)
+        end
+        
+      end
+    rescue => ex
+      handle_error(ex, referer_page, location)
+    end
+    
+    ###################################################################### Checking
+    
+    def check_web_page(page)
+      uristate[page.uri] |= CHECKING
+      listener.checking(page){
+        # Make ping checks
+        all_ping!(ping_list.join(', '), page)
+        # Crawl all links now
+        crawl_all(crawl_list.join(', '), page)
+      }
+      uristate[page.uri] |= CHECKED
+    end
+    
+    ###################################################################### Pinging
+    
+    def all_ping!(query, referer_page)
+      referer_page.search(query).each do |loc|
+        ping!(loc, referer_page)
+      end
+    end
+    
+    def ping!(loc, referer_page)
+      uri = resolve_uri(loc, referer_page)
+      
+      # Only ping uri that are not PINGED/PENDING/CHECKING/CHECKED
+      return unless uristate[uri] < PINGED
+
+      # bypass externals if required
+      if internal_uri?(uri) || check_externals
+        agent.head(uri) # ping!
+        pinged!(uri)
+        listener.ping_ok(referer_page, loc)
+      else
+        listener.ping_skipped(referer_page, loc)
+      end
+      
+    rescue => ex
+      handle_error(ex, referer_page, loc)
+    end
+    
+    ###################################################################### Error handling
+    
+    # Handles errors that occur
+    def handle_error(ex, referer_page, loc)
+      case ex
+        when Mechanize::ResponseCodeError
+          listener.reach_failure(referer_page, loc, ex)
+        when Mechanize::UnsupportedSchemeError
+          listener.scheme_failure(referer_page, loc, ex)
+        when SocketError
+          listener.socket_error(referer_page, loc, ex)
+        else
+          raise ex
+      end
+    end
+    
+  end # class Crawler
+end # module Waw